Merge pull request #55 from vara-bonthu/shs-upgrade

feat: Updated spark history server to latest version 1.5.1

Author: vara-bonthu

README.md

@@ -74,15 +74,15 @@ module "eks_data_addons" {
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
-| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.4.1 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.95 |
+| <a name="requirement_helm"></a> [helm](#requirement\_helm) | ~> 2.17 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72 |
-| <a name="provider_helm"></a> [helm](#provider\_helm) | >= 2.4.1 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.95 |
+| <a name="provider_helm"></a> [helm](#provider\_helm) | ~> 2.17 |
 
 ## Modules
 

spark-history-server.tf

@@ -1,77 +1,68 @@
 locals {
   spark_history_server_name       = "spark-history-server"
   spark_history_server_repository = "https://kubedai.github.io/spark-history-server"
-  spark_history_server_version    = "1.3.1"
+  spark_history_server_version    = "1.5.1"
 
   spark_history_server_service_account = "spark-history-server-sa"
   spark_history_server_create_irsa     = var.enable_spark_history_server && try(var.spark_history_server_helm_config.create_irsa, true)
   spark_history_server_namespace       = try(var.spark_history_server_helm_config["namespace"], local.spark_history_server_name)
-  spark_history_server_set_values = local.spark_history_server_create_irsa ? [
-    {
-      name  = "serviceAccount.name"
-      value = local.spark_history_server_service_account
-    },
+
+  # Parse user values
+  user_provided_values = try(yamldecode(var.spark_history_server_helm_config.values[0]), {})
+
+  # Build the final configuration - always S3 for AWS, but preserve user's S3 config
+  spark_history_server_values = yamlencode(merge(
+    local.user_provided_values,
     {
-      name  = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
-      value = module.spark_history_server_irsa[0].iam_role_arn
+      # Always set logStore type to S3 and add IRSA role ARN
+      logStore = merge(
+        try(local.user_provided_values.logStore, {}),
+        {
+          type = "s3"
+          s3 = merge(
+            try(local.user_provided_values.logStore.s3, {}),
+            {
+              # Chart handles ServiceAccount annotation automatically when this is set
+              irsaRoleArn = local.spark_history_server_create_irsa ? module.spark_history_server_irsa[0].iam_role_arn : ""
+            }
+          )
+        }
+      )
     }
-  ] : []
-
-  spark_history_server_default_values = <<-EOT
-sparkConf: |-
-  spark.hadoop.fs.s3a.aws.credentials.provider=com.amazonaws.auth.WebIdentityTokenCredentialsProvider
-  spark.history.fs.eventLog.rolling.maxFilesToRetain=5
-  spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
-  spark.eventLog.enabled=true
-  spark.history.ui.port=18080
-
-resources:
-  limits:
-    cpu: 200m
-    memory: 2G
-  requests:
-    cpu: 100m
-    memory: 1G
-EOT
-
-  spark_history_server_merged_values_yaml = yamlencode(merge(
-    yamldecode(local.spark_history_server_default_values),
-    try(yamldecode(var.spark_history_server_helm_config.values[0]), {})
   ))
-
 }
 
 resource "helm_release" "spark_history_server" {
   count = var.enable_spark_history_server ? 1 : 0
 
-  name                       = try(var.spark_history_server_helm_config["name"], local.spark_history_server_name)
-  repository                 = try(var.spark_history_server_helm_config["repository"], local.spark_history_server_repository)
-  chart                      = try(var.spark_history_server_helm_config["chart"], local.spark_history_server_name)
-  version                    = try(var.spark_history_server_helm_config["version"], local.spark_history_server_version)
-  timeout                    = try(var.spark_history_server_helm_config["timeout"], 300)
-  values                     = [local.spark_history_server_merged_values_yaml]
-  create_namespace           = try(var.spark_history_server_helm_config["create_namespace"], true)
-  namespace                  = local.spark_history_server_namespace
+  name             = try(var.spark_history_server_helm_config["name"], local.spark_history_server_name)
+  repository       = try(var.spark_history_server_helm_config["repository"], local.spark_history_server_repository)
+  chart            = try(var.spark_history_server_helm_config["chart"], local.spark_history_server_name)
+  version          = try(var.spark_history_server_helm_config["version"], local.spark_history_server_version)
+  namespace        = local.spark_history_server_namespace
+  create_namespace = try(var.spark_history_server_helm_config["create_namespace"], true)
+
+  values = [local.spark_history_server_values]
+
+  # Essential deployment settings
+  timeout         = try(var.spark_history_server_helm_config["timeout"], 300)
+  wait            = try(var.spark_history_server_helm_config["wait"], true)
+  atomic          = try(var.spark_history_server_helm_config["atomic"], true)
+  cleanup_on_fail = try(var.spark_history_server_helm_config["cleanup_on_fail"], true)
+
+  # Pass through other settings
   lint                       = try(var.spark_history_server_helm_config["lint"], false)
-  description                = try(var.spark_history_server_helm_config["description"], "")
-  repository_key_file        = try(var.spark_history_server_helm_config["repository_key_file"], "")
-  repository_cert_file       = try(var.spark_history_server_helm_config["repository_cert_file"], "")
-  repository_username        = try(var.spark_history_server_helm_config["repository_username"], "")
-  repository_password        = try(var.spark_history_server_helm_config["repository_password"], "")
+  description                = try(var.spark_history_server_helm_config["description"], "Spark History Server for AWS EKS")
   verify                     = try(var.spark_history_server_helm_config["verify"], false)
-  keyring                    = try(var.spark_history_server_helm_config["keyring"], "")
   disable_webhooks           = try(var.spark_history_server_helm_config["disable_webhooks"], false)
   reuse_values               = try(var.spark_history_server_helm_config["reuse_values"], false)
   reset_values               = try(var.spark_history_server_helm_config["reset_values"], false)
   force_update               = try(var.spark_history_server_helm_config["force_update"], false)
   recreate_pods              = try(var.spark_history_server_helm_config["recreate_pods"], false)
-  cleanup_on_fail            = try(var.spark_history_server_helm_config["cleanup_on_fail"], false)
-  max_history                = try(var.spark_history_server_helm_config["max_history"], 0)
-  atomic                     = try(var.spark_history_server_helm_config["atomic"], false)
+  max_history                = try(var.spark_history_server_helm_config["max_history"], 10)
   skip_crds                  = try(var.spark_history_server_helm_config["skip_crds"], false)
   render_subchart_notes      = try(var.spark_history_server_helm_config["render_subchart_notes"], true)
   disable_openapi_validation = try(var.spark_history_server_helm_config["disable_openapi_validation"], false)
-  wait                       = try(var.spark_history_server_helm_config["wait"], true)
   wait_for_jobs              = try(var.spark_history_server_helm_config["wait_for_jobs"], false)
   dependency_update          = try(var.spark_history_server_helm_config["dependency_update"], false)
   replace                    = try(var.spark_history_server_helm_config["replace"], false)
@@ -82,7 +73,7 @@ resource "helm_release" "spark_history_server" {
 
   dynamic "set" {
     iterator = each_item
-    for_each = distinct(concat(try(var.spark_history_server_helm_config.set, []), local.spark_history_server_set_values))
+    for_each = try(var.spark_history_server_helm_config.set, [])
 
     content {
       name  = each_item.value.name
@@ -110,15 +101,16 @@ module "spark_history_server_irsa" {
   source = "./irsa"
   count  = local.spark_history_server_create_irsa ? 1 : 0
 
-  # IAM role for service account (IRSA)
   create_role                   = try(var.spark_history_server_helm_config.create_role, true)
   role_name                     = try(var.spark_history_server_helm_config.role_name, local.spark_history_server_name)
   role_name_use_prefix          = try(var.spark_history_server_helm_config.role_name_use_prefix, true)
   role_path                     = try(var.spark_history_server_helm_config.role_path, "/")
   role_permissions_boundary_arn = try(var.spark_history_server_helm_config.role_permissions_boundary_arn, null)
-  role_description              = try(var.spark_history_server_helm_config.role_description, "IRSA for ${local.spark_history_server_name} project")
+  role_description              = try(var.spark_history_server_helm_config.role_description, "IRSA for ${local.spark_history_server_name}")
 
-  role_policy_arns = try(var.spark_history_server_helm_config.role_policy_arns, { "S3ReadOnlyPolicy" : "arn:${local.partition}:iam::aws:policy/AmazonS3ReadOnlyAccess" })
+  role_policy_arns = try(var.spark_history_server_helm_config.role_policy_arns, {
+    "S3ReadOnlyPolicy" = "arn:${local.partition}:iam::aws:policy/AmazonS3ReadOnlyAccess"
+  })
 
   oidc_providers = {
     this = {
@@ -127,4 +119,6 @@ module "spark_history_server_irsa" {
       service_account = local.spark_history_server_service_account
     }
   }
+
+  tags = try(var.spark_history_server_helm_config.tags, {})
 }

versions.tf

@@ -4,11 +4,11 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.72"
+      version = "~> 5.95"
     }
     helm = {
       source  = "hashicorp/helm"
-      version = ">= 2.4.1"
+      version = "~> 2.17"
     }
   }
 }