Configurable pre-stop hooks for node daemonset

iam.tf

@@ -17,6 +17,17 @@ data "aws_iam_policy_document" "ebs_controller_policy" {
     ]
   }
 
+  # https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/2190
+  statement {
+    effect = "Allow"
+
+    resources = [
+      "arn:${var.arn_format}:ec2:*:*:snapshot/*",
+    ]
+
+    actions = ["ec2:CreateVolume"]
+  }
+
   statement {
     effect = "Allow"
 
@@ -163,4 +174,4 @@ module "ebs_controller_role" {
   role_policy_arns              = concat([aws_iam_policy.ebs_controller_policy.arn], var.additional_iam_policies_arns)
   oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${local.controller_name}"]
   tags                          = var.tags
-}
+}

node-rbac.tf

@@ -16,6 +16,18 @@ resource "kubernetes_cluster_role" "node" {
   rule {
     api_groups = [""]
     resources  = ["nodes"]
+    verbs      = ["get", "patch"]
+  }
+
+  rule {
+    api_groups = ["storage.k8s.io"]
+    resources  = ["volumeattachments"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = ["storage.k8s.io"]
+    resources  = ["csinodes"]
     verbs      = ["get"]
   }
 }
@@ -37,4 +49,4 @@ resource "kubernetes_cluster_role_binding" "node" {
     name      = kubernetes_service_account.node.metadata[0].name
     namespace = kubernetes_service_account.node.metadata[0].namespace
   }
-}
+}

node.tf

@@ -54,7 +54,7 @@ resource "kubernetes_daemonset" "node" {
         priority_class_name             = "system-node-critical"
 
         dynamic "toleration" {
-          for_each = length(var.node_tolerations) > 0 ? var.csi_controller_tolerations : [{ operator = "Exists" }]
+          for_each = var.node_tolerations
           content {
             key                = lookup(toleration.value, "key", null)
             operator           = lookup(toleration.value, "operator", null)
@@ -76,6 +76,17 @@ resource "kubernetes_daemonset" "node" {
             var.volume_attach_limit == -1 ? [] : ["--volume-attach-limit=${var.volume_attach_limit}"]
           ])
 
+          dynamic "lifecycle" {
+            for_each = var.ebs_csi_plugin_pre_stop_command != null ? [1] : []
+            content {
+              pre_stop {
+                exec {
+                  command = var.ebs_csi_plugin_pre_stop_command
+                }
+              }
+            }
+          }
+
           security_context {
             privileged = true
           }
@@ -156,10 +167,13 @@ resource "kubernetes_daemonset" "node" {
             "--v=${tostring(var.log_level)}",
           ]
 
-          lifecycle {
-            pre_stop {
-              exec {
-                command = ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"]
+          dynamic "lifecycle" {
+            for_each = var.ebs_csi_registrar_pre_stop_command != null ? [1] : []
+            content {
+              pre_stop {
+                exec {
+                  command = var.ebs_csi_registrar_pre_stop_command
+                }
               }
             }
           }

variables.tf

@@ -24,6 +24,12 @@ variable "ebs_csi_controller_image" {
   type        = string
 }
 
+variable "ebs_csi_plugin_pre_stop_command" {
+  type = list(string)
+  default = ["/bin/aws-ebs-csi-driver", "pre-stop-hook"]
+  description = "The pre-stop command for the EBS CSI driver plugin container"
+}
+
 variable "csi_node_driver_registrar_version" {
   description = "The CSI node driver registrar image version"
   default     = "v2.9.0"
@@ -36,6 +42,12 @@ variable "csi_node_driver_registrar_image" {
   type        = string
 }
 
+variable "ebs_csi_registrar_pre_stop_command" {
+  type = list(string)
+  default = null
+  description = "The pre-stop command for the EBS CSI driver registrar container"
+}
+
 variable "csi_attacher_version" {
   description = "The CSI attacher image version"
   default     = "v3.5.1"
@@ -116,7 +128,7 @@ variable "oidc_url" {
 variable "node_tolerations" {
   description = "CSI driver node tolerations"
   type        = list(map(string))
-  default     = []
+  default     = [{ operator = "Exists" }]
 }
 
 variable "csi_controller_tolerations" {

versions.tf

@@ -2,7 +2,13 @@ terraform {
   required_version = ">= 0.12.6"
 
   required_providers {
-    kubernetes = ">= 1.11.4"
-    aws        = ">= 3.40.0"
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.0.0"
+    }
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0.0"
+    }
   }
 }