tidied up org cloudtrails

terraform/stacks/umccr/org_root/main-cloudtrail-bucket.tf

@@ -3,12 +3,17 @@
 #
 # Setting for the bucket that stores all our organisation cloudtrails
 
-# by default we want to keep cloudtrail logs for 7 years - but if listed here then this is
+# by default we want to keep cloudtrail logs for a long time - but if listed here then this is
 # an override of their expiry (for dev accounts etc)
 variable "override_cloudtrail_expiration_days" {
   type = map(number)
   default = {
-    "843407916570" = 365 # dev
+    # dev
+    "843407916570" = 365
+    # onboarding
+    "702956374523" = 365
+    # guardians dev
+    "842385035780" = 365
   }
 }
 
@@ -34,70 +39,60 @@ resource "aws_s3_bucket_policy" "cloudtrail_root" {
       "Version" : "2012-10-17",
       "Statement" : [
         {
-          "Sid" : "AWSCloudTrailAclCheck20150319",
+          "Sid" : "AWSCloudTrailAclCheck",
           "Effect" : "Allow",
           "Principal" : {
             "Service" : "cloudtrail.amazonaws.com"
           },
           "Action" : ["s3:GetBucketAcl", "s3:ListBucket"],
-          "Resource" : "arn:aws:s3:::umccr-cloudtrail-org-root",
-          #"Condition" : {
-          #  "StringEquals" : {
-          #    "aws:SourceArn" : "arn:aws:cloudtrail:*:${data.aws_organizations_organization.current.master_account_id}:trail/*"
-          #  }
-          #}
+          "Resource" : aws_s3_bucket.cloudtrail_root.arn
+          "Condition" : {
+            "StringEquals" : {
+              "aws:SourceArn" : aws_cloudtrail.org_trail.arn
+            }
+          }
         },
-        #{
-        #  "Sid" : "AllowCloudTrailWritesThisAccount",
-        #  "Effect" : "Allow",
-        #  "Principal" : {
-        #    "Service" : "cloudtrail.amazonaws.com"
-        #  },
-        #  "Action" : "s3:PutObject",
-        #  "Resource" : "arn:aws:s3:::umccr-cloudtrail-org-root/AWSLogs/650704067584/*"
-        #},
         {
-          "Sid" : "AWSCloudTrailWrite20150319",
+          "Sid" : "AWSCloudTrailWrite",
           "Effect" : "Allow",
           "Principal" : {
             "Service" : "cloudtrail.amazonaws.com"
           },
           "Action" : "s3:PutObject",
-          "Resource" : "arn:aws:s3:::umccr-cloudtrail-org-root/AWSLogs/o-p5xvdd9ddb/*",
-          "Condition": {
-            "StringEquals": {
-              "s3:x-amz-acl": "bucket-owner-full-control"
+          "Resource" : "${aws_s3_bucket.cloudtrail_root.arn}/AWSLogs/${data.aws_organizations_organization.current.id}/*",
+          "Condition" : {
+            "StringEquals" : {
+              "aws:SourceArn" : aws_cloudtrail.org_trail.arn
             }
           }
         }
       ]
   })
 }
 
-# THE CURRENT DEPLOYED CLOUDTRAIL BUCKET HAS A BUNCH OF LIFECYCLE RULES
-# BUT THEY ARE ALL DISABLED. LEAVING THIS HERE AS THE NEW CONSTRUCT
-# IF WE WANTED TO SWITCH THEM ON
-#
-# resource "aws_s3_bucket_lifecycle_configuration" "example" {
-#   for_each = toset(data.aws_organizations_organization.current.accounts[*].id)
-#
-#   bucket = aws_s3_bucket.cloudtrail_root.id
-#
-#   rule {
-#     id     = "${each.value} logs lifecycle"
-#     status = "Enabled"
-#
-#     filter {
-#       prefix = "AWSLogs/o-p5xvdd9ddb/${each.value}"
-#     }
-#
-#     transition {
-#       days          = 90
-#       storage_class = "DEEP_ARCHIVE"
-#     }
-#
-#     expiration {
-#       days          = lookup(var.override_cloudtrail_expiration_days, each.value, 7*365)
-#     }
-#   }
-# }
+resource "aws_s3_bucket_lifecycle_configuration" "cloudtrail_root" {
+  bucket = aws_s3_bucket.cloudtrail_root.id
+
+  # a lifecycle rule per account
+  dynamic "rule" {
+    for_each = toset(data.aws_organizations_organization.current.accounts[*].id)
+
+    content {
+      id     = "Account ${rule.value} logs lifecycle - tier 60 days, expire ${lookup(var.override_cloudtrail_expiration_days, rule.value, 7 * 365)} days"
+      status = "Enabled"
+
+      filter {
+        prefix = "AWSLogs/${data.aws_organizations_organization.current.id}/${rule.value}/"
+      }
+
+      transition {
+        days          = 60
+        storage_class = "INTELLIGENT_TIERING"
+      }
+
+      expiration {
+        days = lookup(var.override_cloudtrail_expiration_days, rule.value, 7 * 365)
+      }
+    }
+  }
+}

terraform/stacks/umccr/org_root/main-cloudtrail-common-event-selectors.tf

@@ -0,0 +1,72 @@
+locals {
+
+  # a single common definition of cloud trail events we want to capture at an organisation level
+  # can be shared between cloudtrails and cloudtrail lakes
+  common_advanced_event_selectors = [
+    {
+      name = "Log all management and data events"
+
+      # include management events
+      field_selectors = [{
+        field  = "eventCategory"
+        equals = ["Management"]
+      }]
+    },
+
+    {
+      name = "Log all S3 data events"
+
+      field_selectors = [{
+        field  = "eventCategory"
+        equals = ["Data"]
+        },
+
+        {
+          field  = "resources.type"
+          equals = ["AWS::S3::Object"]
+      }]
+    },
+
+    {
+      name = "Log all Lambda data events"
+
+      field_selectors = [{
+        field  = "eventCategory"
+        equals = ["Data"]
+        },
+
+        {
+          field  = "resources.type"
+          equals = ["AWS::Lambda::Function"]
+      }]
+    },
+
+    # log steps activity data events
+    {
+      name = "Log all Steps activity data events"
+
+      field_selectors = [{
+        field  = "eventCategory"
+        equals = ["Data"]
+        },
+        {
+          field  = "resources.type"
+          equals = ["AWS::StepFunctions::Activity"]
+      }]
+    },
+
+    # log steps state machine data events
+    {
+      name = "Log all Steps state machine data events"
+
+      field_selectors = [{
+        field  = "eventCategory"
+        equals = ["Data"]
+        },
+        {
+          field  = "resources.type"
+          equals = ["AWS::StepFunctions::StateMachine"]
+      }]
+    }
+  ]
+}

terraform/stacks/umccr/org_root/main-cloudtrail-lake.tf

@@ -12,64 +12,23 @@ resource "aws_cloudtrail_event_data_store" "org_trail_store" {
   # until we work out we want this we allow it to be deleted
   termination_protection_enabled = false
 
-  advanced_event_selector {
-    name = "Log all management events"
-
-    field_selector {
-      field  = "eventCategory"
-      equals = ["Management"]
-    }
-  }
-
-  advanced_event_selector {
-    name = "Log all S3 data events"
-
-    field_selector {
-      field  = "eventCategory"
-      equals = ["Data"]
-    }
-    field_selector {
-      field  = "resources.type"
-      equals = ["AWS::S3::Object"]
-    }
-  }
-
-  advanced_event_selector {
-    name = "Log all lambda data events"
-
-    field_selector {
-      field  = "eventCategory"
-      equals = ["Data"]
-    }
-    field_selector {
-      field  = "resources.type"
-      equals = ["AWS::Lambda::Function"]
-    }
-  }
-
-  advanced_event_selector {
-    name = "Log all steps activity data events"
-
-    field_selector {
-      field  = "eventCategory"
-      equals = ["Data"]
-    }
-    field_selector {
-      field  = "resources.type"
-      equals = ["AWS::StepFunctions::Activity"]
-    }
-  }
-
-  advanced_event_selector {
-    name = "Log all steps state machine data events"
-
-    field_selector {
-      field  = "eventCategory"
-      equals = ["Data"]
-    }
-    field_selector {
-      field  = "resources.type"
-      equals = ["AWS::StepFunctions::StateMachine"]
+  dynamic "advanced_event_selector" {
+    for_each = local.common_advanced_event_selectors
+    content {
+      name = advanced_event_selector.value.name
+
+      dynamic "field_selector" {
+        for_each = advanced_event_selector.value.field_selectors
+        content {
+          field  = field_selector.value.field
+          equals = try(field_selector.value.equals, null)
+          not_equals = try(field_selector.value.not_equals, null)
+          starts_with = try(field_selector.value.starts_with, null)
+          ends_with = try(field_selector.value.ends_with, null)
+          not_starts_with = try(field_selector.value.not_starts_with, null)
+          not_ends_with = try(field_selector.value.not_ends_with, null)
+        }
+      }
     }
   }
 

terraform/stacks/umccr/org_root/main-cloudtrail.tf

@@ -13,28 +13,38 @@ resource "aws_cloudtrail" "org_trail" {
   kms_key_id                 = aws_kms_key.org_trail_key.id
   s3_bucket_name             = aws_s3_bucket.cloudtrail_root.id
   sns_topic_name             = null
-  event_selector {
-    exclude_management_event_sources = []
-    include_management_events        = true
-    read_write_type                  = "All"
-    data_resource {
-      type   = "AWS::S3::Object"
-      values = ["arn:aws:s3"]
-    }
-    data_resource {
-      type   = "AWS::Lambda::Function"
-      values = ["arn:aws:lambda"]
-    }
-  }
+
   insight_selector {
     insight_type = "ApiCallRateInsight"
   }
+
+  dynamic "advanced_event_selector" {
+    for_each = local.common_advanced_event_selectors
+    content {
+      name = advanced_event_selector.value.name
+
+      dynamic "field_selector" {
+        for_each = advanced_event_selector.value.field_selectors
+        content {
+          field  = field_selector.value.field
+          equals = try(field_selector.value.equals, null)
+          not_equals = try(field_selector.value.not_equals, null)
+          starts_with = try(field_selector.value.starts_with, null)
+          ends_with = try(field_selector.value.ends_with, null)
+          not_starts_with = try(field_selector.value.not_starts_with, null)
+          not_ends_with = try(field_selector.value.not_ends_with, null)
+        }
+      }
+    }
+  }
 }
 
 resource "aws_cloudwatch_log_group" "org_trail_log_group" {
   name              = "CloudTrail/DefaultLogGroup"
   log_group_class   = "STANDARD"
-  retention_in_days = 0
+  # the cloudwatch log output is used for some metrics and live activity but is
+  # not our primary spot for retain cloudtrail data - so we expire relatively quickly
+  retention_in_days = 30
 }
 
 # __generated__ by Terraform from "arn:aws:kms:ap-southeast-2:650704067584:key/c7169941-5700-4b98-b128-2dc1d2dd2607"