Merge pull request #21 from rhythmictech/govcloud-support

add govcloud support, enhanced log forwarder support

Author: cdaniluk

.pre-commit-config.yaml

@@ -10,10 +10,14 @@ repos:
       - id: terraform_validate
         args:
         - --hook-config=--retry-once-with-cleanup=true
+        exclude: examples\/
       - id: terraform_tflint
         alias: terraform_tflint_nocreds
+        exclude: examples\/
         name: terraform_tflint_nocreds
       - id: terraform_trivy
+        args:
+        - --args=--skip-dirs="**/.terraform,examples/*"
       - id: terraform_providers_lock
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0

.tflint.hcl

@@ -4,7 +4,7 @@ config {
 
 plugin "aws" {
     enabled = true
-    version = "0.12.0"
+    version = "0.30.0"
     source  = "github.com/terraform-linters/tflint-ruleset-aws"
 }
 

README.md

@@ -125,8 +125,10 @@ module "datadog" {
 | [aws_cloudwatch_event_rule.guardduty](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
 | [aws_cloudwatch_event_target.awshealth](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
 | [aws_cloudwatch_event_target.guardduty](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
+| [aws_cloudwatch_log_subscription_filter.cloudwatch_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource |
 | [aws_cloudwatch_log_subscription_filter.rds_enhanced_monitoring](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource |
 | [aws_cur_report_definition.cur](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cur_report_definition) | resource |
+| [aws_iam_access_key.datadog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_access_key) | resource |
 | [aws_iam_policy.datadog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.datadog_cost_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.rds_enhanced_monitoring](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
@@ -136,13 +138,17 @@ module "datadog" {
 | [aws_iam_role_policy_attachment.datadog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.datadog_cost_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.rds_enhanced_monitoring](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_user.datadog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_user) | resource |
+| [aws_iam_user_policy_attachment.cspm_user](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_user_policy_attachment) | resource |
+| [aws_iam_user_policy_attachment.datadog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_user_policy_attachment) | resource |
 | [aws_lambda_function.rds_enhanced_monitoring](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
 | [aws_lambda_permission.awshealth_trigger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
-| [aws_lambda_permission.cloudtrail_trigger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
+| [aws_lambda_permission.bucket_trigger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
+| [aws_lambda_permission.cloudwatch_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
 | [aws_lambda_permission.guardduty_trigger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
 | [aws_s3_bucket.local_cur](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
 | [aws_s3_bucket_lifecycle_configuration.local_cur](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_lifecycle_configuration) | resource |
-| [aws_s3_bucket_notification.cloudtrail_notification](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource |
+| [aws_s3_bucket_notification.bucket_notification](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource |
 | [aws_s3_bucket_policy.local_cur](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
 | [aws_s3_bucket_public_access_block.local_cur](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
 | [aws_s3_bucket_server_side_encryption_configuration.local_cur](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
@@ -165,14 +171,15 @@ module "datadog" {
 | [aws_iam_policy_document.local_cur](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.rds_enhanced_monitoring](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.rds_enhanced_monitoring_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
 | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 | [http_http.rds_enhanced_monitoring](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_cloudtrail_buckets"></a> [cloudtrail\_buckets](#input\_cloudtrail\_buckets) | Bucket(s) to collect CloudTrail logs from | `list(string)` | `[]` | no |
+| <a name="input_access_method"></a> [access\_method](#input\_access\_method) | Access method to use for Datadog integration (recommended not to change unless using GovCloud or China regions, must be either `user` or `role`) | `string` | `"role"` | no |
 | <a name="input_cur_bucket_suffix"></a> [cur\_bucket\_suffix](#input\_cur\_bucket\_suffix) | Suffix to append to the CUR bucket name ([ACCOUNT\_ID]-[REGION]-[cur\_bucket\_suffix]) | `string` | `"datadog-cur-data"` | no |
 | <a name="input_datadog_account_id"></a> [datadog\_account\_id](#input\_datadog\_account\_id) | DataDog AWS account ID (should not need changed) | `string` | `"464622532012"` | no |
 | <a name="input_datadog_site_name"></a> [datadog\_site\_name](#input\_datadog\_site\_name) | DataDog site (e.g., datadoghq.com) | `string` | `"datadoghq.com"` | no |
@@ -186,6 +193,8 @@ module "datadog" {
 | <a name="input_estimated_usage_anomaly_message"></a> [estimated\_usage\_anomaly\_message](#input\_estimated\_usage\_anomaly\_message) | Message for usage anomaly alerts | `string` | `"Datadog usage anomaly detected"` | no |
 | <a name="input_estimated_usage_detection_config"></a> [estimated\_usage\_detection\_config](#input\_estimated\_usage\_detection\_config) | Map of usage types to monitor. | `map(any)` | `{}` | no |
 | <a name="input_estimated_usage_detection_default_config"></a> [estimated\_usage\_detection\_default\_config](#input\_estimated\_usage\_detection\_default\_config) | Map of default usage monitoring settings for each metric type. All are disabled by default. Use `usage_anomaly_services` to enable services and alternately override default settings | <pre>map(object({<br>    anomaly_enabled       = bool<br>    anomaly_span          = string<br>    anomaly_threshold     = number<br>    anomaly_window        = string<br>    anomaly_deviations    = number<br>    anomaly_seasonality   = string<br>    anomaly_rollup        = number<br>    forecast_enabled      = bool<br>    forecast_deviations   = number<br>    forecast_rollup_type  = string<br>    forecast_rollup_value = number<br>    forecast_threshold    = number<br>  }))</pre> | <pre>{<br>  "hosts": {<br>    "anomaly_deviations": 1,<br>    "anomaly_enabled": false,<br>    "anomaly_rollup": 600,<br>    "anomaly_seasonality": "daily",<br>    "anomaly_span": "last_1d",<br>    "anomaly_threshold": 0.15,<br>    "anomaly_window": "last_1h",<br>    "forecast_deviations": 1,<br>    "forecast_enabled": false,<br>    "forecast_rollup_type": "avg",<br>    "forecast_rollup_value": 300,<br>    "forecast_threshold": 1000<br>  },<br>  "logs_indexed": {<br>    "anomaly_deviations": 2,<br>    "anomaly_enabled": false,<br>    "anomaly_rollup": 60,<br>    "anomaly_seasonality": "hourly",<br>    "anomaly_span": "last_1d",<br>    "anomaly_threshold": 0.15,<br>    "anomaly_window": "last_1h",<br>    "forecast_deviations": 1,<br>    "forecast_enabled": false,<br>    "forecast_rollup_type": "sum",<br>    "forecast_rollup_value": 86400,<br>    "forecast_threshold": 1000<br>  },<br>  "logs_ingested": {<br>    "anomaly_deviations": 2,<br>    "anomaly_enabled": false,<br>    "anomaly_rollup": 60,<br>    "anomaly_seasonality": "hourly",<br>    "anomaly_span": "last_1d",<br>    "anomaly_threshold": 0.15,<br>    "anomaly_window": "last_1h",<br>    "forecast_deviations": 1,<br>    "forecast_enabled": false,<br>    "forecast_rollup_type": "sum",<br>    "forecast_rollup_value": 86400,<br>    "forecast_threshold": 1000<br>  }<br>}</pre> | no |
+| <a name="input_forward_buckets"></a> [forward\_buckets](#input\_forward\_buckets) | Bucket(s) to collect logs from (using object notifications) | `list(string)` | `[]` | no |
+| <a name="input_forward_log_groups"></a> [forward\_log\_groups](#input\_forward\_log\_groups) | CloudWatch Log Group names to collect logs from (using filter subscriptions) | `list(string)` | `[]` | no |
 | <a name="input_install_log_forwarder"></a> [install\_log\_forwarder](#input\_install\_log\_forwarder) | controls whether log forwarder lambda should be installed | `bool` | `true` | no |
 | <a name="input_integration_default_namespace_rules"></a> [integration\_default\_namespace\_rules](#input\_integration\_default\_namespace\_rules) | Set all services to disabled by default. | `map(bool)` | <pre>{<br>  "api_gateway": false,<br>  "application_elb": false,<br>  "apprunner": false,<br>  "appstream": false,<br>  "appsync": false,<br>  "athena": false,<br>  "auto_scaling": false,<br>  "backup": false,<br>  "bedrock": false,<br>  "billing": false,<br>  "budgeting": false,<br>  "certificatemanager": false,<br>  "cloudfront": false,<br>  "cloudhsm": false,<br>  "cloudsearch": false,<br>  "cloudwatch_events": false,<br>  "cloudwatch_logs": false,<br>  "codebuild": false,<br>  "codewhisperer": false,<br>  "cognito": false,<br>  "collect_custom_metrics": false,<br>  "connect": false,<br>  "crawl_alarms": false,<br>  "directconnect": false,<br>  "dms": false,<br>  "documentdb": false,<br>  "dynamodb": false,<br>  "dynamodbaccelerator": false,<br>  "ebs": false,<br>  "ec2": false,<br>  "ec2api": false,<br>  "ec2spot": false,<br>  "ecr": false,<br>  "ecs": false,<br>  "efs": false,<br>  "elasticache": false,<br>  "elasticbeanstalk": false,<br>  "elasticinference": false,<br>  "elastictranscoder": false,<br>  "elb": false,<br>  "emr": false,<br>  "es": false,<br>  "firehose": false,<br>  "fsx": false,<br>  "gamelift": false,<br>  "globalaccelerator": false,<br>  "glue": false,<br>  "inspector": false,<br>  "iot": false,<br>  "keyspaces": false,<br>  "kinesis": false,<br>  "kinesis_analytics": false,<br>  "kms": false,<br>  "lambda": false,<br>  "lex": false,<br>  "mediaconnect": false,<br>  "mediaconvert": false,<br>  "medialive": false,<br>  "mediapackage": false,<br>  "mediastore": false,<br>  "mediatailor": false,<br>  "memorydb": false,<br>  "ml": false,<br>  "mq": false,<br>  "msk": false,<br>  "mwaa": false,<br>  "nat_gateway": false,<br>  "neptune": false,<br>  "network_elb": false,<br>  "networkfirewall": false,<br>  "networkmonitor": false,<br>  "opsworks": false,<br>  "polly": false,<br>  "privatelinkendpoints": false,<br>  "privatelinkservices": false,<br>  "rds": false,<br>  "rdsproxy": false,<br>  "redshift": false,<br>  "rekognition": false,<br>  "route53": false,<br>  "route53resolver": false,<br>  "s3": false,<br>  "s3storagelens": false,<br>  "sagemaker": false,<br>  "sagemakerendpoints": false,<br>  "sagemakerlabelingjobs": false,<br>  "sagemakermodelbuildingpipeline": false,<br>  "sagemakerprocessingjobs": false,<br>  "sagemakertrainingjobs": false,<br>  "sagemakertransformjobs": false,<br>  "sagemakerworkteam": false,<br>  "service_quotas": false,<br>  "ses": false,<br>  "shield": false,<br>  "sns": false,<br>  "sqs": false,<br>  "step_functions": false,<br>  "storage_gateway": false,<br>  "swf": false,<br>  "textract": false,<br>  "transitgateway": false,<br>  "translate": false,<br>  "trusted_advisor": false,<br>  "usage": false,<br>  "vpn": false,<br>  "waf": false,<br>  "wafv2": false,<br>  "workspaces": false,<br>  "xray": false<br>}</pre> | no |
 | <a name="input_integration_excluded_regions"></a> [integration\_excluded\_regions](#input\_integration\_excluded\_regions) | Regions to exclude from DataDog monitoring | `list(string)` | `[]` | no |
@@ -210,5 +219,6 @@ module "datadog" {
 | Name | Description |
 |------|-------------|
 | <a name="output_iam_role_datadog"></a> [iam\_role\_datadog](#output\_iam\_role\_datadog) | IAM role assumed by Datadog resources |
+| <a name="output_iam_user_datadog"></a> [iam\_user\_datadog](#output\_iam\_user\_datadog) | IAM user accessed by Datadog resources (when `access_method == user`) |
 | <a name="output_lambda_arn_forwarder"></a> [lambda\_arn\_forwarder](#output\_lambda\_arn\_forwarder) | DataDog Lambda Forwarder ARN |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

cloudtrail.tf

@@ -0,0 +1,72 @@
+resource "aws_cloudformation_stack" "datadog_forwarder" {
+  count = var.install_log_forwarder ? 1 : 0
+
+  name         = "${var.name}-log-forwarder"
+  capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"]
+  template_url = "https://datadog-cloudformation-template.s3.amazonaws.com/aws/forwarder/latest.yaml"
+
+  parameters = {
+    DdApiKeySecretArn = aws_secretsmanager_secret.datadog.arn,
+    DdSite            = var.datadog_site_name,
+    FunctionName      = "${var.name}-forwarder"
+  }
+
+  depends_on = [datadog_integration_aws.datadog]
+}
+
+resource "datadog_integration_aws_lambda_arn" "datadog_forwarder" {
+  count      = var.install_log_forwarder ? 1 : 0
+  account_id = local.account_id
+  lambda_arn = try(aws_cloudformation_stack.datadog_forwarder[0].outputs.DatadogForwarderArn, "")
+
+  depends_on = [aws_cloudformation_stack.datadog_forwarder]
+}
+
+resource "datadog_integration_aws_log_collection" "datadog_forwarder" {
+  count      = var.install_log_forwarder ? 1 : 0
+  account_id = local.account_id
+  services   = var.log_forwarder_sources
+
+  depends_on = [aws_cloudformation_stack.datadog_forwarder]
+}
+
+resource "aws_lambda_permission" "bucket_trigger" {
+  for_each = toset(var.forward_buckets)
+
+  action        = "lambda:InvokeFunction"
+  function_name = try(aws_cloudformation_stack.datadog_forwarder[0].outputs.DatadogForwarderArn, "")
+  principal     = "s3.amazonaws.com"
+  source_arn    = "arn:aws:s3:::${each.value}"
+  statement_id  = "${substr(replace(each.value, "/", "_"), 0, 67)}-AllowExecutionFromS3"
+}
+
+resource "aws_s3_bucket_notification" "bucket_notification" {
+  for_each = toset(var.forward_buckets)
+
+  bucket = each.value
+
+  lambda_function {
+    events              = ["s3:ObjectCreated:*"]
+    lambda_function_arn = try(aws_cloudformation_stack.datadog_forwarder[0].outputs.DatadogForwarderArn, "")
+  }
+}
+
+resource "aws_cloudwatch_log_subscription_filter" "cloudwatch_logs" {
+  for_each = toset(var.forward_log_groups)
+
+  name            = "${each.value}-filter"
+  filter_pattern  = ""
+  destination_arn = try(aws_cloudformation_stack.datadog_forwarder[0].outputs.DatadogForwarderArn, "")
+  distribution    = "Random"
+  log_group_name  = each.value
+}
+
+resource "aws_lambda_permission" "cloudwatch_logs" {
+  for_each = toset(var.forward_log_groups)
+
+  statement_id  = "${substr(replace(each.value, "/", "_"), 0, 67)}-AllowExecutionFromCloudWatchLogs"
+  action        = "lambda:InvokeFunction"
+  function_name = try(aws_cloudformation_stack.datadog_forwarder[0].outputs.DatadogForwarderArn, "")
+  principal     = "logs.${local.region}.amazonaws.com"
+  source_arn    = "arn:${local.partition}:logs:${local.region}:${local.account_id}:log-group:${each.value}:*"
+}