From 45a922b6aac06f6bf1fcd01ec5060d80eb91b756 Mon Sep 17 00:00:00 2001 From: Veronika Gnilitska Date: Fri, 3 Jan 2025 09:23:35 -0500 Subject: [PATCH 1/2] feat: support journald limit configs --- README.md | 24 +++++++++++++----------- main.tf | 8 ++++++-- userdata.sh.tmpl | 18 ++++++++++++++++++ variables.tf | 13 ++++++++++++- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1cdbc0d..e833fda 100644 --- a/README.md +++ b/README.md @@ -71,19 +71,19 @@ Here is an example of using this module: ## Providers -| Name | Version | -| ------------------------------------------------------------------ | ------- | -| [aws](#provider_aws) | 5.76.0 | -| [tailscale](#provider_tailscale) | 0.17.2 | +| Name | Version | +| ------------------------------------------------------------------ | --------- | +| [aws](#provider_aws) | >= 4.0 | +| [tailscale](#provider_tailscale) | >= 0.13.7 | ## Modules -| Name | Source | Version | -| -------------------------------------------------------------------------------------------------------- | ---------------------------------- | ------- | -| [ssm_policy](#module_ssm_policy) | cloudposse/iam-policy/aws | 2.0.1 | -| [ssm_state](#module_ssm_state) | cloudposse/ssm-parameter-store/aws | 0.13.0 | -| [tailscale_subnet_router](#module_tailscale_subnet_router) | masterpointio/ssm-agent/aws | 1.2.0 | -| [this](#module_this) | cloudposse/label/null | 0.25.0 | +| Name | Source | Version | +| -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | --------------------------- | +| [ssm_policy](#module_ssm_policy) | cloudposse/iam-policy/aws | 2.0.1 | +| [ssm_state](#module_ssm_state) | cloudposse/ssm-parameter-store/aws | 0.13.0 | +| [tailscale_subnet_router](#module_tailscale_subnet_router) | git::https://github.com/masterpointio//terraform-aws-ssm-agent.git | feature/ami-minimal-edition | +| [this](#module_this) | cloudposse/label/null | 0.25.0 | ## Resources @@ -115,6 +115,8 @@ Here is an example of using this module: | [expiry](#input_expiry) | The expiry of the auth key in seconds. | `number` | `7776000` | no | | [id_length_limit](#input_id_length_limit) | Limit `id` to this many characters (minimum 6).
Set to `0` for unlimited length.
Set to `null` for keep the existing setting, which defaults to `0`.
Does not affect `id_full`. | `number` | `null` | no | | [instance_type](#input_instance_type) | The instance type to use for the Tailscale Subnet Router EC2 instance. | `string` | `"t3.nano"` | no | +| [journald_max_retention_sec](#input_journald_max_retention_sec) | The maximum time to store journal entries. | `string` | `"7d"` | no | +| [journald_system_max_use](#input_journald_system_max_use) | Disk space the journald may use up at most | `string` | `"200M"` | no | | [key_pair_name](#input_key_pair_name) | The name of the key-pair to associate with the Tailscale Subnet Router EC2 instance. | `string` | `null` | no | | [label_key_case](#input_label_key_case) | Controls the letter case of the `tags` keys (label names) for tags generated by this module.
Does not affect keys of tags passed in via the `tags` input.
Possible values: `lower`, `title`, `upper`.
Default value: `title`. | `string` | `null` | no | | [label_order](#input_label_order) | The order in which the labels (ID elements) appear in the `id`.
Defaults to ["namespace", "environment", "stage", "name", "attributes"].
You can omit any of the 6 labels ("tenant" is the 6th), but at least one must be present. | `list(string)` | `null` | no | @@ -133,7 +135,7 @@ Here is an example of using this module: | [session_logging_kms_key_alias](#input_session_logging_kms_key_alias) | Alias name for `session_logging` KMS Key.
This is only applied if 2 conditions are met: (1) `session_logging_kms_key_arn` is unset,
(2) `session_logging_encryption_enabled` = true. | `string` | `"alias/session_logging"` | no | | [session_logging_ssm_document_name](#input_session_logging_ssm_document_name) | Name for `session_logging` SSM document.
This is only applied if 2 conditions are met: (1) `session_logging_enabled` = true,
(2) `create_run_shell_document` = true. | `string` | `"SSM-SessionManagerRunShell-Tailscale"` | no | | [ssh_enabled](#input_ssh_enabled) | Enable SSH access to the Tailscale Subnet Router EC2 instance. Defaults to true. | `bool` | `true` | no | -| [ssm_state_enabled](#input_ssm_state_enabled) | Control is tailscaled state (including preferences and keys) is stored in AWS SSM.
See more in the [docs](https://tailscale.com/kb/1278/tailscaled#flags-to-tailscaled). | `bool` | `false` | no | +| [ssm_state_enabled](#input_ssm_state_enabled) | Control if tailscaled state is stored in AWS SSM (including preferences and keys).
This tells the Tailscale daemon to write + read state from SSM,
which unlocks important features like retaining the existing tailscale machine name.
See more in the [docs](https://tailscale.com/kb/1278/tailscaled#flags-to-tailscaled). | `bool` | `false` | no | | [stage](#input_stage) | ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no | | [subnet_ids](#input_subnet_ids) | The Subnet IDs which the Tailscale Subnet Router EC2 instance will run in. These _should_ be private subnets. | `list(string)` | n/a | yes | | [tags](#input_tags) | Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`).
Neither the tag keys nor the tag values will be modified by this module. | `map(string)` | `{}` | no | diff --git a/main.tf b/main.tf index c6e0235..92dbd96 100644 --- a/main.tf +++ b/main.tf @@ -26,6 +26,9 @@ locals { tailscaled_extra_flags = local.tailscaled_extra_flags tailscale_up_extra_flags_enabled = local.tailscale_up_extra_flags_enabled tailscale_up_extra_flags = join(" ", var.tailscale_up_extra_flags) + + journald_system_max_use = var.journald_system_max_use + journald_max_retention_sec = var.journald_max_retention_sec }) } @@ -33,8 +36,9 @@ locals { # Bucket does not have versioning enabled # trivy:ignore:AVD-AWS-0090 module "tailscale_subnet_router" { - source = "masterpointio/ssm-agent/aws" - version = "1.2.0" + # source = "masterpointio/ssm-agent/aws" + # version = "1.2.0" + source = "git::https://github.com/masterpointio/terraform-aws-ssm-agent.git?ref=feature/ami-minimal-edition" context = module.this.context tags = module.this.tags diff --git a/userdata.sh.tmpl b/userdata.sh.tmpl index 727bce9..124ee71 100644 --- a/userdata.sh.tmpl +++ b/userdata.sh.tmpl @@ -8,6 +8,24 @@ echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf echo 'net.ipv6.conf.all.forwarding = 1' >> /etc/sysctl.conf sysctl -p /etc/sysctl.conf +# In systemd, Administrator drop-ins should reside in /etc/systemd/, ensuring they +# are preserved across updates and have higher precedence than vendor defaults. +# +# We name our file 99-custom.conf so it loads last among any .conf files. +# That way, it overrides any settings that come earlier. + +# Create the journald configs directory if it doesn't already exist +mkdir -p /etc/systemd/journald.conf.d + +cat < /etc/systemd/journald.conf.d/99-custom.conf +[Journal] +SystemMaxUse=${journald_system_max_use} +MaxRetentionSec=${journald_max_retention_sec} +EOF + +# Restart journald so it picks up the new configuration +systemctl restart systemd-journald + # Function to retry a command up to a maximum number of attempts retry_command() { local cmd="$1" diff --git a/variables.tf b/variables.tf index 3d4f298..9217c40 100644 --- a/variables.tf +++ b/variables.tf @@ -80,7 +80,7 @@ variable "ami" { } variable "instance_type" { - default = "t3.nano" + default = "t4g.nano" type = string description = "The instance type to use for the Tailscale Subnet Router EC2 instance." } @@ -115,6 +115,17 @@ variable "desired_capacity" { default = 1 } +variable "journald_system_max_use" { + description = "Disk space the journald may use up at most" + type = string + default = "200M" +} + +variable "journald_max_retention_sec" { + description = "The maximum time to store journal entries." + type = string + default = "7d" +} ################ ## Tailscale ## ############## From f7dead3064dea6b6934fa9e0845cd8cab30ae67e Mon Sep 17 00:00:00 2001 From: Veronika Gnilitska Date: Fri, 3 Jan 2025 11:49:29 -0500 Subject: [PATCH 2/2] feat: install CW Agent --- README.md | 31 ++++++++++++++++++++----------- main.tf | 10 +++++++--- userdata.sh.tmpl | 6 ++++++ 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e833fda..03396b8 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,14 @@ Here is an example of using this module: - [`examples/complete`](https://github.com/masterpointio/terraform-aws-tailscale/) - complete example of using this module +## System Logging and Monitoring Setup + +On Linux and other Unix-like systems, Tailscale typically runs as a systemd service, which by default does not rotate logs - potentially allowing system logs to grow until the disk fills. + +To address this, our user data script configures both a maximum journal size and a retention period to ensure logs are periodically purged. We also install the [CloudWatch Agent](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Install-CloudWatch-Agent.html) with its default configuration so that filesystem usage metrics are reported to AWS. + +👀 To view these metrics, navigate in the AWS Console to “CWAgent” → “AutoScalingGroupName, ImageId, InstanceId, InstanceType, device, fstype, path” → “disk_used_percent” for the root path “/”. + ## Requirements @@ -78,19 +86,20 @@ Here is an example of using this module: ## Modules -| Name | Source | Version | -| -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | --------------------------- | -| [ssm_policy](#module_ssm_policy) | cloudposse/iam-policy/aws | 2.0.1 | -| [ssm_state](#module_ssm_state) | cloudposse/ssm-parameter-store/aws | 0.13.0 | -| [tailscale_subnet_router](#module_tailscale_subnet_router) | git::https://github.com/masterpointio//terraform-aws-ssm-agent.git | feature/ami-minimal-edition | -| [this](#module_this) | cloudposse/label/null | 0.25.0 | +| Name | Source | Version | +| -------------------------------------------------------------------------------------------------------- | ---------------------------------- | ------- | +| [ssm_policy](#module_ssm_policy) | cloudposse/iam-policy/aws | 2.0.1 | +| [ssm_state](#module_ssm_state) | cloudposse/ssm-parameter-store/aws | 0.13.0 | +| [tailscale_subnet_router](#module_tailscale_subnet_router) | masterpointio/ssm-agent/aws | 1.3.0 | +| [this](#module_this) | cloudposse/label/null | 0.25.0 | ## Resources -| Name | Type | -| ------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | -| [aws_iam_role_policy_attachment.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [tailscale_tailnet_key.default](https://registry.terraform.io/providers/tailscale/tailscale/latest/docs/resources/tailnet_key) | resource | +| Name | Type | +| ------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| [aws_iam_role_policy_attachment.cw_agent](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [tailscale_tailnet_key.default](https://registry.terraform.io/providers/tailscale/tailscale/latest/docs/resources/tailnet_key) | resource | ## Inputs @@ -114,7 +123,7 @@ Here is an example of using this module: | [exit_node_enabled](#input_exit_node_enabled) | Advertise Tailscale Subnet Router EC2 instance as exit node. Defaults to false. | `bool` | `false` | no | | [expiry](#input_expiry) | The expiry of the auth key in seconds. | `number` | `7776000` | no | | [id_length_limit](#input_id_length_limit) | Limit `id` to this many characters (minimum 6).
Set to `0` for unlimited length.
Set to `null` for keep the existing setting, which defaults to `0`.
Does not affect `id_full`. | `number` | `null` | no | -| [instance_type](#input_instance_type) | The instance type to use for the Tailscale Subnet Router EC2 instance. | `string` | `"t3.nano"` | no | +| [instance_type](#input_instance_type) | The instance type to use for the Tailscale Subnet Router EC2 instance. | `string` | `"t4g.nano"` | no | | [journald_max_retention_sec](#input_journald_max_retention_sec) | The maximum time to store journal entries. | `string` | `"7d"` | no | | [journald_system_max_use](#input_journald_system_max_use) | Disk space the journald may use up at most | `string` | `"200M"` | no | | [key_pair_name](#input_key_pair_name) | The name of the key-pair to associate with the Tailscale Subnet Router EC2 instance. | `string` | `null` | no | diff --git a/main.tf b/main.tf index 92dbd96..d79fac6 100644 --- a/main.tf +++ b/main.tf @@ -36,9 +36,8 @@ locals { # Bucket does not have versioning enabled # trivy:ignore:AVD-AWS-0090 module "tailscale_subnet_router" { - # source = "masterpointio/ssm-agent/aws" - # version = "1.2.0" - source = "git::https://github.com/masterpointio/terraform-aws-ssm-agent.git?ref=feature/ami-minimal-edition" + source = "masterpointio/ssm-agent/aws" + version = "1.3.0" context = module.this.context tags = module.this.tags @@ -125,3 +124,8 @@ resource "aws_iam_role_policy_attachment" "default" { role = module.tailscale_subnet_router.role_id policy_arn = module.ssm_policy[0].policy_arn } + +resource "aws_iam_role_policy_attachment" "cw_agent" { + role = module.tailscale_subnet_router.role_id + policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" +} diff --git a/userdata.sh.tmpl b/userdata.sh.tmpl index 124ee71..0f02149 100644 --- a/userdata.sh.tmpl +++ b/userdata.sh.tmpl @@ -54,6 +54,12 @@ retry_command() { return $exit_code } +# Install CloudWatch Agent +echo "Installing CloudWatch Agent..." +retry_command "dnf install -y amazon-cloudwatch-agent" 5 +amazon-cloudwatch-agent-ctl -a start -m ec2 + +# Install Tailscale echo "Installing Tailscale..." retry_command "dnf install -y dnf-utils" 5 retry_command "dnf config-manager --add-repo https://pkgs.tailscale.com/stable/amazon-linux/2/tailscale.repo" 5