diff --git a/README.md b/README.md
index 1cdbc0d..03396b8 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,14 @@ Here is an example of using this module:
- [`examples/complete`](https://github.com/masterpointio/terraform-aws-tailscale/) - complete example of using this module
+## System Logging and Monitoring Setup
+
+On Linux and other Unix-like systems, Tailscale typically runs as a systemd service, which by default does not rotate logs - potentially allowing system logs to grow until the disk fills.
+
+To address this, our user data script configures both a maximum journal size and a retention period to ensure logs are periodically purged. We also install the [CloudWatch Agent](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Install-CloudWatch-Agent.html) with its default configuration so that filesystem usage metrics are reported to AWS.
+
+👀 To view these metrics, navigate in the AWS Console to “CWAgent” → “AutoScalingGroupName, ImageId, InstanceId, InstanceType, device, fstype, path” → “disk_used_percent” for the root path “/”.
+
## Requirements
@@ -71,10 +79,10 @@ Here is an example of using this module:
## Providers
-| Name | Version |
-| ------------------------------------------------------------------ | ------- |
-| [aws](#provider_aws) | 5.76.0 |
-| [tailscale](#provider_tailscale) | 0.17.2 |
+| Name | Version |
+| ------------------------------------------------------------------ | --------- |
+| [aws](#provider_aws) | >= 4.0 |
+| [tailscale](#provider_tailscale) | >= 0.13.7 |
## Modules
@@ -82,15 +90,16 @@ Here is an example of using this module:
| -------------------------------------------------------------------------------------------------------- | ---------------------------------- | ------- |
| [ssm_policy](#module_ssm_policy) | cloudposse/iam-policy/aws | 2.0.1 |
| [ssm_state](#module_ssm_state) | cloudposse/ssm-parameter-store/aws | 0.13.0 |
-| [tailscale_subnet_router](#module_tailscale_subnet_router) | masterpointio/ssm-agent/aws | 1.2.0 |
+| [tailscale_subnet_router](#module_tailscale_subnet_router) | masterpointio/ssm-agent/aws | 1.3.0 |
| [this](#module_this) | cloudposse/label/null | 0.25.0 |
## Resources
-| Name | Type |
-| ------------------------------------------------------------------------------------------------------------------------------------------------ | -------- |
-| [aws_iam_role_policy_attachment.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
-| [tailscale_tailnet_key.default](https://registry.terraform.io/providers/tailscale/tailscale/latest/docs/resources/tailnet_key) | resource |
+| Name | Type |
+| ------------------------------------------------------------------------------------------------------------------------------------------------- | -------- |
+| [aws_iam_role_policy_attachment.cw_agent](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [tailscale_tailnet_key.default](https://registry.terraform.io/providers/tailscale/tailscale/latest/docs/resources/tailnet_key) | resource |
## Inputs
@@ -114,7 +123,9 @@ Here is an example of using this module:
| [exit_node_enabled](#input_exit_node_enabled) | Advertise Tailscale Subnet Router EC2 instance as exit node. Defaults to false. | `bool` | `false` | no |
| [expiry](#input_expiry) | The expiry of the auth key in seconds. | `number` | `7776000` | no |
| [id_length_limit](#input_id_length_limit) | Limit `id` to this many characters (minimum 6).
Set to `0` for unlimited length.
Set to `null` for keep the existing setting, which defaults to `0`.
Does not affect `id_full`. | `number` | `null` | no |
-| [instance_type](#input_instance_type) | The instance type to use for the Tailscale Subnet Router EC2 instance. | `string` | `"t3.nano"` | no |
+| [instance_type](#input_instance_type) | The instance type to use for the Tailscale Subnet Router EC2 instance. | `string` | `"t4g.nano"` | no |
+| [journald_max_retention_sec](#input_journald_max_retention_sec) | The maximum time to store journal entries. | `string` | `"7d"` | no |
+| [journald_system_max_use](#input_journald_system_max_use) | Disk space the journald may use up at most | `string` | `"200M"` | no |
| [key_pair_name](#input_key_pair_name) | The name of the key-pair to associate with the Tailscale Subnet Router EC2 instance. | `string` | `null` | no |
| [label_key_case](#input_label_key_case) | Controls the letter case of the `tags` keys (label names) for tags generated by this module.
Does not affect keys of tags passed in via the `tags` input.
Possible values: `lower`, `title`, `upper`.
Default value: `title`. | `string` | `null` | no |
| [label_order](#input_label_order) | The order in which the labels (ID elements) appear in the `id`.
Defaults to ["namespace", "environment", "stage", "name", "attributes"].
You can omit any of the 6 labels ("tenant" is the 6th), but at least one must be present. | `list(string)` | `null` | no |
@@ -133,7 +144,7 @@ Here is an example of using this module:
| [session_logging_kms_key_alias](#input_session_logging_kms_key_alias) | Alias name for `session_logging` KMS Key.
This is only applied if 2 conditions are met: (1) `session_logging_kms_key_arn` is unset,
(2) `session_logging_encryption_enabled` = true. | `string` | `"alias/session_logging"` | no |
| [session_logging_ssm_document_name](#input_session_logging_ssm_document_name) | Name for `session_logging` SSM document.
This is only applied if 2 conditions are met: (1) `session_logging_enabled` = true,
(2) `create_run_shell_document` = true. | `string` | `"SSM-SessionManagerRunShell-Tailscale"` | no |
| [ssh_enabled](#input_ssh_enabled) | Enable SSH access to the Tailscale Subnet Router EC2 instance. Defaults to true. | `bool` | `true` | no |
-| [ssm_state_enabled](#input_ssm_state_enabled) | Control is tailscaled state (including preferences and keys) is stored in AWS SSM.
See more in the [docs](https://tailscale.com/kb/1278/tailscaled#flags-to-tailscaled). | `bool` | `false` | no |
+| [ssm_state_enabled](#input_ssm_state_enabled) | Control if tailscaled state is stored in AWS SSM (including preferences and keys).
This tells the Tailscale daemon to write + read state from SSM,
which unlocks important features like retaining the existing tailscale machine name.
See more in the [docs](https://tailscale.com/kb/1278/tailscaled#flags-to-tailscaled). | `bool` | `false` | no |
| [stage](#input_stage) | ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no |
| [subnet_ids](#input_subnet_ids) | The Subnet IDs which the Tailscale Subnet Router EC2 instance will run in. These _should_ be private subnets. | `list(string)` | n/a | yes |
| [tags](#input_tags) | Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`).
Neither the tag keys nor the tag values will be modified by this module. | `map(string)` | `{}` | no |
diff --git a/main.tf b/main.tf
index c6e0235..d79fac6 100644
--- a/main.tf
+++ b/main.tf
@@ -26,6 +26,9 @@ locals {
tailscaled_extra_flags = local.tailscaled_extra_flags
tailscale_up_extra_flags_enabled = local.tailscale_up_extra_flags_enabled
tailscale_up_extra_flags = join(" ", var.tailscale_up_extra_flags)
+
+ journald_system_max_use = var.journald_system_max_use
+ journald_max_retention_sec = var.journald_max_retention_sec
})
}
@@ -34,7 +37,7 @@ locals {
# trivy:ignore:AVD-AWS-0090
module "tailscale_subnet_router" {
source = "masterpointio/ssm-agent/aws"
- version = "1.2.0"
+ version = "1.3.0"
context = module.this.context
tags = module.this.tags
@@ -121,3 +124,8 @@ resource "aws_iam_role_policy_attachment" "default" {
role = module.tailscale_subnet_router.role_id
policy_arn = module.ssm_policy[0].policy_arn
}
+
+resource "aws_iam_role_policy_attachment" "cw_agent" {
+ role = module.tailscale_subnet_router.role_id
+ policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+}
diff --git a/userdata.sh.tmpl b/userdata.sh.tmpl
index 727bce9..0f02149 100644
--- a/userdata.sh.tmpl
+++ b/userdata.sh.tmpl
@@ -8,6 +8,24 @@ echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
echo 'net.ipv6.conf.all.forwarding = 1' >> /etc/sysctl.conf
sysctl -p /etc/sysctl.conf
+# In systemd, Administrator drop-ins should reside in /etc/systemd/, ensuring they
+# are preserved across updates and have higher precedence than vendor defaults.
+#
+# We name our file 99-custom.conf so it loads last among any .conf files.
+# That way, it overrides any settings that come earlier.
+
+# Create the journald configs directory if it doesn't already exist
+mkdir -p /etc/systemd/journald.conf.d
+
+cat < /etc/systemd/journald.conf.d/99-custom.conf
+[Journal]
+SystemMaxUse=${journald_system_max_use}
+MaxRetentionSec=${journald_max_retention_sec}
+EOF
+
+# Restart journald so it picks up the new configuration
+systemctl restart systemd-journald
+
# Function to retry a command up to a maximum number of attempts
retry_command() {
local cmd="$1"
@@ -36,6 +54,12 @@ retry_command() {
return $exit_code
}
+# Install CloudWatch Agent
+echo "Installing CloudWatch Agent..."
+retry_command "dnf install -y amazon-cloudwatch-agent" 5
+amazon-cloudwatch-agent-ctl -a start -m ec2
+
+# Install Tailscale
echo "Installing Tailscale..."
retry_command "dnf install -y dnf-utils" 5
retry_command "dnf config-manager --add-repo https://pkgs.tailscale.com/stable/amazon-linux/2/tailscale.repo" 5
diff --git a/variables.tf b/variables.tf
index 3d4f298..9217c40 100644
--- a/variables.tf
+++ b/variables.tf
@@ -80,7 +80,7 @@ variable "ami" {
}
variable "instance_type" {
- default = "t3.nano"
+ default = "t4g.nano"
type = string
description = "The instance type to use for the Tailscale Subnet Router EC2 instance."
}
@@ -115,6 +115,17 @@ variable "desired_capacity" {
default = 1
}
+variable "journald_system_max_use" {
+ description = "Disk space the journald may use up at most"
+ type = string
+ default = "200M"
+}
+
+variable "journald_max_retention_sec" {
+ description = "The maximum time to store journal entries."
+ type = string
+ default = "7d"
+}
################
## Tailscale ##
##############