Skip to content

Commit 61ca8c1

Browse files
rodrigobersarodrigobersa
authored
feat: Adding support for Container Insights on AWS for Fluent Bit module (#244)
1 parent c88471d commit 61ca8c1

File tree

4 files changed

+263
-7
lines changed

4 files changed

+263
-7
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ module "eks_blueprints_addons" {
126126
| [aws_iam_role_policy_attachment.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
127127
| [helm_release.this](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
128128
| [kubernetes_config_map_v1.aws_logging](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1) | resource |
129+
| [kubernetes_config_map_v1_data.aws_for_fluentbit_containerinsights](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource |
129130
| [kubernetes_namespace_v1.aws_observability](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource |
130131
| [time_sleep.this](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
131132
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |

‎docs/addons/aws-for-fluentbit.md‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,32 @@ You can optionally customize the Helm chart that deploys AWS for Fluent Bit via
2222
}
2323
aws_for_fluentbit = {
2424
name = "aws-for-fluent-bit"
25-
chart_version = "0.1.24"
25+
chart_version = "0.1.28"
2626
repository = "https://aws.github.io/eks-charts"
2727
namespace = "kube-system"
2828
values = [templatefile("${path.module}/values.yaml", {})]
2929
}
3030
```
3131

32+
If you want to enable [Container Insights on Amazon EKS](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Container-Insights-setup-EKS-quickstart.html) through Fluent Bit, you need to add the following parameter in your configuration:
33+
34+
```hcl
35+
enable_aws_for_fluentbit = true
36+
aws_for_fluentbit = {
37+
enable_containerinsights = true
38+
}
39+
```
40+
3241
## Verify the Fluent Bit setup
3342

3443
Verify aws-for-fluentbit pods are running.
3544

3645
```sh
37-
$ kuebctl get pods -n kube-system
38-
NAME READY STATUS RESTARTS AGE
39-
aws-for-fluent-bit-6kp66 1/1 Running 0 172m
46+
$ kubectl -n kube-system get pods -l app.kubernetes.io/name=aws-for-fluent-bit
47+
NAME READY STATUS RESTARTS AGE
48+
aws-for-fluent-bit-6lhkj 1/1 Running 0 15m
49+
aws-for-fluent-bit-sbn9b 1/1 Running 0 15m
50+
aws-for-fluent-bit-svhwq 1/1 Running 0 15m
4051
```
4152

4253
Open the CloudWatch console at https://console.aws.amazon.com/cloudwatch/
@@ -48,6 +59,12 @@ Make sure that you're in the Region where you deployed Fluent Bit.
4859

4960
Check the list of log groups in the Region. You should see the following:
5061

62+
```
63+
/aws/eks/complete/aws-fluentbit-logs
64+
```
65+
66+
If you enabled Container Insights, you should also see the following Log Groups in your CloudWatch Console.
67+
5168
```
5269
/aws/containerinsights/Cluster_Name/application
5370

‎main.tf‎

Lines changed: 227 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -656,7 +656,7 @@ module "aws_for_fluentbit" {
656656
namespace = local.aws_for_fluentbit_namespace
657657
create_namespace = try(var.aws_for_fluentbit.create_namespace, false)
658658
chart = try(var.aws_for_fluentbit.chart, "aws-for-fluent-bit")
659-
chart_version = try(var.aws_for_fluentbit.chart_version, "0.1.28")
659+
chart_version = try(var.aws_for_fluentbit.chart_version, "0.1.30")
660660
repository = try(var.aws_for_fluentbit.repository, "https://aws.github.io/eks-charts")
661661
values = try(var.aws_for_fluentbit.values, [])
662662

@@ -750,6 +750,232 @@ module "aws_for_fluentbit" {
750750
tags = var.tags
751751
}
752752

753+
resource "kubernetes_config_map_v1_data" "aws_for_fluentbit_containerinsights" {
754+
count = var.enable_aws_for_fluentbit && try(var.aws_for_fluentbit.enable_containerinsights, false) ? 1 : 0
755+
depends_on = [module.aws_for_fluentbit]
756+
force = true
757+
758+
metadata {
759+
name = "aws-for-fluent-bit"
760+
namespace = local.aws_for_fluentbit_namespace
761+
}
762+
763+
data = {
764+
"fluent-bit.conf" = try(
765+
var.aws_for_fluentbit.fluentbit_conf,
766+
<<-EOT
767+
[SERVICE]
768+
Flush 5
769+
Grace 30
770+
Log_Level info
771+
Daemon off
772+
Parsers_File parsers.conf
773+
HTTP_Server On
774+
HTTP_Listen 0.0.0.0
775+
HTTP_Port 2020
776+
storage.path /var/fluent-bit/state/flb-storage/
777+
storage.sync normal
778+
storage.checksum off
779+
storage.backlog.mem_limit 5M
780+
781+
@INCLUDE application-log.conf
782+
@INCLUDE dataplane-log.conf
783+
@INCLUDE host-log.conf
784+
EOT
785+
)
786+
"application-log.conf" = try(
787+
var.aws_for_fluentbit.application_log_conf,
788+
<<-EOT
789+
[INPUT]
790+
Name tail
791+
Tag application.*
792+
Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
793+
Path /var/log/containers/*.log
794+
multiline.parser docker, cri
795+
DB /var/fluent-bit/state/flb_container.db
796+
Mem_Buf_Limit 50MB
797+
Skip_Long_Lines On
798+
Refresh_Interval 10
799+
Rotate_Wait 30
800+
storage.type filesystem
801+
Read_from_Head Off
802+
803+
[INPUT]
804+
Name tail
805+
Tag application.*
806+
Path /var/log/containers/fluent-bit*
807+
multiline.parser docker, cri
808+
DB /var/fluent-bit/state/flb_log.db
809+
Mem_Buf_Limit 5MB
810+
Skip_Long_Lines On
811+
Refresh_Interval 10
812+
Read_from_Head Off
813+
814+
[INPUT]
815+
Name tail
816+
Tag application.*
817+
Path /var/log/containers/cloudwatch-agent*
818+
multiline.parser docker, cri
819+
DB /var/fluent-bit/state/flb_cwagent.db
820+
Mem_Buf_Limit 5MB
821+
Skip_Long_Lines On
822+
Refresh_Interval 10
823+
Read_from_Head Off
824+
825+
[FILTER]
826+
Name kubernetes
827+
Match application.*
828+
Kube_URL https://kubernetes.default.svc:443
829+
Kube_Tag_Prefix application.var.log.containers.
830+
Merge_Log On
831+
Merge_Log_Key log_processed
832+
K8S-Logging.Parser On
833+
K8S-Logging.Exclude Off
834+
Labels Off
835+
Annotations Off
836+
Use_Kubelet On
837+
Kubelet_Port 10250
838+
Buffer_Size 0
839+
840+
[OUTPUT]
841+
Name cloudwatch_logs
842+
Match application.*
843+
region ${local.region}
844+
log_group_name /aws/containerinsights/${local.cluster_name}/application
845+
log_stream_prefix $${HOSTNAME}-
846+
auto_create_group true
847+
extra_user_agent container-insights
848+
workers 1
849+
EOT
850+
)
851+
"dataplane-log.conf" = try(
852+
var.aws_for_fluentbit.dataplane_log_conf,
853+
<<-EOT
854+
[INPUT]
855+
Name systemd
856+
Tag dataplane.systemd.*
857+
Systemd_Filter _SYSTEMD_UNIT=docker.service
858+
Systemd_Filter _SYSTEMD_UNIT=containerd.service
859+
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
860+
DB /var/fluent-bit/state/systemd.db
861+
Path /var/log/journal
862+
Read_From_Tail On
863+
864+
[INPUT]
865+
Name tail
866+
Tag dataplane.tail.*
867+
Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
868+
multiline.parser docker, cri
869+
DB /var/fluent-bit/state/flb_dataplane_tail.db
870+
Mem_Buf_Limit 50MB
871+
Skip_Long_Lines On
872+
Refresh_Interval 10
873+
Rotate_Wait 30
874+
storage.type filesystem
875+
Read_from_Head Off
876+
877+
[FILTER]
878+
Name modify
879+
Match dataplane.systemd.*
880+
Rename _HOSTNAME hostname
881+
Rename _SYSTEMD_UNIT systemd_unit
882+
Rename MESSAGE message
883+
Remove_regex ^((?!hostname|systemd_unit|message).)*$
884+
885+
[FILTER]
886+
Name aws
887+
Match dataplane.*
888+
imds_version v2
889+
890+
[OUTPUT]
891+
Name cloudwatch_logs
892+
Match dataplane.*
893+
region ${local.region}
894+
log_group_name /aws/containerinsights/${local.cluster_name}/dataplane
895+
log_stream_prefix $${HOSTNAME}-
896+
auto_create_group true
897+
extra_user_agent container-insights
898+
EOT
899+
)
900+
"host-log.conf" = try(
901+
var.aws_for_fluentbit.host_log_conf,
902+
<<-EOT
903+
[INPUT]
904+
Name tail
905+
Tag host.dmesg
906+
Path /var/log/dmesg
907+
Key message
908+
DB /var/fluent-bit/state/flb_dmesg.db
909+
Mem_Buf_Limit 5MB
910+
Skip_Long_Lines On
911+
Refresh_Interval 10
912+
Read_from_Head Off
913+
914+
[INPUT]
915+
Name tail
916+
Tag host.messages
917+
Path /var/log/messages
918+
Parser syslog
919+
DB /var/fluent-bit/state/flb_messages.db
920+
Mem_Buf_Limit 5MB
921+
Skip_Long_Lines On
922+
Refresh_Interval 10
923+
Read_from_Head Off
924+
925+
[INPUT]
926+
Name tail
927+
Tag host.secure
928+
Path /var/log/secure
929+
Parser syslog
930+
DB /var/fluent-bit/state/flb_secure.db
931+
Mem_Buf_Limit 5MB
932+
Skip_Long_Lines On
933+
Refresh_Interval 10
934+
Read_from_Head Off
935+
936+
[FILTER]
937+
Name aws
938+
Match host.*
939+
imds_version v2
940+
941+
[OUTPUT]
942+
Name cloudwatch_logs
943+
Match host.*
944+
region ${local.region}
945+
log_group_name /aws/containerinsights/${local.cluster_name}/host
946+
log_stream_prefix $${HOSTNAME}.
947+
auto_create_group true
948+
extra_user_agent container-insights
949+
EOT
950+
)
951+
"parsers.conf" = try(
952+
var.aws_for_fluentbit.parsers_conf,
953+
<<-EOT
954+
[PARSER]
955+
Name syslog
956+
Format regex
957+
Regex ^(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
958+
Time_Key time
959+
Time_Format %b %d %H:%M:%S
960+
961+
[PARSER]
962+
Name container_firstline
963+
Format regex
964+
Regex (?<log>(?<="log":")\S(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
965+
Time_Key time
966+
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
967+
968+
[PARSER]
969+
Name cwagent_firstline
970+
Format regex
971+
Regex (?<log>(?<="log":")\d{4}[\/-]\d{1,2}[\/-]\d{1,2}[ T]\d{2}:\d{2}:\d{2}(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
972+
Time_Key time
973+
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
974+
EOT
975+
)
976+
}
977+
}
978+
753979
################################################################################
754980
# AWS FSX CSI DRIVER
755981
################################################################################

‎tests/complete/main.tf‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,8 @@ module "eks_blueprints_addons" {
144144
enable_kube_prometheus_stack = true
145145
enable_external_dns = true
146146
enable_external_secrets = true
147-
# enable_gatekeeper = true
148-
enable_ingress_nginx = true
147+
enable_gatekeeper = true
148+
# enable_ingress_nginx = true
149149

150150
# Turn off mutation webhook for services to avoid ordering issue
151151
enable_aws_load_balancer_controller = true
@@ -160,7 +160,19 @@ module "eks_blueprints_addons" {
160160
enable_vpa = true
161161
enable_fargate_fluentbit = true
162162
enable_aws_for_fluentbit = true
163+
aws_for_fluentbit_cw_log_group = {
164+
create = true
165+
use_name_prefix = true # Set this to true to enable name prefix
166+
name_prefix = "eks-cluster-logs-"
167+
retention = 7
168+
}
163169
aws_for_fluentbit = {
170+
enable_containerinsights = true
171+
chart_version = "0.1.28"
172+
set = [{
173+
name = "cloudWatchLogs.autoCreateGroup"
174+
value = true
175+
}]
164176
s3_bucket_arns = [
165177
module.velero_backup_s3_bucket.s3_bucket_arn,
166178
"${module.velero_backup_s3_bucket.s3_bucket_arn}/logs/*"

0 commit comments

Comments
 (0)