Skip to content

Commit 4550672

Browse files
kvvitkvvit
committed
Ready logstash with template
1 parent e4c8a42 commit 4550672

File tree

4 files changed

+145
-145
lines changed

4 files changed

+145
-145
lines changed

‎terraform/modules/k8s-addons/eks-filebeat.tf

Lines changed: 0 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -32,119 +32,3 @@ resource "helm_release" "filebeat" {
3232
]
3333
}
3434

35-
resource "kubectl_manifest" "elk_external_secrets" {
36-
count = local.elk.enabled ? 1 : 0
37-
yaml_body = <<-EOF
38-
apiVersion: external-secrets.io/v1beta1
39-
kind: ExternalSecret
40-
metadata:
41-
name: elastic-credentials
42-
namespace: elk
43-
spec:
44-
refreshInterval: 1m
45-
secretStoreRef:
46-
name: elastic-credentials
47-
kind: SecretStore
48-
target:
49-
name: elastic-credentials
50-
creationPolicy: Owner
51-
dataFrom:
52-
- extract:
53-
key: "/${var.name}/infra/elk"
54-
EOF
55-
depends_on = [
56-
helm_release.external_secrets
57-
]
58-
}
59-
60-
resource "kubectl_manifest" "elk_secretstore" {
61-
count = local.elk.enabled ? 1 : 0
62-
yaml_body = <<-EOF
63-
apiVersion: external-secrets.io/v1beta1
64-
kind: SecretStore
65-
metadata:
66-
name: elastic-credentials
67-
namespace: elk
68-
spec:
69-
provider:
70-
aws:
71-
service: SecretsManager
72-
region: us-east-1
73-
auth:
74-
jwt:
75-
serviceAccountRef:
76-
name: sa-filebeat
77-
EOF
78-
depends_on = [
79-
helm_release.external_secrets
80-
]
81-
}
82-
83-
resource "kubectl_manifest" "kibana_service_account" {
84-
count = local.elk.enabled ? 1 : 0
85-
yaml_body = <<-EOF
86-
apiVersion: v1
87-
kind: ServiceAccount
88-
metadata:
89-
annotations:
90-
eks.amazonaws.com/role-arn: ${aws_iam_role.elk_service_account_role[count.index].arn}
91-
name: sa-filebeat
92-
namespace: elk
93-
EOF
94-
depends_on = [
95-
aws_iam_role.elk_service_account_role
96-
]
97-
}
98-
99-
resource "aws_iam_role" "elk_service_account_role" {
100-
count = local.elk.enabled ? 1 : 0
101-
name = "elk-service-account-role"
102-
assume_role_policy = <<EOF
103-
{
104-
"Version": "2012-10-17",
105-
"Statement": [
106-
{
107-
"Effect": "Allow",
108-
"Principal": {
109-
"Federated": "${var.eks_oidc_provider_arn}"
110-
},
111-
"Action": "sts:AssumeRoleWithWebIdentity",
112-
"Condition": {
113-
"StringEquals": {
114-
"oidc.eks.${var.region}.amazonaws.com/id/${regex("[A-Z0-9]{32}", var.eks_oidc_provider_arn)}:aud": "sts.amazonaws.com"
115-
}
116-
}
117-
}
118-
]
119-
}
120-
EOF
121-
}
122-
123-
resource "aws_iam_policy" "elk_secret_manager_policy" {
124-
count = local.elk.enabled ? 1 : 0
125-
name = "elk-secret-manager-policy"
126-
127-
policy = jsonencode({
128-
Version = "2012-10-17",
129-
Statement = [
130-
{
131-
Action : [
132-
"secretsmanager:GetSecretValue",
133-
"secretsmanager:DescribeSecret",
134-
"secretsmanager:ListSecretVersionIds",
135-
"secretsmanager:UpdateSecret",
136-
"secretsmanager:PutSecretValue"
137-
],
138-
Effect = "Allow",
139-
Resource = "arn:aws:secretsmanager:${var.region}:${data.aws_caller_identity.current.account_id}:secret:/${var.name}/infra/elk*"
140-
}
141-
]
142-
})
143-
}
144-
145-
resource "aws_iam_policy_attachment" "elk_secret_manager_policy_attachment" {
146-
count = local.elk.enabled ? 1 : 0
147-
name = "elk-secret-manager-policy-attachment"
148-
roles = [aws_iam_role.elk_service_account_role[count.index].name]
149-
policy_arn = aws_iam_policy.elk_secret_manager_policy[count.index].arn
150-
}

‎terraform/modules/k8s-addons/eks-logstash.tf

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,119 @@ resource "helm_release" "logstash" {
2929
])
3030
}
3131

32+
resource "kubectl_manifest" "elk_external_secrets" {
33+
count = local.logstash.enabled ? 1 : 0
34+
yaml_body = <<-EOF
35+
apiVersion: external-secrets.io/v1beta1
36+
kind: ExternalSecret
37+
metadata:
38+
name: elastic-credentials
39+
namespace: elk
40+
spec:
41+
refreshInterval: 1m
42+
secretStoreRef:
43+
name: elastic-credentials
44+
kind: SecretStore
45+
target:
46+
name: elastic-credentials
47+
creationPolicy: Owner
48+
dataFrom:
49+
- extract:
50+
key: "/${var.name}/infra/elk"
51+
EOF
52+
depends_on = [
53+
helm_release.external_secrets
54+
]
55+
}
56+
57+
resource "kubectl_manifest" "elk_secretstore" {
58+
count = local.logstash.enabled ? 1 : 0
59+
yaml_body = <<-EOF
60+
apiVersion: external-secrets.io/v1beta1
61+
kind: SecretStore
62+
metadata:
63+
name: elastic-credentials
64+
namespace: elk
65+
spec:
66+
provider:
67+
aws:
68+
service: SecretsManager
69+
region: us-east-1
70+
auth:
71+
jwt:
72+
serviceAccountRef:
73+
name: sa-filebeat
74+
EOF
75+
depends_on = [
76+
helm_release.external_secrets
77+
]
78+
}
79+
80+
resource "kubectl_manifest" "kibana_service_account" {
81+
count = local.logstash.enabled ? 1 : 0
82+
yaml_body = <<-EOF
83+
apiVersion: v1
84+
kind: ServiceAccount
85+
metadata:
86+
annotations:
87+
eks.amazonaws.com/role-arn: ${aws_iam_role.elk_service_account_role[count.index].arn}
88+
name: sa-filebeat
89+
namespace: elk
90+
EOF
91+
depends_on = [
92+
aws_iam_role.elk_service_account_role
93+
]
94+
}
95+
96+
resource "aws_iam_role" "elk_service_account_role" {
97+
count = local.logstash.enabled ? 1 : 0
98+
name = "elk-service-account-role"
99+
assume_role_policy = <<EOF
100+
{
101+
"Version": "2012-10-17",
102+
"Statement": [
103+
{
104+
"Effect": "Allow",
105+
"Principal": {
106+
"Federated": "${var.eks_oidc_provider_arn}"
107+
},
108+
"Action": "sts:AssumeRoleWithWebIdentity",
109+
"Condition": {
110+
"StringEquals": {
111+
"oidc.eks.${var.region}.amazonaws.com/id/${regex("[A-Z0-9]{32}", var.eks_oidc_provider_arn)}:aud": "sts.amazonaws.com"
112+
}
113+
}
114+
}
115+
]
116+
}
117+
EOF
118+
}
119+
120+
resource "aws_iam_policy" "elk_secret_manager_policy" {
121+
count = local.logstash.enabled ? 1 : 0
122+
name = "elk-secret-manager-policy"
123+
124+
policy = jsonencode({
125+
Version = "2012-10-17",
126+
Statement = [
127+
{
128+
Action : [
129+
"secretsmanager:GetSecretValue",
130+
"secretsmanager:DescribeSecret",
131+
"secretsmanager:ListSecretVersionIds",
132+
"secretsmanager:UpdateSecret",
133+
"secretsmanager:PutSecretValue"
134+
],
135+
Effect = "Allow",
136+
Resource = "arn:aws:secretsmanager:${var.region}:${data.aws_caller_identity.current.account_id}:secret:/${var.name}/infra/elk*"
137+
}
138+
]
139+
})
140+
}
141+
142+
resource "aws_iam_policy_attachment" "elk_secret_manager_policy_attachment" {
143+
count = local.logstash.enabled ? 1 : 0
144+
name = "elk-secret-manager-policy-attachment"
145+
roles = [aws_iam_role.elk_service_account_role[count.index].name]
146+
policy_arn = aws_iam_policy.elk_secret_manager_policy[count.index].arn
147+
}

‎terraform/modules/k8s-addons/elk-templates/filebeat-values.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ filebeatConfig:
2424
- drop_event.when.and:
2525
- equals.k8s-app: "coredns"
2626
- regexp.message: ".*NOERROR.*"
27-
setup.template.name: "filebeat"
28-
setup.template.pattern: "filebeat-*"
2927
setup.ilm.enabled: false
3028
output.logstash:
3129
hosts: ["logstash-logstash.elk.svc:5044"]

‎terraform/modules/k8s-addons/elk-templates/logstash-values.yaml

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,6 @@ resources:
88
memory: 2Gi
99

1010

11-
# persistence:
12-
# enabled: true
13-
1411
logstashConfig:
1512
logstash.yml: |
1613
http.host: 0.0.0.0
@@ -33,38 +30,43 @@ logstashPipeline:
3330
hosts => ["$${ELASTICSEARCH_HOST}:9200"]
3431
user => '$${ELASTICSEARCH_USERNAME}'
3532
password => '$${ELASTICSEARCH_PASSWORD}'
36-
index => "filebeat-%%{+yyyy.MM.dd}-000001"
37-
manage_template => false
33+
index => "filebeat-${env}-%%{+yyyy.MM.dd}-000001"
34+
manage_template => true
35+
template => '/etc/elk/logstash-index-template.json'
36+
template_name => 'filebeat-${env}'
37+
template_overwrite => true
38+
action => "create"
3839
ssl => true
3940
ssl_certificate_verification => false
4041
cacert => '/etc/logstash/certificates/ca.crt'
4142
}
4243
stdout { codec => rubydebug }
4344
}
4445
45-
# https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html
46+
secrets:
47+
- name: "index-template"
48+
value:
49+
logstash-index-template.json: |
50+
{
51+
"index_patterns": ["filebeat-${env}-*"],
52+
"template": {
53+
"aliases": {
54+
"filebeat-${env}": {}
55+
},
56+
"settings": {
57+
"index.mapping.total_fields.limit": 5000,
58+
"index.lifecycle.name": "delete_old_${env}_indicies",
59+
"index.lifecycle.rollover_alias": "filebeat-${env}",
60+
"number_of_shards": 1,
61+
"number_of_replicas": 1
62+
}
63+
}
64+
}
4665
47-
#secrets:
48-
# - name: "index-template"
49-
# value:
50-
# logstash-index-template.json: |
51-
# {
52-
# "index_patterns": ["filebeat-*"],
53-
# "template": {
54-
# "settings": {
55-
# "index.refresh_interval": "5s",
56-
# "index.mapping.total_fields.limit": 5000,
57-
# "index.lifecycle.name": "filebeat-policy",
58-
# "number_of_shards": 1,
59-
# "number_of_replicas": 1
60-
# }
61-
# }
62-
# }
63-
#
64-
#secretMounts:
65-
# - name: logstash-index-template
66-
# secretName: logstash-logstash-index-template
67-
# path: /etc/elk
66+
secretMounts:
67+
- name: logstash-index-template
68+
secretName: logstash-logstash-index-template
69+
path: /etc/elk
6870

6971
extraPorts:
7072
- name: beats

0 commit comments

Comments
 (0)