Skip to content
This repository was archived by the owner on Sep 30, 2020. It is now read-only.

Commit 133ae0b

Browse files
kfr2kfr2
committed
Allow dnsmasq to be backed by a local copy of CoreDNS
This commit allows the user to specify that dnsmasq should be backed by a pod-local copy of CoreDNS rather than relying on the global CoreDNS service. If enabled, the dnsmasq-node DaemonSet will be configured to use a local copy of CoreDNS for its resolution while setting the global CoreDNS service as a fallback. This is handy in situations where the number of DNS requests within a cluster grows large and causes resolution issues as dnsmasq reaches out to the global CoreDNS service. Additionally, several values passed to dnsmasq are now configurable including its `--cache-size` and `--dns-forward-max`. See [this postmortem](https://github.com/zalando-incubator/kubernetes-on-aws/blob/dev/docs/postmortems/jan-2019-dns-outage.md) for an investigation into this situation which was instrumental in understanding issues we were facing. Many thanks to dominicgunn for providing the manifests which I codified into this commit. --- These features can be enabled and tuned by setting the following values within cluster.yaml: ```yaml kubeDns: dnsmasq: coreDNSLocal: # When enabled, this will run a copy of CoreDNS within each DNS-masq pod and # configure the utility to use it for resolution. enabled: true # Defines the resource requests/limits for the coredns-local container. # cpu and/or memory constraints can be removed by setting the appropriate value(s) # to an empty string. resources: requests: cpu: 50m memory: 100Mi limits: cpu: 50m memory: 100Mi # The size of dnsmasq's cache. cacheSize: 50000 # The maximum number of concurrent DNS queries. dnsForwardMax: 500 # This option gives a default value for time-to-live (in seconds) which dnsmasq # uses to cache negative replies even in the absence of an SOA record. negTTL: 60 ```
1 parent 4518022 commit 133ae0b

File tree

5 files changed

+503
-40
lines changed

5 files changed

+503
-40
lines changed

‎builtin/files/cluster.yaml.tmpl

+33-2
Original file line numberDiff line numberDiff line change
@@ -1167,7 +1167,7 @@ kubernetes:
11671167
# Tells Kubernetes to enable the autoscaler rest client (not using heapster) without the requirement to use metrics-server.
11681168
podAutoscalerUseRestClient:
11691169
enabled: false
1170-
1170+
11711171
# controllerManager:
11721172
# resources:
11731173
# requests:
@@ -1329,6 +1329,35 @@ kubeDns:
13291329
# - --neg-ttl=10
13301330
# - --no-ping
13311331

1332+
# Settings for the dnsmasq-node DaemonSet which must be enabled by setting
1333+
# `kubeDns.nodeLocalResolver` to true.
1334+
dnsmasq:
1335+
coreDNSLocal:
1336+
# When enabled, this will run a copy of CoreDNS within each DNS-masq pod and
1337+
# configure the utility to use it for resolution.
1338+
enabled: false
1339+
1340+
# Defines the resource requests/limits for the coredns-local container.
1341+
# cpu and/or memory constraints can be removed by setting the appropriate value(s)
1342+
# to an empty string.
1343+
resources:
1344+
requests:
1345+
cpu: 50m
1346+
memory: 100Mi
1347+
limits:
1348+
cpu: 50m
1349+
memory: 100Mi
1350+
1351+
# The size of dnsmasq's cache.
1352+
cacheSize: 50000
1353+
1354+
# The maximum number of concurrent DNS queries.
1355+
dnsForwardMax: 500
1356+
1357+
# This option gives a default value for time-to-live (in seconds) which dnsmasq
1358+
# uses to cache negative replies even in the absence of an SOA record.
1359+
# negTTL: 60
1360+
13321361
# When enabled, will modify the TTL of the coredns service.
13331362
# ttl: 30
13341363

@@ -1351,6 +1380,8 @@ kubeDns:
13511380
#
13521381
# This configuration is injected into the CoreDNS config map after the root
13531382
# zone (".") and can be used to add configuration for additional zones.
1383+
# If coreDNSLocal has been enabled, this configuration will additionally be injected
1384+
# into its ConfigMap.
13541385
# additionalZoneCoreDNSConfig: |
13551386
# global:53 {
13561387
# errors
@@ -1378,7 +1409,7 @@ kubeProxy:
13781409
# When enabled, a security group rule is included on the generated kube-aws SG to allow ICMP Ping from all traffic (0.0.0.0/0).
13791410
# This is applied to all nodes (worker & control plane) in the cluster.
13801411
openICMP: true
1381-
1412+
13821413
# Addon features
13831414
addons:
13841415
# When enabled, Kubernetes rescheduler is deployed to the cluster controller(s)

‎builtin/files/userdata/cloud-config-controller

+178-16
Original file line numberDiff line numberDiff line change
@@ -1160,9 +1160,14 @@ write_files:
11601160
"${mfdir}/kube-dns-autoscaler-de.yaml" \
11611161
"${mfdir}/kube-dns-de.yaml"
11621162
{{- end }}
1163-
{{ if .KubeDns.NodeLocalResolver -}}
1163+
{{- if .KubeDns.NodeLocalResolver }}
1164+
{{- if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
1165+
deploy "${mfdir}/dnsmasq-node-coredns-local.yaml"
1166+
{{- else }}
1167+
remove "${mfdir}/dnsmasq-node-coredns-local.yaml"
1168+
{{- end }}
11641169
deploy "${mfdir}/dnsmasq-node-ds.yaml"
1165-
{{ end -}}
1170+
{{- end }}
11661171
forceapply "${mfdir}/kube-dns-pdb.yaml"
11671172

11681173
{{ if .Addons.MetricsServer.Enabled -}}
@@ -5384,6 +5389,9 @@ write_files:
53845389
namespace: kube-system
53855390
data:
53865391
Corefile: |
5392+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5393+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 10 }}
5394+
{{- end }}
53875395
.:53 {
53885396
errors
53895397
health
@@ -5406,9 +5414,6 @@ write_files:
54065414
reload
54075415
loadbalance
54085416
}
5409-
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5410-
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 10 }}
5411-
{{- end }}
54125417
{{- else }}
54135418
- path: /srv/kubernetes/manifests/kube-dns-sa.yaml
54145419
content: |
@@ -5474,6 +5479,85 @@ write_files:
54745479
- --v=2
54755480
- --logtostderr
54765481

5482+
{{ if and .KubeDns.NodeLocalResolver .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5483+
- path: /srv/kubernetes/manifests/dnsmasq-node-coredns-local.yaml
5484+
content: |
5485+
apiVersion: v1
5486+
kind: ServiceAccount
5487+
metadata:
5488+
name: dnsmasq
5489+
namespace: kube-system
5490+
---
5491+
apiVersion: rbac.authorization.k8s.io/v1
5492+
kind: ClusterRole
5493+
metadata:
5494+
name: dnsmasq
5495+
rules:
5496+
- apiGroups: [""]
5497+
resources: ["endpoints", "services", "pods", "namespaces"]
5498+
verbs: ["list", "watch"]
5499+
---
5500+
apiVersion: rbac.authorization.k8s.io/v1
5501+
kind: ClusterRoleBinding
5502+
metadata:
5503+
name: dnsmasq
5504+
roleRef:
5505+
apiGroup: rbac.authorization.k8s.io
5506+
kind: ClusterRole
5507+
name: dnsmasq
5508+
subjects:
5509+
- kind: ServiceAccount
5510+
name: dnsmasq
5511+
namespace: kube-system
5512+
---
5513+
apiVersion: rbac.authorization.k8s.io/v1
5514+
kind: RoleBinding
5515+
metadata:
5516+
name: dnsmasq-privileged-psp
5517+
namespace: kube-system
5518+
roleRef:
5519+
apiGroup: rbac.authorization.k8s.io
5520+
kind: ClusterRole
5521+
name: privileged-psp
5522+
subjects:
5523+
- kind: ServiceAccount
5524+
name: dnsmasq
5525+
namespace: kube-system
5526+
---
5527+
apiVersion: v1
5528+
kind: ConfigMap
5529+
metadata:
5530+
name: coredns-local
5531+
namespace: kube-system
5532+
labels:
5533+
application: coredns
5534+
data:
5535+
Corefile: |
5536+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5537+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 12 }}
5538+
{{- end }}
5539+
5540+
cluster.local:9254 {{ .PodCIDR }}:9254 {{ .ServiceCIDR }}:9254 {
5541+
errors
5542+
kubernetes {
5543+
pods insecure
5544+
}
5545+
cache 30
5546+
log svc.svc.cluster.local.
5547+
prometheus :9153
5548+
}
5549+
5550+
.:9254 {
5551+
errors
5552+
health :9154 # this is global for all servers
5553+
prometheus :9153
5554+
forward . /etc/resolv.conf
5555+
pprof 127.0.0.1:9156
5556+
cache 30
5557+
reload
5558+
}
5559+
{{ end }}
5560+
54775561
{{ if .KubeDns.NodeLocalResolver }}
54785562
- path: /srv/kubernetes/manifests/dnsmasq-node-ds.yaml
54795563
content: |
@@ -5485,9 +5569,12 @@ write_files:
54855569
labels:
54865570
k8s-app: dnsmasq-node
54875571
spec:
5572+
selector:
5573+
matchLabels:
5574+
k8s-app: dnsmasq-node
54885575
updateStrategy:
54895576
rollingUpdate:
5490-
maxUnavailable: 100%
5577+
maxUnavailable: 10%
54915578
type: RollingUpdate
54925579
selector:
54935580
matchLabels:
@@ -5512,15 +5599,24 @@ write_files:
55125599
configMap:
55135600
name: kube-dns
55145601
optional: true
5602+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5603+
- name: coredns-local-config
5604+
configMap:
5605+
name: coredns-local
5606+
items:
5607+
- key: Corefile
5608+
path: Corefile
5609+
{{ end }}
55155610
containers:
55165611
- name: dnsmasq
55175612
image: {{ .KubeDnsMasqImage.RepoWithTag }}
55185613
livenessProbe:
55195614
httpGet:
55205615
path: /healthcheck/dnsmasq
5521-
port: 10054
5616+
port: 9054
55225617
scheme: HTTP
55235618
initialDelaySeconds: 60
5619+
periodSeconds: 10
55245620
timeoutSeconds: 5
55255621
successThreshold: 1
55265622
failureThreshold: 5
@@ -5531,13 +5627,24 @@ write_files:
55315627
- -restartDnsmasq=true
55325628
- --
55335629
- -k
5534-
- --min-port=1024
5535-
- --cache-size=1000
5630+
- --cache-size={{ .KubeDns.DNSMasq.CacheSize }}
5631+
- --dns-forward-max={{ .KubeDns.DNSMasq.DNSForwardMax }}
5632+
- --log-facility=-
5633+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5634+
- --no-resolv
5635+
- --keep-in-foreground
5636+
- --neg-ttl={{ .KubeDns.DNSMasq.NegTTL }}
5637+
# Send requests to the last server (coredns-local) first and only
5638+
# fallback to the previous one (global coredns) if it's unreachable.
5639+
- --strict-order
5640+
- --server={{.DNSServiceIP}}#53
5641+
- --server=127.0.0.1#9254
5642+
{{ else }}
55365643
- --server=//{{.DNSServiceIP}}
55375644
- --server=/cluster.local/{{.DNSServiceIP}}
55385645
- --server=/in-addr.arpa/{{.DNSServiceIP}}
55395646
- --server=/ip6.arpa/{{.DNSServiceIP}}
5540-
- --log-facility=-
5647+
{{ end }}
55415648
{{- if ne (len .KubeDns.NodeLocalResolverOptions) 0 }}
55425649
{{- range .KubeDns.NodeLocalResolverOptions }}
55435650
- {{.}}
@@ -5553,8 +5660,10 @@ write_files:
55535660
# see: https://github.com/kubernetes/kubernetes/issues/29055 for details
55545661
resources:
55555662
requests:
5556-
cpu: 150m
5557-
memory: 20Mi
5663+
ephemeral-storage: 256Mi
5664+
limits:
5665+
cpu: 10m
5666+
memory: 45Mi
55585667
volumeMounts:
55595668
- name: kube-dns-config
55605669
mountPath: /etc/k8s/dns/dnsmasq-nanny
@@ -5563,7 +5672,7 @@ write_files:
55635672
livenessProbe:
55645673
httpGet:
55655674
path: /metrics
5566-
port: 10054
5675+
port: 9054
55675676
scheme: HTTP
55685677
initialDelaySeconds: 60
55695678
timeoutSeconds: 5
@@ -5572,17 +5681,70 @@ write_files:
55725681
args:
55735682
- --v=2
55745683
- --logtostderr
5684+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5685+
- --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A
5686+
{{ else }}
55755687
- --probe=dnsmasq,127.0.0.1:53,ec2.amazonaws.com,5,A
5688+
{{ end }}
5689+
- --prometheus-port=9054
55765690
ports:
5577-
- containerPort: 10054
5691+
- containerPort: 9054
55785692
name: metrics
55795693
protocol: TCP
55805694
resources:
55815695
requests:
5582-
memory: 20Mi
5696+
ephemeral-storage: 256Mi
5697+
limits:
5698+
cpu: 100m
5699+
memory: 50Mi
5700+
terminationMessagePath: /dev/termination-log
5701+
terminationMessagePolicy: File
5702+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5703+
- name: coredns
5704+
image: {{ .CoreDnsImage.RepoWithTag }}
5705+
args: ["-conf", "/etc/coredns/Corefile"]
5706+
volumeMounts:
5707+
- name: coredns-local-config
5708+
mountPath: /etc/coredns
5709+
ports:
5710+
- containerPort: 9254
5711+
name: dns
5712+
protocol: UDP
5713+
- containerPort: 9254
5714+
name: dns-tcp
5715+
protocol: TCP
5716+
livenessProbe:
5717+
httpGet:
5718+
path: /health
5719+
port: 9154
5720+
scheme: HTTP
5721+
initialDelaySeconds: 60
5722+
timeoutSeconds: 5
5723+
successThreshold: 1
5724+
failureThreshold: 5
5725+
resources:
5726+
requests:
5727+
ephemeral-storage: 256Mi
5728+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
5729+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
5730+
{{ end }}
5731+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
5732+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
5733+
{{ end }}
5734+
{{ if or .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5735+
limits:
5736+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
5737+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
5738+
{{ end }}
5739+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5740+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5741+
{{ end }}
5742+
{{ end }}
5743+
{{ end }}
55835744
hostNetwork: true
55845745
dnsPolicy: Default
5585-
automountServiceAccountToken: false
5746+
automountServiceAccountToken: true
5747+
serviceAccountName: dnsmasq
55865748
{{ end }}
55875749

55885750
{{- if eq .KubeDns.Provider "coredns" }}

‎pkg/api/cluster.go

+20-2
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,26 @@ func NewDefaultCluster() *Cluster {
184184
IPVSMode: ipvsMode,
185185
},
186186
KubeDns: KubeDns{
187-
Provider: "coredns",
188-
NodeLocalResolver: false,
187+
Provider: "coredns",
188+
NodeLocalResolver: false,
189+
DNSMasq: DNSMasq{
190+
CoreDNSLocal: CoreDNSLocal{
191+
Enabled: false,
192+
ComputeResources: ComputeResources{
193+
Requests: ResourceQuota{
194+
Cpu: "50m",
195+
Memory: "100Mi",
196+
},
197+
Limits: ResourceQuota{
198+
Cpu: "50m",
199+
Memory: "100Mi",
200+
},
201+
},
202+
},
203+
CacheSize: 50000,
204+
DNSForwardMax: 500,
205+
NegTTL: 60,
206+
},
189207
DeployToControllers: false,
190208
AntiAffinityAvailabilityZone: false,
191209
TTL: 30,

0 commit comments

Comments
 (0)