Skip to content
This repository was archived by the owner on Sep 30, 2020. It is now read-only.

Commit d14e2fb

Browse files
dominicgunndominicgunn
authored
Merge pull request #1891 from kfr2/coredns-local-v0.14.x
[v0.14.x] Allow dnsmasq to be backed by a local copy of CoreDNS
2 parents ae2f65a + 2d28eca commit d14e2fb

File tree

5 files changed

+500
-38
lines changed

5 files changed

+500
-38
lines changed

‎builtin/files/cluster.yaml.tmpl

+31
Original file line numberDiff line numberDiff line change
@@ -1360,6 +1360,35 @@ kubeDns:
13601360
# - --neg-ttl=10
13611361
# - --no-ping
13621362

1363+
# Settings for the dnsmasq-node DaemonSet which must be enabled by setting
1364+
# `kubeDns.nodeLocalResolver` to true.
1365+
dnsmasq:
1366+
coreDNSLocal:
1367+
# When enabled, this will run a copy of CoreDNS within each DNS-masq pod and
1368+
# configure the utility to use it for resolution.
1369+
enabled: false
1370+
1371+
# Defines the resource requests/limits for the coredns-local container.
1372+
# cpu and/or memory constraints can be removed by setting the appropriate value(s)
1373+
# to an empty string.
1374+
resources:
1375+
requests:
1376+
cpu: 50m
1377+
memory: 100Mi
1378+
limits:
1379+
cpu: 50m
1380+
memory: 100Mi
1381+
1382+
# The size of dnsmasq's cache.
1383+
cacheSize: 50000
1384+
1385+
# The maximum number of concurrent DNS queries.
1386+
dnsForwardMax: 500
1387+
1388+
# This option gives a default value for time-to-live (in seconds) which dnsmasq
1389+
# uses to cache negative replies even in the absence of an SOA record.
1390+
# negTTL: 60
1391+
13631392
# When enabled, will modify the TTL of the coredns service.
13641393
# ttl: 30
13651394

@@ -1381,6 +1410,8 @@ kubeDns:
13811410
#
13821411
# This configuration is injected into the CoreDNS config map after the root
13831412
# zone (".") and can be used to add configuration for additional zones.
1413+
# If corednsLocal has been enabled, this configuration will additionally be injected
1414+
# into its ConfigMap.
13841415
# additionalZoneCoreDNSConfig: |
13851416
# global:53 {
13861417
# errors

‎builtin/files/userdata/cloud-config-controller

+177-16
Original file line numberDiff line numberDiff line change
@@ -1058,6 +1058,11 @@ write_files:
10581058
"${mfdir}/kube-dns-de.yaml"
10591059
{{- end }}
10601060
{{ if .KubeDns.NodeLocalResolver -}}
1061+
{{ if .KubeDns.dnsmasq.CoreDNSLocal.Enabled -}}
1062+
deploy "${mfdir}/dnsmasq-node-coredns-local.yaml"
1063+
{{- else }}
1064+
remove "${mfdir}/dnsmasq-node-coredns-local.yaml"
1065+
{{ end -}}
10611066
deploy "${mfdir}/dnsmasq-node-ds.yaml"
10621067
{{ end -}}
10631068
forceapply "${mfdir}/kube-dns-pdb.yaml"
@@ -3882,6 +3887,10 @@ write_files:
38823887
namespace: kube-system
38833888
data:
38843889
Corefile: |
3890+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
3891+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 12 }}
3892+
{{- end }}
3893+
38853894
.:53 {
38863895
errors
38873896
health
@@ -3904,9 +3913,6 @@ write_files:
39043913
reload
39053914
loadbalance
39063915
}
3907-
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
3908-
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 12 }}
3909-
{{- end }}
39103916
{{- else }}
39113917
- path: /srv/kubernetes/manifests/kube-dns-sa.yaml
39123918
content: |
@@ -3969,6 +3975,85 @@ write_files:
39693975
- --v=2
39703976
- --logtostderr
39713977

3978+
{{ if and .KubeDns.NodeLocalResolver .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
3979+
- path: /srv/kubernetes/manifests/dnsmasq-node-coredns-local.yaml
3980+
content: |
3981+
apiVersion: v1
3982+
kind: ServiceAccount
3983+
metadata:
3984+
name: dnsmasq
3985+
namespace: kube-system
3986+
---
3987+
apiVersion: rbac.authorization.k8s.io/v1
3988+
kind: ClusterRole
3989+
metadata:
3990+
name: dnsmasq
3991+
rules:
3992+
- apiGroups: [""]
3993+
resources: ["endpoints", "services", "pods", "namespaces"]
3994+
verbs: ["list", "watch"]
3995+
---
3996+
apiVersion: rbac.authorization.k8s.io/v1
3997+
kind: ClusterRoleBinding
3998+
metadata:
3999+
name: dnsmasq
4000+
roleRef:
4001+
apiGroup: rbac.authorization.k8s.io
4002+
kind: ClusterRole
4003+
name: dnsmasq
4004+
subjects:
4005+
- kind: ServiceAccount
4006+
name: dnsmasq
4007+
namespace: kube-system
4008+
---
4009+
apiVersion: rbac.authorization.k8s.io/v1
4010+
kind: RoleBinding
4011+
metadata:
4012+
name: dnsmasq-privileged-psp
4013+
namespace: kube-system
4014+
roleRef:
4015+
apiGroup: rbac.authorization.k8s.io
4016+
kind: ClusterRole
4017+
name: privileged-psp
4018+
subjects:
4019+
- kind: ServiceAccount
4020+
name: dnsmasq
4021+
namespace: kube-system
4022+
---
4023+
apiVersion: v1
4024+
kind: ConfigMap
4025+
metadata:
4026+
name: coredns-local
4027+
namespace: kube-system
4028+
labels:
4029+
application: coredns
4030+
data:
4031+
Corefile: |
4032+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
4033+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 12 }}
4034+
{{- end }}
4035+
4036+
cluster.local:9254 {{ .PodCIDR }}:9254 {{ .ServiceCIDR }}:9254 {
4037+
errors
4038+
kubernetes {
4039+
pods insecure
4040+
}
4041+
cache 30
4042+
log svc.svc.cluster.local.
4043+
prometheus :9153
4044+
}
4045+
4046+
.:9254 {
4047+
errors
4048+
health :9154 # this is global for all servers
4049+
prometheus :9153
4050+
forward . /etc/resolv.conf
4051+
pprof 127.0.0.1:9156
4052+
cache 30
4053+
reload
4054+
}
4055+
{{ end }}
4056+
39724057
{{ if .KubeDns.NodeLocalResolver }}
39734058
- path: /srv/kubernetes/manifests/dnsmasq-node-ds.yaml
39744059
content: |
@@ -3980,9 +4065,12 @@ write_files:
39804065
labels:
39814066
k8s-app: dnsmasq-node
39824067
spec:
4068+
selector:
4069+
matchLabels:
4070+
k8s-app: dnsmasq-node
39834071
updateStrategy:
39844072
rollingUpdate:
3985-
maxUnavailable: 100%
4073+
maxUnavailable: 10%
39864074
type: RollingUpdate
39874075
template:
39884076
metadata:
@@ -3997,22 +4085,29 @@ write_files:
39974085
effect: NoSchedule
39984086
- operator: Exists
39994087
effect: NoExecute
4000-
- operator: Exists
4001-
key: CriticalAddonsOnly
40024088
volumes:
40034089
- name: kube-dns-config
40044090
configMap:
40054091
name: kube-dns
40064092
optional: true
4093+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
4094+
- name: coredns-local-config
4095+
configMap:
4096+
name: coredns-local
4097+
items:
4098+
- key: Corefile
4099+
path: Corefile
4100+
{{ end }}
40074101
containers:
40084102
- name: dnsmasq
40094103
image: {{ .KubeDnsMasqImage.RepoWithTag }}
40104104
livenessProbe:
40114105
httpGet:
40124106
path: /healthcheck/dnsmasq
4013-
port: 10054
4107+
port: 9054
40144108
scheme: HTTP
40154109
initialDelaySeconds: 60
4110+
periodSeconds: 10
40164111
timeoutSeconds: 5
40174112
successThreshold: 1
40184113
failureThreshold: 5
@@ -4023,13 +4118,24 @@ write_files:
40234118
- -restartDnsmasq=true
40244119
- --
40254120
- -k
4026-
- --min-port=1024
4027-
- --cache-size=1000
4121+
- --cache-size={{ .KubeDns.DNSMasq.CacheSize }}
4122+
- --dns-forward-max={{ .KubeDns.DNSMasq.DNSForwardMax }}
4123+
- --log-facility=-
4124+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
4125+
- --no-resolv
4126+
- --keep-in-foreground
4127+
- --neg-ttl={{ .KubeDns.DNSMasq.NegTTL }}
4128+
# Send requests to the last server (coredns-local) first and only
4129+
# fallback to the previous one (global coredns) if it's unreachable.
4130+
- --strict-order
4131+
- --server={{.DNSServiceIP}}#53
4132+
- --server=127.0.0.1#9254
4133+
{{ else }}
40284134
- --server=//{{.DNSServiceIP}}
40294135
- --server=/cluster.local/{{.DNSServiceIP}}
40304136
- --server=/in-addr.arpa/{{.DNSServiceIP}}
40314137
- --server=/ip6.arpa/{{.DNSServiceIP}}
4032-
- --log-facility=-
4138+
{{ end }}
40334139
{{- if ne (len .KubeDns.NodeLocalResolverOptions) 0 }}
40344140
{{- range .KubeDns.NodeLocalResolverOptions }}
40354141
- {{.}}
@@ -4044,9 +4150,11 @@ write_files:
40444150
protocol: TCP
40454151
# see: https://github.com/kubernetes/kubernetes/issues/29055 for details
40464152
resources:
4153+
limits:
4154+
cpu: 100m
4155+
memory: 50Mi
40474156
requests:
4048-
cpu: 150m
4049-
memory: 20Mi
4157+
ephemeral-storage: 256Mi
40504158
volumeMounts:
40514159
- name: kube-dns-config
40524160
mountPath: /etc/k8s/dns/dnsmasq-nanny
@@ -4055,7 +4163,7 @@ write_files:
40554163
livenessProbe:
40564164
httpGet:
40574165
path: /metrics
4058-
port: 10054
4166+
port: 9054
40594167
scheme: HTTP
40604168
initialDelaySeconds: 60
40614169
timeoutSeconds: 5
@@ -4064,17 +4172,70 @@ write_files:
40644172
args:
40654173
- --v=2
40664174
- --logtostderr
4175+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
4176+
- --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A
4177+
{{ else }}
40674178
- --probe=dnsmasq,127.0.0.1:53,ec2.amazonaws.com,5,A
4179+
{{ end }}
4180+
- --prometheus-port=9054
40684181
ports:
4069-
- containerPort: 10054
4182+
- containerPort: 9054
40704183
name: metrics
40714184
protocol: TCP
40724185
resources:
40734186
requests:
4074-
memory: 20Mi
4187+
ephemeral-storage: 256Mi
4188+
limits:
4189+
cpu: 10m
4190+
memory: 45Mi
4191+
terminationMessagePath: /dev/termination-log
4192+
terminationMessagePolicy: File
4193+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
4194+
- name: coredns
4195+
image: {{ .CoreDnsImage.RepoWithTag }}
4196+
args: ["-conf", "/etc/coredns/Corefile"]
4197+
volumeMounts:
4198+
- name: coredns-local-config
4199+
mountPath: /etc/coredns
4200+
ports:
4201+
- containerPort: 9254
4202+
name: dns
4203+
protocol: UDP
4204+
- containerPort: 9254
4205+
name: dns-tcp
4206+
protocol: TCP
4207+
livenessProbe:
4208+
httpGet:
4209+
path: /health
4210+
port: 9154
4211+
scheme: HTTP
4212+
initialDelaySeconds: 60
4213+
timeoutSeconds: 5
4214+
successThreshold: 1
4215+
failureThreshold: 5
4216+
resources:
4217+
requests:
4218+
ephemeral-storage: 256Mi
4219+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
4220+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
4221+
{{ end }}
4222+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
4223+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
4224+
{{ end }}
4225+
{{ if or .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
4226+
limits:
4227+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
4228+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
4229+
{{ end }}
4230+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
4231+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
4232+
{{ end }}
4233+
{{ end }}
4234+
{{ end }}
40754235
hostNetwork: true
40764236
dnsPolicy: Default
4077-
automountServiceAccountToken: false
4237+
automountServiceAccountToken: true
4238+
serviceAccountName: dnsmasq
40784239
{{ end }}
40794240

40804241
{{- if eq .KubeDns.Provider "coredns" }}

‎pkg/api/cluster.go

+20-2
Original file line numberDiff line numberDiff line change
@@ -161,8 +161,26 @@ func NewDefaultCluster() *Cluster {
161161
IPVSMode: ipvsMode,
162162
},
163163
KubeDns: KubeDns{
164-
Provider: "coredns",
165-
NodeLocalResolver: false,
164+
Provider: "coredns",
165+
NodeLocalResolver: false,
166+
DNSMasq: DNSMasq{
167+
CoreDNSLocal: CoreDNSLocal{
168+
Enabled: false,
169+
ComputeResources: ComputeResources{
170+
Requests: ResourceQuota{
171+
Cpu: "50m",
172+
Memory: "100Mi",
173+
},
174+
Limits: ResourceQuota{
175+
Cpu: "50m",
176+
Memory: "100Mi",
177+
},
178+
},
179+
},
180+
CacheSize: 50000,
181+
DNSForwardMax: 500,
182+
NegTTL: 60,
183+
},
166184
DeployToControllers: false,
167185
AntiAffinityAvailabilityZone: false,
168186
TTL: 30,

‎pkg/api/types.go

+13
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,18 @@ type IPVSMode struct {
209209
MinSyncPeriod string `yaml:"minSyncPeriod"`
210210
}
211211

212+
type CoreDNSLocal struct {
213+
Enabled bool `yaml:"enabled"`
214+
ComputeResources ComputeResources `yaml:"resources,omitempty"`
215+
}
216+
217+
type DNSMasq struct {
218+
CoreDNSLocal CoreDNSLocal `yaml:"coreDNSLocal"`
219+
CacheSize int `yaml:"cacheSize"`
220+
DNSForwardMax int `yaml:"dnsForwardMax"`
221+
NegTTL int `yaml:"negTTL"`
222+
}
223+
212224
type KubeDnsAutoscaler struct {
213225
CoresPerReplica int `yaml:"coresPerReplica"`
214226
NodesPerReplica int `yaml:"nodesPerReplica"`
@@ -219,6 +231,7 @@ type KubeDns struct {
219231
Provider string `yaml:"provider"`
220232
NodeLocalResolver bool `yaml:"nodeLocalResolver"`
221233
NodeLocalResolverOptions []string `yaml:"nodeLocalResolverOptions"`
234+
DNSMasq DNSMasq `yaml:"dnsmasq"`
222235
DeployToControllers bool `yaml:"deployToControllers"`
223236
AntiAffinityAvailabilityZone bool `yaml:"antiAffinityAvailabilityZone"`
224237
TTL int `yaml:"ttl"`

0 commit comments

Comments
 (0)