Allow to add options to /etc/resolv.conf on worker nodes (#342)

* add systemd service to update resolv.conf according to resolvConfOptions field

* added unit tests for EnsureAdditionalUnits and EnsureAdditionalFiles; minor refactoring

* use []string for option list, fix newline for flatcar

Author: MartinWeindel

docs/usage-as-operator.md

@@ -65,6 +65,9 @@ machineImages:
 # serverGroupPolicies:
 # - soft-anti-affinity
 # - anti-affinity
+# resolvConfOptions:
+# - rotate
+# - timeout:1
 constraints:
   floatingPools:
   - name: fp-pool-1
@@ -121,6 +124,10 @@ omit `keystoneURL` and always specify `region`.
 
 If Gardener creates and manages the router of a shoot cluster, it is additionally possible to specify that the [enable_snat](https://registry.terraform.io/providers/terraform-provider-openstack/openstack/latest/docs/resources/networking_router_v2#enable_snat) field is set to `true` via `useSNAT: true` in the `CloudProfileConfig`.
 
+On some OpenStack enviroments, there may be the need to set options in the file `/etc/resolv.conf` on worker nodes.
+If the field `resolvConfOptions` is set, a systemd service will be installed which copies `/run/systemd/resolve/resolv.conf`
+on every change to `/etc/resolv.conf` and appends the given options.
+
 ## Example `CloudProfile` manifest
 
 Please find below an example `CloudProfile` manifest:

hack/api-reference/api.md

@@ -216,6 +216,18 @@ bool
 <p>ServerGroupPolicies specify the allowed server group policies for worker groups.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>resolvConfOptions</code></br>
+<em>
+[]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>ResolvConfOptions specifies options to be added to /etc/resolv.conf on workers</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="openstack.provider.extensions.gardener.cloud/v1alpha1.ControlPlaneConfig">ControlPlaneConfig

pkg/apis/openstack/types_cloudprofile.go

@@ -58,6 +58,8 @@ type CloudProfileConfig struct {
 	UseSNAT *bool
 	// ServerGroupPolicies specify the allowed server group policies for worker groups.
 	ServerGroupPolicies []string
+	// ResolvConfOptions specifies options to be added to /etc/resolv.conf on workers
+	ResolvConfOptions []string
 }
 
 // Constraints is an object containing constraints for the shoots.

pkg/apis/openstack/v1alpha1/types_cloudprofile.go

@@ -67,6 +67,9 @@ type CloudProfileConfig struct {
 	// ServerGroupPolicies specify the allowed server group policies for worker groups.
 	// +optional
 	ServerGroupPolicies []string `json:"serverGroupPolicies,omitempty"`
+	// ResolvConfOptions specifies options to be added to /etc/resolv.conf on workers
+	// +optional
+	ResolvConfOptions []string `json:"resolvConfOptions,omitempty"`
 }
 
 // Constraints is an object containing constraints for the shoots.

pkg/apis/openstack/v1alpha1/zz_generated.conversion.go

@@ -17,7 +17,10 @@ package controlplane
 import (
 	"context"
 	"fmt"
+	"strings"
 
+	apisopenstack "github.com/gardener/gardener-extension-provider-openstack/pkg/apis/openstack"
+	"github.com/gardener/gardener-extension-provider-openstack/pkg/apis/openstack/helper"
 	"github.com/gardener/gardener-extension-provider-openstack/pkg/openstack"
 
 	"github.com/coreos/go-systemd/v22/unit"
@@ -28,6 +31,7 @@ import (
 	"github.com/gardener/gardener/extensions/pkg/webhook/controlplane"
 	"github.com/gardener/gardener/extensions/pkg/webhook/controlplane/genericmutator"
 	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
 	versionutils "github.com/gardener/gardener/pkg/utils/version"
 	"github.com/go-logr/logr"
@@ -451,3 +455,128 @@ func (e *ensurer) EnsureKubeletCloudProviderConfig(ctx context.Context, gctx gco
 	*data = string(secret.Data[openstack.CloudProviderConfigDataKey])
 	return nil
 }
+
+// EnsureAdditionalUnits ensures that additional required system units are added.
+func (e *ensurer) EnsureAdditionalUnits(ctx context.Context, gctx gcontext.GardenContext, new, old *[]extensionsv1alpha1.Unit) error {
+	cloudProfileConfig, err := getCloudProfileConfig(ctx, gctx)
+	if err != nil {
+		return err
+	}
+	if len(getResolveConfOptions(cloudProfileConfig)) > 0 {
+		e.addAdditionalUnitsForResolvConfOptions(new)
+	}
+	return nil
+}
+
+// addAdditionalUnitsForResolvConfOptions installs a systemd service to update `resolv.conf`
+// after each change of `/run/systemd/resolve/resolv.conf`.
+func (e *ensurer) addAdditionalUnitsForResolvConfOptions(new *[]extensionsv1alpha1.Unit) {
+	var (
+		trueVar           = true
+		customPathContent = `[Path]
+PathChanged=/run/systemd/resolve/resolv.conf
+
+[Install]
+WantedBy=multi-user.target
+`
+		customUnitContent = `[Unit]
+Description=add options to /etc/resolv.conf after every change of /run/systemd/resolve/resolv.conf
+After=network.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=oneshot
+ExecStart=/opt/bin/update-resolv-conf.sh
+`
+	)
+
+	extensionswebhook.AppendUniqueUnit(new, extensionsv1alpha1.Unit{
+		Name:    "update-resolv-conf.path",
+		Enable:  &trueVar,
+		Content: &customPathContent,
+	})
+	extensionswebhook.AppendUniqueUnit(new, extensionsv1alpha1.Unit{
+		Name:    "update-resolv-conf.service",
+		Enable:  &trueVar,
+		Content: &customUnitContent,
+	})
+}
+
+// EnsureAdditionalFiles ensures that additional required system files are added.
+func (e *ensurer) EnsureAdditionalFiles(ctx context.Context, gctx gcontext.GardenContext, new, old *[]extensionsv1alpha1.File) error {
+	cloudProfileConfig, err := getCloudProfileConfig(ctx, gctx)
+	if err != nil {
+		return err
+	}
+	if options := getResolveConfOptions(cloudProfileConfig); len(options) > 0 {
+		e.addAdditionalFilesForResolvConfOptions(options, new)
+	}
+	return nil
+}
+
+// addAdditionalFilesForResolvConfOptions writes the script to update `/etc/resolv.conf` from
+// `/run/systemd/resolve/resolv.conf` and adds a options line to it.
+func (e *ensurer) addAdditionalFilesForResolvConfOptions(options []string, new *[]extensionsv1alpha1.File) {
+	var (
+		permissions int32 = 0755
+		template          = `#!/bin/sh
+
+file=/run/systemd/resolve/resolv.conf
+tmp=/etc/resolv.conf.new
+dest=/etc/resolv.conf
+line=%q
+
+if [ -f "$file" ]; then
+  cp "$file" "$tmp"
+  echo "" >> "$tmp"
+  echo "# updated by update-resolv-conf.service (installed by gardener-extension-provider-openstack)" >> "$tmp"
+  echo "$line" >> "$tmp"
+  mv "$tmp" "$dest"
+fi`
+	)
+
+	content := fmt.Sprintf(template, fmt.Sprintf("options %s", strings.Join(options, " ")))
+	file := extensionsv1alpha1.File{
+		Path:        "/opt/bin/update-resolv-conf.sh",
+		Permissions: &permissions,
+		Content: extensionsv1alpha1.FileContent{
+			Inline: &extensionsv1alpha1.FileContentInline{
+				Encoding: "",
+				Data:     content,
+			},
+		},
+	}
+	appendUniqueFile(new, file)
+}
+
+func getCloudProfileConfig(ctx context.Context, gctx gcontext.GardenContext) (*apisopenstack.CloudProfileConfig, error) {
+	cluster, err := gctx.GetCluster(ctx)
+	if err != nil {
+		return nil, err
+	}
+	cloudProfileConfig, err := helper.CloudProfileConfigFromCluster(cluster)
+	if err != nil {
+		return nil, err
+	}
+	return cloudProfileConfig, nil
+}
+
+func getResolveConfOptions(cloudProfileConfig *apisopenstack.CloudProfileConfig) []string {
+	if cloudProfileConfig == nil {
+		return nil
+	}
+	return cloudProfileConfig.ResolvConfOptions
+}
+
+// appendUniqueFile appends a unit file only if it does not exist, otherwise overwrite content of previous files
+func appendUniqueFile(files *[]extensionsv1alpha1.File, file extensionsv1alpha1.File) {
+	resFiles := make([]extensionsv1alpha1.File, 0, len(*files))
+
+	for _, f := range *files {
+		if f.Path != file.Path {
+			resFiles = append(resFiles, f)
+		}
+	}
+
+	*files = append(resFiles, file)
+}