Skip to content

Commit 830f9cc

Browse files
brandondbrandond
committed
Fix embedded excutor VPN config injection
Allow the executor to modify node config before certs are generated, and use this to add VPN node IPs to kubelet serving cert
1 parent 33f9173 commit 830f9cc

6 files changed

Lines changed: 129 additions & 118 deletions

File tree

‎pkg/agent/config/config.go‎

Lines changed: 46 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/k3s-io/k3s/pkg/clientaccess"
2828
"github.com/k3s-io/k3s/pkg/daemons/config"
2929
"github.com/k3s-io/k3s/pkg/daemons/control/deps"
30+
"github.com/k3s-io/k3s/pkg/daemons/executor"
3031
"github.com/k3s-io/k3s/pkg/spegel"
3132
"github.com/k3s-io/k3s/pkg/util"
3233
"github.com/k3s-io/k3s/pkg/util/errors"
@@ -515,54 +516,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
515516

516517
os.Setenv("NODE_NAME", nodeName)
517518

518-
// Ensure that the kubelet's server certificate is valid for all configured node IPs. Note
519-
// that in the case of an external CCM, additional IPs may be added by the infra provider
520-
// that the cert will not be valid for, as they are not present in the list collected here.
521-
nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...)
522-
523-
// Ask the server to sign our kubelet server cert.
524-
if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
525-
return nil, errors.WithMessage(err, servingKubeletCert)
526-
}
527-
528-
// Ask the server to sign our kubelet client cert.
529-
if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
530-
return nil, errors.WithMessage(err, clientKubeletCert)
531-
}
532-
533-
// Generate a kubeconfig for the kubelet.
534519
kubeconfigKubelet := filepath.Join(envInfo.DataDir, "agent", "kubelet.kubeconfig")
535-
if err := deps.KubeConfig(kubeconfigKubelet, apiServerURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
536-
return nil, err
537-
}
538-
539520
clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt")
540521
clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key")
541-
542-
// Ask the server to sign our kube-proxy client cert.
543-
if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
544-
return nil, errors.WithMessage(err, clientKubeProxyCert)
545-
}
546-
547-
// Generate a kubeconfig for kube-proxy.
548522
kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "agent", "kubeproxy.kubeconfig")
549-
if err := deps.KubeConfig(kubeconfigKubeproxy, apiServerURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
550-
return nil, err
551-
}
552-
553523
clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt")
554524
clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key")
555-
556-
// Ask the server to sign our agent controller client cert.
557-
if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
558-
return nil, errors.WithMessage(err, clientK3sControllerCert)
559-
}
560-
561-
// Generate a kubeconfig for the agent controller.
562525
kubeconfigK3sController := filepath.Join(envInfo.DataDir, "agent", version.Program+"controller.kubeconfig")
563-
if err := deps.KubeConfig(kubeconfigK3sController, apiServerURL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil {
564-
return nil, err
565-
}
566526

567527
// Ensure kubelet config dir exists
568528
kubeletConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf.d")
@@ -769,6 +729,51 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
769729
return nil, err
770730
}
771731

732+
// allow executor to do additional configuration; this is the last chance to modify nodeConfig before certs are signed
733+
if err := executor.Prepare(ctx, nodeConfig, *envInfo); err != nil {
734+
return nil, errors.WithMessage(err, "failed to prepare configuration")
735+
}
736+
737+
// Ensure that the kubelet's server certificate is valid for all configured node IPs. Note
738+
// that in the case of an external CCM, additional IPs may be added by the infra provider
739+
// that the cert will not be valid for, as they are not present in the list collected here.
740+
nodeExternalAndInternalIPs := append(nodeConfig.AgentConfig.NodeIPs, nodeConfig.AgentConfig.NodeExternalIPs...)
741+
742+
// Ask the server to sign our kubelet server cert.
743+
if err := getKubeletServingCert(nodeConfig.AgentConfig.NodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
744+
return nil, errors.WithMessage(err, servingKubeletCert)
745+
}
746+
747+
// Ask the server to sign our kubelet client cert.
748+
if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeConfig.AgentConfig.NodeName, nodeConfig.AgentConfig.NodeIPs, newNodePasswordFile, info); err != nil {
749+
return nil, errors.WithMessage(err, clientKubeletCert)
750+
}
751+
752+
// Generate a kubeconfig for the kubelet.
753+
if err := deps.KubeConfig(kubeconfigKubelet, apiServerURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
754+
return nil, err
755+
}
756+
757+
// Ask the server to sign our kube-proxy client cert.
758+
if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
759+
return nil, errors.WithMessage(err, clientKubeProxyCert)
760+
}
761+
762+
// Generate a kubeconfig for kube-proxy.
763+
if err := deps.KubeConfig(kubeconfigKubeproxy, apiServerURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
764+
return nil, err
765+
}
766+
767+
// Ask the server to sign our agent controller client cert.
768+
if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
769+
return nil, errors.WithMessage(err, clientK3sControllerCert)
770+
}
771+
772+
// Generate a kubeconfig for the agent controller.
773+
if err := deps.KubeConfig(kubeconfigK3sController, apiServerURL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil {
774+
return nil, err
775+
}
776+
772777
return nodeConfig, nil
773778
}
774779

‎pkg/cli/agent/agent.go‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"github.com/k3s-io/k3s/pkg/util/mux"
2424
"github.com/k3s-io/k3s/pkg/util/permissions"
2525
"github.com/k3s-io/k3s/pkg/version"
26-
"github.com/k3s-io/k3s/pkg/vpn"
2726
"github.com/sirupsen/logrus"
2827
"github.com/urfave/cli/v2"
2928
"k8s.io/klog/v2"
@@ -111,20 +110,6 @@ func Run(clx *cli.Context) (rerr error) {
111110
cfg.DataDir = dataDir
112111

113112
go cmds.WriteCoverage(ctx)
114-
if cfg.VPNAuthFile != "" {
115-
cfg.VPNAuth, err = util.ReadFile(cfg.VPNAuthFile)
116-
if err != nil {
117-
return err
118-
}
119-
}
120-
121-
// Starts the VPN in the agent if config was set up
122-
if cfg.VPNAuth != "" {
123-
err := vpn.StartVPN(cfg.VPNAuth)
124-
if err != nil {
125-
return err
126-
}
127-
}
128113

129114
// Until the agent is run and retrieves config from the server, we won't know
130115
// if the embedded registry is enabled. If it is not enabled, these are not

‎pkg/cli/server/server.go‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -117,21 +117,6 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
117117
}
118118
}
119119

120-
if cmds.AgentConfig.VPNAuthFile != "" {
121-
cmds.AgentConfig.VPNAuth, err = util.ReadFile(cmds.AgentConfig.VPNAuthFile)
122-
if err != nil {
123-
return err
124-
}
125-
}
126-
127-
// Starts the VPN in the server if config was set up
128-
if cmds.AgentConfig.VPNAuth != "" {
129-
err := vpn.StartVPN(cmds.AgentConfig.VPNAuth)
130-
if err != nil {
131-
return err
132-
}
133-
}
134-
135120
serverConfig := server.Config{}
136121
serverConfig.DisableAgent = cfg.DisableAgent
137122
serverConfig.ControlConfig.Runtime = config.NewRuntime()

‎pkg/daemons/executor/executor.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ var (
2828
// The enableMaintenance flag enables attempts to perform corrective maintenance during the test process.
2929
type TestFunc func(ctx context.Context, enableMaintenance bool) error
3030

31+
// Executor is a set of functions for bootstrapping a node and starting the CRI, CNI, and Kubernetes components
3132
type Executor interface {
3233
Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error
3334
Kubelet(ctx context.Context, args []string) error
@@ -49,6 +50,12 @@ type Executor interface {
4950
IsSelfHosted() bool
5051
}
5152

53+
// PreparingExecutor is an optional interface that Executors may implement to modify node configuration
54+
// and CLI flags before config is retrieved from the server.
55+
type PreparingExecutor interface {
56+
Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error
57+
}
58+
5259
type ETCDSocketOpts struct {
5360
ReuseAddress bool `json:"reuse-address,omitempty"`
5461
ReusePort bool `json:"reuse-port,omitempty"`
@@ -278,6 +285,13 @@ func IsSelfHosted() bool {
278285
return executor.IsSelfHosted()
279286
}
280287

288+
func Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
289+
if ex, ok := executor.(PreparingExecutor); ok {
290+
return ex.Prepare(ctx, nodeConfig, cfg)
291+
}
292+
return nil
293+
}
294+
281295
func CloseIfNilErr(err error, ch chan struct{}) error {
282296
if err == nil {
283297
close(ch)

‎pkg/executor/embed/embed.go‎

Lines changed: 65 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,75 @@ func init() {
5757

5858
// explicit type check
5959
var _ executor.Executor = &Embedded{}
60+
var _ executor.PreparingExecutor = &Embedded{}
6061

6162
type Embedded struct {
6263
apiServerReady <-chan struct{}
6364
etcdReady chan struct{}
6465
criReady chan struct{}
6566
nodeConfig *daemonconfig.Node
67+
vpnInfo *vpn.Info
68+
}
69+
70+
// Prepare modifies the node config prior to downloading client and server certificates from the server.
71+
// If node IPs or names need to be modified, it should be done here so that the kubelet client and serving certs are valid.
72+
func (e *Embedded) Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
73+
// If there is a VPN, we must start it early to overwrite NodeIP and flannel interface
74+
var err error
75+
if cfg.VPNAuthFile != "" {
76+
cfg.VPNAuth, err = util.ReadFile(cfg.VPNAuthFile)
77+
if err != nil {
78+
return errors.WithMessage(err, "failed to read vpn-auth-file")
79+
}
80+
}
81+
82+
if cfg.VPNAuth != "" {
83+
err = vpn.StartVPN(cfg.VPNAuth)
84+
if err != nil {
85+
return err
86+
}
87+
88+
e.vpnInfo, err = vpn.GetInfo(cfg.VPNAuth)
89+
if err != nil {
90+
return err
91+
}
92+
93+
// Pass ipv4, ipv6 or both depending on nodeIPs mode
94+
nodeIPs := nodeConfig.AgentConfig.NodeIPs
95+
var vpnIPs []net.IP
96+
if utilsnet.IsIPv4(nodeIPs[0]) && e.vpnInfo.IPv4Address != nil {
97+
vpnIPs = append(vpnIPs, e.vpnInfo.IPv4Address)
98+
if e.vpnInfo.IPv6Address != nil {
99+
vpnIPs = append(vpnIPs, e.vpnInfo.IPv6Address)
100+
}
101+
} else if utilsnet.IsIPv6(nodeIPs[0]) && e.vpnInfo.IPv6Address != nil {
102+
vpnIPs = append(vpnIPs, e.vpnInfo.IPv6Address)
103+
if e.vpnInfo.IPv4Address != nil {
104+
vpnIPs = append(vpnIPs, e.vpnInfo.IPv4Address)
105+
}
106+
} else {
107+
return fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, e.vpnInfo.IPv4Address, e.vpnInfo.IPv6Address)
108+
}
109+
110+
// Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters
111+
if len(vpnIPs) != 0 {
112+
logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs)
113+
if len(cfg.NodeIP.Value()) != 0 {
114+
logrus.Warn("VPN provider overrides configured node-ip parameter")
115+
}
116+
if len(cfg.NodeExternalIP.Value()) != 0 {
117+
logrus.Warn("VPN provider overrides node-external-ip parameter")
118+
}
119+
nodeIPs = vpnIPs
120+
nodeConfig.AgentConfig.NodeIPs = vpnIPs
121+
nodeConfig.AgentConfig.NodeIP = vpnIPs[0].String()
122+
nodeConfig.Flannel.Iface, err = net.InterfaceByName(e.vpnInfo.Interface)
123+
if err != nil {
124+
return errors.WithMessagef(err, "unable to find vpn interface: %s", e.vpnInfo.Interface)
125+
}
126+
}
127+
}
128+
return err
66129
}
67130

68131
func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
@@ -100,50 +163,6 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node,
100163
}
101164
}
102165

103-
// If there is a VPN, we must overwrite NodeIP and flannel interface
104-
var vpnInfo *vpn.Info
105-
if cfg.VPNAuth != "" {
106-
vpnInfo, err = vpn.GetInfo(cfg.VPNAuth)
107-
if err != nil {
108-
return err
109-
}
110-
111-
// Pass ipv4, ipv6 or both depending on nodeIPs mode
112-
nodeIPs := nodeConfig.AgentConfig.NodeIPs
113-
var vpnIPs []net.IP
114-
if utilsnet.IsIPv4(nodeIPs[0]) && vpnInfo.IPv4Address != nil {
115-
vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
116-
if vpnInfo.IPv6Address != nil {
117-
vpnIPs = append(vpnIPs, vpnInfo.IPv6Address)
118-
}
119-
} else if utilsnet.IsIPv6(nodeIPs[0]) && vpnInfo.IPv6Address != nil {
120-
vpnIPs = append(vpnIPs, vpnInfo.IPv6Address)
121-
if vpnInfo.IPv4Address != nil {
122-
vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
123-
}
124-
} else {
125-
return fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address)
126-
}
127-
128-
// Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters
129-
if len(vpnIPs) != 0 {
130-
logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs)
131-
if len(cfg.NodeIP.Value()) != 0 {
132-
logrus.Warn("VPN provider overrides configured node-ip parameter")
133-
}
134-
if len(cfg.NodeExternalIP.Value()) != 0 {
135-
logrus.Warn("VPN provider overrides node-external-ip parameter")
136-
}
137-
nodeIPs = vpnIPs
138-
nodeConfig.AgentConfig.NodeIPs = vpnIPs
139-
nodeConfig.AgentConfig.NodeIP = vpnIPs[0].String()
140-
nodeConfig.Flannel.Iface, err = net.InterfaceByName(vpnInfo.Interface)
141-
if err != nil {
142-
return errors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.Interface)
143-
}
144-
}
145-
}
146-
147166
// set paths for embedded flannel if enabled
148167
hostLocal, err := exec.LookPath("host-local")
149168
if err != nil {
@@ -160,8 +179,8 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node,
160179
nodeConfig.AgentConfig.CNIConfDir = filepath.Join(cfg.DataDir, "agent", "etc", "cni", "net.d")
161180

162181
// It does not make sense to use VPN without its flannel backend
163-
if cfg.VPNAuth != "" {
164-
nodeConfig.Flannel.Backend = vpnInfo.ProviderName
182+
if e.vpnInfo != nil {
183+
nodeConfig.Flannel.Backend = e.vpnInfo.ProviderName
165184
}
166185
}
167186

‎pkg/vpn/vpn.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,10 @@ func StartVPN(vpnAuthConfigFile string) error {
7272
logrus.Debugf("Flags passed to tailscale up: %v", args)
7373
output, err := util.ExecCommand("tailscale", args)
7474
if err != nil {
75-
return errors.WithMessage(err, "tailscale up failed: "+output)
75+
if output != "" {
76+
return errors.WithMessagef(err, "tailscale up failed (%q)", output)
77+
}
78+
return errors.WithMessage(err, "tailscale up failed")
7679
}
7780
logrus.Debugf("Output from tailscale up: %v", output)
7881
return nil

0 commit comments

Comments
 (0)