Skip to content

Commit 23634fb

Browse files
justinsbjustinsb
committed
metal: simple IPAM for IPv6
1 parent a33f907 commit 23634fb

File tree

12 files changed

+214
-19
lines changed

12 files changed

+214
-19
lines changed

‎cmd/kops-controller/controllers/gceipam.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func NewGCEIPAMReconciler(mgr manager.Manager) (*GCEIPAMReconciler, error) {
5656
return r, nil
5757
}
5858

59-
// GCEIPAMReconciler observes Node objects, assigning their`PodCIDRs` from the instance's `ExternalIpv6`.
59+
// GCEIPAMReconciler observes Node objects, assigning their `PodCIDRs` from the instance's `ExternalIpv6`.
6060
type GCEIPAMReconciler struct {
6161
// client is the controller-runtime client
6262
client client.Client

‎cmd/kops-controller/controllers/metalipam.go

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
"github.com/go-logr/logr"
24+
corev1 "k8s.io/api/core/v1"
25+
apierrors "k8s.io/apimachinery/pkg/api/errors"
26+
"k8s.io/apimachinery/pkg/types"
27+
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
28+
"k8s.io/klog/v2"
29+
kopsapi "k8s.io/kops/pkg/apis/kops/v1alpha2"
30+
ctrl "sigs.k8s.io/controller-runtime"
31+
"sigs.k8s.io/controller-runtime/pkg/client"
32+
"sigs.k8s.io/controller-runtime/pkg/manager"
33+
)
34+
35+
// NewMetalIPAMReconciler is the constructor for a MetalIPAMReconciler
36+
func NewMetalIPAMReconciler(ctx context.Context, mgr manager.Manager) (*MetalIPAMReconciler, error) {
37+
klog.Info("starting metal ipam controller")
38+
r := &MetalIPAMReconciler{
39+
client: mgr.GetClient(),
40+
log: ctrl.Log.WithName("controllers").WithName("metal_ipam"),
41+
}
42+
43+
coreClient, err := corev1client.NewForConfig(mgr.GetConfig())
44+
if err != nil {
45+
return nil, fmt.Errorf("building corev1 client: %w", err)
46+
}
47+
r.coreV1Client = coreClient
48+
49+
return r, nil
50+
}
51+
52+
// MetalIPAMReconciler observes Node objects, assigning their `PodCIDRs` from the instance's `ExternalIpv6`.
53+
type MetalIPAMReconciler struct {
54+
// client is the controller-runtime client
55+
client client.Client
56+
57+
// log is a logr
58+
log logr.Logger
59+
60+
// coreV1Client is a client-go client for patching nodes
61+
coreV1Client *corev1client.CoreV1Client
62+
}
63+
64+
// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
65+
// Reconcile is the main reconciler function that observes node changes.
66+
func (r *MetalIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
67+
node := &corev1.Node{}
68+
if err := r.client.Get(ctx, req.NamespacedName, node); err != nil {
69+
klog.Warningf("unable to fetch node %s: %v", node.Name, err)
70+
if apierrors.IsNotFound(err) {
71+
// we'll ignore not-found errors, since they can't be fixed by an immediate
72+
// requeue (we'll need to wait for a new notification), and we can get them
73+
// on deleted requests.
74+
return ctrl.Result{}, nil
75+
}
76+
return ctrl.Result{}, err
77+
}
78+
79+
host := &kopsapi.Host{}
80+
id := types.NamespacedName{
81+
Namespace: "kops-system",
82+
Name: node.Name,
83+
}
84+
if err := r.client.Get(ctx, id, host); err != nil {
85+
klog.Warningf("unable to fetch host %s: %v", id, err)
86+
return ctrl.Result{}, err
87+
}
88+
89+
if len(node.Spec.PodCIDRs) == 0 {
90+
if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, host.Spec.PodCIDRs); err != nil {
91+
return ctrl.Result{}, err
92+
}
93+
}
94+
95+
return ctrl.Result{}, nil
96+
}
97+
98+
func (r *MetalIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error {
99+
return ctrl.NewControllerManagedBy(mgr).
100+
Named("metal_ipam").
101+
For(&corev1.Node{}).
102+
Complete(r)
103+
}

‎cmd/kops-controller/main.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,12 @@ func setupCloudIPAM(ctx context.Context, mgr manager.Manager, opt *config.Option
392392
return fmt.Errorf("creating gce IPAM controller: %w", err)
393393
}
394394
controller = ipamController
395+
case "metal":
396+
ipamController, err := controllers.NewMetalIPAMReconciler(ctx, mgr)
397+
if err != nil {
398+
return fmt.Errorf("creating metal IPAM controller: %w", err)
399+
}
400+
controller = ipamController
395401
default:
396402
return fmt.Errorf("kOps IPAM controller is not supported on cloud %q", opt.Cloud)
397403
}

‎cmd/kops/toolbox_enroll.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ func NewCmdToolboxEnroll(f commandutils.Factory, out io.Writer) *cobra.Command {
4545

4646
cmd.Flags().StringVar(&options.ClusterName, "cluster", options.ClusterName, "Name of cluster to join")
4747
cmd.Flags().StringVar(&options.InstanceGroup, "instance-group", options.InstanceGroup, "Name of instance-group to join")
48+
cmd.Flags().StringSliceVar(&options.PodCIDRs, "pod-cidr", options.PodCIDRs, "IP Address range to use for pods that run on this node")
4849

4950
cmd.Flags().StringVar(&options.Host, "host", options.Host, "IP/hostname for machine to add")
5051
cmd.Flags().StringVar(&options.SSHUser, "ssh-user", options.SSHUser, "user for ssh")

‎docs/cli/kops_toolbox_enroll.md

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎k8s/crds/kops.k8s.io_hosts.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ spec:
4242
properties:
4343
instanceGroup:
4444
type: string
45+
podCIDRs:
46+
description: PodCIDRs configures the IP ranges to be used for pods
47+
on this node/host.
48+
items:
49+
type: string
50+
type: array
4551
publicKey:
4652
type: string
4753
type: object

‎nodeup/pkg/model/kube_apiserver.go

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,12 @@ package model
1919
import (
2020
"context"
2121
"fmt"
22+
"net"
2223
"path/filepath"
2324
"sort"
2425
"strings"
2526

27+
"k8s.io/klog/v2"
2628
"k8s.io/kops/pkg/apis/kops"
2729
"k8s.io/kops/pkg/flagbuilder"
2830
"k8s.io/kops/pkg/k8scodecs"
@@ -77,6 +79,55 @@ func (b *KubeAPIServerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
7779
}
7880
}
7981

82+
if b.CloudProvider() == kops.CloudProviderMetal {
83+
// Workaround for https://github.com/kubernetes/kubernetes/issues/111671
84+
if b.IsIPv6Only() {
85+
interfaces, err := net.Interfaces()
86+
if err != nil {
87+
return fmt.Errorf("getting local network interfaces: %w", err)
88+
}
89+
var ipv6s []net.IP
90+
for _, intf := range interfaces {
91+
addresses, err := intf.Addrs()
92+
if err != nil {
93+
return fmt.Errorf("getting addresses for network interface %q: %w", intf.Name, err)
94+
}
95+
for _, addr := range addresses {
96+
ip, _, err := net.ParseCIDR(addr.String())
97+
if ip == nil {
98+
return fmt.Errorf("parsing ip address %q (bound to network %q): %w", addr.String(), intf.Name, err)
99+
}
100+
if ip.To4() != nil {
101+
// We're only looking for ipv6
102+
continue
103+
}
104+
if ip.IsLinkLocalUnicast() {
105+
klog.V(4).Infof("ignoring link-local unicast addr %v", addr)
106+
continue
107+
}
108+
if ip.IsLinkLocalMulticast() {
109+
klog.V(4).Infof("ignoring link-local multicast addr %v", addr)
110+
continue
111+
}
112+
if ip.IsLoopback() {
113+
klog.V(4).Infof("ignoring loopback addr %v", addr)
114+
continue
115+
}
116+
ipv6s = append(ipv6s, ip)
117+
}
118+
}
119+
if len(ipv6s) > 1 {
120+
klog.Warningf("found multiple ipv6s, choosing first: %v", ipv6s)
121+
}
122+
if len(ipv6s) == 0 {
123+
klog.Warningf("did not find ipv6 address for kube-apiserver --advertise-address")
124+
}
125+
if len(ipv6s) > 0 {
126+
kubeAPIServer.AdvertiseAddress = ipv6s[0].String()
127+
}
128+
}
129+
}
130+
80131
b.configureOIDC(&kubeAPIServer)
81132
if err := b.writeAuthenticationConfig(c, &kubeAPIServer); err != nil {
82133
return err
@@ -697,10 +748,9 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops
697748
image := b.RemapImage(kubeAPIServer.Image)
698749

699750
container := &v1.Container{
700-
Name: "kube-apiserver",
701-
Image: image,
702-
Env: append(kubeAPIServer.Env, proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy)...),
703-
LivenessProbe: livenessProbe,
751+
Name: "kube-apiserver",
752+
Image: image,
753+
Env: append(kubeAPIServer.Env, proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy)...), LivenessProbe: livenessProbe,
704754
ReadinessProbe: readinessProbe,
705755
StartupProbe: startupProbe,
706756
Ports: []v1.ContainerPort{

‎nodeup/pkg/model/prefix.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ func (b *PrefixBuilder) Build(c *fi.NodeupModelBuilderContext) error {
4141
})
4242
case kops.CloudProviderGCE:
4343
// Prefix is assigned by GCE
44+
case kops.CloudProviderMetal:
45+
// IPv6 must be configured externally (not by nodeup)
4446
default:
4547
return fmt.Errorf("kOps IPAM controller not supported on cloud %q", b.CloudProvider())
4648
}

‎pkg/apis/kops/v1alpha2/host.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ type Host struct {
3636
type HostSpec struct {
3737
PublicKey string `json:"publicKey,omitempty"`
3838
InstanceGroup string `json:"instanceGroup,omitempty"`
39+
40+
// PodCIDRs configures the IP ranges to be used for pods on this node/host.
41+
PodCIDRs []string `json:"podCIDRs,omitempty"`
3942
}
4043

4144
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

‎pkg/commands/toolbox_enroll.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ type ToolboxEnrollOptions struct {
6767

6868
SSHUser string
6969
SSHPort int
70+
71+
// PodCIDRs is the list of IP Address ranges to use for pods that run on this node
72+
PodCIDRs []string
7073
}
7174

7275
func (o *ToolboxEnrollOptions) InitDefaults() {
@@ -209,6 +212,7 @@ func createHostResourceInAPIServer(ctx context.Context, options *ToolboxEnrollOp
209212
host.Name = nodeName
210213
host.Spec.InstanceGroup = options.InstanceGroup
211214
host.Spec.PublicKey = string(publicKey)
215+
host.Spec.PodCIDRs = options.PodCIDRs
212216

213217
if err := client.Create(ctx, host); err != nil {
214218
return fmt.Errorf("failed to create host %s/%s: %w", host.Namespace, host.Name, err)

‎pkg/kubeconfig/create_kubecfg.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"crypto/x509/pkix"
2222
"fmt"
23+
"net"
2324
"os/user"
2425
"sort"
2526
"time"
@@ -41,7 +42,7 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
4142
server = "https://" + cluster.APIInternalName()
4243
} else {
4344
if cluster.Spec.API.PublicName != "" {
44-
server = "https://" + cluster.Spec.API.PublicName
45+
server = "https://" + wrapIPv6Address(cluster.Spec.API.PublicName)
4546
} else {
4647
server = "https://api." + clusterName
4748
}
@@ -82,7 +83,7 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
8283
if len(targets) != 1 {
8384
klog.Warningf("Found multiple API endpoints (%v), choosing arbitrarily", targets)
8485
}
85-
server = "https://" + targets[0]
86+
server = "https://" + wrapIPv6Address(targets[0])
8687
}
8788
}
8889
}
@@ -171,3 +172,14 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
171172

172173
return b, nil
173174
}
175+
176+
// wrapIPv6Address will wrap IPv6 addresses in square brackets,
177+
// for use in URLs; other endpoints are unchanged.
178+
func wrapIPv6Address(endpoint string) string {
179+
ip := net.ParseIP(endpoint)
180+
// IPv6 addresses are wrapped in square brackets in URLs
181+
if ip != nil && ip.To4() == nil {
182+
return "[" + endpoint + "]"
183+
}
184+
return endpoint
185+
}

‎tests/e2e/scenarios/bare-metal/scenario-ipv6

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -70,18 +70,24 @@ ${REPO_ROOT}/tests/e2e/scenarios/bare-metal/start-vms
7070

7171
. hack/dev-build-metal.sh
7272

73+
IPV6_PREFIX=fd00:10:123:45:
74+
IPV4_PREFIX=10.123.45.
75+
7376
echo "Waiting 10 seconds for VMs to start"
7477
sleep 10
7578

79+
VM0_IP=${IPV4_PREFIX}10
80+
VM1_IP=${IPV4_PREFIX}11
81+
VM2_IP=${IPV4_PREFIX}12
82+
7683
# Remove from known-hosts in case of reuse
77-
ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.10 || true
78-
ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.11 || true
79-
ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.12 || true
84+
ssh-keygen -f ~/.ssh/known_hosts -R ${VM0_IP} || true
85+
ssh-keygen -f ~/.ssh/known_hosts -R ${VM1_IP} || true
86+
ssh-keygen -f ~/.ssh/known_hosts -R ${VM2_IP} || true
8087

81-
# Check SSH is working and accept the keys
82-
ssh -o StrictHostKeyChecking=accept-new root@${VM0_IP} uptime
83-
ssh -o StrictHostKeyChecking=accept-new root@${VM1_IP} uptime
84-
ssh -o StrictHostKeyChecking=accept-new root@${VM2_IP} uptime
88+
ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${VM0_IP} uptime
89+
ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${VM1_IP} uptime
90+
ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${VM2_IP} uptime
8591

8692
cd ${REPO_ROOT}
8793

@@ -206,8 +212,8 @@ ${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group control-plane-
206212
cat <<EOF | ssh root@${VM0_IP} tee -a /etc/hosts
207213
208214
# Hosts added for etcd discovery
209-
10.123.45.10 node0.main.${CLUSTER_NAME}
210-
10.123.45.10 node0.events.${CLUSTER_NAME}
215+
${VM0_IP} node0.main.${CLUSTER_NAME}
216+
${VM0_IP} node0.events.${CLUSTER_NAME}
211217
EOF
212218

213219
ssh root@${VM0_IP} cat /etc/hosts
@@ -278,17 +284,18 @@ EOF
278284

279285
function enroll_node() {
280286
local node_ip=$1
287+
local pod_cidr=$2
281288

282289
# Manual "discovery" for control-plane endpoints
283290
# TODO: Replace with well-known IP
284291
cat <<EOF | ssh root@${node_ip} tee -a /etc/hosts
285292
286293
# Hosts added for leader discovery
287-
10.123.45.10 kops-controller.internal.${CLUSTER_NAME}
288-
10.123.45.10 api.internal.${CLUSTER_NAME}
294+
${VM0_IP} kops-controller.internal.${CLUSTER_NAME}
295+
${VM0_IP} api.internal.${CLUSTER_NAME}
289296
EOF
290297

291-
timeout 10m ${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group nodes-main --host ${node_ip} --v=2
298+
timeout 10m ${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group nodes-main --host ${node_ip} --pod-cidr ${pod_cidr} --v=2
292299
}
293300

294301
enroll_node ${VM1_IP} ${VM1_POD_CIDR}

0 commit comments

Comments
 (0)