Skip to content

Commit fde310e

Browse files
committed
set network device status data
Change-Id: Ic8fe1378d5f001db9c7dbfc1c5b5ff11e4d1cfd2
1 parent a3c281b commit fde310e

File tree

3 files changed

+102
-19
lines changed

3 files changed

+102
-19
lines changed

pkg/driver/driver.go

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package driver
1818

1919
import (
2020
"context"
21+
"errors"
2122
"fmt"
2223
"os"
2324
"path/filepath"
@@ -33,8 +34,11 @@ import (
3334
"github.com/containerd/nri/pkg/stub"
3435

3536
resourceapi "k8s.io/api/resource/v1beta1"
37+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3638
"k8s.io/apimachinery/pkg/types"
3739
"k8s.io/apimachinery/pkg/util/wait"
40+
metav1apply "k8s.io/client-go/applyconfigurations/meta/v1"
41+
resourceapply "k8s.io/client-go/applyconfigurations/resource/v1beta1"
3842
"k8s.io/client-go/kubernetes"
3943
"k8s.io/client-go/tools/cache"
4044
"k8s.io/dynamic-resource-allocation/kubeletplugin"
@@ -216,6 +220,7 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox)
216220
np.netdb.AddPodNetns(podKey(pod), ns)
217221

218222
// Process the configurations of the ResourceClaim
223+
errorList := []error{}
219224
for _, obj := range objs {
220225
claim, ok := obj.(*resourceapi.ResourceClaim)
221226
if !ok {
@@ -225,11 +230,11 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox)
225230
if claim.Status.Allocation == nil {
226231
continue
227232
}
233+
resourceClaimStatus := resourceapply.ResourceClaimStatus()
228234
for _, result := range claim.Status.Allocation.Devices.Results {
229235
if result.Driver != np.driverName {
230236
continue
231237
}
232-
233238
// Process the configurations of the ResourceClaim
234239
for _, config := range claim.Status.Allocation.Devices.Config {
235240
if config.Opaque == nil {
@@ -255,14 +260,54 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox)
255260

256261
// TODO config options to rename the device and pass parameters
257262
// use https://github.com/opencontainers/runtime-spec/pull/1271
258-
err := nsAttachNetdev(result.Device, ns, result.Device)
263+
networkData, err := nsAttachNetdev(result.Device, ns, result.Device)
259264
if err != nil {
260265
klog.Infof("RunPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err)
261-
return err
266+
resourceClaimStatus = resourceClaimStatus.WithDevices(
267+
resourceapply.AllocatedDeviceStatus().
268+
WithDevice(result.Device).WithDriver(result.Driver).WithPool(result.Pool).
269+
WithConditions(
270+
metav1apply.Condition().
271+
WithType("Ready").
272+
WithStatus(metav1.ConditionFalse).
273+
WithReason("NetworkDeviceError").
274+
WithMessage(err.Error()).
275+
WithLastTransitionTime(metav1.Now()),
276+
),
277+
)
278+
errorList = append(errorList, err)
279+
} else {
280+
resourceClaimStatus = resourceClaimStatus.WithDevices(
281+
resourceapply.AllocatedDeviceStatus().
282+
WithDevice(result.Device).WithDriver(result.Driver).WithPool(result.Pool).
283+
WithConditions(
284+
metav1apply.Condition().
285+
WithType("Ready").
286+
WithReason("NetworkDeviceReady").
287+
WithStatus(metav1.ConditionTrue).
288+
WithLastTransitionTime(metav1.Now()),
289+
).
290+
WithNetworkData(resourceapply.NetworkDeviceData().
291+
WithInterfaceName(networkData.InterfaceName).
292+
WithHardwareAddress(networkData.HardwareAddress).
293+
WithIPs(networkData.IPs...),
294+
),
295+
)
262296
}
263297
}
298+
resourceClaimApply := resourceapply.ResourceClaim(claim.Name, claim.Namespace).WithStatus(resourceClaimStatus)
299+
_, err = np.kubeClient.ResourceV1beta1().ResourceClaims(claim.Namespace).ApplyStatus(ctx,
300+
resourceClaimApply,
301+
metav1.ApplyOptions{FieldManager: np.driverName, Force: true},
302+
)
303+
// do not fail hard
304+
if err != nil {
305+
klog.Infof("failed to update status for claim %s/%s : %v", claim.Namespace, claim.Name, err)
306+
} else {
307+
klog.V(2).Infof("update status for claim %s/%s", claim.Namespace, claim.Name)
308+
}
264309
}
265-
return nil
310+
return errors.Join(errorList...)
266311
}
267312

268313
func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox) error {

pkg/driver/hostdevice.go

Lines changed: 42 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,29 +24,31 @@ import (
2424
"github.com/vishvananda/netlink/nl"
2525
"github.com/vishvananda/netns"
2626
"golang.org/x/sys/unix"
27+
28+
resourceapi "k8s.io/api/resource/v1beta1"
2729
)
2830

29-
func nsAttachNetdev(hostIfName string, containerNsPAth string, ifName string) error {
31+
func nsAttachNetdev(hostIfName string, containerNsPAth string, ifName string) (*resourceapi.NetworkDeviceData, error) {
3032
hostDev, err := netlink.LinkByName(hostIfName)
3133
// recover same behavior on vishvananda/netlink@1.2.1 and do not fail when the kernel returns NLM_F_DUMP_INTR.
3234
if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) {
33-
return err
35+
return nil, err
3436
}
3537

3638
// Devices can be renamed only when down
3739
if err = netlink.LinkSetDown(hostDev); err != nil {
38-
return fmt.Errorf("failed to set %q down: %v", hostDev.Attrs().Name, err)
40+
return nil, fmt.Errorf("failed to set %q down: %v", hostDev.Attrs().Name, err)
3941
}
4042

4143
// get the existing IP addresses
4244
addresses, err := netlink.AddrList(hostDev, netlink.FAMILY_ALL)
4345
if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) {
44-
return fmt.Errorf("fail to get ip addresses: %w", err)
46+
return nil, fmt.Errorf("fail to get ip addresses: %w", err)
4547
}
4648

4749
containerNs, err := netns.GetFromPath(containerNsPAth)
4850
if err != nil {
49-
return err
51+
return nil, err
5052
}
5153
defer containerNs.Close()
5254

@@ -60,7 +62,7 @@ func nsAttachNetdev(hostIfName string, containerNsPAth string, ifName string) er
6062
// Get a netlink socket in current namespace
6163
s, err := nl.GetNetlinkSocketAt(netns.None(), netns.None(), unix.NETLINK_ROUTE)
6264
if err != nil {
63-
return fmt.Errorf("could not get network namespace handle: %w", err)
65+
return nil, fmt.Errorf("could not get network namespace handle: %w", err)
6466
}
6567
defer s.Close()
6668

@@ -87,37 +89,65 @@ func nsAttachNetdev(hostIfName string, containerNsPAth string, ifName string) er
8789

8890
_, err = req.Execute(unix.NETLINK_ROUTE, 0)
8991
if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) {
90-
return err
92+
return nil, err
9193
}
9294

9395
// to avoid golang problem with goroutines we create the socket in the
9496
// namespace and use it directly
9597
nhNs, err := netlink.NewHandleAt(containerNs)
9698
if err != nil {
97-
return err
99+
return nil, err
98100
}
99101
defer nhNs.Close()
100102

101103
nsLink, err := nhNs.LinkByName(attrs.Name)
102104
if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) {
103-
return fmt.Errorf("link not found for interface %s on namespace %s: %w", attrs.Name, containerNsPAth, err)
105+
return nil, fmt.Errorf("link not found for interface %s on namespace %s: %w", attrs.Name, containerNsPAth, err)
104106
}
105107

108+
// Re-add the original IP addresses to the interface in the new namespace.
109+
// The kernel removes IP addresses when an interface is moved between network namespaces.
106110
for _, address := range addresses {
111+
// Only move permanent IP addresses configured by the user, dynamic addresses are excluded because
112+
// their validity may rely on the original network namespace's context and they may have limited
113+
// lifetimes and are not guaranteed to be available in a new namespace.
114+
// Ref: https://www.ietf.org/rfc/rfc3549.txt
115+
if address.Flags&unix.IFA_F_PERMANENT == 0 {
116+
continue
117+
}
118+
// Only move IP addresses with global scope because those are not host-specific, auto-configured,
119+
// or have limited network scope, making them unsuitable inside the container namespace.
120+
// Ref: https://www.ietf.org/rfc/rfc3549.txt
121+
if address.Scope != unix.RT_SCOPE_UNIVERSE {
122+
continue
123+
}
107124
// remove the interface attribute of the original address
108125
// to avoid issues when the interface is renamed.
109126
err = nhNs.AddrAdd(nsLink, &netlink.Addr{IPNet: address.IPNet})
110127
if err != nil {
111-
return fmt.Errorf("fail to set up address %s on namespace %s: %w", address.String(), containerNsPAth, err)
128+
return nil, fmt.Errorf("fail to set up address %s on namespace %s: %w", address.String(), containerNsPAth, err)
112129
}
113130
}
114131

115132
err = nhNs.LinkSetUp(nsLink)
116133
if err != nil {
117-
return fmt.Errorf("failt to set up interface %s on namespace %s: %w", nsLink.Attrs().Name, containerNsPAth, err)
134+
return nil, fmt.Errorf("failt to set up interface %s on namespace %s: %w", nsLink.Attrs().Name, containerNsPAth, err)
118135
}
119136

120-
return nil
137+
networkData := &resourceapi.NetworkDeviceData{
138+
InterfaceName: nsLink.Attrs().Name,
139+
HardwareAddress: string(nsLink.Attrs().HardwareAddr.String()),
140+
}
141+
142+
// get the existing IP addresses
143+
addresses, err = nhNs.AddrList(nsLink, netlink.FAMILY_ALL)
144+
if err == nil || errors.Is(err, netlink.ErrDumpInterrupted) {
145+
for _, address := range addresses {
146+
networkData.IPs = append(networkData.IPs, address.IPNet.String())
147+
}
148+
}
149+
150+
return networkData, nil
121151
}
122152

123153
func nsDetachNetdev(containerNsPAth string, devName string) error {

tests/e2e.bats

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
#!/usr/bin/env bats
22

3-
@test "dummy interface" {
4-
cat "$BATS_TEST_DIRNAME"/../examples/add_dummy_iface.sh | docker exec -i "$CLUSTER_NAME"-worker bash
3+
@test "dummy interface with IP addresses" {
4+
docker exec "$CLUSTER_NAME"-worker bash -c "ip link add dummy0 type dummy"
5+
docker exec "$CLUSTER_NAME"-worker bash -c "ip link set up dev dummy0"
6+
docker exec "$CLUSTER_NAME"-worker bash -c "ip addr add 169.254.169.13/32 dev dummy0"
7+
58
kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml
69
kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim.yaml
710
kubectl wait --timeout=2m --for=condition=ready pods -l app=pod
8-
kubectl exec -it pod1 -- ip link show dummy0
11+
run kubectl exec pod1 -- ip addr show dummy0
12+
[ "$status" -eq 0 ]
13+
[[ "$output" == *"169.254.169.13"* ]]
14+
run kubectl get resourceclaims dummy-interface-static-ip -o=jsonpath='{.status.devices[0].networkData.ips[*]}'
15+
[ "$status" -eq 0 ]
16+
[[ "$output" == *"169.254.169.13"* ]]
917
}

0 commit comments

Comments
 (0)