ipcache: add tests reproducing host-IP-as-world misclassification on restart

Add two test files that reproduce and document a bug where node/host IPs
inside a local-DC CiliumCIDRGroup are misclassified as world identity during
rolling Cilium agent restarts.

Root cause (two cooperating code paths):

1. local_identity_restorer.go:128 - dumpOldIPCache() only restores identities
   with IdentityScopeLocal or ReservedIdentityIngress. ReservedIdentityHost
   (id=1, scope=global) is excluded. After ipcachemap.Recreate() wipes the BPF
   map, host IP entries are gone until syncHostIPs runs.

2. daemon.go startup ordering - K8sWatcher.InitK8sSubsystem() (line 202) starts
   CiliumCIDRGroup processing before syncHostIPs.StartAndWaitFirst() (line 249).
   During this window a host IP in a local-DC CiliumCIDRGroup receives only a
   cidrgroup label.

3. metadata.go:798 resolveLabels() - without reserved:host, HasHostLabel()=false,
   isInCluster=false, AddWorldLabel() fires. The IP is assigned world identity.

Impact: traffic from node IPs to cluster-dns is denied because the CNP uses
fromEntities:cluster (which excludes world). Confirmed in production at
us1.fed.dog (plain1, us-gov-west-1a) on 2026-03-20 with 10,551 drops over 48h.

pkg/ipcache/metadata_restart_test.go:
- TestHostIPWorldFallbackDuringRestartWindow: asserts the buggy behaviour where
  a host IP with only a cidrgroup label gets world identity. Expected to fail
  once the root cause is fixed.
- TestWorldFallbackDoesNotOccurWhenHostLabelPresentFirst: asserts the correct
  steady-state behaviour where reserved:host arrives before cidrgroup labels.

pkg/ipcache/restore/local_identity_restorer_test.go:
- TestHostIdentityExcludedFromIPCacheRestoration: directly tests the filter
  condition at local_identity_restorer.go:128 for all relevant identity types.
- TestHostIdentityScopeIsGlobal: verifies that ReservedIdentityHost.Scope()
  returns IdentityScopeGlobal, which is why it is excluded from restoration.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

Author: seanduncandatadog

pkg/ipcache/metadata_restart_test.go

@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package ipcache
+
+// TestHostIPWorldFallbackDuringRestartWindow and TestHostIdentityRestorationGap
+// reproduce a bug observed in production (us1.fed.dog, 2026-03-20) where node/host
+// IPs in a local-DC CIDR were misclassified as "world" identity during rolling
+// Cilium agent restarts, causing policy_denied drops against cluster-dns.
+//
+// Root cause (two code paths, both required):
+//
+// 1. pkg/ipcache/restore/local_identity_restorer.go:128
+//    dumpOldIPCache() filters restored identities to IdentityScopeLocal and
+//    ReservedIdentityIngress only. ReservedIdentityHost (scope=global, id=1) is
+//    explicitly excluded. After ipcachemap.Recreate(), the new BPF map has no
+//    entry for host IPs.
+//
+// 2. daemon/cmd/daemon.go startup ordering
+//    K8sWatcher.InitK8sSubsystem() starts at line 202 (begins processing
+//    CiliumCIDRGroups managed by fabric-k8s-controller). syncHostIPs.StartAndWaitFirst()
+//    is not called until line 249. During this window, a host IP in the local-DC
+//    CiliumCIDRGroup (e.g. 10.160.0.0/14) receives only a cidrgroup label.
+//
+// 3. pkg/ipcache/metadata.go:798 (resolveLabels)
+//    Any IP without reserved:host, reserved:remote-node, reserved:health, or
+//    reserved:ingress label has AddWorldLabel() called on it. A host IP with only
+//    a cidrgroup label therefore becomes world — which is not covered by the
+//    cluster-dns CNP's "fromEntities: cluster" ingress rule, causing drops.
+
+import (
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	cmtypes "github.com/cilium/cilium/pkg/clustermesh/types"
+	"github.com/cilium/cilium/pkg/identity"
+	"github.com/cilium/cilium/pkg/labels"
+	"github.com/cilium/cilium/pkg/option"
+	"github.com/cilium/cilium/pkg/source"
+)
+
+// cidrGroupLabels returns a Labels set simulating what the CiliumCIDRGroup reconciler
+// (fabric-k8s-controller) injects via UpsertMetadata for an IP that matches a
+// CiliumCIDRGroup (e.g. the "local-dc" group covering 10.160.0.0/14).
+func cidrGroupLabels(groupName string) labels.Labels {
+	return labels.Labels{
+		groupName: labels.NewLabel(groupName, "", labels.LabelSourceCIDRGroup),
+	}
+}
+
+// TestHostIPWorldFallbackDuringRestartWindow reproduces the bug where a host IP
+// is assigned world identity because resolveLabels() runs with only cidrgroup labels
+// — before syncHostIPs has inserted the reserved:host label.
+//
+// This test asserts the CURRENT BUGGY BEHAVIOR. It is expected to fail once the
+// bug is fixed (e.g. by ensuring host IPs are seeded into ipcache metadata before
+// CiliumCIDRGroup processing can trigger resolveLabels for those prefixes).
+func TestHostIPWorldFallbackDuringRestartWindow(t *testing.T) {
+	s := setupIPCacheTestSuite(t)
+	ctx := t.Context()
+
+	// Disable PolicyCIDRMatchMode to avoid interference from node-CIDR matching.
+	oldVal := option.Config.PolicyCIDRMatchMode
+	t.Cleanup(func() { option.Config.PolicyCIDRMatchMode = oldVal })
+	option.Config.PolicyCIDRMatchMode = []string{}
+
+	// The host IP observed in production: 10.161.39.126 (in 10.160.0.0/14, localDc CIDR).
+	// 8,258 drops were recorded against cluster-dns over 48h.
+	hostIPPrefix := cmtypes.NewLocalPrefixCluster(netip.MustParsePrefix("10.161.39.126/32"))
+
+	// ── Stage 1: Restart window ──────────────────────────────────────────────
+	// K8sWatcher has processed the "local-dc" CiliumCIDRGroup. The ipcache BPF
+	// map has been recreated empty (RestoreLocalIdentities skipped this IP since
+	// ReservedIdentityHost is not locally-scoped). syncHostIPs has NOT run yet.
+	//
+	// Only the cidrgroup label is present — no reserved:host.
+	s.IPIdentityCache.metadata.upsertLocked(
+		hostIPPrefix,
+		source.Generated,
+		"cidrgroup-resource-uid",
+		cidrGroupLabels("local-dc"),
+	)
+
+	_, err := s.IPIdentityCache.doInjectLabels(ctx, []cmtypes.PrefixCluster{hostIPPrefix})
+	require.NoError(t, err)
+
+	entry, ok := s.IPIdentityCache.ipToIdentityCache["10.161.39.126/32"]
+	require.True(t, ok, "expected an identity entry for 10.161.39.126/32")
+
+	assignedID := entry.ID
+
+	// Verify the assigned identity is NOT reserved:host (id=1).
+	// This demonstrates the bug: the IP should be host but is not.
+	assert.NotEqual(t, identity.ReservedIdentityHost, assignedID,
+		"BUG REPRODUCED: host IP 10.161.39.126 was not assigned ReservedIdentityHost (id=1). "+
+			"Got id=%d. This occurs because resolveLabels() ran with only cidrgroup labels "+
+			"(no reserved:host) during the restart window before syncHostIPs executed.",
+		assignedID)
+
+	// Verify the assigned identity has a world label — the world fallback fired.
+	resolvedIdentity := s.Allocator.LookupIdentityByID(ctx, assignedID)
+	require.NotNil(t, resolvedIdentity, "identity %d should be resolvable", assignedID)
+	assert.True(t,
+		resolvedIdentity.Labels.HasWorldLabel() || resolvedIdentity.Labels.HasWorldIPv4Label(),
+		"BUG: host IP 10.161.39.126/32 was assigned world identity (id=%d, labels=%v). "+
+			"resolveLabels() called AddWorldLabel() because HasHostLabel()=false. "+
+			"This causes policy_denied drops: the cluster-dns CNP allows 'fromEntities: cluster' "+
+			"but world (id=2) is not in the cluster entity.",
+		assignedID, resolvedIdentity.Labels)
+
+	// ── Stage 2: syncHostIPs runs ────────────────────────────────────────────
+	// After daemon initialization completes (daemon.go:249), syncHostIPs inserts
+	// the reserved:host label for this IP. resolveLabels() now sees HasHostLabel()=true,
+	// sets isInCluster=true, removes the cidrgroup label, and does NOT add world.
+	s.IPIdentityCache.metadata.upsertLocked(
+		hostIPPrefix,
+		source.Local,
+		"daemon-reserved",
+		labels.LabelHost,
+	)
+
+	_, err = s.IPIdentityCache.doInjectLabels(ctx, []cmtypes.PrefixCluster{hostIPPrefix})
+	require.NoError(t, err)
+
+	correctedEntry, ok := s.IPIdentityCache.ipToIdentityCache["10.161.39.126/32"]
+	require.True(t, ok)
+
+	// After syncHostIPs runs, the identity must be corrected to reserved:host.
+	assert.Equal(t, identity.ReservedIdentityHost, correctedEntry.ID,
+		"After syncHostIPs inserts reserved:host, identity should be corrected to "+
+			"ReservedIdentityHost (id=1). Got id=%d.", correctedEntry.ID)
+}
+
+// TestWorldFallbackDoesNotOccurWhenHostLabelPresentFirst verifies the CORRECT
+// behaviour: when reserved:host is present before CIDRGroup labels are processed,
+// resolveLabels() correctly identifies the IP as in-cluster and does not add
+// the world label.
+//
+// This is the inverse of TestHostIPWorldFallbackDuringRestartWindow and
+// documents the expected steady-state behaviour (no restart window).
+func TestWorldFallbackDoesNotOccurWhenHostLabelPresentFirst(t *testing.T) {
+	s := setupIPCacheTestSuite(t)
+	ctx := t.Context()
+
+	oldVal := option.Config.PolicyCIDRMatchMode
+	t.Cleanup(func() { option.Config.PolicyCIDRMatchMode = oldVal })
+	option.Config.PolicyCIDRMatchMode = []string{}
+
+	hostIPPrefix := cmtypes.NewLocalPrefixCluster(netip.MustParsePrefix("10.161.39.126/32"))
+
+	// syncHostIPs runs FIRST (correct startup order / no restart window).
+	s.IPIdentityCache.metadata.upsertLocked(
+		hostIPPrefix,
+		source.Local,
+		"daemon-reserved",
+		labels.LabelHost,
+	)
+
+	// CiliumCIDRGroup label arrives afterwards (normal steady-state order).
+	s.IPIdentityCache.metadata.upsertLocked(
+		hostIPPrefix,
+		source.Generated,
+		"cidrgroup-resource-uid",
+		cidrGroupLabels("local-dc"),
+	)
+
+	_, err := s.IPIdentityCache.doInjectLabels(ctx, []cmtypes.PrefixCluster{hostIPPrefix})
+	require.NoError(t, err)
+
+	entry, ok := s.IPIdentityCache.ipToIdentityCache["10.161.39.126/32"]
+	require.True(t, ok)
+
+	// When reserved:host is present, the identity must be ReservedIdentityHost.
+	assert.Equal(t, identity.ReservedIdentityHost, entry.ID,
+		"When reserved:host is already in ipcache metadata before CIDRGroup labels "+
+			"arrive, the identity must be ReservedIdentityHost (id=1). Got id=%d.", entry.ID)
+
+	resolvedIdentity := s.Allocator.LookupIdentityByID(ctx, entry.ID)
+	require.NotNil(t, resolvedIdentity)
+	assert.False(t,
+		resolvedIdentity.Labels.HasWorldLabel() || resolvedIdentity.Labels.HasWorldIPv4Label(),
+		"Identity must not have world label when reserved:host is present. Labels: %v",
+		resolvedIdentity.Labels)
+}

pkg/ipcache/restore/local_identity_restorer_test.go

@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package restoration
+
+// TestHostIdentityExcludedFromIPCacheRestoration tests that the ipcache
+// restoration logic (dumpOldIPCache) explicitly excludes ReservedIdentityHost
+// entries from the set of identities that survive an agent restart.
+//
+// This is one of two root causes for the host-IP-as-world misclassification bug:
+//
+//   dumpOldIPCache() at local_identity_restorer.go:128:
+//
+//     if nid.Scope() == identity.IdentityScopeLocal ||
+//        nid == identity.ReservedIdentityIngress {
+//         localPrefixes[k.Prefix()] = nid  // host identity NEVER matches
+//     }
+//
+// Because ReservedIdentityHost (id=1) has IdentityScopeGlobal (scope bits = 0),
+// it does not pass the IdentityScopeLocal check. It is also not
+// ReservedIdentityIngress. The host IP entry from the OLD ipcache BPF map is
+// therefore NEVER written into localPrefixes and is NEVER restored into the new
+// ipcache metadata.
+//
+// Consequence: after ipcachemap.Recreate() wipes the BPF map (cell.go:118),
+// there is a window before syncHostIPs runs (daemon.go:249) during which host
+// IPs have no ipcache metadata entry. If a CiliumCIDRGroup covering the host IP
+// is processed during this window, resolveLabels() sees only the cidrgroup label,
+// calls AddWorldLabel(), and assigns world identity — causing policy_denied drops.
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cilium/cilium/pkg/identity"
+)
+
+// TestHostIdentityExcludedFromIPCacheRestoration documents the restoration
+// filter condition that causes host IPs to lose their identity on restart.
+//
+// The filter in dumpOldIPCache (local_identity_restorer.go:128) is:
+//
+//	nid.Scope() == identity.IdentityScopeLocal || nid == identity.ReservedIdentityIngress
+//
+// This test verifies the scope/identity values used by the filter and shows
+// which identities are retained vs dropped during restoration.
+func TestHostIdentityExcludedFromIPCacheRestoration(t *testing.T) {
+	type testCase struct {
+		name           string
+		id             identity.NumericIdentity
+		wantRestored   bool
+		explanation    string
+	}
+
+	// These cases mirror the exact filter condition at local_identity_restorer.go:128.
+	cases := []testCase{
+		{
+			name:         "ReservedIdentityHost is NOT restored",
+			id:           identity.ReservedIdentityHost, // id=1, scope=global
+			wantRestored: false,
+			explanation: "ReservedIdentityHost (id=1) has IdentityScopeGlobal (scope bits = 0). " +
+				"It does not satisfy Scope()==IdentityScopeLocal and is not ReservedIdentityIngress. " +
+				"BUG: after ipcachemap.Recreate(), host IP entries are missing from the new ipcache " +
+				"until syncHostIPs runs. If CiliumCIDRGroup processing happens first, " +
+				"resolveLabels() assigns world identity to the host IP.",
+		},
+		{
+			name:         "ReservedIdentityWorld is NOT restored",
+			id:           identity.ReservedIdentityWorld, // id=2, scope=global
+			wantRestored: false,
+			explanation:  "World identity is re-added as a /0 catch-all by syncHostIPs, not via restoration.",
+		},
+		{
+			name:         "ReservedIdentityIngress IS restored",
+			id:           identity.ReservedIdentityIngress,
+			wantRestored: true,
+			explanation:  "Ingress is explicitly included via the nid==ReservedIdentityIngress check.",
+		},
+		{
+			name:         "A local-scope CIDR identity IS restored",
+			id:           identity.NumericIdentity(1<<24 + 42), // IdentityScopeLocal | 42
+			wantRestored: true,
+			explanation: "Local-scope CIDR identities are restored to preserve numeric identity " +
+				"stability across restarts. This is the primary use case for dumpOldIPCache.",
+		},
+		{
+			name:         "A remote-node-scope identity is NOT restored",
+			id:           identity.NumericIdentity(2<<24 + 1), // IdentityScopeRemoteNode | 1
+			wantRestored: false,
+			explanation:  "Remote-node-scope identities are re-derived from the node manager, not restored.",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			// This is the exact filter predicate from dumpOldIPCache:
+			//   local_identity_restorer.go:128
+			wouldBeRestored := tc.id.Scope() == identity.IdentityScopeLocal ||
+				tc.id == identity.ReservedIdentityIngress
+
+			assert.Equal(t, tc.wantRestored, wouldBeRestored,
+				"%s\n"+
+					"  identity id:    %d\n"+
+					"  identity scope: %d (IdentityScopeLocal=%d, IdentityScopeGlobal=%d)\n"+
+					"  filter result:  restored=%v",
+				tc.explanation,
+				tc.id,
+				tc.id.Scope(),
+				identity.IdentityScopeLocal,
+				identity.IdentityScopeGlobal,
+				wouldBeRestored,
+			)
+		})
+	}
+}
+
+// TestHostIdentityScopeIsGlobal explicitly verifies that ReservedIdentityHost
+// has global scope, which is why it is excluded from the restoration filter.
+// This is the direct mechanical reason for the misclassification bug.
+func TestHostIdentityScopeIsGlobal(t *testing.T) {
+	hostScope := identity.ReservedIdentityHost.Scope()
+
+	assert.Equal(t, identity.IdentityScopeGlobal, hostScope,
+		"ReservedIdentityHost must have IdentityScopeGlobal (scope=0). "+
+			"This means it is excluded by the dumpOldIPCache filter "+
+			"(local_identity_restorer.go:128) which only retains IdentityScopeLocal "+
+			"and ReservedIdentityIngress. As a result, host IP entries are lost from "+
+			"the ipcache BPF map after ipcachemap.Recreate() and are not re-inserted "+
+			"until syncHostIPs.StartAndWaitFirst() completes (daemon.go:249).")
+
+	assert.NotEqual(t, identity.IdentityScopeLocal, hostScope,
+		"If this assertion fails, the bug would be self-healing: "+
+			"host identity would be restored and the world fallback would not occur.")
+}