diff --git a/pkg/controller/init.go b/pkg/controller/init.go
index ad28963bb35..82d095e9a92 100644
--- a/pkg/controller/init.go
+++ b/pkg/controller/init.go
@@ -437,6 +437,10 @@ func (c *Controller) InitIPAM() error {
 			continue
 		}
 
+		if !hasAllocatedAnnotation(pod) {
+			continue
+		}
+
 		podNets, err := c.getPodKubeovnNets(pod)
 		if err != nil {
 			klog.Errorf("failed to get pod kubeovn nets %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[util.IPAddressAnnotation], err)
@@ -1034,3 +1038,12 @@ func (c *Controller) syncFinalizers() error {
 	klog.Info("sync finalizers done")
 	return nil
 }
+
+func hasAllocatedAnnotation(pod *v1.Pod) bool {
+	for key, value := range pod.Annotations {
+		if value == "true" && strings.HasSuffix(key, util.AllocatedAnnotationSuffix) {
+			return true
+		}
+	}
+	return false
+}
diff --git a/pkg/controller/init_test.go b/pkg/controller/init_test.go
new file mode 100644
index 00000000000..485ad5ce740
--- /dev/null
+++ b/pkg/controller/init_test.go
@@ -0,0 +1,76 @@
+package controller
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestHasAllocatedAnnotation(t *testing.T) {
+	tests := []struct {
+		name        string
+		annotations map[string]string
+		expected    bool
+	}{
+		{
+			name:        "nil annotations",
+			annotations: nil,
+			expected:    false,
+		},
+		{
+			name:        "empty annotations",
+			annotations: map[string]string{},
+			expected:    false,
+		},
+		{
+			name: "default provider allocated",
+			annotations: map[string]string{
+				"ovn.kubernetes.io/allocated": "true",
+			},
+			expected: true,
+		},
+		{
+			name: "custom provider allocated",
+			annotations: map[string]string{
+				"my-provider.kubernetes.io/allocated": "true",
+			},
+			expected: true,
+		},
+		{
+			name: "allocated is false",
+			annotations: map[string]string{
+				"ovn.kubernetes.io/allocated": "false",
+			},
+			expected: false,
+		},
+		{
+			name: "unrelated annotations only",
+			annotations: map[string]string{
+				"app":                          "test",
+				"ovn.kubernetes.io/ip_address": "10.0.0.1",
+			},
+			expected: false,
+		},
+		{
+			name: "multiple providers with one allocated",
+			annotations: map[string]string{
+				"ovn.kubernetes.io/allocated":         "false",
+				"my-provider.kubernetes.io/allocated": "true",
+			},
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: tt.annotations,
+				},
+			}
+			require.Equal(t, tt.expected, hasAllocatedAnnotation(pod))
+		})
+	}
+}