Skip to content

waitfor consumer,the pv is not on that node #303

@13567436138

Description

@13567436138

Describe the bug: A clear and concise description of what the bug is.

Events:
  Type     Reason            Age                  From               Message
  ----     ------            ----                 ----               -------
  Warning  FailedScheduling  2m42s                default-scheduler  0/4 nodes are available: persistentvolumeclaim "data-etcd-0-target" not found. not found
  Normal   Scheduled         2m37s                default-scheduler  Successfully assigned kamaji-system/busybox-0 to kamaji-manager-01-c
  Warning  FailedMount       29s (x9 over 2m37s)  kubelet            MountVolume.NewMounter initialization failed for volume "pvc-9d69442d-6512-4b7e-9e11-8cf0ddbf43ba" : path "/var/openebs/local/pvc-9d69442d-6512-4b7e-9e11-8cf0ddbf43ba" does not exist

all went to this node

root@kamaji-manager-01-d:~# ls /var/openebs/local/
lost+found                                pvc-9d69442d-6512-4b7e-9e11-8cf0ddbf43ba
pvc-5bfd1d83-ac49-4f1b-babc-2a71f64a323b  pvc-a20f40e9-6cf0-42e5-8e61-bbd623aa9f2d
cat << EOF|kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  labels:
    run: busybox
  name: busybox-0
  namespace: kamaji-system
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: kubernetes.io/hostname
            operator: In
            values:
            - kamaji-manager-01-c
  containers:
  - args:
    - sleep
    - "10000000"
    image: registry.cn-hangzhou.aliyuncs.com/hxpdocker/busybox:1.33.1
    imagePullPolicy: IfNotPresent
    name: busybox
    volumeMounts:
    - name: data-volume
      mountPath: /data
      readOnly: false
  volumes:
  - name: data-volume
    persistentVolumeClaim:
      claimName: data-etcd-0-target
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: data-etcd-0-target
  namespace: kamaji-system
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: openebs-hostpath
  volumeMode: Filesystem
---
apiVersion: v1
kind: Pod
metadata:
  labels:
    run: busybox
  name: busybox-1
  namespace: kamaji-system
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: kubernetes.io/hostname
            operator: In
            values:
            - kamaji-manager-01-b
  containers:
  - args:
    - sleep
    - "10000000"
    image: registry.cn-hangzhou.aliyuncs.com/hxpdocker/busybox:1.33.1
    imagePullPolicy: IfNotPresent
    name: busybox
    volumeMounts:
    - name: data-volume
      mountPath: /data
      readOnly: false
  volumes:
  - name: data-volume
    persistentVolumeClaim:
      claimName: data-etcd-1-target
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: data-etcd-1-target
  namespace: kamaji-system
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: openebs-hostpath
  volumeMode: Filesystem
---
apiVersion: v1
kind: Pod
metadata:
  labels:
    run: busybox
  name: busybox-2
  namespace: kamaji-system
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: kubernetes.io/hostname
            operator: In
            values:
            - kamaji-manager-01-d
  containers:
  - args:
    - sleep
    - "10000000"
    image: registry.cn-hangzhou.aliyuncs.com/hxpdocker/busybox:1.33.1
    imagePullPolicy: IfNotPresent
    name: busybox
    volumeMounts:
    - name: data-volume
      mountPath: /data
      readOnly: false
  volumes:
  - name: data-volume
    persistentVolumeClaim:
      claimName: data-etcd-2-target
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: data-etcd-2-target
  namespace: kamaji-system
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: openebs-hostpath
  volumeMode: Filesystem
EOF
root@ubuntu:~# kubectl get sc  openebs-hostpath -o yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
    cas.openebs.io/config: |
      - name: StorageType
        value: "hostpath"
      - name: BasePath
        value: "/var/openebs/local"
      - name: NodeAffinityLabels
        list:
          - openebs.io/local
    meta.helm.sh/release-name: dlp
    meta.helm.sh/release-namespace: dynamic-localpv-provisioner
    openebs.io/cas-type: local
  creationTimestamp: "2025-11-25T04:20:16Z"
  labels:
    app.kubernetes.io/managed-by: Helm
  name: openebs-hostpath
  resourceVersion: "1479977"
  uid: 2fbe26c5-c4b5-4b7f-b22c-72f608892a59
provisioner: openebs.io/local
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer

Expected behaviour: A concise description of what you expected to happen

Steps to reproduce the bug:
Steps to reproduce the bug should be clear and easily reproducible to help people gain an understanding of the problem

The output of the following commands will help us better understand what's going on:

  • kubectl get pods -n <openebs_namespace> --show-labels
  • kubectl logs <upgrade_job_pod> -n <openebs_namespace>

Anything else we need to know?:
Add any other context about the problem here.

Environment details:

  • OpenEBS version (use kubectl get po -n openebs --show-labels):4.4.0-prerelease
  • Kubernetes version (use kubectl version):1.34.1
  • Cloud provider or hardware configuration:
  • OS (e.g: cat /etc/os-release):
  • kernel (e.g: uname -a):
  • others:

Metadata

Metadata

Assignees

No one assigned

    Labels

    kind/bugCategorizes issue or PR as related to a bug

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions