Call WaitGroup.Done() once only when PVB changes to fianl status the first time to avoid panic

changelogs/CHANGELOG-1.16.md

@@ -13,6 +13,7 @@ https://velero.io/docs/v1.16/
 https://velero.io/docs/v1.16/upgrade-to-1.16/
 
 ### All Changes
+  * Call WaitGroup.Done() once only when PVB changes to final status the first time to avoid panic (#8940, @ywk253100)
   * Add VolumeSnapshotContent into the RIA and the mustHave resource list. (#8926, @blackpiglet)
   * Warn for not found error in patching managed fields (#8916, @sseago)
   * Fix issue 8878, relief node os deduction error checks (#8911, @Lyndon-Li)

pkg/podvolume/backupper.go

@@ -173,11 +173,28 @@
 					return
 				}
 
+				statusChangedToFinal := true
+				existObj, exist, err := b.pvbIndexer.Get(pvb)
+				if err == nil && exist {
+					existPVB, ok := existObj.(*velerov1api.PodVolumeBackup)
+					// the PVB in the indexer is already in final status, no need to call WaitGroup.Done()
+					if ok && (existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted ||
+						existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed) {
+						statusChangedToFinal = false
+					}
+				}
+
 				// the Indexer inserts PVB directly if the PVB to be updated doesn't exist
 				if err := b.pvbIndexer.Update(pvb); err != nil {
 					log.WithError(err).Errorf("failed to update PVB %s/%s in indexer", pvb.Namespace, pvb.Name)
 				}
-				b.wg.Done()
+
+				// call WaitGroup.Done() once only when the PVB changes to final status the first time.
+				// This avoid the cases that the handler gets multiple update events whose PVBs are all in final status
+				// which causes panic with "negative WaitGroup counter" error
+				if statusChangedToFinal {
+					b.wg.Done()
+				}
 			},
 		},
 	)