Skip to content

Commit f884f2e

Browse files
author
Joshua Sierles
committed
Check if a previous backup is actually running before blocking the next one
1 parent 034a6be commit f884f2e

File tree

1 file changed

+48
-2
lines changed

1 file changed

+48
-2
lines changed

percona/controller/pgbackup/controller.go

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -404,11 +404,57 @@ func getBackupInProgress(ctx context.Context, c client.Client, clusterName, ns s
404404
annotation := pNaming.AnnotationBackupInProgress
405405
crunchyAnnotation := pNaming.ToCrunchyAnnotation(annotation)
406406

407+
// Get the backup name from annotations
408+
backupName := ""
407409
if crunchyCluster.Annotations[crunchyAnnotation] != "" {
408-
return crunchyCluster.Annotations[crunchyAnnotation], nil
410+
backupName = crunchyCluster.Annotations[crunchyAnnotation]
411+
} else if pgCluster.Annotations[annotation] != "" {
412+
backupName = pgCluster.Annotations[annotation]
409413
}
410414

411-
return pgCluster.Annotations[annotation], nil
415+
// If no backup is marked as in progress, return empty
416+
if backupName == "" {
417+
return "", nil
418+
}
419+
420+
// Verify the backup is actually still running
421+
backup := &v2.PerconaPGBackup{}
422+
err = c.Get(ctx, types.NamespacedName{Name: backupName, Namespace: ns}, backup)
423+
if err != nil {
424+
if k8serrors.IsNotFound(err) {
425+
// Backup resource doesn't exist, clean up the annotation
426+
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
427+
pg := new(v2.PerconaPGCluster)
428+
if err := c.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: ns}, pg); err != nil {
429+
return errors.Wrap(err, "get PerconaPGCluster")
430+
}
431+
delete(pg.Annotations, annotation)
432+
return c.Update(ctx, pg)
433+
}); err != nil {
434+
return "", errors.Wrap(err, "clean up backup in progress annotation")
435+
}
436+
return "", nil
437+
}
438+
return "", errors.Wrap(err, "get backup")
439+
}
440+
441+
// Check if backup is actually still running
442+
if backup.Status.State != v2.BackupStarting && backup.Status.State != v2.BackupRunning {
443+
// Backup is not actually running, clean up the annotation
444+
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
445+
pg := new(v2.PerconaPGCluster)
446+
if err := c.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: ns}, pg); err != nil {
447+
return errors.Wrap(err, "get PerconaPGCluster")
448+
}
449+
delete(pg.Annotations, annotation)
450+
return c.Update(ctx, pg)
451+
}); err != nil {
452+
return "", errors.Wrap(err, "clean up backup in progress annotation")
453+
}
454+
return "", nil
455+
}
456+
457+
return backupName, nil
412458
}
413459

414460
func getRepo(pg *v2.PerconaPGCluster, pb *v2.PerconaPGBackup) *v1beta1.PGBackRestRepo {

0 commit comments

Comments
 (0)