|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "errors" |
| 5 | + "fmt" |
| 6 | + "log" |
| 7 | + "os" |
| 8 | + "path/filepath" |
| 9 | + "time" |
| 10 | + |
| 11 | + "github.com/AppsGanin/rospanel/internal/backup" |
| 12 | + "github.com/AppsGanin/rospanel/internal/store" |
| 13 | +) |
| 14 | + |
| 15 | +// ensureHealthyDB gates the boot on a readable database, and recovers from the one |
| 16 | +// failure that otherwise ends the install: SQLite reporting the file as corrupt. |
| 17 | +// |
| 18 | +// A hard reboot or a full disk can tear a page and leave "file is not a database" |
| 19 | +// behind. Without this the panel crash-loops forever on a file it will never be |
| 20 | +// able to read, and the operator's only clue is a stack trace. So: quarantine the |
| 21 | +// damaged file (never delete it — it's the only forensic copy, and it may still be |
| 22 | +// partially recoverable by hand) and extract the newest local backup in its place. |
| 23 | +// |
| 24 | +// Recovery is deliberately restricted to store.ErrCorrupt. A locked file, bad |
| 25 | +// permissions or a full disk are all transient or operator-fixable, and restoring |
| 26 | +// over them would destroy good data to "fix" a problem that isn't there. |
| 27 | +// |
| 28 | +// Runs before datasec.Init, because a backup carries its own secrets.key: pulling |
| 29 | +// the archive's DB and key in as a pair keeps encrypted columns decryptable, while |
| 30 | +// restoring only the DB after the key was already loaded would not. |
| 31 | +func ensureHealthyDB(dbPath, dataDir string) error { |
| 32 | + err := store.Check(dbPath) |
| 33 | + if err == nil { |
| 34 | + return nil |
| 35 | + } |
| 36 | + if !errors.Is(err, store.ErrCorrupt) { |
| 37 | + return err |
| 38 | + } |
| 39 | + |
| 40 | + log.Printf("[ALERT] database: %v", err) |
| 41 | + log.Printf("[ALERT] database: the file is damaged — attempting recovery from the newest local backup") |
| 42 | + |
| 43 | + archives, lerr := backup.ListLocal(dataDir) // newest first |
| 44 | + if lerr != nil { |
| 45 | + return fmt.Errorf("database is corrupt and the backup directory is unreadable (%v) — "+ |
| 46 | + "restore a backup by hand: rospanel restore <file>", lerr) |
| 47 | + } |
| 48 | + if len(archives) == 0 { |
| 49 | + return fmt.Errorf("database is corrupt and there is no local backup to restore from. "+ |
| 50 | + "The damaged file is left at %s. Restore an off-box backup with `rospanel restore <file>`, "+ |
| 51 | + "or wipe and start fresh with `rospanel reset`. "+ |
| 52 | + "Turn on scheduled local backups (Настройки → Бэкапы) so this is recoverable next time", dbPath) |
| 53 | + } |
| 54 | + |
| 55 | + quarantine, qerr := quarantineDB(dbPath) |
| 56 | + if qerr != nil { |
| 57 | + return fmt.Errorf("database is corrupt and could not be set aside for recovery: %w", qerr) |
| 58 | + } |
| 59 | + |
| 60 | + newest := filepath.Join(dataDir, backup.LocalBackupDir, archives[0]) |
| 61 | + if rerr := backup.Restore(newest, dataDir); rerr != nil { |
| 62 | + return fmt.Errorf("database is corrupt and restoring %s failed: %w "+ |
| 63 | + "(the damaged database is preserved at %s)", archives[0], rerr, quarantine) |
| 64 | + } |
| 65 | + |
| 66 | + // The archive could itself be damaged or truncated. If what we just restored is |
| 67 | + // also unreadable, stop: a boot loop that keeps unpacking a broken archive over |
| 68 | + // the data dir is worse than a clean failure. |
| 69 | + if cerr := store.Check(dbPath); cerr != nil { |
| 70 | + return fmt.Errorf("restored %s but the database is still unusable: %w "+ |
| 71 | + "(the original damaged database is preserved at %s)", archives[0], cerr, quarantine) |
| 72 | + } |
| 73 | + |
| 74 | + log.Printf("[ALERT] database: recovered from backup %s — changes made after that backup are LOST", archives[0]) |
| 75 | + log.Printf("[ALERT] database: the damaged file is preserved at %s", quarantine) |
| 76 | + return nil |
| 77 | +} |
| 78 | + |
| 79 | +// quarantineDB moves the damaged database aside (with its WAL and shared-memory |
| 80 | +// sidecars, which belong to it and would otherwise be replayed onto the restored |
| 81 | +// file) and returns the path it was moved to. |
| 82 | +func quarantineDB(dbPath string) (string, error) { |
| 83 | + dst := fmt.Sprintf("%s.corrupt-%s", dbPath, time.Now().Format("20060102-150405")) |
| 84 | + if err := os.Rename(dbPath, dst); err != nil { |
| 85 | + return "", err |
| 86 | + } |
| 87 | + for _, suffix := range []string{"-wal", "-shm"} { |
| 88 | + if err := os.Rename(dbPath+suffix, dst+suffix); err != nil && !os.IsNotExist(err) { |
| 89 | + return "", err |
| 90 | + } |
| 91 | + } |
| 92 | + return dst, nil |
| 93 | +} |
0 commit comments