options: add Unsafe.AllowMissingWALDir option

This unsafe option allows opening a store after moving or losing the
previous WAL dir. This is understood to be risky and have associated
data loss if any of the necessary WALs are not in the new directory.

Fixes #5421

Author: RaduBerinde

open_test.go

@@ -240,6 +240,12 @@ func TestOpen_WALFailover(t *testing.T) {
 				switch cmdArg.Key {
 				case "path":
 					o.FS, dataDir = extractFSAndPath(cmdArg)
+				case "wal-dir":
+					fs, dir := extractFSAndPath(cmdArg)
+					if fs != o.FS {
+						td.Fatalf(t, "WAL fs differs from data fs")
+					}
+					o.WALDir = dir
 				case "secondary":
 					fs, dir := extractFSAndPath(cmdArg)
 					o.WALFailover = &WALFailoverOptions{
@@ -248,6 +254,8 @@ func TestOpen_WALFailover(t *testing.T) {
 				case "wal-recovery-dir":
 					fs, dir := extractFSAndPath(cmdArg)
 					o.WALRecoveryDirs = append(o.WALRecoveryDirs, wal.Dir{FS: fs, Dirname: dir})
+				case "allow-missing-wal-dirs":
+					o.Unsafe.AllowMissingWALDirs = true
 				default:
 					return fmt.Sprintf("unrecognized cmdArg %q", cmdArg.Key)
 				}

options.go

@@ -1048,7 +1048,8 @@ type Options struct {
 	// the current database state.
 	//
 	// If a previous WAL configuration may have stored WALs elsewhere but there
-	// is not a corresponding entry in WALRecoveryDirs, Open will error.
+	// is not a corresponding entry in WALRecoveryDirs, Open will error (unless
+	// Unsafe.AllowMissingWALDirs is true).
 	WALRecoveryDirs []wal.Dir
 
 	// WALMinSyncInterval is the minimum duration between syncs of the WAL. If
@@ -1109,6 +1110,19 @@ type Options struct {
 	// preemptively reduce internal fragmentation when loaded into the block cache.
 	AllocatorSizeClasses []int
 
+	// Unsafe contains options that must be used very carefully and in exceptional
+	// circumstances.
+	Unsafe struct {
+		// AllowMissingWALDirs, if set to true, allows opening a DB when the WAL or
+		// WAL secondary directory was changed and the previous directory is not in
+		// WALRecoveryDirs. This can be used to move WALs without having to keep the
+		// previous directory in the options forever.
+		//
+		// CAUTION: Enabling this option will lead to data loss if the missing
+		// directory contained any WAL files that were not flushed to sstables.
+		AllowMissingWALDirs bool
+	}
+
 	// private options are only used by internal tests or are used internally
 	// for facilitating upgrade paths of unconfigurable functionality.
 	private struct {
@@ -2334,6 +2348,11 @@ func (o *Options) checkWALDir(storeDir, walDir, errContext string) error {
 		}
 	}
 
+	if o.Unsafe.AllowMissingWALDirs {
+		o.Logger.Infof("directory %q may contain relevant WALs but is not in WALRecoveryDirs (AllowMissingWALDirs enabled)", walDir)
+		return nil
+	}
+
 	var buf bytes.Buffer
 	fmt.Fprintf(&buf, "\n  %s\n", errContext)
 	fmt.Fprintf(&buf, "  o.WALDir: %q\n", o.WALDir)
@@ -2344,6 +2363,7 @@ func (o *Options) checkWALDir(storeDir, walDir, errContext string) error {
 	for _, d := range o.WALRecoveryDirs {
 		fmt.Fprintf(&buf, "\n    %q", d.Dirname)
 	}
+
 	return ErrMissingWALRecoveryDir{Dir: walPath, ExtraInfo: buf.String()}
 }
 

testdata/open_wal_failover

@@ -69,3 +69,52 @@ directory "secondary-wals" may contain relevant WALs but is not in WALRecoveryDi
 open path=(a,data) wal-recovery-dir=(b,secondary-wals)
 ----
 ok
+
+open path=(c,data) wal-dir=(c,wal1)
+----
+ok
+
+# We cannot change the WAL directory without providing the previous one as a recovery dir.
+open path=(c,data) wal-dir=(c,wal2)
+----
+directory "wal1" may contain relevant WALs but is not in WALRecoveryDirs
+  WALDir changed from previous options
+  o.WALDir: "wal2"
+  o.WALRecoveryDirs: 0
+
+open path=(c,data) wal-dir=(c,wal2) wal-recovery-dir=(c,wal1)
+----
+ok
+
+# Once we opened the database once (and thus replayed any WALs in the previous
+# WAL dir), we can open without a recovery dir.
+open path=(c,data) wal-dir=(c,wal2)
+----
+ok
+
+open path=(c,data) wal-dir=(c,wal3)
+----
+directory "wal2" may contain relevant WALs but is not in WALRecoveryDirs
+  WALDir changed from previous options
+  o.WALDir: "wal3"
+  o.WALRecoveryDirs: 0
+
+# Test the Unsafe.AllowMissingWALDirs option.
+open path=(c,data) wal-dir=(c,wal3) allow-missing-wal-dirs
+----
+ok
+
+open path=(c,data) wal-dir=(c,wal3) secondary=(d,secondary)
+----
+ok
+
+open path=(c,data) wal-dir=(c,wal3)
+----
+directory "secondary" may contain relevant WALs but is not in WALRecoveryDirs
+  WALFailover.Secondary changed from previous options
+  o.WALDir: "wal3"
+  o.WALRecoveryDirs: 0
+
+open path=(c,data) wal-dir=(c,wal3) allow-missing-wal-dirs
+----
+ok