tool: add db upgrade command

Add a `db upgrade` command that ratches up the DB version to the
newest one.

Author: RaduBerinde

tool/data_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/cockroachdb/datadriven"
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/testkeys"
 	"github.com/cockroachdb/pebble/vfs"
@@ -49,34 +50,6 @@ func runTests(t *testing.T, path string) {
 		fs := vfs.NewMem()
 		t.Run(name, func(t *testing.T) {
 			datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
-				args := []string{d.Cmd}
-				for _, arg := range d.CmdArgs {
-					args = append(args, arg.String())
-				}
-				args = append(args, strings.Fields(d.Input)...)
-
-				// The testdata files contain paths with "/" path separators, but we
-				// might be running on a system with a different path separator
-				// (e.g. Windows). Copy the input data into a mem filesystem which
-				// always uses "/" for the path separator.
-				for i := range args {
-					src := normalize(args[i])
-					dest := vfs.Default.PathBase(src)
-					if ok, err := vfs.Clone(vfs.Default, fs, src, dest); err != nil {
-						return err.Error()
-					} else if ok {
-						args[i] = fs.PathBase(args[i])
-					}
-				}
-
-				var buf bytes.Buffer
-				var secs int64
-				timeNow = func() time.Time { secs++; return time.Unix(secs, 0) }
-
-				defer func() {
-					timeNow = time.Now
-				}()
-
 				// Register a test comparer and merger so that we can check the
 				// behavior of tools when the comparer and merger do not match.
 				comparer := func() *Comparer {
@@ -106,6 +79,51 @@ func runTests(t *testing.T, path string) {
 					m.Name = "test-merger"
 					return &m
 				}()
+
+				if d.Cmd == "create" {
+					dbDir := d.CmdArgs[0].String()
+					opts := &pebble.Options{
+						Comparer:           comparer,
+						Merger:             merger,
+						FS:                 fs,
+						FormatMajorVersion: pebble.FormatMostCompatible,
+					}
+					db, err := pebble.Open(dbDir, opts)
+					if err != nil {
+						d.Fatalf(t, "%v", err)
+					}
+					db.Close()
+					return ""
+				}
+
+				args := []string{d.Cmd}
+				for _, arg := range d.CmdArgs {
+					args = append(args, arg.String())
+				}
+				args = append(args, strings.Fields(d.Input)...)
+
+				// The testdata files contain paths with "/" path separators, but we
+				// might be running on a system with a different path separator
+				// (e.g. Windows). Copy the input data into a mem filesystem which
+				// always uses "/" for the path separator.
+				for i := range args {
+					src := normalize(args[i])
+					dest := vfs.Default.PathBase(src)
+					if ok, err := vfs.Clone(vfs.Default, fs, src, dest); err != nil {
+						return err.Error()
+					} else if ok {
+						args[i] = fs.PathBase(args[i])
+					}
+				}
+
+				var buf bytes.Buffer
+				var secs int64
+				timeNow = func() time.Time { secs++; return time.Unix(secs, 0) }
+
+				defer func() {
+					timeNow = time.Now
+				}()
+
 				openErrEnhancer := func(err error) error {
 					if errors.Is(err, base.ErrCorruption) {
 						return base.CorruptionErrorf("%v\nCustom message in case of corruption error.", err)

tool/db.go

@@ -5,9 +5,11 @@
 package tool
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io"
+	"strings"
 	"text/tabwriter"
 
 	"github.com/cockroachdb/errors"
@@ -29,6 +31,7 @@ import (
 type dbT struct {
 	Root       *cobra.Command
 	Check      *cobra.Command
+	Upgrade    *cobra.Command
 	Checkpoint *cobra.Command
 	Get        *cobra.Command
 	Logs       *cobra.Command
@@ -58,6 +61,7 @@ type dbT struct {
 	ioParallelism int
 	ioSizes       string
 	verbose       bool
+	bypassPrompt  bool
 }
 
 func newDB(
@@ -93,6 +97,17 @@ database not be in use by another process.
 		Args: cobra.ExactArgs(1),
 		Run:  d.runCheck,
 	}
+	d.Upgrade = &cobra.Command{
+		Use:   "upgrade <dir>",
+		Short: "upgrade the DB internal format version",
+		Long: `
+Upgrades the DB internal format version to the latest version.
+It is recommended to make a backup copy of the DB directory before upgrading.
+Requires that the specified database not be in use by another process.
+`,
+		Args: cobra.ExactArgs(1),
+		Run:  d.runUpgrade,
+	}
 	d.Checkpoint = &cobra.Command{
 		Use:   "checkpoint <src-dir> <dest-dir>",
 		Short: "create a checkpoint",
@@ -177,10 +192,10 @@ specified database.
 		Run:  d.runIOBench,
 	}
 
-	d.Root.AddCommand(d.Check, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.IOBench)
+	d.Root.AddCommand(d.Check, d.Upgrade, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.IOBench)
 	d.Root.PersistentFlags().BoolVarP(&d.verbose, "verbose", "v", false, "verbose output")
 
-	for _, cmd := range []*cobra.Command{d.Check, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space} {
+	for _, cmd := range []*cobra.Command{d.Check, d.Upgrade, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space} {
 		cmd.Flags().StringVar(
 			&d.comparerName, "comparer", "", "comparer name (use default if empty)")
 		cmd.Flags().StringVar(
@@ -200,6 +215,10 @@ specified database.
 		cmd.Flags().Var(
 			&d.fmtValue, "value", "value formatter")
 	}
+	for _, cmd := range []*cobra.Command{d.Upgrade} {
+		cmd.Flags().BoolVarP(
+			&d.bypassPrompt, "yes", "y", false, "bypass prompt")
+	}
 
 	d.Scan.Flags().Int64Var(
 		&d.count, "count", 0, "key count for scan (0 is unlimited)")
@@ -350,13 +369,43 @@ func (d *dbT) runCheck(cmd *cobra.Command, args []string) {
 		stats.NumPoints, makePlural("point", stats.NumPoints), stats.NumTombstones, makePlural("tombstone", int64(stats.NumTombstones)))
 }
 
-type nonReadOnly struct{}
+func (d *dbT) runUpgrade(cmd *cobra.Command, args []string) {
+	stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
+	db, err := d.openDB(args[0], nonReadOnly{})
+	if err != nil {
+		fmt.Fprintf(stderr, "%s\n", err)
+		return
+	}
+	defer d.closeDB(stderr, db)
 
-func (n nonReadOnly) apply(opts *pebble.Options) {
-	opts.ReadOnly = false
-	// Increase the L0 compaction threshold to reduce the likelihood of an
-	// unintended compaction changing test output.
-	opts.L0CompactionThreshold = 10
+	targetVersion := pebble.FormatNewest
+	current := db.FormatMajorVersion()
+	if current >= targetVersion {
+		fmt.Fprintf(stdout, "DB is already at internal version %d.\n", current)
+		return
+	}
+	fmt.Fprintf(stdout, "Upgrading DB from internal version %d to %d.\n", current, targetVersion)
+
+	prompt := `WARNING!!!
+This DB will not be usable with older versions of Pebble!
+
+It is strongly recommended to back up the data before upgrading.
+`
+
+	if len(d.opts.BlockPropertyCollectors) == 0 {
+		prompt += `
+If this DB uses custom block property collectors, the upgrade should be invoked
+through a custom binary that configures them. Otherwise, any new tables created
+during upgrade will not have the relevant block properties.
+`
+	}
+	if !d.promptForConfirmation(prompt, cmd.InOrStdin(), stdout, stderr) {
+		return
+	}
+	if err := db.RatchetFormatMajorVersion(targetVersion); err != nil {
+		fmt.Fprintf(stderr, "error: %s\n", err)
+	}
+	fmt.Fprintf(stdout, "Upgrade complete.\n")
 }
 
 func (d *dbT) runCheckpoint(cmd *cobra.Command, args []string) {
@@ -665,6 +714,46 @@ func (d *dbT) runSet(cmd *cobra.Command, args []string) {
 	}
 }
 
+func (d *dbT) promptForConfirmation(prompt string, stdin io.Reader, stdout, stderr io.Writer) bool {
+	if d.bypassPrompt {
+		return true
+	}
+	if _, err := fmt.Fprintf(stdout, "%s\n", prompt); err != nil {
+		fmt.Fprintf(stderr, "Error: %v\n", err)
+		return false
+	}
+	reader := bufio.NewReader(stdin)
+	for {
+		if _, err := fmt.Fprintf(stdout, "Continue? [Y/N] "); err != nil {
+			fmt.Fprintf(stderr, "Error: %v\n", err)
+			return false
+		}
+		answer, err := reader.ReadString('\n')
+		if err != nil {
+			fmt.Fprintf(stderr, "Error: %v\n", err)
+			return false
+		}
+		answer = strings.ToLower(strings.TrimSpace(answer))
+		if answer == "y" || answer == "yes" {
+			return true
+		}
+
+		if answer == "n" || answer == "no" {
+			_, _ = fmt.Fprintf(stderr, "Aborting\n")
+			return false
+		}
+	}
+}
+
+type nonReadOnly struct{}
+
+func (n nonReadOnly) apply(opts *pebble.Options) {
+	opts.ReadOnly = false
+	// Increase the L0 compaction threshold to reduce the likelihood of an
+	// unintended compaction changing test output.
+	opts.L0CompactionThreshold = 10
+}
+
 func propArgs(props []props, getProp func(*props) interface{}) []interface{} {
 	args := make([]interface{}, 0, len(props))
 	for _, p := range props {

tool/testdata/db_upgrade

@@ -0,0 +1,59 @@
+create foo
+----
+
+db set foo blue blue-val
+----
+
+db set foo orange orange-val
+----
+
+db set foo green green-val
+----
+
+db set foo red red-val
+----
+
+db set foo yellow yellow-val
+----
+
+db get foo blue
+----
+[626c75652d76616c]
+
+db get foo yellow
+----
+[79656c6c6f772d76616c]
+
+db upgrade foo
+----
+----
+Upgrading DB from internal version 1 to 16.
+WARNING!!!
+This DB will not be usable with older versions of Pebble!
+
+It is strongly recommended to back up the data before upgrading.
+
+If this DB uses custom block property collectors, the upgrade should be invoked
+through a custom binary that configures them. Otherwise, any new tables created
+during upgrade will not have the relevant block properties.
+
+Continue? [Y/N] Error: EOF
+----
+----
+
+db upgrade foo --yes
+----
+Upgrading DB from internal version 1 to 16.
+Upgrade complete.
+
+db get foo blue
+----
+[626c75652d76616c]
+
+db get foo yellow
+----
+[79656c6c6f772d76616c]
+
+db upgrade foo
+----
+DB is already at internal version 16.