feat(cmd): Add config-remove and config-update commands to CLI (#2068)

This PR supersedes #1481 as it was outdated (my fault for not addressing
earlier) and also simplifies the `config-update` command.

It introduces two new commands: 
* `config-remove` which will delete a node's config
* `config-update` which will update an outdated config by loading it,
applying any newly added fields, and saving it back in the config path.

There is one unfortunate caveat which is that if a newly added field is
merged into the old config, some old custom values from that field may
be overwritten which is why I included a prompt to the user to ensure
their old custom values were preserved. Custom values from other fields
that were unchanged in the config are preserved however.

If there's time, I will investigate how to preserve custom values of a
specific field that has been upgraded.

Credit to @dixitaniket for implementing this feature.

Closes #1414

---------

Co-authored-by: Hlib Kanunnikov <[email protected]>

Author: renaynay

cmd/celestia/bridge.go

@@ -32,6 +32,8 @@ func init() {
 		cmdnode.Start(flags...),
 		cmdnode.AuthCmd(flags...),
 		cmdnode.ResetStore(flags...),
+		cmdnode.RemoveConfigCmd(flags...),
+		cmdnode.UpdateConfigCmd(flags...),
 	)
 }
 

cmd/celestia/full.go

@@ -36,6 +36,8 @@ func init() {
 		cmdnode.Start(flags...),
 		cmdnode.AuthCmd(flags...),
 		cmdnode.ResetStore(flags...),
+		cmdnode.RemoveConfigCmd(flags...),
+		cmdnode.UpdateConfigCmd(flags...),
 	)
 }
 

cmd/celestia/light.go

@@ -36,6 +36,8 @@ func init() {
 		cmdnode.Start(flags...),
 		cmdnode.AuthCmd(flags...),
 		cmdnode.ResetStore(flags...),
+		cmdnode.RemoveConfigCmd(flags...),
+		cmdnode.UpdateConfigCmd(flags...),
 	)
 }
 

cmd/config.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	flag "github.com/spf13/pflag"
+
+	"github.com/celestiaorg/celestia-node/nodebuilder"
+)
+
+func RemoveConfigCmd(fsets ...*flag.FlagSet) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "config-remove",
+		Short: "Deletes the node's config",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			return nodebuilder.RemoveConfig(StorePath(ctx))
+		},
+	}
+
+	for _, set := range fsets {
+		cmd.Flags().AddFlagSet(set)
+	}
+	return cmd
+}
+
+func UpdateConfigCmd(fsets ...*flag.FlagSet) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "config-update",
+		Short: "Updates the node's outdated config",
+		Long: "Updates the node's outdated config with default values from newly-added fields. Check the config " +
+			" afterwards to ensure all old custom values were preserved.",
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			return nodebuilder.UpdateConfig(NodeType(ctx), StorePath(ctx))
+		},
+	}
+
+	for _, set := range fsets {
+		cmd.Flags().AddFlagSet(set)
+	}
+	return cmd
+}

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-retryablehttp v0.7.2
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
+	github.com/imdario/mergo v0.3.15
 	github.com/ipfs/go-blockservice v0.5.0
 	github.com/ipfs/go-cid v0.3.2
 	github.com/ipfs/go-datastore v0.6.0

go.sum

@@ -917,6 +917,8 @@ github.com/iancoleman/orderedmap v0.1.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428/go.mod h1:uhpZMVGznybq1itEKXj6RYw9I71qK4kH+OGMjRC4KEo=
+github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
+github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ=
 github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=

nodebuilder/config.go

@@ -5,7 +5,9 @@ import (
 	"os"
 
 	"github.com/BurntSushi/toml"
+	"github.com/imdario/mergo"
 
+	"github.com/celestiaorg/celestia-node/libs/fslock"
 	"github.com/celestiaorg/celestia-node/nodebuilder/core"
 	"github.com/celestiaorg/celestia-node/nodebuilder/das"
 	"github.com/celestiaorg/celestia-node/nodebuilder/gateway"
@@ -80,6 +82,76 @@ func LoadConfig(path string) (*Config, error) {
 	return &cfg, cfg.Decode(f)
 }
 
+// RemoveConfig removes the Config from the given store path.
+func RemoveConfig(path string) (err error) {
+	path, err = storePath(path)
+	if err != nil {
+		return
+	}
+
+	flock, err := fslock.Lock(lockPath(path))
+	if err != nil {
+		if err == fslock.ErrLocked {
+			err = ErrOpened
+		}
+		return
+	}
+	defer flock.Unlock() //nolint: errcheck
+
+	return removeConfig(configPath(path))
+}
+
+// removeConfig removes Config from the given 'path'.
+func removeConfig(path string) error {
+	return os.Remove(path)
+}
+
+// UpdateConfig loads the node's config and applies new values
+// from the default config of the given node type, saving the
+// newly updated config into the node's config path.
+func UpdateConfig(tp node.Type, path string) (err error) {
+	path, err = storePath(path)
+	if err != nil {
+		return
+	}
+
+	flock, err := fslock.Lock(lockPath(path))
+	if err != nil {
+		if err == fslock.ErrLocked {
+			err = ErrOpened
+		}
+		return
+	}
+	defer flock.Unlock() //nolint: errcheck
+
+	newCfg := DefaultConfig(tp)
+
+	cfgPath := configPath(path)
+	cfg, err := LoadConfig(cfgPath)
+	if err != nil {
+		return
+	}
+
+	cfg, err = updateConfig(cfg, newCfg)
+	if err != nil {
+		return
+	}
+
+	// save the updated config
+	err = removeConfig(cfgPath)
+	if err != nil {
+		return
+	}
+	return SaveConfig(cfgPath, cfg)
+}
+
+// updateConfig merges new values from the new config into the old
+// config, returning the updated old config.
+func updateConfig(oldCfg *Config, newCfg *Config) (*Config, error) {
+	err := mergo.Merge(oldCfg, newCfg, mergo.WithOverrideEmptySlice)
+	return oldCfg, err
+}
+
 // TODO(@Wondertan): We should have a description for each field written into w,
 // 	so users can instantly understand purpose of each field. Ideally, we should have a utility
 // program to parse comments 	from actual sources(*.go files) and generate docs from comments.

nodebuilder/config_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/BurntSushi/toml"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -33,3 +34,104 @@ func TestConfigWriteRead(t *testing.T) {
 		})
 	}
 }
+
+// TestUpdateConfig tests that updating an outdated config
+// using a new default config applies the correct values and
+// preserves old custom values.
+func TestUpdateConfig(t *testing.T) {
+	cfg := new(Config)
+	_, err := toml.Decode(outdatedConfig, cfg)
+	require.NoError(t, err)
+
+	newCfg := DefaultConfig(node.Light)
+	// ensure this config field is not filled in the outdated config
+	require.NotEqual(t, newCfg.Share.PeerManagerParams, cfg.Share.PeerManagerParams)
+
+	cfg, err = updateConfig(cfg, newCfg)
+	require.NoError(t, err)
+	// ensure this config field is now set after updating the config
+	require.Equal(t, newCfg.Share.PeerManagerParams, cfg.Share.PeerManagerParams)
+	// ensure old custom values were not changed
+	require.Equal(t, "thisshouldnthavechanged", cfg.State.KeyringAccName)
+	require.Equal(t, "7979", cfg.RPC.Port)
+	require.True(t, cfg.Gateway.Enabled)
+}
+
+// outdatedConfig is an outdated config from a light node
+var outdatedConfig = `
+[Core]
+  IP = "0.0.0.0"
+  RPCPort = "0"
+  GRPCPort = "0"
+
+[State]
+  KeyringAccName = "thisshouldnthavechanged"
+  KeyringBackend = "test"
+
+[P2P]
+  ListenAddresses = ["/ip4/0.0.0.0/udp/2121/quic-v1", "/ip6/::/udp/2121/quic-v1", "/ip4/0.0.0.0/tcp/2121", 
+"/ip6/::/tcp/2121"]
+  AnnounceAddresses = []
+  NoAnnounceAddresses = ["/ip4/0.0.0.0/udp/2121/quic-v1", "/ip4/127.0.0.1/udp/2121/quic-v1", "/ip6/::/udp/2121/quic-v1",
+"/ip4/0.0.0.0/tcp/2121", "/ip4/127.0.0.1/tcp/2121", "/ip6/::/tcp/2121"]
+  MutualPeers = []
+  PeerExchange = false
+  RoutingTableRefreshPeriod = "1m0s"
+  [P2P.ConnManager]
+    Low = 50
+    High = 100
+    GracePeriod = "1m0s"
+  [P2P.Metrics]
+    PrometheusAgentPort = "8890"
+
+[RPC]
+  Address = "0.0.0.0"
+  Port = "7979"
+
+[Gateway]
+  Address = "0.0.0.0"
+  Port = "26659"
+  Enabled = true 
+
+[Share]
+  PeersLimit = 5
+  DiscoveryInterval = "30s"
+  AdvertiseInterval = "30s"
+  UseShareExchange = true
+  [Share.ShrExEDSParams]
+    ServerReadTimeout = "5s"
+    ServerWriteTimeout = "1m0s"
+    HandleRequestTimeout = "1m0s"
+    ConcurrencyLimit = 10
+    BufferSize = 32768
+  [Share.ShrExNDParams]
+    ServerReadTimeout = "5s"
+    ServerWriteTimeout = "2m35s"
+    HandleRequestTimeout = "1m0s"
+    ConcurrencyLimit = 10
+
+[Header]
+  TrustedHash = ""
+  TrustedPeers = []
+  [Header.Store]
+    StoreCacheSize = 4096
+    IndexCacheSize = 16384
+    WriteBatchSize = 2048
+  [Header.Syncer]
+    TrustingPeriod = "168h0m0s"
+  [Header.Server]
+    WriteDeadline = "8s"
+    ReadDeadline = "1m0s"
+    RangeRequestTimeout = "10s"
+  [Header.Client]
+    MaxHeadersPerRangeRequest = 64
+    RangeRequestTimeout = "8s"
+    TrustedPeersRequestTimeout = "300ms"
+
+[DASer]
+  SamplingRange = 100
+  ConcurrencyLimit = 16
+  BackgroundStoreInterval = "10m0s"
+  SampleFrom = 1
+  SampleTimeout = "4m0s"
+`