Merge pull request #19244 from henrybear327/e2e/downgrade_cancellation_e2e

Add downgrade cancellation e2e tests

Author: ahrtr

etcdctl/README.md

@@ -1119,7 +1119,7 @@ DOWNGRADE ENABLE starts a downgrade action to cluster.
 Downgrade enable success, cluster version 3.6
 ```
 
-### DOWNGRADE CANCEL \<TARGET_VERSION\>
+### DOWNGRADE CANCEL
 
 DOWNGRADE CANCEL cancels the ongoing downgrade action to cluster.
 

tests/e2e/cluster_downgrade_test.go

@@ -37,23 +37,47 @@ import (
 	"go.etcd.io/etcd/tests/v3/framework/e2e"
 )
 
+type CancellationState int
+
+const (
+	noCancellation CancellationState = iota
+	cancelRightBeforeEnable
+	cancelRightAfterEnable
+)
+
 func TestDowngradeUpgradeClusterOf1(t *testing.T) {
-	testDowngradeUpgrade(t, 1, false)
+	testDowngradeUpgrade(t, 1, false, noCancellation)
 }
 
 func TestDowngradeUpgradeClusterOf3(t *testing.T) {
-	testDowngradeUpgrade(t, 3, false)
+	testDowngradeUpgrade(t, 3, false, noCancellation)
 }
 
 func TestDowngradeUpgradeClusterOf1WithSnapshot(t *testing.T) {
-	testDowngradeUpgrade(t, 1, true)
+	testDowngradeUpgrade(t, 1, true, noCancellation)
 }
 
 func TestDowngradeUpgradeClusterOf3WithSnapshot(t *testing.T) {
-	testDowngradeUpgrade(t, 3, true)
+	testDowngradeUpgrade(t, 3, true, noCancellation)
+}
+
+func TestDowngradeCancellationWithoutEnablingClusterOf1(t *testing.T) {
+	testDowngradeUpgrade(t, 1, false, cancelRightBeforeEnable)
+}
+
+func TestDowngradeCancellationRightAfterEnablingClusterOf1(t *testing.T) {
+	testDowngradeUpgrade(t, 1, false, cancelRightAfterEnable)
+}
+
+func TestDowngradeCancellationWithoutEnablingClusterOf3(t *testing.T) {
+	testDowngradeUpgrade(t, 3, false, cancelRightBeforeEnable)
 }
 
-func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
+func TestDowngradeCancellationRightAfterEnablingClusterOf3(t *testing.T) {
+	testDowngradeUpgrade(t, 3, false, cancelRightAfterEnable)
+}
+
+func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool, triggerCancellation CancellationState) {
 	currentEtcdBinary := e2e.BinPath.Etcd
 	lastReleaseBinary := e2e.BinPath.EtcdLastRelease
 	if !fileutil.Exist(lastReleaseBinary) {
@@ -107,9 +131,26 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
 	require.NoError(t, err)
 	beforeMembers, beforeKV := getMembersAndKeys(t, cc)
 
+	if triggerCancellation == cancelRightBeforeEnable {
+		t.Logf("Cancelling downgrade before enabling")
+		e2e.DowngradeCancel(t, epc)
+		t.Log("Downgrade cancelled, validating if cluster is in the right state")
+		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr))
+
+		return // No need to perform downgrading, end the test here
+	}
 	e2e.DowngradeEnable(t, epc, lastVersion)
+	if triggerCancellation == cancelRightAfterEnable {
+		t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)")
+		e2e.DowngradeCancel(t, epc)
+		t.Log("Downgrade cancelled, validating if cluster is in the right state")
+		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr))
+		return // No need to perform downgrading, end the test here
+	}
+
 	t.Logf("Starting downgrade process to %q", lastVersionStr)
-	e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), currentVersion, lastClusterVersion)
+	err = e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), currentVersion, lastClusterVersion)
+	require.NoError(t, err)
 	e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded")
 
 	t.Log("Downgrade complete")
@@ -135,7 +176,8 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
 	beforeMembers, beforeKV = getMembersAndKeys(t, cc)
 
 	t.Logf("Starting upgrade process to %q", currentVersionStr)
-	e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), lastClusterVersion, currentVersion)
+	err = e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), lastClusterVersion, currentVersion)
+	require.NoError(t, err)
 	t.Log("Upgrade complete")
 
 	afterMembers, afterKV = getMembersAndKeys(t, cc)
@@ -233,3 +275,17 @@ func getMembersAndKeys(t *testing.T, cc *e2e.EtcdctlV3) (*clientv3.MemberListRes
 
 	return members, kvs
 }
+
+func generateIdenticalVersions(clusterSize int, currentVersion string) []*version.Versions {
+	ret := make([]*version.Versions, clusterSize)
+
+	for i := range clusterSize {
+		ret[i] = &version.Versions{
+			Cluster: currentVersion,
+			Server:  currentVersion,
+			Storage: currentVersion,
+		}
+	}
+
+	return ret
+}

tests/framework/e2e/downgrade.go

@@ -53,6 +53,17 @@ func DowngradeEnable(t *testing.T, epc *EtcdProcessCluster, ver *semver.Version)
 	t.Log("Cluster is ready for downgrade")
 }
 
+func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster) {
+	t.Logf("etcdctl downgrade cancel")
+	c := epc.Etcdctl()
+	testutils.ExecuteWithTimeout(t, 20*time.Second, func() {
+		err := c.DowngradeCancel(context.TODO())
+		require.NoError(t, err)
+	})
+
+	t.Log("Cluster downgrade cancellation is completed")
+}
+
 func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessCluster, numberOfMembersToChange int, currentVersion, targetVersion *semver.Version) error {
 	if lg == nil {
 		lg = clus.lg
@@ -104,6 +115,13 @@ func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessClus
 	return nil
 }
 
+func ValidateMemberVersions(t *testing.T, epc *EtcdProcessCluster, expect []*version.Versions) {
+	for i := 0; i < len(epc.Procs); i++ {
+		ValidateVersion(t, epc.Cfg, epc.Procs[i], *expect[i])
+	}
+	t.Log("Cluster member version validation after downgrade cancellation is completed")
+}
+
 func ValidateVersion(t *testing.T, cfg *EtcdProcessClusterConfig, member EtcdProcess, expect version.Versions) {
 	testutils.ExecuteWithTimeout(t, 30*time.Second, func() {
 		for {

tests/framework/e2e/etcdctl.go

@@ -86,6 +86,11 @@ func (ctl *EtcdctlV3) DowngradeEnable(ctx context.Context, version string) error
 	return err
 }
 
+func (ctl *EtcdctlV3) DowngradeCancel(ctx context.Context) error {
+	_, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("downgrade", "cancel"), nil, expect.ExpectedResponse{Value: "Downgrade cancel success"})
+	return err
+}
+
 func (ctl *EtcdctlV3) Get(ctx context.Context, key string, o config.GetOptions) (*clientv3.GetResponse, error) {
 	resp := clientv3.GetResponse{}
 	var args []string