Skip to content

Commit d609e9f

Browse files
stevenlandersstevenlanders
authored
Add integration test for upgrades (#1082)
* add integration test for upgrades * fix path to script * fix path * fix substitution issue * add major upgrade test * cleanup * fix major release test * try different order * try fake upgrade test * try with -u * remove fake job * add more flushes to logs * remove extra flush * attempt with some hang detection * cleanup, add tests to help coverage * cleanup makefile * make major upgrade more tolerant to hangs * adjust exit code for pass situation * merge into integration-tests * de-upgrade docker-compose * add debug ls to integration-test * use yaml instead of yml so the file can be found * remove debug ls * remove excess s from path * make it work with a full path * point to latest cosmos-sdk release * reduce flakiness * update with actual cosmos sdk
1 parent 62bcd8f commit d609e9f

22 files changed

+676
-11
lines changed

‎.github/workflows/integration-test.yml

+15-1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,20 @@ jobs:
8484
"python3 integration_test/scripts/runner.py integration_test/distribution_module/rewards.yaml",
8585
]
8686
},
87+
{
88+
name: "Upgrade Module (Major)",
89+
env: "UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2",
90+
scripts: [
91+
"python3 integration_test/scripts/runner.py integration_test/upgrade_module/major_upgrade_test.yaml"
92+
]
93+
},
94+
{
95+
name: "Upgrade Module (Minor)",
96+
env: "UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2",
97+
scripts: [
98+
"python3 integration_test/scripts/runner.py integration_test/upgrade_module/minor_upgrade_test.yaml"
99+
]
100+
},
87101
]
88102
steps:
89103
- uses: actions/checkout@v3
@@ -104,7 +118,7 @@ jobs:
104118
go-version: 1.18
105119

106120
- name: Start 4 node docker cluster
107-
run: make clean && INVARIANT_CHECK_INTERVAL=10 make docker-cluster-start &
121+
run: make clean && INVARIANT_CHECK_INTERVAL=10 ${{matrix.test.env}} make docker-cluster-start &
108122

109123
- name: Wait for docker cluster to start
110124
run: |

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ kill-rpc-node:
179179
# Run a 4-node docker containers
180180
docker-cluster-start: docker-cluster-stop build-docker-node
181181
@rm -rf $(PROJECT_HOME)/build/generated
182-
@cd docker && NUM_ACCOUNTS=10 INVARIANT_CHECK_INTERVAL=${INVARIANT_CHECK_INTERVAL} docker-compose up
182+
@cd docker && NUM_ACCOUNTS=10 INVARIANT_CHECK_INTERVAL=${INVARIANT_CHECK_INTERVAL} UPGRADE_VERSION_LIST=${UPGRADE_VERSION_LIST} docker-compose up
183183

184184
.PHONY: localnet-start
185185

‎app/upgrades.go

+14
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ package app
22

33
import (
44
"log"
5+
"os"
56
"sort"
7+
"strings"
68

79
sdk "github.com/cosmos/cosmos-sdk/types"
810
"github.com/cosmos/cosmos-sdk/types/module"
@@ -62,12 +64,24 @@ var upgradesList = []string{
6264
"v3.1.1",
6365
}
6466

67+
// if there is an override list, use that instead, for integration tests
68+
func overrideList() {
69+
// if there is an override list, use that instead, for integration tests
70+
envList := os.Getenv("UPGRADE_VERSION_LIST")
71+
if envList != "" {
72+
upgradesList = strings.Split(envList, ",")
73+
}
74+
}
75+
6576
func (app App) RegisterUpgradeHandlers() {
6677
// Upgrades names must be in alphabetical order
6778
// https://github.com/cosmos/cosmos-sdk/issues/11707
6879
if !sort.StringsAreSorted(upgradesList) {
6980
log.Fatal("New upgrades must be appended to 'upgradesList' in alphabetical order")
7081
}
82+
83+
// if there is an override list, use that instead, for integration tests
84+
overrideList()
7185
for _, upgradeName := range upgradesList {
7286
app.UpgradeKeeper.SetUpgradeHandler(upgradeName, func(ctx sdk.Context, plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
7387
// Set params to Distribution here when migrating

‎app/upgrades_test.go

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package app
2+
3+
import (
4+
"os"
5+
"reflect"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
)
10+
11+
func TestOverrideList(t *testing.T) {
12+
defaultList := upgradesList
13+
tests := []struct {
14+
name string
15+
envValue string
16+
expectedList []string
17+
}{
18+
{
19+
name: "UPGRADE_VERSION_LIST not set",
20+
envValue: "",
21+
expectedList: defaultList,
22+
},
23+
{
24+
name: "UPGRADE_VERSION_LIST set with single value",
25+
envValue: "2.0.0",
26+
expectedList: []string{"2.0.0"},
27+
},
28+
{
29+
name: "UPGRADE_VERSION_LIST set with multiple values",
30+
envValue: "2.0.0,2.1.0,2.2.0",
31+
expectedList: []string{"2.0.0", "2.1.0", "2.2.0"},
32+
},
33+
}
34+
35+
for _, tt := range tests {
36+
t.Run(tt.name, func(t *testing.T) {
37+
if tt.envValue != "" {
38+
os.Setenv("UPGRADE_VERSION_LIST", tt.envValue)
39+
defer os.Unsetenv("UPGRADE_VERSION_LIST")
40+
}
41+
// reset upgrades list before each test
42+
upgradesList = defaultList
43+
44+
overrideList()
45+
46+
assert.True(t, reflect.DeepEqual(tt.expectedList, upgradesList), "Expected %v but got %v", tt.expectedList, upgradesList)
47+
})
48+
}
49+
}

‎docker/docker-compose.yml

+4
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ services:
1414
- NUM_ACCOUNTS
1515
- SKIP_BUILD
1616
- INVARIANT_CHECK_INTERVAL
17+
- UPGRADE_VERSION_LIST
1718
volumes:
1819
- "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
1920
- "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
@@ -36,6 +37,7 @@ services:
3637
- NUM_ACCOUNTS
3738
- SKIP_BUILD
3839
- INVARIANT_CHECK_INTERVAL
40+
- UPGRADE_VERSION_LIST
3941
volumes:
4042
- "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
4143
- "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
@@ -55,6 +57,7 @@ services:
5557
- NUM_ACCOUNTS
5658
- SKIP_BUILD
5759
- INVARIANT_CHECK_INTERVAL
60+
- UPGRADE_VERSION_LIST
5861
ports:
5962
- "26662-26664:26656-26658"
6063
- "9094-9095:9090-9091"
@@ -77,6 +80,7 @@ services:
7780
- NUM_ACCOUNTS
7881
- SKIP_BUILD
7982
- INVARIANT_CHECK_INTERVAL
83+
- UPGRADE_VERSION_LIST
8084
ports:
8185
- "26665-26667:26656-26658"
8286
- "9096-9097:9090-9091"

‎docker/localnode/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,4 @@ COPY scripts/step3_add_validator_to_genesis.sh /usr/bin/add_validator_to_gensis.
2424
COPY scripts/step4_config_override.sh /usr/bin/config_override.sh
2525
COPY scripts/step5_start_sei.sh /usr/bin/start_sei.sh
2626
COPY scripts/step6_start_price_feeder.sh /usr/bin/start_price_feeder.sh
27-
ENV PATH "$PATH:$HOME/go/bin"
27+
ENV PATH "$PATH:$HOME/go/bin:/sei-protocol/sei-chain/integration_test/upgrade_module/scripts/"

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ require (
273273
replace (
274274
github.com/CosmWasm/wasmd => github.com/sei-protocol/sei-wasmd v0.0.2
275275
github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
276-
github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.2.61
276+
github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.2.62
277277
github.com/cosmos/iavl => github.com/sei-protocol/sei-iavl v0.1.7
278278
github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.2.0
279279
github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -1074,8 +1074,8 @@ github.com/securego/gosec/v2 v2.11.0/go.mod h1:SX8bptShuG8reGC0XS09+a4H2BoWSJi+f
10741074
github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
10751075
github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQpIbXDA=
10761076
github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8=
1077-
github.com/sei-protocol/sei-cosmos v0.2.61 h1:2BmRscI5ZPfTqRbBQpwrBMZ0vJeGRBBUoDwDUwPnlGA=
1078-
github.com/sei-protocol/sei-cosmos v0.2.61/go.mod h1:IdRmfhjeuY+S3HLd+pSwsYdDt/j+egIk0KHyHMmXSgM=
1077+
github.com/sei-protocol/sei-cosmos v0.2.62 h1:JfT6WcEWfqTmN/xaRr1nSS84wmQCOXZ6g6dknCBCB1c=
1078+
github.com/sei-protocol/sei-cosmos v0.2.62/go.mod h1:IdRmfhjeuY+S3HLd+pSwsYdDt/j+egIk0KHyHMmXSgM=
10791079
github.com/sei-protocol/sei-iavl v0.1.7 h1:cUdHDBkxs0FF/kOt1qCVLm0K+Bqaw92/dbZSgn4kxiA=
10801080
github.com/sei-protocol/sei-iavl v0.1.7/go.mod h1:7PfkEVT5dcoQE+s/9KWdoXJ8VVVP1QpYYPLdxlkSXFk=
10811081
github.com/sei-protocol/sei-ibc-go/v3 v3.2.0 h1:T8V75OEWKvYDraPZZKilprl7ZkahZYGo40crxNL4unc=

‎integration_test/scripts/runner.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -22,23 +22,23 @@ def process_data(self, data):
2222
# Function to execute a single test case
2323
def run_test(self, test):
2424
test_name = test["name"]
25-
print("\n========== " + test_name + " =========")
25+
print("\n========== " + test_name + " =========", flush=True)
2626
inputs = test["inputs"]
2727
env_map = {}
2828
for input in inputs:
2929
cmd = input['cmd']
3030
container = input.get("node", "sei-node-0")
31-
print(f'Input : {cmd}')
31+
print(f'Input : {cmd}', flush=True)
3232
output = self.run_bash_command(cmd, True, container, env_map, False)
3333
if input.get('env'):
3434
env_map[input['env']] = output
3535
result = output
36-
print(f'Output: {result}')
36+
print(f'Output: {result}', flush=True)
3737
for verifier in test["verifiers"]:
3838
if not self.verify_result(env_map, verifier):
39-
print("Test failed for {}".format(verifier))
39+
print("Test failed for {}".format(verifier), flush=True)
4040
exit(1)
41-
print("Test Passed")
41+
print("Test Passed", flush=True)
4242

4343
# Function to verify the result of a single test case
4444
def verify_result(self, env_map, verifier):

‎integration_test/upgrade_module/major_upgrade_test.yaml

+176
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
# Major Release Upgrade Scenarios:
2+
# 1. Proposal & Voting:
3+
# - A major release upgrade is proposed for a future block height.
4+
# - All nodes vote in favor of the proposal.
5+
6+
# 2. Early Upgrade:
7+
# - Node 0 is prematurely upgraded to the major release.
8+
# - It's verified that this node panics due to the early upgrade.
9+
10+
# 3. Target Block Height:
11+
# - The network reaches the proposed upgrade block height.
12+
# - Non-upgraded nodes (1, 2, and 3) panic, indicating the need for the upgrade.
13+
# - An "UPGRADE NEEDED" log message is expected on these nodes.
14+
15+
# 4. Post-Upgrade Behavior:
16+
# - Nodes 1, 2, and 3 are upgraded to the major release and resume normal operation.
17+
# - Node 0, which was prematurely upgraded, is downgraded to sync with the network.
18+
# - Once synced, Node 0 also panics at the upgrade height and logs the "UPGRADE NEEDED" message.
19+
# - Finally, Node 0 is upgraded to the major release and resumes normal operation.
20+
21+
# 5. Final State:
22+
# - All nodes are verified to be running smoothly post-upgrade.
23+
24+
25+
- name: Test Major Release Upgrade
26+
inputs:
27+
- cmd: echo "v2.0.0"
28+
env: VERSION
29+
# PASS PROPOSAL FOR MAJOR UPGRADE
30+
# Get block for 30s from now
31+
- cmd: proposal_target_height.sh 30
32+
env: TARGET_HEIGHT
33+
node: sei-node-0
34+
35+
# Submit the software upgrade proposal
36+
- cmd: proposal_submit.sh $TARGET_HEIGHT major $VERSION
37+
env: PROPOSAL_ID
38+
node: sei-node-0
39+
40+
# Vote with all nodes
41+
- cmd: proposal_vote.sh $PROPOSAL_ID
42+
node: sei-node-0
43+
- cmd: proposal_vote.sh $PROPOSAL_ID
44+
node: sei-node-1
45+
- cmd: proposal_vote.sh $PROPOSAL_ID
46+
node: sei-node-2
47+
- cmd: proposal_vote.sh $PROPOSAL_ID
48+
node: sei-node-3
49+
50+
# Wait for the proposal to pass
51+
- cmd: proposal_wait_for_pass.sh $PROPOSAL_ID
52+
node: sei-node-0
53+
54+
# EXECUTE UPGRADE SCENARIOS
55+
# Confirm no panic before upgrade and block height
56+
- cmd: verify_running.sh
57+
node: sei-node-0
58+
env: RUNNING_BEFORE_UPGRADE_NODE_0
59+
- cmd: verify_running.sh
60+
node: sei-node-1
61+
env: RUNNING_BEFORE_UPGRADE_NODE_1
62+
- cmd: verify_running.sh
63+
node: sei-node-2
64+
env: RUNNING_BEFORE_UPGRADE_NODE_2
65+
- cmd: verify_running.sh
66+
node: sei-node-3
67+
env: RUNNING_BEFORE_UPGRADE_NODE_3
68+
69+
# Upgrade to major release (node 0)
70+
- cmd: seid_upgrade.sh $VERSION
71+
node: sei-node-0
72+
73+
# Confirm the node is not running because it upgraded too early
74+
- cmd: verify_panic.sh $TARGET_HEIGHT
75+
node: sei-node-0
76+
env: PANIC_AFTER_EARLY_UPGRADE_NODE_0
77+
78+
# Wait for the target height (non-panicked nodes)
79+
- cmd: wait_for_height.sh $TARGET_HEIGHT
80+
node: sei-node-1
81+
- cmd: wait_for_height.sh $TARGET_HEIGHT
82+
node: sei-node-2
83+
- cmd: wait_for_height.sh $TARGET_HEIGHT
84+
node: sei-node-3
85+
86+
# Confirm panic and UPGRADE NEEDED log message for non-upgraded node
87+
- cmd: verify_panic.sh $TARGET_HEIGHT
88+
node: sei-node-2
89+
env: PANIC_AT_BLOCK_HEIGHT_NODE_2
90+
- cmd: verify_panic.sh $TARGET_HEIGHT
91+
node: sei-node-3
92+
env: PANIC_AT_BLOCK_HEIGHT_NODE_3
93+
94+
# Upgrade all nodes to major release
95+
- cmd: seid_upgrade.sh $VERSION
96+
node: sei-node-1
97+
- cmd: seid_upgrade.sh $VERSION
98+
node: sei-node-2
99+
- cmd: seid_upgrade.sh $VERSION
100+
node: sei-node-3
101+
102+
# Downgrade node 0 so that it can sync the rest of the blocks
103+
- cmd: seid_downgrade.sh
104+
node: sei-node-0
105+
106+
# Confirm nodes are running after upgrade
107+
- cmd: verify_running.sh
108+
node: sei-node-1
109+
env: RUNNING_UPGRADED_NODE_1
110+
- cmd: verify_running.sh
111+
node: sei-node-2
112+
env: RUNNING_UPGRADED_NODE_2
113+
- cmd: verify_running.sh
114+
node: sei-node-3
115+
env: RUNNING_UPGRADED_NODE_3
116+
117+
# Wait for the target height (on node 0, catching up)
118+
- cmd: wait_for_height.sh $TARGET_HEIGHT
119+
node: sei-node-0
120+
121+
# Confirm node 0 has now panicked and logged
122+
- cmd: verify_panic.sh $TARGET_HEIGHT
123+
node: sei-node-0
124+
env: PANIC_AT_BLOCK_HEIGHT_NODE_0
125+
- cmd: verify_upgrade_needed_log.sh $TARGET_HEIGHT $VERSION
126+
node: sei-node-0
127+
env: LOG_AT_BLOCK_HEIGHT_NODE_0
128+
129+
# Upgrade node 0 now that it has caught up
130+
- cmd: seid_upgrade.sh $VERSION
131+
node: sei-node-0
132+
133+
# node 0 should be running now
134+
- cmd: verify_running.sh
135+
node: sei-node-0
136+
env: RUNNING_UPGRADED_NODE_0
137+
138+
verifiers:
139+
# Nodes are running before upgrade
140+
- type: eval
141+
expr: RUNNING_BEFORE_UPGRADE_NODE_0 == "PASS"
142+
- type: eval
143+
expr: RUNNING_BEFORE_UPGRADE_NODE_1 == "PASS"
144+
- type: eval
145+
expr: RUNNING_BEFORE_UPGRADE_NODE_2 == "PASS"
146+
- type: eval
147+
expr: RUNNING_BEFORE_UPGRADE_NODE_3 == "PASS"
148+
149+
# Upgraded nodes panic because they were upgraded too early
150+
- type: eval
151+
expr: PANIC_AFTER_EARLY_UPGRADE_NODE_0 == "PASS"
152+
153+
# At least one node should panic and print the upgrade-needed log message
154+
# The other will possibly hang on block-1 because of missing peers (due to race)
155+
- type: eval
156+
expr: PANIC_AT_BLOCK_HEIGHT_NODE_2 == "PASS"
157+
- type: eval
158+
expr: PANIC_AT_BLOCK_HEIGHT_NODE_3 == "PASS"
159+
160+
# After upgrade, all nodes are running
161+
- type: eval
162+
expr: RUNNING_UPGRADED_NODE_1 == "PASS"
163+
- type: eval
164+
expr: RUNNING_UPGRADED_NODE_2 == "PASS"
165+
- type: eval
166+
expr: RUNNING_UPGRADED_NODE_3 == "PASS"
167+
168+
# After downgrade and restart, node 0 panics at height
169+
- type: eval
170+
expr: PANIC_AT_BLOCK_HEIGHT_NODE_0 == "PASS"
171+
- type: eval
172+
expr: LOG_AT_BLOCK_HEIGHT_NODE_0 == "PASS"
173+
174+
# After upgrade, node 0 is running
175+
- type: eval
176+
expr: RUNNING_UPGRADED_NODE_0 == "PASS"

0 commit comments

Comments
 (0)