Skip to content

Commit 6ef59f3

Browse files
mriccobenemriccobene
authored
qa-tests: add Geth test to the Tip-Tracking with load test (#18896)
1 parent f4e7da2 commit 6ef59f3

File tree

1 file changed

+133
-74
lines changed

1 file changed

+133
-74
lines changed

‎.github/workflows/qa-tip-tracking-with-load.yml‎

Lines changed: 133 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -18,76 +18,125 @@ on:
1818
description: "Stress test load sequence (qps:duration_secs) e.g. 100:300,500:300,750:300 "
1919
required: false
2020
default: "100:300,500:300,1000:300,2000:300,1:300,3000:300,1:300"
21+
skip_pprof:
22+
description: "Skip downloading pprof data"
23+
type: boolean
24+
required: false
25+
default: false
26+
run_geth:
27+
description: 'Run the Geth benchmark besides the Erigon one'
28+
type: boolean
29+
required: false
30+
default: false
2131

2232
concurrency:
2333
group: ${{ github.workflow }}-${{ github.ref }}
2434
cancel-in-progress: false
2535

2636
jobs:
27-
mainnet-tip-tracking-with-load-test:
28-
runs-on: [self-hosted, qa, Ethereum, tip-tracking]
37+
tip-tracking-with-load-test:
38+
name: Tip-tracking with load ${{ matrix.client }}
39+
strategy:
40+
fail-fast: false
41+
matrix:
42+
include:
43+
- client: erigon
44+
runs-on: [ self-hosted, qa, Ethereum, tip-tracking ]
45+
- client: geth
46+
runs-on: [ self-hosted, qa, Ethereum, rpc-latest-geth ]
47+
runs-on: ${{ matrix.runs-on }}
2948
timeout-minutes: 300 # 5 hours
3049
env:
3150
ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version-main/datadir
3251
ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
3352
ERIGON_QA_PATH: /home/qarunner/erigon-qa
34-
TRACKING_TIME_SECONDS: 1800 # 0.5 hours
35-
TOTAL_TIME_SECONDS: 7200 # 2 hours
53+
TRACKING_TIME_SECONDS: 7200 # 2 hours
54+
TOTAL_TIME_SECONDS: 10800 # 3 hours
3655
CHAIN: mainnet
56+
GETH_INSTALL_DIR: /opt/go-ethereum
57+
CLIENT: ${{ matrix.client }}
3758
EXECUTION_NAME: ${{ github.event.inputs.execution_name || 'eth_getBlockByNumber' }}
3859
LOAD_PATTERN: ${{ github.event.inputs.load_pattern || 'stress_test_eth_getBlockByNumber_13M.tar' }}
3960
LOAD_SEQUENCE: ${{ github.event.inputs.load_sequence || '100:300,500:300,1000:300,2000:300,1:300,3000:300,1:300' }}
61+
LOAD_MODE: ${{ github.event.inputs.skip_pprof == 'true' && 'measure' || 'load' }}
4062

4163
steps:
4264
- name: Print job parameters
65+
if: matrix.client == 'erigon' || (matrix.client == 'geth' && github.event.inputs.run_geth == 'true')
4366
run: |
4467
echo "Execution name: ${{ env.EXECUTION_NAME }}"
4568
echo "Load pattern: ${{ env.LOAD_PATTERN }}"
4669
echo "Load sequence: ${{ env.LOAD_SEQUENCE }}"
4770
71+
- name: Clean runner workspace
72+
run: |
73+
# Remove all contents under the runner workspace directory
74+
rm -rf ${{ runner.workspace }}/* || true
75+
mkdir -p ${{ runner.workspace }}/erigon
76+
4877
- name: Check out Erigon repository
78+
if: matrix.client == 'erigon'
4979
uses: actions/checkout@v6
5080

5181
- name: Checkout RPC Tests Repository & Install Requirements
82+
if: matrix.client == 'erigon' || (matrix.client == 'geth' && github.event.inputs.run_geth == 'true')
5283
run: |
5384
rm -rf ${{runner.workspace}}/rpc-tests
5485
git -c advice.detachedHead=false clone --depth 1 --branch v1.78.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
5586
cd ${{runner.workspace}}/rpc-tests
5687
5788
- name: Clean Erigon Build Directory
89+
if: matrix.client == 'erigon'
5890
run: |
5991
make clean
6092
6193
- name: Build Erigon
94+
if: matrix.client == 'erigon'
6295
run: |
6396
make erigon
6497
working-directory: ${{ github.workspace }}
6598

6699
- name: Pause the Erigon instance dedicated to db maintenance
100+
if: matrix.client == 'erigon'
67101
run: |
68102
python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
69103
70-
- name: Run Erigon and wait for sync (stabilization step)
104+
- name: Save Erigon datadir Directory
105+
if: matrix.client == 'erigon'
106+
id: save_datadir_step
107+
run: |
108+
rm -rf $ERIGON_TESTBED_DATA_DIR || true
109+
110+
echo "Mirror datadir"
111+
if ! "$GITHUB_WORKSPACE/cmd/scripts/mirror-datadir.sh" "$ERIGON_REFERENCE_DATA_DIR" "$ERIGON_TESTBED_DATA_DIR" > /dev/null 2>&1; then
112+
echo "Failed to mirror datadir from $ERIGON_REFERENCE_DATA_DIR to $ERIGON_TESTBED_DATA_DIR"
113+
exit 1
114+
fi
115+
116+
cp $ERIGON_REFERENCE_DATA_DIR/../erigon $ERIGON_TESTBED_DATA_DIR/../
117+
118+
echo "datadir_saved=true" >> $GITHUB_OUTPUT
119+
120+
- name: Assure the client is in sync (stabilization step)
121+
if: matrix.client == 'erigon' || (matrix.client == 'geth' && github.event.inputs.run_geth == 'true')
71122
id: pre_test_step
72123
run: |
73124
set +e # Disable exit on error
74125
75-
# Launch the testbed Erigon instance & test its ability to maintain sync for 2 minutes
76-
python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
77-
"$ERIGON_REFERENCE_DATA_DIR/../" "$ERIGON_REFERENCE_DATA_DIR" 120 "$TOTAL_TIME_SECONDS" "Erigon3" "$CHAIN"
126+
# Launch the testbed client instance & test its ability to maintain sync for 2 minutes
127+
if [ "${{ matrix.client }}" == "erigon" ]; then
128+
python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
129+
"$ERIGON_TESTBED_DATA_DIR/../" "$ERIGON_TESTBED_DATA_DIR" 120 "$TOTAL_TIME_SECONDS" "Erigon3" "$CHAIN"
130+
else
131+
python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/check_tip_tracking.py \
132+
"${{ matrix.client }}" "http://localhost:8545" 120 "$TOTAL_TIME_SECONDS" "$CHAIN"
133+
fi
78134
79135
# Capture monitoring script exit status
80136
test_exit_status=$?
81137
82138
# Save the subsection reached status
83139
echo "test_executed=true" >> $GITHUB_OUTPUT
84-
85-
# Clean up Erigon process if it's still running
86-
if kill -0 $ERIGON_PID 2> /dev/null; then
87-
echo "Terminating Erigon"
88-
kill $ERIGON_PID
89-
wait $ERIGON_PID
90-
fi
91140
92141
# Check test runner script exit status
93142
if [ $test_exit_status -eq 0 ]; then
@@ -101,13 +150,8 @@ jobs:
101150
exit 1
102151
fi
103152
104-
- name: Restore Erigon Testbed Data Directory & Erigon binary
105-
id: save_chaindata_step
106-
run: |
107-
rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/
108-
cp $ERIGON_REFERENCE_DATA_DIR/../erigon $ERIGON_TESTBED_DATA_DIR/../
109-
110-
- name: Run Erigon, wait sync and check ability to maintain sync with load
153+
- name: Run the client, wait sync and check ability to maintain sync with load
154+
if: matrix.client == 'erigon' || (matrix.client == 'geth' && github.event.inputs.run_geth == 'true')
111155
id: test_step
112156
env:
113157
SAFE_EXECUTION_NAME: ${{ env.EXECUTION_NAME }}
@@ -117,43 +161,57 @@ jobs:
117161
run: |
118162
set +e # Disable exit on error
119163
120-
# Launch the testbed Erigon instance & test its ability to maintain sync with a load for TRACKING_TIME_SECONDS
121-
python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
122-
"${{ github.workspace }}/build/bin" \
123-
"$ERIGON_TESTBED_DATA_DIR" \
124-
"$TRACKING_TIME_SECONDS" \
125-
"$TOTAL_TIME_SECONDS" \
126-
"Erigon3" \
127-
"$CHAIN" \
128-
"archive_node" \
129-
"load@${SAFE_EXECUTION_NAME}@${SAFE_LOAD_PATTERN}@${SAFE_LOAD_SEQUENCE}@${RPC_TESTS_DIR}"
130-
164+
# Launch the testbed client instance & test its ability to maintain sync with a load for TRACKING_TIME_SECONDS
165+
if [ "${{ matrix.client }}" == "erigon" ]; then
166+
python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
167+
"${{ github.workspace }}/build/bin" \
168+
"$ERIGON_TESTBED_DATA_DIR" \
169+
"$TRACKING_TIME_SECONDS" \
170+
"$TOTAL_TIME_SECONDS" \
171+
"Erigon3" \
172+
"$CHAIN" \
173+
"archive_node" \
174+
"${{ env.LOAD_MODE }}@${SAFE_EXECUTION_NAME}@${SAFE_LOAD_PATTERN}@${SAFE_LOAD_SEQUENCE}@${RPC_TESTS_DIR}"
175+
else
176+
python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/check_tip_tracking.py \
177+
"${{ matrix.client }}" \
178+
"http://localhost:8545" \
179+
"$TRACKING_TIME_SECONDS" \
180+
"$TOTAL_TIME_SECONDS" \
181+
"$CHAIN" \
182+
"${{ env.LOAD_MODE }}@${SAFE_EXECUTION_NAME}@${SAFE_LOAD_PATTERN}@${SAFE_LOAD_SEQUENCE}@${RPC_TESTS_DIR}"
183+
fi
131184
# Capture monitoring script exit status
132185
test_exit_status=$?
133186
134187
# Save the subsection reached status
135188
echo "test_executed=true" >> $GITHUB_OUTPUT
136189
137-
# Clean up Erigon process if it's still running
138-
if kill -0 $ERIGON_PID 2> /dev/null; then
139-
echo "Terminating Erigon"
140-
kill $ERIGON_PID
141-
wait $ERIGON_PID
142-
fi
143-
144190
# Check test runner script exit status
145191
if [ $test_exit_status -eq 0 ]; then
146192
echo "Tip-tracking w/ load test completed successfully"
147-
echo "::notice::Tip-tracking w/ load test completed successfully"
193+
echo "::notice::Tip-tracking w/ load test ${{ matrix.client }} completed successfully"
148194
echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
195+
elif [ $test_exit_status -eq 1 ]; then
196+
# Ignore failures relating to unacceptable behaviour, as these are potentially tests designed to break the system.
197+
echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
198+
if reason=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/print_reason.py ${{ github.workspace }}/result-$CHAIN.json); then
199+
echo "Tip-tracking w/ load test completed with a warning: $reason"
200+
echo "::warning::Tip-tracking w/ load test ${{ matrix.client }} completed with a warning: $reason"
201+
else
202+
echo "Tip-tracking w/ load test completed with a warning"
203+
echo "::warning::Tip-tracking w/ load test ${{ matrix.client }} completed with a warning"
204+
fi
149205
else
150206
echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
151-
if reason=$(python3 print_reason.py ${{ github.workspace }}/result-$CHAIN.json); then
207+
if reason=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/print_reason.py ${{ github.workspace }}/result-$CHAIN.json); then
152208
echo "Error detected during tests: $reason"
153-
echo "::error::Tip-tracking w/ load test encountered an error: $reason"
209+
echo "::error::Tip-tracking w/ load test ${{ matrix.client }} encountered an error: $reason"
210+
exit 1
154211
else
155212
echo "Tip-tracking w/ load test encountered an error"
156-
echo "::error::Tip-tracking w/ load test encountered an error"
213+
echo "::error::Tip-tracking w/ load test ${{ matrix.client }} encountered an error"
214+
exit 1
157215
fi
158216
fi
159217
@@ -167,84 +225,85 @@ jobs:
167225
fi
168226
169227
- name: Upload Downloader Torrent Client Status
170-
if: always()
228+
if: always() && matrix.client == 'erigon'
171229
uses: actions/upload-artifact@v6
172230
with:
173231
name: torrent-client-status
174232
path: torrent-client-status.txt
175233

176234
- name: Save test results
177-
if: steps.test_step.outputs.test_executed == 'true'
235+
if: always() && steps.test_step.outputs.test_executed == 'true'
178236
env:
179237
TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }}
180238
run: |
181-
db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit)
182-
if [ -z "$db_version" ]; then
183-
db_version="no-version"
239+
# Detect the commit hash and branch name
240+
if [ "${{ matrix.client }}" == "erigon" ]; then
241+
commit=$(git -C ${{runner.workspace}}/erigon rev-parse --short HEAD)
242+
branch_name=${{ github.ref_name }}
243+
else
244+
branch_name="release"
245+
commit=$(git -C $GETH_INSTALL_DIR rev-parse --short HEAD)
246+
fi
247+
248+
# Detect the pre-built db version
249+
if [ "$client" == "erigon" ]; then
250+
db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit)
251+
else
252+
db_version=$commit
184253
fi
185254
186255
python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
187-
--repo erigon \
188-
--commit $(git rev-parse HEAD) \
189-
--branch ${{ github.ref_name }} \
190-
--test_name tip-tracking-with-load \
256+
--repo ${{ matrix.client }} \
257+
--commit $commit \
258+
--branch $branch_name \
259+
--test_name tip-tracking-with-load-${{ matrix.client }} \
191260
--chain $CHAIN \
192261
--runner ${{ runner.name }} \
193262
--db_version $db_version \
194263
--outcome $TEST_RESULT \
195264
--result_file ${{ github.workspace }}/result-$CHAIN.json
196265
197266
- name: Upload test results
198-
if: steps.test_step.outputs.test_executed == 'true'
267+
if: always() && steps.test_step.outputs.test_executed == 'true'
199268
uses: actions/upload-artifact@v6
200269
with:
201-
name: test-results
270+
name: test-results-${{ env.CHAIN }}-${{ matrix.client }}
202271
path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json
203272

204273
- name: Upload erigon logs
205-
if: steps.test_step.outputs.test_executed == 'true'
274+
if: always() && steps.test_step.outputs.test_executed == 'true' && matrix.client == 'erigon'
206275
uses: actions/upload-artifact@v6
207276
with:
208277
name: erigon-logs
209278
path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/
210279

211280
- name: Upload metric plots
212-
if: steps.test_step.outputs.test_executed == 'true'
281+
if: always() && steps.test_step.outputs.test_executed == 'true'
213282
uses: actions/upload-artifact@v6
214283
with:
215-
name: metric-plots
284+
name: metric-plots-${{ env.CHAIN }}-${{ matrix.client }}
216285
path: ${{ github.workspace }}/metrics-${{ env.CHAIN }}-plots*
217286

218287
- name: Upload profiling data
219-
if: steps.test_step.outputs.test_executed == 'true'
288+
if: always() && steps.test_step.outputs.test_executed == 'true'
220289
uses: actions/upload-artifact@v6
221290
with:
222-
name: profiling-data
291+
name: profiling-data-${{ env.CHAIN }}-${{ matrix.client }}
223292
path: ${{ github.workspace }}/profiling-data*
224293

225294
- name: Upload FD Leak Analysis
226-
if: always()
295+
if: always() && matrix.client == 'erigon'
227296
uses: actions/upload-artifact@v6
228297
with:
229298
name: fd-leak-analysis
230299
path: ${{ github.workspace }}/fd-leak-analysis-${{ env.CHAIN }}.md
231300

232301
- name: Delete Erigon Testbed Data Directory
233-
if: ${{ always() }}
302+
if: always() && matrix.client == 'erigon' && steps.save_datadir_step.outputs.datadir_saved == 'true'
234303
run: |
235-
if [ -d "$ERIGON_TESTBED_DATA_DIR" ]; then
236-
rm -rf $ERIGON_TESTBED_DATA_DIR
237-
fi
304+
rm -rf $ERIGON_TESTBED_DATA_DIR
238305
239306
- name: Resume the Erigon instance dedicated to db maintenance
240-
if: ${{ always() }}
307+
if: always() && matrix.client == 'erigon'
241308
run: |
242309
python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
243-
244-
- name: Action for Not Success
245-
if: ${{ always() }}
246-
run: |
247-
if [[ "${{ steps.test_step.outputs.TEST_RESULT }}" != "success" ]]; then
248-
echo "::error::Error detected during tests, please see the summary section for details"
249-
exit 1
250-
fi

0 commit comments

Comments
 (0)