Skip to content

Commit e4cadde

Browse files
forsyth2forsyth2
committed
Update Chrysalis tests
1 parent 3486b0a commit e4cadde

File tree

5 files changed

+123
-92
lines changed

5 files changed

+123
-92
lines changed

‎tests/README.md‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,15 @@ python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py
3333
# Ran 4 tests in 2.666s
3434
# OK
3535
cd tests/integration/bash_tests/run_from_perlmutter/
36-
time ./follow_symlinks.sh
36+
time ./follow_symlinks.sh # NOTE: you will have to change out paths for your username
3737
# real 0m31.851s
3838
# No errors
3939
time ./test_update_non_empty_hpss.bash
4040
# real 0m10.062s
4141
# No errors
42+
43+
# Log into globus.org
44+
# Log into endpoints (NERSC Perlmutter, Globus Tutorial Collection 1) at globus.org: File Manager > Add the endpoints in the "Collection" fields
4245
time ./test_ls_globus.bash
4346
# real 0m26.930s
4447
# No errors
@@ -63,7 +66,41 @@ python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py
6366
# Ran 4 tests in 6.889s
6467
# OK
6568
cd tests/integration/bash_tests/run_from_chrysalis/
66-
# TODO: Add Chrysalis-specific tests
69+
70+
# Log into globus.org
71+
# 1. Log into endpoints (LCRC Improv DTN, NERSC Perlmutter) at globus.org: File Manager > Add the endpoints in the "Collection" fields
72+
# 2. To start fresh, with no consents: https://auth.globus.org/v2/web/consents > Manage Your Consents > Globus Endpoint Performance Monitoring > rescind all"
73+
# Then, increment `try_num` below to avoid using an old directory.
74+
# Alternatively, start fresh by deleting the directory on Perlmutter:
75+
# `rm -rf /global/homes/f/forsyth/zstash/tests/test_globus_auth_try{try_num}`
76+
time ./globus_auth.bash try_num # NOTE: you will have to change out paths for your username
77+
# Paste the URL into your browser
78+
# Log into Argonne
79+
# Log into NERSC
80+
# Provide a label
81+
# Copy the auth code to the command line
82+
#
83+
# real 2m45.954s
84+
# No errors
85+
86+
# If not done above, do the following:
87+
# Log into globus.org
88+
# Log into endpoints (LCRC Improv DTN, NERSC Perlmutter) at globus.org: File Manager > Add the endpoints in the "Collection" fields
89+
90+
# In all cases, do:
91+
# Then, increment `try_num` below to avoid using an old directory.
92+
# Alternatively, start fresh by deleting the directory on Perlmutter:
93+
# `rm -rf /global/homes/f/forsyth/zstash/tests/test_database_corruption_try{try_num}`
94+
time ./database_corruption.bash try_num # NOTE: you will have to change out paths for your username
95+
# Success count: 25
96+
# Fail count: 0
97+
# real 6m43.994s
98+
99+
time ./symlinks.sh # NOTE: you will have to change out paths for your username
100+
# real 0m1.346s
101+
# No errors
102+
103+
# TODO: Handle blocking_test_scripts
67104
```
68105

69106
## Testing with GitHub Actions

‎tests/integration/bash_tests/run_from_chrysalis/database_corruption.bash‎

Lines changed: 5 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -19,34 +19,13 @@ setup()
1919

2020
run_test_cases()
2121
{
22-
# Before running the first time:
23-
# globus.org
24-
# Authenticate into LCRC Improv DTN, NERSC Perlmutter
25-
# Run toy problem:
26-
#
27-
# source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh
28-
# cd /lcrc/group/e3sm/ac.forsyth2/zstash_testing/test_20250729
29-
# mkdir zstash_demo; echo 'file0 stuff' > zstash_demo/file0.txt
30-
# Chrysalis: 15288284-7006-4041-ba1a-6b52501e49f1
31-
# Perlmutter: 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79
32-
# zstash create --hpss=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79/global/homes/f/forsyth/zstash/tests/manual_run zstash_demo
33-
# Will prompt for LCRC AND NERSC authentication, and then paste generated auth code one time.
3422

35-
# Before each run:
36-
# Perlmutter:
37-
# cd /global/homes/f/forsyth/zstash/tests/
38-
# rm -rf test_database_corruption
39-
#
40-
# Chrysalis:
41-
# cd ~/ez/zstash/
42-
# conda activate zstash-377-20250728
43-
# pre-commit run --all-files
44-
# python -m pip install .
45-
# cd tests/integration/workflows/run_from_chrysalis
46-
# ./database_corruption.bash
23+
local try_num=$1
4724

4825
SRC_DIR=/lcrc/group/e3sm/ac.forsyth2/zstash_testing/test_database_corruption # Chrysalis
49-
DST_DIR=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79/global/homes/f/forsyth/zstash/tests/test_database_corruption # Perlmutter
26+
DST_DIR=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79/global/homes/f/forsyth/zstash/tests/test_database_corruption_try${try_num} # Perlmutter
27+
# To start fresh with try_num=1, delete the above directory on Perlmutter before running. Example:
28+
# rm -rf /global/homes/f/forsyth/zstash/tests/test_database_corruption_try1
5029

5130
success_count=0
5231
fail_count=0
@@ -318,7 +297,7 @@ run_test_cases()
318297
echo "Review: ${review_str}"
319298
}
320299

321-
run_test_cases
300+
run_test_cases "$1"
322301

323302
# Success count: 25
324303
# Fail count: 0

‎tests/integration/bash_tests/run_from_chrysalis/globus_auth.bash‎

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ check_log_has()
2020
{
2121
local expected_grep="${1}"
2222
local log_file="${2}"
23-
grep "${expected_grep}" ${log_file}
23+
grep -q "${expected_grep}" ${log_file}
2424
if [ $? != 0 ]; then
2525
echo "Expected grep '${expected_grep}' not found in ${log_file}. Test failed."
2626
exit 2
@@ -40,30 +40,14 @@ check_log_does_not_have()
4040

4141
run_test_cases()
4242
{
43-
# This script requires user input and thus cannot be run automatically as part of a test suite.
44-
45-
# To start fresh with Globus:
46-
# 1. Log into endpoints (LCRC Improv DTN, NERSC Perlmutter) at globus.org: File Manager > Add the endpoints in the "Collection" fields
47-
# 2. To start fresh, with no consents: https://auth.globus.org/v2/web/consents > Manage Your Consents > Globus Endpoint Performance Monitoring > rescind all"
48-
49-
# Before each run:
50-
# Perlmutter:
51-
# cd /global/homes/f/forsyth/zstash/tests/
52-
# rm -rf test_globus_auth_try1 # Or just change $DST_DIR to a new directory
53-
#
54-
# Chrysalis:
55-
# cd ~/ez/zstash/
56-
# conda activate <env-name>
57-
# pre-commit run --all-files
58-
# python -m pip install .
59-
# cd tests/integration/workflows/run_from_chrysalis
60-
# ./globus_auth.bash
43+
local try_num=$1
6144

6245
PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79
6346

64-
TRY_NUM=8
6547
SRC_DIR=/lcrc/group/e3sm/ac.forsyth2/zstash_testing/test_globus_auth # Chrysalis
66-
DST_DIR=globus://${PERLMUTTER_ENDPOINT}/global/homes/f/forsyth/zstash/tests/test_globus_auth_try${TRY_NUM}
48+
DST_DIR=globus://${PERLMUTTER_ENDPOINT}/global/homes/f/forsyth/zstash/tests/test_globus_auth_try${try_num} # Perlmutter
49+
# To start fresh with try_num=1, delete the above directory on Perlmutter before running. Example:
50+
# rm -rf /global/homes/f/forsyth/zstash/tests/test_globus_auth_try1
6751

6852
GLOBUS_CFG=/home/ac.forsyth2/.globus-native-apps.cfg
6953
INI_PATH=/home/ac.forsyth2/.zstash.ini
@@ -134,4 +118,4 @@ run_test_cases()
134118
# Could also test -l and -v options, but the above code covers the important part.
135119
}
136120

137-
run_test_cases
121+
run_test_cases "$1"

‎tests/integration/bash_tests/run_from_chrysalis/symlinks.sh‎

Lines changed: 72 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,75 @@
11
# Test symlinks
22
# Adjusted from https://github.com/E3SM-Project/zstash/issues/341
33

4-
follow_symlinks=true
5-
6-
rm -rf workdir workdir2 workdir3
7-
mkdir workdir workdir2 workdir3
8-
cd workdir
9-
mkdir -p src/d1 src/d2
10-
touch src/d1/large_file.txt
11-
12-
# This creates a symlink in d2 that links to a file in d1
13-
# Notice absolute path is used for source
14-
ln -s /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir/src/d1/large_file.txt src/d2/large_file.txt
15-
16-
echo ""
17-
echo "ls -l src/d2"
18-
ls -l src/d2
19-
# symlink
20-
21-
echo ""
22-
if [[ "${follow_symlinks,,}" == "true" ]]; then
23-
echo "zstash create --hpss=none --follow-symlinks --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2"
24-
zstash create --hpss=none --follow-symlinks --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2
25-
else
26-
echo "zstash create --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2"
27-
zstash create --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2
28-
fi
29-
30-
echo ""
31-
echo "ls -l src/d2"
32-
ls -l src/d2
33-
# symlink (src is unaffected)
34-
35-
cd ../workdir3
36-
echo ""
37-
echo "zstash extract --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2"
38-
zstash extract --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2
39-
40-
cd ..
41-
echo ""
42-
echo "ls workdir3"
43-
ls workdir3
44-
# large_file.txt
4+
check_log_has()
5+
{
6+
local expected_grep="${1}"
7+
local log_file="${2}"
8+
grep -q "${expected_grep}" ${log_file}
9+
if [ $? != 0 ]; then
10+
echo "Expected grep '${expected_grep}' not found in ${log_file}. Test failed."
11+
exit 2
12+
fi
13+
}
14+
15+
test_cases()
16+
{
17+
local follow_symlinks=$1
18+
cd /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks
19+
rm -rf workdir workdir2 workdir3
20+
mkdir workdir workdir2 workdir3
21+
cd workdir
22+
mkdir -p src/d1 src/d2
23+
touch src/d1/large_file.txt
24+
25+
# This creates a symlink in d2 that links to a file in d1
26+
# Notice absolute path is used for source
27+
ln -s /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir/src/d1/large_file.txt src/d2/large_file.txt
28+
29+
echo ""
30+
echo "ls -l src/d2"
31+
case_name="ls_1"
32+
ls -l src/d2 2>&1 | tee ${case_name}.log
33+
# symlink
34+
check_log_has "large_file.txt -> /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir/src/d1/large_file.txt" ${case_name}.log
35+
36+
echo ""
37+
case_name="create"
38+
if [[ "${follow_symlinks,,}" == "true" ]]; then
39+
echo "zstash create --hpss=none --follow-symlinks --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2"
40+
zstash create --hpss=none --follow-symlinks --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2 2>&1 | tee ${case_name}.log
41+
else
42+
echo "zstash create --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2"
43+
zstash create --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2 2>&1 | tee ${case_name}.log
44+
fi
45+
check_log_has "Adding 000000.tar" ${case_name}.log
46+
47+
echo ""
48+
echo "ls -l src/d2"
49+
case_name="ls_2"
50+
ls -l src/d2 2>&1 | tee ${case_name}.log
51+
# symlink (src is unaffected)
52+
check_log_has "large_file.txt -> /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir/src/d1/large_file.txt" ${case_name}.log
53+
54+
cd ../workdir3
55+
echo ""
56+
echo "zstash extract --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2"
57+
case_name="extract"
58+
zstash extract --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 2>&1 | tee ${case_name}.log
59+
check_log_has "No failures detected when extracting the files" ${case_name}.log
60+
61+
cd ..
62+
echo ""
63+
echo "ls workdir3"
64+
case_name="ls_3"
65+
ls workdir3 2>&1 | tee ${case_name}.log
66+
# large_file.txt
67+
check_log_has "large_file.txt" ${case_name}.log
68+
69+
cd /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks
70+
rm -rf workdir workdir2 workdir3 ls_3.log
71+
72+
}
73+
74+
test_cases true
75+
test_cases false

‎tests/integration/bash_tests/run_from_perlmutter/test_update_non_empty_hpss.bash‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ check_log_has()
77
{
88
local expected_grep="${1}"
99
local log_file="${2}"
10-
grep "${expected_grep}" ${log_file}
10+
grep -q "${expected_grep}" ${log_file}
1111
if [ $? != 0 ]; then
1212
echo "Expected grep '${expected_grep}' not found in ${log_file}. Test failed."
1313
exit 1

0 commit comments

Comments
 (0)