diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index d3574d77..d0aa857a 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -82,9 +82,12 @@ jobs: run: | python -m pip install . + # Run machine-independent tests - name: Run Tests run: | - python -m unittest tests/test_*.py + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py # Build documentation on all PRs and pushes; only publish (commit to gh-pages) on pushes. build-docs: diff --git a/.gitignore b/.gitignore index f0c2eeda..f20af3db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ build/ dist/ -tests/test_follow_symlinks/ -tests/test_follow_symlinks_non_archived/ +tests/utils/test_follow_symlinks +tests/utils/test_follow_symlinks_non_archived zstash.egg-info/ *.pyc *~ diff --git a/conda/dev.yml b/conda/dev.yml index 216b47cf..6ec8e1b9 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -8,7 +8,7 @@ dependencies: - python >=3.11,<3.14 - sqlite - six >=1.16.0 - - globus-sdk >=3.15.0 + - globus-sdk >=3.15.0,<4.0 # Developer Tools # ================= # If versions are updated, also update 'rev' in `.pre-commit.config.yaml` @@ -18,6 +18,10 @@ dependencies: - mypy ==1.18.2 - pre-commit ==4.3.0 - tbump >=6.9.0 + # Testing + # ======================= + - pytest + - pytest-cov # Documentation # ================= # If versions are updated, also update in `.github/workflows/workflow.yml` diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..d63e7851 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,120 @@ +# zstash Test Suite + +## The test directory structure + +``` +tests/ + integration/ # Tests that call zstash from the command line + bash_tests/ # Test zstash commands via bash scripts + run_from_chrysalis/ # Run these from Chrysalis (these use Globus and/or require the Chrysalis file system) + run_from_perlmutter/ # Run these from Perlmutter (these use `hsi` directly and/or require the Perlmutter file system) + python_tests/ # Test zstash commands via Python unittest wrappers + group_by_command # Tests organized by command + group_by_workflow # Tests organized by workflow + unit/ # Tests of pure functions (uses pytest, not unittest) + utils/ # Utilities for testing +``` + +## Testing example for Perlmutter + +```bash +rm -rf build +conda clean --all --y +conda env create -f conda/dev.yml -n zstash_dev_20251017_test1 +conda activate zstash_dev_20251017_test1 +pre-commit run --all-files +python -m pip install . +pytest tests/unit/test_*.py +# 1 passed in 0.19s +python -m unittest tests/integration/python_tests/group_by_command/test_*.py +# Ran 69 tests in 327.570s +# OK +python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py +# Ran 4 tests in 2.666s +# OK +cd tests/integration/bash_tests/run_from_perlmutter/ +time ./follow_symlinks.sh # NOTE: you will have to change out paths for your username +# real 0m31.851s +# No errors +time ./test_update_non_empty_hpss.bash +# real 0m10.062s +# No errors + +# Log into globus.org +# Log into endpoints (NERSC Perlmutter, Globus Tutorial Collection 1) at globus.org: File Manager > Add the endpoints in the "Collection" fields +time ./test_ls_globus.bash +# real 0m26.930s +# No errors +``` + +## Testing example for Chrysalis + +```bash +rm -rf build +conda clean --all --y +conda env create -f conda/dev.yml -n zstash_dev_20251017_test1 +conda activate zstash_dev_20251017_test1 +pre-commit run --all-files +python -m pip install . +pytest tests/unit/test_*.py +# 1 passed in 0.84s +python -m unittest tests/integration/python_tests/group_by_command/test_*.py +# Ran 69 tests in 110.139s +# OK (skipped=32) +# NOTE: Some tests are skipped because Chrysalis doesn't have direct `hsi`/HPSS access +python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py +# Ran 4 tests in 6.889s +# OK +cd tests/integration/bash_tests/run_from_chrysalis/ + +# Log into globus.org +# 1. Log into endpoints (LCRC Improv DTN, NERSC Perlmutter) at globus.org: File Manager > Add the endpoints in the "Collection" fields +# 2. To start fresh, with no consents: https://auth.globus.org/v2/web/consents > Manage Your Consents > Globus Endpoint Performance Monitoring > rescind all" +# Then, increment `try_num` below to avoid using an old directory. +# Alternatively, start fresh by deleting the directory on Perlmutter: +# `rm -rf /global/homes/f/forsyth/zstash/tests/test_globus_auth_try{try_num}` +time ./globus_auth.bash try_num # NOTE: you will have to change out paths for your username +# Paste the URL into your browser +# Log into Argonne +# Log into NERSC +# Provide a label +# Copy the auth code to the command line +# +# real 2m45.954s +# No errors + +# If not done above, do the following: +# Log into globus.org +# Log into endpoints (LCRC Improv DTN, NERSC Perlmutter) at globus.org: File Manager > Add the endpoints in the "Collection" fields + +# In all cases, do: +# Then, increment `try_num` below to avoid using an old directory. +# Alternatively, start fresh by deleting the directory on Perlmutter: +# `rm -rf /global/homes/f/forsyth/zstash/tests/test_database_corruption_try{try_num}` +time ./database_corruption.bash try_num # NOTE: you will have to change out paths for your username +# Success count: 25 +# Fail count: 0 +# real 6m43.994s + +time ./symlinks.sh # NOTE: you will have to change out paths for your username +# real 0m1.346s +# No errors + +cd blocking_test_scripts +# Review README_TEST_BLOCKING +# This uses "12 piControl ocean monthly files, 49 GB", +# so processing may take a long time. +# TODO (later PR): Confirm this test works +``` + +## Testing with GitHub Actions + +GitHub Actions runs the tests according to `.github/workflows/build_workflow.yml`: +``` + # Run machine-independent tests + - name: Run Tests + run: | + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py +``` diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests3/README_TEST_BLOCKING b/tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/README_TEST_BLOCKING similarity index 83% rename from tests3/README_TEST_BLOCKING rename to tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/README_TEST_BLOCKING index e367335b..cb17bf4a 100644 --- a/tests3/README_TEST_BLOCKING +++ b/tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/README_TEST_BLOCKING @@ -67,15 +67,7 @@ II. Running the Test Script The test script "test_zstash_blocking.sh" accepts two positional parameters: - test_zstash_blocking.sh (BLOCKING|NON_BLOCKING) [NEW_CREDS] - -On an initial run, or whenever Globus complains of authentication failures, -add "NEW_CREDS" as the second parameter. This will act to delete your -cached Globus credentials and trigger prompts for you to paste login URLs -to your browser (generally one per endpoint) which requires that you conduct -a login sequence, and then paste a returned key-value at the bash command -prompt. After both keys are accepted, you can re-run the test script -without "NEW_CREDS", until the credentials expire (usually 24 hours.) + test_zstash_blocking.sh (BLOCKING|NON_BLOCKING) If "BLOCKING" is selected, zstash will run in default mode, waiting for each tar file to complete transfer before generating another tar file. @@ -93,7 +85,7 @@ so that your command prompt returns and you can monitor progress with snapshot.sh which will provide a view of both the tarfile cache and the destination -directory for delivred tar files. It is also suggested that you name your +directory for delivered tar files. It is also suggested that you name your logfile to reflect the date, and whether BLOCKING or not was specified. diff --git a/tests3/gen_data.sh b/tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/gen_data.sh similarity index 100% rename from tests3/gen_data.sh rename to tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/gen_data.sh diff --git a/tests3/gen_data_runner.sh b/tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/gen_data_runner.sh similarity index 100% rename from tests3/gen_data_runner.sh rename to tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/gen_data_runner.sh diff --git a/tests3/reset_test.sh b/tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/reset_test.sh similarity index 100% rename from tests3/reset_test.sh rename to tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/reset_test.sh diff --git a/tests3/snapshot.sh b/tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/snapshot.sh similarity index 100% rename from tests3/snapshot.sh rename to tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/snapshot.sh diff --git a/tests3/test_zstash_blocking.sh b/tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/test_zstash_blocking.sh similarity index 80% rename from tests3/test_zstash_blocking.sh rename to tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/test_zstash_blocking.sh index bae334a8..38af63d4 100755 --- a/tests3/test_zstash_blocking.sh +++ b/tests/integration/bash_tests/run_from_chrysalis/blocking_test_scripts/test_zstash_blocking.sh @@ -1,11 +1,9 @@ #!/bin/bash if [[ $# -lt 1 ]]; then - echo "Usage: text_zstash_blocking.sh (BLOCKING|NON_BLOCKING) [NEW_CREDS]" + echo "Usage: text_zstash_blocking.sh (BLOCKING|NON_BLOCKING)" echo " One of \"BLOCKING\" or \"NON_BLOCKING\" must be supplied as the" echo " first parameter." - echo " Add \"NEW_CREDS\" if Globus credentials have expired." - echo " This will cause Globus to prompt for new credentials." exit 0 fi @@ -20,14 +18,6 @@ else exit 0 fi -# remove old auth data, if exists, so that globus will prompt us -# for new auth credentials in case they have expired: -if [[ $# -gt 1 ]]; then - if [[ $2 == "NEW_CREDS" ]]: then - rm -f ~/.globus-native-apps.cfg - fi -fi - base_dir=`pwd` base_dir=`realpath $base_dir` diff --git a/tests/scripts/database_corruption.bash b/tests/integration/bash_tests/run_from_chrysalis/database_corruption.bash similarity index 91% rename from tests/scripts/database_corruption.bash rename to tests/integration/bash_tests/run_from_chrysalis/database_corruption.bash index b65ab1b1..e984c18f 100755 --- a/tests/scripts/database_corruption.bash +++ b/tests/integration/bash_tests/run_from_chrysalis/database_corruption.bash @@ -19,34 +19,13 @@ setup() run_test_cases() { - # Before running the first time: - # globus.org - # Authenticate into LCRC Improv DTN, NERSC Perlmutter - # Run toy problem: - # - # source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh - # cd /lcrc/group/e3sm/ac.forsyth2/zstash_testing/test_20250729 - # mkdir zstash_demo; echo 'file0 stuff' > zstash_demo/file0.txt - # Chrysalis: 15288284-7006-4041-ba1a-6b52501e49f1 - # Perlmutter: 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 - # zstash create --hpss=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79/global/homes/f/forsyth/zstash/tests/manual_run zstash_demo - # Will prompt for LCRC AND NERSC authentication, and then paste generated auth code one time. - # Before each run: - # Perlmutter: - # cd /global/homes/f/forsyth/zstash/tests/ - # rm -rf test_database_corruption - # - # Chrysalis: - # cd ~/ez/zstash/ - # conda activate zstash-377-20250728 - # pre-commit run --all-files - # python -m pip install . - # cd tests/scripts - # ./database_corruption.bash + local try_num=$1 SRC_DIR=/lcrc/group/e3sm/ac.forsyth2/zstash_testing/test_database_corruption # Chrysalis - DST_DIR=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79/global/homes/f/forsyth/zstash/tests/test_database_corruption # Perlmutter + DST_DIR=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79/global/homes/f/forsyth/zstash/tests/test_database_corruption_try${try_num} # Perlmutter + # To start fresh with try_num=1, delete the above directory on Perlmutter before running. Example: + # rm -rf /global/homes/f/forsyth/zstash/tests/test_database_corruption_try1 success_count=0 fail_count=0 @@ -318,7 +297,7 @@ run_test_cases() echo "Review: ${review_str}" } -run_test_cases +run_test_cases "$1" # Success count: 25 # Fail count: 0 diff --git a/tests/scripts/globus_auth.bash b/tests/integration/bash_tests/run_from_chrysalis/globus_auth.bash similarity index 84% rename from tests/scripts/globus_auth.bash rename to tests/integration/bash_tests/run_from_chrysalis/globus_auth.bash index 0396dabc..a3905bdb 100755 --- a/tests/scripts/globus_auth.bash +++ b/tests/integration/bash_tests/run_from_chrysalis/globus_auth.bash @@ -20,7 +20,7 @@ check_log_has() { local expected_grep="${1}" local log_file="${2}" - grep "${expected_grep}" ${log_file} + grep -q "${expected_grep}" ${log_file} if [ $? != 0 ]; then echo "Expected grep '${expected_grep}' not found in ${log_file}. Test failed." exit 2 @@ -40,30 +40,14 @@ check_log_does_not_have() run_test_cases() { - # This script requires user input and thus cannot be run automatically as part of a test suite. - - # To start fresh with Globus: - # 1. Log into endpoints (LCRC Improv DTN, NERSC Perlmutter) at globus.org: File Manager > Add the endpoints in the "Collection" fields - # 2. To start fresh, with no consents: https://auth.globus.org/v2/web/consents > Manage Your Consents > Globus Endpoint Performance Monitoring > rescind all" - - # Before each run: - # Perlmutter: - # cd /global/homes/f/forsyth/zstash/tests/ - # rm -rf test_globus_auth_try1 # Or just change $DST_DIR to a new directory - # - # Chrysalis: - # cd ~/ez/zstash/ - # conda activate - # pre-commit run --all-files - # python -m pip install . - # cd tests/scripts - # ./globus_auth.bash + local try_num=$1 PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 - TRY_NUM=8 SRC_DIR=/lcrc/group/e3sm/ac.forsyth2/zstash_testing/test_globus_auth # Chrysalis - DST_DIR=globus://${PERLMUTTER_ENDPOINT}/global/homes/f/forsyth/zstash/tests/test_globus_auth_try${TRY_NUM} + DST_DIR=globus://${PERLMUTTER_ENDPOINT}/global/homes/f/forsyth/zstash/tests/test_globus_auth_try${try_num} # Perlmutter + # To start fresh with try_num=1, delete the above directory on Perlmutter before running. Example: + # rm -rf /global/homes/f/forsyth/zstash/tests/test_globus_auth_try1 GLOBUS_CFG=/home/ac.forsyth2/.globus-native-apps.cfg INI_PATH=/home/ac.forsyth2/.zstash.ini @@ -134,4 +118,4 @@ run_test_cases() # Could also test -l and -v options, but the above code covers the important part. } -run_test_cases +run_test_cases "$1" diff --git a/tests/integration/bash_tests/run_from_chrysalis/symlinks.sh b/tests/integration/bash_tests/run_from_chrysalis/symlinks.sh new file mode 100755 index 00000000..1479f180 --- /dev/null +++ b/tests/integration/bash_tests/run_from_chrysalis/symlinks.sh @@ -0,0 +1,75 @@ +# Test symlinks +# Adjusted from https://github.com/E3SM-Project/zstash/issues/341 + +check_log_has() +{ + local expected_grep="${1}" + local log_file="${2}" + grep -q "${expected_grep}" ${log_file} + if [ $? != 0 ]; then + echo "Expected grep '${expected_grep}' not found in ${log_file}. Test failed." + exit 2 + fi +} + +test_cases() +{ + local follow_symlinks=$1 + cd /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks + rm -rf workdir workdir2 workdir3 + mkdir workdir workdir2 workdir3 + cd workdir + mkdir -p src/d1 src/d2 + touch src/d1/large_file.txt + + # This creates a symlink in d2 that links to a file in d1 + # Notice absolute path is used for source + ln -s /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir/src/d1/large_file.txt src/d2/large_file.txt + + echo "" + echo "ls -l src/d2" + case_name="ls_1" + ls -l src/d2 2>&1 | tee ${case_name}.log + # symlink + check_log_has "large_file.txt -> /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir/src/d1/large_file.txt" ${case_name}.log + + echo "" + case_name="create" + if [[ "${follow_symlinks,,}" == "true" ]]; then + echo "zstash create --hpss=none --follow-symlinks --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2" + zstash create --hpss=none --follow-symlinks --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2 2>&1 | tee ${case_name}.log + else + echo "zstash create --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2" + zstash create --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 src/d2 2>&1 | tee ${case_name}.log + fi + check_log_has "Adding 000000.tar" ${case_name}.log + + echo "" + echo "ls -l src/d2" + case_name="ls_2" + ls -l src/d2 2>&1 | tee ${case_name}.log + # symlink (src is unaffected) + check_log_has "large_file.txt -> /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir/src/d1/large_file.txt" ${case_name}.log + + cd ../workdir3 + echo "" + echo "zstash extract --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2" + case_name="extract" + zstash extract --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks/workdir2 2>&1 | tee ${case_name}.log + check_log_has "No failures detected when extracting the files" ${case_name}.log + + cd .. + echo "" + echo "ls workdir3" + case_name="ls_3" + ls workdir3 2>&1 | tee ${case_name}.log + # large_file.txt + check_log_has "large_file.txt" ${case_name}.log + + cd /home/ac.forsyth2/ez/zstash/tests/utils/test_symlinks + rm -rf workdir workdir2 workdir3 ls_3.log + +} + +test_cases true +test_cases false diff --git a/tests/integration/bash_tests/run_from_perlmutter/follow_symlinks.sh b/tests/integration/bash_tests/run_from_perlmutter/follow_symlinks.sh new file mode 100755 index 00000000..7e035b2e --- /dev/null +++ b/tests/integration/bash_tests/run_from_perlmutter/follow_symlinks.sh @@ -0,0 +1,295 @@ +#!/bin/bash + +# From Claude: +check_log_has() +{ + local log_file="${@: -1}" # Last argument is the log file + local patterns=("${@:1:$#-1}") # All but last argument are patterns + + for pattern in "${patterns[@]}"; do + if ! grep -q "${pattern}" "${log_file}"; then + echo "Expected grep '${pattern}' not found in ${log_file}. Test failed." + exit 1 + fi + done +} + +setup() +{ + echo "##########################################################################################################" + use_hpss=$1 + follow_symlinks=$2 + case_name="${3}" + archive_name=$4 + if [[ "${use_hpss}" == "true" ]]; then + hsi rm -R ${archive_name} + fi + echo "use_hpss=${use_hpss}" + echo "follow_symlinks=${follow_symlinks}" + echo "case_name=${case_name}" + local_archive_name=test_follow_symlinks + non_archived_dir=${local_archive_name}_non_archived + test_dir=/global/homes/f/forsyth/ez/zstash/tests/utils/ + cd ${test_dir} + rm -rf ${local_archive_name} + rm -rf ${non_archived_dir} + mkdir ${local_archive_name} + # At the same level as local_archive_name + mkdir ${non_archived_dir} + cd ${local_archive_name} + + mkdir zstash_demo + mkdir zstash_demo/empty_dir + mkdir zstash_demo/dir + mkdir non_archived + echo -n '' > zstash_demo/file_empty.txt + echo 'file0 stuff' > zstash_demo/dir/file0.txt + echo 'file1 stuff' > non_archived/file1.txt + echo 'file2 stuff' > ../${non_archived_dir}/file2.txt + # NOTE: `ln -s` appears to require absolute paths for the source files + ln -s ${test_dir}/${local_archive_name}/non_archived/file1.txt zstash_demo/file3.txt + check_log_has "file1 stuff" zstash_demo/file3.txt + ln -s ${test_dir}/${non_archived_dir}/file2.txt zstash_demo/file4.txt + check_log_has "file2 stuff" zstash_demo/file4.txt +} + +zstash_create() +{ + archive_name=$1 + follow_symlinks=$2 + echo "Starting zstash create from:" + pwd + if [[ "${follow_symlinks}" == "true" ]]; then + zstash create --hpss=${archive_name} zstash_demo --follow-symlinks + else + zstash create --hpss=${archive_name} zstash_demo + fi +} + +zstash_extract() +{ + archive_name=$1 + rm -rf zstash_extraction + mkdir zstash_extraction + cd zstash_extraction + if [[ "${archive_name}" == "none" ]]; then + echo "Copying zstash" + cp -r ../zstash_demo/zstash/ zstash + fi + echo "Starting zstash extract from:" + pwd + zstash extract --hpss=${archive_name} + echo "> ls" + ls 2>&1 | tee out_ls.txt + echo "> ls -l" + ls -l 2>&1 | tee out_ls_l.txt + echo "> zstash ls" + zstash ls --hpss=${archive_name} 2>&1 | tee out_zstash_ls.txt + echo "> zstash ls -l" + zstash ls -l --hpss=${archive_name} 2>&1 | tee out_zstash_ls_l.txt + cd .. +} + +test_cases() +{ + test_num=$1 + use_hpss=$2 + follow_symlinks=$3 + if [[ "${use_hpss}" == "true" ]]; then + archive_name=/home/f/forsyth/zstash_test_follow_symlinks + else + archive_name=none + fi + + echo "##########################################################################################################" + echo "Test ${test_num}: use_hpss=${use_hpss}, follow_symlinks=${follow_symlinks}" + case_name="Case ${test_num}.1: Don't delete original file" + setup ${use_hpss} ${follow_symlinks} "${case_name}" ${archive_name} + zstash_create ${archive_name} ${follow_symlinks} 2>&1 | tee case_${test_num}.1_create.txt + check_log_has \ + "Archiving file3.txt" \ + "Archiving file4.txt" \ + "Archiving file_empty.txt" \ + "Archiving dir/file0.txt" \ + "Archiving empty_dir" \ + "Completed archive file 000000.tar" \ + case_${test_num}.1_create.txt + zstash_extract ${archive_name} 2>&1 | tee case_${test_num}.1_extract.txt + check_log_has \ + "Extracting file3.txt" \ + "Extracting file4.txt" \ + "Extracting file_empty.txt" \ + "Extracting dir/file0.txt" \ + "Extracting empty_dir" \ + "No failures detected when extracting the files." \ + case_${test_num}.1_extract.txt + check_log_has "file1 stuff" zstash_extraction/file3.txt + check_log_has "file2 stuff" zstash_extraction/file4.txt + check_log_has \ + "dir" \ + "empty_dir" \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "zstash" \ + zstash_extraction/out_ls.txt + check_log_has \ + "dir" \ + "empty_dir" \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "zstash" \ + zstash_extraction/out_ls_l.txt + check_log_has \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "dir/file0.txt" \ + "empty_dir" \ + zstash_extraction/out_zstash_ls.txt + check_log_has \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "dir/file0.txt" \ + "empty_dir" \ + "000000.tar" \ + zstash_extraction/out_zstash_ls_l.txt + + case_name="Case ${test_num}.2: Delete before create" + setup ${use_hpss} ${follow_symlinks} "${case_name}" ${archive_name} + rm non_archived/file1.txt # Remove the file that file3 links to + zstash_create ${archive_name} ${follow_symlinks} 2>&1 | tee case_${test_num}.2_create.txt + if [ "${follow_symlinks}" = "true" ]; then + check_log_has \ + "Archiving file3.txt" \ + "FileNotFoundError" \ + "ERROR: Archiving file3.txt" \ + "Exception: Archive creation failed due to broken symlink." \ + case_${test_num}.2_create.txt + else + check_log_has \ + "Archiving file3.txt" \ + "Archiving file4.txt" \ + "Archiving file_empty.txt" \ + "Archiving dir/file0.txt" \ + "Archiving empty_dir" \ + "Completed archive file 000000.tar" \ + case_${test_num}.2_create.txt + zstash_extract ${archive_name} 2>&1 | tee case_${test_num}.2_extract.txt + check_log_has \ + "Extracting file3.txt" \ + "Extracting file4.txt" \ + "Extracting file_empty.txt" \ + "Extracting dir/file0.txt" \ + "Extracting empty_dir" \ + "No failures detected when extracting the files." \ + case_${test_num}.2_extract.txt + if [ -f "zstash_extraction/file3.txt" ]; then + echo "zstash_extraction/file3.txt exists, but it should not because the file it links to was deleted." + exit 2 + fi + check_log_has "file2 stuff" zstash_extraction/file4.txt + check_log_has \ + "dir" \ + "empty_dir" \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "zstash" \ + zstash_extraction/out_ls.txt + check_log_has \ + "dir" \ + "empty_dir" \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "zstash" \ + zstash_extraction/out_ls_l.txt + check_log_has \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "dir/file0.txt" \ + "empty_dir" \ + zstash_extraction/out_zstash_ls.txt + check_log_has \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "dir/file0.txt" \ + "empty_dir" \ + "000000.tar" \ + zstash_extraction/out_zstash_ls_l.txt + fi + + case_name="Case ${test_num}.3: Delete after create" + setup ${use_hpss} ${follow_symlinks} "${case_name}" ${archive_name} + zstash_create ${archive_name} ${follow_symlinks} 2>&1 | tee case_${test_num}.3_create.txt + check_log_has \ + "Archiving file3.txt" \ + "Archiving file4.txt" \ + "Archiving file_empty.txt" \ + "Archiving dir/file0.txt" \ + "Archiving empty_dir" \ + "Completed archive file 000000.tar" \ + case_${test_num}.3_create.txt + rm non_archived/file1.txt + zstash_extract ${archive_name} 2>&1 | tee case_${test_num}.3_extract.txt + check_log_has \ + "Extracting file3.txt" \ + "Extracting file4.txt" \ + "Extracting file_empty.txt" \ + "Extracting dir/file0.txt" \ + "Extracting empty_dir" \ + "No failures detected when extracting the files." \ + case_${test_num}.3_extract.txt + if [ "${follow_symlinks}" = "true" ]; then + check_log_has "file1 stuff" zstash_extraction/file3.txt + else + if [ -f "zstash_extraction/file3.txt" ]; then + echo "zstash_extraction/file3.txt exists, but it should not because the file it links to was deleted." + exit 3 + fi + fi + check_log_has "file2 stuff" zstash_extraction/file4.txt + check_log_has \ + "dir" \ + "empty_dir" \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "zstash" \ + zstash_extraction/out_ls.txt + check_log_has \ + "dir" \ + "empty_dir" \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "zstash" \ + zstash_extraction/out_ls_l.txt + check_log_has \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "dir/file0.txt" \ + "empty_dir" \ + zstash_extraction/out_zstash_ls.txt + check_log_has \ + "file3.txt" \ + "file4.txt" \ + "file_empty.txt" \ + "dir/file0.txt" \ + "empty_dir" \ + "000000.tar" \ + zstash_extraction/out_zstash_ls_l.txt + +} + +# Begin tests +test_cases 1 true true # HPSS, follow symlinks +test_cases 2 false true # No HPSS, follow symlinks +test_cases 3 true false # HPSS, don't follow symlinks +test_cases 4 false false # No HPSS, don't follow symlinks diff --git a/tests/scripts_unit_tests/test_ls_globus.bash b/tests/integration/bash_tests/run_from_perlmutter/test_ls_globus.bash similarity index 51% rename from tests/scripts_unit_tests/test_ls_globus.bash rename to tests/integration/bash_tests/run_from_perlmutter/test_ls_globus.bash index b1677a80..2315a2a2 100755 --- a/tests/scripts_unit_tests/test_ls_globus.bash +++ b/tests/integration/bash_tests/run_from_perlmutter/test_ls_globus.bash @@ -4,6 +4,20 @@ hpss_globus_endpoint="6c54cade-bde5-45c1-bdea-f4bd71dba2cc" hpss_path="globus://${hpss_globus_endpoint}/~/zstash_test/" cache=zstash # Set via `self.cache = "zstash"` +# From Claude: +check_log_has() +{ + local log_file="${@: -1}" # Last argument is the log file + local patterns=("${@:1:$#-1}") # All but last argument are patterns + + for pattern in "${patterns[@]}"; do + if ! grep -q "${pattern}" "${log_file}"; then + echo "Expected grep '${pattern}' not found in ${log_file}. Test failed." + exit 1 + fi + done +} + # base.setupDirs ############################################################## test_dir=zstash_test @@ -29,18 +43,36 @@ ln -s ${test_dir}/file0.txt ${test_dir}/file0_hard.txt # base.create ################################################################# echo "Adding files to HPSS" -zstash create --hpss=${hpss_path} ${test_dir} -# Archives 000000.tar +case_name="create" +zstash create --hpss=${hpss_path} ${test_dir} 2>&1 | tee ${case_name}.log +check_log_has "INFO: Adding 000000.tar" ${case_name}.log echo "Cache:" -ls -l ${test_dir}/${cache} # just index.db -echo "HPSS:" -hsi ls -l ${hpss_path} # 000000.tar, index.db +ls -l ${test_dir}/${cache} 2>&1 | tee ${case_name}_cache.log +check_log_has "index.db" ${case_name}_cache.log +# NOTE: +# It appears that we can't access the HPSS path directly with `hsi`. +# That's presumably because we're using the Globus tutorial endpoint +# rather than the NERSC HPSS endpoint. # back in test_globus.helperLsGlobus ########################################## cd ${test_dir} -zstash ls --hpss=${hpss_path} +case_name="ls" +zstash ls --hpss=${hpss_path} 2>&1 | tee ${case_name}.log +check_log_has \ + "file0.txt" \ + "file0_hard.txt" \ + "file0_soft_bad.txt" \ + "file_0_soft.txt" \ + "file_empty.txt" \ + "dir/file1.txt" \ + "empty_dir" \ + ${case_name}.log cd .. echo "Cache:" -ls -l ${test_dir}/${cache} -echo "HPSS:" -hsi ls -l ${hpss_path} +ls -l ${test_dir}/${cache} 2>&1 | tee ${case_name}_cache.log +check_log_has "index.db" ${case_name}_cache.log + +rm -rf ${test_dir} # Cleanup +rm create.log +rm create_cache.log +rm ls_cache.log diff --git a/tests/integration/bash_tests/run_from_perlmutter/test_update_non_empty_hpss.bash b/tests/integration/bash_tests/run_from_perlmutter/test_update_non_empty_hpss.bash new file mode 100755 index 00000000..f4f819c1 --- /dev/null +++ b/tests/integration/bash_tests/run_from_perlmutter/test_update_non_empty_hpss.bash @@ -0,0 +1,138 @@ +#!/bin/bash + +hpss_path=zstash_test # Set via `HPSS_ARCHIVE = "zstash_test"` +cache=zstash # Set via `self.cache = "zstash"` + +check_log_has() +{ + local expected_grep="${1}" + local log_file="${2}" + grep -q "${expected_grep}" ${log_file} + if [ $? != 0 ]; then + echo "Expected grep '${expected_grep}' not found in ${log_file}. Test failed." + exit 1 + fi +} + +# base.setupDirs ############################################################## +test_dir=zstash_test + +# Create files and directories +echo "Creating files." +mkdir -p ${test_dir} +mkdir -p ${test_dir}/empty_dir +mkdir -p ${test_dir}/dir + +echo "file0 stuff" > ${test_dir}/file0.txt +echo "" > ${test_dir}/file_empty.txt +echo "file1 stuff" > ${test_dir}/dir/file1.txt + +# Symbolic (soft) link (points to a file name which points to an inode) +# ${test_dir}/file_0_soft.txt points to ${test_dir}/file0.txt +# The python `os.symlink` call omits the first `test_dir` +# because it simply looks in the same directory for the file to link to. +ln -s ${test_dir}/file0.txt ${test_dir}/file_0_soft.txt +# Bad symbolic (soft) link (points to a file name which points to an inode) +ln -s ${test_dir}/file0_that_doesnt_exist.txt ${test_dir}/file0_soft_bad.txt +# Hard link (points to an inode directly) +ln -s ${test_dir}/file0.txt ${test_dir}/file0_hard.txt + +# base.create ################################################################# +echo "Adding files to HPSS" + +case_name="create" +zstash create --hpss=${hpss_path} ${test_dir} 2>&1 | tee ${case_name}.log +check_log_has "INFO: Adding 000000.tar" ${case_name}.log +echo "Cache:" +ls -l ${test_dir}/${cache} 2>&1 | tee ${case_name}_cache.log +check_log_has "index.db" ${case_name}_cache.log +echo "HPSS:" +hsi ls -l ${hpss_path} 2>&1 | tee ${case_name}_hpss.log +check_log_has "000000.tar" ${case_name}_hpss.log +check_log_has "index.db" ${case_name}_hpss.log + +# base.add_files ############################################################## +echo "Testing update with an actual change" +mkdir -p ${test_dir}/dir2 +echo "file2 stuff" > ${test_dir}/dir2/file2.txt +echo "file1 stuff with changes" > ${test_dir}/dir/file1.txt +cd ${test_dir} +case_name="update1" +zstash update -v --hpss=${hpss_path} 2>&1 | tee ${case_name}.log +check_log_has "INFO: Adding 000001.tar" ${case_name}.log +cd .. +echo "Cache:" +ls -l ${test_dir}/${cache} 2>&1 | tee ${case_name}_cache.log +check_log_has "index.db" ${case_name}_cache.log +echo "HPSS:" +hsi ls -l ${hpss_path} 2>&1 | tee ${case_name}_hpss.log +check_log_has "000000.tar" ${case_name}_hpss.log +check_log_has "000001.tar" ${case_name}_hpss.log +check_log_has "index.db" ${case_name}_hpss.log + +echo "Adding more files to the HPSS archive." +echo "file3 stuff" > ${test_dir}/file3.txt +cd ${test_dir} +case_name="update2" +zstash update --hpss=${hpss_path} 2>&1 | tee ${case_name}.log +check_log_has "INFO: Adding 000002.tar" ${case_name}.log +cd .. +echo "Cache:" +ls -l ${test_dir}/${cache} 2>&1 | tee ${case_name}_cache.log +check_log_has "index.db" ${case_name}_cache.log +echo "HPSS:" +hsi ls -l ${hpss_path} 2>&1 | tee ${case_name}_hpss.log +check_log_has "000000.tar" ${case_name}_hpss.log +check_log_has "000001.tar" ${case_name}_hpss.log +check_log_has "000002.tar" ${case_name}_hpss.log +check_log_has "index.db" ${case_name}_hpss.log + +echo "file4 stuff" > ${test_dir}/file4.txt +cd ${test_dir} +case_name="update3" +zstash update --hpss=${hpss_path} 2>&1 | tee ${case_name}.log +check_log_has "INFO: Adding 000003.tar" ${case_name}.log +cd .. +echo "Cache:" +ls -l ${test_dir}/${cache} 2>&1 | tee ${case_name}_cache.log +check_log_has "index.db" ${case_name}_cache.log +echo "HPSS:" +hsi ls -l ${hpss_path} 2>&1 | tee ${case_name}_hpss.log +check_log_has "000000.tar" ${case_name}_hpss.log +check_log_has "000001.tar" ${case_name}_hpss.log +check_log_has "000002.tar" ${case_name}_hpss.log +check_log_has "000003.tar" ${case_name}_hpss.log +check_log_has "index.db" ${case_name}_hpss.log + +echo "file5 stuff" > ${test_dir}/file5.txt +cd ${test_dir} +case_name="update4" +zstash update --hpss=${hpss_path} 2>&1 | tee ${case_name}.log +check_log_has "INFO: Adding 000004.tar" ${case_name}.log +cd .. +echo "Cache:" +ls -l ${test_dir}/${cache} 2>&1 | tee ${case_name}_cache.log +check_log_has "index.db" ${case_name}_cache.log +echo "HPSS:" +hsi ls -l ${hpss_path} 2>&1 | tee ${case_name}_hpss.log +check_log_has "000000.tar" ${case_name}_hpss.log +check_log_has "000001.tar" ${case_name}_hpss.log +check_log_has "000002.tar" ${case_name}_hpss.log +check_log_has "000003.tar" ${case_name}_hpss.log +check_log_has "000004.tar" ${case_name}_hpss.log +check_log_has "index.db" ${case_name}_hpss.log + +# back in test_update.helperUpdateNonEmpty #################################### +echo "Cache check actually performed in the unit test:" +case_name="ls" +ls -l ${test_dir}/${cache} 2>&1 | tee ${case_name}.log +check_log_has "index.db" ${case_name}.log + +# base.tearDown ############################################################### +echo "Removing test files, both locally and at the HPSS repo" +rm -rf ${test_dir} +rm create.log +rm create_*.log +rm ls.log +rm update*.log +hsi rm -R ${hpss_path} diff --git a/tests/integration/python_tests/__init__.py b/tests/integration/python_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/python_tests/group_by_command/__init_.py b/tests/integration/python_tests/group_by_command/__init_.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/base.py b/tests/integration/python_tests/group_by_command/base.py similarity index 99% rename from tests/base.py rename to tests/integration/python_tests/group_by_command/base.py index de88ee4b..eada5220 100644 --- a/tests/base.py +++ b/tests/integration/python_tests/group_by_command/base.py @@ -1,7 +1,5 @@ """ -Run the test suite with `python -m unittest tests/test_*.py` - -To run an individual test, run something like `python -m unittest tests.test_extract.TestExtract.testExtractRetries` +To run an individual test, run something like `python -m unittest tests.integration.python_tests.group_by_command.test_extract.TestExtract.testExtractRetries` If running on Cori, it is preferable to run from $CSCRATCH rather than /global/homes. Running from the latter may result in a diff --git a/tests/test_check.py b/tests/integration/python_tests/group_by_command/test_check.py similarity index 99% rename from tests/test_check.py rename to tests/integration/python_tests/group_by_command/test_check.py index 9e8c17e0..d1dbe6c0 100644 --- a/tests/test_check.py +++ b/tests/integration/python_tests/group_by_command/test_check.py @@ -2,7 +2,7 @@ import shutil import unittest -from tests.base import ( +from tests.integration.python_tests.group_by_command.base import ( HPSS_ARCHIVE, TOP_LEVEL, ZSTASH_PATH, diff --git a/tests/test_check_parallel.py b/tests/integration/python_tests/group_by_command/test_check_parallel.py similarity index 98% rename from tests/test_check_parallel.py rename to tests/integration/python_tests/group_by_command/test_check_parallel.py index f0f1525c..2d2993b0 100644 --- a/tests/test_check_parallel.py +++ b/tests/integration/python_tests/group_by_command/test_check_parallel.py @@ -2,7 +2,7 @@ import shutil import unittest -from tests.base import ( +from tests.integration.python_tests.group_by_command.base import ( HPSS_ARCHIVE, TOP_LEVEL, ZSTASH_PATH, @@ -11,7 +11,7 @@ print_starred, run_cmd, ) -from tests.test_check import helperCheckTars +from tests.integration.python_tests.group_by_command.test_check import helperCheckTars # https://bugs.python.org/issue43743 # error: Module has no attribute "_USE_CP_SENDFILE" diff --git a/tests/test_chgrp.py b/tests/integration/python_tests/group_by_command/test_chgrp.py similarity index 92% rename from tests/test_chgrp.py rename to tests/integration/python_tests/group_by_command/test_chgrp.py index 9706e5e2..dc8763aa 100644 --- a/tests/test_chgrp.py +++ b/tests/integration/python_tests/group_by_command/test_chgrp.py @@ -1,6 +1,11 @@ import unittest -from tests.base import HPSS_ARCHIVE, ZSTASH_PATH, TestZstash, run_cmd +from tests.integration.python_tests.group_by_command.base import ( + HPSS_ARCHIVE, + ZSTASH_PATH, + TestZstash, + run_cmd, +) class TestChgrp(TestZstash): diff --git a/tests/test_create.py b/tests/integration/python_tests/group_by_command/test_create.py similarity index 99% rename from tests/test_create.py rename to tests/integration/python_tests/group_by_command/test_create.py index 8613d8d7..53360974 100644 --- a/tests/test_create.py +++ b/tests/integration/python_tests/group_by_command/test_create.py @@ -1,7 +1,7 @@ import os import unittest -from tests.base import ( +from tests.integration.python_tests.group_by_command.base import ( HPSS_ARCHIVE, TOP_LEVEL, ZSTASH_PATH, diff --git a/tests/test_extract.py b/tests/integration/python_tests/group_by_command/test_extract.py similarity index 99% rename from tests/test_extract.py rename to tests/integration/python_tests/group_by_command/test_extract.py index f3269ee0..1ca4f340 100644 --- a/tests/test_extract.py +++ b/tests/integration/python_tests/group_by_command/test_extract.py @@ -2,7 +2,7 @@ import shutil import unittest -from tests.base import ( +from tests.integration.python_tests.group_by_command.base import ( HPSS_ARCHIVE, TOP_LEVEL, ZSTASH_PATH, diff --git a/tests/test_extract_parallel.py b/tests/integration/python_tests/group_by_command/test_extract_parallel.py similarity index 96% rename from tests/test_extract_parallel.py rename to tests/integration/python_tests/group_by_command/test_extract_parallel.py index f34497ef..2ddd0db5 100644 --- a/tests/test_extract_parallel.py +++ b/tests/integration/python_tests/group_by_command/test_extract_parallel.py @@ -4,7 +4,7 @@ import unittest from typing import List -from tests.base import ( +from tests.integration.python_tests.group_by_command.base import ( HPSS_ARCHIVE, TOP_LEVEL, ZSTASH_PATH, @@ -12,7 +12,9 @@ print_starred, run_cmd, ) -from tests.test_extract import helperExtractTars +from tests.integration.python_tests.group_by_command.test_extract import ( + helperExtractTars, +) # https://bugs.python.org/issue43743 # error: Module has no attribute "_USE_CP_SENDFILE" diff --git a/tests/test_ls.py b/tests/integration/python_tests/group_by_command/test_ls.py similarity index 98% rename from tests/test_ls.py rename to tests/integration/python_tests/group_by_command/test_ls.py index e3a82a11..b8c5fea8 100644 --- a/tests/test_ls.py +++ b/tests/integration/python_tests/group_by_command/test_ls.py @@ -1,7 +1,7 @@ import os import unittest -from tests.base import ( +from tests.integration.python_tests.group_by_command.base import ( HPSS_ARCHIVE, TOP_LEVEL, ZSTASH_PATH, diff --git a/tests/test_update.py b/tests/integration/python_tests/group_by_command/test_update.py similarity index 99% rename from tests/test_update.py rename to tests/integration/python_tests/group_by_command/test_update.py index 4d8507e7..84354e4f 100644 --- a/tests/test_update.py +++ b/tests/integration/python_tests/group_by_command/test_update.py @@ -1,7 +1,7 @@ import os import unittest -from tests.base import ( +from tests.integration.python_tests.group_by_command.base import ( HPSS_ARCHIVE, TOP_LEVEL, ZSTASH_PATH, diff --git a/tests2/base.py b/tests/integration/python_tests/group_by_workflow/base.py similarity index 95% rename from tests2/base.py rename to tests/integration/python_tests/group_by_workflow/base.py index 644f3577..45d3a406 100644 --- a/tests2/base.py +++ b/tests/integration/python_tests/group_by_workflow/base.py @@ -1,14 +1,3 @@ -""" -Run the test suite with `python -m unittest tests2/test_*.py` - -tests2/ is a successor testing directory to tests/ -All new tests should be written in tests2/ -tests/ groups testing by zstash command (e.g., `create`, `extract`) -tests2/ groups testing by more logical workflows that test multiple zstash commands. - -The goal of tests2/ is to be able to follow the commands as if you were just reading a bash script. -""" - import os import shutil import stat diff --git a/tests2/test_cache_fs.py b/tests/integration/python_tests/group_by_workflow/test_cache_fs.py similarity index 98% rename from tests2/test_cache_fs.py rename to tests/integration/python_tests/group_by_workflow/test_cache_fs.py index 209496b9..a156f916 100644 --- a/tests2/test_cache_fs.py +++ b/tests/integration/python_tests/group_by_workflow/test_cache_fs.py @@ -1,7 +1,11 @@ import os import unittest -from tests2.base import TOP_LEVEL, TestZstash, run_cmd +from tests.integration.python_tests.group_by_workflow.base import ( + TOP_LEVEL, + TestZstash, + run_cmd, +) class TestCacheFs(TestZstash): diff --git a/tests/scripts/README.md b/tests/scripts/README.md deleted file mode 100644 index deaede4f..00000000 --- a/tests/scripts/README.md +++ /dev/null @@ -1 +0,0 @@ -Scripts for debugging / manually testing zstash diff --git a/tests/scripts/follow_symlinks.sh b/tests/scripts/follow_symlinks.sh deleted file mode 100755 index 1d85199e..00000000 --- a/tests/scripts/follow_symlinks.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash - -setup() -{ - echo "##########################################################################################################" - use_hpss=$1 - follow_symlinks=$2 - case_name="${3}" - archive_name=$4 - if [[ "${use_hpss}" == "true" ]]; then - hsi rm -R ${archive_name} - fi - echo "use_hpss=${use_hpss}" - echo "follow_symlinks=${follow_symlinks}" - echo "case_name=${case_name}" - local_archive_name=test_follow_symlinks - non_archived_dir=${local_archive_name}_non_archived - test_dir=/global/homes/f/forsyth/zstash/tests - cd ${test_dir} - rm -rf ${local_archive_name} - rm -rf ${non_archived_dir} - mkdir ${local_archive_name} - # At the same level as local_archive_name - mkdir ${non_archived_dir} - cd ${local_archive_name} - - mkdir zstash_demo - mkdir zstash_demo/empty_dir - mkdir zstash_demo/dir - mkdir non_archived - echo -n '' > zstash_demo/file_empty.txt - echo 'file0 stuff' > zstash_demo/dir/file0.txt - echo 'file1 stuff' > non_archived/file1.txt - echo 'file2 stuff' > ../${non_archived_dir}/file2.txt - # NOTE: `ln -s` appears to require absolute paths for the source files - ln -s ${test_dir}/${local_archive_name}/non_archived/file1.txt zstash_demo/file3.txt - ln -s ${test_dir}/${non_archived_dir}/file2.txt zstash_demo/file4.txt - cat zstash_demo/file3.txt - cat zstash_demo/file4.txt -} - -zstash_create() -{ - archive_name=$1 - follow_symlinks=$2 - echo "Starting zstash create" - if [[ "${follow_symlinks}" == "true" ]]; then - zstash create --hpss=${archive_name} zstash_demo --follow-symlinks - else - zstash create --hpss=${archive_name} zstash_demo - fi -} - -zstash_extract() -{ - archive_name=$1 - mkdir zstash_extraction - cd zstash_extraction - if [[ "${archive_name}" == "none" ]]; then - echo "Copying zstash" - cp -r ../zstash_demo/zstash/ zstash - fi - echo "Starting zstash extract" - zstash extract --hpss=${archive_name} - cat file3.txt - cat file4.txt - echo "> ls" - ls - echo "> ls -l" - ls -l - echo "> zstash ls" - zstash ls --hpss=${archive_name} - echo "> zstash ls -l" - zstash ls -l --hpss=${archive_name} - cd .. -} - -test_cases() -{ - use_hpss=$1 - follow_symlinks=$2 - if [[ "${use_hpss}" == "true" ]]; then - archive_name=/home/f/forsyth/zstash_test_follow_symlinks - else - archive_name=none - fi - - case_name="Don't delete original file" - setup ${use_hpss} ${follow_symlinks} "${case_name}" ${archive_name} - zstash_create ${archive_name} ${follow_symlinks} - zstash_extract ${archive_name} - - case_name="Delete before create" - setup ${use_hpss} ${follow_symlinks} "${case_name}" ${archive_name} - rm non_archived/file1.txt - rm ../run_n247_non_archived/file2.txt - zstash_create ${archive_name} ${follow_symlinks} - zstash_extract ${archive_name} - - case_name="Delete after create" - setup ${use_hpss} ${follow_symlinks} "${case_name}" ${archive_name} - zstash_create ${archive_name} ${follow_symlinks} - rm non_archived/file1.txt - rm ../run_n247_non_archived/file2.txt - zstash_extract ${archive_name} - -} - -conda_env=zstash_dev_n247 -# Set up Conda -source /global/homes/f/forsyth/miniconda3/etc/profile.d/conda.sh -conda activate ${conda_env} -# Install branch -cd /global/homes/f/forsyth/zstash -pip install . -# Begin tests -test_cases true true # HPSS, follow symlinks -test_cases false true # No HPSS, follow symlinks -test_cases true false # HPSS, don't follow symlinks -test_cases false false # No HPSS, don't follow symlinks diff --git a/tests/scripts/symlinks.sh b/tests/scripts/symlinks.sh deleted file mode 100755 index c4437818..00000000 --- a/tests/scripts/symlinks.sh +++ /dev/null @@ -1,44 +0,0 @@ -# Test symlinks -# Adjusted from https://github.com/E3SM-Project/zstash/issues/341 - -follow_symlinks=true - -rm -rf workdir workdir2 workdir3 -mkdir workdir workdir2 workdir3 -cd workdir -mkdir -p src/d1 src/d2 -touch src/d1/large_file.txt - -# This creates a symlink in d2 that links to a file in d1 -# Notice absolute path is used for source -ln -s /home/ac.forsyth2/ez/zstash/tests/scripts/workdir/src/d1/large_file.txt src/d2/large_file.txt - -echo "" -echo "ls -l src/d2" -ls -l src/d2 -# symlink - -echo "" -if [[ "${follow_symlinks,,}" == "true" ]]; then - echo "zstash create --hpss=none --follow-symlinks --cache /home/ac.forsyth2/ez/zstash/tests/scripts/workdir2 src/d2" - zstash create --hpss=none --follow-symlinks --cache /home/ac.forsyth2/ez/zstash/tests/scripts/workdir2 src/d2 -else - echo "zstash create --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/scripts/workdir2 src/d2" - zstash create --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/scripts/workdir2 src/d2 -fi - -echo "" -echo "ls -l src/d2" -ls -l src/d2 -# symlink (src is unaffected) - -cd ../workdir3 -echo "" -echo "zstash extract --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/scripts/workdir2" -zstash extract --hpss=none --cache /home/ac.forsyth2/ez/zstash/tests/scripts/workdir2 - -cd .. -echo "" -echo "ls workdir3" -ls workdir3 -# large_file.txt diff --git a/tests/scripts_unit_tests/README.md b/tests/scripts_unit_tests/README.md deleted file mode 100644 index ca41e4d6..00000000 --- a/tests/scripts_unit_tests/README.md +++ /dev/null @@ -1,4 +0,0 @@ -The unit tests, in script form. - -The unit tests are hard to follow via Python, -so it may be easier to run and debug the logic using these scripts. diff --git a/tests/scripts_unit_tests/test_update_non_empty_hpss.bash b/tests/scripts_unit_tests/test_update_non_empty_hpss.bash deleted file mode 100755 index 4c213d4e..00000000 --- a/tests/scripts_unit_tests/test_update_non_empty_hpss.bash +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -hpss_path=zstash_test # Set via `HPSS_ARCHIVE = "zstash_test"` -cache=zstash # Set via `self.cache = "zstash"` - -# base.setupDirs ############################################################## -test_dir=zstash_test - -# Create files and directories -echo "Creating files." -mkdir -p ${test_dir} -mkdir -p ${test_dir}/empty_dir -mkdir -p ${test_dir}/dir - -echo "file0 stuff" > ${test_dir}/file0.txt -echo "" > ${test_dir}/file_empty.txt -echo "file1 stuff" > ${test_dir}/dir/file1.txt - -# Symbolic (soft) link (points to a file name which points to an inode) -# ${test_dir}/file_0_soft.txt points to ${test_dir}/file0.txt -# The python `os.symlink` call omits the first `test_dir` -# because it simply looks in the same directory for the file to link to. -ln -s ${test_dir}/file0.txt ${test_dir}/file_0_soft.txt -# Bad symbolic (soft) link (points to a file name which points to an inode) -ln -s ${test_dir}/file0_that_doesnt_exist.txt ${test_dir}/file0_soft_bad.txt -# Hard link (points to an inode directly) -ln -s ${test_dir}/file0.txt ${test_dir}/file0_hard.txt - -# base.create ################################################################# -echo "Adding files to HPSS" -zstash create --hpss=${hpss_path} ${test_dir} -# Archives 000000.tar -echo "Cache:" -ls -l ${test_dir}/${cache} # just index.db -echo "HPSS:" -hsi ls -l ${hpss_path} # 000000.tar, index.db - -# base.add_files ############################################################## -echo "Testing update with an actual change" -mkdir -p ${test_dir}/dir2 -echo "file2 stuff" > ${test_dir}/dir2/file2.txt -echo "file1 stuff with changes" > ${test_dir}/dir/file1.txt -cd ${test_dir} -zstash update -v --hpss=${hpss_path} -# Archives 000001.tar -cd .. -echo "Cache:" -ls -l ${test_dir}/${cache} # just index.db -echo "HPSS:" -hsi ls -l ${hpss_path} # 000000.tar, 000001.tar, index.db - -echo "Adding more files to the HPSS archive." -echo "file3 stuff" > ${test_dir}/file3.txt -cd ${test_dir} -zstash update --hpss=${hpss_path} -# Archives 000002.tar -cd .. -echo "Cache:" -ls -l ${test_dir}/${cache} # just index.db -echo "HPSS:" -hsi ls -l ${hpss_path} # 000000.tar, 000001.tar, 000002.tar, index.db - -echo "file4 stuff" > ${test_dir}/file4.txt -cd ${test_dir} -zstash update --hpss=${hpss_path} -# Archives 000003.tar -cd .. -echo "Cache:" -ls -l ${test_dir}/${cache} # just index.db -echo "HPSS:" -hsi ls -l ${hpss_path} # 000000.tar, 000001.tar, 000002.tar, 000003.tar, index.db - -echo "file5 stuff" > ${test_dir}/file5.txt -cd ${test_dir} -zstash update --hpss=${hpss_path} -# Archives 000004.tar -cd .. -echo "Cache:" -ls -l ${test_dir}/${cache} # just index.db -echo "HPSS:" -hsi ls -l ${hpss_path} # 000000.tar, 000001.tar, 000002.tar, 000003.tar, 000004.tar, index.db - -# back in test_update.helperUpdateNonEmpty #################################### -echo "Cache check actually performed in the unit test:" -ls -l ${test_dir}/${cache} # just index.db - -# base.tearDown ############################################################### -echo "Removing test files, both locally and at the HPSS repo" -rm -rf ${test_dir} -hsi rm -R ${hpss_path} diff --git a/tests/test_functions.py b/tests/test_functions.py deleted file mode 100644 index cc494e9c..00000000 --- a/tests/test_functions.py +++ /dev/null @@ -1,69 +0,0 @@ -import unittest - -from zstash.extract import parse_tars_option - - -class TestFunctions(unittest.TestCase): - def testParseTarsOption(self): - # Starting at 00005a until the end - tar_list = parse_tars_option("00005a-", "000000", "00005d") - self.assertEqual(tar_list, ["00005a", "00005b", "00005c", "00005d"]) - - # Starting from the beginning to 00005a (included) - tar_list = parse_tars_option("-00005a", "000058", "00005d") - self.assertEqual(tar_list, ["000058", "000059", "00005a"]) - - # Specific range - tar_list = parse_tars_option("00005a-00005c", "00000", "000005d") - self.assertEqual(tar_list, ["00005a", "00005b", "00005c"]) - - # Selected tar files - tar_list = parse_tars_option("00003e,00004e,000059", "000000", "00005d") - self.assertEqual(tar_list, ["00003e", "00004e", "000059"]) - - # Mix and match - tar_list = parse_tars_option("000030-00003e,00004e,00005a-", "000000", "00005d") - self.assertEqual( - tar_list, - [ - "000030", - "000031", - "000032", - "000033", - "000034", - "000035", - "000036", - "000037", - "000038", - "000039", - "00003a", - "00003b", - "00003c", - "00003d", - "00003e", - "00004e", - "00005a", - "00005b", - "00005c", - "00005d", - ], - ) - - # Check removal of duplicates and sorting - tar_list = parse_tars_option("000009,000003,-000005", "000000", "00005d") - self.assertEqual( - tar_list, - ["000000", "000001", "000002", "000003", "000004", "000005", "000009"], - ) - - # Remove .tar suffix - tar_list = parse_tars_option( - "000009.tar-00000a,000003.tar,-000002.tar", "000000", "00005d" - ) - self.assertEqual( - tar_list, ["000000", "000001", "000002", "000003", "000009", "00000a"] - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/test_functions.py b/tests/unit/test_functions.py new file mode 100644 index 00000000..5fa4d5f3 --- /dev/null +++ b/tests/unit/test_functions.py @@ -0,0 +1,62 @@ +from zstash.extract import parse_tars_option + + +def test_parse_tars_option(): + # Starting at 00005a until the end + tar_list = parse_tars_option("00005a-", "000000", "00005d") + assert tar_list == ["00005a", "00005b", "00005c", "00005d"] + + # Starting from the beginning to 00005a (included) + tar_list = parse_tars_option("-00005a", "000058", "00005d") + assert tar_list == ["000058", "000059", "00005a"] + + # Specific range + tar_list = parse_tars_option("00005a-00005c", "00000", "000005d") + assert tar_list == ["00005a", "00005b", "00005c"] + + # Selected tar files + tar_list = parse_tars_option("00003e,00004e,000059", "000000", "00005d") + assert tar_list == ["00003e", "00004e", "000059"] + + # Mix and match + tar_list = parse_tars_option("000030-00003e,00004e,00005a-", "000000", "00005d") + assert tar_list == [ + "000030", + "000031", + "000032", + "000033", + "000034", + "000035", + "000036", + "000037", + "000038", + "000039", + "00003a", + "00003b", + "00003c", + "00003d", + "00003e", + "00004e", + "00005a", + "00005b", + "00005c", + "00005d", + ] + + # Check removal of duplicates and sorting + tar_list = parse_tars_option("000009,000003,-000005", "000000", "00005d") + assert tar_list == [ + "000000", + "000001", + "000002", + "000003", + "000004", + "000005", + "000009", + ] + + # Remove .tar suffix + tar_list = parse_tars_option( + "000009.tar-00000a,000003.tar,-000002.tar", "000000", "00005d" + ) + assert tar_list == ["000000", "000001", "000002", "000003", "000009", "00000a"]