Skip to content

Commit e738007

Browse files
forsyth2forsyth2
committed
Add integration tests
1 parent 0fb1996 commit e738007

File tree

5 files changed

+205
-75
lines changed

5 files changed

+205
-75
lines changed

‎tests_integration/__init__.py‎

Whitespace-only changes.

‎tests2/test_cache_fs.py‎ renamed to ‎tests_integration/test_cache_fs.py‎

Lines changed: 21 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import os
22
import unittest
33

4-
from tests2.base import TOP_LEVEL, TestZstash, run_cmd
4+
from tests_integration.utils import TestZstash, run_cmd
55

66

77
class TestCacheFs(TestZstash):
@@ -17,7 +17,7 @@ def test_hpss_none_fs_on(self):
1717
# internal symlink (in same dir, in different dir), external symlink
1818
# internal hard link (in same dir, in different dir), external hard link, broken hard link
1919
self.setup_dirs(include_broken_symlink=False)
20-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/")
20+
os.chdir(f"{self.work_dir}/zstash_src/")
2121

2222
# Test zstash_src before create
2323
self.assertTrue(os.path.islink("file0_soft.txt"))
@@ -28,10 +28,10 @@ def test_hpss_none_fs_on(self):
2828
self.assertFalse(os.path.islink("file_not_included_hard.txt"))
2929
self.assertFalse(os.path.islink("original_was_deleted_hard.txt"))
3030

31-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/")
32-
cmd = f"zstash create --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache --follow-symlinks zstash_src"
31+
os.chdir(f"{self.work_dir}/")
32+
cmd = f"zstash create --hpss=none --cache={self.work_dir}/test_cache --follow-symlinks zstash_src"
3333
run_cmd(cmd)
34-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/")
34+
os.chdir(f"{self.work_dir}/zstash_src/")
3535

3636
# Test zstash_src after create
3737
# Running `create` should not alter the source directory.
@@ -44,9 +44,7 @@ def test_hpss_none_fs_on(self):
4444
self.assertFalse(os.path.islink("original_was_deleted_hard.txt"))
4545

4646
os.chdir("../zstash_extracted")
47-
cmd = (
48-
f"zstash extract --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache"
49-
)
47+
cmd = f"zstash extract --hpss=none --cache={self.work_dir}/test_cache"
5048
run_cmd(cmd)
5149

5250
# Test extraction from zstash_archive
@@ -81,24 +79,22 @@ def test_hpss_none_fs_on_broken_symlink(self):
8179
# Cases:
8280
# broken symlink
8381
self.setup_dirs(include_broken_symlink=True)
84-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/")
82+
os.chdir(f"{self.work_dir}/zstash_src/")
8583

8684
# Test zstash_src before create
8785
self.assertTrue(os.path.islink("original_was_deleted_soft.txt"))
8886

89-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/")
90-
cmd = f"zstash create --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache --follow-symlinks zstash_src"
87+
os.chdir(f"{self.work_dir}/")
88+
cmd = f"zstash create --hpss=none --cache={self.work_dir}/test_cache --follow-symlinks zstash_src"
9189
run_cmd(cmd)
92-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/")
90+
os.chdir(f"{self.work_dir}/zstash_src/")
9391

9492
# Test zstash_src after create
9593
# Running `create` should not alter the source directory.
9694
self.assertTrue(os.path.islink("original_was_deleted_soft.txt"))
9795

9896
os.chdir("../zstash_extracted")
99-
cmd = (
100-
f"zstash extract --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache"
101-
)
97+
cmd = f"zstash extract --hpss=none --cache={self.work_dir}/test_cache"
10298
_, err = run_cmd(cmd)
10399
# This is ultimately caused by:
104100
# `Exception: Archive creation failed due to broken symlink.`
@@ -111,7 +107,7 @@ def test_hpss_none_fs_on_broken_symlink(self):
111107
def test_hpss_none_fs_off(self):
112108
#
113109
self.setup_dirs(include_broken_symlink=False)
114-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/")
110+
os.chdir(f"{self.work_dir}/zstash_src/")
115111

116112
# Test zstash_src before create
117113
self.assertTrue(os.path.islink("file0_soft.txt"))
@@ -122,10 +118,10 @@ def test_hpss_none_fs_off(self):
122118
self.assertFalse(os.path.islink("file_not_included_hard.txt"))
123119
self.assertFalse(os.path.islink("original_was_deleted_hard.txt"))
124120

125-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/")
126-
cmd = f"zstash create --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache zstash_src"
121+
os.chdir(f"{self.work_dir}/")
122+
cmd = f"zstash create --hpss=none --cache={self.work_dir}/test_cache zstash_src"
127123
run_cmd(cmd)
128-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/")
124+
os.chdir(f"{self.work_dir}/zstash_src/")
129125

130126
# Test zstash_src after create
131127
# Running `create` should not alter the source directory.
@@ -138,9 +134,7 @@ def test_hpss_none_fs_off(self):
138134
self.assertFalse(os.path.islink("original_was_deleted_hard.txt"))
139135

140136
os.chdir("../zstash_extracted")
141-
cmd = (
142-
f"zstash extract --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache"
143-
)
137+
cmd = f"zstash extract --hpss=none --cache={self.work_dir}/test_cache"
144138
run_cmd(cmd)
145139

146140
# Test extraction from zstash_archive
@@ -172,24 +166,22 @@ def test_hpss_none_fs_off(self):
172166

173167
def test_hpss_none_fs_off_broken_symlink(self):
174168
self.setup_dirs(include_broken_symlink=True)
175-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/")
169+
os.chdir(f"{self.work_dir}/zstash_src/")
176170

177171
# Test zstash_src before create
178172
self.assertTrue(os.path.islink("original_was_deleted_soft.txt"))
179173

180-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/")
181-
cmd = f"zstash create --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache zstash_src"
174+
os.chdir(f"{self.work_dir}/")
175+
cmd = f"zstash create --hpss=none --cache={self.work_dir}/test_cache zstash_src"
182176
run_cmd(cmd)
183-
os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/")
177+
os.chdir(f"{self.work_dir}/zstash_src/")
184178

185179
# Test zstash_src after create
186180
# Running `create` should not alter the source directory.
187181
self.assertTrue(os.path.islink("original_was_deleted_soft.txt"))
188182

189183
os.chdir("../zstash_extracted")
190-
cmd = (
191-
f"zstash extract --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache"
192-
)
184+
cmd = f"zstash extract --hpss=none --cache={self.work_dir}/test_cache"
193185
run_cmd(cmd)
194186
# With fs off, this command completes successfully.
195187

‎tests_integration/test_tutorial_2020.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Test 2020 tutorial workflow.
2+
# https://github.com/E3SM-Project/zstash/pull/79/files
3+
4+
from tests_integration.utils import TestZstash
5+
6+
7+
class TestTutorial2020(TestZstash):
8+
9+
def test_tutorial_2020(self):
10+
pass

‎tests_integration/test_tutorial_2024.py‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# Test workflow similar to the 2024 tutorial.
2+
# https://github.com/E3SM-Project/zstash/blob/add-tutorial-materials/tutorial_materials/zstash_demo.md
3+
4+
5+
import os
6+
7+
from tests_integration.utils import TestZstash, run_cmd
8+
9+
10+
class TestTutorial2024(TestZstash):
11+
12+
def test_tutorial_2024(self):
13+
# This test can only be run on NERSC, using NERSC HPSS.
14+
self.conditional_hpss_skip()
15+
self.setup_dirs()
16+
os.chdir(f"{self.work_dir}")
17+
18+
files_to_include = "dir1/*.txt" # Should match file1.txt
19+
cmd = f"zstash create --hpss={self.hpss_dir} --cache={self.cache_dir} --include={files_to_include} {self.dir_to_archive}"
20+
output, err = run_cmd(cmd)
21+
expected_present = [
22+
"Creating new tar archive",
23+
"Archiving dir1/file1.txt",
24+
"Transferring file to HPSS",
25+
"Completed archive file",
26+
]
27+
expected_absent = [
28+
"ERROR",
29+
"Archiving file0.txt",
30+
"Archiving file_empty.txt",
31+
"Archiving file0_soft.txt",
32+
"Archiving dir2/file1_soft.txt",
33+
"Archiving file_not_included_soft.txt",
34+
"Archiving file0_hard.txt",
35+
"Archiving dir2/file1_hard.txt",
36+
"Archiving file_not_included_hard.txt",
37+
"Archiving original_was_deleted_hard.txt",
38+
]
39+
self.check_strings(cmd, output + err, expected_present, expected_absent)
40+
41+
os.mkdir("check_output")
42+
os.chdir("check_output")
43+
cmd = f"zstash check --hpss={self.hpss_dir}"
44+
output, err = run_cmd(cmd)
45+
expected_present = [
46+
"Transferring file from HPSS",
47+
"Opening tar archive",
48+
"Checking dir1/file1.txt",
49+
"No failures detected when checking the files",
50+
]
51+
expected_absent = [
52+
"ERROR",
53+
"Checking file0.txt",
54+
"Checking file_empty.txt",
55+
"Checking file0_soft.txt",
56+
"Checking dir2/file1_soft.txt",
57+
"Checking file_not_included_soft.txt",
58+
"Checking file0_hard.txt",
59+
"Checking dir2/file1_hard.txt",
60+
"Checking file_not_included_hard.txt",
61+
"Checking original_was_deleted_hard.txt",
62+
]
63+
self.check_strings(cmd, output + err, expected_present, expected_absent)
64+
os.chdir(f"{self.work_dir}")
65+
66+
os.mkdir("ls_output")
67+
os.chdir("ls_output")
68+
cmd = f"zstash ls --hpss={self.hpss_dir}"
69+
output, err = run_cmd(cmd)
70+
expected_present = [
71+
"dir1/file1.txt",
72+
]
73+
expected_absent = [
74+
"ERROR",
75+
"file0.txt",
76+
"file_empty.txt",
77+
"file0_soft.txt",
78+
"dir2/file1_soft.txt",
79+
"file_not_included_soft.txt",
80+
"file0_hard.txt",
81+
"dir2/file1_hard.txt",
82+
"file_not_included_hard.txt",
83+
"original_was_deleted_hard.txt",
84+
]
85+
self.check_strings(cmd, output + err, expected_present, expected_absent)
86+
os.chdir(f"{self.work_dir}")
87+
88+
os.mkdir("extract_output")
89+
os.chdir("extract_output")
90+
cmd = f"zstash extract --hpss={self.hpss_dir}"
91+
output, err = run_cmd(cmd)
92+
expected_present = [
93+
"Transferring file from HPSS",
94+
"Opening tar archive",
95+
"Extracting dir1/file1.txt",
96+
"No failures detected when extracting the files",
97+
]
98+
expected_absent = [
99+
"ERROR",
100+
"Extracting file0.txt",
101+
"Extracting file_empty.txt",
102+
"Extracting file0_soft.txt",
103+
"Extracting dir2/file1_soft.txt",
104+
"Extracting file_not_included_soft.txt",
105+
"Extracting file0_hard.txt",
106+
"Extracting dir2/file1_hard.txt",
107+
"Extracting file_not_included_hard.txt",
108+
"Extracting original_was_deleted_hard.txt",
109+
]
110+
self.check_strings(cmd, output + err, expected_present, expected_absent)
111+
os.chdir(f"{self.work_dir}")

0 commit comments

Comments
 (0)