Skip to content

Commit e60fd45

Browse files
zshaheenzshaheen
authored
Merge pull request #10 from E3SM-Project/testing
Testing
2 parents c03bbfe + b37ff46 commit e60fd45

File tree

13 files changed

+296
-14
lines changed

13 files changed

+296
-14
lines changed

‎conda/meta.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package:
2+
name: zstash
3+
version: 0.0.1
4+
5+
source:
6+
git_url: https://github.com/E3SM-Project/zstash
7+
git_rev: v0.0.1
8+
9+
build:
10+
script: python setup.py install
11+
12+
requirements:
13+
build:
14+
- python 2
15+
- setuptools
16+
17+
run:
18+
- python 2
19+
20+
about:
21+
home: https://github.com/E3SM-Project/zstash
22+

‎setup.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from setuptools import find_packages, setup
2+
3+
setup(
4+
name="zstash",
5+
version="0.0.1",
6+
author="Chris Golaz, Zeshawn Shaheen",
7+
author_email="[email protected], [email protected]",
8+
description="Long term HPSS archiving software for E3SM",
9+
packages=find_packages(exclude=["*.test", "*.test.*", "test.*", "test"]),
10+
entry_points={
11+
'console_scripts': [
12+
'zstash=zstash.main:main'
13+
]}
14+
)
15+

‎tests/test.py‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
import os
2+
import sys
3+
import subprocess
4+
import shutil
5+
6+
def write_file(name, contents):
7+
"""
8+
Write contents to a file named name.
9+
"""
10+
with open(name, 'w') as f:
11+
f.write(contents)
12+
13+
def run_cmd(cmd):
14+
"""
15+
Run a command while printing and returning the stdout and stderr
16+
"""
17+
print('+ {}'.format(cmd))
18+
if isinstance(cmd, str):
19+
cmd = cmd.split()
20+
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
21+
output, err = p.communicate()
22+
print(output)
23+
print(err)
24+
return output, err
25+
26+
def str_not_in(output, msg):
27+
"""
28+
If the msg is not in the output string, then everything is fine.
29+
"""
30+
if msg in output:
31+
print('*'*40)
32+
print('This was not supposed to be found: {}',format(msg))
33+
print('*'*40)
34+
exit()
35+
36+
def str_in(output, msg):
37+
"""
38+
If the msg is in the output string, then the everything is fine.
39+
"""
40+
if not msg in output:
41+
print('*'*40)
42+
print('This was supposed to be found, but was not: {}'.format(msg))
43+
print('*'*40)
44+
exit()
45+
46+
def cleanup():
47+
"""
48+
After this script is ran, remove all created files, even those on the HPSS repo.
49+
"""
50+
print('Removing test files, both locally and at the HPSS repo')
51+
if os.path.exists('zstash_test'):
52+
shutil.rmtree('zstash_test')
53+
if os.path.exists('zstash_test_backup'):
54+
shutil.rmtree('zstash_test_backup')
55+
cmd = 'hsi rm -R {}'.format(HPSS_PATH)
56+
run_cmd(cmd)
57+
58+
def exit():
59+
"""
60+
Cleanup and stop running this script
61+
"""
62+
cleanup()
63+
sys.exit()
64+
65+
66+
# TODO: Change the hpss directory to a dir that's accessable to everyone
67+
HPSS_PATH='/home/z/zshaheen/zstash_test'
68+
69+
# Create files and directories
70+
print('Creating files.')
71+
if not os.path.exists('zstash_test'):
72+
os.mkdir('zstash_test')
73+
if not os.path.exists('zstash_test/empty_dir'):
74+
os.mkdir('zstash_test/empty_dir')
75+
if not os.path.exists('zstash_test/dir'):
76+
os.mkdir('zstash_test/dir')
77+
78+
write_file('zstash_test/file0.txt', 'file0 stuff')
79+
write_file('zstash_test/file_empty.txt', '')
80+
write_file('zstash_test/dir/file1.txt', 'file1 stuff')
81+
82+
if not os.path.lexists('zstash_test/file0_soft.txt'):
83+
# If we symlink zstash_test/file0_soft.txt to zstash_test/file0.txt
84+
# zstash_test/file0_soft.txt links to zstash_test/zstash_test/file0.txt
85+
os.symlink('file0.txt', 'zstash_test/file0_soft.txt')
86+
87+
if not os.path.lexists('zstash_test/file0_soft_bad.txt'):
88+
# If we symlink zstash_test/file0_soft.txt to zstash_test/file0.txt
89+
# zstash_test/file0_soft.txt links to zstash_test/zstash_test/file0.txt
90+
os.symlink('file0_that_doesnt_exist.txt', 'zstash_test/file0_soft_bad.txt')
91+
92+
if not os.path.lexists('zstash_test/file0_hard.txt'):
93+
os.link('zstash_test/file0.txt', 'zstash_test/file0_hard.txt')
94+
95+
print('Adding files to HPSS')
96+
cmd = 'zstash create --hpss={} zstash_test'.format(HPSS_PATH)
97+
output, err = run_cmd(cmd)
98+
str_in(output+err, 'Transferring file to HPSS')
99+
100+
print('Testing chgrp')
101+
GROUP = 'acme'
102+
print('First, make sure that the files are not already in the {} group'.format(GROUP))
103+
cmd = 'hsi ls -l {}'.format(HPSS_PATH)
104+
output, err = run_cmd(cmd)
105+
str_not_in(output+err, GROUP)
106+
print('Running zstash chgrp')
107+
cmd = 'zstash chgrp -R {} {}'.format(GROUP, HPSS_PATH)
108+
output, err = run_cmd(cmd)
109+
print('Now check that the files are in the {} group'.format(GROUP))
110+
cmd = 'hsi ls -l {}'.format(HPSS_PATH)
111+
output, err = run_cmd(cmd)
112+
str_in(output+err, 'acme')
113+
114+
print('Running update on the newly created directory, nothing should happen')
115+
os.chdir('zstash_test')
116+
cmd = 'zstash update --hpss={}'.format(HPSS_PATH)
117+
output, err = run_cmd(cmd)
118+
os.chdir('../')
119+
str_in(output+err, 'Nothing to update')
120+
121+
122+
print('Testing update with an actual change')
123+
if not os.path.exists('zstash_test/dir2'):
124+
os.mkdir('zstash_test/dir2')
125+
write_file('zstash_test/dir2/file2.txt', 'file2 stuff')
126+
write_file('zstash_test/dir/file1.txt', 'file1 stuff with changes')
127+
128+
os.chdir('zstash_test')
129+
cmd = 'zstash update --hpss={}'.format(HPSS_PATH)
130+
output, err = run_cmd(cmd)
131+
os.chdir('../')
132+
str_in(output+err, 'Transferring file to HPSS')
133+
# Make sure none of the old files are moved
134+
str_not_in(output+err, 'file0')
135+
str_not_in(output+err, 'file_empty')
136+
str_not_in(output+err, 'empty_dir')
137+
138+
print('Testing the extract functionality')
139+
os.rename('zstash_test', 'zstash_test_backup')
140+
os.mkdir('zstash_test')
141+
os.chdir('zstash_test')
142+
cmd = 'zstash extract --hpss={}'.format(HPSS_PATH)
143+
output, err = run_cmd(cmd)
144+
os.chdir('../')
145+
str_in(output+err, 'Transferring from HPSS')
146+
str_in(output+err, 'Extracting file0.txt')
147+
str_in(output+err, 'Extracting file0_hard.txt')
148+
str_in(output+err, 'Extracting file0_soft.txt')
149+
str_in(output+err, 'Extracting file_empty.txt')
150+
str_in(output+err, 'Extracting dir/file1.txt')
151+
str_in(output+err, 'Extracting empty_dir')
152+
str_in(output+err, 'Extracting dir2/file2.txt')
153+
154+
print('Running update on the newly extracted directory, nothing should happen')
155+
os.chdir('zstash_test')
156+
cmd = 'zstash update --hpss={}'.format(HPSS_PATH)
157+
output, err = run_cmd(cmd)
158+
os.chdir('../')
159+
str_in(output+err, 'Nothing to update')
160+
161+
print('Verifying the data from database with the actual files')
162+
# Checksums from HPSS
163+
cmd = ['sqlite3', 'zstash_test/zstash/index.db', 'select md5, name from files;']
164+
output_hpss, err_hpss = run_cmd(cmd)
165+
hpss_dict = {}
166+
167+
for l in output_hpss.split('\n'):
168+
l = l.split('|')
169+
if len(l) >= 2:
170+
f_name = l[1]
171+
f_hash = l[0]
172+
hpss_dict[f_name] = f_hash
173+
174+
# Checksums from local files
175+
cmd = '''find zstash_test_backup -regex .*\.txt.* -exec md5sum {} + '''
176+
output_local, err_local = run_cmd(cmd)
177+
local_dict = {}
178+
179+
for l in output_local.split('\n'):
180+
l = l.split(' ')
181+
if len(l) >= 2:
182+
f_name = l[1].split('/') # remove the 'zstash_test_backup'
183+
f_name = '/'.join(f_name[1:])
184+
f_hash = l[0]
185+
local_dict[f_name] = f_hash
186+
print('filename|HPSS hash|local file hash')
187+
for k in local_dict:
188+
print('{}|{}|{}'.format(k, hpss_dict[k], local_dict[k]))
189+
190+
cleanup()
191+
print('*'*40)
192+
print('All of the tests passed! :)')
193+
print('*'*40)

‎zstash‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎zstash/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
__version__ = 'v0.0.1'
2+

‎zstash/chgrp.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import sys
2+
import argparse
3+
from hpss import hpss_chgrp
4+
5+
def chgrp():
6+
7+
# Parser
8+
parser = argparse.ArgumentParser(
9+
usage='zstash chgrp [<args>] group hpss_archive',
10+
description='Change the group of an HPSS repository.')
11+
parser.add_argument('group', type=str, help='new group name of file(s)')
12+
parser.add_argument('hpss', type=str, help='path to HPSS storage')
13+
parser.add_argument('-R', action='store_const', const=True, help='recurse through subdirectories')
14+
15+
args = parser.parse_args(sys.argv[2:])
16+
recurse = True if args.R else False
17+
hpss_chgrp(args.hpss, args.group, recurse)
18+

‎src/create.py‎ renamed to ‎zstash/create.py‎

File renamed without changes.

‎src/extract.py‎ renamed to ‎zstash/extract.py‎

File renamed without changes.

‎src/hpss.py‎ renamed to ‎zstash/hpss.py‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@
55
from subprocess import Popen, PIPE
66

77

8-
# Put file to hpss
98
def hpss_put(hpss, file, keep=True):
9+
"""
10+
Put a file to the HPSS archive.
11+
"""
1012

1113
logging.info('Transferring file to HPSS: %s' % (file))
1214
path, name = os.path.split(file)
@@ -36,8 +38,10 @@ def hpss_put(hpss, file, keep=True):
3638
os.remove(file)
3739

3840

39-
# Get file from hpss
4041
def hpss_get(hpss, file):
42+
"""
43+
Get ia file from the HPSS archive.
44+
"""
4145

4246
logging.info('Transferring from HPSS: %s' % (file))
4347
path, name = os.path.split(file)
@@ -63,3 +67,22 @@ def hpss_get(hpss, file):
6367
# Back to original working directory
6468
if path != '':
6569
os.chdir(cwd)
70+
71+
def hpss_chgrp(hpss, group, recurse=False):
72+
"""
73+
Change the group of the HPSS archive.
74+
"""
75+
if recurse:
76+
cmd = 'hsi chgrp -R {} {}'.format(group, hpss)
77+
else:
78+
cmd = 'hsi chgrp {} {}'.format(group, hpss)
79+
80+
p1 = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE)
81+
(stdout, stderr) = p1.communicate()
82+
status = p1.returncode
83+
if status != 0:
84+
logging.error('Changing group of HPSS archive {} to {}'.format())
85+
logging.debug('stdout:\n%s', stdout)
86+
logging.debug('stderr:\n%s', stderr)
87+
raise Exception
88+

‎src/main.py‎ renamed to ‎zstash/main.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from create import create
1010
from update import update
1111
from extract import extract
12+
from chgrp import chgrp
1213

1314

1415
# -----------------------------------------------------------------------------
@@ -39,6 +40,8 @@ def main():
3940
update()
4041
elif args.command == 'extract':
4142
extract()
43+
elif args.command == 'chgrp':
44+
chgrp()
4245
else:
4346
print 'Unrecognized command'
4447
parser.print_help()

0 commit comments

Comments
 (0)