Merge pull request #28 from E3SM-Project/v1.3

v0.3.0

Author: zshaheen

conda/meta.yaml

@@ -1,21 +1,21 @@
 package:
     name: zstash
-    version: 0.2.0
+    version: 0.3.0
 
 source:
     git_url: https://github.com/E3SM-Project/zstash
-    git_rev: v0.2.0
+    git_rev: v0.3.0
 
 build:
     script: python setup.py install
 
 requirements:
   build:
-    - python 2
+    - python 3
     - setuptools
 
   run:
-    - python 2
+    - python 3
 
 about:
     home: https://github.com/E3SM-Project/zstash

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="zstash",
-    version="0.2.0",
+    version="0.3.0",
     author="Chris Golaz, Zeshawn Shaheen",
     author_email="[email protected], [email protected]",
     description="Long term HPSS archiving software for E3SM",

tests/test.py

@@ -1,3 +1,10 @@
+"""
+Run the test suite like:
+    python test.py
+You'll get a statement printed
+if all of the tests pass.
+"""
+
 import os
 import sys
 import subprocess
@@ -19,6 +26,14 @@ def run_cmd(cmd):
         cmd = cmd.split()
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     output, err = p.communicate()
+
+    # When running in Python 3, the output of subprocess.Popen.communicate()
+    # is a bytes object. We need to convert it to a string.
+    if isinstance(output, bytes):
+        output = output.decode("utf-8")
+    if isinstance(err, bytes):
+        err = err.decode("utf-8")
+
     print(output)
     print(err)
     return output, err
@@ -52,6 +67,8 @@ def cleanup():
         shutil.rmtree('zstash_test')
     if os.path.exists('zstash_test_backup'):
         shutil.rmtree('zstash_test_backup')
+    if os.path.exists('zstash'):
+        shutil.rmtree('zstash')
     cmd = 'hsi rm -R {}'.format(HPSS_PATH)
     run_cmd(cmd)
 
@@ -62,9 +79,9 @@ def stop():
     cleanup()
     sys.exit()
 
-
-# TODO: Change the hpss directory to a dir that's accessable to everyone.
-HPSS_PATH='/home/z/zshaheen/zstash_test'
+# Makes this in the home dir of the user on HPSS.
+# Ex: /home/z/zshaheen/zstash_test
+HPSS_PATH='zstash_test'
 
 # Create files and directories
 print('Creating files.')
@@ -92,29 +109,16 @@ def stop():
 if not os.path.lexists('zstash_test/file0_hard.txt'):
     os.link('zstash_test/file0.txt', 'zstash_test/file0_hard.txt')
 
+
 print('Adding files to HPSS')
 cmd = 'zstash create --hpss={} zstash_test'.format(HPSS_PATH)
 output, err = run_cmd(cmd)
 str_in(output+err, 'Transferring file to HPSS')
 str_not_in(output+err, 'ERROR')
 
-print('Testing ls')
-cmd = 'zstash ls --hpss={}'.format(HPSS_PATH)
-output, err = run_cmd(cmd)
-str_in(output+err, 'file0.txt')
-str_not_in(output+err, 'ERROR')
-cmd = 'zstash ls -l --hpss={}'.format(HPSS_PATH)
-output, err = run_cmd(cmd)
-str_in(output+err, 'tar')
-str_not_in(output+err, 'ERROR')
 
 print('Testing chgrp')
 GROUP = 'acme'
-print('First, make sure that the files are not already in the {} group'.format(GROUP))
-cmd = 'hsi ls -l {}'.format(HPSS_PATH)
-output, err = run_cmd(cmd)
-str_not_in(output+err, GROUP)
-str_not_in(output+err, 'ERROR')
 print('Running zstash chgrp')
 cmd = 'zstash chgrp -R {} {}'.format(GROUP, HPSS_PATH)
 output, err = run_cmd(cmd)
@@ -125,6 +129,7 @@ def stop():
 str_in(output+err, 'acme')
 str_not_in(output+err, 'ERROR')
 
+
 print('Running update on the newly created directory, nothing should happen')
 os.chdir('zstash_test')
 cmd = 'zstash update --hpss={}'.format(HPSS_PATH)
@@ -133,6 +138,7 @@ def stop():
 str_in(output+err, 'Nothing to update')
 str_not_in(output+err, 'ERROR')
 
+
 print('Testing update with an actual change')
 if not os.path.exists('zstash_test/dir2'):
     os.mkdir('zstash_test/dir2')
@@ -145,24 +151,108 @@ def stop():
 os.chdir('../')
 str_in(output+err, 'Transferring file to HPSS')
 str_not_in(output+err, 'ERROR')
-# Make sure none of the old files are moved
+# Make sure none of the old files are moved.
 str_not_in(output+err, 'file0')
 str_not_in(output+err, 'file_empty')
 str_not_in(output+err, 'empty_dir')
 str_not_in(output+err, 'ERROR')
 
+
+print('Adding many more files to the HPSS archive.')
+msg = 'This is because we need many separate tar archives'
+msg += ' for testing zstash extract/check with parallel.'
+print(msg)
+write_file('zstash_test/file3.txt', 'file3 stuff')
+os.chdir('zstash_test')
+cmd = 'zstash update --hpss={}'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+os.chdir('../')
+write_file('zstash_test/file4.txt', 'file4 stuff')
+os.chdir('zstash_test')
+cmd = 'zstash update --hpss={}'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+os.chdir('../')
+write_file('zstash_test/file5.txt', 'file5 stuff')
+os.chdir('zstash_test')
+cmd = 'zstash update --hpss={}'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+os.chdir('../')
+
+
+print('Testing ls')
+cmd = 'zstash ls --hpss={}'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+str_in(output+err, 'file0.txt')
+str_not_in(output+err, 'ERROR')
+cmd = 'zstash ls -l --hpss={}'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+str_in(output+err, 'tar')
+str_not_in(output+err, 'ERROR')
+
+
 print('Testing the checking functionality')
 cmd = 'zstash check --hpss={}'.format(HPSS_PATH)
 output, err = run_cmd(cmd)
+str_in(output+err, 'Checking file0.txt')
+str_in(output+err, 'Checking file0_hard.txt')
+str_in(output+err, 'Checking file0_soft.txt')
+str_in(output+err, 'Checking file_empty.txt')
+str_in(output+err, 'Checking dir/file1.txt')
+str_in(output+err, 'Checking empty_dir')
+str_in(output+err, 'Checking dir2/file2.txt')
+str_in(output+err, 'Checking file3.txt')
+str_in(output+err, 'Checking file4.txt')
+str_in(output+err, 'Checking file5.txt')
 str_not_in(output+err, 'ERROR')
 
+
 print('Testing the extract functionality')
 os.rename('zstash_test', 'zstash_test_backup')
 os.mkdir('zstash_test')
 os.chdir('zstash_test')
 cmd = 'zstash extract --hpss={}'.format(HPSS_PATH)
 output, err = run_cmd(cmd)
 os.chdir('../')
+str_in(output+err, 'Extracting file0.txt')
+str_in(output+err, 'Extracting file0_hard.txt')
+str_in(output+err, 'Extracting file0_soft.txt')
+str_in(output+err, 'Extracting file_empty.txt')
+str_in(output+err, 'Extracting dir/file1.txt')
+str_in(output+err, 'Extracting empty_dir')
+str_in(output+err, 'Extracting dir2/file2.txt')
+str_in(output+err, 'Extracting file3.txt')
+str_in(output+err, 'Extracting file4.txt')
+str_in(output+err, 'Extracting file5.txt')
+str_not_in(output+err, 'ERROR')
+str_not_in(output+err, 'Not extracting')
+
+print('Testing the extract functionality again, nothing should happen')
+os.chdir('zstash_test')
+cmd = 'zstash extract --hpss={}'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+os.chdir('../')
+str_in(output+err, 'Not extracting file0.txt')
+str_in(output+err, 'Not extracting file0_hard.txt')
+# It's okay to extract the symlinks.
+str_not_in(output+err, 'Not extracting file0_soft.txt')
+str_in(output+err, 'Not extracting file_empty.txt')
+str_in(output+err, 'Not extracting dir/file1.txt')
+# It's okay to extract empty dirs.
+str_not_in(output+err, 'Not extracting empty_dir')
+str_in(output+err, 'Not extracting dir2/file2.txt')
+str_in(output+err, 'Not extracting file3.txt')
+str_in(output+err, 'Not extracting file4.txt')
+str_in(output+err, 'Not extracting file5.txt')
+str_not_in(output+err, 'ERROR')
+
+
+print('Deleting the extracted files and doing it again in parallel.')
+shutil.rmtree('zstash_test')
+os.mkdir('zstash_test')
+os.chdir('zstash_test')
+cmd = 'zstash extract --hpss={} --workers=3'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+os.chdir('../')
 str_in(output+err, 'Transferring from HPSS')
 str_in(output+err, 'Extracting file0.txt')
 str_in(output+err, 'Extracting file0_hard.txt')
@@ -171,19 +261,93 @@ def stop():
 str_in(output+err, 'Extracting dir/file1.txt')
 str_in(output+err, 'Extracting empty_dir')
 str_in(output+err, 'Extracting dir2/file2.txt')
+str_in(output+err, 'Extracting file3.txt')
+str_in(output+err, 'Extracting file4.txt')
+str_in(output+err, 'Extracting file5.txt')
 str_not_in(output+err, 'ERROR')
+str_not_in(output+err, 'Not extracting')
+# Checking that the printing was done in order.
+tar_order = []
+console_output = output+err
+for word in console_output.replace('\n', ' ').split(' '):
+    if '.tar' in word:
+        word = word.replace('zstash/', '')
+        tar_order.append(word)
+if tar_order != sorted(tar_order):
+    print('*'*40)
+    print('The tars were printed in this order: {}'.format(tar_order))
+    print('When it should have been in this order: {}'.format(sorted(tar_order)))
+    print('*'*40)
+    stop()
+
 
-print('Running update on the newly extracted directory, nothing should happen')
+print('Checking the files again in parallel.')
 os.chdir('zstash_test')
-cmd = 'zstash update --hpss={}'.format(HPSS_PATH)
+cmd = 'zstash check --hpss={} --workers=3'.format(HPSS_PATH)
 output, err = run_cmd(cmd)
 os.chdir('../')
-str_in(output+err, 'Nothing to update')
+str_in(output+err, 'Checking file0.txt')
+str_in(output+err, 'Checking file0_hard.txt')
+str_in(output+err, 'Checking file0_soft.txt')
+str_in(output+err, 'Checking file_empty.txt')
+str_in(output+err, 'Checking dir/file1.txt')
+str_in(output+err, 'Checking empty_dir')
+str_in(output+err, 'Checking dir2/file2.txt')
+str_in(output+err, 'Checking file3.txt')
+str_in(output+err, 'Checking file4.txt')
+str_in(output+err, 'Checking file5.txt')
 str_not_in(output+err, 'ERROR')
 
+
+print('Causing MD5 mismatch errors and checking the files.')
+os.chdir('zstash_test')
+shutil.copy('zstash/index.db', 'zstash/index_old.db')
+print('Messing up the MD5 of all of the files with an even id.')
+cmd = ['sqlite3', 'zstash/index.db', 'UPDATE files SET md5 = 0 WHERE id % 2 = 0;']
+run_cmd(cmd)
+cmd = 'zstash check --hpss={}'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+str_in(output+err, 'md5 mismatch for: dir/file1.txt')
+str_in(output+err, 'md5 mismatch for: file3.txt')
+str_in(output+err, 'md5 mismatch for: file3.txt')
+str_in(output+err, 'ERROR: 000001.tar')
+str_in(output+err, 'ERROR: 000004.tar')
+str_in(output+err, 'ERROR: 000002.tar')
+str_not_in(output+err, 'ERROR: 000000.tar')
+str_not_in(output+err, 'ERROR: 000003.tar')
+str_not_in(output+err, 'ERROR: 000005.tar')
+# Put the original index.db back.
+os.remove('zstash/index.db')
+shutil.copy('zstash/index_old.db', 'zstash/index.db')
+os.chdir('../')
+
+
+print('Causing MD5 mismatch errors and checking the files in parallel.')
+os.chdir('zstash_test')
+shutil.copy('zstash/index.db', 'zstash/index_old.db')
+print('Messing up the MD5 of all of the files with an even id.')
+cmd = ['sqlite3', 'zstash/index.db', 'UPDATE files SET md5 = 0 WHERE id % 2 = 0;']
+run_cmd(cmd)
+cmd = 'zstash check --hpss={} --workers=3'.format(HPSS_PATH)
+output, err = run_cmd(cmd)
+str_in(output+err, 'md5 mismatch for: dir/file1.txt')
+str_in(output+err, 'md5 mismatch for: file3.txt')
+str_in(output+err, 'md5 mismatch for: file3.txt')
+str_in(output+err, 'ERROR: 000001.tar')
+str_in(output+err, 'ERROR: 000004.tar')
+str_in(output+err, 'ERROR: 000002.tar')
+str_not_in(output+err, 'ERROR: 000000.tar')
+str_not_in(output+err, 'ERROR: 000003.tar')
+str_not_in(output+err, 'ERROR: 000005.tar')
+# Put the original index.db back.
+os.remove('zstash/index.db')
+shutil.copy('zstash/index_old.db', 'zstash/index.db')
+os.chdir('../')
+
+
 print('Verifying the data from database with the actual files')
 # Checksums from HPSS
-cmd = ['sqlite3', 'zstash_test/zstash/index.db', 'select md5, name from files;']
+cmd = ['sqlite3', 'zstash_test/zstash/index.db', 'SELECT md5, name FROM files;']
 output_hpss, err_hpss = run_cmd(cmd)
 hpss_dict = {}
 
@@ -210,6 +374,7 @@ def stop():
 for k in local_dict:
     print('{}|{}|{}'.format(k, hpss_dict[k], local_dict[k]))
 
+
 cleanup()
 print('*'*40)
 print('All of the tests passed! :)')

zstash/__init__.py

@@ -1,2 +1,2 @@
-__version__ = 'v0.2.0'
+__version__ = 'v0.3.0'
 

zstash/check.py

@@ -1,5 +1,7 @@
+from __future__ import print_function, absolute_import
+
 import logging
-import extract
+from . import extract
 
 def check():
     """

zstash/chgrp.py

@@ -1,6 +1,8 @@
+from __future__ import print_function, absolute_import
+
 import sys
 import argparse
-from hpss import hpss_chgrp
+from .hpss import hpss_chgrp
 
 def chgrp():
 

zstash/create.py

@@ -1,3 +1,5 @@
+from __future__ import print_function, absolute_import
+
 import argparse
 import errno
 import logging
@@ -6,12 +8,10 @@
 import sys
 import sqlite3
 import tarfile
-
 from subprocess import Popen, PIPE
-
-from hpss import hpss_put
-from utils import addfiles, excludeFiles
-from settings import config, CACHE, BLOCK_SIZE, DB_FILENAME
+from .hpss import hpss_put
+from .utils import addfiles, excludeFiles
+from .settings import config, CACHE, BLOCK_SIZE, DB_FILENAME
 
 
 def create():