zstash ls refactored

Author: forsyth2

zstash/create.py

@@ -31,16 +31,16 @@ def create():
 
     # Start doing actual work
     logger.debug(f"{ts_utc()}: Running zstash create")
-    logger.debug(f"Local path: {command_info.dir_to_archive_absolute}")
-    logger.debug(f"HPSS path: {command_info.hpss_path}")
-    logger.debug(f"Max size: {command_info.maxsize}")
+    logger.debug(f"Local path: {command_info.config.path}")
+    logger.debug(f"HPSS path: {command_info.config.hpss}")
+    logger.debug(f"Max size: {command_info.config.maxsize}")
     logger.debug(f"Keep local tar files: {command_info.keep}")
 
     # Make sure input path exists and is a directory
     logger.debug("Making sure input path exists and is a directory")
-    if not os.path.isdir(command_info.dir_to_archive_absolute):
+    if not os.path.isdir(command_info.config.path):
         # Input path is not a directory
-        input_path_error_str: str = f"Input path should be a directory: {command_info.dir_to_archive_absolute}"
+        input_path_error_str: str = f"Input path should be a directory: {command_info.config.path}"
         logger.error(input_path_error_str)
         raise NotADirectoryError(input_path_error_str)
 
@@ -49,21 +49,21 @@ def create():
         logger.debug(f"{ts_utc()}: Calling globus_activate")
         globus_activate(command_info.globus_info)
     elif command_info.hpss_type == HPSSType.SAME_MACHINE_HPSS:
-        logger.debug(f"{ts_utc()}: Creating target HPSS directory {command_info.hpss_path}")
-        mkdir_command: str = f"hsi -q mkdir -p {command_info.hpss_path}"
-        mkdir_error_str: str = f"Could not create HPSS directory: {command_info.hpss_path}"
+        logger.debug(f"{ts_utc()}: Creating target HPSS directory {command_info.config.hpss}")
+        mkdir_command: str = f"hsi -q mkdir -p {command_info.config.hpss}"
+        mkdir_error_str: str = f"Could not create HPSS directory: {command_info.config.hpss}"
         run_command(mkdir_command, mkdir_error_str)
 
         # Make sure it is exists and is empty
         logger.debug("Making sure target HPSS directory exists and is empty")
 
-        ls_command: str = f'hsi -q "cd {command_info.hpss_path}; ls -l"'
+        ls_command: str = f'hsi -q "cd {command_info.config.hpss}; ls -l"'
         ls_error_str: str = "Target HPSS directory is not empty"
         run_command(ls_command, ls_error_str)
 
     # Create cache directory
     logger.debug(f"{ts_utc()}: Creating local cache directory")
-    os.chdir(command_info.dir_to_archive_absolute)
+    os.chdir(command_info.config.path)
     try:
         os.makedirs(command_info.cache_dir)
     except OSError as exc:
@@ -168,7 +168,7 @@ def setup_create(command_info: CommandInfo) -> argparse.Namespace:
         command_info.cache_dir = args.cache
     command_info.keep = args.keep
     command_info.set_dir_to_archive(args.path)
-    command_info.set_maxsize(args.maxsize)
+    command_info.set_and_scale_maxsize(args.maxsize)
     command_info.set_hpss_parameters(args.hpss)
 
     return args

zstash/ls.py

@@ -9,38 +9,32 @@
 
 from .hpss import hpss_get
 from .settings import (
-    DEFAULT_CACHE,
     FilesRow,
     TarsRow,
     TupleFilesRow,
     TupleTarsRow,
-    config,
-    get_db_filename,
     logger,
 )
-from .utils import tars_table_exists, update_config
+from .utils import CommandInfo, HPSSType, tars_table_exists
 
 
 def ls():
     """
     List all of the files in the HPSS path.
     Supports the '-l' argument for more information.
     """
-
-    args: argparse.Namespace
-    cache: str
-    args, cache = setup_ls()
-
-    matches: List[FilesRow] = ls_database(args, cache)
+    command_info = CommandInfo("ls")
+    args: argparse.Namespace = setup_ls(command_info)
+    matches: List[FilesRow] = ls_database(command_info, args)
 
     print_matches(args, matches)
 
     if args.tars:
-        tar_matches: List[TarsRow] = ls_tars_database(args, cache)
+        tar_matches: List[TarsRow] = ls_tars_database(command_info, args)
         print_matches(args, tar_matches)
 
 
-def setup_ls() -> Tuple[argparse.Namespace, str]:
+def setup_ls(command_info: CommandInfo) -> Tuple[argparse.Namespace, str]:
     parser: argparse.ArgumentParser = argparse.ArgumentParser(
         usage="zstash ls [<args>] [files]",
         description="List the files from an existing archive. If `files` is specified, then only the files specified will be listed. If `hpss=none`, then this will list the directories and files in the current directory excluding the cache.",
@@ -76,33 +70,30 @@ def setup_ls() -> Tuple[argparse.Namespace, str]:
 
     parser.add_argument("files", nargs="*", default=["*"])
     args: argparse.Namespace = parser.parse_args(sys.argv[2:])
-    if args.hpss and args.hpss.lower() == "none":
+
+    if args.hpss and (args.hpss.lower() == "none"):
         args.hpss = "none"
-    cache: str
-    if args.cache:
-        cache = args.cache
-    else:
-        cache = DEFAULT_CACHE
     if args.verbose:
         logger.setLevel(logging.DEBUG)
 
-    return args, cache
+    if args.cache:
+        command_info.cache_dir = args.cache
+    command_info.keep = args.keep
+    command_info.set_dir_to_archive(os.getcwd())
+    command_info.set_hpss_parameters(args.hpss, null_hpss_allowed=True)
+
+    return args
 
 
-def ls_database(args: argparse.Namespace, cache: str) -> List[FilesRow]:
+def ls_database(command_info: CommandInfo, args: argparse.Namespace) -> List[FilesRow]:
     # Open database
     logger.debug("Opening index database")
-    if not os.path.exists(get_db_filename(cache)):
+    if not os.path.exists(command_info.get_db_name()):
         # Will need to retrieve from HPSS
-        if args.hpss is not None:
-            config.hpss = args.hpss
-            if config.hpss is not None:
-                hpss = config.hpss
-            else:
-                raise TypeError("Invalid config.hpss={}".format(config.hpss))
+        if command_info.hpss_type != HPSSType.UNDEFINED:
             try:
                 # Retrieve from HPSS
-                hpss_get(hpss, get_db_filename(cache), cache)
+                hpss_get(command_info, command_info.get_db_name())
             except RuntimeError:
                 raise FileNotFoundError("There was nothing to ls.")
         else:
@@ -113,25 +104,16 @@ def ls_database(args: argparse.Namespace, cache: str) -> List[FilesRow]:
             raise ValueError(error_str)
 
     con: sqlite3.Connection = sqlite3.connect(
-        get_db_filename(cache), detect_types=sqlite3.PARSE_DECLTYPES
+        command_info.get_db_name(), detect_types=sqlite3.PARSE_DECLTYPES
     )
     cur: sqlite3.Cursor = con.cursor()
 
-    update_config(cur)
-
-    if config.maxsize is not None:
-        maxsize: int = config.maxsize
-    else:
-        raise TypeError("Invalid config.maxsize={}".format(config.maxsize))
-    config.maxsize = maxsize
-
-    # The command line arg should always have precedence
-    if args.hpss is not None:
-        config.hpss = args.hpss
+    command_info.update_config_using_db(cur)
+    command_info.validate_maxsize()
 
     # Start doing actual work
     logger.debug("Running zstash ls")
-    logger.debug("HPSS path  : %s" % (config.hpss))
+    logger.debug(f"HPSS path  : {command_info.config.hpss}")
 
     # Find matching files
     matches_: List[TupleFilesRow] = []
@@ -165,9 +147,9 @@ def ls_database(args: argparse.Namespace, cache: str) -> List[FilesRow]:
     return matches
 
 
-def ls_tars_database(args: argparse.Namespace, cache: str) -> List[TarsRow]:
+def ls_tars_database(command_info: CommandInfo, args: argparse.Namespace) -> List[TarsRow]:
     con: sqlite3.Connection = sqlite3.connect(
-        get_db_filename(cache), detect_types=sqlite3.PARSE_DECLTYPES
+        command_info.get_db_name(), detect_types=sqlite3.PARSE_DECLTYPES
     )
     cur: sqlite3.Cursor = con.cursor()
 

zstash/update.py

@@ -47,7 +47,6 @@ def update():
         logger.warning("Some files could not be archived")
         for file_path in failures:
             logger.error(f"Archiving {file_path}")
-    # TODO: (A) Continue refactor from here
 
 
 def setup_update(command_info: CommandInfo) -> argparse.Namespace:
@@ -121,7 +120,7 @@ def setup_update(command_info: CommandInfo) -> argparse.Namespace:
         command_info.cache_dir = args.cache
     command_info.keep = args.keep
     command_info.set_dir_to_archive(os.getcwd())
-    command_info.set_maxsize(args.maxsize)
+    command_info.set_and_scale_maxsize(args.maxsize)
     command_info.set_hpss_parameters(args.hpss)
 
     return args
@@ -156,12 +155,7 @@ def update_database(  # noqa: C901
     cur: sqlite3.Cursor = con.cursor()
 
     command_info.update_config_using_db(cur)
-
-    if command_info.config.maxsize is not None:
-        command_info.maxsize = command_info.config.maxsize
-    else:
-        raise TypeError(f"Invalid config.maxsize={command_info.config.maxsize}")
-    command_info.config.maxsize = int(command_info.maxsize)
+    command_info.validate_maxsize()
 
     if command_info.hpss_type == HPSSType.NO_HPSS:
         # If not using HPSS, always keep the files.
@@ -173,7 +167,7 @@ def update_database(  # noqa: C901
     logger.debug("Running zstash update")
     logger.debug(f"Local path : {command_info.config.path}")
     logger.debug(f"HPSS path  : {command_info.config.hpss}")
-    logger.debug(f"Max size  : {command_info.maxsize}")
+    logger.debug(f"Max size  : {command_info.config.maxsize}")
     logger.debug(f"Keep local tar files  : {command_info.keep}")
 
     files: List[str] = get_files_to_archive(command_info.get_db_name(), args.include, args.exclude)

zstash/utils.py

@@ -17,6 +17,7 @@ class HPSSType(Enum):
     NO_HPSS = 1
     SAME_MACHINE_HPSS = 2
     GLOBUS = 3
+    UNDEFINED = 4
 
 
 class GlobusInfo(object):
@@ -56,12 +57,8 @@ def __init__(self, command_name: str):
         self.curr_transfers = []
 
         # Use set_dir_to_archive
-        self.dir_to_archive_absolute = None
         self.dir_to_archive_relative = None
 
-        # Use set_maxsize
-        self.maxsize = None
-
         # Use set_hpss_parameters
         self.hpss_path = None
         self.hpss_type = None
@@ -70,16 +67,22 @@ def __init__(self, command_name: str):
     def set_dir_to_archive(self, path: str):
         abs_path = os.path.abspath(path)
         if abs_path is not None:
-            self.dir_to_archive_absolute = abs_path
+            self.config.path = abs_path
             self.dir_to_archive_relative = path
         else:
             raise ValueError(f"Invalid path={path}")
 
-    def set_maxsize(self, maxsize):
-        self.maxsize = int(1024 * 1024 * 1024 * maxsize)
+    def set_and_scale_maxsize(self, maxsize):
+        self.config.maxsize = int(1024 * 1024 * 1024 * maxsize)
 
-    def set_hpss_parameters(self, hpss_path: str):
-        self.hpss_path = hpss_path
+    def validate_maxsize(self):
+        if self.config.maxsize is not None:
+            self.config.maxsize = int(self.config.maxsize)
+        else:
+            raise ValueError("config.maxsize is undefined")
+
+    def set_hpss_parameters(self, hpss_path: str, null_hpss_allowed=False):
+        self.config.hpss = hpss_path
         if hpss_path == "none":
             self.hpss_type = HPSSType.NO_HPSS
         elif hpss_path is not None:
@@ -89,13 +92,10 @@ def set_hpss_parameters(self, hpss_path: str):
                 self.globus_info = GlobusInfo(hpss_path)
             else:
                 self.hpss_type = HPSSType.SAME_MACHINE_HPSS
+        elif null_hpss_allowed:
+                self.hpss_type = HPSSType.UNDEFINED
         else:
-            raise ValueError(f"Invalid hpss_path={hpss_path}")
-        
-    def update_config(self):
-        self.config.path = self.dir_to_archive_absolute
-        self.config.hpss = self.hpss_path
-        self.config.maxsize = self.maxsize
+            raise ValueError(f"hpss_path is undefined")
 
     def update_config_using_db(self, cur: sqlite3.Cursor):
         # Retrieve some configuration settings from database