zstash update refactored

Author: forsyth2

zstash/create.py

@@ -80,7 +80,7 @@ def create():
 
     # Transfer to HPSS. Always keep a local copy of the database.
     logger.debug(f"{ts_utc()}: calling hpss_put() for {command_info.get_db_name()}")
-    hpss_put(command_info, command_info.get_db_name(), is_index=True)
+    hpss_put(command_info, command_info.get_db_name())
 
     if command_info.globus_info:
         logger.debug(f"{ts_utc()}: calling globus_finalize()")
@@ -93,7 +93,7 @@ def create():
             logger.error(f"Failed to archive {file_path}")
 
 
-def setup_create(ci: CommandInfo) -> argparse.Namespace:
+def setup_create(command_info: CommandInfo) -> argparse.Namespace:
     # Parser
     parser: argparse.ArgumentParser = argparse.ArgumentParser(
         usage="zstash create [<args>] path", description="Create a new zstash archive"
@@ -165,11 +165,11 @@ def setup_create(ci: CommandInfo) -> argparse.Namespace:
         logger.setLevel(logging.DEBUG)
 
     if args.cache:
-        ci.cache_dir = args.cache
-    ci.keep = args.keep
-    ci.set_dir_to_archive(args.path)
-    ci.set_maxsize(args.maxsize)
-    ci.set_hpss_parameters(args.hpss)
+        command_info.cache_dir = args.cache
+    command_info.keep = args.keep
+    command_info.set_dir_to_archive(args.path)
+    command_info.set_maxsize(args.maxsize)
+    command_info.set_hpss_parameters(args.hpss)
 
     return args
 

zstash/globus.py

@@ -72,6 +72,8 @@ def globus_activate(globus_info: GlobusInfo):
     If the ini file does not exist, create an ini file with empty values,
     and try to find the local endpoint UUID based on the FQDN
     """
+    if not globus_info:
+        raise ValueError("globus_info is undefined")
     globus_info.remote_endpoint = globus_info.url.netloc
 
     ini_path = os.path.expanduser("~/.zstash.ini")
@@ -362,6 +364,8 @@ def globus_wait(globus_info: GlobusInfo, alternative_task_id=None):
 
 
 def globus_finalize(globus_info: GlobusInfo, non_blocking: bool = False):
+    if not globus_info:
+        raise ValueError("globus_info is undefined")
     last_task_id = None
 
     if globus_info.transfer_data:

zstash/hpss.py

@@ -16,7 +16,6 @@ def hpss_transfer(
     file_path: str,
     transfer_type: str,
     non_blocking: bool = False,
-    is_index: bool = False,
 ):
 
     logger.info(
@@ -67,21 +66,18 @@ def hpss_transfer(
         else:
             raise ValueError("Invalid transfer_type={}".format(transfer_type))
         logger.info(f"Transferring file {transfer_word} HPSS: {file_path}")
-        scheme: str
-        endpoint: str
-        path: str
-        name: str
 
         url = urlparse(command_info.hpss_path)
-        scheme = url.scheme
-        endpoint = url.netloc
+        endpoint: str = url.netloc
         url_path = url.path
 
         command_info.curr_transfers.append(file_path)
         # TODO: Expected output for tests needs to be changed if we uncomment this:
         # logger.debug(
         #     f"{ts_utc()}: curr_transfers has been appended to, is now {command_info.curr_transfers}"
         # )
+        path: str
+        name: str
         path, name = os.path.split(file_path)
 
         # Need to be in local directory for `hsi` to work
@@ -143,19 +139,18 @@ def hpss_put(
     command_info: CommandInfo,
     file_path: str,
     non_blocking: bool = False,
-    is_index=False,
 ):
     """
     Put a file to the HPSS archive.
     """
-    hpss_transfer(command_info, file_path, "put", non_blocking, is_index)
+    hpss_transfer(command_info, file_path, "put", non_blocking)
 
 
-def hpss_get(hpss: str, file_path: str, cache: str):
+def hpss_get(command_info: CommandInfo, file_path: str):
     """
     Get a file from the HPSS archive.
     """
-    hpss_transfer(hpss, file_path, "get", cache, False)
+    hpss_transfer(command_info, file_path, "get")
 
 
 def hpss_chgrp(hpss: str, group: str, recurse: bool = False):

zstash/update.py

@@ -13,48 +13,44 @@
 from .hpss import hpss_get, hpss_put
 from .hpss_utils import add_files
 from .settings import (
-    DEFAULT_CACHE,
     TIME_TOL,
     FilesRow,
     TupleFilesRow,
     config,
     get_db_filename,
     logger,
 )
-from .utils import get_files_to_archive, update_config
+from .utils import CommandInfo, HPSSType, get_files_to_archive
 
 
 def update():
+    command_info = CommandInfo("update")
+    args: argparse.Namespace = setup_update(command_info)
 
-    args: argparse.Namespace
-    cache: str
-    args, cache = setup_update()
-
-    result: Optional[List[str]] = update_database(args, cache)
-
-    if result is None:
+    failures: Optional[List[str]] = update_database(command_info, args)
+    if failures is None:
         # There was either nothing to update or `--dry-run` was set.
         return
-    else:
-        failures = result
 
     # Transfer to HPSS. Always keep a local copy of the database.
-    if config.hpss is not None:
-        hpss = config.hpss
+    if command_info.config.hpss is not None:
+        hpss = command_info.config.hpss
     else:
-        raise TypeError("Invalid config.hpss={}".format(config.hpss))
-    hpss_put(hpss, get_db_filename(cache), cache, keep=args.keep, is_index=True)
+        raise TypeError(f"Invalid config.hpss={command_info.config.hpss}")
+    hpss_put(hpss, command_info.get_db_name(), command_info.cache_dir, keep=command_info.keep)
 
-    globus_finalize(non_blocking=args.non_blocking)
+    if command_info.hpss_type == HPSSType.GLOBUS:
+        globus_finalize(command_info.globus_info, non_blocking=args.non_blocking)
 
     # List failures
     if len(failures) > 0:
         logger.warning("Some files could not be archived")
         for file_path in failures:
-            logger.error("Archiving {}".format(file_path))
+            logger.error(f"Archiving {file_path}")
+    # TODO: (A) Continue refactor from here
 
 
-def setup_update() -> Tuple[argparse.Namespace, str]:
+def setup_update(command_info: CommandInfo) -> argparse.Namespace:
     # Parser
     parser: argparse.ArgumentParser = argparse.ArgumentParser(
         usage="zstash update [<args>]", description="Update an existing zstash archive"
@@ -118,80 +114,69 @@ def setup_update() -> Tuple[argparse.Namespace, str]:
     if (not args.hpss) or (args.hpss.lower() == "none"):
         args.hpss = "none"
         args.keep = True
-
-    # Copy configuration
-    # config.path = os.path.abspath(args.path)
-    config.hpss = args.hpss
-    config.maxsize = int(1024 * 1024 * 1024 * args.maxsize)
-
-    cache: str
-    if args.cache:
-        cache = args.cache
-    else:
-        cache = DEFAULT_CACHE
     if args.verbose:
         logger.setLevel(logging.DEBUG)
 
-    return args, cache
+    if args.cache:
+        command_info.cache_dir = args.cache
+    command_info.keep = args.keep
+    command_info.set_dir_to_archive(os.getcwd())
+    command_info.set_maxsize(args.maxsize)
+    command_info.set_hpss_parameters(args.hpss)
+
+    return args
 
 
 # C901 'update_database' is too complex (20)
 def update_database(  # noqa: C901
-    args: argparse.Namespace, cache: str
+    command_info: CommandInfo, args: argparse.Namespace
 ) -> Optional[List[str]]:
     # Open database
     logger.debug("Opening index database")
-    if not os.path.exists(get_db_filename(cache)):
+    if not os.path.exists(command_info.get_db_name()):
         # The database file doesn't exist in the cache.
         # We need to retrieve it from HPSS
-        if args.hpss is not None:
-            config.hpss = args.hpss
-            if config.hpss is not None:
-                hpss: str = config.hpss
-            else:
-                raise TypeError("Invalid config.hpss={}".format(config.hpss))
-            globus_activate(hpss)
-            hpss_get(hpss, get_db_filename(cache), cache)
+        if command_info.hpss_type != HPSSType.NO_HPSS:
+            command_info.update_config()
+            if command_info.hpss_type == HPSSType.GLOBUS:
+                globus_activate(command_info.globus_info)
+            hpss_get(command_info, command_info.get_db_name())
         else:
+            # NOTE: while --hpss is required in `create`, it is optional in `update`!
+            # If --hpss is not provided, we assume it is 'none' => HPSSType.NO_HPSS
             error_str: str = (
-                "--hpss argument is required when local copy of database is unavailable"
+                "--hpss argument (!= none) is required when local copy of database is unavailable"
             )
             logger.error(error_str)
             raise ValueError(error_str)
 
     con: sqlite3.Connection = sqlite3.connect(
-        get_db_filename(cache), detect_types=sqlite3.PARSE_DECLTYPES
+        command_info.get_db_name(), detect_types=sqlite3.PARSE_DECLTYPES
     )
     cur: sqlite3.Cursor = con.cursor()
 
-    update_config(cur)
+    command_info.update_config_using_db(cur)
 
-    if config.maxsize is not None:
-        maxsize = config.maxsize
-    else:
-        raise TypeError("Invalid config.maxsize={}".format(config.maxsize))
-    config.maxsize = int(maxsize)
-
-    keep: bool
-    # The command line arg should always have precedence
-    if args.hpss == "none":
-        # If no HPSS is available, always keep the files.
-        keep = True
+    if command_info.config.maxsize is not None:
+        command_info.maxsize = command_info.config.maxsize
     else:
-        # If HPSS is used, let the user specify whether or not to keep the files.
-        keep = args.keep
+        raise TypeError(f"Invalid config.maxsize={command_info.config.maxsize}")
+    command_info.config.maxsize = int(command_info.maxsize)
+
+    if command_info.hpss_type == HPSSType.NO_HPSS:
+        # If not using HPSS, always keep the files.
+        command_info.keep = True
+    # else: keep command_info.keep set to args.keep
 
-    if args.hpss is not None:
-        config.hpss = args.hpss
 
     # Start doing actual work
     logger.debug("Running zstash update")
-    logger.debug("Local path : {}".format(config.path))
-    logger.debug("HPSS path  : {}".format(config.hpss))
-    logger.debug("Max size  : {}".format(maxsize))
-    logger.debug("Keep local tar files  : {}".format(keep))
+    logger.debug(f"Local path : {command_info.config.path}")
+    logger.debug(f"HPSS path  : {command_info.config.hpss}")
+    logger.debug(f"Max size  : {command_info.maxsize}")
+    logger.debug(f"Keep local tar files  : {command_info.keep}")
 
-    files: List[str] = get_files_to_archive(cache, args.include, args.exclude)
+    files: List[str] = get_files_to_archive(command_info.get_db_name(), args.include, args.exclude)
 
     # Eliminate files that are already archived and up to date
     newfiles: List[str] = []
@@ -261,8 +246,8 @@ def update_database(  # noqa: C901
                 con,
                 itar,
                 newfiles,
-                cache,
-                keep,
+                command_info.get_db_name(),
+                command_info.keep,
                 args.follow_symlinks,
                 non_blocking=args.non_blocking,
             )
@@ -275,8 +260,8 @@ def update_database(  # noqa: C901
             con,
             itar,
             newfiles,
-            cache,
-            keep,
+            command_info.get_db_name(),
+            command_info.keep,
             args.follow_symlinks,
             non_blocking=args.non_blocking,
         )

zstash/utils.py

@@ -97,6 +97,21 @@ def update_config(self):
         self.config.hpss = self.hpss_path
         self.config.maxsize = self.maxsize
 
+    def update_config_using_db(self, cur: sqlite3.Cursor):
+        # Retrieve some configuration settings from database
+        # Loop through all attributes of config.
+        for attr in dir(self.config):
+            value: Any = getattr(self.config, attr)
+            if not callable(value) and not attr.startswith("__"):
+                # config.{attr} is not a function.
+                # The attribute name does not start with "__"
+                # Get the value (column 2) for attribute `attr` (column 1)
+                # i.e., for the row where column 1 is the attribute, get the value from column 2
+                cur.execute("select value from config where arg=?", (attr,))
+                value = cur.fetchone()[0]
+                # Update config with the new attribute-value pair
+                setattr(self.config, attr, value)
+
     def get_db_name(self) -> str:
         return os.path.join(self.cache_dir, "index.db")
 
@@ -212,22 +227,6 @@ def get_files_to_archive(cache: str, include: str, exclude: str) -> List[str]:
     return files
 
 
-def update_config(cur: sqlite3.Cursor):
-    # Retrieve some configuration settings from database
-    # Loop through all attributes of config.
-    for attr in dir(config):
-        value: Any = getattr(config, attr)
-        if not callable(value) and not attr.startswith("__"):
-            # config.{attr} is not a function.
-            # The attribute name does not start with "__"
-            # Get the value (column 2) for attribute `attr` (column 1)
-            # i.e., for the row where column 1 is the attribute, get the value from column 2
-            cur.execute("select value from config where arg=?", (attr,))
-            value = cur.fetchone()[0]
-            # Update config with the new attribute-value pair
-            setattr(config, attr, value)
-
-
 def create_tars_table(cur: sqlite3.Cursor, con: sqlite3.Connection):
     # Create 'tars' table
     cur.execute(