Fix pre-commit errors

Author: forsyth2

zstash/create.py

@@ -8,8 +8,6 @@
 import sys
 from typing import Any, List
 
-from six.moves.urllib.parse import urlparse
-
 from .globus import globus_activate, globus_finalize
 from .hpss import hpss_put
 from .hpss_utils import add_files
@@ -38,20 +36,28 @@ def create():
 
     # Make sure input path exists and is a directory
     logger.debug("Making sure input path exists and is a directory")
+    if not command_info.config.path:
+        raise ValueError("config.path is undefined")
     if not os.path.isdir(command_info.config.path):
         # Input path is not a directory
-        input_path_error_str: str = f"Input path should be a directory: {command_info.config.path}"
+        input_path_error_str: str = (
+            f"Input path should be a directory: {command_info.config.path}"
+        )
         logger.error(input_path_error_str)
         raise NotADirectoryError(input_path_error_str)
 
-    if command_info.hpss_type == HPSSType.GLOBUS:
+    if command_info.globus_info:
         # identify globus endpoints
         logger.debug(f"{ts_utc()}: Calling globus_activate")
         globus_activate(command_info.globus_info)
     elif command_info.hpss_type == HPSSType.SAME_MACHINE_HPSS:
-        logger.debug(f"{ts_utc()}: Creating target HPSS directory {command_info.config.hpss}")
+        logger.debug(
+            f"{ts_utc()}: Creating target HPSS directory {command_info.config.hpss}"
+        )
         mkdir_command: str = f"hsi -q mkdir -p {command_info.config.hpss}"
-        mkdir_error_str: str = f"Could not create HPSS directory: {command_info.config.hpss}"
+        mkdir_error_str: str = (
+            f"Could not create HPSS directory: {command_info.config.hpss}"
+        )
         run_command(mkdir_command, mkdir_error_str)
 
         # Make sure it is exists and is empty
@@ -231,19 +237,21 @@ def create_database(command_info: CommandInfo, args: argparse.Namespace) -> List
             cur.execute("insert into config values (?,?)", (attr, value))
     con.commit()
 
-    files: List[str] = get_files_to_archive(command_info.cache_dir, args.include, args.exclude)
+    files: List[str] = get_files_to_archive(
+        command_info.cache_dir, args.include, args.exclude
+    )
 
     failures: List[str]
     if args.follow_symlinks:
         try:
             # Add files to archive
             failures = add_files(
+                command_info,
                 cur,
                 con,
                 -1,
                 files,
                 command_info.cache_dir,
-                args.keep,
                 args.follow_symlinks,
                 skip_tars_md5=args.no_tars_md5,
                 non_blocking=args.non_blocking,
@@ -253,12 +261,12 @@ def create_database(command_info: CommandInfo, args: argparse.Namespace) -> List
     else:
         # Add files to archive
         failures = add_files(
+            command_info,
             cur,
             con,
             -1,
             files,
             command_info.cache_dir,
-            args.keep,
             args.follow_symlinks,
             skip_tars_md5=args.no_tars_md5,
             non_blocking=args.non_blocking,

zstash/extract.py

@@ -20,14 +20,7 @@
 
 from . import parallel
 from .hpss import hpss_get
-from .settings import (
-    BLOCK_SIZE,
-    DEFAULT_CACHE,
-    TIME_TOL,
-    FilesRow,
-    TupleFilesRow,
-    logger,
-)
+from .settings import BLOCK_SIZE, TIME_TOL, FilesRow, TupleFilesRow, logger
 from .utils import CommandInfo, HPSSType, tars_table_exists
 
 
@@ -60,7 +53,7 @@ def extract(keep_files: bool = True):
         )
 
 
-def setup_extract(command_info: CommandInfo) -> Tuple[argparse.Namespace, str]:
+def setup_extract(command_info: CommandInfo) -> argparse.Namespace:
     parser: argparse.ArgumentParser = argparse.ArgumentParser(
         usage="zstash extract [<args>] [files]",
         description="Extract files from existing archive",
@@ -155,16 +148,16 @@ def parse_tars_option(tars: str, first_tar: str, last_tar: str) -> List[str]:
     return tar_list
 
 
-def extract_database(command_info: CommandInfo,
-    args: argparse.Namespace, keep_files: bool
+def extract_database(
+    command_info: CommandInfo, args: argparse.Namespace, keep_files: bool
 ) -> List[FilesRow]:
 
     # Open database
     logger.debug("Opening index database")
     if not os.path.exists(command_info.get_db_name()):
         # Will need to retrieve from HPSS
         if command_info.hpss_type != HPSSType.UNDEFINED:
-            hpss_get(command_info.config.hpss, command_info.get_db_name())
+            hpss_get(command_info, command_info.get_db_name())
         else:
             error_str: str = (
                 "--hpss argument is required when local copy of database is unavailable"
@@ -261,10 +254,15 @@ def extract_database(command_info: CommandInfo,
     if args.workers > 1:
         logger.debug("Running zstash {} with multiprocessing".format(cmd))
         failures = multiprocess_extract(
-            args.workers, matches, keep_files, command_info.keep, command_info.cache_dir, cur, args
+            args.workers,
+            command_info,
+            matches,
+            keep_files,
+            cur,
+            args,
         )
     else:
-        failures = extractFiles(matches, keep_files, command_info.keep, command_info.cache_dir, cur, args)
+        failures = extractFiles(command_info, matches, keep_files, cur, args)
 
     # Close database
     logger.debug("Closing index database")
@@ -275,10 +273,9 @@ def extract_database(command_info: CommandInfo,
 
 def multiprocess_extract(
     num_workers: int,
+    command_info: CommandInfo,
     matches: List[FilesRow],
     keep_files: bool,
-    keep_tars: Optional[bool],
-    cache: str,
     cur: sqlite3.Cursor,
     args: argparse.Namespace,
 ) -> List[FilesRow]:
@@ -353,7 +350,7 @@ def multiprocess_extract(
         )
         process: multiprocessing.Process = multiprocessing.Process(
             target=extractFiles,
-            args=(matches, keep_files, keep_tars, cache, cur, args, worker),
+            args=(command_info, matches, keep_files, cur, args, worker),
             daemon=True,
         )
         process.start()
@@ -397,10 +394,9 @@ def check_sizes_match(cur, tfname):
 
 # FIXME: C901 'extractFiles' is too complex (33)
 def extractFiles(  # noqa: C901
+    command_info: CommandInfo,
     files: List[FilesRow],
     keep_files: bool,
-    keep_tars: Optional[bool],
-    cache: str,
     cur: sqlite3.Cursor,
     args: argparse.Namespace,
     multiprocess_worker: Optional[parallel.ExtractWorker] = None,
@@ -413,14 +409,15 @@ def extractFiles(  # noqa: C901
     This is used for when checking if the files in an HPSS
     repository are valid.
 
-    If keep_tars is True, the tar archives that are downloaded are kept,
+    If command_info.keep is True, the tar archives that are downloaded are kept,
     even after the program has terminated. Otherwise, they are deleted.
 
     If running in parallel, then multiprocess_worker is the Worker
     that called this function.
     We need a reference to it so we can signal it to print
     the contents of what's in its print queue.
     """
+
     failures: List[FilesRow] = []
     tfname: str
     newtar: bool = True
@@ -442,17 +439,13 @@ def extractFiles(  # noqa: C901
         # Open new tar archive
         if newtar:
             newtar = False
-            tfname = os.path.join(cache, files_row.tar)
+            tfname = os.path.join(command_info.cache_dir, files_row.tar)
             # Everytime we're extracting a new tar, if running in parallel,
             # let the process know.
             # This is to synchronize the print statements.
             if multiprocess_worker:
                 multiprocess_worker.set_curr_tar(files_row.tar)
 
-            if config.hpss is not None:
-                hpss: str = config.hpss
-            else:
-                raise TypeError("Invalid config.hpss={}".format(config.hpss))
             tries: int = args.retries + 1
             # Set to True to test the `--retries` option with a forced failure.
             # Then run `python -m unittest tests.test_extract.TestExtract.testExtractRetries`
@@ -471,7 +464,7 @@ def extractFiles(  # noqa: C901
                         test_retry = False
                         raise RuntimeError
                     if do_retrieve:
-                        hpss_get(hpss, tfname, cache)
+                        hpss_get(command_info, tfname)
                         if not check_sizes_match(cur, tfname):
                             raise RuntimeError(
                                 f"{tfname} size does not match expected size."
@@ -612,7 +605,7 @@ def extractFiles(  # noqa: C901
             newtar = True
 
             # Delete this tar if the corresponding command-line arg was used.
-            if not keep_tars:
+            if not command_info.keep:
                 if tfname is not None:
                     os.remove(tfname)
                 else: