Merge pull request #64 from HPCNow/main

froster v0.12.22

Author: macheitor

froster/froster.py

@@ -91,10 +91,7 @@ def __init__(self):
             self.home_dir = os.path.expanduser('~')
 
             # Froster's home directory
-            self.froster_dir = os.path.join(sys.prefix)
-
-            # Froster's binary directory
-            self.bin_dir = os.path.join(self.froster_dir, 'bin')
+            self.froster_dir = os.path.dirname(os.path.realpath(shutil.which('froster')))
 
             # Froster's data directory
             xdg_data_home = os.environ.get('XDG_DATA_HOME')
@@ -3080,13 +3077,14 @@ def _index_locally(self, folder):
                             f'    ...folder already indexed at {folder_hotspot}. Use "-f" or "--force" flag to force indexing.\n')
                         return
 
+
             # Run pwalk on given folder
             with tempfile.NamedTemporaryFile() as pwalk_output:
                 with tempfile.NamedTemporaryFile() as pwalk_output_folders:
                     with tempfile.NamedTemporaryFile() as pwalk_output_folders_converted:
 
                         # Build the pwalk command
-                        pwalk_bin = os.path.join(sys.prefix, 'bin', 'pwalk')
+                        pwalk_bin = os.path.join(self.cfg.froster_dir, 'pwalk')
                         pwalkcmd = f'{pwalk_bin} --NoSnap --one-file-system --header'
                         mycmd = f'{pwalkcmd} "{folder}" > {pwalk_output.name}'
 
@@ -4285,6 +4283,8 @@ def _delete_locally(self, folder_to_delete):
                     f"Archive aws profile: {archived_folder_info['profile']}\n")
                 rme.write(f"Archiver user: {archived_folder_info['user']}\n")
                 rme.write(f'Archiver email: {self.cfg.email}\n')
+                rme.write(
+                    f'froster-archives.json: {self.archive_json}\n')
                 rme.write(
                     f'Archive tool: https://github.com/dirkpetersen/froster\n')
                 rme.write(
@@ -5187,7 +5187,7 @@ def __init__(self, args: argparse.Namespace, cfg: ConfigManager):
             self.cfg = cfg
 
             # Set the Rclone executable path
-            self.rc = os.path.join(sys.prefix, 'bin', 'rclone')
+            self.rc = os.path.join(self.cfg.froster_dir, 'rclone')
 
             # Set the Rclone environment variables
             # Note: Keys are set in the AWS Boto __init__ function
@@ -6052,13 +6052,15 @@ def print_version(self):
     def print_info(self):
         '''Print froster info'''
 
+        froster_dir = os.path.dirname(os.path.realpath(shutil.which('froster')))
+
         log(
             f'froster v{pkg_resources.get_distribution("froster").version}\n')
         log(f'Tools version:')
         log(f'    python v{platform.python_version()}')
-        log('    pwalk ', 'v'+subprocess.run([os.path.join(sys.prefix, 'bin', 'pwalk'), '--version'],
+        log('    pwalk ', 'v'+subprocess.run([os.path.join(froster_dir, 'pwalk'), '--version'],
                                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True).stderr.split('\n')[0].split()[2])
-        log('   ', subprocess.run([os.path.join(sys.prefix, 'bin', 'rclone'), '--version'],
+        log('   ', subprocess.run([os.path.join(froster_dir, 'rclone'), '--version'],
                                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True).stdout.split('\n')[0])
 
         log(textwrap.dedent(f'''
@@ -6109,11 +6111,6 @@ def subcmd_config(self, cfg: ConfigManager, aws: AWSBoto):
             if self.args.print:
                 return cfg.print_config()
 
-            # if self.args.monitor:
-            #     froster_binary = os.path.join(cfg.bin_dir, 'froster')
-            #     return cfg.add_systemd_cron_job(
-            #         f'{froster_binary} restore --monitor', '30')
-
             log(f'\n*****************************')
             log(f'*** FROSTER CONFIGURATION ***')
             log(f'*****************************\n')
@@ -7065,7 +7062,7 @@ def main():
             cmd.print_help()
 
         # Check if there are updates on froster every X days
-        if cfg.check_update() and args.subcmd not in ['update', 'upd']:
+        if cfg.configuration_done and cfg.check_update() and args.subcmd not in ['update', 'upd']:
             cmd.subcmd_update(mute_no_update=True)
 
         # Close the AWS session

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "froster"
-version = "0.12.21"
+version = "0.12.22"
 description = "Froster is a tool for easy data transfer between local file systems and AWS S3 storage."
 authors = ["Victor Machado <[email protected]>"]
 readme = "README.md"

tests/config.py

@@ -1,8 +1,13 @@
 # Variables
 import os
+import random
+import string
 import tempfile
 
 
+def random_string(length=4):
+    return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length))
+
 NAME = "Test-Bob"
 NAME_2 = "Test-Alice"
 EMAIL = "[email protected]"
@@ -23,11 +28,11 @@
 NIH_SECTION = 'NIH'
 S3_SECTION = 'S3'
 
-S3_BUCKET_NAME_CONFIG = 'froster-unittest-config'
-S3_BUCKET_NAME_CONFIG_2 = 'froster-unittest-config-2'
+S3_BUCKET_NAME_CONFIG = 'froster-unittest-config-' + random_string(4)
+S3_BUCKET_NAME_CONFIG_2 = 'froster-unittest-config-' + random_string(4)
 
-S3_BUCKET_NAME_INDEX = 'froster-unittest-index'
-S3_BUCKET_NAME_INDEX_2 = 'froster-unittest-index-2'
+S3_BUCKET_NAME_INDEX = 'froster-unittest-index-' + random_string(4)
+S3_BUCKET_NAME_INDEX_2 = 'froster-unittest-index-' + random_string(4)
 
 S3_ARCHIVE_DIR = 'froster_bob'
 S3_ARCHIVE_DIR_2 = 'froster_alice'

tests/generate_dummy_data.py

@@ -44,12 +44,11 @@ def generate_test_data():
 
     # Create files in the base directory
     for dir in [base_dir, subdir1, subdir2]:
-        create_file('small_1_' + random_string(), 1, dir, '1 days ago')
-        create_file('small_2_' + random_string(), 1, dir, '10 days ago')
-        create_file('medium_1_' + random_string(), 1100, dir, '100 days ago')
-        create_file('medium_2_' + random_string(), 1100, dir, '1000 days ago')
-        create_file('large_1_' + random_string(), 1100000, dir, '10 days ago')
-        create_file('large_2_' + random_string(), 1100000, dir, '1000 days ago')
+        create_file('small_1_' + random_string(), 1, dir, '1 days ago') # 1B
+        create_file('small_2_' + random_string(), 1, dir, '10 days ago') # 1B
+        create_file('medium_1_' + random_string(), 1100, dir, '100 days ago') # 1.1KB
+        create_file('medium_2_' + random_string(), 1100, dir, '1000 days ago') # 1.1KB
+        create_file('large_' + random_string(), 1100000000, dir, '10 days ago') # 1.1GB
 
     return base_dir