From 1b321bdb118e40b32d63a01be37692afc6b4dc0b Mon Sep 17 00:00:00 2001 From: Victor Machado Date: Thu, 9 May 2024 14:41:41 +0200 Subject: [PATCH 1/5] Prevent user from doing anything if configuration not set --- froster/froster.py | 57 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/froster/froster.py b/froster/froster.py index a6581a0..63cc5e7 100755 --- a/froster/froster.py +++ b/froster/froster.py @@ -88,7 +88,9 @@ def __init__(self): # Initialize the variables that check if specific configuration sections have been initialized self.user_init = False self.aws_init = False + self.nih_init = False self.s3_init = False + self.configuration_done = False # Whoami self.whoami = getpass.getuser() @@ -193,6 +195,9 @@ def __init__(self): # NIH configuration self.is_nih = config.getboolean('NIH', 'is_nih', fallback=None) + # Set nih init flag + self.nih_init = True if self.is_nih is not None else False + # Current S3 Bucket name self.bucket_name = config.get( 'S3', 'bucket_name', fallback=None) @@ -241,6 +246,10 @@ def __init__(self): self.ec2_last_instance = config.get( 'CLOUD', 'ec2_last_instance', fallback=None) + if self.user_init and self.aws_init and self.s3_init and self.nih_init: + self.configuration_done = True + + def __repr__(self): ''' Return a string representation of the object''' @@ -4664,26 +4673,33 @@ def convert_size(self, size_bytes): def _archive_json_add_entry(self, key, value): '''Add a new entry to the archive JSON file''' + try: - # Initialize the data dictionary in case archive_json does not exist - data = {} + # Initialize the data dictionary in case archive_json does not exist + data = {} - # Read the archive JSON file - if os.path.isfile(self.archive_json): - with open(self.archive_json, 'r') as file: - try: - data = json.load(file) - except: - print('Error in Archiver._archive_json_add_entry():') - print(f'Cannot read {self.archive_json}, file corrupt?') - return False + # Read the archive JSON file + if os.path.isfile(self.archive_json): + with open(self.archive_json, 'r') as file: + try: + data = json.load(file) + except: + print('Error in Archiver._archive_json_add_entry():') + print( + f'Cannot read {self.archive_json}, file corrupt?') + return - # Add the new entry to the data dictionary - data[key] = value + # Add the new entry to the data dictionary + data[key] = value - # Write the updated data dictionary to the archive JSON file - with open(self.archive_json, 'w') as file: - json.dump(data, file, indent=4) + # Create the directory for the archive JSON file if it does not exist + os.makedirs(os.path.dirname(self.archive_json), exist_ok=True) + + # Write the updated data dictionary to the archive JSON file + with open(self.archive_json, 'w') as file: + json.dump(data, file, indent=4) + except: + print_error() def _is_folder_archived(self, folder): '''Check if an entry exists in the archive JSON file''' @@ -5221,7 +5237,7 @@ def _run_rclone_command(self, command, background=False): # This is the solution i found to prevent the popen subprocess to throw errors due # our particular usage of rclone. - output = True + output = False if output: # Print output in stdout @@ -6558,6 +6574,13 @@ def main(): if cfg.is_shared and cfg.shared_dir: cfg.assure_permissions_and_group(cfg.shared_dir) + # Do not allow other commands rather than config if the configuration is not set + if not cfg.configuration_done and args.subcmd not in ['config', 'cnf']: + print('\nFroster is not full configured yet.\n') + print('Run "froster config" for a full new configuration.') + print('Run "froster config --help" for more information.\n') + sys.exit(1) + # call a function for each sub command in our CLI if args.subcmd in ['config', 'cnf']: subcmd_config(args, cfg, aws) From f65e79d59db5158fbe05156068120b1d3e6c46dc Mon Sep 17 00:00:00 2001 From: Victor Machado Date: Thu, 9 May 2024 15:17:37 +0200 Subject: [PATCH 2/5] Adding fusermount3 as a pre-requisite for froster --- README.md | 6 +++--- install.sh | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 383be74..3180deb 100644 --- a/README.md +++ b/README.md @@ -10,21 +10,21 @@ Froster is a user-friendly archiving tool for teams that move data between highe ``` sudo apt-get update -sudo apt-get install -y curl pipx git gcc lib32gcc-s1 unzip +sudo apt-get install -y curl pipx git gcc lib32gcc-s1 unzip fuse3 ``` ### On RHEL ``` sudo yum update -sudo yum install -y curl pipx git gcc lib32gcc-s1 unzip +sudo yum install -y curl pipx git gcc lib32gcc-s1 unzip fuse3 ``` ### On HPC machine Please contact your administrator to install these packages: ``` -curl pipx git gcc lib32gcc-s1 unzip +curl pipx git gcc lib32gcc-s1 unzip fuse3 ```
diff --git a/install.sh b/install.sh index 45b3e10..4f4358a 100755 --- a/install.sh +++ b/install.sh @@ -148,6 +148,19 @@ check_apt_dependencies() { echo exit 1 fi + + # Check if fuse3 is installed + if [[ -z $(command -v fusermount3) ]]; then + echo "Error: fusermount3 is not installed." + echo + echo "Please install fuse3" + echo "In most linux distros you can install the latest version of fuse3 by running the following commands:" + echo " sudo apt update" + echo " sudo apt install -y fuse3" + echo + exit 1 + fi + } # Backup older installations (if any) but keep the froster-archive.json and config.ini files @@ -260,14 +273,17 @@ install_pwalk() { rm -rf ${pwalk_path} >/dev/null 2>&1 # Gather pwalk repository files + echo " Downloading pwalk files" curl -s -L ${pwalk_repository} | tar xzf - >/dev/null 2>&1 & spinner $! # Compile pwalk tool and put exec file in froster's binaries folder + echo " Compiling pwalk" gcc -pthread ${pwalk_path}/pwalk.c ${pwalk_path}/exclude.c ${pwalk_path}/fileProcess.c -o ${pwalk_path}/pwalk >/dev/null 2>&1 & spinner $! # Move pwalk to froster's binaries folder + echo " Moving pwalk to froster's binaries folder" if [ -d "${HOME}/.local/share/pipx" ]; then mv ${pwalk_path}/pwalk ${HOME}/.local/share/pipx/venvs/froster/bin/pwalk >/dev/null 2>&1 elif [ -d "${HOME}/.local/pipx" ]; then @@ -280,6 +296,7 @@ install_pwalk() { fi # Delete downloaded pwalk files + echo " Cleaning up pwalk installation files" rm -rf ${pwalk_path} >/dev/null 2>&1 echo " ...pwalk installed" @@ -310,14 +327,17 @@ install_rclone() { rm -rf rclone-current-linux-*.zip rclone-v*/ >/dev/null 2>&1 # Download the rclone zip file + echo " Downloading rclone files" curl -LO $rclone_url >/dev/null 2>&1 & spinner $! # Extract the zip file + echo " Extracting rclone files" unzip rclone-current-linux-*.zip >/dev/null 2>&1 & spinner $! # Move rclone to froster's binaries folder + echo " Moving rclone to froster's binaries folder" if [ -d "${HOME}/.local/share/pipx" ]; then mv rclone-v*/rclone ${HOME}/.local/share/pipx/venvs/froster/bin/rclone >/dev/null 2>&1 elif [ -d "${HOME}/.local/pipx" ]; then @@ -330,6 +350,7 @@ install_rclone() { fi # Remove the downloaded zip file + echo " Cleaning up rclone installation files" rm -rf rclone-current-linux-*.zip rclone-v*/ >/dev/null 2>&1 echo " ...rclone installed" From 3fc8d6a7d1b731792c5333ff6a4a18be898ee35f Mon Sep 17 00:00:00 2001 From: Victor Machado Date: Thu, 9 May 2024 15:17:58 +0200 Subject: [PATCH 3/5] Increasing version to 0.10.3 --- setup.py | 2 +- to_archive/.froster.md5sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 to_archive/.froster.md5sum diff --git a/setup.py b/setup.py index af7d01e..1f26bcc 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='froster', - version='0.10.2', + version='0.10.3', license='MIT', packages=find_packages(), install_requires=[ diff --git a/to_archive/.froster.md5sum b/to_archive/.froster.md5sum new file mode 100644 index 0000000..adac855 --- /dev/null +++ b/to_archive/.froster.md5sum @@ -0,0 +1,2 @@ +6888c844c80e2f6ee62d39c379e40ed5 Froster.allfiles.csv +3df48d348f20b7b9a9168488e07c2694 Froster.smallfiles.tar From d487f5ee90b0253a9dd52b3b55bfa8ac2a8e4555 Mon Sep 17 00:00:00 2001 From: Victor Machado Date: Thu, 9 May 2024 15:20:23 +0200 Subject: [PATCH 4/5] Delete unwanted file --- to_archive/.froster.md5sum | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 to_archive/.froster.md5sum diff --git a/to_archive/.froster.md5sum b/to_archive/.froster.md5sum deleted file mode 100644 index adac855..0000000 --- a/to_archive/.froster.md5sum +++ /dev/null @@ -1,2 +0,0 @@ -6888c844c80e2f6ee62d39c379e40ed5 Froster.allfiles.csv -3df48d348f20b7b9a9168488e07c2694 Froster.smallfiles.tar From 4a9cf23fc4a45f10713b62c8843d553025ae2d29 Mon Sep 17 00:00:00 2001 From: Victor Machado Date: Thu, 9 May 2024 16:05:45 +0200 Subject: [PATCH 5/5] Increasing version to 0.10.4 --- froster/froster.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/froster/froster.py b/froster/froster.py index 63cc5e7..e7e0e48 100755 --- a/froster/froster.py +++ b/froster/froster.py @@ -60,7 +60,7 @@ from textual.widgets import DataTable, Footer, Button __app__ = 'Froster, a user friendly S3/Glacier archiving tool' -__version__ = '0.10.2' +__version__ = '0.10.4' class ConfigManager: diff --git a/setup.py b/setup.py index 1f26bcc..8a37092 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='froster', - version='0.10.3', + version='0.10.4', license='MIT', packages=find_packages(), install_requires=[