Skip to content

froster v0.12.22 #64

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
macheitor merged 3 commits into from
Jun 12, 2024
Merged

froster v0.12.22 #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
af9eb50
Randomize bucket name for testing
Jun 12, 2024
311096d
typos
Jun 12, 2024
b17fce4
froster v0.12.22
Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 11 additions & 14 deletions froster/froster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,7 @@ def __init__(self):
self.home_dir = os.path.expanduser('~')

# Froster's home directory
self.froster_dir = os.path.join(sys.prefix)

# Froster's binary directory
self.bin_dir = os.path.join(self.froster_dir, 'bin')
self.froster_dir = os.path.dirname(os.path.realpath(shutil.which('froster')))

# Froster's data directory
xdg_data_home = os.environ.get('XDG_DATA_HOME')
Expand Down Expand Up @@ -3080,13 +3077,14 @@ def _index_locally(self, folder):
f' ...folder already indexed at {folder_hotspot}. Use "-f" or "--force" flag to force indexing.\n')
return


# Run pwalk on given folder
with tempfile.NamedTemporaryFile() as pwalk_output:
with tempfile.NamedTemporaryFile() as pwalk_output_folders:
with tempfile.NamedTemporaryFile() as pwalk_output_folders_converted:

# Build the pwalk command
pwalk_bin = os.path.join(sys.prefix, 'bin', 'pwalk')
pwalk_bin = os.path.join(self.cfg.froster_dir, 'pwalk')
pwalkcmd = f'{pwalk_bin} --NoSnap --one-file-system --header'
mycmd = f'{pwalkcmd} "{folder}" > {pwalk_output.name}'

Expand Down Expand Up @@ -4285,6 +4283,8 @@ def _delete_locally(self, folder_to_delete):
f"Archive aws profile: {archived_folder_info['profile']}\n")
rme.write(f"Archiver user: {archived_folder_info['user']}\n")
rme.write(f'Archiver email: {self.cfg.email}\n')
rme.write(
f'froster-archives.json: {self.archive_json}\n')
rme.write(
f'Archive tool: https://github.com/dirkpetersen/froster\n')
rme.write(
Expand Down Expand Up @@ -5187,7 +5187,7 @@ def __init__(self, args: argparse.Namespace, cfg: ConfigManager):
self.cfg = cfg

# Set the Rclone executable path
self.rc = os.path.join(sys.prefix, 'bin', 'rclone')
self.rc = os.path.join(self.cfg.froster_dir, 'rclone')

# Set the Rclone environment variables
# Note: Keys are set in the AWS Boto __init__ function
Expand Down Expand Up @@ -6052,13 +6052,15 @@ def print_version(self):
def print_info(self):
'''Print froster info'''

froster_dir = os.path.dirname(os.path.realpath(shutil.which('froster')))

log(
f'froster v{pkg_resources.get_distribution("froster").version}\n')
log(f'Tools version:')
log(f' python v{platform.python_version()}')
log(' pwalk ', 'v'+subprocess.run([os.path.join(sys.prefix, 'bin', 'pwalk'), '--version'],
log(' pwalk ', 'v'+subprocess.run([os.path.join(froster_dir, 'pwalk'), '--version'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True).stderr.split('\n')[0].split()[2])
log(' ', subprocess.run([os.path.join(sys.prefix, 'bin', 'rclone'), '--version'],
log(' ', subprocess.run([os.path.join(froster_dir, 'rclone'), '--version'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True).stdout.split('\n')[0])

log(textwrap.dedent(f'''
Expand Down Expand Up @@ -6109,11 +6111,6 @@ def subcmd_config(self, cfg: ConfigManager, aws: AWSBoto):
if self.args.print:
return cfg.print_config()

# if self.args.monitor:
# froster_binary = os.path.join(cfg.bin_dir, 'froster')
# return cfg.add_systemd_cron_job(
# f'{froster_binary} restore --monitor', '30')

log(f'\n*****************************')
log(f'*** FROSTER CONFIGURATION ***')
log(f'*****************************\n')
Expand Down Expand Up @@ -7065,7 +7062,7 @@ def main():
cmd.print_help()

# Check if there are updates on froster every X days
if cfg.check_update() and args.subcmd not in ['update', 'upd']:
if cfg.configuration_done and cfg.check_update() and args.subcmd not in ['update', 'upd']:
cmd.subcmd_update(mute_no_update=True)

# Close the AWS session
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "froster"
version = "0.12.21"
version = "0.12.22"
description = "Froster is a tool for easy data transfer between local file systems and AWS S3 storage."
authors = ["Victor Machado <[email protected]>"]
readme = "README.md"
Expand Down
13 changes: 9 additions & 4 deletions tests/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
# Variables
import os
import random
import string
import tempfile


def random_string(length=4):
return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length))

NAME = "Test-Bob"
NAME_2 = "Test-Alice"
EMAIL = "[email protected]"
Expand All @@ -23,11 +28,11 @@
NIH_SECTION = 'NIH'
S3_SECTION = 'S3'

S3_BUCKET_NAME_CONFIG = 'froster-unittest-config'
S3_BUCKET_NAME_CONFIG_2 = 'froster-unittest-config-2'
S3_BUCKET_NAME_CONFIG = 'froster-unittest-config-' + random_string(4)
S3_BUCKET_NAME_CONFIG_2 = 'froster-unittest-config-' + random_string(4)

S3_BUCKET_NAME_INDEX = 'froster-unittest-index'
S3_BUCKET_NAME_INDEX_2 = 'froster-unittest-index-2'
S3_BUCKET_NAME_INDEX = 'froster-unittest-index-' + random_string(4)
S3_BUCKET_NAME_INDEX_2 = 'froster-unittest-index-' + random_string(4)

S3_ARCHIVE_DIR = 'froster_bob'
S3_ARCHIVE_DIR_2 = 'froster_alice'
Expand Down
11 changes: 5 additions & 6 deletions tests/generate_dummy_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,11 @@ def generate_test_data():

# Create files in the base directory
for dir in [base_dir, subdir1, subdir2]:
create_file('small_1_' + random_string(), 1, dir, '1 days ago')
create_file('small_2_' + random_string(), 1, dir, '10 days ago')
create_file('medium_1_' + random_string(), 1100, dir, '100 days ago')
create_file('medium_2_' + random_string(), 1100, dir, '1000 days ago')
create_file('large_1_' + random_string(), 1100000, dir, '10 days ago')
create_file('large_2_' + random_string(), 1100000, dir, '1000 days ago')
create_file('small_1_' + random_string(), 1, dir, '1 days ago') # 1B
create_file('small_2_' + random_string(), 1, dir, '10 days ago') # 1B
create_file('medium_1_' + random_string(), 1100, dir, '100 days ago') # 1.1KB
create_file('medium_2_' + random_string(), 1100, dir, '1000 days ago') # 1.1KB
create_file('large_' + random_string(), 1100000000, dir, '10 days ago') # 1.1GB

return base_dir

Expand Down