Skip to content

Commit 6e7557a

Browse files
victormachadoperezvictormachadoperez
authored
Merge pull request #136 from dirkpetersen:issue-103-test
Added "froster test" command
2 parents 730ff11 + a26e63e commit 6e7557a

11 files changed

+190
-213
lines changed

Diff for: ‎.github/workflows/froster-local-install.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
- uses: actions/checkout@v4
1414

1515
- name: Set up Python
16-
uses: actions/setup-python@v3
16+
uses: actions/setup-python@v5
1717
with:
1818
python-version: '3.10'
1919

Diff for: ‎.github/workflows/froster-remote-install.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
- uses: actions/checkout@v4
1111

1212
- name: Set up Python
13-
uses: actions/setup-python@v3
13+
uses: actions/setup-python@v5
1414
with:
1515
python-version: '3.10'
1616

Diff for: ‎.github/workflows/pypi-release-publish.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@ permissions:
1818
contents: read
1919

2020
jobs:
21-
deploy:
21+
pypi-release:
2222

2323
runs-on: ubuntu-latest
2424

2525
steps:
2626
- uses: actions/checkout@v4
2727

2828
- name: Set up Python
29-
uses: actions/setup-python@v3
29+
uses: actions/setup-python@v5
3030
with:
3131
python-version: '3.x'
3232

@@ -45,7 +45,7 @@ jobs:
4545
password: ${{ secrets.PYPI_API_TOKEN }}
4646

4747
check-froster-installation:
48-
needs: deploy
48+
needs: pypi-release
4949
runs-on: ubuntu-latest
5050
steps:
5151

Diff for: ‎.github/workflows/test-credentials.yml

+11-4
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,25 @@ run-name: Test "froster credentials"
55
on: [push, pull_request]
66

77
jobs:
8-
froster-index:
8+
test-credentials:
99
runs-on: ubuntu-latest
10+
strategy:
11+
matrix:
12+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
13+
1014
steps:
1115

1216
- uses: actions/checkout@v4
1317
with:
1418
ref: ${{ github.ref }}
1519

16-
- name: Set up Python
17-
uses: actions/setup-python@v3
20+
- name: Set up Python ${{ matrix.python-version }}
21+
uses: actions/setup-python@v5
1822
with:
19-
python-version: '3.10'
23+
python-version: ${{ matrix.python-version }}
24+
25+
- name: Display Python version
26+
run: python3 -c "import sys; print(sys.version)"
2027

2128
- name: Create and activate virtual environment
2229
run: |

Diff for: ‎.github/workflows/test-froster-config.yml.pending

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
ref: ${{ github.ref }}
2121

2222
- name: Set up Python
23-
uses: actions/setup-python@v3
23+
uses: actions/setup-python@v5
2424
with:
2525
python-version: '3.10'
2626

Diff for: ‎.github/workflows/test-index.yml.pending

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
ref: ${{ github.ref }}
2121

2222
- name: Set up Python
23-
uses: actions/setup-python@v3
23+
uses: actions/setup-python@v5
2424
with:
2525
python-version: '3.10'
2626

Diff for: ‎froster/froster.py

+163-9
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
"""
77

88
# internal modules
9+
import random
10+
import string
911
from textual.widgets import DataTable, Footer, Button
1012
from textual.widgets import Label, Input, LoadingIndicator
1113
from textual.screen import ModalScreen
@@ -1776,25 +1778,70 @@ def create_bucket(self, bucket_name, region):
17761778
print_error()
17771779
return False
17781780

1779-
def delete_bucket(self, bucket_name):
1780-
'''Delete the given bucket'''
1781+
def empty_bucket(self, bucket_name):
17811782

1783+
if not bucket_name:
1784+
raise ValueError('No bucket name provided')
1785+
1786+
# Added a check to prevent accidental deletion of buckets
17821787
if os.environ.get('DEBUG') != '1':
1783-
raise ValueError('Buckets cannot be deleted outside DEBUG mode.')
1788+
raise ValueError('Objects in bucket cannot be deleted outside DEBUG mode.')
1789+
1790+
# Additional check to prevent accidental deletion of buckets
1791+
if not bucket_name.startswith('froster-unittest') and not bucket_name.startswith('froster-cli-test'):
1792+
raise ValueError('Bucket name must start with "froster-unittest" or "froster-cli-test" to be emptied.')
1793+
1794+
try:
1795+
paginator = self.s3_client.get_paginator('list_objects_v2')
1796+
for page in paginator.paginate(Bucket=bucket_name):
1797+
for obj in page.get('Contents', []):
1798+
# print(f"Deleting {obj['Key']}")
1799+
self.s3_client.delete_object(Bucket=bucket_name, Key=obj['Key'])
1800+
1801+
return True
1802+
1803+
except Exception:
1804+
print_error()
1805+
return False
1806+
1807+
def delete_bucket(self, bucket_name):
1808+
'''Delete the given bucket'''
17841809

17851810
if not bucket_name:
17861811
raise ValueError('No bucket name provided')
1812+
1813+
# Added a check to prevent accidental deletion of buckets
1814+
if os.environ.get('DEBUG') != '1':
1815+
raise ValueError('Objects in bucket cannot be deleted outside DEBUG mode.')
17871816

1817+
# Additional check to prevent accidental deletion of buckets
1818+
if not bucket_name.startswith('froster-unittest') and not bucket_name.startswith('froster-cli-test'):
1819+
raise ValueError('Bucket name must start with "froster-unittest" or "froster-cli-test" to be deleted.')
1820+
17881821
try:
17891822

17901823
s3_buckets = self.get_buckets()
17911824

1792-
# Delete the buckets if they exists
1825+
# Delete the bucket if its exists
17931826
if bucket_name in s3_buckets:
1827+
self.empty_bucket(bucket_name)
17941828
self.s3_client.delete_bucket(Bucket=bucket_name)
17951829
log(f'Bucket {bucket_name} deleted\n')
1830+
1831+
# This is here in case there is a mistake and the S3 buckets are not deleted
1832+
# This will erase all the froster-unittest* or froster-cli-test buckets at once
1833+
elif bucket_name == "froster-unittest" or bucket_name == "froster-cli-test":
1834+
for bucket in s3_buckets:
1835+
if bucket.startswith(bucket_name):
1836+
try:
1837+
self.empty_bucket(bucket_name)
1838+
self.s3_client.delete_bucket(Bucket=bucket)
1839+
except Exception as e:
1840+
print(f'Error: {e}')
1841+
continue
1842+
log(f'\nBucket {bucket} deleted\n')
17961843
else:
1797-
log(f'Bucket {bucket_name} not found\n')
1844+
log(f'\nBucket {bucket_name} not found\n')
17981845

17991846
return True
18001847

@@ -1862,7 +1909,7 @@ def get_regions(self):
18621909
print_error()
18631910
sys.exit(1)
18641911

1865-
def list_objects_in_bucket(self, bucket_name):
1912+
def get_objects(self, bucket_name):
18661913
'''List all the objects in the given bucket'''
18671914

18681915
if not bucket_name:
@@ -3686,7 +3733,7 @@ def _index_locally(self, folder):
36863733

36873734
log(f'Total folders processed: {len(rows)}')
36883735

3689-
log(f'INDEXING SUCCESSFULLY COMPLETED')
3736+
log(f'\nINDEXING SUCCESSFULLY COMPLETED')
36903737

36913738
lastagedbytes = 0
36923739
for i in range(0, len(daysaged)):
@@ -5489,7 +5536,7 @@ class TextualStringListSelector(App[list]):
54895536

54905537
BINDINGS = [("q", "request_quit", "Quit")]
54915538

5492-
def __init__(self, title: str, items: list[str]):
5539+
def __init__(self, title, items):
54935540
super().__init__()
54945541
self.title = title
54955542
self.items = items
@@ -5520,7 +5567,7 @@ class TableArchive(App[list]):
55205567

55215568
BINDINGS = [("q", "request_quit", "Quit")]
55225569

5523-
def __init__(self, files: list[str]):
5570+
def __init__(self, files):
55245571
super().__init__()
55255572
self.files = files
55265573

@@ -6866,6 +6913,104 @@ def subcmd_umount(self, arch: Archiver):
68666913
print_error()
68676914
return False
68686915

6916+
def subcmd_test(self, cfg: ConfigManager, arch: Archiver, aws: AWSBoto):
6917+
'''Test basic functionality of Froster'''
6918+
6919+
try:
6920+
def tearDown(bucket_name, folder_path):
6921+
# Delete the directory
6922+
shutil.rmtree(folder_path)
6923+
6924+
# Delete the bucket
6925+
os.environ['DEBUG'] = '1'
6926+
aws.delete_bucket(bucket_name)
6927+
6928+
log('\nTESTING FROSTER...')
6929+
6930+
if not aws.check_credentials(prints=True):
6931+
log('\nTESTING FAILED\n')
6932+
return False
6933+
6934+
# Generate a random bucket name
6935+
rnd = ''.join(random.choices(
6936+
string.ascii_lowercase + string.digits, k=4))
6937+
new_bucket_name = f'froster-cli-test-{rnd}'
6938+
6939+
# Set temporary this new bucket name in the configuration
6940+
cfg.bucket_name = new_bucket_name
6941+
6942+
# Create a dummy file
6943+
folder_path = tempfile.mkdtemp(prefix='froster_test_')
6944+
file_path = os.path.join(folder_path, 'dummy_file')
6945+
6946+
log(f'\nCreating dummy file {file_path}...')
6947+
6948+
with open(file_path, 'wb') as f:
6949+
f.truncate(1)
6950+
6951+
subprocess.run(['touch', file_path])
6952+
6953+
log(f' ....dummy file create')
6954+
6955+
# Create a new bucket
6956+
if not aws.create_bucket(bucket_name=new_bucket_name, region=cfg.region):
6957+
tearDown(new_bucket_name, folder_path)
6958+
log('\nTESTING FAILED\n')
6959+
return False
6960+
6961+
# Mocking the index arguments
6962+
self.args.folders = [folder_path]
6963+
self.args.permissions = False
6964+
self.args.pwalkcopy = None
6965+
6966+
# Running index command
6967+
if not self.subcmd_index(cfg, arch):
6968+
tearDown(new_bucket_name, folder_path)
6969+
log('\nTESTING FAILED\n')
6970+
return False
6971+
6972+
# Mocking the archive arguments
6973+
self.args.older = 0
6974+
self.args.newer = 0
6975+
self.args.reset = False
6976+
self.args.recursive = False
6977+
self.args.nih = False
6978+
self.args.nihref = None
6979+
self.args.notar = True # True to check by file name in the archive
6980+
self.args.force = False
6981+
self.args.noslurm = True # Avoid slurm execution
6982+
6983+
# Running archive command
6984+
if not self.subcmd_archive(arch, aws):
6985+
tearDown(new_bucket_name, folder_path)
6986+
log('\nTESTING FAILED\n')
6987+
return False
6988+
6989+
# Mocking the delete command
6990+
self.args.bucket = None
6991+
self.args.debug = False
6992+
self.args.recursive = False
6993+
self.args.noslurm = True # Avoid slurm execution
6994+
6995+
# Running delete command
6996+
if not self.subcmd_delete(arch, aws):
6997+
tearDown(new_bucket_name, folder_path)
6998+
log('\nTESTING FAILED\n')
6999+
return False
7000+
7001+
# Clean up
7002+
tearDown(new_bucket_name, folder_path)
7003+
7004+
log('\nTEST SUCCESSFULLY COMPLETED\n')
7005+
7006+
return True
7007+
7008+
except Exception:
7009+
print_error()
7010+
aws.delete_bucket(new_bucket_name)
7011+
log('\nTESTING FAILED\n')
7012+
return False
7013+
68697014
def subcmd_ssh(self, cfg: ConfigManager, aws: AWSBoto):
68707015
'''SSH into an AWS EC2 instance'''
68717016

@@ -7269,6 +7414,13 @@ def parse_arguments(self):
72697414
parser_update.add_argument('--rclone', '-r', dest='rclone', action='store_true',
72707415
help="Update rclone to latests version")
72717416

7417+
# ***
7418+
7419+
parser_test = subparsers.add_parser('test', aliases=['tst'],
7420+
description=textwrap.dedent(f'''
7421+
Test basic functionality of Froster
7422+
'''), formatter_class=argparse.RawTextHelpFormatter)
7423+
72727424
return parser
72737425

72747426

@@ -7499,6 +7651,8 @@ def main():
74997651
# Calling check_update as we are checking for udpates (used to store the last timestamp check)
75007652
cfg.check_update()
75017653
res = cmd.subcmd_update(mute_no_update=False)
7654+
elif args.subcmd in ['test', 'tst']:
7655+
res = cmd.subcmd_test(cfg, arch, aws)
75027656
else:
75037657

75047658
# Check credentials

Diff for: ‎install.sh

+6-1
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,12 @@ install_pipx() {
296296

297297
# Ensure path for pipx
298298
echo " Ensuring path for pipx"
299-
$HOME/.local/bin/pipx ensurepath >/dev/null 2>&1
299+
if [[ $(command -v pipx) ]]; then
300+
pipx ensurepath >/dev/null 2>&1
301+
else
302+
$HOME/.local/bin/pipx ensurepath >/dev/null 2>&1
303+
fi
304+
300305
echo "...pipx installed"
301306
else
302307
echo "...pipx already installed"

Diff for: ‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
66
name = "froster"
7-
version = "0.16.11"
7+
version = "0.16.12"
88
description = "Froster is a tool for easy data transfer between local file systems and AWS S3 storage."
99
authors = ["Victor Machado <[email protected]>"]
1010
readme = "README.md"

0 commit comments

Comments
 (0)