Skip to content

Commit ce02b00

Browse files
dirkpetersendirkpetersen
authored
Merge pull request #155 from dirkpetersen/fixes
Smaller Fixes
2 parents 1c11004 + f4e9550 commit ce02b00

7 files changed

+197
-7
lines changed

Diff for: ‎.github/workflows/froster-local-install.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
- name: Set up Python
1616
uses: actions/setup-python@v5
1717
with:
18-
python-version: '3.10'
18+
python-version: '3.11'
1919

2020
- name: Install pipx
2121
run: python -m pip install pipx

Diff for: ‎.github/workflows/froster-remote-install.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
- name: Set up Python
1313
uses: actions/setup-python@v5
1414
with:
15-
python-version: '3.10'
15+
python-version: '3.11'
1616

1717
- name: Install pipx
1818
run: python -m pip install pipx

Diff for: ‎.github/workflows/pypi-release-publish.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ jobs:
5252
- name: Set up Python
5353
uses: actions/setup-python@v3
5454
with:
55-
python-version: '3.10'
55+
python-version: '3.11'
5656

5757
- name: Install pipx
5858
run: python -m pip install pipx

Diff for: ‎.github/workflows/test-basic-features.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ jobs:
99
runs-on: ubuntu-latest
1010
strategy:
1111
matrix:
12-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
12+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
1313

1414
steps:
1515

Diff for: ‎.github/workflows/test-credentials.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ jobs:
99
runs-on: ubuntu-latest
1010
strategy:
1111
matrix:
12-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
12+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
1313

1414
steps:
1515

Diff for: ‎pyproject.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@ build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
66
name = "froster"
7-
version = "0.19.0"
7+
version = "0.20.0"
88
description = "Froster is a tool for easy data transfer between local file systems and AWS S3 storage."
99
authors = ["Dirk Petersen <[email protected]>", "Victor Machado <[email protected]>"]
1010
readme = "README.md"
1111
license = "MIT"
1212

1313
[tool.poetry.dependencies]
14-
python = "^3.8"
14+
python = "^3.9"
1515
boto3 = "^1.35.14"
1616
duckdb = "^1.0.0"
1717
inquirer = "^3.2.4"

Diff for: ‎tests/test_data.sh

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
#!/bin/bash
2+
3+
# test script creates test data, runs through all froster
4+
# sub commands and removes data from bucket and /tmp
5+
6+
7+
set -e
8+
9+
script_path="$(readlink -f "$0")"
10+
script_dir="$(dirname "$script_path")"
11+
12+
# Function to create a random 3-character string
13+
random_string() {
14+
cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 3 | head -n 1
15+
}
16+
17+
# Function to create a sparse file
18+
create_sparse_file() {
19+
local file_name=$1
20+
local size=$2
21+
truncate -s "$size" "$file_name"
22+
set_file_times "$file_name"
23+
}
24+
25+
# Function to set atime and mtime of a file to 100 days ago
26+
set_file_times() {
27+
local file_name=$1
28+
touch -d '100 days ago' "$file_name"
29+
}
30+
31+
generate_test_data() {
32+
# Create a random directory using mktemp with the updated command
33+
#local base_dir=$(mktemp -d -t "froster.XXX")
34+
local base_dir=$(mktemp -d "froster.XXX")
35+
#echo "Base directory: $base_dir"
36+
37+
# Function to create sparse files in a directory with unique names
38+
create_files_in_dir() {
39+
local dir=$1
40+
local prefix=$2
41+
42+
# Create a few sparse files
43+
for i in {1..3}; do
44+
create_sparse_file "$dir/${prefix}_$(random_string)_medium_$i.out" "1100K"
45+
create_sparse_file "$dir/${prefix}_$(random_string)_small_$i.out" "900K"
46+
done
47+
48+
# Create an executable file
49+
local script_name="${prefix}_$(random_string)_script.sh"
50+
touch "$dir/$script_name"
51+
chmod +x "$dir/$script_name"
52+
set_file_times "$dir/$script_name"
53+
}
54+
55+
# Create unique subdirectories with unique names
56+
local subdir1_name=$(random_string)
57+
local subdir2_name=$(random_string)
58+
local subdir1="$base_dir/${subdir1_name}_subdir"
59+
local subdir2="$subdir1/${subdir2_name}_subdir"
60+
mkdir -p "$subdir1"
61+
mkdir -p "$subdir2"
62+
63+
# Create sparse files and an executable script in the main directory
64+
create_files_in_dir "$base_dir" "main"
65+
66+
# Create a sparse large file and other files only in the first subdirectory
67+
create_sparse_file "$subdir1/large_$(random_string).out" "1G"
68+
create_files_in_dir "$subdir1" "${subdir1_name}"
69+
70+
# Create sparse files and an executable script in the second subdirectory
71+
create_files_in_dir "$subdir2" "${subdir2_name}"
72+
73+
# Return the path of the created folder
74+
echo "$base_dir"
75+
}
76+
77+
# Execute the function and capture the returned folder path
78+
created_folder=$(generate_test_data)
79+
80+
echo "Test data folder: $created_folder"
81+
82+
exit
83+
84+
script_dir=~/.local/bin
85+
86+
if ! [[ -f $script_dir/froster ]]; then
87+
curl -s https://raw.githubusercontent.com/dirkpetersen/froster/main/install.sh | bash
88+
fi
89+
90+
testbucket='froster-'$(cat /dev/urandom | tr -dc 'a-z' | fold -w 5 | head -n 1)
91+
echo "Using test bucket $testbucket"
92+
93+
cfgbucket=''
94+
if [[ -f ~/.config/froster/general/bucket ]]; then
95+
cfgbucket=$(cat ~/.config/froster/general/bucket)
96+
fi
97+
echo "$testbucket" > ~/.config/froster/general/bucket
98+
99+
export RCLONE_S3_PROFILE=aws # or ${AWS_PROFILE} or change this to the AWS profile you want to use
100+
export RCLONE_S3_REGION=us-west-2 # or change this to the AWS region you want to use
101+
export RCLONE_S3_PROVIDER=AWS # or change this to another S3 provider (e.g. Ceph for on-premises)
102+
export RCLONE_S3_ENV_AUTH=true # use AWS environment variables and settings from ~/.aws
103+
104+
rclone --log-level error mkdir ":s3:$testbucket"
105+
106+
echo "Running in ${script_dir} ..."
107+
108+
echo -e "\n*** froster config --index"
109+
${script_dir}/froster --no-slurm config --index
110+
echo -e "\n*** froster index $created_folder:"
111+
${script_dir}/froster --no-slurm index "$created_folder"
112+
echo "*** froster archive $created_folder:"
113+
${script_dir}/froster --no-slurm archive "$created_folder"
114+
echo "*** froster delete $created_folder:"
115+
${script_dir}/froster --no-slurm delete "$created_folder"
116+
echo "*** froster mount $created_folder:"
117+
${script_dir}/froster --no-slurm mount "$created_folder"
118+
echo "Wait 3 sec for mount to finish"
119+
sleep 3
120+
echo -e "\n*** froster umount $created_folder:"
121+
${script_dir}/froster --no-slurm umount "$created_folder"
122+
echo -e "\n*** froster restore $created_folder:"
123+
${script_dir}/froster --no-slurm restore "$created_folder"
124+
125+
if [[ -n $cfgbucket ]]; then
126+
echo "$cfgbucket" > ~/.config/froster/general/bucket
127+
fi
128+
129+
echo "deleting bucket s3://$testbucket"
130+
rclone --log-level error purge ":s3:${testbucket}${created_folder}"
131+
# only deletes bucket if created_folder was the only content in bucket
132+
rclone --log-level error rmdirs ":s3:$testbucket"
133+
134+
echo "deleting test data in $created_folder"
135+
136+
rm -rf $created_folder
137+
138+
### OLD TEST DOWN HERE
139+
140+
# # Generate dummy data
141+
# dummy_data_path=$(generate_test_data)
142+
# echo "Dummy data path: ${dummy_data_path}"
143+
144+
# # Create an aws s3 bucket
145+
# testbucket='froster-'$(cat /dev/urandom | tr -dc 'a-z' | fold -w 5 | head -n 1)
146+
# echo "test bucket: ${testbucket}"
147+
148+
# # TODO: this should not be here.
149+
# cfgbucket=''
150+
# if [[ -f ~/.froster/config/general/bucket ]]; then
151+
# cfgbucket=$(cat ~/.froster/config/general/bucket)
152+
# fi
153+
# echo "$testbucket" >~/.froster/config/general/bucket
154+
155+
# export RCLONE_S3_PROFILE=aws # or ${AWS_PROFILE} or change this to the AWS profile you want to use
156+
# export RCLONE_S3_REGION=us-west-2 # or change this to the AWS region you want to use
157+
# export RCLONE_S3_PROVIDER=AWS # or change this to another S3 provider (e.g. Ceph for on-premises)
158+
# export RCLONE_S3_ENV_AUTH=true # use AWS environment variables and settings from ~/.aws
159+
160+
# rclone --log-level error mkdir ":s3:$testbucket"
161+
162+
# echo -e "\n*** froster config --index"
163+
# froster --no-slurm config --index
164+
# echo -e "\n*** froster index $dummy_data_path:"
165+
# froster --no-slurm index "$dummy_data_path"
166+
# echo "*** froster archive $dummy_data_path:"
167+
# froster --no-slurm archive "$dummy_data_path"
168+
# echo "*** froster delete $dummy_data_path:"
169+
# froster --no-slurm delete "$dummy_data_path"
170+
# echo "*** froster mount $dummy_data_path:"
171+
# froster --no-slurm mount "$dummy_data_path"
172+
# echo "Wait 3 sec for mount to finish"
173+
# sleep 3
174+
# echo -e "\n*** froster umount $dummy_data_path:"
175+
# froster --no-slurm umount "$dummy_data_path"
176+
# echo -e "\n*** froster restore $dummy_data_path:"
177+
# froster --no-slurm restore "$dummy_data_path"
178+
179+
# if [[ -n $cfgbucket ]]; then
180+
# echo "$cfgbucket" >~/.froster/config/general/bucket
181+
# fi
182+
183+
# echo "deleting bucket s3://$testbucket"
184+
# rclone --log-level error purge ":s3:${testbucket}${dummy_data_path}"
185+
# # only deletes bucket if dummy_data_path was the only content in bucket
186+
# rclone --log-level error rmdirs ":s3:$testbucket"
187+
188+
# echo "deleting test data in $dummy_data_path"
189+
190+
# rm -rf $dummy_data_path

0 commit comments

Comments
 (0)