Skip to content

Commit 2be0bfa

Browse files
author
Dirk Petersen
committed
fixed pip issue in installer and re-added test data script
1 parent cc02023 commit 2be0bfa

File tree

2 files changed

+202
-4
lines changed

2 files changed

+202
-4
lines changed

Diff for: ‎install.sh

+12-4
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ install_pipx() {
289289
else
290290
echo "...pipx ensurepath failed"
291291
fi
292-
292+
293293
}
294294

295295
install_froster() {
@@ -302,12 +302,20 @@ install_froster() {
302302
rm -f ${HOME}/.local/bin/s3-restore.py
303303
echo "...old froster files removed"
304304

305-
if [ "$LOCAL_INSTALL" = "true" ]; then
305+
if [[ "$LOCAL_INSTALL" = "true" ]]; then
306306

307307
echo " Installing from the current directory"
308+
MIN_PIP="21.3"
309+
CUR_PIP=$(python3 -m pip --version | awk '{print $2}')
310+
# Compare versions using sort -V
311+
if [[ $(echo -e "$CUR_PIP\n$MIN_PIP" | sort -V | head -n 1) != "$MIN_PIP" ]]; then
312+
echo -e "\nYour pip version is $CUR_PIP. Please upgrade to at least version $MIN_PIP."
313+
echo "Upgrading pip now ..."
314+
python3 -m pip install --upgrade pip
315+
fi
308316
echo -e "\nInstalling Froster from the current directory in --editable mode..."
309-
python3 -m pip install --force -e . >/dev/null 2>&1 & #>/dev/null 2>&1
310-
spinner "froster"
317+
python3 -m pip install --force -e . #>/dev/null 2>&1 &
318+
#spinner "froster"
311319

312320
else
313321

Diff for: ‎tests/test_data.sh

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
#!/bin/bash
2+
3+
# test script creates test data, runs through all froster
4+
# sub commands and removes data from bucket and /tmp
5+
6+
7+
set -e
8+
9+
script_path="$(readlink -f "$0")"
10+
script_dir="$(dirname "$script_path")"
11+
12+
# Function to create a random 3-character string
13+
random_string() {
14+
cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 3 | head -n 1
15+
}
16+
17+
# Function to create a sparse file
18+
create_sparse_file() {
19+
local file_name=$1
20+
local size=$2
21+
truncate -s "$size" "$file_name"
22+
set_file_times "$file_name"
23+
}
24+
25+
# Function to set atime and mtime of a file to 100 days ago
26+
set_file_times() {
27+
local file_name=$1
28+
touch -d '100 days ago' "$file_name"
29+
}
30+
31+
generate_test_data() {
32+
# Create a random directory using mktemp with the updated command
33+
#local base_dir=$(mktemp -d -t "froster.XXX")
34+
local base_dir=$(mktemp -d "froster.XXX")
35+
#echo "Base directory: $base_dir"
36+
37+
# Function to create sparse files in a directory with unique names
38+
create_files_in_dir() {
39+
local dir=$1
40+
local prefix=$2
41+
42+
# Create a few sparse files
43+
for i in {1..3}; do
44+
create_sparse_file "$dir/${prefix}_$(random_string)_medium_$i.out" "1100K"
45+
create_sparse_file "$dir/${prefix}_$(random_string)_small_$i.out" "900K"
46+
done
47+
48+
# Create an executable file
49+
local script_name="${prefix}_$(random_string)_script.sh"
50+
touch "$dir/$script_name"
51+
chmod +x "$dir/$script_name"
52+
set_file_times "$dir/$script_name"
53+
}
54+
55+
# Create unique subdirectories with unique names
56+
local subdir1_name=$(random_string)
57+
local subdir2_name=$(random_string)
58+
local subdir1="$base_dir/${subdir1_name}_subdir"
59+
local subdir2="$subdir1/${subdir2_name}_subdir"
60+
mkdir -p "$subdir1"
61+
mkdir -p "$subdir2"
62+
63+
# Create sparse files and an executable script in the main directory
64+
create_files_in_dir "$base_dir" "main"
65+
66+
# Create a sparse large file and other files only in the first subdirectory
67+
create_sparse_file "$subdir1/large_$(random_string).out" "1G"
68+
create_files_in_dir "$subdir1" "${subdir1_name}"
69+
70+
# Create sparse files and an executable script in the second subdirectory
71+
create_files_in_dir "$subdir2" "${subdir2_name}"
72+
73+
# Return the path of the created folder
74+
echo "$base_dir"
75+
}
76+
77+
# Execute the function and capture the returned folder path
78+
created_folder=$(generate_test_data)
79+
80+
echo "Test data folder: $created_folder"
81+
82+
exit
83+
84+
script_dir=~/.local/bin
85+
86+
if ! [[ -f $script_dir/froster ]]; then
87+
curl -s https://raw.githubusercontent.com/dirkpetersen/froster/main/install.sh | bash
88+
fi
89+
90+
testbucket='froster-'$(cat /dev/urandom | tr -dc 'a-z' | fold -w 5 | head -n 1)
91+
echo "Using test bucket $testbucket"
92+
93+
cfgbucket=''
94+
if [[ -f ~/.config/froster/general/bucket ]]; then
95+
cfgbucket=$(cat ~/.config/froster/general/bucket)
96+
fi
97+
echo "$testbucket" > ~/.config/froster/general/bucket
98+
99+
export RCLONE_S3_PROFILE=aws # or ${AWS_PROFILE} or change this to the AWS profile you want to use
100+
export RCLONE_S3_REGION=us-west-2 # or change this to the AWS region you want to use
101+
export RCLONE_S3_PROVIDER=AWS # or change this to another S3 provider (e.g. Ceph for on-premises)
102+
export RCLONE_S3_ENV_AUTH=true # use AWS environment variables and settings from ~/.aws
103+
104+
rclone --log-level error mkdir ":s3:$testbucket"
105+
106+
echo "Running in ${script_dir} ..."
107+
108+
echo -e "\n*** froster config --index"
109+
${script_dir}/froster --no-slurm config --index
110+
echo -e "\n*** froster index $created_folder:"
111+
${script_dir}/froster --no-slurm index "$created_folder"
112+
echo "*** froster archive $created_folder:"
113+
${script_dir}/froster --no-slurm archive "$created_folder"
114+
echo "*** froster delete $created_folder:"
115+
${script_dir}/froster --no-slurm delete "$created_folder"
116+
echo "*** froster mount $created_folder:"
117+
${script_dir}/froster --no-slurm mount "$created_folder"
118+
echo "Wait 3 sec for mount to finish"
119+
sleep 3
120+
echo -e "\n*** froster umount $created_folder:"
121+
${script_dir}/froster --no-slurm umount "$created_folder"
122+
echo -e "\n*** froster restore $created_folder:"
123+
${script_dir}/froster --no-slurm restore "$created_folder"
124+
125+
if [[ -n $cfgbucket ]]; then
126+
echo "$cfgbucket" > ~/.config/froster/general/bucket
127+
fi
128+
129+
echo "deleting bucket s3://$testbucket"
130+
rclone --log-level error purge ":s3:${testbucket}${created_folder}"
131+
# only deletes bucket if created_folder was the only content in bucket
132+
rclone --log-level error rmdirs ":s3:$testbucket"
133+
134+
echo "deleting test data in $created_folder"
135+
136+
rm -rf $created_folder
137+
138+
### OLD TEST DOWN HERE
139+
140+
# # Generate dummy data
141+
# dummy_data_path=$(generate_test_data)
142+
# echo "Dummy data path: ${dummy_data_path}"
143+
144+
# # Create an aws s3 bucket
145+
# testbucket='froster-'$(cat /dev/urandom | tr -dc 'a-z' | fold -w 5 | head -n 1)
146+
# echo "test bucket: ${testbucket}"
147+
148+
# # TODO: this should not be here.
149+
# cfgbucket=''
150+
# if [[ -f ~/.froster/config/general/bucket ]]; then
151+
# cfgbucket=$(cat ~/.froster/config/general/bucket)
152+
# fi
153+
# echo "$testbucket" >~/.froster/config/general/bucket
154+
155+
# export RCLONE_S3_PROFILE=aws # or ${AWS_PROFILE} or change this to the AWS profile you want to use
156+
# export RCLONE_S3_REGION=us-west-2 # or change this to the AWS region you want to use
157+
# export RCLONE_S3_PROVIDER=AWS # or change this to another S3 provider (e.g. Ceph for on-premises)
158+
# export RCLONE_S3_ENV_AUTH=true # use AWS environment variables and settings from ~/.aws
159+
160+
# rclone --log-level error mkdir ":s3:$testbucket"
161+
162+
# echo -e "\n*** froster config --index"
163+
# froster --no-slurm config --index
164+
# echo -e "\n*** froster index $dummy_data_path:"
165+
# froster --no-slurm index "$dummy_data_path"
166+
# echo "*** froster archive $dummy_data_path:"
167+
# froster --no-slurm archive "$dummy_data_path"
168+
# echo "*** froster delete $dummy_data_path:"
169+
# froster --no-slurm delete "$dummy_data_path"
170+
# echo "*** froster mount $dummy_data_path:"
171+
# froster --no-slurm mount "$dummy_data_path"
172+
# echo "Wait 3 sec for mount to finish"
173+
# sleep 3
174+
# echo -e "\n*** froster umount $dummy_data_path:"
175+
# froster --no-slurm umount "$dummy_data_path"
176+
# echo -e "\n*** froster restore $dummy_data_path:"
177+
# froster --no-slurm restore "$dummy_data_path"
178+
179+
# if [[ -n $cfgbucket ]]; then
180+
# echo "$cfgbucket" >~/.froster/config/general/bucket
181+
# fi
182+
183+
# echo "deleting bucket s3://$testbucket"
184+
# rclone --log-level error purge ":s3:${testbucket}${dummy_data_path}"
185+
# # only deletes bucket if dummy_data_path was the only content in bucket
186+
# rclone --log-level error rmdirs ":s3:$testbucket"
187+
188+
# echo "deleting test data in $dummy_data_path"
189+
190+
# rm -rf $dummy_data_path

0 commit comments

Comments
 (0)