Smaller Fixes

.github/workflows/froster-local-install.yml

@@ -15,7 +15,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
 
     - name: Install pipx
       run: python -m pip install pipx

.github/workflows/froster-remote-install.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
 
     - name: Install pipx
       run: python -m pip install pipx

.github/workflows/pypi-release-publish.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Install pipx
         run: python -m pip install pipx

.github/workflows/test-basic-features.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
 
     steps:
 

.github/workflows/test-credentials.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
 
     steps:
 

pyproject.toml

@@ -4,14 +4,14 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "froster"
-version = "0.19.0"
+version = "0.20.0"
 description = "Froster is a tool for easy data transfer between local file systems and AWS S3 storage."
 authors = ["Dirk Petersen <[email protected]>", "Victor Machado <[email protected]>"]
 readme = "README.md"
 license = "MIT"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 boto3 = "^1.35.14"
 duckdb = "^1.0.0"
 inquirer = "^3.2.4"

tests/test_data.sh

@@ -0,0 +1,190 @@
+#!/bin/bash
+
+# test script creates test data, runs through all froster
+# sub commands and removes data from bucket and /tmp
+
+
+set -e
+
+script_path="$(readlink -f "$0")"
+script_dir="$(dirname "$script_path")"
+
+# Function to create a random 3-character string
+random_string() {
+    cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 3 | head -n 1
+}
+
+# Function to create a sparse file
+create_sparse_file() {
+    local file_name=$1
+    local size=$2
+    truncate -s "$size" "$file_name"
+    set_file_times "$file_name"
+}
+
+# Function to set atime and mtime of a file to 100 days ago
+set_file_times() {
+    local file_name=$1
+    touch -d '100 days ago' "$file_name"
+}
+
+generate_test_data() {
+    # Create a random directory using mktemp with the updated command
+    #local base_dir=$(mktemp -d -t "froster.XXX")
+    local base_dir=$(mktemp -d "froster.XXX")
+    #echo "Base directory: $base_dir"
+
+    # Function to create sparse files in a directory with unique names
+    create_files_in_dir() {
+        local dir=$1
+        local prefix=$2
+
+        # Create a few sparse files
+        for i in {1..3}; do
+            create_sparse_file "$dir/${prefix}_$(random_string)_medium_$i.out" "1100K"
+            create_sparse_file "$dir/${prefix}_$(random_string)_small_$i.out" "900K"
+        done
+
+        # Create an executable file
+        local script_name="${prefix}_$(random_string)_script.sh"
+        touch "$dir/$script_name"
+        chmod +x "$dir/$script_name"
+        set_file_times "$dir/$script_name"
+    }
+
+    # Create unique subdirectories with unique names
+    local subdir1_name=$(random_string)
+    local subdir2_name=$(random_string)
+    local subdir1="$base_dir/${subdir1_name}_subdir"
+    local subdir2="$subdir1/${subdir2_name}_subdir"
+    mkdir -p "$subdir1"
+    mkdir -p "$subdir2"
+
+    # Create sparse files and an executable script in the main directory
+    create_files_in_dir "$base_dir" "main"
+
+    # Create a sparse large file and other files only in the first subdirectory
+    create_sparse_file "$subdir1/large_$(random_string).out" "1G"
+    create_files_in_dir "$subdir1" "${subdir1_name}"
+
+    # Create sparse files and an executable script in the second subdirectory
+    create_files_in_dir "$subdir2" "${subdir2_name}"
+
+    # Return the path of the created folder
+    echo "$base_dir"
+}
+
+# Execute the function and capture the returned folder path
+created_folder=$(generate_test_data)
+
+echo "Test data folder: $created_folder"
+
+exit
+
+script_dir=~/.local/bin
+
+if ! [[ -f $script_dir/froster ]]; then
+  curl -s https://raw.githubusercontent.com/dirkpetersen/froster/main/install.sh | bash
+fi
+
+testbucket='froster-'$(cat /dev/urandom | tr -dc 'a-z' | fold -w 5 | head -n 1)
+echo "Using test bucket $testbucket"
+
+cfgbucket=''
+if [[ -f ~/.config/froster/general/bucket ]]; then
+  cfgbucket=$(cat ~/.config/froster/general/bucket)
+fi
+echo "$testbucket" > ~/.config/froster/general/bucket
+
+export RCLONE_S3_PROFILE=aws # or ${AWS_PROFILE} or change this to the AWS profile you want to use
+export RCLONE_S3_REGION=us-west-2 # or change this to the AWS region you want to use
+export RCLONE_S3_PROVIDER=AWS # or change this to another S3 provider (e.g. Ceph for on-premises)
+export RCLONE_S3_ENV_AUTH=true # use AWS environment variables and settings from ~/.aws
+
+rclone --log-level error mkdir ":s3:$testbucket"
+
+echo "Running in ${script_dir} ..."
+
+echo -e "\n*** froster config --index"
+${script_dir}/froster --no-slurm config --index
+echo -e "\n*** froster index $created_folder:"
+${script_dir}/froster --no-slurm index "$created_folder"
+echo "*** froster archive $created_folder:"
+${script_dir}/froster --no-slurm archive "$created_folder"
+echo "*** froster delete $created_folder:"
+${script_dir}/froster --no-slurm delete "$created_folder"
+echo "*** froster mount $created_folder:"
+${script_dir}/froster --no-slurm mount "$created_folder"
+echo "Wait 3 sec for mount to finish"
+sleep 3
+echo -e "\n*** froster umount $created_folder:"
+${script_dir}/froster --no-slurm umount "$created_folder"
+echo -e "\n*** froster restore $created_folder:"
+${script_dir}/froster --no-slurm restore "$created_folder"
+
+if [[ -n $cfgbucket ]]; then
+  echo "$cfgbucket" > ~/.config/froster/general/bucket
+fi
+
+echo "deleting bucket s3://$testbucket"
+rclone --log-level error purge ":s3:${testbucket}${created_folder}"
+# only deletes bucket if created_folder was the only content in bucket
+rclone --log-level error rmdirs ":s3:$testbucket"
+
+echo "deleting test data in $created_folder"
+
+rm -rf $created_folder
+
+### OLD TEST DOWN HERE
+
+# # Generate dummy data
+# dummy_data_path=$(generate_test_data)
+# echo "Dummy data path: ${dummy_data_path}"
+
+# # Create an aws s3 bucket
+# testbucket='froster-'$(cat /dev/urandom | tr -dc 'a-z' | fold -w 5 | head -n 1)
+# echo "test bucket: ${testbucket}"
+
+# # TODO: this should not be here.
+# cfgbucket=''
+# if [[ -f ~/.froster/config/general/bucket ]]; then
+#   cfgbucket=$(cat ~/.froster/config/general/bucket)
+# fi
+# echo "$testbucket" >~/.froster/config/general/bucket
+
+# export RCLONE_S3_PROFILE=aws      # or ${AWS_PROFILE} or change this to the AWS profile you want to use
+# export RCLONE_S3_REGION=us-west-2 # or change this to the AWS region you want to use
+# export RCLONE_S3_PROVIDER=AWS     # or change this to another S3 provider (e.g. Ceph for on-premises)
+# export RCLONE_S3_ENV_AUTH=true    # use AWS environment variables and settings from ~/.aws
+
+# rclone --log-level error mkdir ":s3:$testbucket"
+
+# echo -e "\n*** froster config --index"
+# froster --no-slurm config --index
+# echo -e "\n*** froster index $dummy_data_path:"
+# froster --no-slurm index "$dummy_data_path"
+# echo "*** froster archive $dummy_data_path:"
+# froster --no-slurm archive "$dummy_data_path"
+# echo "*** froster delete $dummy_data_path:"
+# froster --no-slurm delete "$dummy_data_path"
+# echo "*** froster mount $dummy_data_path:"
+# froster --no-slurm mount "$dummy_data_path"
+# echo "Wait 3 sec for mount to finish"
+# sleep 3
+# echo -e "\n*** froster umount $dummy_data_path:"
+# froster --no-slurm umount "$dummy_data_path"
+# echo -e "\n*** froster restore $dummy_data_path:"
+# froster --no-slurm restore "$dummy_data_path"
+
+# if [[ -n $cfgbucket ]]; then
+#   echo "$cfgbucket" >~/.froster/config/general/bucket
+# fi
+
+# echo "deleting bucket s3://$testbucket"
+# rclone --log-level error purge ":s3:${testbucket}${dummy_data_path}"
+# # only deletes bucket if dummy_data_path was the only content in bucket
+# rclone --log-level error rmdirs ":s3:$testbucket"
+
+# echo "deleting test data in $dummy_data_path"
+
+# rm -rf $dummy_data_path