Merge pull request #1 from universal-development/remote-backups-support

denis256 · web-flow · commit 62ce07dc1b0f · 2025-04-13T09:36:05.000+01:00
Add support for remote backups
diff --git a/README.md b/README.md
@@ -3,23 +3,29 @@
 Lightweight and reliable Bash script that automates the backup of local directories and securely
 uploads them to a remote storage service.
 
-Tool combines the power of `rsync` for fast, incremental local backups with `rclone` for seamless remote syncing across
+This tool combines the power of `restic` for secure, incremental local backups with `rclone` for seamless remote syncing across
 cloud providers, SFTP, WebDAV, or even custom remote endpoints.
 
 #### 🔧 Features
 
-- 📁 Backup selected local directories with `rsync`
-- ☁️ Upload archives or snapshots to remote storage using `rclone`
+- 📁 Backup selected local directories with `restic`
+- ☁️ Upload backups to remote storage using `rclone`
+- 🔒 Strong encryption with `restic` for secure backups
+- 🔄 Easy restoration from any snapshot
+- 📊 Backup statistics and verification
 - 🕒 Supports scheduled (cron) execution for automated daily backups
-- 🔐 Optional encryption via `rclone`'s crypt backend
-- 🧾 Log-friendly output for easy monitoring
-- 🧰 Minimal dependencies: only `bash`, `rsync`, and `rclone`
+- 📋 Comprehensive logging system with log rotation
+- 🔃 Profile support for multiple backup configurations
+- 🚫 Lock mechanism to prevent concurrent executions
+- 🧪 Dry-run capability to preview actions
+- 🧰 Minimal dependencies: only `bash`, `restic`, and `rclone`
 
 #### 💼 Use Cases
 
 - Personal or professional workstation backups
 - Offsite backups for servers or VPS
 - Archiving dev environments, config files, or databases
+- Multi-system backup management with profiles
 
 #### Installation
 
@@ -33,6 +39,7 @@ cloud providers, SFTP, WebDAV, or even custom remote endpoints.
 
    ```bash
    echo 'export PATH="$HOME/.rearden-backup/bin:$PATH"' >> ~/.bash_profile
+   ```
 
 #### Commands
 
@@ -41,81 +48,164 @@ cloud providers, SFTP, WebDAV, or even custom remote endpoints.
 | `init`                      | Initialize the restic repository                             |
 | `backup`                    | Create a new backup                                          |
 | `restore [path] [snapshot]` | Restore from backup (defaults to latest snapshot and / path) |
-| `push`                      | Upload backup to remote storage                              |
-| `pull`                      | Download backup from remote storage                          |
+| `push`                      | Upload entire CONFIG_DIR to remote storage                   |
+| `pull`                      | Download entire CONFIG_DIR from remote storage               |
 | `list`                      | List available snapshots                                     |
 | `stats`                     | Show backup statistics                                       |
 | `verify`                    | Verify the integrity of the backup repository                |
 | `export`                    | Export backup information to a file                          |
 | `template`                  | Display a template for init.sh file                          |
 
-## Options
+#### Options
 
 | Option                  | Description                                       |
 |-------------------------|---------------------------------------------------|
 | `-p, --profile PROFILE` | Use specific profile configuration                |
 | `-d, --dry-run`         | Show what would be done without actually doing it |
-| `-v, --verbose`         | Increase verbosity                                |
+| `-v, --verbose`         | Enable basic verbosity (level 1)                  |
+| `-vv`                   | Enable more detailed verbosity (level 2)          |
+| `-vvv`                  | Enable maximum verbosity (level 3)                |
 | `-h, --help`            | Show help message                                 |
 
-## Environment Variables
-
-| Variable        | Description                         | Default   |
-|-----------------|-------------------------------------|-----------|
-| `INIT_SCRIPT`   | Path to initialization script       | `init.sh` |
-| `PROFILE`       | Default profile name                | `default` |
-| `DRY_RUN`       | Set to 1 for dry-run mode           | `0`       |
-| `VERBOSE`       | Set to 1 for verbose output         | `0`       |
-| `MAX_LOG_FILES` | Maximum number of log files to keep | `10`      |
-
-## Configuration Variables (in init.sh)
-
-| Variable               | Description                                      | Required         |
-|------------------------|--------------------------------------------------|------------------|
-| `CONFIG_DIR`           | Directory for all configuration and backup files | Yes              |
-| `BACKUP_DIRECTORIES`   | Space-separated list of directories to backup    | Yes              |
-| `RCLONE_REMOTE`        | Remote storage path for rclone                   | No               |
-| `RETENTION_DAYS`       | Number of days to keep backups                   | No (default: 30) |
-| `RESTIC_PASSWORD`      | Password for restic repository                   | No*              |
-| `RESTIC_PASSWORD_FILE` | Path to file containing restic password          | No*              |
-| `ENABLE_BACKUP`        | Enable/disable backup functionality              | No (default: 1)  |
-| `ENABLE_RESTORE`       | Enable/disable restore functionality             | No (default: 1)  |
-| `ENABLE_PUSH`          | Enable/disable push functionality                | No (default: 1)  |
-| `ENABLE_PULL`          | Enable/disable pull functionality                | No (default: 1)  |
+#### Environment Variables
+
+| Variable            | Description                                                        | Default   |
+|---------------------|--------------------------------------------------------------------|-----------|
+| `INIT_SCRIPT`       | Path to initialization script                                      | `init.sh` |
+| `PROFILE`           | Default profile name                                               | `default` |
+| `DRY_RUN`           | Set to 1 for dry-run mode                                          | `0`       |
+| `VERBOSE`           | Set verbosity level: 0=none, 1=basic, 2=detailed, 3=maximum        | `1`       |
+| `MAX_LOG_FILES`     | Maximum number of log files to keep                                | `10`      |
+| `RESTIC_REPOSITORY` | Optional: Use a remote repository directly instead of local+rclone | ` `       |
+
+#### Configuration Variables (in init.sh)
+
+| Variable               | Description                                                                    | Required        |
+|------------------------|--------------------------------------------------------------------------------|-----------------|
+| `CONFIG_DIR`           | Directory for all configuration and backup files                               | Yes             |
+| `BACKUP_DIRECTORIES`   | Space-separated list of directories to backup                                  | Yes             |
+| `LOCAL_BACKUP_REPO`    | Local repository path for restic (default: `${CONFIG_DIR}/backups/${PROFILE}`) | No              |
+| `RCLONE_REMOTE`        | Remote storage path for rclone                                                 | No              |
+| `RETENTION_DAYS`       | Number of days to keep backups                                                 | No (default: 0) |
+| `VERIFY_BACKUP`        | Verify backup after creation (1=yes, 0=no)                                     | No (default: 1) |
+| `RESTIC_PASSWORD`      | Password for restic repository                                                 | No*             |
+| `RESTIC_PASSWORD_FILE` | Path to file containing restic password                                        | No*             |
+| `ENABLE_BACKUP`        | Enable/disable backup functionality                                            | No (default: 1) |
+| `ENABLE_RESTORE`       | Enable/disable restore functionality                                           | No (default: 1) |
+| `ENABLE_PUSH`          | Enable/disable push functionality                                              | No (default: 1) |
+| `ENABLE_PULL`          | Enable/disable pull functionality                                              | No (default: 1) |
 
 *Either `RESTIC_PASSWORD` or `RESTIC_PASSWORD_FILE` must be set
 
-## Example files
+#### Backup Repository Modes
 
-Example `init.sh`
+The script supports two modes for managing the backup repository:
 
-```
+1. **Local + Remote Sync mode** (default)
+   - Backups are created locally in `LOCAL_BACKUP_REPO`
+   - The entire `CONFIG_DIR` (including the repository) is synced to/from remote with rclone
+   - Good for frequent backups with occasional remote synchronization
+
+2. **Direct Remote Repository mode**
+   - Set by defining `RESTIC_REPOSITORY` with a remote URL (s3, sftp, rest, etc.)
+   - Backup operations work directly with the remote repository
+   - Push/Pull operations are disabled in this mode
+   - Good for direct cloud backups without local copies
+
+#### Verbosity Levels
+
+The script supports multiple verbosity levels to control the amount of information displayed:
+
+- Level 0: Minimal output (only basic logs)
+- Level 1 (`-v`): Basic progress and information
+- Level 2 (`-vv`): Detailed progress and debug information
+- Level 3 (`-vvv`): Maximum verbosity for troubleshooting
+
+#### Profiles
+
+Multiple backup profiles can be defined to manage different backup scenarios:
+
+1. Create a profile file at `${CONFIG_DIR}/profiles/<profile_name>.sh`
+2. Add specific configuration variables for that profile
+3. Use the profile with `-p` or `--profile` option when running commands
+
+#### Example Files
+
+Example `init.sh`:
+
+```bash
+#!/bin/bash
+# Basic configuration for backup script
+
+# Required: Set the configuration directory
 export CONFIG_DIR=$(pwd)/local
-export BACKUP_DIRECTORIES=/projects
-export RESTIC_PASSWORD_FILE=$(pwd)/creds.txt
+
+# Directories to backup (space-separated)
+export BACKUP_DIRECTORIES="/projects /home/user/documents"
+
+# Remote repository configuration
 export RCLONE_REMOTE="remote:backup/projects-$(hostname)"
+
+# Backup retention (days)
+export RETENTION_DAYS=30
+
+# Password file for restic repository
+export RESTIC_PASSWORD_FILE="${CONFIG_DIR}/restic-password.txt"
+
+# Optional: Configure verbosity (0=none, 1=basic, 2=detailed, 3=maximum)
+export VERBOSE=1
 ```
 
-Example `local/rclone.conf` file for sftp uploads:
+Example profile for server backups (`local/profiles/server.sh`):
+
+```bash
+#!/bin/bash
+# Server-specific backup settings
 
+export BACKUP_DIRECTORIES="/etc /var/www /opt/application/data"
+export RETENTION_DAYS=60
+export VERBOSE=1
 ```
+
+Example `local/rclone.conf` file for SFTP uploads:
+
+```ini
 [remote]
 type = sftp
-host = xyz
-user = abc
+host = example.com
+user = backup-user
 port = 22
 ```
 
 Example `local/rclone.conf` file for Hetzner storage box:
-```
+
+```ini
 [remote]
 type = sftp
-host = xyz.your-storagebox.de
-user = abc
+host = u123456.your-storagebox.de
+user = u123456
 port = 23
 ```
 
-## License
+Example `local/exclude.txt` for excluding specific patterns:
 
-This code is released under the MIT License. See [LICENSE](LICENSE).
+```
+# Patterns to exclude from backup
+**/.DS_Store
+**/node_modules
+**/.git
+**/*.log
+**/tmp
+**/temp
+**/.cache
+```
+
+#### Compatibility Notes
 
+- The script is compatible with both newer and older versions of restic
+- Some advanced features (like the `stats` command) may not be available in older restic versions
+- For best results, using restic 0.12.0 or later is recommended
+
+#### License
+
+This code is released under the MIT License. See [LICENSE](LICENSE).
diff --git a/bin/rearden-backup.sh b/bin/rearden-backup.sh
@@ -81,7 +81,8 @@ ENABLE_BACKUP="1"
 ENABLE_RESTORE="1"
 ENABLE_PUSH="1"
 ENABLE_PULL="1"
-VERBOSE="${VERBOSE:-0}"
+# Verbosity level: 0=none, 1=-v, 2=-vv, 3=-vvv
+VERBOSE="${VERBOSE:-1}"  # Default to level 1 verbosity
 RESTIC_REPOSITORY="${RESTIC_REPOSITORY:-}"
 USE_REMOTE_REPO="0"
 
@@ -201,10 +202,17 @@ check_requirements() {
             local version
             if [[ "$cmd" == "restic" ]]; then
                 version=$(restic version | head -n1)
+                log "Found $version"
+
+                # Check if it's a very old version of restic that might not support some flags
+                if [[ $version =~ restic\ 0\.[0-9](\.[0-9]+)? ]]; then
+                    log_warn "You're using an older version of restic that may not support all features."
+                    log_warn "Consider upgrading to restic 0.12.0 or later for better functionality."
+                fi
             else
                 version=$(rclone --version | head -n1)
+                log "Found $version"
             fi
-            log "Found $version"
         fi
     done
 
@@ -341,8 +349,7 @@ get_repo_path() {
     fi
 }
 
-# Helper function to build restic command with proper repository flag
-# FIX: Removed the quotes within the command string
+# Helper function to build restic command with proper repository flag and verbosity
 build_restic_command() {
     local base_cmd="restic"
 
@@ -352,6 +359,17 @@ build_restic_command() {
         base_cmd+=" -r $repo_path"  # Removed the quotes that were causing the issue
     fi
 
+    # Add verbosity flags based on the VERBOSE setting
+    if [[ "$VERBOSE" -eq 1 ]]; then
+        base_cmd+=" -v"
+    elif [[ "$VERBOSE" -eq 2 ]]; then
+        base_cmd+=" -vv"
+    elif [[ "$VERBOSE" -ge 3 ]]; then
+        base_cmd+=" -vvv"
+    fi
+
+    # Note: Removed --progress flag as it's not supported in older restic versions
+
     echo "$base_cmd"
 }
 
@@ -393,16 +411,13 @@ backup() {
             log "Using exclude file: ${CONFIG_DIR}/exclude.txt"
         fi
 
-        # Build the base restic command with repo flag
+        # Build the base restic command with repo flag and verbosity already included
         local restic_cmd=$(build_restic_command)
 
         # Build the complete backup command
+        # Note: Removed --stats flag as it's not supported in older restic versions
         local cmd="$restic_cmd backup $BACKUP_DIRECTORIES $exclude_opts"
 
-        if [[ "$VERBOSE" -eq 1 ]]; then
-            cmd+=" -v"
-        fi
-
         if [[ "$DRY_RUN" -eq 1 ]]; then
             log "DRY-RUN: Would execute: $cmd"
         else
@@ -591,7 +606,15 @@ list_snapshots() {
 show_stats() {
     local restic_cmd=$(build_restic_command)
     log "Generating backup statistics:"
-    $restic_cmd stats
+
+    # Use `stats` command if available, but be prepared for it to fail in older versions
+    if $restic_cmd stats &>/dev/null; then
+        $restic_cmd stats
+    else
+        # Fallback for older restic versions that don't have the stats command
+        log "The 'stats' command is not available in your restic version."
+        log "Using snapshots command instead to show repository information:"
+    fi
 
     log "Summary of latest snapshots:"
     $restic_cmd snapshots --latest 5
@@ -614,8 +637,14 @@ export_info() {
         echo "===== Snapshots ====="
         $restic_cmd snapshots
         echo ""
+
+        # Only try to run stats if the command is available
         echo "===== Statistics ====="
-        $restic_cmd stats
+        if $restic_cmd stats &>/dev/null; then
+            $restic_cmd stats
+        else
+            echo "Statistics command not available in this restic version."
+        fi
     } > "$export_file"
 
     log_success "Exported backup information to $export_file"
@@ -684,14 +713,16 @@ Commands:
 Options:
   -p, --profile PROFILE   Use specific profile configuration
   -d, --dry-run           Show what would be done without actually doing it
-  -v, --verbose           Increase verbosity
+  -v, --verbose           Enable basic verbosity (level 1)
+  -vv                     Enable more detailed verbosity (level 2)
+  -vvv                    Enable maximum verbosity (level 3)
   -h, --help              Show this help message
 
 Environment variables:
   INIT_SCRIPT          Path to initialization script (default: init.sh)
   PROFILE              Default profile name (default: default)
   DRY_RUN              Set to 1 for dry-run mode
-  VERBOSE              Set to 1 for verbose output
+  VERBOSE              Set verbosity level: 0=none, 1=basic, 2=detailed, 3=maximum (default: 1)
   MAX_LOG_FILES        Maximum number of log files to keep (default: 10)
   RESTIC_REPOSITORY    Optional: Use a remote repository directly instead of local+rclone
 
@@ -731,6 +762,14 @@ parse_args() {
                 VERBOSE=1
                 shift
                 ;;
+            -vv)
+                VERBOSE=2
+                shift
+                ;;
+            -vvv)
+                VERBOSE=3
+                shift
+                ;;
             -h|--help)
                 print_usage
                 exit 0
