Add an exclude filter for files through DL_EXCLUDE_REGEX (Issue #27)

Author: crazy-max

.travis.yml

@@ -23,10 +23,11 @@ script:
   - ./test02_wget_sha1_text.sh
   - ./test03_curl_md5_sqlite3.sh
   - ./test04_curl_sha1_sqlite3_secure.sh
+  - ./test05_exclude.sh
 
 after_script:
   - tree -pu /opt/ftpgrab/ /var/log/ftpgrab/ $HOME/download/
 
 notifications:
-  slack:
-    secure: WuzYm7aWEyuYdLGA9PMv7RdYQxOhaJMsWbVv3aneTX+y4Pk2MdgWNoUqu87NZsbjvpmH7UpH8JxJeo4QZ2bEVgrFvuozxY8Z3D6C0ZstvWfZyHQWVVwcy0zuLxJ1lFU3nZHftLl6S5zQQ4drJd/JVaI3f7qBVUMOEUDkYh4C678=
+  webhooks:
+    secure: QjLOJOPsNZxoVcSI+fn6tWXp7TDb+t2Eykm55TI85ctb0JA8yepQDLfPT0HfnYt/Hnd8bME3No9JPE8H1kDlR+XaaG1fFWIBnQdDWySJKDIndOGb8gKLDUbvFPO/YRoLIsTrItEh//EJgSApkwqVlZbJIJ+if1lPzIsIy9/dG+3gaTc99VR7O0RZuwkkC0t+xgbMjrFpTSmgbMPAOB4jwFqrNKUQBc5Kxjd/wIqM1ZsWtQ4lFHUKQJBgTsnH5o2JMkpkwUJ+XHrWnmknJTD4mVwl1es2axUicumhERiyGN1WrDwY2kVs9UI0AqLHYNeBP3aNXV2lwngkmt59MIFpnqT6ELlOb34WKDCaNTXMzHBphtzevzXshGOUPB6be0S4ai2xukAq1rOzDhUSNzhrupxjjb8FhxHz0TtoKjcAO1ZlT+92dIgcEPx3qf3Y9e46lUKEokidmZOhwHnT0D0uF7LJ+c2z9oopG7DyzBNIVDTyitQmdlvfCUc+RJRfnDQ/8v5qoI8TzkJ6VXwpyWWdZJRJIknkWwf2BKxkGP1pCCSsfSiSKAvsRa1GPq6ztfnfC4jcrfurnSFrLmeGc4+kvDXuYiC4NZ6LoezDsjUKnR3gf1s0IvZNurcb6PA0yPIoAnOWzUYR8ObOcU1gs51zvXsDqxZC3D+O4lChHWsOy8U=

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 4.3.0 (2017/12/26)
+
+* Add an exclude filter for files through [DL_EXCLUDE_REGEX](https://ftpgrab.github.io/doc/configuration/#dl_exclude_regex) (Issue #27)
+
 ## 4.2.4 (2017/11/01)
 
 * Do not exit if connection failed

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2013-2017 Cr@zy
+Copyright (c) 2013-2017 CrazyMax
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

ftpgrab.conf

@@ -23,6 +23,7 @@ DL_USER=""
 DL_GROUP=""
 DL_CHMOD=""
 DL_REGEX=""
+DL_EXCLUDE_REGEX=""
 DL_RETRY=3
 DL_RESUME=0
 DL_SHUFFLE=0

ftpgrab.sh

@@ -2,12 +2,12 @@
 
 ###################################################################################
 #                                                                                 #
-#  FTPGrab v4.2.4                                                                 #
+#  FTPGrab v4.3.0                                                                 #
 #                                                                                 #
 #  Simple script to grab your files from a remote FTP server.                     #
 #                                                                                 #
 #  MIT License                                                                    #
-#  Copyright (c) 2013-2017 Cr@zy                                                  #
+#  Copyright (c) 2013-2017 CrazyMax                                               #
 #                                                                                 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy   #
 #  of this software and associated documentation files (the "Software"), to deal  #
@@ -282,6 +282,7 @@ function ftpgrabProcess() {
       local _SKIP_DL=0
       local _RESUME=0
       local _START_TIME=$(awk "BEGIN{srand();print srand()}")
+      local _EXCLUDE=0
       if [ ${_DEST_FILE: -1} == "/" ]
       then
         mkdir -p "$_DEST_FILE"
@@ -292,24 +293,42 @@ function ftpgrabProcess() {
         ftpgrabAddLog "Hash: $_SRC_HASH"
         ftpgrabAddLog "Size: $(ftpgrabGetHumanSize "$_SRC_FILE_PROC")"
 
-        # Check validity
-        local _DL_STATUS=$(ftpgrabIsDownloaded "$_SRC_FILE_PROC" "$_SRC_FILE")
-
-        if [ ${_DL_STATUS:0:1} -eq ${FILE_STATUS_NEVER_DL} ]
-        then
-          ftpgrabAddLog "Status: Never downloaded..."
-        elif [ ${_DL_STATUS:0:1} -eq ${FILE_STATUS_SIZE_EQUAL} ]
-        then
-          local _SKIP_DL=1
-          ftpgrabAddLog "Status: Already downloaded and valid. Skip download..."
-        elif [ ${_DL_STATUS:0:1} -eq ${FILE_STATUS_SIZE_DIFF} ]
+        # Check exclude
+        if [ ! -z "$DL_EXCLUDE_REGEX" ]
         then
-          if [ "$DL_RESUME" == "1" ]; then local _RESUME=1; fi
-          ftpgrabAddLog "Status: Exists but sizes are different..."
-        elif [ ${_DL_STATUS:0:1} -eq ${FILE_STATUS_HASH_EXISTS} ]
+          IFS=';' read -ra EXCLUDE_REGEX <<< "$DL_EXCLUDE_REGEX"
+          for _P in "${EXCLUDE_REGEX[@]}"; do
+            if $(echo "$_SRC_FILE_SHORT" | grep -Eq "^$_P$")
+            then
+              local _EXCLUDE=1
+              local _SKIP_DL=1
+              ftpgrabAddLog "Status: File skipped with exclude filter '$_P'..."
+              break
+            fi
+          done
+        fi
+
+        # Check validity
+        if [ "$_EXCLUDE" == "0" ]
         then
-          local _SKIP_DL=1
-          ftpgrabAddLog "Status: Hash sum exists. Skip download..."
+          local _DL_STATUS=$(ftpgrabIsDownloaded "$_SRC_FILE_PROC" "$_SRC_FILE")
+
+          if [ ${_DL_STATUS:0:1} -eq ${FILE_STATUS_NEVER_DL} ]
+          then
+            ftpgrabAddLog "Status: Never downloaded..."
+          elif [ ${_DL_STATUS:0:1} -eq ${FILE_STATUS_SIZE_EQUAL} ]
+          then
+            local _SKIP_DL=1
+            ftpgrabAddLog "Status: Already downloaded and valid. Skip download..."
+          elif [ ${_DL_STATUS:0:1} -eq ${FILE_STATUS_SIZE_DIFF} ]
+          then
+            if [ "$DL_RESUME" == "1" ]; then local _RESUME=1; fi
+            ftpgrabAddLog "Status: Exists but sizes are different..."
+          elif [ ${_DL_STATUS:0:1} -eq ${FILE_STATUS_HASH_EXISTS} ]
+          then
+            local _SKIP_DL=1
+            ftpgrabAddLog "Status: Hash sum exists. Skip download..."
+          fi
         fi
 
         # Check if download skipped and want to hide it in log file
@@ -501,6 +520,7 @@ DL_USER=""
 DL_GROUP=""
 DL_CHMOD=""
 DL_REGEX=""
+DL_EXCLUDE_REGEX=""
 DL_RETRY=3
 DL_RESUME=0
 DL_SHUFFLE=0
@@ -553,7 +573,7 @@ mkdir -p "$HASH_DIR"
 if [ ! -d "$HASH_DIR" ]; then ftpgrabEcho "ERROR: Cannot create dir $HASH_DIR with $(whoami) user"; exit 1; fi
 if [ ! -w "$HASH_DIR" ]; then ftpgrabEcho "ERROR: Dir $HASH_DIR is not writable by $(whoami)"; exit 1; fi
 
-ftpgrabEcho "FTPGrab v4.2.4 ($BASENAME_FILE - $(date +"%Y/%m/%d %H:%M:%S"))"
+ftpgrabEcho "FTPGrab v4.3.0 ($BASENAME_FILE - $(date +"%Y/%m/%d %H:%M:%S"))"
 ftpgrabEcho "--------------"
 
 # Check required packages

tests/test05_exclude.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+source common.sh
+TEST_NAME=$(echo $(basename $0) | sed -e 's/^[SK][0-9]*//' -e 's/\.sh$//')
+
+CONFIG_FILE="/opt/ftpgrab/conf/$TEST_NAME.conf"
+DIR_DEST="$HOME/download/$TEST_NAME"
+
+# Clean
+rm -f "$CONFIG_FILE"
+rm -rf "$DIR_DEST"
+rm -f "/opt/ftpgrab/hash/$TEST_NAME*"
+
+# Edit config
+cp -f "$DEFAULT_CONFIG_FILE" "$CONFIG_FILE"
+sed "s#DIR_DEST=\"$DEFAULT_DIR_DEST\"#DIR_DEST=\"$DIR_DEST\"#" -i "$CONFIG_FILE"
+#sed "s#DEBUG=0#DEBUG=1#" -i "$CONFIG_FILE"
+sed "s#FTP_HOST=\"$DEFAULT_FTP_HOST\"#FTP_HOST=\"$SERVER1_IP\"#" -i "$CONFIG_FILE"
+sed "s#FTP_USER=\"\"#FTP_USER=\"$SERVER1_USER\"#" -i "$CONFIG_FILE"
+sed "s#FTP_PASSWORD=\"\"#FTP_PASSWORD=\"$SERVER1_PASSWORD\"#" -i "$CONFIG_FILE"
+sed "s#DL_EXCLUDE_REGEX=\"\"#DL_EXCLUDE_REGEX=\"*.txt\"#" -i "$CONFIG_FILE"
+
+# Launch
+ftpgrab "$TEST_NAME.conf"
+
+# Check
+if [ ! -z $(find "$DIR_DEST" -name "*.txt" -print0) ]; then
+  exit 1
+fi