Merge pull request #38 from dyne/sieve-import

feat: import Jaromail sieve rules

Author: jaromil

.github/workflows/pr-tests.yml

@@ -27,6 +27,7 @@ jobs:
           chmod +x extras/test/run-source.sh
           chmod +x extras/test/test-addressbook-parse.sh
           chmod +x extras/test/test-filtering.sh
+          chmod +x extras/test/test-sieve-import.sh
           chmod +x extras/test/test-helpers.sh
 
       - name: Build
@@ -41,5 +42,8 @@ jobs:
       - name: Filtering tests
         run: extras/test/test-filtering.sh
 
+      - name: Sieve import tests
+        run: extras/test/test-sieve-import.sh
+
       - name: Helper binary tests
         run: extras/test/test-helpers.sh

extras/shell_completion/_jaromail

@@ -16,7 +16,7 @@ _jaromail() {
 
     case $state in
 	    commands)
-            _arguments '1:Commands:(open compose fetch send peek search passwd abook extract import backup merge update filter)'
+            _arguments '1:Commands:(open compose fetch send peek search passwd abook extract import backup merge update sieve sieve-import filter)'
 	        ;;
 	    *)
             _last=$(( ${#words} - 1 ))
@@ -55,6 +55,10 @@ _jaromail() {
                     _multi_parts . md
 		            ;;
 
+                sieve-import)
+                    _files -g '*.sieve(-.)'
+                    ;;
+
 		        compose)
                     _adrs=("${(@f)$(jaro search addr . 2>/dev/null)}")
                     compadd -X "Recipient addresses:" -a _adrs

extras/shell_completion/jaromail.bash

@@ -16,7 +16,7 @@ _jaro() {  #+ starts with an underscore.
 
   case "$cur" in
       *)
-	  COMPREPLY=( $( compgen -W 'fetch send peek compose open' -- $cur ) )
+	  COMPREPLY=( $( compgen -W 'fetch send peek compose open update sieve sieve-import filter' -- $cur ) )
 	  ;;
 
       open)
@@ -39,4 +39,3 @@ echo "current array: ${COMP_WORDS[@]}"
 }
 
 complete -F _jaro -o filenames jaro
-

extras/test/test-sieve-import.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env zsh
+
+set -euo pipefail
+
+script_dir="${0:A:h}"
+source "${script_dir}/lib/test_helpers.zsh"
+test_setup "${0}"
+
+cleanup() {
+  test_cleanup
+}
+trap cleanup EXIT INT TERM
+
+missing_cmds=()
+for req in pinentry fetchmail gpg msmtp notmuch abook; do
+  command -v "${req}" >/dev/null 2>&1 || missing_cmds+=("${req}")
+done
+if (( ${#missing_cmds[@]} > 0 )); then
+  print -- "SKIP test-sieve-import.sh: missing runtime commands: ${missing_cmds[*]}"
+  exit 0
+fi
+
+jaro_source init >/dev/null
+: > "${mail_root}/Filters.txt"
+: > "${mail_root}/Aliases.txt"
+
+sieve_file="${tmp_root}/import.sieve"
+cat > "${sieve_file}" <<'EOF'
+# mailbox supports fileinto :create
+require ["fileinto","mailbox","variables"];
+
+# zz.blacklist
+if header :contains "From" [
+"blocked@example.org",
+"blocked@example.org"
+]
+{ fileinto :create "zz.blacklist"; stop; }
+
+# bounces
+if header :contains "Sender" "mailman-bounce" {
+    fileinto :create "zz.bounces";
+    stop;
+}
+
+#############
+# own filters
+
+if header :contains [ "To","Cc" ]  "team@example.org" { fileinto :create "team"; stop; }
+if header :contains [ "From","Sender" ]  "list@example.org" { fileinto :create "lists"; stop; }
+
+# INBOX
+if header :contains [ "From","Sender" ] [
+"Known Person <known@example.org>",
+"known@example.org"
+]
+{ fileinto :create "INBOX"; stop; }
+
+# spam
+if header :is "X-Spam-Flag" "YES" {
+    fileinto :create "Spam"; stop;
+}
+
+# priv
+if header :contains [ "To","Cc" ]  [
+"alias@example.org",
+"unknown@gmail.com"
+]
+{ fileinto :create "priv"; stop; }
+
+fileinto :create "unsorted";
+EOF
+
+jaro_source sieve-import "${sieve_file}" >/dev/null
+jaro_source sieve-import "${sieve_file}" >/dev/null
+
+filters="$(grep -v '^#' "${mail_root}/Filters.txt" | grep -v '^$' | sort)"
+assert_equal "${filters}" $'from list@example.org move lists\nto team@example.org move team' "import filters"
+
+alias_lines="$(grep -v '^#' "${mail_root}/Aliases.txt" | grep -v '^$' | sort)"
+assert_equal "${alias_lines}" "alias@example.org" "import aliases"
+
+whitelist="$(jaro_source addr 2>/dev/null | sort)"
+assert_equal "${whitelist}" "Known Person <known@example.org>" "import whitelist"
+
+blacklist="$(jaro_source -l blacklist addr 2>/dev/null | sort)"
+assert_equal "${blacklist}" "blocked <blocked@example.org>" "import blacklist"
+
+jaro_source update >/dev/null
+
+generated="$(cat "${mail_root}/Filters.sieve")"
+assert_contains "${generated}" '"team@example.org" { fileinto :create "team"; stop; }' "round-trip to filter"
+assert_contains "${generated}" '"list@example.org" { fileinto :create "lists"; stop; }' "round-trip from filter"
+assert_contains "${generated}" '"alias@example.org"' "round-trip alias"
+assert_contains "${generated}" '"known@example.org"' "round-trip whitelist"
+assert_contains "${generated}" '"blocked@example.org"' "round-trip blacklist"

src/jaro

@@ -108,6 +108,13 @@ a pipe | in front indicate they take an email body from stdin
  update   updates all filter engine according to Filters.txt
           (also generates Sieve format rules ready for server use)
 
+ sieve    regenerate $MAILDIRS/Filters.sieve from local filters,
+          aliases, and addressbooks without updating the mail index
+
+ sieve-import FILE
+          import a Jaromail-generated Sieve file into Filters.txt,
+          Aliases.txt, whitelist.abook, and blacklist.abook
+
  filter   process a maildir distributing emails according to Filters.txt
           (if none specified, processes incoming/ - called by fetch)
 
@@ -136,6 +143,7 @@ main() {
 
     option_subcommands[update]=""
     option_subcommands[sieve]=""
+    option_subcommands[sieve-import]=""
 
     option_subcommands[stat]=""
 
@@ -269,6 +277,8 @@ main() {
 
 	sieve) cmd_sieve ;;
 
+    sieve-import) cmd_sieve_import ;;
+
     help) cmd_help ;;
 
     index) cmd_index ;;

src/zlibs/filters

@@ -105,12 +105,16 @@ update_filters() {
     newlock "$ff"
     cat <<EOF >> "$ff"
 # generated on `date`
-typeset -Al  filter_from
-typeset -alU filter_own
-typeset -Al  filter_to
+typeset -gAl  filter_from
+typeset -galU filter_own
+typeset -gAl  filter_to
+typeset -galU filter_whitelist
+typeset -galU filter_blacklist
 filter_from=()
 filter_own=()
 filter_to=()
+filter_whitelist=()
+filter_blacklist=()
 
 EOF
 
@@ -587,10 +591,16 @@ if header :contains "Sender" "mailman-bounce" {
 
 EOF
 
-    set -A sieve_filter_map ${(kv)filter_to}
+    sieve_filter_map=()
+    for fil in ${(k)filter_to}; do
+        sieve_filter_map[$fil]="${filter_to[$fil]}"
+    done
     sieve_complex_filter 'if header :contains [ "To","Cc" ] '
 
-    set -A sieve_filter_map ${(kv)filter_from}
+    sieve_filter_map=()
+    for fil in ${(k)filter_from}; do
+        sieve_filter_map[$fil]="${filter_from[$fil]}"
+    done
     sieve_complex_filter 'if header :contains [ "From","Sender" ] '
 
     ##############################################################
@@ -635,6 +645,164 @@ EOF
     return 0
 } # end of update()
 
+sieve_import_addressbook_entry() {
+    local target_list="$1"
+    local raw_address="$2"
+    local old_list="$list"
+    local old_addressbook="$ADDRESSBOOK"
+
+    list="$target_list"
+    ADDRESSBOOK="${target_list}.abook"
+    [[ -r "$MAILDIRS/$ADDRESSBOOK" ]] || create_addressbook
+
+    e_addr=()
+    print - "From: $raw_address" | e_parse From > /dev/null || {
+        list="$old_list"
+        ADDRESSBOOK="$old_addressbook"
+        return 1
+    }
+
+    local email name
+    for email in ${(k)e_addr}; do
+        name="${e_addr[$email]}"
+        [[ -z "$name" ]] && name="$email"
+        lookup_email "$email"
+        [[ $? = 0 ]] || insert_address "$email" "$name" > /dev/null
+    done
+
+    list="$old_list"
+    ADDRESSBOOK="$old_addressbook"
+    return 0
+}
+
+sieve_import_grouped_values() {
+    local sieve_file="$1"
+    awk '
+/^# / {
+    section=substr($0, 3)
+    next
+}
+/^[[:space:]]*"/ {
+    value=$0
+    sub(/^[[:space:]]*"/, "", value)
+    sub(/".*/, "", value)
+    print section "|" value
+}
+' "$sieve_file"
+}
+
+sieve_import_route_rules() {
+    local sieve_file="$1"
+    awk '
+/header :contains/ && /fileinto :create/ {
+    destination=$0
+    sub(/^.*fileinto :create "/, "", destination)
+    sub(/".*/, "", destination)
+
+    if (destination == "INBOX" || destination == "priv" ||
+        destination == "Spam" || destination == "unsorted" ||
+        destination == "zz.blacklist" || destination == "zz.bounces") {
+        next
+    }
+
+    if ($0 ~ /\[ "To","Cc" \]/) {
+        header="to"
+    } else if ($0 ~ /\[ "From","Sender" \]/) {
+        header="from"
+    } else {
+        next
+    }
+
+    pattern=$0
+    sub(/^.*\][[:space:]]*"/, "", pattern)
+    sub(/".*/, "", pattern)
+    if (pattern != "") {
+        print header " " pattern " move " destination
+    }
+}
+' "$sieve_file"
+}
+
+sieve_import_append_unique() {
+    local target_file="$1"
+    local line="$2"
+
+    touch "$target_file"
+    grep -Fqx "$line" "$target_file" 2>/dev/null || print - "$line" >> "$target_file"
+}
+
+sieve_import_is_account_address() {
+    local needle="${1:l}"
+
+    [[ -d "$MAILDIRS/Accounts" ]] || return 1
+    find "$MAILDIRS/Accounts" -type f -exec awk '/^email/ { print tolower($2) }' {} \; | \
+        grep -Fqx "$needle"
+}
+
+sieve_import_alias_entry() {
+    local raw_address="$1"
+
+    e_addr=()
+    print - "From: $raw_address" | e_parse From > /dev/null || return 1
+
+    local email
+    for email in ${(k)e_addr}; do
+        sieve_import_is_account_address "$email" && continue
+        sieve_import_append_unique "$MAILDIRS/Aliases.txt" "$email"
+    done
+    return 0
+}
+
+sieve_import() {
+    fn sieve_import $*
+    local sieve_file="$1"
+    req=(MAILDIRS sieve_file)
+    freq=($sieve_file)
+    ckreq || return 1
+
+    [[ -r "$sieve_file" ]] || {
+        error "Sieve file not readable: $sieve_file"
+        return 1
+    }
+
+    notice "Import Sieve filters from $sieve_file"
+    local section value entry
+    local imported_blacklist=0
+    local imported_whitelist=0
+    local imported_filters=0
+    local imported_aliases=0
+
+    for entry in ${(f)"$(sieve_import_grouped_values "$sieve_file")"}; do
+        section="${entry%%|*}"
+        value="${entry#*|}"
+        case "$section" in
+            zz.blacklist)
+                sieve_import_addressbook_entry blacklist "$value" && \
+                    imported_blacklist=$(( $imported_blacklist + 1 ))
+                ;;
+            INBOX)
+                sieve_import_addressbook_entry whitelist "$value" && \
+                    imported_whitelist=$(( $imported_whitelist + 1 ))
+                ;;
+            priv)
+                sieve_import_alias_entry "$value" && \
+                    imported_aliases=$(( $imported_aliases + 1 ))
+                ;;
+        esac
+    done
+
+    for entry in ${(f)"$(sieve_import_route_rules "$sieve_file")"}; do
+        sieve_import_append_unique "$MAILDIRS/Filters.txt" "$entry"
+        imported_filters=$(( $imported_filters + 1 ))
+    done
+
+    act "blacklist addresses imported: $imported_blacklist"
+    act "whitelist addresses imported: $imported_whitelist"
+    act "aliases imported: $imported_aliases"
+    act "filter rules imported: $imported_filters"
+    return 0
+}
+
 cmd_filter() {
     filter_maildir ${option_params}
     exitcode=$?

src/zlibs/setup

@@ -45,3 +45,8 @@ cmd_sieve() {
     update_sieve
     notice "Sieve filters updated in $MAILDIRS"
 }
+
+cmd_sieve_import() {
+    sieve_import ${option_params}
+    exitcode=$?
+}