+BaseMailBox.(copy,move,flag,delete) chunks arg

Author: ikvk

README.rst

@@ -238,7 +238,7 @@ Action's uid_list arg may takes:
 To get uids, use the maibox methods: uids, fetch.
 
 For actions with a large number of messages imap command may be too large and will cause exception at server side,
-use 'limit' argument for fetch in this case.
+use ``chunks`` argument for ``copy,move,delete,flag`` OR ``limit`` argument for ``fetch`` in this case.
 
 .. code-block:: python
 
@@ -247,8 +247,8 @@ use 'limit' argument for fetch in this case.
         # COPY messages with uid in 23,27 from current folder to folder1
         mailbox.copy('23,27', 'folder1')
 
-        # MOVE all messages from current folder to INBOX/folder2
-        mailbox.move(mailbox.uids(), 'INBOX/folder2')
+        # MOVE all messages from current folder to INBOX/folder2, move by 100 emails at once
+        mailbox.move(mailbox.uids(), 'INBOX/folder2', chunks=100)
 
         # DELETE messages with 'cat' word in its html from current folder
         mailbox.delete([msg.uid for msg in mailbox.fetch() if 'cat' in msg.html])
@@ -429,7 +429,8 @@ Big thanks to people who helped develop this library:
 `homoLudenus <https://github.com/homoLudenus>`_,
 `sphh <https://github.com/sphh>`_,
 `bh <https://github.com/bh>`_,
-`tomasmach <https://github.com/tomasmach>`_
+`tomasmach <https://github.com/tomasmach>`_,
+`errror <https://github.com/errror>`_
 
 Help the project
 ----------------

docs/release_notes.rst

@@ -1,3 +1,13 @@
+1.10.0
+======
+* Added: support IMAP command MOVE at BaseMailBox.move
+* Added: MailboxMoveError, raises from BaseMailBox.move when MOVE command is supported
+* Added: chunks argument for BaseMailBox.(copy,move,flag,delete) methods - Number of UIDs to proc at once, to avoid server errors on large set
+* Changed: BaseMailBox.(copy,move,flag,delete) result types
+* Changed: utils.clean_uids now returns List[str]
+* Changed: utils.chunks_crop -> utils.chunked_crop, n arg renamed to chunk_size and it takes False-like vals
+* Renamed: utils.chunks -> utils.chunked
+
 1.9.1
 =====
 * Replaced: functools.lru_cache to functools.cached_property

imap_tools/__init__.py

@@ -11,4 +11,4 @@
 from .utils import EmailAddress
 from .errors import *
 
-__version__ = '1.9.1'
+__version__ = '1.10.0'

imap_tools/consts.py

@@ -9,6 +9,7 @@
 
 PYTHON_VERSION_MINOR = int(sys.version_info.minor)
 
+MOVE_RESULT_TAG = ('_MOVE',)  # const delete_result part for mailbox.move result, when server have MOVE in capabilities
 
 class MailMessageFlags:
     """

imap_tools/errors.py

@@ -85,6 +85,10 @@ class MailboxCopyError(UnexpectedCommandStatusError):
     pass
 
 
+class MailboxMoveError(UnexpectedCommandStatusError):
+    pass
+
+
 class MailboxFlagError(UnexpectedCommandStatusError):
     pass
 

imap_tools/mailbox.py

@@ -7,12 +7,12 @@
 from .message import MailMessage
 from .folder import MailBoxFolderManager
 from .idle import IdleManager
-from .consts import UID_PATTERN, PYTHON_VERSION_MINOR
-from .utils import clean_uids, check_command_status, chunks, encode_folder, clean_flags, check_timeout_arg_support, \
-    chunks_crop, StrOrBytes
+from .consts import UID_PATTERN, PYTHON_VERSION_MINOR, MOVE_RESULT_TAG
+from .utils import clean_uids, check_command_status, chunked, encode_folder, clean_flags, check_timeout_arg_support, \
+    chunked_crop, StrOrBytes
 from .errors import MailboxStarttlsError, MailboxLoginError, MailboxLogoutError, MailboxNumbersError, \
     MailboxFetchError, MailboxExpungeError, MailboxDeleteError, MailboxCopyError, MailboxFlagError, \
-    MailboxAppendError, MailboxUidsError, MailboxTaggedResponseError
+    MailboxAppendError, MailboxUidsError, MailboxTaggedResponseError, MailboxMoveError
 
 # Maximal line length when calling readline(). This is to prevent reading arbitrary length lines.
 # 20Mb is enough for search response with about 2 000 000 message numbers
@@ -153,7 +153,7 @@ def _fetch_in_bulk(self, uid_list: Sequence[str], message_parts: str, reverse: b
             return
 
         if isinstance(bulk, int) and bulk >= 2:
-            uid_list_seq = chunks_crop(uid_list, bulk)
+            uid_list_seq = chunked_crop(uid_list, bulk)
         elif isinstance(bulk, bool):
             uid_list_seq = (uid_list,)
         else:
@@ -164,7 +164,7 @@ def _fetch_in_bulk(self, uid_list: Sequence[str], message_parts: str, reverse: b
             check_command_status(fetch_result, MailboxFetchError)
             if not fetch_result[1] or fetch_result[1][0] is None:
                 return
-            for built_fetch_item in chunks((reversed if reverse else iter)(fetch_result[1]), 2):
+            for built_fetch_item in chunked((reversed if reverse else iter)(fetch_result[1]), 2):
                 yield built_fetch_item
 
     def fetch(self, criteria: Criteria = 'ALL', charset: str = 'US-ASCII', limit: Optional[Union[int, slice]] = None,
@@ -204,62 +204,105 @@ def expunge(self) -> tuple:
         check_command_status(result, MailboxExpungeError)
         return result
 
-    def delete(self, uid_list: Union[str, Iterable[str]]) -> Optional[Tuple[tuple, tuple]]:
+    def delete(self, uid_list: Union[str, Iterable[str]], chunks: Optional[int] = None) \
+            -> Optional[List[Tuple[tuple, tuple]]]:
         """
         Delete email messages
         Do nothing on empty uid_list
+        :param uid_list: UIDs for delete
+        :param chunks: Number of UIDs to process at once, to avoid server errors on large set. Proc all at once if None.
         :return: None on empty uid_list, command results otherwise
         """
-        uid_str = clean_uids(uid_list)
-        if not uid_str:
+        cleaned_uid_list = clean_uids(uid_list)
+        if not cleaned_uid_list:
             return None
-        store_result = self.client.uid('STORE', uid_str, '+FLAGS', r'(\Deleted)')
-        check_command_status(store_result, MailboxDeleteError)
-        expunge_result = self.expunge()
-        return store_result, expunge_result
-
-    def copy(self, uid_list: Union[str, Iterable[str]], destination_folder: StrOrBytes) -> Optional[tuple]:
+        results = []
+        for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks):
+            store_result = self.client.uid('STORE', ','.join(cleaned_uid_list_i), '+FLAGS', r'(\Deleted)')
+            check_command_status(store_result, MailboxDeleteError)
+            expunge_result = self.expunge()
+            results.append((store_result, expunge_result))
+        return results
+
+    def copy(self, uid_list: Union[str, Iterable[str]], destination_folder: StrOrBytes, chunks: Optional[int] = None) \
+            -> Optional[List[tuple]]:
         """
-        Copy email messages into the specified folder
-        Do nothing on empty uid_list
+        Copy email messages into the specified folder.
+        Do nothing on empty uid_list.
+        :param uid_list: UIDs for copy
+        :param destination_folder: Folder for email copies
+        :param chunks: Number of UIDs to process at once, to avoid server errors on large set. Proc all at once if None.
         :return: None on empty uid_list, command results otherwise
         """
-        uid_str = clean_uids(uid_list)
-        if not uid_str:
+        cleaned_uid_list = clean_uids(uid_list)
+        if not cleaned_uid_list:
             return None
-        copy_result = self.client.uid('COPY', uid_str, encode_folder(destination_folder))  # noqa
-        check_command_status(copy_result, MailboxCopyError)
-        return copy_result
-
-    def move(self, uid_list: Union[str, Iterable[str]], destination_folder: StrOrBytes) -> Optional[Tuple[tuple, tuple]]:
+        results = []
+        for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks):
+            copy_result = self.client.uid(
+                'COPY', ','.join(cleaned_uid_list_i), encode_folder(destination_folder))  # noqa
+            check_command_status(copy_result, MailboxCopyError)
+            results.append(copy_result)
+        return results
+
+    def move(self, uid_list: Union[str, Iterable[str]], destination_folder: StrOrBytes, chunks: Optional[int] = None) \
+            -> Optional[List[Tuple[tuple, tuple]]]:
         """
-        Move email messages into the specified folder
-        Do nothing on empty uid_list
+        Move email messages into the specified folder.
+        Do nothing on empty uid_list.
+        :param uid_list: UIDs for move
+        :param destination_folder: Folder for move to
+        :param chunks: Number of UIDs to process at once, to avoid server errors on large set. Proc all at once if None.
         :return: None on empty uid_list, command results otherwise
         """
-        uid_str = clean_uids(uid_list)
-        if not uid_str:
+        cleaned_uid_list = clean_uids(uid_list)
+        if not cleaned_uid_list:
             return None
-        copy_result = self.copy(uid_str, destination_folder)
-        delete_result = self.delete(uid_str)
-        return copy_result, delete_result
-
-    def flag(self, uid_list: Union[str, Iterable[str]], flag_set: Union[str, Iterable[str]], value: bool) \
-            -> Optional[Tuple[tuple, tuple]]:
+        if 'MOVE' in self.client.capabilities:
+            # server side move
+            results = []
+            for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks):
+                move_result = self.client.uid(
+                    'MOVE', ','.join(cleaned_uid_list_i), encode_folder(destination_folder))  # noqa
+                check_command_status(move_result, MailboxMoveError)
+                results.append((move_result, MOVE_RESULT_TAG))
+            return results
+        else:
+            # client side move
+            results = []
+            for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks):
+                copy_result = self.copy(cleaned_uid_list_i, destination_folder)
+                delete_result = self.delete(cleaned_uid_list_i)
+                results.append((copy_result, delete_result))
+            return results
+
+    def flag(self, uid_list: Union[str, Iterable[str]], flag_set: Union[str, Iterable[str]], value: bool,
+             chunks: Optional[int] = None) -> Optional[List[Tuple[tuple, tuple]]]:
         """
-        Set/unset email flags
-        Do nothing on empty uid_list
+        Set/unset email flags.
+        Do nothing on empty uid_list.
         System flags contains in consts.MailMessageFlags.all
+        :param uid_list: UIDs for set flag
+        :param flag_set: Flags for operate
+        :param value: Should the flags be set: True - yes, False - no
+        :param chunks: Number of UIDs to process at once, to avoid server errors on large set. Proc all at once if None.
         :return: None on empty uid_list, command results otherwise
         """
-        uid_str = clean_uids(uid_list)
-        if not uid_str:
+        cleaned_uid_list = clean_uids(uid_list)
+        if not cleaned_uid_list:
             return None
-        store_result = self.client.uid(
-            'STORE', uid_str, ('+' if value else '-') + 'FLAGS', f'({" ".join(clean_flags(flag_set))})')
-        check_command_status(store_result, MailboxFlagError)
-        expunge_result = self.expunge()
-        return store_result, expunge_result
+        results = []
+        for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks):
+            store_result = self.client.uid(
+                'STORE',
+                ','.join(cleaned_uid_list_i),
+                ('+' if value else '-') + 'FLAGS',
+                f'({" ".join(clean_flags(flag_set))})'
+            )
+            check_command_status(store_result, MailboxFlagError)
+            expunge_result = self.expunge()
+            results.append((store_result, expunge_result))
+        return results
 
     def append(self, message: Union[MailMessage, bytes],
                folder: StrOrBytes = 'INBOX',

imap_tools/query.py

@@ -209,7 +209,7 @@ def cleaned_uid(key: str, value: Union[str, Iterable[str], UidRange]) -> str:
             return str(value)
         # set
         try:
-            return clean_uids(value)
+            return ','.join(clean_uids(value))
         except TypeError as e:
             raise TypeError(f'{key} parse error: {str(e)}')
 

imap_tools/utils.py

@@ -4,35 +4,35 @@
 from itertools import zip_longest
 from email.utils import getaddresses, parsedate_to_datetime
 from email.header import decode_header, Header
-from typing import Union, Optional, Tuple, Iterable, Any, List, Dict, Iterator
+from typing import Union, Optional, Tuple, Iterable, Any, List, Dict, Iterator, Sequence
 
 from .consts import SHORT_MONTH_NAMES, MailMessageFlags
 from .imap_utf7 import utf7_encode
 
 StrOrBytes = Union[str, bytes]
 
 
-def clean_uids(uid_set: Union[str, Iterable[str]]) -> str:
+def clean_uids(uid_set: Union[str, Iterable[str]]) -> List[str]:
     """
     Prepare set of uid for use in IMAP commands
     uid RE patterns are not strict and allow invalid combinations, but simple. Example: 2,4:7,9,12:*
     :param uid_set:
         str, that is comma separated uids
         Iterable, that contains str uids
-    :return: str - uids, concatenated by a comma
+    :return: list of str - cleaned uids
     """
     # str
     if type(uid_set) is str:
         if re.search(r'^([\d*:]+,)*[\d*:]+$', uid_set):  # *optimization for already good str
-            return uid_set
+            return uid_set.split(',')
         uid_set = uid_set.split(',')
     # check uid types
     for uid in uid_set:
         if type(uid) is not str:
             raise TypeError(f'uid "{str(uid)}" is not string')
         if not re.match(r'^[\d*:]+$', uid.strip()):
             raise TypeError(f'Wrong uid: "{uid}"')
-    return ','.join(i.strip() for i in uid_set)
+    return [i.strip() for i in uid_set]
 
 
 def check_command_status(command_result: tuple, exception: type, expected='OK'):
@@ -205,23 +205,27 @@ def replace_html_ct_charset(html: str, new_charset: str) -> str:
     return html
 
 
-def chunks(iterable: Iterable[Any], n: int, fill_value: Optional[Any] = None) -> Iterator[Tuple[Any, ...]]:
+def chunked(iterable: Iterable[Any], n: int, fill_value: Optional[Any] = None) -> Iterator[Tuple[Any, ...]]:
     """
     Group data into fixed-length chunks or blocks
         [iter(iterable)]*n creates one iterator, repeated n times in the list
         izip_longest then effectively performs a round-robin of "each" (same) iterator
     Examples:
-        chunks('ABCDEFGH', 3, '?') --> [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'H', '?')]
-        chunks([1, 2, 3, 4, 5], 2) --> [(1, 2), (3, 4), (5, None)]
+        chunked('ABCDEFGH', 3, '?') --> [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'H', '?')]
+        chunked([1, 2, 3, 4, 5], 2) --> [(1, 2), (3, 4), (5, None)]
     """
     return zip_longest(*[iter(iterable)] * n, fillvalue=fill_value)
 
 
-def chunks_crop(lst: iter, n: int) -> iter:
+def chunked_crop(seq: Sequence, chunk_size: Optional[int]) -> Iterator[list]:
     """
-    Yield successive n-sized chunks from lst.
+    Yield successive n-sized chunks from seq.
+    Yield seq if chunk_size is False-like
+    :param seq: Sequence to chunks
+    :param chunk_size: chunk size
+    :return: Iterator
     import pprint
-    pprint.pprint(list(chunks(range(10, 75), 10)))
+    pprint.pprint(list(chunked_crop(range(10, 75), 10)))
     [[10, 11, 12, 13, 14, 15, 16, 17, 18, 19],
      [20, 21, 22, 23, 24, 25, 26, 27, 28, 29],
      [30, 31, 32, 33, 34, 35, 36, 37, 38, 39],
@@ -230,5 +234,10 @@ def chunks_crop(lst: iter, n: int) -> iter:
      [60, 61, 62, 63, 64, 65, 66, 67, 68, 69],
      [70, 71, 72, 73, 74]]
     """
-    for i in range(0, len(lst), n):
-        yield lst[i:i + n]
+    if not chunk_size:
+        yield seq
+        return
+    if chunk_size < 0:
+        raise ValueError('False-like or int>=0 expected')
+    for i in range(0, len(seq), chunk_size):
+        yield seq[i:i + chunk_size]