Merge branch 'backports-7.1' into 7.1.x

Bundled all backported PRs from master to avoid running down Travis CI
build credits, since they're strictly rationed now.

Author: dgw

sopel/cli/plugins.py

@@ -212,12 +212,15 @@ def handle_list(options):
                 else:
                     description['name'] = utils.red(name)
         except Exception as error:
-            label = ('%s' % error) or 'unknown loading exception'
+            description.update(plugin.get_meta_description())
+
+            meta_label = description.get('label') or '(error)'
+            error_label = ('%s' % error) or 'unknown loading exception'
+            label = '%s (error: %s)' % (meta_label, error_label)
             error_status = 'error'
+
             description.update({
-                'label': 'Error: %s' % label,
-                'type': 'unknown',
-                'source': 'unknown',
+                'label': label,
                 'status': error_status,
             })
             if not no_color:
@@ -268,12 +271,15 @@ def handle_show(options):
         description.update(plugin.get_meta_description())
         loaded = True
     except Exception as error:
-        label = ('%s' % error) or 'unknown loading exception'
+        description.update(plugin.get_meta_description())
+
+        meta_label = description.get('label') or '(error)'
+        error_label = ('%s' % error) or 'unknown loading exception'
+        label = '%s (error: %s)' % (meta_label, error_label)
         error_status = 'error'
+
         description.update({
-            'label': 'Error: %s' % label,
-            'type': 'unknown',
-            'source': 'unknown',
+            'label': label,
             'status': error_status,
         })
 

sopel/config/types.py

@@ -279,6 +279,7 @@ def _serialize_boolean(value):
 @deprecated(
     reason='Use BooleanAttribute instead of ValidatedAttribute with parse=bool',
     version='7.1',
+    warning_in='8.0',
     removed_in='9.0',
     stack_frame=-2,
 )

sopel/modules/announce.py

@@ -9,6 +9,8 @@
 """
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import itertools
+
 from sopel import plugin
 
 
@@ -18,10 +20,18 @@ def _chunks(items, size):
     :param items: the collection of items to chunk
     :type items: :term:`iterable`
     :param int size: the size of each chunk
+    :return: a :term:`generator` of chunks
+    :rtype: :term:`generator` of :class:`tuple`
     """
-    # from https://stackoverflow.com/a/312464/5991 with modified names for readability
-    for delim in range(0, len(items), size):
-        yield items[delim:delim + size]
+    # This approach is safer than slicing with non-subscriptable types,
+    # for example `dict_keys` objects
+    iterator = iter(items)
+    # TODO: Simplify to assignment expression (`while cond := expr`)
+    # when dropping Python 3.7
+    chunk = tuple(itertools.islice(iterator, size))
+    while chunk:
+        yield chunk
+        chunk = tuple(itertools.islice(iterator, size))
 
 
 @plugin.command('announce')

sopel/modules/reddit.py

@@ -39,7 +39,7 @@
 
 domain = r'https?://(?:www\.|old\.|pay\.|ssl\.|[a-z]{2}\.)?reddit\.com'
 subreddit_url = r'%s/r/([\w-]+)/?$' % domain
-post_url = r'%s/r/\S+?/comments/([\w-]+)(?:/[\w%%]+)?/?$' % domain
+post_url = r'%s/r/\S+?/comments/([\w-]+)(?:/[\w%%]+)?/?' % domain
 short_post_url = r'https?://redd\.it/([\w-]+)'
 user_url = r'%s/u(?:ser)?/([\w-]+)' % domain
 comment_url = r'%s/r/\S+?/comments/\S+?/\S+?/([\w-]+)' % domain
@@ -55,6 +55,7 @@ def setup(bot):
             user_agent=USER_AGENT,
             client_id='6EiphT6SSQq7FQ',
             client_secret=None,
+            check_for_updates=False,
         )
 
 

sopel/modules/tell.py

@@ -15,8 +15,9 @@
 import os
 import threading
 import time
+import unicodedata
 
-from sopel import plugin, tools
+from sopel import formatting, plugin, tools
 from sopel.config import types
 from sopel.tools.time import format_time, get_timezone
 
@@ -130,6 +131,43 @@ def shutdown(bot):
             pass
 
 
+def _format_safe_lstrip(text):
+    """``str.lstrip()`` but without eating IRC formatting.
+
+    :param str text: text to clean
+    :rtype: str
+    :raises TypeError: if the passed ``text`` is not a string
+
+    Stolen and tweaked from the ``choose`` plugin's ``_format_safe()``
+    function by the person who wrote it.
+    """
+    if not isinstance(text, str):
+        raise TypeError("A string is required.")
+    elif not text:
+        # unnecessary optimization
+        return ''
+
+    start = 0
+
+    # strip left
+    pos = 0
+    while pos < len(text):
+        is_whitespace = unicodedata.category(text[pos]) == 'Zs'
+        is_non_printing = (
+            text[pos] in formatting.CONTROL_NON_PRINTING and
+            text[pos] not in formatting.CONTROL_FORMATTING
+        )
+        if not is_whitespace and not is_non_printing:
+            start = pos
+            break
+        pos += 1
+    else:
+        # skipped everything; string is all whitespace
+        return ''
+
+    return text[start:]
+
+
 @plugin.command('tell', 'ask')
 @plugin.nickname_command('tell', 'ask')
 @plugin.example('$nickname, tell dgw he broke something again.')
@@ -143,7 +181,7 @@ def f_remind(bot, trigger):
         return
 
     tellee = trigger.group(3).rstrip('.,:;')
-    msg = trigger.group(2).lstrip(tellee).lstrip()
+    msg = _format_safe_lstrip(trigger.group(2).split(' ', 1)[1])
 
     if not msg:
         bot.reply("%s %s what?" % (verb, tellee))

sopel/modules/translate.py

@@ -240,9 +240,11 @@ def mangle(bot, trigger):
     """Repeatedly translate the input until it makes absolutely no sense."""
     long_lang_list = ['fr', 'de', 'es', 'it', 'no', 'he', 'la', 'ja', 'cy', 'ar', 'yi', 'zh', 'nl', 'ru', 'fi', 'hi', 'af', 'jw', 'mr', 'ceb', 'cs', 'ga', 'sv', 'eo', 'el', 'ms', 'lv']
     lang_list = []
+
     for __ in range(0, 8):
         lang_list = get_random_lang(long_lang_list, lang_list)
     random.shuffle(lang_list)
+
     if trigger.group(2) is None:
         try:
             phrase = (bot.memory['mangle_lines'][trigger.sender], '')
@@ -251,9 +253,11 @@ def mangle(bot, trigger):
             return
     else:
         phrase = (trigger.group(2).strip(), '')
+
     if phrase[0] == '':
         bot.reply("What do you want me to mangle?")
         return
+
     for lang in lang_list:
         backup = phrase
         try:
@@ -274,6 +278,12 @@ def mangle(bot, trigger):
             phrase = backup
             break
 
+    if phrase[0] is None:
+        # translate() returns (None, None) if an error happens,
+        # usually because the bot has exceeded a rate limit
+        bot.reply("Translation rate limit reached. Try again later.")
+        return
+
     bot.say(phrase[0])
 
 

sopel/modules/wikipedia.py

@@ -35,6 +35,7 @@ def __init__(self, section_name):
         self.section_name = section_name
 
         self.citations = False
+        self.messagebox = False
         self.span_depth = 0
         self.div_depth = 0
 
@@ -55,14 +56,35 @@ def handle_starttag(self, tag, attrs):
                     if attr[0] == 'class' and 'edit' in attr[1]:
                         self.span_depth += 1
 
-        elif tag == 'div':  # We want to skip thumbnail text and the inexplicable table of contents, and as such also need to track div depth
+        elif tag == 'div':
+            # We want to skip thumbnail text and the inexplicable table of contents,
+            # and as such also need to track div depth
             if self.div_depth:
                 self.div_depth += 1
             else:
                 for attr in attrs:
                     if attr[0] == 'class' and ('thumb' in attr[1] or attr[1] == 'toc'):
                         self.div_depth += 1
 
+        elif tag == 'table':
+            # Message box templates are what we want to ignore here
+            for attr in attrs:
+                if (
+                    attr[0] == 'class'
+                    and any(classname in attr[1].lower() for classname in [
+                        # Most of list from https://en.wikipedia.org/wiki/Template:Mbox_templates_see_also
+                        'ambox',  # messageboxes on article pages
+                        'cmbox',  # messageboxes on category pages
+                        'imbox',  # messageboxes on file (image) pages
+                        'tmbox',  # messageboxes on talk pages
+                        'fmbox',  # header and footer messageboxes
+                        'ombox',  # messageboxes on other types of page
+                        'mbox',  # for messageboxes that are used in different namespaces and change their presentation accordingly
+                        'dmbox',  # for disambiguation messageboxes
+                    ])
+                ):
+                    self.messagebox = True
+
         elif tag == 'ol':
             for attr in attrs:
                 if attr[0] == 'class' and 'references' in attr[1]:
@@ -77,9 +99,11 @@ def handle_endtag(self, tag):
             self.span_depth -= 1
         if self.div_depth and tag == 'div':
             self.div_depth -= 1
+        if self.messagebox and tag == 'table':
+            self.messagebox = False
 
     def handle_data(self, data):
-        if self.consume and not any([self.citations, self.span_depth, self.div_depth]):
+        if self.consume and not any([self.citations, self.messagebox, self.span_depth, self.div_depth]):
             if not (self.is_header and data == self.section_name):  # Skip the initial header info only
                 self.result += data
 
@@ -225,14 +249,19 @@ def mw_section(server, query, section):
     for entry in sections['parse']['sections']:
         if entry['anchor'] == section:
             section_number = entry['index']
+            # Needed to handle sections from transcluded pages properly
+            # e.g. template documentation (usually pulled in from /doc subpage).
+            # One might expect this prop to be nullable because in most cases it
+            # will simply repeat the requested page title, but it's always set.
+            fetch_title = quote(entry['fromtitle'])
             break
 
     if not section_number:
         return None
 
     snippet_url = ('https://{0}/w/api.php?format=json&redirects'
                    '&action=parse&page={1}&prop=text'
-                   '&section={2}').format(server, query, section_number)
+                   '&section={2}').format(server, fetch_title, section_number)
 
     data = get(snippet_url).json()
 

test/modules/test_modules_announce.py

@@ -0,0 +1,32 @@
+"""Tests for Sopel's ``announce`` plugin"""
+from __future__ import generator_stop
+
+from sopel.modules import announce
+
+
+def test_chunks():
+    """Test the `_chunks` helper for functionality and compatibility."""
+    # list input
+    items = ['list', 'of', 'items', 'to', 'chunk']
+    r = list(announce._chunks(items, 2))
+
+    assert len(r) == 3
+    assert r[0] == tuple(items[:2])
+    assert r[1] == tuple(items[2:4])
+    assert r[2] == tuple(items[4:])
+
+    # tuple input
+    items = ('tuple', 'of', 'items')
+    r = list(announce._chunks(items, 3))
+
+    assert len(r) == 1
+    assert r[0] == items
+
+    # dict keys input
+    keys = {'one': True, 'two': True, 'three': True}.keys()
+    items = list(keys)
+    r = list(announce._chunks(keys, 1))
+
+    assert len(r) == 3
+    for idx in range(len(items)):
+        assert r[idx] == (items[idx],)