Merge remote-tracking branch 'origin/develop'

Author: awdeorio

.github/workflows/main.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       # Define OS and Python versions to use. 3.x is the latest minor version.
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.x"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.x"]
         os: [ubuntu-latest]
 
     # Sequence of tasks for this job
@@ -55,6 +55,6 @@ jobs:
       # Upload coverage report
       # https://github.com/codecov/codecov-action
       - name: Upload coverage report
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v2
         with:
           fail_ci_if_error: true

mailmerge/__main__.py

@@ -240,6 +240,14 @@ def create_sample_input_files(template_path, database_path, config_path):
             # username = YOUR_USERNAME_HERE
             # ratelimit = 0
 
+            # Example: Plain security
+            # [smtp_server]
+            # host = newman.eecs.umich.edu
+            # port = 25
+            # security = PLAIN
+            # username = YOUR_USERNAME_HERE
+            # ratelimit = 0
+
             # Example: No security
             # [smtp_server]
             # host = newman.eecs.umich.edu
@@ -255,25 +263,47 @@ def create_sample_input_files(template_path, database_path, config_path):
     """))
 
 
+def detect_database_format(database_file):
+    """Automatically detect the database format.
+
+    Automatically detect the format ("dialect") using the CSV library's sniffer
+    class.  For example, comma-delimited, tab-delimited, etc.  Default to
+    StrictExcel if automatic detection fails.
+
+    """
+    class StrictExcel(csv.excel):
+        # Our helper class is really simple
+        # pylint: disable=too-few-public-methods, missing-class-docstring
+        strict = True
+
+    # Read a sample from database
+    sample = database_file.read(1024)
+    database_file.seek(0)
+
+    # Attempt automatic format detection, fall back on StrictExcel default
+    try:
+        csvdialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
+    except csv.Error:
+        csvdialect = StrictExcel
+
+    return csvdialect
+
+
 def read_csv_database(database_path):
     """Read database CSV file, providing one line at a time.
 
-    We'll use a class to modify the csv library's default dialect ('excel') to
-    enable strict syntax checking.  This will trigger errors for things like
+    Use strict syntax checking, which will trigger errors for things like
     unclosed quotes.
 
     We open the file with the utf-8-sig encoding, which skips a byte order mark
     (BOM), if any.  Sometimes Excel will save CSV files with a BOM.  See Issue
     #93 https://github.com/awdeorio/mailmerge/issues/93
 
     """
-    class StrictExcel(csv.excel):
-        # Our helper class is really simple
-        # pylint: disable=too-few-public-methods, missing-class-docstring
-        strict = True
-
     with database_path.open(encoding="utf-8-sig") as database_file:
-        reader = csv.DictReader(database_file, dialect=StrictExcel)
+        csvdialect = detect_database_format(database_file)
+        csvdialect.strict = True
+        reader = csv.DictReader(database_file, dialect=csvdialect)
         try:
             for row in reader:
                 yield row

mailmerge/sendmail_client.py

@@ -49,7 +49,7 @@ def read_config(self):
             security = None
 
         # Verify security type
-        if security not in [None, "SSL/TLS", "STARTTLS"]:
+        if security not in [None, "SSL/TLS", "STARTTLS", "PLAIN"]:
             raise exceptions.MailmergeError(
                 f"{self.config_path}: unrecognized security type: '{security}'"
             )
@@ -100,6 +100,10 @@ def sendmail(self, sender, recipients, message):
                     smtp.ehlo()
                     smtp.login(self.config.username, self.password)
                     smtp.sendmail(sender, recipients, message_flattened)
+            elif self.config.security == "PLAIN":
+                with smtplib.SMTP(host, port) as smtp:
+                    smtp.login(self.config.username, self.password)
+                    smtp.sendmail(sender, recipients, message_flattened)
             elif self.config.security is None:
                 with smtplib.SMTP(host, port) as smtp:
                     smtp.sendmail(sender, recipients, message_flattened)

mailmerge/template_message.py

@@ -58,7 +58,7 @@ def render(self, context):
         self._transform_markdown()
         self._transform_attachments()
         self._transform_attachment_references()
-        self._message.__setitem__('Date', email.utils.formatdate())
+        self._message.add_header('Date', email.utils.formatdate())
         assert self._sender
         assert self._recipients
         assert self._message
@@ -74,7 +74,9 @@ def _transform_encoding(self, raw_message):
 
     def _transform_recipients(self):
         """Extract sender and recipients from FROM, TO, CC and BCC fields."""
-        # Extract recipients
+        # The docs recommend using __delitem__()
+        # https://docs.python.org/3/library/email.message.html#email.message.EmailMessage.__delitem__
+        # pylint: disable=unnecessary-dunder-call
         addrs = email.utils.getaddresses(self._message.get_all("TO", [])) + \
             email.utils.getaddresses(self._message.get_all("CC", [])) + \
             email.utils.getaddresses(self._message.get_all("BCC", []))
@@ -87,15 +89,15 @@ def _make_message_multipart(self):
         Convert self._message into a multipart message.
 
         Specifically, if the message's content-type is not multipart, this
-        method will create a new `multipart/mixed` message, copy message
+        method will create a new `multipart/related` message, copy message
         headers and re-attach the original payload.
         """
         # Do nothing if message already multipart
         if self._message.is_multipart():
             return
 
         # Create empty multipart message
-        multipart_message = email.mime.multipart.MIMEMultipart('mixed')
+        multipart_message = email.mime.multipart.MIMEMultipart('related')
 
         # Copy headers.  Avoid duplicate Content-Type and MIME-Version headers,
         # which we set explicitely.  MIME-Version was set when we created an
@@ -128,13 +130,13 @@ def _transform_markdown(self):
         Specifically, if the message's content-type is `text/markdown`, we
         transform `self._message` to have the following structure:
 
-        multipart/mixed
+        multipart/related
          └── multipart/alternative
              ├── text/plain (original markdown plaintext)
              └── text/html (converted markdown)
 
         Attachments should be added as subsequent payload items of the
-        top-level `multipart/mixed` message.
+        top-level `multipart/related` message.
         """
         # Do nothing if Content-Type is not text/markdown
         if not self._message['Content-Type'].startswith("text/markdown"):
@@ -186,11 +188,11 @@ def _transform_attachments(self):
         """
         Parse attachment headers and generate content-id headers for each.
 
-        Attachments are added to the payload of a `multipart/mixed` message.
+        Attachments are added to the payload of a `multipart/related` message.
         For instance, a plaintext message with attachments would have the
         following structure:
 
-        multipart/mixed
+        multipart/related
          ├── text/plain
          ├── attachment1
          └── attachment2
@@ -199,7 +201,7 @@ def _transform_attachments(self):
         then the message would have the following structure after transforming
         markdown and attachments:
 
-        multipart/mixed
+        multipart/related
          ├── multipart/alternative
          │   ├── text/plain
          │   └── text/html

setup.py

@@ -14,7 +14,7 @@
     description="A simple, command line mail merge tool",
     long_description=LONG_DESCRIPTION,
     long_description_content_type="text/markdown",
-    version="2.2.1",
+    version="2.2.2",
     author="Andrew DeOrio",
     author_email="[email protected]",
     url="https://github.com/awdeorio/mailmerge/",

tests/test_helpers.py

@@ -31,7 +31,7 @@ def test_enumerate_range_stop_value():
 def test_enumerate_range_stop_zero():
     """Verify stop=0."""
     output = list(enumerate_range(["a", "b", "c"], stop=0))
-    assert output == []
+    assert not output
 
 
 def test_enumerate_range_stop_too_big():
@@ -61,13 +61,13 @@ def test_enumerate_range_start_last_one():
 def test_enumerate_range_start_length():
     """Verify start=length."""
     output = list(enumerate_range(["a", "b", "c"], start=3))
-    assert output == []
+    assert not output
 
 
 def test_enumerate_range_start_too_big():
     """Verify start past the end."""
     output = list(enumerate_range(["a", "b", "c"], start=10))
-    assert output == []
+    assert not output
 
 
 def test_enumerate_range_start_stop():

tests/test_main.py

@@ -859,3 +859,109 @@ def test_database_bom(tmpdir):
         >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
         >>> This was a dry run.  To send messages, use the --no-dry-run option.
     """)  # noqa: E501
+
+
+def test_database_tsv(tmpdir):
+    """Automatically detect TSV database format."""
+    # Simple template
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent("""\
+        TO: {{email}}
+        FROM: My Self <[email protected]>
+
+        Hello {{name}}
+    """), encoding="utf8")
+
+    # Tab-separated format database
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent("""\
+        email\tname
+        [email protected]\tMy Name
+    """), encoding="utf8")
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent("""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """), encoding="utf8")
+
+    # Run mailmerge
+    runner = click.testing.CliRunner()
+    with tmpdir.as_cwd():
+        result = runner.invoke(main, ["--output-format", "text"])
+    assert not result.exception
+    assert result.exit_code == 0
+
+    # Verify output
+    stdout = copy.deepcopy(result.output)
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
+    assert stdout == textwrap.dedent("""\
+        >>> message 1
+        TO: [email protected]
+        FROM: My Self <[email protected]>
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="us-ascii"
+        Content-Transfer-Encoding: 7bit
+        Date: REDACTED
+
+        Hello My Name
+
+        >>> message 1 sent
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501
+
+
+def test_database_semicolon(tmpdir):
+    """Automatically detect semicolon-delimited database format."""
+    # Simple template
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent("""\
+        TO: {{email}}
+        FROM: My Self <[email protected]>
+
+        Hello {{name}}
+    """), encoding="utf8")
+
+    # Semicolon-separated format database
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent("""\
+        email;name
+        [email protected];My Name
+    """), encoding="utf8")
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent("""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """), encoding="utf8")
+
+    # Run mailmerge
+    runner = click.testing.CliRunner()
+    with tmpdir.as_cwd():
+        result = runner.invoke(main, ["--output-format", "text"])
+    assert not result.exception
+    assert result.exit_code == 0
+
+    # Verify output
+    stdout = copy.deepcopy(result.output)
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
+    assert stdout == textwrap.dedent("""\
+        >>> message 1
+        TO: [email protected]
+        FROM: My Self <[email protected]>
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="us-ascii"
+        Content-Transfer-Encoding: 7bit
+        Date: REDACTED
+
+        Hello My Name
+
+        >>> message 1 sent
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501

tests/test_sendmail_client.py

@@ -250,6 +250,48 @@ def test_security_starttls(mocker, tmp_path):
     assert smtp.sendmail.call_count == 1
 
 
+def test_security_plain(mocker, tmp_path):
+    """Verify plain security configuration."""
+    # Config for Plain SMTP server
+    config_path = tmp_path/"server.conf"
+    config_path.write_text(textwrap.dedent("""\
+        [smtp_server]
+        host = newman.eecs.umich.edu
+        port = 25
+        security = PLAIN
+        username = YOUR_USERNAME_HERE
+    """))
+
+    # Simple template
+    sendmail_client = SendmailClient(config_path, dry_run=False)
+    message = email.message_from_string("Hello world")
+
+    # Mock SMTP
+    mock_smtp = mocker.patch('smtplib.SMTP')
+    mock_smtp_ssl = mocker.patch('smtplib.SMTP_SSL')
+
+    # Mock the password entry
+    mock_getpass = mocker.patch('getpass.getpass')
+    mock_getpass.return_value = "password"
+
+    # Send a message
+    sendmail_client.sendmail(
+        sender="[email protected]",
+        recipients=["[email protected]"],
+        message=message,
+    )
+
+    # Verify SMTP library calls
+    assert mock_getpass.call_count == 1
+    assert mock_smtp.call_count == 1
+    assert mock_smtp_ssl.call_count == 0
+    smtp = mock_smtp.return_value.__enter__.return_value
+    assert smtp.ehlo.call_count == 0
+    assert smtp.starttls.call_count == 0
+    assert smtp.login.call_count == 1
+    assert smtp.sendmail.call_count == 1
+
+
 def test_security_ssl(mocker, tmp_path):
     """Verify open (Never) security configuration."""
     # Config for SSL SMTP server