bpo-32814: Handle 8BITMIME availabilty in smtplib.SMTP.send_message

Doc/library/smtplib.rst

@@ -540,12 +540,20 @@ An :class:`SMTP` instance has the following methods:
    serialized with a clone of its :mod:`~email.policy` with the
    :attr:`~email.policy.EmailPolicy.utf8` attribute set to ``True``, and
    ``SMTPUTF8`` and ``BODY=8BITMIME`` are added to *mail_options*.
+   If the ``Message`` policy requests an ``'8bit'``
+   :attr:`~email.policy.EmailPolicy.cte_type` and the server supports
+   ``8BITMIME``, the message will be sent using it.  Otherwise the policy
+   will be cloned with ``cte_type`` set to ``'7bit'`` and the message sent
+   using it.
 
    .. versionadded:: 3.2
 
    .. versionadded:: 3.5
       Support for internationalized addresses (``SMTPUTF8``).
 
+   .. versionadded:: 3.8
+      Handling ``8BITMIME`` (:rfc:`6152`) availability automatically.
+
 
 .. method:: SMTP.quit()
 

Lib/smtplib.py

@@ -926,8 +926,11 @@ def send_message(self, msg, from_addr=None, to_addrs=None,
         SMTPUTF8 capability, the policy is cloned with utf8 set to True for the
         serialization, and SMTPUTF8 and BODY=8BITMIME are asserted on the send.
         If the server does not support SMTPUTF8, an SMTPNotSupported error is
-        raised.  Otherwise the generator is called without modifying the
-        policy.
+        raised.  If the policy requests an '8bit' cte and the server supports
+        8BITMIME, the message will be sent using it.  Otherwise the policy
+        will be cloned with cte_type set to '7bit'.  If we didn't clone the
+        policy as previously described, the generator is called with the
+        original policy.
 
         """
         # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822
@@ -940,6 +943,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None,
         # option allowing the user to enable the heuristics.  (It should be
         # possible to guess correctly almost all of the time.)
 
+        mail_options = list(mail_options)
         self.ehlo_or_helo_if_needed()
         resent = msg.get_all('Resent-Date')
         if resent is None:
@@ -962,9 +966,10 @@ def send_message(self, msg, from_addr=None, to_addrs=None,
             to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)]
         # Make a local copy so we can delete the bcc headers.
         msg_copy = copy.copy(msg)
+        policy = msg.policy
         del msg_copy['Bcc']
         del msg_copy['Resent-Bcc']
-        international = False
+        body_is_8bit = False
         try:
             ''.join([from_addr, *to_addrs]).encode('ascii')
         except UnicodeEncodeError:
@@ -973,14 +978,18 @@ def send_message(self, msg, from_addr=None, to_addrs=None,
                     "One or more source or delivery addresses require"
                     " internationalized email support, but the server"
                     " does not advertise the required SMTPUTF8 capability")
-            international = True
-        with io.BytesIO() as bytesmsg:
-            if international:
-                g = email.generator.BytesGenerator(
-                    bytesmsg, policy=msg.policy.clone(utf8=True))
-                mail_options = (*mail_options, 'SMTPUTF8', 'BODY=8BITMIME')
+            policy = policy.clone(utf8=True)
+            mail_options.append('SMTPUTF8')
+            body_is_8bit = True
+        if policy.cte_type == '8bit':
+            if self.has_extn('8bitmime'):
+                body_is_8bit = True
             else:
-                g = email.generator.BytesGenerator(bytesmsg)
+                policy = policy.clone(cte_type='7bit')
+        if body_is_8bit:
+            mail_options.append('BODY=8BITMIME')
+        with io.BytesIO() as bytesmsg:
+            g = email.generator.BytesGenerator(bytesmsg, policy=policy)
             g.flatten(msg_copy, linesep='\r\n')
             flatmsg = bytesmsg.getvalue()
         return self.sendmail(from_addr, to_addrs, flatmsg, mail_options,

Lib/test/test_logging.py

@@ -1083,7 +1083,7 @@ def test_basic(self):
         self.assertEqual(mailfrom, 'me')
         self.assertEqual(rcpttos, ['you'])
         self.assertIn('\nSubject: Log\n', data)
-        self.assertTrue(data.endswith('\n\nHello \u2713'))
+        self.assertTrue(data.endswith('\n\nSGVsbG8g4pyTCg=='))
         h.close()
 
     def process_message(self, *args):

Lib/test/test_smtplib.py

@@ -1,5 +1,6 @@
 import base64
 import email.mime.text
+from email import policy
 from email.message import EmailMessage
 from email.base64mime import body_encode as encode_base64
 import email.utils
@@ -733,7 +734,7 @@ def setUp(self):
         self.smtp.has_extn, self.smtp.sendmail = Mock(), Mock()
 
     def testSendMessage(self):
-        expected_mail_options = ('SMTPUTF8', 'BODY=8BITMIME')
+        expected_mail_options = ['SMTPUTF8', 'BODY=8BITMIME']
         self.smtp.send_message(self.msg)
         self.smtp.send_message(self.msg)
         self.assertEqual(self.smtp.sendmail.call_args_list[0][0][3],
@@ -743,7 +744,7 @@ def testSendMessage(self):
 
     def testSendMessageWithMailOptions(self):
         mail_options = ['STARTTLS']
-        expected_mail_options = ('STARTTLS', 'SMTPUTF8', 'BODY=8BITMIME')
+        expected_mail_options = ['STARTTLS', 'SMTPUTF8', 'BODY=8BITMIME']
         self.smtp.send_message(self.msg, None, None, mail_options)
         self.assertEqual(mail_options, ['STARTTLS'])
         self.assertEqual(self.smtp.sendmail.call_args_list[0][0][3],
@@ -1486,6 +1487,102 @@ def test_send_message_uses_smtputf8_if_addrs_non_ascii(self):
         self.assertIn('SMTPUTF8', self.serv.last_mail_options)
         self.assertEqual(self.serv.last_rcpt_options, [])
 
+    def test_send_message_uses_8bitmime_if_cte_type_8bitmime(self):
+        msg = EmailMessage()
+        msg['From'] = 'Dinsdale <foo@bar.com>'
+        msg['To'] = 'Dinsdale'
+        msg['Subject'] = 'Nudge nudge, wink, wink \u1F609'
+        # XXX I don't know why I need two \n's here, but this is an existing
+        # bug (if it is one) and not a problem with the new functionality.
+        msg.set_content("oh là là, know what I mean, know what I mean?\n\n")
+        # XXX smtpd converts received /r/n to /n, so we can't easily test that
+        # we are successfully sending /r/n :(.
+        expected = textwrap.dedent("""\
+            From: Dinsdale <foo@bar.com>
+            To: Dinsdale
+            Subject: Nudge nudge, wink, wink =?utf-8?q?=E1=BD=A09?=
+            Content-Type: text/plain; charset="utf-8"
+            Content-Transfer-Encoding: 8bit
+            MIME-Version: 1.0
+
+            oh là là, know what I mean, know what I mean?
+            """)
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        self.assertEqual(smtp.send_message(msg), {})
+        self.assertEqual(self.serv.last_mailfrom, 'foo@bar.com')
+        self.assertEqual(self.serv.last_rcpttos, ['Dinsdale'])
+        self.assertEqual(self.serv.last_message.decode(), expected)
+        self.assertIn('BODY=8BITMIME', self.serv.last_mail_options)
+        self.assertNotIn('SMTPUTF8', self.serv.last_mail_options)
+        self.assertEqual(self.serv.last_rcpt_options, [])
+
+    def test_send_message_uses_7bit_if_requested_even_with_8bitmime(self):
+        msg = EmailMessage(policy=policy.default.clone(cte_type='7bit'))
+        msg['From'] = 'Dinsdale <foo@bar.com>'
+        msg['To'] = 'Dinsdale'
+        msg['Subject'] = 'Nudge nudge, wink, wink \u1F609'
+        # XXX I don't know why I need two \n's here, but this is an existing
+        # bug (if it is one) and not a problem with the new functionality.
+        msg.set_content("oh là là, know what I mean, know what I mean?\n\n")
+        # XXX smtpd converts received /r/n to /n, so we can't easily test that
+        # we are successfully sending /r/n :(.
+        # XXX This uses quoted-printable but the case below happens to end up
+        # with base64...
+        expected = textwrap.dedent("""\
+            From: Dinsdale <foo@bar.com>
+            To: Dinsdale
+            Subject: Nudge nudge, wink, wink =?utf-8?q?=E1=BD=A09?=
+            Content-Type: text/plain; charset="utf-8"
+            Content-Transfer-Encoding: quoted-printable
+            MIME-Version: 1.0
+
+            oh l=C3=A0 l=C3=A0, know what I mean, know what I mean?
+            """)
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        self.assertEqual(smtp.send_message(msg), {})
+        self.assertEqual(self.serv.last_mailfrom, 'foo@bar.com')
+        self.assertEqual(self.serv.last_rcpttos, ['Dinsdale'])
+        self.assertEqual(self.serv.last_message.decode(), expected)
+        self.assertNotIn('BODY=8BITMIME', self.serv.last_mail_options)
+        self.assertNotIn('SMTPUTF8', self.serv.last_mail_options)
+        self.assertEqual(self.serv.last_rcpt_options, [])
+
+    def test_send_message_uses_7bit_if_cte_type_8bitmime_but_no_8bitmime(self):
+        self.serv.enable_SMTPUTF8 = False
+        self.serv._extra_features = []
+        msg = EmailMessage()
+        msg['From'] = 'Dinsdale <foo@bar.com>'
+        msg['To'] = 'Dinsdale'
+        msg['Subject'] = 'Nudge nudge, wink, wink \u1F609'
+        # XXX I don't know why I need two \n's here, but this is an existing
+        # bug (if it is one) and not a problem with the new functionality.
+        msg.set_content("oh là là, know what I mean, know what I mean?\n\n")
+        # XXX smtpd converts received /r/n to /n, so we can't easily test that
+        # we are successfully sending /r/n :(.
+        expected = textwrap.dedent("""\
+            From: Dinsdale <foo@bar.com>
+            To: Dinsdale
+            Subject: Nudge nudge, wink, wink =?utf-8?q?=E1=BD=A09?=
+            Content-Type: text/plain; charset="utf-8"
+            Content-Transfer-Encoding: base64
+            MIME-Version: 1.0
+
+            b2ggbMOgIGzDoCwga25vdyB3aGF0IEkgbWVhbiwga25vdyB3aGF0IEkgbWVhbj8KCg==""")
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        self.assertEqual(smtp.send_message(msg), {})
+        self.assertEqual(self.serv.last_mailfrom, 'foo@bar.com')
+        self.assertEqual(self.serv.last_rcpttos, ['Dinsdale'])
+        self.assertEqual(self.serv.last_message.decode(), expected)
+        self.assertNotIn('BODY=8BITMIME', self.serv.last_mail_options)
+        self.assertNotIn('SMTPUTF8', self.serv.last_mail_options)
+        self.assertEqual(self.serv.last_rcpt_options, [])
+
 
 EXPECTED_RESPONSE = encode_base64(b'\0psu\0doesnotexist', eol='')
 

Misc/NEWS.d/next/Library/2018-07-16-22-41-39.bpo-32814.wD4vHD.rst

@@ -0,0 +1,2 @@
+Handle ``8BITMIME`` (RFC6152) availability automatically in
+``smtplib.SMTP.send_message``.  Patch by Segev Finer.