From f1d84c015faf2d26c0b5f76a20a6af246cb919fe Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Mon, 16 Jul 2018 22:43:33 +0300 Subject: [PATCH 1/3] bpo-32814: Handle 8BITMIME availabilty in smtplib.SMTP.send_messageIf the policy requests an 8bit content transfer encoding, than we willcheck for the 8BITMIME extension and use it if available. Otherwise wewill use a 7bit content transfer encoding since the server doesn'tsupport 8bit messages. --- Doc/library/smtplib.rst | 8 ++ Lib/smtplib.py | 26 +++-- Lib/test/test_logging.py | 2 +- Lib/test/test_smtplib.py | 101 +++++++++++++++++- .../2018-07-16-22-41-39.bpo-32814.wD4vHD.rst | 2 + 5 files changed, 127 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-07-16-22-41-39.bpo-32814.wD4vHD.rst diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 86e769e6a1f8dc..ba9049d72f41f1 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -514,12 +514,20 @@ An :class:`SMTP` instance has the following methods: serialized with a clone of its :mod:`~email.policy` with the :attr:`~email.policy.EmailPolicy.utf8` attribute set to ``True``, and ``SMTPUTF8`` and ``BODY=8BITMIME`` are added to *mail_options*. + If the ``Message`` policy requests an ``'8bit'`` + :attr:`~email.policy.EmailPolicy.cte_type` and the server supports + ``8BITMIME``, the message will be sent using it. Otherwise the policy + will be cloned with ``cte_type`` set to ``'7bit'`` and the message sent + using it. .. versionadded:: 3.2 .. versionadded:: 3.5 Support for internationalized addresses (``SMTPUTF8``). + .. versionadded:: 3.8 + Handling ``8BITMIME`` (:rfc:`6152`) availability automatically. + .. method:: SMTP.quit() diff --git a/Lib/smtplib.py b/Lib/smtplib.py index b679875fd2c539..be420b85f8f2d7 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -906,8 +906,11 @@ def send_message(self, msg, from_addr=None, to_addrs=None, SMTPUTF8 capability, the policy is cloned with utf8 set to True for the serialization, and SMTPUTF8 and BODY=8BITMIME are asserted on the send. If the server does not support SMTPUTF8, an SMTPNotSupported error is - raised. Otherwise the generator is called without modifying the - policy. + raised. If the policy requests an '8bit' cte and the server supports + 8BITMIME, the message will be sent using it. Otherwise the policy + will be cloned with cte_type set to '7bit'. If we didn't clone the + policy as previously described, the generator is called with the + original policy. """ # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822 @@ -942,9 +945,10 @@ def send_message(self, msg, from_addr=None, to_addrs=None, to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)] # Make a local copy so we can delete the bcc headers. msg_copy = copy.copy(msg) + policy = msg.policy.clone() + mail_options = mail_options[:] del msg_copy['Bcc'] del msg_copy['Resent-Bcc'] - international = False try: ''.join([from_addr, *to_addrs]).encode('ascii') except UnicodeEncodeError: @@ -953,14 +957,16 @@ def send_message(self, msg, from_addr=None, to_addrs=None, "One or more source or delivery addresses require" " internationalized email support, but the server" " does not advertise the required SMTPUTF8 capability") - international = True - with io.BytesIO() as bytesmsg: - if international: - g = email.generator.BytesGenerator( - bytesmsg, policy=msg.policy.clone(utf8=True)) - mail_options += ['SMTPUTF8', 'BODY=8BITMIME'] + policy = policy.clone(utf8=True) + mail_options += ['SMTPUTF8', 'BODY=8BITMIME'] + if policy.cte_type == '8bit': + if self.has_extn('8bitmime'): + if 'BODY=8BITMIME' not in mail_options: + mail_options.append('BODY=8BITMIME') else: - g = email.generator.BytesGenerator(bytesmsg) + policy = policy.clone(cte_type='7bit') + with io.BytesIO() as bytesmsg: + g = email.generator.BytesGenerator(bytesmsg, policy=policy) g.flatten(msg_copy, linesep='\r\n') flatmsg = bytesmsg.getvalue() return self.sendmail(from_addr, to_addrs, flatmsg, mail_options, diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index f7232f67eee730..68925efbb4f3a1 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -995,7 +995,7 @@ def test_basic(self): self.assertEqual(mailfrom, 'me') self.assertEqual(rcpttos, ['you']) self.assertIn('\nSubject: Log\n', data) - self.assertTrue(data.endswith('\n\nHello \u2713')) + self.assertTrue(data.endswith('\n\nSGVsbG8g4pyTCg==')) h.close() def process_message(self, *args): diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 7991174fb5d75a..c3204b0a8e4562 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -1,6 +1,7 @@ import asyncore import base64 import email.mime.text +from email import policy from email.message import EmailMessage from email.base64mime import body_encode as encode_base64 import email.utils @@ -1219,6 +1220,8 @@ def test_send_message_uses_smtputf8_if_addrs_non_ascii(self): self.assertEqual(self.serv.last_rcpt_options, []) def test_send_message_error_on_non_ascii_addrs_if_no_smtputf8(self): + self.serv.enable_SMTPUTF8 = False + self.serv._extra_features = [] msg = EmailMessage() msg['From'] = "Páolo " msg['To'] = 'Dinsdale' @@ -1227,7 +1230,103 @@ def test_send_message_error_on_non_ascii_addrs_if_no_smtputf8(self): HOST, self.port, local_hostname='localhost', timeout=3) self.addCleanup(smtp.close) self.assertRaises(smtplib.SMTPNotSupportedError, - smtp.send_message(msg)) + lambda: smtp.send_message(msg)) + + def test_send_message_uses_8bitmime_if_cte_type_8bitmime(self): + msg = EmailMessage() + msg['From'] = 'Dinsdale ' + msg['To'] = 'Dinsdale' + msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' + # XXX I don't know why I need two \n's here, but this is an existing + # bug (if it is one) and not a problem with the new functionality. + msg.set_content("oh là là, know what I mean, know what I mean?\n\n") + # XXX smtpd converts received /r/n to /n, so we can't easily test that + # we are successfully sending /r/n :(. + expected = textwrap.dedent("""\ + From: Dinsdale + To: Dinsdale + Subject: Nudge nudge, wink, wink =?utf-8?q?=E1=BD=A09?= + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + MIME-Version: 1.0 + + oh là là, know what I mean, know what I mean? + """) + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + self.assertEqual(smtp.send_message(msg), {}) + self.assertEqual(self.serv.last_mailfrom, 'foo@bar.com') + self.assertEqual(self.serv.last_rcpttos, ['Dinsdale']) + self.assertEqual(self.serv.last_message.decode(), expected) + self.assertIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertNotIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) + + def test_send_message_uses_7bit_if_requested_even_with_8bitmime(self): + msg = EmailMessage(policy=policy.default.clone(cte_type='7bit')) + msg['From'] = 'Dinsdale ' + msg['To'] = 'Dinsdale' + msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' + # XXX I don't know why I need two \n's here, but this is an existing + # bug (if it is one) and not a problem with the new functionality. + msg.set_content("oh là là, know what I mean, know what I mean?\n\n") + # XXX smtpd converts received /r/n to /n, so we can't easily test that + # we are successfully sending /r/n :(. + # XXX This uses quoted-printable but the case below happens to end up + # with base64... + expected = textwrap.dedent("""\ + From: Dinsdale + To: Dinsdale + Subject: Nudge nudge, wink, wink =?utf-8?q?=E1=BD=A09?= + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: quoted-printable + MIME-Version: 1.0 + + oh l=C3=A0 l=C3=A0, know what I mean, know what I mean? + """) + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + self.assertEqual(smtp.send_message(msg), {}) + self.assertEqual(self.serv.last_mailfrom, 'foo@bar.com') + self.assertEqual(self.serv.last_rcpttos, ['Dinsdale']) + self.assertEqual(self.serv.last_message.decode(), expected) + self.assertNotIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertNotIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) + + def test_send_message_uses_7bit_if_cte_type_8bitmime_but_no_8bitmime(self): + self.serv.enable_SMTPUTF8 = False + self.serv._extra_features = [] + msg = EmailMessage() + msg['From'] = 'Dinsdale ' + msg['To'] = 'Dinsdale' + msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' + # XXX I don't know why I need two \n's here, but this is an existing + # bug (if it is one) and not a problem with the new functionality. + msg.set_content("oh là là, know what I mean, know what I mean?\n\n") + # XXX smtpd converts received /r/n to /n, so we can't easily test that + # we are successfully sending /r/n :(. + expected = textwrap.dedent("""\ + From: Dinsdale + To: Dinsdale + Subject: Nudge nudge, wink, wink =?utf-8?q?=E1=BD=A09?= + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + MIME-Version: 1.0 + + b2ggbMOgIGzDoCwga25vdyB3aGF0IEkgbWVhbiwga25vdyB3aGF0IEkgbWVhbj8KCg==""") + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + self.assertEqual(smtp.send_message(msg), {}) + self.assertEqual(self.serv.last_mailfrom, 'foo@bar.com') + self.assertEqual(self.serv.last_rcpttos, ['Dinsdale']) + self.assertEqual(self.serv.last_message.decode(), expected) + self.assertNotIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertNotIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) EXPECTED_RESPONSE = encode_base64(b'\0psu\0doesnotexist', eol='') diff --git a/Misc/NEWS.d/next/Library/2018-07-16-22-41-39.bpo-32814.wD4vHD.rst b/Misc/NEWS.d/next/Library/2018-07-16-22-41-39.bpo-32814.wD4vHD.rst new file mode 100644 index 00000000000000..feec549c88c698 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-07-16-22-41-39.bpo-32814.wD4vHD.rst @@ -0,0 +1,2 @@ +Handle ``8BITMIME`` (RFC6152) availability automatically in +``smtplib.SMTP.send_message``. Patch by Segev Finer. From 404227fdfcdd2d2b6b8b9faec8329ee7a6d2109b Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Wed, 25 Jul 2018 22:45:30 +0300 Subject: [PATCH 2/3] bpo-32814: Fix code review comments by @bitdancer --- Lib/smtplib.py | 13 ++++++++----- Lib/test/test_smtplib.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index be420b85f8f2d7..ccf6aabf4f83ef 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -923,6 +923,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, # option allowing the user to enable the heuristics. (It should be # possible to guess correctly almost all of the time.) + mail_options = mail_options[:] self.ehlo_or_helo_if_needed() resent = msg.get_all('Resent-Date') if resent is None: @@ -945,10 +946,10 @@ def send_message(self, msg, from_addr=None, to_addrs=None, to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)] # Make a local copy so we can delete the bcc headers. msg_copy = copy.copy(msg) - policy = msg.policy.clone() - mail_options = mail_options[:] + policy = msg.policy del msg_copy['Bcc'] del msg_copy['Resent-Bcc'] + body_is_8bit = False try: ''.join([from_addr, *to_addrs]).encode('ascii') except UnicodeEncodeError: @@ -958,13 +959,15 @@ def send_message(self, msg, from_addr=None, to_addrs=None, " internationalized email support, but the server" " does not advertise the required SMTPUTF8 capability") policy = policy.clone(utf8=True) - mail_options += ['SMTPUTF8', 'BODY=8BITMIME'] + mail_options.append('SMTPUTF8') + body_is_8bit = True if policy.cte_type == '8bit': if self.has_extn('8bitmime'): - if 'BODY=8BITMIME' not in mail_options: - mail_options.append('BODY=8BITMIME') + body_is_8bit = True else: policy = policy.clone(cte_type='7bit') + if body_is_8bit: + mail_options.append('BODY=8BITMIME') with io.BytesIO() as bytesmsg: g = email.generator.BytesGenerator(bytesmsg, policy=policy) g.flatten(msg_copy, linesep='\r\n') diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index c3204b0a8e4562..663661038a161d 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -1230,7 +1230,7 @@ def test_send_message_error_on_non_ascii_addrs_if_no_smtputf8(self): HOST, self.port, local_hostname='localhost', timeout=3) self.addCleanup(smtp.close) self.assertRaises(smtplib.SMTPNotSupportedError, - lambda: smtp.send_message(msg)) + smtp.send_message, msg) def test_send_message_uses_8bitmime_if_cte_type_8bitmime(self): msg = EmailMessage() From ab2ecad5cbdf0ac1b07adf9f84d31e71506e49ed Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Sat, 22 Sep 2018 22:06:37 +0300 Subject: [PATCH 3/3] bpo-32814: Fix test_smtplib after merge --- Lib/test/test_smtplib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 3048bf5868433c..68a49ee4b0b610 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -614,7 +614,7 @@ def setUp(self): self.smtp.has_extn, self.smtp.sendmail = Mock(), Mock() def testSendMessage(self): - expected_mail_options = ('SMTPUTF8', 'BODY=8BITMIME') + expected_mail_options = ['SMTPUTF8', 'BODY=8BITMIME'] self.smtp.send_message(self.msg) self.smtp.send_message(self.msg) self.assertEqual(self.smtp.sendmail.call_args_list[0][0][3], @@ -624,7 +624,7 @@ def testSendMessage(self): def testSendMessageWithMailOptions(self): mail_options = ['STARTTLS'] - expected_mail_options = ('STARTTLS', 'SMTPUTF8', 'BODY=8BITMIME') + expected_mail_options = ['STARTTLS', 'SMTPUTF8', 'BODY=8BITMIME'] self.smtp.send_message(self.msg, None, None, mail_options) self.assertEqual(mail_options, ['STARTTLS']) self.assertEqual(self.smtp.sendmail.call_args_list[0][0][3],