diff -Nru muttdown-0.3.3/CHANGES.md muttdown-0.3.4/CHANGES.md --- muttdown-0.3.3/CHANGES.md 2018-12-27 21:49:45.000000000 +0000 +++ muttdown-0.3.4/CHANGES.md 2019-02-09 04:16:03.000000000 +0000 @@ -1,3 +1,8 @@ +0.3.4 +===== +- Fix regression in headers from 0.3.0 with some multipart/signed messages +- Fix regression in passthrough mode from 0.3.3 on Python 2; add better testing + 0.3.3 ===== - Fix `-s` / smtp passthrough mode on Python 3 diff -Nru muttdown-0.3.3/debian/changelog muttdown-0.3.4/debian/changelog --- muttdown-0.3.3/debian/changelog 2018-12-28 17:28:01.000000000 +0000 +++ muttdown-0.3.4/debian/changelog 2019-03-06 06:25:43.000000000 +0000 @@ -1,3 +1,10 @@ +muttdown (0.3.4-1) unstable; urgency=medium + + * [8824d5] New upstream version 0.3.4 + * [e16e96] Add python3-pytest-mock build dependency + + -- gustavo panizzo Wed, 06 Mar 2019 14:25:43 +0800 + muttdown (0.3.3-1) unstable; urgency=medium * [fb6862] New upstream version 0.3.3 diff -Nru muttdown-0.3.3/debian/control muttdown-0.3.4/debian/control --- muttdown-0.3.3/debian/control 2018-12-28 17:28:01.000000000 +0000 +++ muttdown-0.3.4/debian/control 2019-03-06 06:25:43.000000000 +0000 @@ -3,7 +3,7 @@ Priority: optional Maintainer: Python Applications Packaging Team Uploaders: gustavo panizzo -Build-Depends: debhelper (>= 11), dh-python, python3-all, python3-setuptools, python3-pip, python3-pynliner, python3-markdown, python3-yaml, python3-nose, python3-pytest, python3 +Build-Depends: debhelper (>= 11), dh-python, python3-all, python3-setuptools, python3-pip, python3-pynliner, python3-markdown, python3-yaml, python3-nose, python3-pytest, python3, python3-pytest-mock Standards-Version: 4.3.0 Homepage: https://github.com/Roguelazer/muttdown Vcs-Git: https://salsa.debian.org/python-team/applications/muttdown.git diff -Nru muttdown-0.3.3/muttdown/__init__.py muttdown-0.3.4/muttdown/__init__.py --- muttdown-0.3.3/muttdown/__init__.py 2018-12-27 21:49:45.000000000 +0000 +++ muttdown-0.3.4/muttdown/__init__.py 2019-02-09 04:16:03.000000000 +0000 @@ -1,3 +1,3 @@ -version_info = (0, 3, 3) +version_info = (0, 3, 4) __version__ = '.'.join(map(str, version_info)) __author__ = 'James Brown ' diff -Nru muttdown-0.3.3/muttdown/main.py muttdown-0.3.4/muttdown/main.py --- muttdown-0.3.3/muttdown/main.py 2018-12-27 21:49:45.000000000 +0000 +++ muttdown-0.3.4/muttdown/main.py 2019-02-09 04:16:03.000000000 +0000 @@ -23,30 +23,26 @@ def convert_one(part, config): - try: - text = part.get_payload(decode=True) - if not isinstance(text, six.text_type): - # no, I don't know why decode=True sometimes fails to decode. - text = text.decode('utf-8') - if not text.startswith('!m'): - return None - text = re.sub('\s*!m\s*', '', text, re.M) - if '\n-- \n' in text: - pre_signature, signature = text.split('\n-- \n') - md = markdown.markdown(pre_signature, output_format="html5") - md += '\n

--
' - md += '
'.join(signature.split('\n')) - md += '

' - else: - md = markdown.markdown(text) - if config.css: - md = '' + md - md = pynliner.fromString(md) - message = MIMEText(md, 'html', _charset="UTF-8") - return message - except Exception: - raise + text = part.get_payload(decode=True) + if not isinstance(text, six.text_type): + # no, I don't know why decode=True sometimes fails to decode. + text = text.decode('utf-8') + if not text.startswith('!m'): return None + text = re.sub('\s*!m\s*', '', text, re.M) + if '\n-- \n' in text: + pre_signature, signature = text.split('\n-- \n') + md = markdown.markdown(pre_signature, output_format="html5") + md += '\n

--
' + md += '
'.join(signature.split('\n')) + md += '

' + else: + md = markdown.markdown(text) + if config.css: + md = '' + md + md = pynliner.fromString(md) + message = MIMEText(md, 'html', _charset="UTF-8") + return message def _move_headers(source, dest): @@ -59,7 +55,7 @@ del source[k] -def convert_tree(message, config, indent=0): +def convert_tree(message, config, indent=0, wrap_alternative=True): """Recursively convert a potentially-multipart tree. Returns a tuple of (the converted tree, whether any markdown was found) @@ -73,19 +69,32 @@ if disposition == 'inline' and ct in ('text/plain', 'text/markdown'): converted = convert_one(message, config) if converted is not None: - new_tree = MIMEMultipart('alternative') - _move_headers(message, new_tree) - new_tree.attach(message) - new_tree.attach(converted) - return new_tree, True + if wrap_alternative: + new_tree = MIMEMultipart('alternative') + _move_headers(message, new_tree) + new_tree.attach(message) + new_tree.attach(converted) + return new_tree, True + else: + return converted, True return message, False else: if ct == 'multipart/signed': # if this is a multipart/signed message, then let's just # recurse into the non-signature part + new_root = MIMEMultipart('alternative') + if message.preamble: + new_root.preamble = message.preamble + _move_headers(message, new_root) + converted = None for part in message.get_payload(): if part.get_content_type() != 'application/pgp-signature': - return convert_tree(part, config, indent=indent + 1) + converted, did_conversion = convert_tree(part, config, indent=indent + 1, + wrap_alternative=False) + if did_conversion: + new_root.attach(converted) + new_root.attach(message) + return new_root, did_conversion else: did_conversion = False new_root = MIMEMultipart(cs, message.get_charset()) @@ -116,12 +125,17 @@ if not c.smtp_ssl: conn.ehlo() conn.starttls() + conn.ehlo() if c.smtp_username: conn.login(c.smtp_username, c.smtp_password) return conn -def main(): +def read_message(): + return sys.stdin.read() + + +def main(argv=None): parser = argparse.ArgumentParser(prog='muttdown') parser.add_argument('-v', '--version', action='version', version='%s %s' % (__name__, __version__)) parser.add_argument( @@ -139,7 +153,7 @@ help='Pass mail through to sendmail for delivery' ) parser.add_argument('addresses', nargs='+') - args = parser.parse_args() + args = parser.parse_args(argv) c = config.Config() try: @@ -150,7 +164,7 @@ sys.stderr.flush() return 1 - message = sys.stdin.read() + message = read_message() mail = email.message_from_string(message) @@ -163,7 +177,12 @@ cmd = c.sendmail.split() + ['-f', args.envelope_from] + args.addresses proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, shell=False) - proc.communicate(rebuilt.as_string().encode('utf-8')) + msg = rebuilt.as_string() + if sys.version_info > (3, 0): + msg = msg.encode('utf-8') + proc.stdin.write(msg) + proc.stdin.close() + proc.wait() return proc.returncode else: conn = smtp_connection(c) diff -Nru muttdown-0.3.3/requirements-tests.txt muttdown-0.3.4/requirements-tests.txt --- muttdown-0.3.3/requirements-tests.txt 2018-12-27 21:49:45.000000000 +0000 +++ muttdown-0.3.4/requirements-tests.txt 2019-02-09 04:16:03.000000000 +0000 @@ -1,2 +1,3 @@ -pytest>=3.0,<4.0 -pytest-cov>=2.0,<3.0 +pytest==3.10.* +pytest-cov==2.6.* +pytest-mock==1.10.* diff -Nru muttdown-0.3.3/setup.py muttdown-0.3.4/setup.py --- muttdown-0.3.3/setup.py 2018-12-27 21:49:45.000000000 +0000 +++ muttdown-0.3.4/setup.py 2019-02-09 04:16:03.000000000 +0000 @@ -7,7 +7,7 @@ setup( name="muttdown", - version="0.3.3", + version="0.3.4", author="James Brown", author_email="Roguelazer@gmail.com", url="https://github.com/Roguelazer/muttdown", diff -Nru muttdown-0.3.3/tests/data/cert.pem muttdown-0.3.4/tests/data/cert.pem --- muttdown-0.3.3/tests/data/cert.pem 1970-01-01 00:00:00.000000000 +0000 +++ muttdown-0.3.4/tests/data/cert.pem 2019-02-09 04:16:03.000000000 +0000 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICmDCCAYACCQC7muxZ8ym2UDANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJV +UzAgFw0xOTAyMDkwMzA0MDdaGA8yMTE5MDExNjAzMDQwN1owDTELMAkGA1UEBhMC +VVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCa7UukN2cZcltM5ajH +NVH7YeWGsiJjmobg1QQaf5AgCP1TC2WckN7cbmAp5nR8Ie2y8p4hu0dSziCLap6M +txtTJJ87IoTkLvVv2ZIUJBJ00xD3SlEKNDv/532PTnObDUsRzdDRXcKW3PKqprFr +kS1UKHgyR/U4pOENdRk5zN2Jkv5A/fWc2nkDwhYXInqW26WyxDJkamInRZF2iW6Y +t88QMnHtEtNrf1rom4UvCPTmZqh/9Pm6uhIHi7/aanSSnCasOfKd1wUTuk3wAOzQ +SxhVVEXPVHSI0yPQRDKm5O8VSSjRr0WC1ooljJarfrs8sLECTWpQOetlW2YlUjJf +udUvAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAJchLG3aIRnjcZU5unEu3icxVayi +OiUWqLPTZsNc4vVKn+h8HP3TeTUSWUMHVp7OqmVvbMeLDZJm7JBMklr1RYBgeGi7 +HeJzG3q/9O5Ny1CZ3Rok/mxncZlDKKG0Z61aBD8rzkOooQFIa1KAPNdkfBbnfuir +Lo83Y8Sy8OMA8yjatMxDt0sjhXJ++F83Tki9EMKEMhAcbY3nC5Df1c25va3O1IPH +5GkDb97lsey5UQ559XmHSBl+w8yZNRtN4k1Vq/aZuKCbyci9ogczU7eIjkawxhQ7 +X+O0ZgVqfLqrdCjnLYf7Gz3cEfYhaIfwQc9i6kOTyPZ/oQCz1E7Lf9KOI0c= +-----END CERTIFICATE----- diff -Nru muttdown-0.3.3/tests/data/key.pem muttdown-0.3.4/tests/data/key.pem --- muttdown-0.3.3/tests/data/key.pem 1970-01-01 00:00:00.000000000 +0000 +++ muttdown-0.3.4/tests/data/key.pem 2019-02-09 04:16:03.000000000 +0000 @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCa7UukN2cZcltM +5ajHNVH7YeWGsiJjmobg1QQaf5AgCP1TC2WckN7cbmAp5nR8Ie2y8p4hu0dSziCL +ap6MtxtTJJ87IoTkLvVv2ZIUJBJ00xD3SlEKNDv/532PTnObDUsRzdDRXcKW3PKq +prFrkS1UKHgyR/U4pOENdRk5zN2Jkv5A/fWc2nkDwhYXInqW26WyxDJkamInRZF2 +iW6Yt88QMnHtEtNrf1rom4UvCPTmZqh/9Pm6uhIHi7/aanSSnCasOfKd1wUTuk3w +AOzQSxhVVEXPVHSI0yPQRDKm5O8VSSjRr0WC1ooljJarfrs8sLECTWpQOetlW2Yl +UjJfudUvAgMBAAECggEAfm6G21XnSmn7vk5xpViLNfYXZQv8aoKR7euI9MMDcFFF +wr67RsEnToa47ZjHmQHrRK0ghXCbbSUQhBYXm8hWgUySsaSjBMCZxZSt1Mf3U+Vn +pBe++O/Vwyo8WnXwfCmmCLqI3kOA6LMZSlDM23bXoiWAqa/1nCtaCix00KmyZXEN +9x/2OKLorIRn3ewhY+kplhYt/P6MUlPUpPeX1RBMoF31iPFNcSiwwZSZjsK/nUI5 +MnZJT4NxcmZfBrP7RUBHZy5WFOlKQ0KO+zlYjXN+R0ZGuCvFAnuDtK37/BHQYc51 +gAqcdLCKnjOF00IhQNGnorEKMYvqIIgVIv39bGp2CQKBgQDLBPaE4Zz/CFL/Hqum +mlzMk/Oy+cixZJ3LYnLlptnHKTu8PkzoAH0fSRfRYuOBLuYhhhP3iCqnPBAQ0yu+ +48nMothMrbe4OCe+kUiQ/9VUyRLwr1sh0yFcZJBawstcpOX7YTfgJS3qtvQS4rJG +1FDrekBpz3Sgj1us5vya2e/7WwKBgQDDW3BZxuZMGJkeZdKpYrqp2hDrdn/t5lkb +TKs9fbEf8SMcQSlYJvoCjmy55NCW04TEFnPSwj/XNhwOPNDxurHMxy3+vfKbchsq +4vnqowpBCdZ1Q7osS9xlzqvLPqET3WXATtcYoYXMjSq5o5tzwcmiSoQnkDnaAnXZ +HeyD4aI5vQKBgBLTQfyuYv1vCysm7+nB9IrvyTA2Yzq3xr3+QgMzhowmMajR6hW1 +PeTxxSigT9JBxAslwKI6WSIqup6kxjCsNKEqFH5/uUJ2ypCsLhtr7Z8wCfaRfBTV +3AkSNiSEXZEYpU67BBBfwjM6hcVeigNxWpOLQX/OQdVFlc2hmZjOTqdzAoGASRN0 +RHDthruQ01kdYzVGQ/EJcTrjgdcvr9GPILJaxmsKSjBpycrSrJAgRa09BZ5bxInt +i4IUJWndNsozEqlWhxZeszLUhKc7WGCNQeL5G/kVGspZ4uYBrKeRhbaIxIiF3ljf +hxwsk6aeu9BifvuXdDjRlIcTzOQstynFZlPJvjUCgYBi7b/aCGtACAYL0wI/l4VJ +iHRghC7/E/wm6Fdu4DT3lUAFYyzV2X30e9Hy9RKGxRB4CHUA4hJ3TIjhCtrg0fA3 ++StWD+BqCO8NhiPOQzdCDsElsefMeS70iNg2+vxXn6Vw2QcW0d5G1bz44zH4DVuL +mLrBMIq8BZ6qJ8OrylCoMg== +-----END PRIVATE KEY----- diff -Nru muttdown-0.3.3/tests/test_basic.py muttdown-0.3.4/tests/test_basic.py --- muttdown-0.3.3/tests/test_basic.py 2018-12-27 21:49:45.000000000 +0000 +++ muttdown-0.3.4/tests/test_basic.py 2019-02-09 04:16:03.000000000 +0000 @@ -1,11 +1,21 @@ from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart +from email.mime.application import MIMEApplication from email.message import Message import tempfile import shutil +import socket +import threading +import ssl +import select +import time +import os +import sys import pytest +import yaml +from muttdown import main from muttdown.main import convert_tree from muttdown.main import process_message from muttdown.config import Config @@ -100,3 +110,164 @@ assert text_part.get_payload(decode=True) == b'!m\n\nThis is a message' html_part = converted.get_payload()[1] assert html_part.get_payload(decode=True) == b'

This is a message

' + + +def test_headers_when_multipart_signed(basic_config): + msg = MIMEMultipart('signed') + msg['Subject'] = 'Test Message' + msg['From'] = 'from@example.com' + msg['To'] = 'to@example.com' + msg['Bcc'] = 'bananas' + msg.preamble = 'Outer preamble' + + msg.attach(MIMEText("!m This is the main message body")) + msg.attach(MIMEApplication('signature here', 'pgp-signature', name='signature.asc')) + + converted, _ = convert_tree(msg, basic_config) + + assert converted['Subject'] == 'Test Message' + assert converted['From'] == 'from@example.com' + assert converted['To'] == 'to@example.com' + + assert isinstance(converted, MIMEMultipart) + assert converted.preamble == 'Outer preamble' + assert len(converted.get_payload()) == 2 + assert converted.get_content_type() == 'multipart/alternative' + html_part = converted.get_payload()[0] + original_signed_part = converted.get_payload()[1] + assert isinstance(html_part, MIMEText) + assert html_part.get_content_type() == 'text/html' + assert isinstance(original_signed_part, MIMEMultipart) + assert original_signed_part.get_content_type() == 'multipart/signed' + assert original_signed_part['Subject'] is None + text_part = original_signed_part.get_payload()[0] + signature_part = original_signed_part.get_payload()[1] + assert text_part.get_content_type() == 'text/plain' + assert signature_part.get_content_type() == 'application/pgp-signature' + + +class MockSmtpServer(object): + def __init__(self): + self._s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._s.bind(('127.0.0.1', 0)) + self.address = self._s.getsockname()[0:2] + self._t = None + self._started = threading.Event() + self.messages = [] + self.running = False + + def start(self): + self._t = threading.Thread(target=self.run) + self._t.start() + if self._started.wait(5) is not True: + raise ValueError('SMTP Server Thread failed to start!') + + def run(self): + if hasattr(ssl, 'create_default_context'): + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + else: + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain(certfile='tests/data/cert.pem', keyfile='tests/data/key.pem') + self._s.listen(128) + self._started.set() + self.running = True + while self.running: + r, _, x = select.select([self._s], [self._s], [self._s], 0.5) + if r: + start = time.time() + conn, addr = self._s.accept() + conn = context.wrap_socket(conn, server_side=True) + message = b'' + conn.sendall(b'220 localhost SMTP Fake\r\n') + message += conn.recv(1024) + conn.sendall(b'250-localhost\r\n250 DSN\r\n') + # MAIL FROM + message += conn.recv(1024) + conn.sendall(b'250 2.1.0 Ok\r\n') + # RCPT TO + message += conn.recv(1024) + conn.sendall(b'250 2.1.0 Ok\r\n') + # DATA + message += conn.recv(6) + conn.sendall(b'354 End data with .\r\n') + while time.time() < start + 5: + chunk = conn.recv(4096) + if not chunk: + break + message += chunk + if b'\r\n.\r\n' in message: + break + conn.sendall(b'250 2.1.0 Ok\r\n') + message += conn.recv(1024) + conn.sendall(b'221 Bye\r\n') + conn.close() + self.messages.append((addr, message)) + + def stop(self): + if self._t is not None: + self.running = False + self._t.join() + + +@pytest.fixture +def smtp_server(): + s = MockSmtpServer() + s.start() + try: + yield s + finally: + s.stop() + + +def test_main_smtplib(tempdir, smtp_server, mocker): + config_path = os.path.join(tempdir, 'config.yaml') + with open(config_path, 'w') as f: + yaml.dump({ + 'smtp_host': smtp_server.address[0], + 'smtp_port': smtp_server.address[1], + 'smtp_ssl': True + }, f) + msg = Message() + msg['Subject'] = 'Test Message' + msg['From'] = 'from@example.com' + msg['To'] = 'to@example.com' + msg['Bcc'] = 'bananas' + msg.set_payload('This message has no sigil') + mocker.patch.object(main, 'read_message', return_value=msg.as_string()) + main.main(['-c', config_path, '-f', 'from@example.com', 'to@example.com']) + + assert len(smtp_server.messages) == 1 + attr, transcript = smtp_server.messages[0] + assert b'Subject: Test Message' in transcript + assert b'no sigil' in transcript + + +def test_main_passthru(tempdir, mocker): + output_path = os.path.join(tempdir, 'output') + sendmail_path = os.path.join(tempdir, 'sendmail') + with open(sendmail_path, 'w') as f: + f.write('#!{0}\n'.format(sys.executable)) + f.write('import sys\n') + f.write('output_path = "{0}"\n'.format(output_path)) + f.write('open(output_path, "w").write(sys.stdin.read())\n') + f.write('sys.exit(0)') + os.chmod(sendmail_path, 0o750) + config_path = os.path.join(tempdir, 'config.yaml') + with open(config_path, 'w') as f: + yaml.dump({ + 'sendmail': sendmail_path + }, f) + + msg = Message() + msg['Subject'] = 'Test Message' + msg['From'] = 'from@example.com' + msg['To'] = 'to@example.com' + msg['Bcc'] = 'bananas' + msg.set_payload('This message has no sigil') + mocker.patch.object(main, 'read_message', return_value=msg.as_string()) + main.main(['-c', config_path, '-f', 'from@example.com', '-s', 'to@example.com']) + + with open(output_path, 'rb') as f: + transcript = f.read() + assert b'Subject: Test Message' in transcript + assert b'no sigil' in transcript