Merge branch 'master' into release/beta

mailpile/config.py

@@ -2036,7 +2036,7 @@ def get_profile(self, email=None):
 
         return default_profile
 
-    def get_sendmail(self, frm, rcpts=['-t']):
+    def get_route(self, frm, rcpts=['-t']):
         if len(rcpts) == 1:
             if rcpts[0].lower().endswith('.onion'):
                 return {"protocol": "smtorp",
@@ -2048,7 +2048,6 @@ def get_sendmail(self, frm, rcpts=['-t']):
         if self.routes[routeid] is not None:
             return self.routes[routeid]
         else:
-            print "Migration notice: Try running 'setup/migrate'."
             raise ValueError(_("Route %s for %s does not exist."
                                ) % (routeid, frm))
 

mailpile/mailutils.py

@@ -143,6 +143,20 @@ def MakeGnuPG(*args, **kwargs):
     return message
 
 
+def GetTextPayload(part):
+    mimetype = part.get_content_type() or 'text/plain'
+    cte = part.get('content-transfer-encoding', '').lower()
+    if mimetype[:5] == 'text/' and cte == 'base64':
+        # Mailing lists like to mess with text/plain parts, and Majordomo
+        # in particular isn't aware of base64 encoding. Compensate!
+        payload = part.get_payload(None, False) or ''
+        parts = payload.split('\n--')
+        parts[0] = base64.b64decode(parts[0])
+        return '\n--'.join(parts)
+    else:
+        return part.get_payload(None, True) or ''
+
+
 def ExtractEmails(string, strip_keys=True):
     emails = []
     startcrap = re.compile('^[\'\"<(]')
@@ -1232,8 +1246,7 @@ def decode_text(self, payload, charset='utf-8', binary=True):
 
     def decode_payload(self, part):
         charset = part.get_content_charset() or None
-        payload = part.get_payload(None, True) or ''
-        return self.decode_text(payload, charset=charset)
+        return self.decode_text(GetTextPayload(part), charset=charset)
 
     def parse_text_part(self, data, charset, crypto):
         psi = crypto['signature']

mailpile/plugins/autotag.py

@@ -366,8 +366,11 @@ def command(self):
 
 ##[ Keywords ]################################################################
 
-def filter_hook(session, msg_mid, msg, keywords, **ignored_kwargs):
+def filter_hook(session, msg_mid, msg, keywords, **kwargs):
     """Classify this message."""
+    if not kwargs.get('incoming', False):
+        return keywords
+
     config = session.config
     for at_config in config.prefs.autotag:
         try:

mailpile/plugins/compose.py

@@ -870,7 +870,8 @@ def command(self, emails=None):
                         data={'mid': msg_mid, 'sid': msg_sid}))
 
                 SendMail(session, msg_mid,
-                         [PrepareMessage(config, email.get_msg(pgpmime=False),
+                         [PrepareMessage(config,
+                                         email.get_msg(pgpmime=False),
                                          sender=sender,
                                          rcpts=(bounce_to or None),
                                          bounce=(True if bounce_to else False),
@@ -901,7 +902,8 @@ def command(self, emails=None):
             # FIXME: Also fatal, when the SMTP server REJECTS the mail
             except:
                 # We want to try that again!
-                message = _('Failed to send %s') % email
+                subject = email.get_msg_info(config.index.MSG_SUBJECT)
+                message = _('Failed to send message: %s') % subject
                 for ev in events:
                     ev.flags = Event.INCOMPLETE
                     ev.message = message
@@ -1031,14 +1033,27 @@ def command(self):
         if not idx:
             return self._error(_('The index is not ready yet'))
 
+        # Collect a list of messages from the outbox
         messages = []
         for tag in cfg.get_tags(type='outbox'):
             search = ['in:%s' % tag._key]
             for msg_idx_pos in idx.search(self.session, search,
                                           order='flat-index').as_set():
                 messages.append('=%s' % b36(msg_idx_pos))
+
+        # Messages no longer in the outbox get their events canceled...
+        if cfg.event_log:
+            events = cfg.event_log.incomplete(source='.plugins.compose.Sendit')
+            for ev in events:
+                if ('mid' in ev.data and
+                        ('=%s' % ev.data['mid']) not in messages):
+                    ev.flags = ev.COMPLETE
+                    ev.message = _('Sending cancelled.')
+                    cfg.event_log.log_event(ev)
+
+        # Send all the mail!
         if messages:
-            self.args = tuple(messages)
+            self.args = tuple(set(messages))
             return Sendit.command(self)
         else:
             return self._success(_('The outbox is empty'))

mailpile/search.py

@@ -16,7 +16,7 @@
 from mailpile.i18n import ngettext as _n
 from mailpile.plugins import PluginManager
 from mailpile.mailutils import FormatMbxId, MBX_ID_LEN, NoSuchMailboxError
-from mailpile.mailutils import AddressHeaderParser
+from mailpile.mailutils import AddressHeaderParser, GetTextPayload
 from mailpile.mailutils import ExtractEmails, ExtractEmailAndName
 from mailpile.mailutils import Email, ParseMessage, HeaderPrint
 from mailpile.postinglist import GlobalPostingList
@@ -1268,7 +1268,7 @@ def read_message(self, session,
 
             def _loader(p):
                 if payload[0] is None:
-                    payload[0] = self.try_decode(p.get_payload(None, True),
+                    payload[0] = self.try_decode(GetTextPayload(p),
                                                  charset)
                 return payload[0]
 

mailpile/smtp_client.py

@@ -95,7 +95,8 @@ def __init__(self, msg, details=None):
 def _RouteTuples(session, from_to_msg_ev_tuples, test_route=None):
     tuples = []
     for frm, to, msg, events in from_to_msg_ev_tuples:
-        dest = {}
+        rcpts = {}
+        routes = {}
         for recipient in to:
             # If any of the events thinks this message has been delivered,
             # then don't try to send it again.
@@ -110,27 +111,21 @@ def _RouteTuples(session, from_to_msg_ev_tuples, test_route=None):
                          "password": "",
                          "command": "",
                          "host": "",
-                         "port": 25
-                         }
+                         "port": 25}
 
                 if test_route:
                     route.update(test_route)
                 else:
-                    route.update(session.config.get_sendmail(frm, [recipient]))
-
-                if route["command"]:
-                    txtroute = "|%(command)s" % route
-                else:
-                    # FIXME: This is dumb, makes it hard to handle usernames
-                    #        or passwords with funky characters in them :-(
-                    txtroute = "%(protocol)s://%(username)s:%(password)s@" \
-                               + "%(host)s:%(port)d"
-                    txtroute %= route
-
-                dest[txtroute] = dest.get(txtroute, [])
-                dest[txtroute].append(recipient)
-        for route in dest:
-            tuples.append((frm, route, dest[route], msg, events))
+                    route.update(session.config.get_route(frm, [recipient]))
+
+                # Group together recipients that use the same route
+                rid = '/'.join(sorted(['%s' % (k, )
+                                       for k in route.iteritems()]))
+                routes[rid] = route
+                rcpts[rid] = rcpts.get(rid, [])
+                rcpts[rid].append(recipient)
+        for rid in rcpts:
+            tuples.append((frm, routes[rid], rcpts[rid], msg, events))
     return tuples
 
 
@@ -147,12 +142,12 @@ def SendMail(session, msg_mid, from_to_msg_ev_tuples,
 
     # Update initial event state before we go through and start
     # trying to deliver stuff.
-    for frm, sendmail, to, msg, events in routes:
+    for frm, route, to, msg, events in routes:
         for ev in (events or []):
             for rcpt in to:
                 ev.private_data['>'.join([frm, rcpt])] = False
 
-    for frm, sendmail, to, msg, events in routes:
+    for frm, route, to, msg, events in routes:
         for ev in events:
             ev.data['recipients'] = len(ev.private_data.keys())
             ev.data['delivered'] = len([k for k in ev.private_data
@@ -179,22 +174,25 @@ def smtp_do_or_die(msg, events, method, *args, **kwargs):
                  details={'smtp_error': '%s: %s' % (rc, msg)})
 
     # Do the actual delivering...
-    for frm, sendmail, to, msg, events in routes:
+    for frm, route, to, msg, events in routes:
+        route_description = route['command'] or route['host']
+
         frm_vcard = session.config.vcards.get_vcard(frm)
         update_to_vcards = msg and msg["x-mp-internal-pubkeys-attached"]
 
         if 'sendmail' in session.config.sys.debug:
-            sys.stderr.write(_('SendMail: from %s (%s), to %s via %s\n'
-                               ) % (frm,
-                                    frm_vcard and frm_vcard.random_uid or '',
-                                    to, sendmail.split('@')[-1]))
+            sys.stderr.write(_('SendMail: from %s (%s), to %s via %s\n')
+                             % (frm, frm_vcard and frm_vcard.random_uid or '',
+                                to, route_description))
         sm_write = sm_close = lambda: True
 
-        mark(_('Connecting to %s') % sendmail.split('@')[-1], events)
+        mark(_('Sending via %s') % route_description, events)
 
-        if sendmail.startswith('|'):
-            sendmail %= {"rcpt": ",".join(to)}
-            cmd = sendmail[1:].strip().split()
+        if route['command']:
+            # Note: The .strip().split() here converts our cmd into a list,
+            #       which should ensure that Popen does not spawn a shell
+            #       with potentially exploitable arguments.
+            cmd = (route['command'] % {"rcpt": ",".join(to)}).strip().split()
             proc = Popen(cmd, stdin=PIPE, long_running=True)
             sm_startup = None
             sm_write = proc.stdin.write
@@ -213,19 +211,11 @@ def sm_close():
                 ev.data['proto'] = 'subprocess'
                 ev.data['command'] = cmd[0]
 
-        elif (sendmail.startswith('smtp:') or
-              sendmail.startswith('smtorp:') or
-              sendmail.startswith('smtpssl:') or
-              sendmail.startswith('smtptls:')):
-            proto = sendmail.split(':', 1)[0]
-            host, port = sendmail.split(':', 1
-                                        )[1].replace('/', '').rsplit(':', 1)
+        elif route['protocol'] in ('smtp', 'smtorp', 'smtpssl', 'smtptls'):
+            proto = route['protocol']
+            host, port = route['host'], route['port']
+            user, pwd = route['username'], route['password']
             smtp_ssl = proto in ('smtpssl', )  # FIXME: 'smtorp'
-            if '@' in host:
-                userpass, host = host.rsplit('@', 1)
-                user, pwd = userpass.split(':', 1)
-            else:
-                user = pwd = None
 
             for ev in events:
                 ev.data['proto'] = proto
@@ -242,7 +232,7 @@ def sm_connect_server():
                           )(local_hostname='mailpile.local', timeout=25)
                 if 'sendmail' in session.config.sys.debug:
                     server.set_debuglevel(1)
-                if smtp_ssl or sendmail[:7] in ('smtorp', 'smtptls'):
+                if smtp_ssl or proto in ('smtorp', 'smtptls'):
                     conn_needs = [ConnBroker.OUTGOING_ENCRYPTED]
                 else:
                     conn_needs = [ConnBroker.OUTGOING_SMTP]
@@ -260,15 +250,16 @@ def sm_startup():
                     try:
                         server.starttls()
                     except:
-                        if sendmail.startswith('smtptls'):
+                        if proto == 'smtptls':
                             raise InsecureSmtpError()
                         else:
                             server = sm_connect_server()
                 serverbox[0] = server
 
                 if user and pwd:
                     try:
-                        server.login(user.encode('utf-8'), pwd.encode('utf-8'))
+                        server.login(user.encode('utf-8'),
+                                     pwd.encode('utf-8'))
                     except UnicodeDecodeError:
                         fail(_('Bad character in username or password'),
                              events,
@@ -308,8 +299,7 @@ def sm_cleanup():
                 if hasattr(server, 'sock'):
                     server.close()
         else:
-            fail(_('Invalid sendmail command/SMTP server: %s') % sendmail,
-                 events)
+            fail(_('Invalid route: %s') % route, events)
 
         try:
             # Run the entire connect/login sequence in a single timer, but