openpgpkey-milter

#!/usr/bin/python3
# -*- coding: utf-8 -*-

# Copyright 2013-2014 Paul Wouters <paul@cypherpunks.ca>
#
# Based on the pymilter example code
# See the pymilter project at http://bmsi.com/python/milter.html
# based on Sendmail's milter API http://www.milter.org/milter_api/api.html

# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.

VERSION = '0.5'
OPENPGPKEY = 61

import Milter
try:
    from io import  StringIO
except:
    from StringIO import StringIO
import time
import email
import sys
import os
import shutil
import argparse
import random
import string
from hashlib import sha1
from hashlib import sha256
from email.utils import parseaddr
import requests

from socket import AF_INET6
from Milter.utils import parse_addr
if True:
    from multiprocessing import Process as Thread, Queue
else:
    from threading import Thread
    from queue import Queue

logq = Queue(maxsize=4)

from syslog import syslog, openlog, LOG_MAIL
try:
    openlog('openpgpkey-milter', facility=LOG_MAIL)
except:
    # for python 2.6
    openlog('openpgpkey-milter', LOG_MAIL)

try:
    import setproctitle
    setproctitle.setproctitle("openpgpkey-milter")
except:
    syslog('openpgpkey-milter: failed to setproctitle - python-setproctitle missing?')

import unbound
ctx = unbound.ub_ctx()
ctx.resolvconf('/etc/resolv.conf')

ROOTKEY="none"
cauldron = ( "/var/lib/unbound/root.anchor", "/var/lib/unbound/root.key", "/etc/unbound/root.key" )
for root in cauldron:
    if os.path.isfile(root):
       ROOTKEY=root
       break

try:
    if ROOTKEY != "none":
       ctx.add_ta_file(ROOTKEY)
except:
    pass

spool_dir = '/var/spool/openpgpkey-milter'

import gnupg

def sha256trunc(data):
    """Compute SHA2-256 hash truncated to 28 octets."""
    return sha256(data).hexdigest()[:56]

# Taken from https://hg.intevation.de/gnupg/wkd-tools/file/tip/generate-openpgpkey-hu
def zb32_encode(data):
    """Return data in zbase 32 encoding.

    Data must be convertible to a bytearray.

    Implementation is derived from GnuPG's common/zb32.c
    as published in gnupg-2.1.15.
    """
    zb32asc = "ybndrfg8ejkmcpqxot1uwisza345h769"

    data = bytearray(data)
    databits = len(data) * 8
    datalen = (databits + 7) / 8

    output = ""

    while datalen >= 5:
        output += zb32asc[((data[0]      ) >> 3)                  ]
        output += zb32asc[((data[0] &   7) << 2) | (data[1] >> 6) ]
        output += zb32asc[((data[1] &  63) >> 1)                  ]
        output += zb32asc[((data[1] &   1) << 4) | (data[2] >> 4) ]
        output += zb32asc[((data[2] &  15) << 1) | (data[3] >> 7) ]
        output += zb32asc[((data[3] & 127) >> 2)                  ]
        output += zb32asc[((data[3] &   3) << 3) | (data[4] >> 5) ]
        output += zb32asc[((data[4] &  31)     )                  ]
        data = data[5:]
        datalen -= 5

    if datalen == 4:
        output += zb32asc[((data[0]      ) >> 3)                  ]
        output += zb32asc[((data[0] &   7) << 2) | (data[1] >> 6) ]
        output += zb32asc[((data[1] &  63) >> 1)                  ]
        output += zb32asc[((data[1] &   1) << 4) | (data[2] >> 4) ]
        output += zb32asc[((data[2] &  15) << 1) | (data[3] >> 7) ]
        output += zb32asc[((data[3] & 127) >> 2)                  ]
        output += zb32asc[((data[3] &   3) << 3)                  ]
    elif datalen == 3:
        output += zb32asc[((data[0]      ) >> 3)                  ]
        output += zb32asc[((data[0] &   7) << 2) | (data[1] >> 6) ]
        output += zb32asc[((data[1] &  63) >> 1)                  ]
        output += zb32asc[((data[1] &   1) << 4) | (data[2] >> 4) ]
        output += zb32asc[((data[2] &  15) << 1)                  ]
    elif datalen == 2:
        output += zb32asc[((data[0]      ) >> 3)                  ]
        output += zb32asc[((data[0] &   7) << 2) | (data[1] >> 6) ]
        output += zb32asc[((data[1] &  63) >> 1)                  ]
        output += zb32asc[((data[1] &   1) << 4)                  ]
    elif datalen == 1:
        output += zb32asc[((data[0]      ) >> 3)                  ]
        output += zb32asc[((data[0] &   7) << 2)                  ]

    # Need to strip some bytes if not a multiple of 40.
    output_offset = int((databits + 5 - 1) / 5)
    output = output[:output_offset]
    return output

def zb32sha1(data):
    return zb32_encode(sha1(data).digest())

class myMilter(Milter.Base):

    def __init__(self):  # A new instance with each new connection.
        self.id = Milter.uniqueID()  # Integer incremented with each call.

  # each connection runs in its own thread and has its own myMilter
  # instance.  Python code must be thread safe.  This is trivial if only stuff
  # in myMilter instances is referenced.

    @Milter.noreply
    def connect(
        self,
        IPname,
        family,
        hostaddr,
        ):
        self.IP = hostaddr[0]
        self.port = hostaddr[1]
        if family == AF_INET6:
            self.flow = hostaddr[2]
            self.scope = hostaddr[3]
        else:
            self.flow = None
            self.scope = None
        self.IPname = IPname  # Name from a reverse IP lookup
        self.H = None
        self.fp = None
        self.receiver = self.getsymval('j')
        syslog('connect from %s at %s' % (IPname, hostaddr))
        return Milter.CONTINUE

    def hello(self, heloname):
        self.H = heloname
        return Milter.CONTINUE

    def envfrom(self, mailfrom, *str):
        self.F = mailfrom
        self.R = []  # list of recipients
        self.msg_body = []
        self.fromparms = Milter.dictfromlist(str)  # ESMTP parms
        self.user = self.getsymval('{auth_authen}')  # authenticated user

        self.fp = StringIO()
        self.canon_from = '@'.join(parse_addr(mailfrom))
        self.fp.write(u'From %s %s\n' % (self.canon_from, time.ctime()))
        return Milter.CONTINUE

    @Milter.noreply
    def envrcpt(self, to, *str):
        rcptinfo = (to, Milter.dictfromlist(str))
        rcpto = rcptinfo[0]
        self.R.append(parseaddr(rcpto)[1])
        return Milter.CONTINUE

    @Milter.noreply
    def header(self, name, hval):
        self.fp.write(u'%s: %s\n' % (name, hval))  # add header to buffer
        return Milter.CONTINUE

    @Milter.noreply
    def eoh(self):
        self.fp.write(u'\n')  # terminate headers
        return Milter.CONTINUE

#    @Milter.noreply
    def body(self, chunk):
        #TODO: properly handle binary data
        try:
            self.fp.write(chunk.decode('utf-8', 'strict'))
            self.msg_body.append(chunk.decode('utf-8', 'strict'))
        except:
            return Milter.ACCEPT
        return Milter.CONTINUE

    def eom(self):
        self.fp.seek(0)
        msg_body = ''.join(self.msg_body)

        self.addheader('X-OPENPGPKEY', 'Message passed unmodified' , 1)
        msg = email.message_from_file(self.fp)
        # msg is an email.message.Message
        # http://docs.python.org/release/2.7.6/library/email.message.html

        self.fp.close()
        del self.fp

        if not len(self.R) > 0:
            syslog('No recipients')
            return Milter.CONTINUE

        # Protect against super-encryption
        if '-----BEGIN PGP MESSAGE-----' in msg_body:
            # already encrypted, let it go as is
            syslog('Message already encrypted with PGP - letting it go unmodified')
            return Milter.CONTINUE

        # Protect against super-encryption
        if "Content-Type" in msg:
            if "pkcs7-mime" in msg["Content-Type"] and \
                "enveloped-data" in msg["Content-Type"]:
                syslog('Message already encrypted with S/MIME - letting it go unmodified')
                return Milter.CONTINUE

            if "application/octet-stream" in msg["Content-Type"] and \
                ".p7m" in msg["Content-Type"] or \
                "application/x-pkcs7-mime" in msg["Content-Type"] or \
                "application/pkcs7-mime" in msg["Content-Type"]:
                syslog('Message already encrypted with S/MIME - letting it go unmodified')
                return Milter.CONTINUE

        gpgdir = '%s/%s' % (spool_dir, self.id)
        if os.path.isdir(gpgdir):
            shutil.rmtree(gpgdir)
        os.makedirs(gpgdir)

        gpg = gnupg.GPG(gnupghome=gpgdir)
        gpg.decode_errors="ignore"
        for recipient in self.R:
            try:
                (username, domainname) = recipient.split('@')
            except: # null mailer <> or bad address
                syslog("Skipping recipient address <%s>" % recipient)
                return Milter.CONTINUE
            # lowercase for sanity - not currently part of specification
            rfcname = sha256trunc(username.lower().encode('utf-8'))
            wkdname = zb32sha1(username.lower().encode('utf-8'))
            wkd_key = None

            # try fetching key from WKD
            try:
                advanced_query = "https://openpgpkey.%s/.well-known/openpgpkey/%s/hu/%s" % (domainname.lower(), domainname.lower(), wkdname)
                wkd_request = requests.get(advanced_query, allow_redirects=False, params={"l": username}, timeout=2)
                if wkd_request.status_code == 200:
                    wkd_key = wkd_request.content
            except:
                pass
            if wkd_key is None:
                try:
                    direct_query = "https://%s/.well-known/openpgpkey/hu/%s" % (domainname.lower(), wkdname)
                    wkd_request = requests.get(direct_query, allow_redirects=False, params={"l": username}, timeout=2)
                    if wkd_request.status_code == 200:
                        wkd_key = wkd_request.content
                except:
                    pass

            qname = '%s._openpgpkey.%s' % (rfcname, domainname)
            (status, result) = ctx.resolve(qname, OPENPGPKEY,
                    unbound.RR_CLASS_IN)
            if wkd_key is None and status != 0:
                # can this ever happen?
                syslog("unbound openpgpkey lookup for '%s' returned non-zero status, deferring" % recipient)
                return Milter.TEMPFAIL
            if wkd_key is None and result.rcode_str == 'serv fail':
                syslog("unbound openpgpkey lookup for '%s' returned SERVFAIL, deferring" % recipient)
                return Milter.TEMPFAIL
            if wkd_key is None and result.bogus:
                syslog("unbound openpgpkey lookup for '%s' returned with INVALID DNSSEC data, deferring" % recipient)
                return Milter.TEMPFAIL
            if wkd_key is None and not result.secure:
                syslog("unbound openpgpkey lookup for '%s' ignored as the domain is not signed with DNSSEC - letting go plaintext" % recipient)
                return Milter.CONTINUE
            if wkd_key is None and not result.havedata:
                syslog("unbound openpgpkey lookup for '%s' succeeded but no OpenPGP key publishd - letting go plaintext" % recipient)
                return Milter.CONTINUE

            # Finally, we have a key!
            import_count = 0
            if result.havedata:
                for openpgpkey in result.data.raw:
                    import_result = gpg.import_keys(openpgpkey)
                    import_count += import_result.count

            if not wkd_key is None:
                import_result = gpg.import_keys(wkd_key)
                import_count += import_result.count

            if import_count < 1:
                syslog("no keys could be imported for '%s' - letting go plaintext" % recipient)
                return Milter.CONTINUE

        # if we get here, all recipients had an valid openpgpkey record so we can encrypt the message
        # collect fingerprints needed to target message encryption
        fingerprints = []
        imported_keys = gpg.list_keys()
        for ikey in imported_keys:
            syslog('Received OPENPGPKEY for %s: Key-ID:%s Fingerprint:%s'
                    % (recipient, ikey['keyid'], ikey['fingerprint']))
            fingerprints.append(ikey['fingerprint'])
        fpliststr = ','.join(fingerprints)

        # place exception rules here
        #if recipient == "plaintext@nohats.ca" or ikey['keyid'] == "1111111111111111":
        #   syslog("Skipping encryption");
        #   return Milter.CONTINUE

        outerboundary = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))
        innerboundary = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))

        mime_header = '';
        if 'Content-Type' in msg:
            mime_header += 'Content-Type: %s' % msg.get('Content-Type')
        else:
            mime_header += 'Content-Type: text/plain'

        # MIME headers from https://www.iana.org/assignments/message-headers/message-headers.xhtml
        mime_message_headers = ['Base', 'Content-Alternative', 'Content-Base', 'Content-Description', 'Content-Disposition', 'Content-Duration', 'Content-features', 'Content-ID', 'Content-Language', 'Content-Location', 'Content-MD5', 'Content-Transfer-Encoding', 'Content-Translation-Type', 'MIME-Version']
        for header in mime_message_headers:
            if header in msg:
                mime_header += '\n%s: %s' % (header, msg.get(header))

        mime_msgstr = '''Content-Type: multipart/mixed; boundary="%s";
 protected-headers="v1"
Subject: %s

--%s
%s

%s

--%s--''' % (innerboundary, msg.get('subject'), innerboundary, mime_header, msg_body, innerboundary)

        syslog('Will encrypt message to fingerprints:%s' % fpliststr)
        mime_enc_msg = gpg.encrypt(mime_msgstr.encode('utf-8'), fingerprints, always_trust=True)
        if not mime_enc_msg.ok:
            syslog("Encryption to %s failed with status '%s' - letting go plaintext" % (fpliststr, mime_enc_msg.status))
            return Milter.CONTINUE

        mime = '''This is an OpenPGP/MIME encrypted message (RFC 4880 and 3156)
--%s
Content-Type: application/pgp-encrypted
Content-Description: PGP/MIME version identification

Version: 1

--%s
Content-Type: application/octet-stream; name="encrypted.asc"
Content-Description: OpenPGP encrypted message
Content-Disposition: inline; filename="encrypted.asc"

%s
--%s--''' % (outerboundary, outerboundary, str(mime_enc_msg), outerboundary)

        # Replace original email with our encrypted version
        for header in mime_message_headers:
            if header in msg:
                self.chgheader(header, 1, None)
        self.replacebody(mime)
        self.chgheader('User-Agent', 1, 'dkim-openpgpkey')
        self.chgheader('Subject', 1, '[openpgpkey-milter encrypted message]')
        self.chgheader('X-OPENPGPKEY', 1, 'Encrypted to key(s): %s ' % fpliststr)
        self.chgheader('Content-Transfer-Encoding', 1, '8bit')
        self.chgheader('MIME-Version', 1, '1.0')
        self.chgheader('Content-Type', 1, 'multipart/encrypted;\n protocol="application/pgp-encrypted"; boundary="%s"' % outerboundary)


        return Milter.ACCEPT

    def close(self):
        # always called, even when abort is called.  Clean up
        # any external resources here.
        gpgdir = '%s/%s' % (spool_dir, self.id)
        if os.path.isdir(gpgdir):
            shutil.rmtree(gpgdir)
        return Milter.CONTINUE

    def abort(self):
        # client disconnected prematurely
        return Milter.CONTINUE


  # # === Support Functions ===

def background():
    while True:
        t = logq.get()
        if not t:
            break
        (msg, mid, ts) = t
        mymsgs = ''
        for i in msg:
            mymsgs += '%s ' % i
        syslog('backgrounding [%d] ' % mid, mymsgs)

## ===

def main():
    global spool_dir
    global ctx
    parser = \
        argparse.ArgumentParser(description='OPENPGPKEY milter application'
                                , epilog='For bugs. see paul@nohats.ca')
    parser.add_argument('--anchor', '-a', action='store', default='',
                        help='location of the unbound DNSSEC trust anchor file (default /var/lib/unbound/root.anchor')
    parser.add_argument('--port', '-p', action='store', default='8890',
                        help='port on localhost to use (default 8890)')
    parser.add_argument('--pid', '-P', action='store', default='',
                        help='pidfile to create (default no pid file is created')
    parser.add_argument('--rrtype', '-r', action='store',
                        default='65280',
                        help='RRtype allocation (default private use 65280)')
    parser.add_argument('--spool', '-s', action='store',
                        default='/var/spool/openpgpkey-milter',
                        help='spool dir for tmp files (default /var/spool/openpgpkey-milter)')
    parser.add_argument('--timeout', '-t', action='store', default=600,
                        help='timeout (default 600)')
    parser.add_argument('--version', action='store_true',
                        help='show version and exit')
    args = parser.parse_args()
    if args.version:
        print('openpgpkey-milter version %s by Paul Wouters <paul@cypherpunks.ca>' % VERSION)
        print('     options: --rrtype %s --spool %s  --port %s  --timeout %s --pid <pidfile>' % (args.rrtype, args.spool, args.port, args.timeout))
        sys.exit()

    if args.anchor:
        if not os.path.isfile(args.anchor):
           sys.exit("anchor file '%s' does not exist"%args.anchor)
        ctx.add_ta_file(args.anchor)

    socketname = 'inet:%s@127.0.0.1' % args.port
    spool_dir = args.spool

    bt = Thread(target=background)
    bt.start()

    # Register to have the Milter factory create instances of your class:
    Milter.factory = myMilter
    flags = Milter.CHGBODY + Milter.CHGHDRS + Milter.ADDHDRS
    flags += Milter.ADDRCPT
    flags += Milter.DELRCPT
    Milter.set_flags(flags)

    mypid = str(os.getpid())
    if args.pid:
       try:
            fp = open(args.pid,"w")
            fp.write(mypid)
            fp.close()
       except:
              sys.exit("Failed to write pid, aborted")

    syslog('starting daemon [%s] version %s on port %s at %s with timeout %s'
            % (mypid, VERSION, args.port, args.spool, args.timeout))
    sys.stdout.flush()
    Milter.runmilter('pythonfilter', socketname, args.timeout)
    logq.put(None)
    bt.join()
    syslog('shutting down daemon')

    if os.path.isfile(args.pid) and not os.path.islink(args.pid):
       os.unlink(args.pid)

if __name__ == '__main__':
    main()