cc.py

from setuptools import distutils as dutils
from setuptools.command import build_ext
from setuptools.extension import Extension

import os
import shutil
import sys
import tempfile

from numba.core import typing, sigutils
from numba.core.compiler_lock import global_compiler_lock
from numba.pycc.compiler import ModuleCompiler, ExportEntry
from numba.pycc.platform import Toolchain
from numba import cext


dir_util = dutils.dir_util
log = dutils.log
extension_libs = cext.get_extension_libs()


class CC(object):
    """
    An ahead-of-time compiler to create extension modules that don't
    depend on Numba.
    """

    # NOTE: using ccache can speed up repetitive builds
    # (especially for the mixin modules)

    _mixin_sources = ['modulemixin.c',]  + extension_libs

    # -flto strips all unused helper functions, which 1) makes the
    # produced output much smaller and 2) can make the linking step faster.
    # (the Windows linker seems to do this by default, judging by the results)

    _extra_cflags = {
        # Comment out due to odd behavior with GCC 4.9+ with LTO
        # 'posix': ['-flto'],
        }

    _extra_ldflags = {
        # Comment out due to odd behavior with GCC 4.9+ with LTO
        # 'posix': ['-flto'],
        }

    def __init__(self, extension_name, source_module=None):
        if '.' in extension_name:
            raise ValueError("basename should be a simple module name, not "
                             "qualified name")

        self._basename = extension_name
        self._init_function = 'pycc_init_' + extension_name
        self._exported_functions = {}
        # Resolve source module name and directory
        f = sys._getframe(1)
        if source_module is None:
            dct = f.f_globals
            source_module = dct['__name__']
        elif hasattr(source_module, '__name__'):
            dct = source_module.__dict__
            source_module = source_module.__name__
        else:
            dct = sys.modules[source_module].__dict__

        self._source_path = dct.get('__file__', '')
        self._source_module = source_module
        self._toolchain = Toolchain()
        self._verbose = False
        # By default, output in directory of caller module
        self._output_dir = os.path.dirname(self._source_path)
        self._output_file = self._toolchain.get_ext_filename(extension_name)
        self._use_nrt = True
        self._target_cpu = ''

    @property
    def name(self):
        """
        The name of the extension module to create.
        """
        return self._basename

    @property
    def output_file(self):
        """
        The specific output file (a DLL) that will be generated.
        """
        return self._output_file

    @output_file.setter
    def output_file(self, value):
        self._output_file = value

    @property
    def output_dir(self):
        """
        The directory the output file will be put in.
        """
        return self._output_dir

    @output_dir.setter
    def output_dir(self, value):
        self._output_dir = value

    @property
    def use_nrt(self):
        return self._use_nrt

    @use_nrt.setter
    def use_nrt(self, value):
        self._use_nrt = value

    @property
    def target_cpu(self):
        """
        The target CPU model for code generation.
        """
        return self._target_cpu

    @target_cpu.setter
    def target_cpu(self, value):
        self._target_cpu = value

    @property
    def verbose(self):
        """
        Whether to display detailed information when compiling.
        """
        return self._verbose

    @verbose.setter
    def verbose(self, value):
        self._verbose = value

    def export(self, exported_name, sig):
        """
        Mark a function for exporting in the extension module.
        """
        fn_args, fn_retty = sigutils.normalize_signature(sig)
        sig = typing.signature(fn_retty, *fn_args)
        if exported_name in self._exported_functions:
            raise KeyError("duplicated export symbol %s" % (exported_name))

        def decorator(func):
            entry = ExportEntry(exported_name, sig, func)
            self._exported_functions[exported_name] = entry
            return func

        return decorator

    @property
    def _export_entries(self):
        return sorted(self._exported_functions.values(),
                      key=lambda entry: entry.symbol)

    def _get_mixin_sources(self):
        here = os.path.dirname(__file__)
        mixin_sources = self._mixin_sources[:]
        if self._use_nrt:
            mixin_sources.append('../core/runtime/nrt.cpp')
        return [os.path.join(here, f) for f in mixin_sources]

    def _get_mixin_defines(self):
        # Macro definitions required by modulemixin.c
        return [
            ('PYCC_MODULE_NAME', self._basename),
            ('PYCC_USE_NRT', int(self._use_nrt)),
            ]

    def _get_extra_cflags(self):
        extra_cflags = self._extra_cflags.get(sys.platform, [])
        if not extra_cflags:
            extra_cflags = self._extra_cflags.get(os.name, [])
        return extra_cflags

    def _get_extra_ldflags(self):
        extra_ldflags = self._extra_ldflags.get(sys.platform, [])
        if not extra_ldflags:
            extra_ldflags = self._extra_ldflags.get(os.name, [])
        # helperlib uses pthread on linux. make sure we are linking to it.
        if sys.platform.startswith("linux"):
            if "-pthread" not in extra_ldflags:
                extra_ldflags.append('-pthread')
        return extra_ldflags

    def _compile_mixins(self, build_dir):
        sources = self._get_mixin_sources()
        macros = self._get_mixin_defines()
        include_dirs = self._toolchain.get_python_include_dirs()

        extra_cflags = self._get_extra_cflags()
        # XXX distutils creates a whole subtree inside build_dir,
        # e.g. /tmp/test_pycc/home/antoine/numba/numba/pycc/modulemixin.o
        objects = self._toolchain.compile_objects(sources, build_dir,
                                                  include_dirs=include_dirs,
                                                  macros=macros,
                                                  extra_cflags=extra_cflags)
        return objects

    @global_compiler_lock
    def _compile_object_files(self, build_dir):
        compiler = ModuleCompiler(self._export_entries, self._basename,
                                self._use_nrt, cpu_name=self._target_cpu)
        compiler.external_init_function = self._init_function
        temp_obj = os.path.join(build_dir,
                                os.path.splitext(self._output_file)[0] + '.o')
        log.info("generating LLVM code for '%s' into %s",
                self._basename, temp_obj)
        compiler.write_native_object(temp_obj, wrap=True)
        return [temp_obj], compiler.dll_exports

    @global_compiler_lock
    def compile(self):
        """
        Compile the extension module.
        """
        self._toolchain.verbose = self.verbose
        build_dir = tempfile.mkdtemp(prefix='pycc-build-%s-' % self._basename)

        # Compile object file
        objects, dll_exports = self._compile_object_files(build_dir)

        # Compile mixins
        objects += self._compile_mixins(build_dir)

        # Then create shared library
        extra_ldflags = self._get_extra_ldflags()
        output_dll = os.path.join(self._output_dir, self._output_file)
        libraries = self._toolchain.get_python_libraries()
        library_dirs = self._toolchain.get_python_library_dirs()
        self._toolchain.link_shared(output_dll, objects,
                                    libraries, library_dirs,
                                    export_symbols=dll_exports,
                                    extra_ldflags=extra_ldflags)

        shutil.rmtree(build_dir)

    def distutils_extension(self, **kwargs):
        """
        Create a distutils extension object that can be used in your
        setup.py.
        """
        macros = kwargs.pop('macros', []) + self._get_mixin_defines()
        depends = kwargs.pop('depends', []) + [self._source_path]
        extra_compile_args = (kwargs.pop('extra_compile_args', [])
                              + self._get_extra_cflags())
        extra_link_args = (kwargs.pop('extra_link_args', [])
                           + self._get_extra_ldflags())
        include_dirs = (kwargs.pop('include_dirs', [])
                        + self._toolchain.get_python_include_dirs())
        libraries = (kwargs.pop('libraries', [])
                     + self._toolchain.get_python_libraries())
        library_dirs = (kwargs.pop('library_dirs', [])
                        + self._toolchain.get_python_library_dirs())
        python_package_path = self._source_module[:self._source_module.rfind('.')+1]

        ext = _CCExtension(name=python_package_path + self._basename,
                           sources=self._get_mixin_sources(),
                           depends=depends,
                           define_macros=macros,
                           include_dirs=include_dirs,
                           libraries=libraries,
                           library_dirs=library_dirs,
                           extra_compile_args=extra_compile_args,
                           extra_link_args=extra_link_args,
                           **kwargs)
        ext.monkey_patch_distutils()
        ext._cc = self
        return ext


class _CCExtension(Extension):
    """
    A Numba-specific Extension subclass to LLVM-compile pure Python code
    to an extension module.
    """

    _cc = None
    _distutils_monkey_patched = False

    def _prepare_object_files(self, build_ext):
        cc = self._cc
        dir_util.mkpath(os.path.join(build_ext.build_temp, *self.name.split('.')[:-1]))
        objects, _ = cc._compile_object_files(build_ext.build_temp)
        # Add generated object files for linking
        self.extra_objects = objects

    @classmethod
    def monkey_patch_distutils(cls):
        """
        Monkey-patch distutils with our own build_ext class knowing
        about pycc-compiled extensions modules.
        """
        if cls._distutils_monkey_patched:
            return

        _orig_build_ext = build_ext.build_ext

        class _CC_build_ext(_orig_build_ext):

            def build_extension(self, ext):
                if isinstance(ext, _CCExtension):
                    ext._prepare_object_files(self)

                _orig_build_ext.build_extension(self, ext)

        build_ext.build_ext = _CC_build_ext

        cls._distutils_monkey_patched = True