setup.py

# BEGIN_COPYRIGHT
#
# Copyright 2009-2024 CRS4.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# END_COPYRIGHT

"""\
Pydoop is a Python MapReduce and HDFS API for Hadoop.

Pydoop is built on top of two C/C++ extension modules: a libhdfs wrapper and a
(de)serialization library for types used by the Hadoop Pipes protocol. Since
libhdfs is, in turn, a JNI wrapper for the HDFS Java code, Pydoop needs a JDK
(a JRE is not enough) to build.

You can point Pydoop to the Java home directory by exporting the JAVA_HOME
environment variable. Make sure JAVA_HOME points to the JDK home directory
(e.g., ${JAVA_HOME}/include/jni.h should be a valid path). If JAVA_HOME is not
defined, Pydoop will try to get the JDK home from Java system properties.

To compile its Java components, Pydoop also needs to find the Hadoop
libraries. In order to do so, it will try to call ``hadoop classpath``, so
make sure that the ``hadoop`` executable is in the PATH.
"""
from __future__ import print_function

import sys
import time
import os
import glob
import shutil
import itertools
import tempfile

SETUPTOOLS_MIN_VER = '3.3'

import setuptools
from pkg_resources import parse_version  # included in setuptools
print('using setuptools version', setuptools.__version__)
if parse_version(setuptools.__version__) < parse_version(SETUPTOOLS_MIN_VER):
    raise RuntimeError(
        'setuptools minimum required version: %s' % SETUPTOOLS_MIN_VER
    )

# bug: http://bugs.python.org/issue1222585
# workaround: http://stackoverflow.com/questions/8106258
from distutils.sysconfig import get_config_var
_UNWANTED_OPTS = frozenset(['-Wstrict-prototypes'])
os.environ['OPT'] = ' '.join(
    _ for _ in get_config_var('OPT').strip().split() if _ not in _UNWANTED_OPTS
)

from setuptools import setup, find_packages, Extension
from setuptools.command.build_ext import build_ext
from distutils.command.build import build
from distutils.errors import DistutilsSetupError, CompileError
from distutils import log

import pydoop
import pydoop.utils.jvm as jvm

VERSION_FN = "VERSION"
EXTRA_COMPILE_ARGS = ["-Wno-write-strings"]  # http://bugs.python.org/issue6952

# properties file.  Since the source is in the root dir, filename = basename
PROP_FN = PROP_BN = pydoop.__propfile_basename__

CONSOLE_SCRIPTS = ['pydoop = pydoop.app.main:main']
if sys.version_info[0] == 3:
    CONSOLE_SCRIPTS.append('pydoop3 = pydoop.app.main:main')
else:
    CONSOLE_SCRIPTS.append('pydoop2 = pydoop.app.main:main')


# ---------
# UTILITIES
# ---------

def rm_rf(path, dry_run=False):
    """
    Remove a file or directory tree.

    Won't throw an exception, even if the removal fails.
    """
    log.info("removing %s" % path)
    if dry_run:
        return
    try:
        if os.path.isdir(path) and not os.path.islink(path):
            shutil.rmtree(path)
        else:
            os.remove(path)
    except OSError:
        pass


def mtime(fn):
    return os.stat(fn).st_mtime


def must_generate(target, prerequisites):
    try:
        return max(mtime(p) for p in prerequisites) > mtime(target)
    except OSError:
        return True


def get_version_string():
    try:
        with open(VERSION_FN) as f:
            return f.read().strip()
    except IOError:
        raise DistutilsSetupError("failed to read version info")


def write_config(filename="pydoop/config.py"):
    prereq = PROP_FN
    if must_generate(filename, [prereq]):
        props = pydoop.read_properties(PROP_FN)
        with open(filename, "w") as fo:
            fo.write("# GENERATED BY setup.py\n")
            for k in sorted(props):
                fo.write("%s = %r\n" % (k, props[k]))


def write_version(filename="pydoop/version.py"):
    if must_generate(filename, [VERSION_FN]):
        with open(filename, "w") as f:
            f.write("# GENERATED BY setup.py\n")
            f.write("version = %r\n" % (get_version_string(),))


EXTENSION_MODULES = [
    Extension(
        'pydoop.native_core_hdfs',
        include_dirs=[
            'src/libhdfs',
            'src/libhdfs/include',
            'src/libhdfs/os/posix',
        ],
        sources=list(itertools.chain(
            glob.iglob('src/libhdfs/*.c'),
            glob.iglob('src/libhdfs/common/*.c'),
            glob.iglob('src/libhdfs/os/posix/*.c'),
            glob.iglob('src/native_core_hdfs/*.cc')
        )),
        extra_compile_args=EXTRA_COMPILE_ARGS,
        # to be finalized at build time
    ),
    Extension(
        'pydoop.sercore',
        sources=[
            "src/sercore/hu_extras.cpp",
            "src/sercore/sercore.cpp",
            "src/sercore/streams.cpp",
            "src/sercore/HadoopUtils/SerialUtils.cc",
        ],
        include_dirs=["src/sercore/HadoopUtils"],
        extra_compile_args=EXTRA_COMPILE_ARGS + ["-std=c++11", "-O3"],
    )
]


# ------------
# BUILD ENGINE
# ------------

class JavaLib(object):

    def __init__(self):
        self.jar_name = pydoop.jar_name()
        self.classpath = pydoop.hadoop_classpath()
        self.java_files = glob.glob(
            "src/it/crs4/pydoop/mapreduce/pipes/*.java"
        ) + ["src/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"]
        self.dependencies = glob.glob('lib/*.jar')
        self.properties = [(
            os.path.join("it/crs4/pydoop/mapreduce/pipes", PROP_BN),
            PROP_FN
        )]


class JavaBuilder(object):

    def __init__(self, build_temp, build_lib):
        self.build_temp = build_temp
        self.build_lib = build_lib
        self.java_libs = [JavaLib()]

    def run(self):
        for jlib in self.java_libs:
            self.__build_java_lib(jlib)

    def __build_java_lib(self, jlib):
        package_path = os.path.join(self.build_lib, "pydoop")
        compile_cmd = "javac"
        if jlib.classpath:
            classpath = [jlib.classpath]
            for src in jlib.dependencies:
                dest = os.path.join(package_path, os.path.basename(src))
                shutil.copyfile(src, dest)
                classpath.append(dest)
            compile_cmd += " -classpath %s" % (':'.join(classpath))
        else:
            log.warn(
                "WARNING: could not set classpath, java code may not compile"
            )
        class_dir = os.path.join(
            self.build_temp, "pipes"
        )
        jar_path = os.path.join(package_path, jlib.jar_name)
        if not os.path.exists(class_dir):
            os.mkdir(class_dir)
        compile_cmd += " -d '%s'" % class_dir
        log.info("Compiling Java classes")
        for f in jlib.java_files:
            compile_cmd += " %s" % f
        ret = os.system(compile_cmd)
        if ret:
            raise DistutilsSetupError(
                "Error compiling java component.  Command: %s" % compile_cmd
            )
        log.info("Copying properties file")
        for p in jlib.properties:
            prop_file_dest = os.path.join(class_dir, p[0])
            shutil.copyfile(p[1], prop_file_dest)
        log.info("Making Jar: %s", jar_path)
        package_cmd = "jar -cf %(jar_path)s -C %(class_dir)s ./it" % {
            'jar_path': jar_path, 'class_dir': class_dir
        }
        log.info("Packaging Java classes")
        log.info("Command: %s", package_cmd)
        ret = os.system(package_cmd)
        if ret:
            raise DistutilsSetupError(
                "Error packaging java component.  Command: %s" % package_cmd
            )


class BuildPydoopExt(build_ext):

    def __have_better_tls(self):
        log.info("checking for TLS support")
        test_code = "int main(void) { static __thread int i = 0; return i; }"
        wd = tempfile.mkdtemp(prefix="pydoop_")
        test_src = os.path.join(wd, "temp.c")
        with open(test_src, "w") as f:
            f.write(test_code)
        try:
            self.compiler.compile([test_src], output_dir=wd)
        except CompileError:
            ret = False
        else:
            ret = True
        shutil.rmtree(wd)
        return ret

    def __finalize_hdfs(self, ext):
        """\
        Adds a few bits that depend on the specific environment.

        Delaying this until the build_ext phase allows non-build commands
        (e.g., sdist) to be run without java.
        """
        java_home = jvm.get_java_home()
        jvm_lib_path, _ = jvm.get_jvm_lib_path_and_name(java_home)
        ext.include_dirs = jvm.get_include_dirs() + ext.include_dirs
        ext.libraries = jvm.get_libraries()
        ext.library_dirs = [os.path.join(java_home, "Libraries"), jvm_lib_path]
        ext.define_macros = jvm.get_macros()
        ext.extra_link_args = ['-Wl,-rpath,%s' % jvm_lib_path]
        if self.__have_better_tls():
            ext.define_macros.append(("HAVE_BETTER_TLS", None))
        try:
            # too many warnings in libhdfs
            self.compiler.compiler_so.remove("-Wsign-compare")
        except (AttributeError, ValueError):
            pass

    # called for each extension, after compiler has been set up
    def build_extension(self, ext):
        if ext.name == "pydoop.native_core_hdfs":
            self.__finalize_hdfs(ext)
        build_ext.build_extension(self, ext)


class BuildPydoop(build):

    def build_java(self):
        jb = JavaBuilder(self.build_temp, self.build_lib)
        jb.run()

    def create_tmp(self):
        if not os.path.exists(self.build_temp):
            os.mkdir(self.build_temp)
        if not os.path.exists(self.build_lib):
            os.mkdir(self.build_lib)

    def clean_up(self):
        shutil.rmtree(self.build_temp)

    def run(self):
        write_version()
        write_config()
        shutil.copyfile(PROP_FN, os.path.join("pydoop", PROP_BN))
        build.run(self)
        try:
            self.create_tmp()
            self.build_java()
        finally:
            # On NFS, if we clean up right away we have issues with
            # NFS handles being still in the directory trees to be
            # deleted.  So, we sleep a bit and then delete
            time.sleep(0.5)
            self.clean_up()
        log.info("Build finished")


setup(
    name="pydoop",
    version=get_version_string(),
    description=pydoop.__doc__.strip().splitlines()[0],
    long_description=pydoop.__doc__.lstrip(),
    author=pydoop.__author__,
    author_email=pydoop.__author_email__,
    url=pydoop.__url__,
    download_url="https://pypi.python.org/pypi/pydoop",
    install_requires=['setuptools>=%s' % SETUPTOOLS_MIN_VER],
    extras_require={
        'avro': [
            'avro>=1.7.4;python_version<"3"',
            'avro-python3>=1.7.4;python_version>="3"',
        ],
    },
    packages=find_packages(exclude=['test', 'test.*']),
    package_data={"pydoop": [PROP_FN]},
    cmdclass={
        "build": BuildPydoop,
        "build_ext": BuildPydoopExt,
    },
    entry_points={'console_scripts': CONSOLE_SCRIPTS},
    platforms=["Linux"],
    ext_modules=EXTENSION_MODULES,
    license="Apache-2.0",
    keywords=["hadoop", "mapreduce"],
    classifiers=[
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3.5",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: POSIX :: Linux",
        "Topic :: Software Development :: Libraries :: Application Frameworks",
        "Intended Audience :: Developers",
    ],
    data_files=[
        ('config', ['README.md']),
    ],
    zip_safe=False,
)