docker: Add ability to build shared library flavours of CPython.

Enable via:

    PY_SHARED=1 \
    PLATFORM=$(uname -m) POLICY=manylinux2014 COMMIT_SHA=latest \
    ./build.sh

If set, builds both static & shared versions of CPython into the images.
Shared versions end up in (eg) /opt/python/cp37-cp37m-shared/ alongside the
existing static /opt/python/cp37-cp37m/, with a Python binary as
/usr/local/bin/python3.7-shared.

README.rst

@@ -172,18 +172,36 @@ Building Docker images
 ----------------------
 
 To build the Docker images, please run the following command from the
-current (root) directory:
+current (root) directory::
 
     $ PLATFORM=$(uname -m) POLICY=manylinux2014 COMMIT_SHA=latest ./build.sh
 
 Please note that the Docker build is using `buildx <https://github.com/docker/buildx>`_.
 
+Shared Library CPython
+~~~~~~~~~~~~~~~~~~~~~~
+
+You can build the Docker images with shared library CPython builds alongside the
+static CPython builds. This might be useful for special packaging requirements,
+but ``libpythonX.Y`` is `not a library that a manylinux extension is allowed to
+link to <https://www.python.org/dev/peps/pep-0513/#libpythonx-y-so-1>`_, so it is
+not part of the default manylinux images.
+
+To build the Docker images with shared library CPython builds::
+
+    $ PY_SHARED=1 PLATFORM=$(uname -m) POLICY=manylinux2014 COMMIT_SHA=latest ./build.sh
+
+The shared CPython interpreters are installed in
+``/opt/python/<python tag>-<abi tag>-shared``. The directories are named after
+the PEP 425 tags for each environment -- e.g. ``/opt/python/cp37-cp37m-shared``
+contains a shared library CPython 3.7 build.
+
 Updating the requirements
 -------------------------
 
 The requirement files are pinned and controlled by pip-tools compile. To update
 the pins, run nox on a Linux system with all supported versions of Python included.
-For example, using a docker image:
+For example, using a docker image::
 
     $ docker run --rm -v $PWD:/nox -t quay.io/pypa/manylinux2010_x86_64:latest pipx run nox -f /nox/noxfile.py -s compile tools
 

build.sh

@@ -66,14 +66,20 @@ else
 	echo "Unsupported policy: '${POLICY}'"
 	exit 1
 fi
+
+# default is not to include shared interpreter builds
+: "${PY_SHARED:=0}"
+
 export BASEIMAGE
 export DEVTOOLSET_ROOTPATH
 export PREPEND_PATH
 export LD_LIBRARY_PATH_ARG
+export PY_SHARED
 
 BUILD_ARGS_COMMON="
 	--build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE
 	--build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG
+	--build-arg PY_SHARED
 	--rm -t quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA}
 	-f docker/Dockerfile docker/
 "
@@ -94,7 +100,7 @@ elif [ "${MANYLINUX_BUILD_FRONTEND}" == "buildkit" ]; then
 		--import-cache type=local,src=$(pwd)/.buildx-cache-${POLICY}_${PLATFORM} \
 		--export-cache type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \
 		--opt build-arg:POLICY=${POLICY} --opt build-arg:PLATFORM=${PLATFORM} --opt build-arg:BASEIMAGE=${BASEIMAGE} \
-		--opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" \
+		--opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" --opt "build-arg:PY_SHARED=${PY_SHARED}"\
 		--output type=docker,name=quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} | docker load
 else
 	echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'"

docker/Dockerfile

@@ -5,13 +5,15 @@ ARG PLATFORM=x86_64
 ARG DEVTOOLSET_ROOTPATH=
 ARG LD_LIBRARY_PATH_ARG=
 ARG PREPEND_PATH=
+ARG PY_SHARED=0
 
 FROM $BASEIMAGE AS runtime_base
 ARG POLICY
 ARG PLATFORM
 ARG DEVTOOLSET_ROOTPATH
 ARG LD_LIBRARY_PATH_ARG
 ARG PREPEND_PATH
+ARG PY_SHARED
 LABEL maintainer="The ManyLinux project"
 
 ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}

docker/build_scripts/build-cpython.sh

@@ -30,36 +30,59 @@ fetch_source Python-${CPYTHON_VERSION}.tgz.asc ${CPYTHON_DOWNLOAD_URL}/${CPYTHON
 gpg --import ${MY_DIR}/cpython-pubkeys.txt
 gpg --verify Python-${CPYTHON_VERSION}.tgz.asc
 tar -xzf Python-${CPYTHON_VERSION}.tgz
-pushd Python-${CPYTHON_VERSION}
-PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}"
-mkdir -p ${PREFIX}/lib
-if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
-	# The _ctypes stdlib module build started to fail with 3.10.0rc1
-	# No clue what changed exactly yet
-	# This workaround fixes the build
-	LIBFFI_INCLUDEDIR=$(pkg-config --cflags-only-I libffi  | tr -d '[:space:]')
-	LIBFFI_INCLUDEDIR=${LIBFFI_INCLUDEDIR:2}
-	cp ${LIBFFI_INCLUDEDIR}/ffi.h ${LIBFFI_INCLUDEDIR}/ffitarget.h /usr/include/
-fi
-# configure with hardening options only for the interpreter & stdlib C extensions
-# do not change the default for user built extension (yet?)
-./configure \
-	CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \
-	LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
-	--prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
-make > /dev/null
-make install > /dev/null
-if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
-	rm -f /usr/include/ffi.h /usr/include/ffitarget.h
-fi
-popd
-rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc
 
-# we don't need libpython*.a, and they're many megabytes
-find ${PREFIX} -name '*.a' -print0 | xargs -0 rm -f
+function build {
+	IS_SHARED=$1
+	pushd Python-${CPYTHON_VERSION}
+	PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}"
+	if [ ${IS_SHARED} -eq 1 ]; then
+		PREFIX="${PREFIX}-shared"
+	fi
+	mkdir -p ${PREFIX}/lib
+	if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
+		# The _ctypes stdlib module build started to fail with 3.10.0rc1
+		# No clue what changed exactly yet
+		# This workaround fixes the build
+		LIBFFI_INCLUDEDIR=$(pkg-config --cflags-only-I libffi  | tr -d '[:space:]')
+		LIBFFI_INCLUDEDIR=${LIBFFI_INCLUDEDIR:2}
+		cp ${LIBFFI_INCLUDEDIR}/ffi.h ${LIBFFI_INCLUDEDIR}/ffitarget.h /usr/include/
+	fi
+	# configure with hardening options only for the interpreter & stdlib C extensions
+	# do not change the default for user built extension (yet?)
+	if [ ${IS_SHARED} -eq 1 ]; then
+		FLAVOR="--enable-shared"
+		FLAVOR_LDFLAGS="-Wl,-rpath=${PREFIX}/lib"
+	else
+		FLAVOR="--disable-shared"
+		FLAVOR_LDFLAGS=
+	fi
+
+	./configure \
+		CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \
+		LDFLAGS_NODIST="${MANYLINUX_LDFLAGS} ${FLAVOR_LDFLAGS}" \
+		--prefix=${PREFIX} ${FLAVOR} --with-ensurepip=no > /dev/null
+	make > /dev/null
+	make install > /dev/null
+	if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
+		rm -f /usr/include/ffi.h /usr/include/ffitarget.h
+	fi
+	popd
+
+	if [ ${IS_SHARED} -eq 0 ]; then
+		# we don't need libpython*.a, and they're many megabytes
+		find ${PREFIX} -name '*.a' -print0 | xargs -0 rm -f
+	fi
 
-# We do not need precompiled .pyc and .pyo files.
-clean_pyc ${PREFIX}
+	# We do not need precompiled .pyc and .pyo files.
+	clean_pyc ${PREFIX}
 
-# Strip ELF files found in ${PREFIX}
-strip_ ${PREFIX}
+	# Strip ELF files found in ${PREFIX}
+	strip_ ${PREFIX}
+}
+
+build 0
+if [ ${PY_SHARED-0} -eq 1 ]; then
+	build 1
+fi
+
+rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc

docker/build_scripts/finalize.sh

@@ -11,6 +11,12 @@ source $MY_DIR/build_utils.sh
 
 mkdir /opt/python
 for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do
+	if [[ "${PREFIX}" =~ -shared$ ]]; then
+		SUFFIX=-shared
+	else
+		SUFFIX=
+	fi
+
 	# Some python's install as bin/python3. Make them available as
 	# bin/python.
 	if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
@@ -24,14 +30,16 @@ for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*'
 	# Since we fall back on a canned copy of pip, we might not have
 	# the latest pip and friends. Upgrade them to make sure.
 	${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
+
 	# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
+
 	ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
-	ln -s ${PREFIX} /opt/python/${ABI_TAG}
+	ln -s ${PREFIX} /opt/python/${ABI_TAG}${SUFFIX:-}
 	# Make versioned python commands available directly in environment.
 	if [[ "${PREFIX}" == *"/pypy"* ]]; then
-		ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}
+		ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}${SUFFIX}
 	else
-		ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
+		ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}${SUFFIX}
 	fi
 done