WIP: Use python binary instead of libpython when it's a PIE

deps/build.jl

@@ -69,14 +69,39 @@ function show_dlopen_error(lib, e)
     end
 end
 
-# return libpython name, libpython pointer
+# return libpython name, libpython pointer, is_pie
 function find_libpython(python::AbstractString)
     dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL
 
+    python = something(Compat.Sys.which(python))
+
+    # If libpython is dynamically linked, use it:
+    lib, = try
+        exec_find_libpython(python, `--dynamic`)
+    catch
+        [nothing]
+    end
+    if lib !== nothing
+        try
+            return (Libdl.dlopen(lib, dlopen_flags), lib, false)
+        catch e
+            show_dlopen_error(lib, e)
+        end
+    end
+
+    # If `python` is a position independent executable, use it as a
+    # shared library:
+    try
+        return (Libdl.dlopen(python, dlopen_flags), python, true)
+    catch e
+        show_dlopen_error(python, e)
+    end
+
+    # Otherwise, look for common locations of libpython:
     libpaths = exec_find_libpython(python, `--list-all`)
     for lib in libpaths
         try
-            return (Libdl.dlopen(lib, dlopen_flags), lib)
+            return (Libdl.dlopen(lib, dlopen_flags), lib, false)
         catch e
             show_dlopen_error(lib, e)
         end
@@ -95,7 +120,7 @@ function find_libpython(python::AbstractString)
             # it easier for users to investigate Python setup
             # PyCall.jl trying to use.  It also helps PyJulia to
             # compare libpython.
-            return (libpython, Libdl.dlpath(libpython))
+            return (libpython, Libdl.dlpath(libpython), false)
         catch e
             show_dlopen_error(lib, e)
         end
@@ -191,7 +216,7 @@ try # make sure deps.jl file is removed on error
         Conda.add("numpy")
     end
 
-    (libpython, libpy_name) = find_libpython(python)
+    (libpython, libpy_name, is_pie) = find_libpython(python)
     programname = pysys(python, "executable")
 
     # Get PYTHONHOME, either from the environment or from Python
@@ -227,6 +252,7 @@ try # make sure deps.jl file is removed on error
     const pyversion_build = $(repr(pyversion))
     const PYTHONHOME = "$(escape_string(PYTHONHOME))"
     const wPYTHONHOME = $(wstringconst(PYTHONHOME))
+    const is_pie = $(repr(is_pie))
 
     "True if we are using the Python distribution in the Conda package."
     const conda = $use_conda

deps/find_libpython.py

@@ -112,7 +112,6 @@ def _linked_libpython_windows():
         return None
 
 
-
 def library_name(name, suffix=SHLIB_SUFFIX, is_windows=is_windows):
     """
     Convert a file basename `name` to a library name (no "lib" and ".so" etc.)
@@ -197,7 +196,6 @@ def candidate_names(suffix=SHLIB_SUFFIX):
         yield dlprefix + stem + suffix
 
 
-
 @uniquified
 def candidate_paths(suffix=SHLIB_SUFFIX):
     """
@@ -263,8 +261,13 @@ def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple):
 
     Parameters
     ----------
-    path : str ot None
+    path : str or None
         A candidate path to a shared library.
+
+    Returns
+    -------
+    path : str or None
+        Normalized existing path or `None`.
     """
     if not path:
         return None
@@ -357,6 +360,11 @@ def cli_find_libpython(cli_op, verbose):
         print_all(candidate_names())
     elif cli_op == "candidate-paths":
         print_all(p for p in candidate_paths() if p and os.path.isabs(p))
+    elif cli_op == "dynamic":
+        path = normalize_path(linked_libpython())
+        if path is None:
+            return 1
+        print(path, end="")
     else:
         path = find_libpython()
         if path is None:
@@ -385,6 +393,10 @@ def main(args=None):
         "--candidate-paths",
         action="store_const", dest="cli_op", const="candidate-paths",
         help="Print list of candidate paths of libpython.")
+    group.add_argument(
+        "--dynamic",
+        action="store_const", dest="cli_op", const="dynamic",
+        help="Print the path to dynamically linked libpython.")
 
     ns = parser.parse_args(args)
     parser.exit(cli_find_libpython(**vars(ns)))

src/pyinit.jl

@@ -78,6 +78,23 @@ function Py_Finalize()
     ccall(@pysym(:Py_Finalize), Cvoid, ())
 end
 
+macro _copy_global_to_pie(libpy_handle, symbols...)
+    exprs = [
+        quote
+            if hassym($(esc(libpy_handle)), $sym)
+                unsafe_store!(cglobal((@pysym $sym), Ptr{Cvoid}),
+                              unsafe_load(cglobal($sym, Ptr{Cvoid})))
+            end
+        end
+        for sym in symbols
+    ]
+    quote
+        if is_pie
+            $(exprs...)
+        end
+    end
+end
+
 function __init__()
     # sanity check: in Pkg for Julia 0.7+, the location of Conda can change
     # if e.g. you checkout Conda master, and we'll need to re-build PyCall
@@ -89,6 +106,7 @@ function __init__()
     # issue #189
     libpy_handle = libpython === nothing ? C_NULL :
         Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL)
+    @_copy_global_to_pie(libpy_handle, :stdin, :stdout, :stderr)
 
     already_inited = 0 != ccall((@pysym :Py_IsInitialized), Cint, ())
 

test/runtests.jl

@@ -7,7 +7,14 @@ filter(f, d::AbstractDict) = Base.filter(f, d)
 PYTHONPATH=get(ENV,"PYTHONPATH","")
 PYTHONHOME=get(ENV,"PYTHONHOME","")
 PYTHONEXECUTABLE=get(ENV,"PYTHONEXECUTABLE","")
-Compat.@info "Python version $pyversion from $(PyCall.libpython), PYTHONHOME=$(PyCall.PYTHONHOME)\nENV[PYTHONPATH]=$PYTHONPATH\nENV[PYTHONHOME]=$PYTHONHOME\nENV[PYTHONEXECUTABLE]=$PYTHONEXECUTABLE"
+Compat.@info """
+Python version $pyversion from $(PyCall.libpython),
+PYTHONHOME=$(PyCall.PYTHONHOME)
+is_pie=$(PyCall.is_pie)
+ENV[PYTHONPATH]=$PYTHONPATH
+ENV[PYTHONHOME]=$PYTHONHOME
+ENV[PYTHONEXECUTABLE]=$PYTHONEXECUTABLE
+"""
 
 roundtrip(T, x) = convert(T, PyObject(x))
 roundtrip(x) = roundtrip(PyAny, x)