[WIP] Use preferences.jl to set python executable

.github/workflows/aot.yml

@@ -16,10 +16,11 @@ jobs:
         os:
           - ubuntu-latest
         architecture: [x64]
-        python-version: ['3.x']
+        python-version: ['3.9']
         julia-version:
+          - '1.6'
           - '1'
-          - '~1.7.0-rc1'
+          - '~1.7'
           # - 'nightly'  # TODO: reenable
       fail-fast: false
     env:
@@ -46,7 +47,6 @@ jobs:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.architecture }}
           show-versioninfo: true
-
       - run: julia -e 'using Pkg; pkg"add PackageCompiler@v1"'
 
       - run: aot/compile.jl

.github/workflows/conda.yml

@@ -18,14 +18,14 @@ jobs:
           - macos-latest
           - windows-latest
         architecture: [x64]
-        julia-version: ['1']
+        julia-version: ['1.6', '1']
         include:
           - os: windows-latest
             architecture: x86
             julia-version: '1'
       fail-fast: false
     env:
-      PYTHON: ""
+      PYPREFERENCES_PYTHON: "conda"
     name: Test
       Julia ${{ matrix.julia-version }}
       Conda
@@ -41,7 +41,13 @@ jobs:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.architecture }}
           show-versioninfo: true
-      - uses: julia-actions/julia-buildpkg@v1
+      - name: Install PyPreferences
+        shell: julia --project=@. {0}
+        run: |
+          using Pkg;
+          pkg"dev ./PyPreferences.jl";
+          using PyPreferences;
+          PyPreferences.use_conda();
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1

.github/workflows/system.yml

@@ -20,7 +20,7 @@ jobs:
         architecture: [x64]
         python-version: ['3.x']
         julia-version:
-          - '1.0'
+          - '1.6'
           - '1'
           - 'nightly'
         include:
@@ -34,14 +34,6 @@ jobs:
             architecture: x64
             python-version: '3.9'
             julia-version: '1'
-          - os: ubuntu-latest
-            architecture: x64
-            python-version: '3.x'
-            julia-version: '1.5'
-          - os: ubuntu-latest
-            architecture: x64
-            python-version: '3.x'
-            julia-version: '1.4'
           # Test Python 2.7 only with a few combinations (TODO: drop 2.7).
           # Note that it does not work in macOS:
           - os: ubuntu-latest
@@ -80,14 +72,21 @@ jobs:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.architecture }}
           show-versioninfo: true
-      - uses: julia-actions/julia-buildpkg@v1
-        env:
-          PYTHON: python
-      - run: julia test/check_deps_version.jl ${{ matrix.python-version }}
+      - name: Set PYTHON Env variable
+        run: echo "PYTHON=python" >> $GITHUB_ENV
+        if: ${{ matrix.python-version == '2.7' }}
+      - name: Install PyPreferences
+        shell: julia --project=@. {0}
+        run: |
+          using Pkg;
+          pkg"dev ./PyPreferences.jl"
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:
           file: ./lcov.info
           flags: unittests
           name: codecov-umbrella
+      - name: tmate session if tests fail
+        if: failure()
+        uses: mxschmitt/action-tmate@v3

.gitignore

@@ -1,8 +1,13 @@
 Manifest.toml
+LocalPreferences.toml
 aot/Manifest.toml
 aot/Project.toml
 aot/_julia_path
 aot/sys.*
 deps/deps.jl
 deps/PYTHON
 deps/build.log
+
+PyPreferences.jl/Manifest.toml
+
+.envrc

Project.toml

@@ -1,25 +1,27 @@
 name = "PyCall"
 uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 authors = ["Steven G. Johnson <stevenj@mit.edu>", "Yichao Yu <yyc1992@gmail.com>", "Takafumi Arakaki <aka.tkf@gmail.com>", "Simon Kornblith <simon@simonster.com>", "Páll Haraldsson <Pall.Haraldsson@gmail.com>", "Jon Malmaud <malmaud@gmail.com>", "Jake Bolewski <jakebolewski@gmail.com>", "Keno Fischer <keno@alumni.harvard.edu>", "Joel Mason <jobba1@hotmail.com>", "Jameson Nash <vtjnash@gmail.com>", "The JuliaPy development team"]
-version = "1.93.0"
+version = "2.0.0"
 
 [deps]
 Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+PyPreferences = "cc9521c6-0242-4dda-8d66-c47a9d9eec02"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289"
 
 [compat]
-Conda = "1.0"
+Conda = "1"
 MacroTools = "0.4, 0.5"
-VersionParsing = "1.0"
-julia = "0.7, 1.0"
+VersionParsing = "1"
+julia = "1.6"
 
 [extras]
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test"]
+test = ["Test", "Preferences"]

PyPreferences.jl/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Takafumi Arakaki <aka.tkf@gmail.com> and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

PyPreferences.jl/Project.toml

@@ -0,0 +1,24 @@
+name = "PyPreferences"
+uuid = "cc9521c6-0242-4dda-8d66-c47a9d9eec02"
+authors = ["Takafumi Arakaki <aka.tkf@gmail.com> and contributors"]
+version = "0.1.0"
+
+[deps]
+Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
+VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289"
+
+[compat]
+Conda = "1"
+julia = "1.6"
+VersionParsing = "1"
+Preferences = "1"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]

PyPreferences.jl/README.md

@@ -0,0 +1,8 @@
+# PyPreferences
+
+```julia
+using PyPreferences
+
+PyPreferences.use_system("python_exe")
+PyPreferences.use_conda()
+```

PyPreferences.jl/src/PyPreferences.jl

@@ -0,0 +1,41 @@
+baremodule PyPreferences
+
+using Logging
+
+function use_system end
+function use_conda end
+# function use_jll end
+function use_inprocess end
+function recompile end
+
+# API to be used from PyCall
+function assert_configured end
+function instruction_message end
+
+# function diagnose end
+function status end
+
+module Implementations
+
+    module PythonUtils
+        include("python_utils.jl")
+    end
+    include("which.jl")
+    include("core.jl")
+    include("api.jl")
+end
+
+let prefs = Implementations.setup_non_failing()
+    global const python = prefs.python
+    global const inprocess = prefs.inprocess
+    global const conda = prefs.conda
+    global const python_fullpath = prefs.python_fullpath
+    global const libpython = prefs.libpython
+    global const python_version = prefs.python_version
+    global const PYTHONHOME = prefs.PYTHONHOME
+end
+
+const pyprogramname = python_fullpath
+const pyversion_build = python_version
+
+end  # baremodule PyPreferences

PyPreferences.jl/src/api.jl

@@ -0,0 +1,62 @@
+using Conda
+
+function PyPreferences.status()
+    # TODO: compare with in-process values
+    code = """
+    $(load_pypreferences_code())
+    PyPreferences.Implementations.status_inprocess()
+    """
+    cmd = include_stdin_cmd()
+    open(pipeline(cmd; stdout = stdout, stderr = stderr); write = true) do io
+        write(io, code)
+    end
+    return
+end
+
+function PyPreferences.use_system(python::AbstractString = "python3")
+    """
+    Use python from a provided executable or path (defaults to `python3`).
+    """
+    return Implementations.set(python = python)
+end
+
+function PyPreferences.use_conda()
+    """
+    Use Python provided by Conda.jl
+    """
+    Conda.add("numpy")
+    return Implementations.set(conda = true)
+end
+
+#=
+function use_jll()
+end
+=#
+
+function PyPreferences.use_inprocess()
+    return Implementations.set(inprocess = true)
+end
+
+function PyPreferences.instruction_message()
+    return """
+    PyPreferences.jl is not configured properly. Run:
+        using Pkg
+        Pkg.add("PyPreferences")
+        using PyPreferences
+        @doc PyPreferences
+    for usage.
+    """
+end
+
+
+function PyPreferences.assert_configured()
+    if (
+        PyPreferences.python === nothing ||
+        PyPreferences.python_fullpath === nothing ||
+        PyPreferences.libpython === nothing ||
+        PyPreferences.python_version === nothing ||
+        PyPreferences.PYTHONHOME === nothing
+    )
+        error(PyPreferences.instruction_message())
+    end
+end