Better handling of base executable not found #1515

Improve the base executable discovery mechanism:

- print at debug level why we refuse some candidates,
- when no candidates match exactly, instead of hard failing fallback to the closest match where the priority of matching attributes is python implementation, major version, minor version, architecture, patch version, release level and serial (this is to facilitate things to still work when the OS upgrade replace/upgrades the system python with a newer version than what the virtualenv host python was created with),
- always resolve system_executable information during the interpreter discovery, and the discovered environment is the system interpreter instead of the venv/virtualenv (this happened before lazily the first time we accessed and caused reporting that the created virtual environment is of type of the virtualenv host python version, instead of the system pythons version - these two can differ if the OS upgraded the system python underneath and the virtualenv host was created via copy),

.gitignore

@@ -22,7 +22,7 @@ dist
 /pip-wheel-metadata
 /src/virtualenv/version.py
 /src/virtualenv/out
-/*env*
+venv*
 .python-version
 
 *wheel-store*

docs/changelog/1515.bugfix.rst

@@ -0,0 +1,14 @@
+Improve base executable discovery mechanism:
+
+- print at debug level why we refuse some candidates,
+- when no candidates match exactly, instead of hard failing fallback to the closest match where the priority of
+  matching attributes is: python implementation, major version, minor version, architecture, patch version,
+  release level and serial (this is to facilitate things to still work when the OS upgrade replace/upgrades the system
+  python with a never version, than what the virtualenv host python was created with),
+- always resolve system_executable information during the interpreter discovery, and the discovered environment is the
+  system interpreter instead of the venv/virtualenv (this happened before lazily the first time we accessed, and caused
+  reporting that the created virtual environment is of type of the virtualenv host python version, instead of the
+  system pythons version - these two can differ if the OS upgraded the system python underneath and the virtualenv
+  host was created via copy),
+
+by ``gaborbernat``.

setup.cfg

@@ -53,7 +53,7 @@ zip_safe = True
 
 [options.entry_points]
 console_scripts =
-    virtualenv=virtualenv.__main__:run
+    virtualenv=virtualenv.__main__:run_with_catch
 virtualenv.activate =
     bash = virtualenv.activation.bash:BashActivator
     cshell = virtualenv.activation.cshell:CShellActivator

src/virtualenv/__main__.py

@@ -4,12 +4,12 @@
 import sys
 from datetime import datetime
 
-from virtualenv.error import ProcessCallFailed
-from virtualenv.run import run_via_cli
-
 
 def run(args=None):
     start = datetime.now()
+    from virtualenv.error import ProcessCallFailed
+    from virtualenv.run import run_via_cli
+
     if args is None:
         args = sys.argv[1:]
     try:
@@ -23,5 +23,13 @@ def run(args=None):
         logging.info("done in %.0fms", (datetime.now() - start).total_seconds() * 1000)
 
 
+def run_with_catch(args=None):
+    try:
+        run(args)
+    except (KeyboardInterrupt, Exception) as exception:
+        logging.error("%s: %s", type(exception).__name__, exception)
+        sys.exit(1)
+
+
 if __name__ == "__main__":
-    run()
+    run_with_catch()

src/virtualenv/create/via_global_ref/builtin/python2/site.py

@@ -92,7 +92,7 @@ def rewrite_standard_library_sys_path():
 
 def disable_user_site_package():
     """Flip the switch on enable user site package"""
-    # sys.flags is a c-extension type, so we cannot monkey patch it, replace it with a python class to flip it
+    # sys.flags is a c-extension type, so we cannot monkeypatch it, replace it with a python class to flip it
     sys.original_flags = sys.flags
 
     class Flags(object):

src/virtualenv/create/via_global_ref/venv.py

@@ -4,7 +4,7 @@
 from collections import namedtuple
 from copy import copy
 
-from virtualenv.discovery.py_info import CURRENT
+from virtualenv.discovery.py_info import PythonInfo
 from virtualenv.error import ProcessCallFailed
 from virtualenv.info import fs_supports_symlink
 from virtualenv.util.path import ensure_dir
@@ -19,7 +19,9 @@ class Venv(ViaGlobalRefApi):
     def __init__(self, options, interpreter):
         self.describe = options.describe
         super(Venv, self).__init__(options, interpreter)
-        self.can_be_inline = interpreter is CURRENT and interpreter.executable == interpreter.system_executable
+        self.can_be_inline = (
+            interpreter is PythonInfo.current() and interpreter.executable == interpreter.system_executable
+        )
         self._context = None
 
     def _args(self):

src/virtualenv/discovery/builtin.py

@@ -9,7 +9,7 @@
 from virtualenv.info import IS_WIN
 
 from .discover import Discover
-from .py_info import CURRENT, PythonInfo
+from .py_info import PythonInfo
 from .py_spec import PythonSpec
 
 
@@ -44,22 +44,24 @@ def get_interpreter(key):
     logging.info("find interpreter for spec %r", spec)
     proposed_paths = set()
     for interpreter, impl_must_match in propose_interpreters(spec):
-        if interpreter.executable not in proposed_paths:
-            logging.info("proposed %s", interpreter)
-            if interpreter.satisfies(spec, impl_must_match):
-                logging.debug("accepted target interpreter %s", interpreter)
-                return interpreter
-            proposed_paths.add(interpreter.executable)
+        key = interpreter.system_executable, impl_must_match
+        if key in proposed_paths:
+            continue
+        logging.info("proposed %s", interpreter)
+        if interpreter.satisfies(spec, impl_must_match):
+            logging.debug("accepted %s", interpreter)
+            return interpreter
+        proposed_paths.add(key)
 
 
 def propose_interpreters(spec):
-    # 1. we always try with the lowest hanging fruit first, the current interpreter
-    yield CURRENT, True
-
-    # 2. if it's an absolute path and exists, use that
+    # 1. if it's an absolute path and exists, use that
     if spec.is_abs and os.path.exists(spec.path):
         yield PythonInfo.from_exe(spec.path), True
 
+    # 2. try with the current
+    yield PythonInfo.current_system(), True
+
     # 3. otherwise fallback to platform default logic
     if IS_WIN:
         from .windows import propose_interpreters

src/virtualenv/discovery/cached_py_info.py

@@ -9,27 +9,28 @@
 import json
 import logging
 import pipes
+import sys
 from collections import OrderedDict
 from hashlib import sha256
 
 from six import ensure_text
 
 from virtualenv.dirs import default_data_dir
+from virtualenv.discovery.py_info import PythonInfo
 from virtualenv.info import PY2, PY3
 from virtualenv.util.path import Path
 from virtualenv.util.subprocess import Popen, subprocess
 from virtualenv.util.zipapp import ensure_file_on_disk
 from virtualenv.version import __version__
 
-from .py_info import PythonInfo
-
 _CACHE = OrderedDict()
+_CACHE[Path(sys.executable)] = PythonInfo()
 _FS_PATH = None
 
 
-def from_exe(exe, raise_on_error=True, ignore_cache=False):
+def from_exe(cls, exe, raise_on_error=True, ignore_cache=False):
     """"""
-    result = _get_from_cache(exe, ignore_cache=ignore_cache)
+    result = _get_from_cache(cls, exe, ignore_cache=ignore_cache)
     if isinstance(result, Exception):
         if raise_on_error:
             raise result
@@ -39,21 +40,22 @@ def from_exe(exe, raise_on_error=True, ignore_cache=False):
     return result
 
 
-def _get_from_cache(exe, ignore_cache=True):
-    # first, we ensure that we resolve symlinks, we reuse paths that have been resolved under different name
-    resolved_resolved_path = Path(exe).resolve()
-    if not ignore_cache and resolved_resolved_path in _CACHE:  # check in the in-memory cache
-        result = _CACHE[resolved_resolved_path]
+def _get_from_cache(cls, exe, ignore_cache=True):
+    # note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a
+    # pyenv.cfg somewhere alongside on python3.4+
+    exe_path = Path(exe)
+    if not ignore_cache and exe_path in _CACHE:  # check in the in-memory cache
+        result = _CACHE[exe_path]
     else:  # then check the persisted cache
-        py_info = _get_via_file_cache(resolved_resolved_path, exe)
-        result = _CACHE[resolved_resolved_path] = py_info
+        py_info = _get_via_file_cache(cls, exe_path, exe)
+        result = _CACHE[exe_path] = py_info
     # independent if it was from the file or in-memory cache fix the original executable location
     if isinstance(result, PythonInfo):
         result.executable = exe
     return result
 
 
-def _get_via_file_cache(resolved_path, exe):
+def _get_via_file_cache(cls, resolved_path, exe):
     key = sha256(str(resolved_path).encode("utf-8") if PY3 else str(resolved_path)).hexdigest()
     py_info = None
     resolved_path_text = ensure_text(str(resolved_path))
@@ -68,19 +70,19 @@ def _get_via_file_cache(resolved_path, exe):
                 data = json.loads(data_file_path.read_text())
                 if data["path"] == resolved_path_text and data["st_mtime"] == resolved_path_modified_timestamp:
                     logging.debug("get PythonInfo from %s for %s", data_file_path, exe)
-                    py_info = PythonInfo.from_dict({k: v for k, v in data["content"].items()})
+                    py_info = cls._from_dict({k: v for k, v in data["content"].items()})
                 else:
                     raise ValueError("force cleanup as stale")
             except (KeyError, ValueError, OSError):
                 logging.debug("remove PythonInfo %s for %s", data_file_path, exe)
                 data_file_path.unlink()  # cleanup out of date files
         if py_info is None:  # if not loaded run and save
-            failure, py_info = _run_subprocess(exe)
+            failure, py_info = _run_subprocess(cls, exe)
             if failure is None:
                 file_cache_content = {
                     "st_mtime": resolved_path_modified_timestamp,
                     "path": resolved_path_text,
-                    "content": py_info.to_dict(),
+                    "content": py_info._to_dict(),
                 }
                 logging.debug("write PythonInfo to %s for %s", data_file_path, exe)
                 data_file_path.write_text(ensure_text(json.dumps(file_cache_content, indent=2)))
@@ -96,7 +98,7 @@ def _get_fs_path():
     return _FS_PATH
 
 
-def _run_subprocess(exe):
+def _run_subprocess(cls, exe):
     resolved_path = Path(__file__).parent.absolute().absolute() / "py_info.py"
     with ensure_file_on_disk(resolved_path) as resolved_path:
         cmd = [exe, "-s", str(resolved_path)]
@@ -112,7 +114,7 @@ def _run_subprocess(exe):
             out, err, code = "", os_error.strerror, os_error.errno
     result, failure = None, None
     if code == 0:
-        result = PythonInfo.from_json(out)
+        result = cls._from_json(out)
         result.executable = exe  # keep original executable as this may contain initialization code
     else:
         msg = "failed to query {} with code {}{}{}".format(