Merge pull request #35 from ActiveState/BE-3440-Apply-2.7.18.7-patches

BE-3440-apply 2.7.18.7 patches

Lib/distutils/msvc9compiler.py

@@ -169,6 +169,66 @@ def sub(self, s):
             s = s.replace(k, v)
         return s
 
+def _find_vc2015():
+    try:
+        key = _winreg.OpenKeyEx(
+            _winreg.HKEY_LOCAL_MACHINE,
+            r"Software\Microsoft\VisualStudio\SxS\VC7",
+            access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY
+        )
+    except OSError:
+        log.debug("Visual C++ is not registered")
+        return None, None
+
+    best_version = 0
+    best_dir = None
+    with key:
+        for i in count():
+            try:
+                v, vc_dir, vt = _winreg.EnumValue(key, i)
+            except OSError:
+                break
+            if v and vt == _winreg.REG_SZ and os.path.isdir(vc_dir):
+                try:
+                    version = int(float(v))
+                except (ValueError, TypeError):
+                    continue
+                if version >= 14 and version > best_version:
+                    best_version, best_dir = version, vc_dir
+    return best_version, best_dir
+
+def _find_vc2017():
+    """Returns "15, path" based on the result of invoking vswhere.exe
+    If no install is found, returns "None, None"
+
+    The version is returned to avoid unnecessarily changing the function
+    result. It may be ignored when the path is not None.
+
+    If vswhere.exe is not available, by definition, VS 2017 is not
+    installed.
+    """
+    root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles")
+    if not root:
+        return None, None
+
+    try:
+        path = subprocess.check_output([
+            os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"),
+            "-latest",
+            "-prerelease",
+            "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+            "-property", "installationPath",
+            "-products", "*",
+        ]).strip()
+    except (subprocess.CalledProcessError, OSError, UnicodeDecodeError):
+        return None, None
+
+    path = os.path.join(path, "VC", "Auxiliary", "Build")
+    if os.path.isdir(path):
+        return 15, path
+
+    return None, None
+
 def get_build_version():
     """Return the version of MSVC that was used to build Python.
 
@@ -216,89 +276,83 @@ def removeDuplicates(variable):
     newVariable = os.pathsep.join(newList)
     return newVariable
 
-def find_vcvarsall(version):
-    """Find the vcvarsall.bat file
 
-    At first it tries to find the productdir of VS 2008 in the registry. If
-    that fails it falls back to the VS90COMNTOOLS env var.
-    """
-    vsbase = VS_BASE % version
-    try:
-        productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
-                                   "productdir")
-    except KeyError:
-        productdir = None
-
-    # trying Express edition
-    if productdir is None:
-        vsbase = VSEXPRESS_BASE % version
-        try:
-            productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
-                                       "productdir")
-        except KeyError:
-            productdir = None
-            log.debug("Unable to find productdir in registry")
-
-    if not productdir or not os.path.isdir(productdir):
-        toolskey = "VS%0.f0COMNTOOLS" % version
-        toolsdir = os.environ.get(toolskey, None)
-
-        if toolsdir and os.path.isdir(toolsdir):
-            productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")
-            productdir = os.path.abspath(productdir)
-            if not os.path.isdir(productdir):
-                log.debug("%s is not a valid directory" % productdir)
-                return None
-        else:
-            log.debug("Env var %s is not set or invalid" % toolskey)
-    if not productdir:
-        log.debug("No productdir found")
-        return None
-    vcvarsall = os.path.join(productdir, "vcvarsall.bat")
-    if os.path.isfile(vcvarsall):
-        return vcvarsall
-    log.debug("Unable to find vcvarsall.bat")
-    return None
+def _find_vcvarsall(plat_spec):
+    # bpo-38597: Removed vcruntime return value
+    _, best_dir = _find_vc2017()
 
-def query_vcvarsall(version, arch="x86"):
-    """Launch vcvarsall.bat and read the settings from its environment
-    """
-    vcvarsall = find_vcvarsall(version)
-    interesting = set(("include", "lib", "libpath", "path"))
-    result = {}
+    if not best_dir:
+        best_version, best_dir = _find_vc2015()
+
+    if not best_dir:
+        log.debug("No suitable Visual C++ version found")
+        return None, None
+
+    vcvarsall = os.path.join(best_dir, "vcvarsall.bat")
+    if not os.path.isfile(vcvarsall):
+        log.debug("%s cannot be found", vcvarsall)
+        return None, None
 
-    if vcvarsall is None:
+    return vcvarsall, None
+
+
+def _get_vc_env(plat_spec):
+    if os.getenv("DISTUTILS_USE_SDK"):
+        return {
+            key.lower(): value
+            for key, value in os.environ.items()
+        }
+
+    vcvarsall, _ = _find_vcvarsall(plat_spec)
+    if not vcvarsall:
         raise DistutilsPlatformError("Unable to find vcvarsall.bat")
-    log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version)
-    popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch),
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-    try:
-        stdout, stderr = popen.communicate()
-        if popen.wait() != 0:
-            raise DistutilsPlatformError(stderr.decode("mbcs"))
-
-        stdout = stdout.decode("mbcs")
-        for line in stdout.split("\n"):
-            line = Reg.convert_mbcs(line)
-            if '=' not in line:
-                continue
-            line = line.strip()
-            key, value = line.split('=', 1)
-            key = key.lower()
-            if key in interesting:
-                if value.endswith(os.pathsep):
-                    value = value[:-1]
-                result[key] = removeDuplicates(value)
 
-    finally:
-        popen.stdout.close()
-        popen.stderr.close()
+    try:
+        out = subprocess.check_output(
+            'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec),
+            stderr=subprocess.STDOUT,
+        ).decode('utf-16le', errors='replace')
+    except subprocess.CalledProcessError as exc:
+        log.error(exc.output)
+        raise DistutilsPlatformError("Error executing {}"
+                .format(exc.cmd))
+
+    env = {
+        key.lower(): value
+        for key, _, value in
+        (line.partition('=') for line in out.splitlines())
+        if key and value
+    }
+
+    return env
+
+def _find_exe(exe, paths=None):
+    """Return path to an MSVC executable program.
+
+    Tries to find the program in several places: first, one of the
+    MSVC program search paths from the registry; next, the directories
+    in the PATH environment variable.  If any of those work, return an
+    absolute path that is known to exist.  If none of them work, just
+    return the original program name, 'exe'.
+    """
+    if not paths:
+        paths = os.getenv('path').split(os.pathsep)
+    for p in paths:
+        fn = os.path.join(os.path.abspath(p), exe)
+        if os.path.isfile(fn):
+            return fn
+    return exe
 
-    if len(result) != len(interesting):
-        raise ValueError(str(list(result.keys())))
+# A map keyed by get_platform() return values to values accepted by
+# 'vcvarsall.bat'. Always cross-compile from x86 to work with the
+# lighter-weight MSVC installs that do not include native 64-bit tools.
+PLAT_TO_VCVARS = {
+    'win32' : 'x86',
+    'win-amd64' : 'x86_amd64',
+    'win-arm32' : 'x86_arm',
+    'win-arm64' : 'x86_arm64'
+}
 
-    return result
 
 # More globals
 VERSION = get_build_version()
@@ -352,7 +406,7 @@ def initialize(self, plat_name=None):
         assert not self.initialized, "don't init multiple times"
         if plat_name is None:
             plat_name = get_platform()
-        # sanity check for platforms to prevent obscure errors later.
+        # sanity check for platforms to prevent obscure errors later.query_vcvarsall
         ok_plats = 'win32', 'win-amd64', 'win-ia64'
         if plat_name not in ok_plats:
             raise DistutilsPlatformError("--plat-name must be one of %s" %
@@ -380,7 +434,7 @@ def initialize(self, plat_name=None):
                 plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \
                             PLAT_TO_VCVARS[plat_name]
 
-            vc_env = query_vcvarsall(VERSION, plat_spec)
+            vc_env = _get_vc_env(plat_spec)
 
             # take care to only use strings in the environment.
             self.__paths = vc_env['path'].encode('mbcs').split(os.pathsep)

Lib/platform.py

@@ -1511,7 +1511,7 @@ def python_version_tuple():
         will always include the patchlevel (it defaults to 0).
 
     """
-    return tuple(string.split(_sys_version()[1], '.'))
+    return tuple(string.split(_sys_version()[1], '.', 2))
 
 def python_branch():
 

PCbuild/openssl.props

@@ -6,19 +6,27 @@
     </ClCompile>
     <Link>
       <AdditionalLibraryDirectories>$(opensslLibDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>ws2_32.lib;libcrypto.lib;libssl.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>ws2_32.lib;$(opensslLibDir)\libcrypto.lib;$(opensslLibDir)\libssl.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <PropertyGroup>
-    <_DLLSuffix>-1_1</_DLLSuffix>
+    <SSLMajorVersion>$(SSL_MAJOR_VERSION)</SSLMajorVersion>
+    <SSLMajorVersion Condition="$(SSLMajorVersion) == ''">3</SSLMajorVersion>
+    <SSLMinorVersion>$(SSL_MINOR_VERSION)</SSLMinorVersion>
+    <SSLMinorVersion Condition="$(SSLMinorVersion) == ''">0</SSLMinorVersion>
+    <SSLVersion>$(SSLMajorVersion)_$(SSLMinorVersion)</SSLVersion>
+    <SSLVersion Condition="$(SSLMinorVersion) == '0'">$(SSLMajorVersion)</SSLVersion>
+    <SSLVersion Condition="$(SSLMajorVersion) == '3'">$(SSLMajorVersion)</SSLVersion>
+    <_DLLSuffix>-$(SSLVersion)</_DLLSuffix>
+    <_DLLSuffix Condition="$(Platform) == 'x64'">$(_DLLSuffix)-x64</_DLLSuffix>
     <_DLLSuffix Condition="$(Platform) == 'ARM'">$(_DLLSuffix)-arm</_DLLSuffix>
     <_DLLSuffix Condition="$(Platform) == 'ARM64'">$(_DLLSuffix)-arm64</_DLLSuffix>
   </PropertyGroup>
   <ItemGroup>
-    <_SSLDLL Include="$(opensslLibDir)\libcrypto$(_DLLSuffix).dll" />
-    <_SSLDLL Include="$(opensslLibDir)\libcrypto$(_DLLSuffix).pdb" />
-    <_SSLDLL Include="$(opensslLibDir)\libssl$(_DLLSuffix).dll" />
-    <_SSLDLL Include="$(opensslLibDir)\libssl$(_DLLSuffix).pdb" />
+    <_SSLDLL Include="$(opensslBinDir)\libcrypto$(_DLLSuffix).dll" />
+    <_SSLDLL Include="$(opensslBinDir)\libcrypto$(_DLLSuffix).pdb" />
+    <_SSLDLL Include="$(opensslBinDir)\libssl$(_DLLSuffix).dll" />
+    <_SSLDLL Include="$(opensslBinDir)\libssl$(_DLLSuffix).pdb" />
   </ItemGroup>
   <Target Name="_CopySSLDLL" Inputs="@(_SSLDLL)" Outputs="@(_SSLDLL->'$(OutDir)%(Filename)%(Extension)')" AfterTargets="Build">
     <Copy SourceFiles="@(_SSLDLL)" DestinationFolder="$(OutDir)" />

PCbuild/python.props

@@ -31,27 +31,36 @@
     <BuildPath Condition="!HasTrailingSlash($(BuildPath))">$(BuildPath)\</BuildPath>
 
     <!-- Directories of external projects. tcltk is handled in tcltk.props -->
-    <ExternalsDir>$([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`))</ExternalsDir>
     <sqlite3Dir>$(SQLITE3_DIR)</sqlite3Dir>
-    <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.28.0.0\</sqlite3Dir>
+    <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.43.0.0\</sqlite3Dir>
     <sqlite3Dir Condition="!HasTrailingSlash($(sqlite3Dir))">$(sqlite3Dir)\</sqlite3Dir>
     <bz2Dir>$(BZIP2_DIR)</bz2Dir>
-    <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
+    <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir>
     <bz2Dir Condition="!HasTrailingSlash($(bz2Dir))">$(bz2Dir)\</bz2Dir>
-    <bsddbDir>$(ExternalsDir)bsddb-4.7.25.0</bsddbDir>
+    <lzmaDir>$(LZMA_DIR)</lzmaDir>
+    <lzmaDir Condition="!HasTrailingSlash($(lzmaDir))">$(lzmaDir)\</lzmaDir>
+    <libffiDir>$(FFI_DIR)</libffiDir>
+    <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi\$(ArchName)\</libffiDir>
+    <libffiDir Condition="!HasTrailingSlash($(libffiDir))">$(libffiDir)\</libffiDir>
+    <libffiIncludeDir>$(libffiDir)include</libffiIncludeDir>
+    <libffiOutDir>$(libffiDir)</libffiOutDir>
     <opensslDir>$(OPENSSL_DIR)</opensslDir>
     <opensslIncludeDir/>
     <opensslLibdir/>
-    <opensslOutDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-bin-1.0.2t\$(ArchName)\</opensslOutDir>
+    <opensslOutDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-bin-1.1.1u\$(ArchName)\</opensslOutDir>
     <opensslLibDir Condition="$(opensslDir) == ''">$(opensslOutDir)</opensslLibDir>
+    <opensslBinDir Condition="$(opensslDir) == ''">$(opensslOutDir)</opensslBinDir>
     <opensslIncludeDir Condition="$(opensslDir) == ''">$(opensslOutDir)include</opensslIncludeDir>
-    <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-1.0.2t\</opensslDir>
+    <libffiIncludeDir Condition="$(libffiIncludeDir) == ''">$(libffiOutDir)include</libffiIncludeDir>
+    <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-1.1.1u\</opensslDir>
     <opensslDir Condition="!HasTrailingSlash($(opensslDir))">$(opensslDir)\</opensslDir>
     <opensslIncludeDir Condition="$(opensslIncludeDir) == ''">$(opensslDir)include</opensslIncludeDir>
     <opensslLibDir Condition="$(opensslLibDir) == ''">$(opensslDir)lib</opensslLibDir>
-    <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>
+    <opensslBinDir Condition="$(opensslBinDir) == '' and $(SSLMajorVersion) == '1'">$(opensslDir)lib</opensslBinDir>
+    <opensslBinDir Condition="$(opensslBinDir) == ''">$(opensslDir)bin</opensslBinDir>
+    <nasmDir>$(ExternalsDir)\nasm-2.15.05\</nasmDir>
     <zlibDir>$(ZLIB_DIR)</zlibDir>
-    <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.2.11\</zlibDir>
+    <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.2.13\</zlibDir>
     <zlibDir Condition="!HasTrailingSlash($(zlibDir))">$(zlibDir)\</zlibDir>
     <zlibIncludeDir>$(zlibDir)include</zlibIncludeDir>
     <zlibLibDir>$(zlibDir)lib</zlibLibDir>

PCbuild/tcltk.props

@@ -2,37 +2,36 @@
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="pyproject.props" />
   <PropertyGroup>
-    <TclMajorVersion>8</TclMajorVersion>
-    <TclMinorVersion>5</TclMinorVersion>
-    <TclPatchLevel>19</TclPatchLevel>
-    <TclRevision>0</TclRevision>
+    <TclMajorVersion>$(TCL_MAJOR_VERSION)</TclMajorVersion>
+    <TclMajorVersion Condition="$(TclMajorVersion) == ''">8</TclMajorVersion>
+    <TclMinorVersion>$(TCL_MINOR_VERSION)</TclMinorVersion>
+    <TclMinorVersion Condition="$(TclMinorVersion) == ''">6</TclMinorVersion>
+    <TclPatchLevel>$(TCL_PATCHLEVEL)</TclPatchLevel>
+    <TclPatchLevel Condition="$(TclPatchLevel) == ''">13</TclPatchLevel>
+    <TclRevision>$(TCL_REVISION)</TclRevision>
+    <TclRevision Condition="$(TclRevision) == ''">0</TclRevision>
     <TkMajorVersion>$(TclMajorVersion)</TkMajorVersion>
     <TkMinorVersion>$(TclMinorVersion)</TkMinorVersion>
     <TkPatchLevel>$(TclPatchLevel)</TkPatchLevel>
-    <TkRevision>$(TclRevision)</TkRevision>
-    <TixMajorVersion>8</TixMajorVersion>
-    <TixMinorVersion>4</TixMinorVersion>
-    <TixPatchLevel>3</TixPatchLevel>
-    <TixRevision>5</TixRevision>
-    <tclDir>$(ExternalsDir)tcl-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\</tclDir>
-    <tkDir>$(ExternalsDir)tk-$(TkMajorVersion).$(TkMinorVersion).$(TkPatchLevel).$(TkRevision)\</tkDir>
-    <tixDir>$(ExternalsDir)tix-$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel).$(TixRevision)\</tixDir>
-    <tcltkDir>$(ExternalsDir)tcltk\</tcltkDir>
-    <tcltkDir Condition="'$(Platform)' == 'x64'">$(ExternalsDir)tcltk64\</tcltkDir>
-    <TclDebugExt Condition="'$(Configuration)' == 'Debug'">g</TclDebugExt>
-    <tclDLLName>tcl$(TclMajorVersion)$(TclMinorVersion)$(TclDebugExt).dll</tclDLLName>
-    <tclLibName>tcl$(TclMajorVersion)$(TclMinorVersion)$(TclDebugExt).lib</tclLibName>
-    <tclShExeName>tclsh$(TclMajorVersion)$(TclMinorVersion)$(TclDebugExt).exe</tclShExeName>
-    <tkDLLName>tk$(TkMajorVersion)$(TkMinorVersion)$(TclDebugExt).dll</tkDLLName>
-    <tkLibName>tk$(TkMajorVersion)$(TkMinorVersion)$(TclDebugExt).lib</tkLibName>
-    <tixDLLName>tix$(TixMajorVersion)$(TixMinorVersion)$(TclDebugExt).dll</tixDLLName>
-    <tixDLLPath>$(tcltkDir)lib\tix$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel)\$(tixDLLName)</tixDLLPath>
-    <tcltkLib>$(tcltkDir)lib\tcl$(TclMajorVersion)$(TclMinorVersion)$(TclDebugExt).lib;$(tcltkDir)lib\tk$(TkMajorVersion)$(TkMinorVersion)$(TclDebugExt).lib</tcltkLib>
-    <TclMachine>IX86</TclMachine>
-    <TclMachine Condition="'$(Platform)' == 'x64'">AMD64</TclMachine>
-    <TclVersions>TCL_MAJOR_VERSION=$(TclMajorVersion) TCL_MINOR_VERSION=$(TclMinorVersion) TCL_PATCH_LEVEL=$(TclPatchLevel)</TclVersions>
-    <TclShortVersions>TCL_MAJOR=$(TclMajorVersion) TCL_MINOR=$(TclMinorVersion) TCL_PATCH=$(TclPatchLevel)</TclShortVersions>
-    <TkVersions>TK_MAJOR_VERSION=$(TkMajorVersion) TK_MINOR_VERSION=$(TkMinorVersion) TK_PATCH_LEVEL=$(TkPatchLevel)</TkVersions>
+    <TkRevision>$(TclRevision)</TkRevision>    
+    <TixMajorVersion>$(TIX_MAJOR_VERSION)</TixMajorVersion>
+    <TixMajorVersion Condition="$(TixMajorVersion) == ''">8</TixMajorVersion>
+    <TixMinorVersion>$(TIX_MINOR_VERSION)</TixMinorVersion>
+    <TixMinorVersion Condition="$(TixMinorVersion) == ''">4</TixMinorVersion>
+    <TixPatchLevel>$(TIX_PATCHLEVEL)</TixPatchLevel>
+    <TixPatchLevel Condition="$(TixPatchlevel) == ''">3</TixPatchLevel>
+    <TixRevision>$(TIX_REVISION)</TixRevision>
+    <TixRevision Condition="$(TixRevision) == ''">6</TixRevision>
+
+    <tclDir>$(TCL_DIR)</tclDir>
+    <tclDir Condition="$(tclDir) == ''">$(ExternalsDir)tcl-core-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\</tclDir>
+    <tkDir>$(TK_DIR)</tkDir>
+    <tkDir Condition="$(tkDir) == ''">$(ExternalsDir)tk-$(TkMajorVersion).$(TkMinorVersion).$(TkPatchLevel).$(TkRevision)\</tkDir>
+    <tixDir>$(TIX_DIR)</tixDir>
+    <tixDir Condition="$(tixDir) == ''">$(ExternalsDir)tix-$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel).$(TixRevision)\</tixDir>
+    <tcltkDir>$(TCL_TK_DIR)</tcltkDir>
+    <tcltkDir Condition="$(tcltkDir) == ''">$(ExternalsDir)tcltk-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\$(ArchName)\</tcltkDir>
+    <tcltkDir Condition="!HasTrailingSlash($(tcltkDir))">$(tcltkDir)\</tcltkDir>
 
     <BuildDirTop>Release</BuildDirTop>
     <BuildDirTop Condition="$(Configuration) == 'Debug'">Debug</BuildDirTop>