vpython: use realpath when invoking virtualenv

This is a reland of https://chromium-review.googlesource.com/c/infra/luci/luci-go/+/5449147
with fix for python2.7.

Use realpath for virtualenv.pyz to mitigate pypa/virtualenv#1949

Change-Id: I4bf2f527d1a90577d25194e05bb2de00bc1d71ef
Reviewed-on: https://chromium-review.googlesource.com/c/infra/luci/luci-go/+/5458472
Commit-Queue: Brian Ryner <bryner@google.com>
Reviewed-by: Brian Ryner <bryner@google.com>
Auto-Submit: Chenlin Fan <fancl@chromium.org>

vpython/application/application.go

@@ -169,7 +169,7 @@ func (a *Application) ParseEnvs(ctx context.Context) (err error) {
 	// Determine our VirtualEnv base directory.
 	if v, ok := e.Lookup(VirtualEnvRootENV); ok {
 		a.VpythonRoot = v
-	} else {
+	} else if a.VpythonRoot == "" {
 		cdir, err := a.userCacheDir()
 		if err != nil {
 			logging.Infof(ctx, "failed to get user cache dir: %s", err)

vpython/application/application_py3_test.go → vpython/application/application_py_test.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -44,24 +45,28 @@ var testStorageDir string
 
 func getPythonEnvironment(ver string) *python.Environment {
 	return map[string]*python.Environment{
+		"2.7": {
+			Executable: "python",
+			CPython:    python.CPythonFromCIPD("version:2@2.7.18.chromium.44"),
+			Virtualenv: python.VirtualenvFromCIPD("version:2@16.7.12.chromium.7"),
+		},
 		"3.8": {
 			Executable: "python3",
-			CPython:    python.CPython3FromCIPD("version:2@3.8.10.chromium.24"),
+			CPython:    python.CPython3FromCIPD("version:2@3.8.10.chromium.34"),
 			Virtualenv: python.VirtualenvFromCIPD("version:2@16.7.12.chromium.7"),
 		},
 		"3.11": {
 			Executable: "python3",
-			CPython:    python.CPython3FromCIPD("version:2@3.11.5.chromium.30"),
-			Virtualenv: python.VirtualenvFromCIPD("version:2@20.17.1.chromium.8"),
+			CPython:    python.CPython3FromCIPD("version:2@3.11.8.chromium.35"),
+			Virtualenv: python.VirtualenvFromCIPD("version:2@20.25.1.chromium.8"),
 		},
 	}[ver]
 }
 
 func setupApp(ctx context.Context, app *Application) context.Context {
-	app.Arguments = append([]string{
-		"-vpython-root",
-		testStorageDir,
-	}, app.Arguments...)
+	if app.VpythonRoot == "" {
+		app.VpythonRoot = testStorageDir
+	}
 
 	app.Initialize(ctx)
 
@@ -131,7 +136,7 @@ func TestMain(m *testing.M) {
 func TestPythonBasic(t *testing.T) {
 	Convey("Test python basic", t, func() {
 		var env *python.Environment
-		for _, ver := range []string{"3.8", "3.11"} {
+		for _, ver := range []string{"2.7", "3.8", "3.11"} {
 			Convey(ver, func() {
 				env = getPythonEnvironment(ver)
 
@@ -166,6 +171,27 @@ func TestPythonBasic(t *testing.T) {
 					So(has, ShouldBeTrue)
 					So(rc, ShouldEqual, 42)
 				})
+
+				if runtime.GOOS != "windows" {
+					// See https://github.com/pypa/virtualenv/issues/1949
+					Convey("Test symlink root", func() {
+						symlinkRoot := filepath.Join(t.TempDir(), "link")
+						err := os.Symlink(testStorageDir, symlinkRoot)
+						So(err, ShouldBeNil)
+
+						c := cmd(t, &Application{
+							Arguments: []string{
+								"-vpython-spec",
+								testData("default.vpython3"),
+								"-c",
+								"print(123)",
+							},
+							VpythonRoot: symlinkRoot,
+						}, env)
+
+						So(output(c), ShouldEqual, "123")
+					})
+				}
 			})
 		}
 	})

vpython/python/bootstrap.py

@@ -16,6 +16,7 @@
 import os
 import subprocess
 import sys
+import tempfile
 
 if sys.version_info[0] > 2:
   ISOLATION_FLAG = '-I'
@@ -27,10 +28,19 @@
 # Create virtual environment in ${out} directory
 virtualenv = glob.glob(
     os.path.join(os.environ['virtualenv'], '*', 'virtualenv.py*'))[0]
-subprocess.check_call([
-    sys.executable, '-B', ISOLATION_FLAG, virtualenv,
-    '--no-download', '--always-copy', os.environ['out']
-])
+
+args = [
+    sys.executable, '-B', ISOLATION_FLAG,
+    # Use realpath to mitigate https://github.com/pypa/virtualenv/issues/1949
+    os.path.realpath(virtualenv),
+    '--no-download', '--always-copy', os.environ['out'],
+]
+if virtualenv.endswith('.pyz'):
+  # .pyz ensures Python3 so we can use TemporaryDirectory.
+  with tempfile.TemporaryDirectory() as d:
+    subprocess.check_call(args + ['--app-data', d])
+else:
+  subprocess.check_call(args)
 
 # Install wheels to virtual environment
 if 'wheels' in os.environ:

vpython/python/python.go

@@ -39,6 +39,19 @@ type Environment struct {
 	Virtualenv generators.Generator
 }
 
+func CPythonFromCIPD(version string) generators.Generator {
+	return &generators.CIPDExport{
+		Name: "cpython",
+		Ensure: ensure.File{
+			PackagesBySubdir: map[string]ensure.PackageSlice{
+				"": {
+					{PackageTemplate: "infra/3pp/tools/cpython/${platform}", UnresolvedVersion: version},
+				},
+			},
+		},
+	}
+}
+
 func CPython3FromCIPD(version string) generators.Generator {
 	return &generators.CIPDExport{
 		Name: "cpython",