test: ensure prerelease versions of pandas and arrow are tested night…

…ly (#961)

* test: ensure prerelease versions of pandas and arrow are tested nightly

* use regex to find package names rather than filter out comment lines

.kokoro/continuous/prerelease-deps-3.8.cfg

@@ -0,0 +1,7 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Only run this nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "prerelease_deps"
+}

noxfile.py

@@ -16,6 +16,7 @@
 
 import pathlib
 import os
+import re
 import shutil
 
 import nox
@@ -212,16 +213,30 @@ def prerelease_deps(session):
     # PyArrow prerelease packages are published to an alternative PyPI host.
     # https://arrow.apache.org/docs/python/install.html#installing-nightly-packages
     session.install(
-        "--extra-index-url", "https://pypi.fury.io/arrow-nightlies/", "--pre", "pyarrow"
+        "--extra-index-url",
+        "https://pypi.fury.io/arrow-nightlies/",
+        "--prefer-binary",
+        "--pre",
+        "--upgrade",
+        "pyarrow",
     )
     session.install(
+        "--extra-index-url",
+        "https://pypi.anaconda.org/scipy-wheels-nightly/simple",
+        "--prefer-binary",
         "--pre",
+        "--upgrade",
+        "pandas",
+    )
+
+    session.install(
+        "--pre",
+        "--upgrade",
         "google-api-core",
         "google-cloud-bigquery-storage",
         "google-cloud-core",
         "google-resumable-media",
         "grpcio",
-        "pandas",
     )
     session.install(
         "freezegun",
@@ -234,7 +249,30 @@ def prerelease_deps(session):
         "pytest",
         "pytest-cov",
     )
-    session.install("-e", ".[all]")
+
+    # Because we test minimum dependency versions on the minimum Python
+    # version, the first version we test with in the unit tests sessions has a
+    # constraints file containing all dependencies and extras.
+    with open(
+        CURRENT_DIRECTORY
+        / "testing"
+        / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt",
+        encoding="utf-8",
+    ) as constraints_file:
+        constraints_text = constraints_file.read()
+
+    # Ignore leading whitespace and comment lines.
+    deps = [
+        match.group(1)
+        for match in re.finditer(
+            r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
+        )
+    ]
+
+    # We use --no-deps to ensure that pre-release versions aren't overwritten
+    # by the version ranges in setup.py.
+    session.install(*deps)
+    session.install("--no-deps", "-e", ".[all]")
 
     # Print out prerelease package versions.
     session.run("python", "-c", "import grpc; print(grpc.__version__)")