Finish out of tree build system (except xbuildenv deploy) (#2823)

This completes the out of tree build CLI. This PR is paired up with:
numpy/numpy#21895
I have also successfully built scikit-learn, statsmodels, pandas, and
astropy with this.

The last thing we need to do after this is set up deployment of the
cross build environment. We can deploy one version to s3 for each
tagged commit. I will do that in a separate PR after this is merged.

pyodide-build/pyodide_build/common.py

@@ -229,11 +229,14 @@ def init_environment() -> None:
         os.environ["PYODIDE_ROOT"] = str(search_pyodide_root(os.getcwd()))
 
     os.environ.update(get_make_environment_vars())
-    hostsitepackages = get_hostsitepackages()
-    pythonpath = [
-        hostsitepackages,
-    ]
-    os.environ["PYTHONPATH"] = ":".join(pythonpath)
+    try:
+        hostsitepackages = get_hostsitepackages()
+        pythonpath = [
+            hostsitepackages,
+        ]
+        os.environ["PYTHONPATH"] = ":".join(pythonpath)
+    except KeyError:
+        pass
     os.environ["BASH_ENV"] = ""
     get_unisolated_packages()
 
@@ -251,13 +254,18 @@ def get_unisolated_packages() -> list[str]:
     if "UNISOLATED_PACKAGES" in os.environ:
         return json.loads(os.environ["UNISOLATED_PACKAGES"])
     PYODIDE_ROOT = get_pyodide_root()
-    unisolated_packages = []
-    for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"):
-        config = parse_package_config(pkg, check=False)
-        if config.get("build", {}).get("cross-build-env", False):
-            unisolated_packages.append(config["package"]["name"])
-    # TODO: remove setuptools_rust from this when they release the next version.
-    unisolated_packages.append("setuptools_rust")
+    unisolated_file = PYODIDE_ROOT / "unisolated.txt"
+    if unisolated_file.exists():
+        # in xbuild env, read from file
+        unisolated_packages = unisolated_file.read_text().splitlines()
+    else:
+        unisolated_packages = []
+        for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"):
+            config = parse_package_config(pkg, check=False)
+            if config.get("build", {}).get("cross-build-env", False):
+                unisolated_packages.append(config["package"]["name"])
+        # TODO: remove setuptools_rust from this when they release the next version.
+        unisolated_packages.append("setuptools_rust")
     os.environ["UNISOLATED_PACKAGES"] = json.dumps(unisolated_packages)
     return unisolated_packages
 

pyodide-build/pyodide_build/create_xbuildenv.py

@@ -3,7 +3,7 @@
 import subprocess
 from pathlib import Path
 
-from .common import get_make_flag, get_pyodide_root
+from .common import get_make_flag, get_pyodide_root, get_unisolated_packages
 from .io import parse_package_config
 
 
@@ -44,7 +44,7 @@ def copy_wasm_libs(xbuildenv_path: Path) -> None:
     wasm_lib_dir = get_relative_path(pyodide_root, "WASM_LIBRARY_DIR")
     sysconfig_dir = get_relative_path(pyodide_root, "SYSCONFIGDATA_DIR")
     xbuildenv_root = xbuildenv_path / "pyodide-root"
-    xbuildenv_path.mkdir()
+    xbuildenv_path.mkdir(exist_ok=True)
     to_copy: list[Path] = [
         pythoninclude,
         sysconfig_dir,
@@ -89,3 +89,6 @@ def main(args: argparse.Namespace) -> None:
         stdout=subprocess.PIPE,
     )
     (xbuildenv_path / "requirements.txt").write_bytes(res.stdout)
+    (xbuildenv_path / "pyodide-root/unisolated.txt").write_text(
+        "\n".join(get_unisolated_packages())
+    )

pyodide-build/pyodide_build/install_xbuildenv.py

@@ -15,12 +15,27 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
         "on numpy or scipy.\n"
         "Note: this is a private endpoint that should not be used outside of the Pyodide Makefile."
     )
+    parser.add_argument("--download", action="store_true", help="Download xbuild env")
     parser.add_argument("xbuild_env", type=str, nargs=1)
     return parser
 
 
-def main(args: argparse.Namespace) -> None:
-    xbuildenv_path = Path(args.xbuild_env[0])
+def download_xbuild_env(version: str, xbuildenv_path: Path) -> None:
+    from shutil import rmtree, unpack_archive
+    from tempfile import NamedTemporaryFile
+    from urllib.request import urlretrieve
+
+    rmtree(xbuildenv_path, ignore_errors=True)
+    with NamedTemporaryFile(suffix=".tar") as f:
+        urlretrieve(
+            f"http://pyodide-cache.s3-website-us-east-1.amazonaws.com/xbuildenv/{version}.tar",
+            f.name,
+        )
+        unpack_archive(f.name, xbuildenv_path)
+
+
+def install_xbuild_env(xbuildenv_path: Path) -> None:
+    xbuildenv_path = xbuildenv_path / "xbuildenv"
     pyodide_root = get_pyodide_root()
     xbuildenv_root = xbuildenv_path / "pyodide-root"
     host_site_packages = xbuildenv_root / Path(
@@ -42,3 +57,11 @@ def main(args: argparse.Namespace) -> None:
     shutil.copytree(
         xbuildenv_path / "site-packages-extras", host_site_packages, dirs_exist_ok=True
     )
+
+
+def main(args: argparse.Namespace) -> None:
+    xbuildenv_path = Path(args.xbuild_env[0])
+    version = "2"
+    if args.download:
+        download_xbuild_env(version, xbuildenv_path)
+    install_xbuild_env(xbuildenv_path)

pyodide-build/pyodide_build/out_of_tree/__main__.py

@@ -0,0 +1,36 @@
+import argparse
+import os
+from pathlib import Path
+
+from . import build
+
+
+def ensure_env_installed(env: Path) -> None:
+    if env.exists():
+        return
+    from ..install_xbuildenv import download_xbuild_env, install_xbuild_env
+
+    version = "2"
+    download_xbuild_env(version, env)
+    install_xbuild_env(env)
+
+
+def main():
+    main_parser = argparse.ArgumentParser(prog="pywasmbuild")
+    main_parser.description = "Tools for creating Python extension modules for the wasm32-unknown-emscripten platform"
+    subparsers = main_parser.add_subparsers(help="action")
+    for module in [build]:
+        modname = module.__name__.rpartition(".")[-1]
+        parser = module.make_parser(subparsers.add_parser(modname))
+        parser.set_defaults(func=module.main)
+
+    env = Path(".pyodide-xbuildenv")
+    os.environ["PYODIDE_ROOT"] = str(env / "xbuildenv/pyodide-root")
+    ensure_env_installed(env)
+
+    args = main_parser.parse_args()
+    if hasattr(args, "func"):
+        # run the selected action
+        args.func(args)
+    else:
+        main_parser.print_help()

pyodide-build/pyodide_build/out_of_tree/build.py

@@ -0,0 +1,47 @@
+import argparse
+import os
+
+from .. import common, pypabuild, pywasmcross
+
+
+def run(exports, args):
+    cflags = common.get_make_flag("SIDE_MODULE_CFLAGS")
+    cflags += f" {os.environ.get('CFLAGS', '')}"
+    cxxflags = common.get_make_flag("SIDE_MODULE_CXXFLAGS")
+    cxxflags += f" {os.environ.get('CXXFLAGS', '')}"
+    ldflags = common.get_make_flag("SIDE_MODULE_LDFLAGS")
+    ldflags += f" {os.environ.get('LDFLAGS', '')}"
+    build_env_ctx = pywasmcross.get_build_env(
+        env=os.environ.copy(),
+        pkgname="",
+        cflags=cflags,
+        cxxflags=cxxflags,
+        ldflags=ldflags,
+        target_install_dir="",
+        exports=exports,
+    )
+
+    with build_env_ctx as env:
+        pypabuild.build(env, " ".join(args))
+
+
+def main(parser_args: argparse.Namespace) -> None:
+    run(parser_args.exports, parser_args.backend_args)
+
+
+def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
+    parser.description = "Use pypa/build to build a Python package."
+    parser.add_argument(
+        "--exports",
+        choices=["pyinit", "requested", "whole_archive"],
+        default="requested",
+        help="Which symbols should be exported when linking .so files?",
+    )
+    parser.add_argument(
+        "backend_args",
+        metavar="args",
+        type=str,
+        nargs=argparse.REMAINDER,
+        help="Arguments to pass on to the backend",
+    )
+    return parser

pyodide-build/setup.py

@@ -5,6 +5,7 @@
     setuptools.setup(
         entry_points={
             "console_scripts": [
+                "pywasm = pyodide_build.out_of_tree.__main__:main",
                 "_pywasmcross = pyodide_build.pywasmcross:compiler_main",
             ]
         }