From af0d0e4eed9c2e01b7829e0a99f78ae0cf7803db Mon Sep 17 00:00:00 2001 From: Erick Fuentes Date: Wed, 13 Mar 2024 11:15:44 -0400 Subject: [PATCH] Add cross compilation toolchain for aarch64 --- WORKSPACE | 13 +- common/python/BUILD | 6 +- common/python/README.md | 6 +- experimental/beacon_sim/BUILD | 5 +- third_party/BUILD.jetson_sysroot | 6 + third_party/python/BUILD | 8 +- ...ements_3_8_10.txt => requirements_3_8.txt} | 0 toolchain/BUILD | 3 +- toolchain/aarch64-linux-gcc.sh | 3 +- toolchain/aarch64/extract_sysroot.py | 142 ++++++++++++++++++ toolchain/gcc_aarch64_toolchain_config.bzl | 9 +- 11 files changed, 182 insertions(+), 19 deletions(-) create mode 100644 third_party/BUILD.jetson_sysroot rename third_party/python/{requirements_3_8_10.txt => requirements_3_8.txt} (100%) create mode 100644 toolchain/aarch64/extract_sysroot.py diff --git a/WORKSPACE b/WORKSPACE index a758b524..a12d7b43 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,6 +11,13 @@ http_archive( sha256 = "212f3c05f3b2263b0e2f902d055aecc2755eba10c0011927788a38faee8fc9aa" ) +http_archive( + name = "jetson_sysroot", + urls = ["https://www.dropbox.com/scl/fi/39qmmgn3mdnhj14sa21zl/cleaned_jetson.tar?rlkey=2fq8ynle042p6ojhprz3vjis3&dl=1"], + build_file = "//third_party:BUILD.jetson_sysroot", + integrity = "sha256-CVtgQSRXe8o2wMjNcJrxtlUxiHZY9ZQJ0kde8BkrTPw=" +) + http_archive( name = "platforms", urls = [ @@ -116,7 +123,7 @@ py_repositories() DEFAULT_PYTHON_VERSION = "3.10" python_register_multi_toolchains( name="python", - python_versions = ['3.10', '3.8.10'], + python_versions = ['3.10', '3.8'], default_version = DEFAULT_PYTHON_VERSION ) @@ -126,11 +133,11 @@ multi_pip_parse( name="pip", default_version = DEFAULT_PYTHON_VERSION, python_interpreter_target = { - "3.8.10": "@python_3_8_10_host//:python", + "3.8": "@python_3_8_host//:python", "3.10": "@python_3_10_host//:python" }, requirements_lock = { - "3.8.10": "//third_party/python:requirements_3_8_10.txt", + "3.8": "//third_party/python:requirements_3_8.txt", "3.10": "//third_party/python:requirements_3_10.txt" }, ) diff --git a/common/python/BUILD b/common/python/BUILD index 556e592c..7e678190 100644 --- a/common/python/BUILD +++ b/common/python/BUILD @@ -1,7 +1,7 @@ load("@pip//:requirements.bzl", "requirement", "data_requirement") load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") load("@rules_python//python:defs.bzl", "py_binary") -load("@python//3.8.10:defs.bzl", py_test_3_8_10 = "py_test") +load("@python//3.8:defs.bzl", py_test_3_8 = "py_test") py_binary( name = "jupyter_notebook", @@ -47,8 +47,8 @@ py_test( data = [":pybind_example_python.so"], ) -py_test_3_8_10( - name = "pybind_example_3_8_10_test", +py_test_3_8( + name = "pybind_example_3_8_test", srcs = ["pybind_example_test.py"], main = "pybind_example_test.py", data = [":pybind_example_python.so"], diff --git a/common/python/README.md b/common/python/README.md index d68e8f27..09e29219 100644 --- a/common/python/README.md +++ b/common/python/README.md @@ -9,7 +9,7 @@ To add new versions of Python, locate the `python_register_multi_toolchains` inv DEFAULT_PYTHON_VERSION = "3.10" python_register_multi_toolchains( name="python", - python_versions = ['3.10', '3.8.10'], # add new entry here + python_versions = ['3.10', '3.8'], # add new entry here default_version = DEFAULT_PYTHON_VERSION ) @@ -19,11 +19,11 @@ multi_pip_parse( name="pip", default_version = DEFAULT_PYTHON_VERSION, python_interpreter_target = { - "3.8.10": "@python_3_8_10_host//:python", + "3.8": "@python_3_8_host//:python", "3.10": "@python_3_10_host//:python" }, requirements_lock = { - "3.8.10": "//third_party/python:requirements_3_8_10.txt", + "3.8": "//third_party/python:requirements_3_8.txt", "3.10": "//third_party/python:requirements_3_10.txt" # Add new entry here }, diff --git a/experimental/beacon_sim/BUILD b/experimental/beacon_sim/BUILD index 3ef6596b..c3a24ed8 100644 --- a/experimental/beacon_sim/BUILD +++ b/experimental/beacon_sim/BUILD @@ -902,7 +902,10 @@ py_wheel( name = "beacon_sim_wheel", distribution = "beacon_sim", version = "0.0.1", - platform = "manylinux2014_x86_64", + platform = select({ + "@platforms//cpu:x86_64": "manylinux2014_x86_64", + "@platforms//cpu:aarch64": "manylinux2014_aarch64", + }), python_tag = select({ "@rules_python//python/config_settings:is_python_3.10": "cp310", "@rules_python//python/config_settings:is_python_3.8": "cp38", diff --git a/third_party/BUILD.jetson_sysroot b/third_party/BUILD.jetson_sysroot new file mode 100644 index 00000000..1e7df28b --- /dev/null +++ b/third_party/BUILD.jetson_sysroot @@ -0,0 +1,6 @@ + +filegroup( + name = "jetson_sysroot", + srcs = glob(["**/*"]), + visibility=["//visibility:public"], +) diff --git a/third_party/python/BUILD b/third_party/python/BUILD index 937b9978..3648e802 100644 --- a/third_party/python/BUILD +++ b/third_party/python/BUILD @@ -1,13 +1,13 @@ -load("@python//3.8.10:defs.bzl", compile_pip_requirements_3_8_10 = "compile_pip_requirements") +load("@python//3.8:defs.bzl", compile_pip_requirements_3_8 = "compile_pip_requirements") load("@python//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements") load("@rules_python//python:defs.bzl", "py_binary") load("@pip//:requirements.bzl", "requirement") -compile_pip_requirements_3_8_10( - name = "requirements_3_8_10", +compile_pip_requirements_3_8( + name = "requirements_3_8", requirements_in=":requirements.in", - requirements_txt=":requirements_3_8_10.txt", + requirements_txt=":requirements_3_8.txt", extra_args=["--allow-unsafe"], timeout="eternal", ) diff --git a/third_party/python/requirements_3_8_10.txt b/third_party/python/requirements_3_8.txt similarity index 100% rename from third_party/python/requirements_3_8_10.txt rename to third_party/python/requirements_3_8.txt diff --git a/toolchain/BUILD b/toolchain/BUILD index 5a5f8d33..90fc607e 100644 --- a/toolchain/BUILD +++ b/toolchain/BUILD @@ -113,7 +113,8 @@ filegroup( ":aarch64-linux-gcc.sh", ":aarch64-linux-ld.sh", "@aarch64-none-linux-gnu//:all", - ] + "@jetson_sysroot" + ], ) cc_toolchain( diff --git a/toolchain/aarch64-linux-gcc.sh b/toolchain/aarch64-linux-gcc.sh index 5c5727d8..afbccdaa 100755 --- a/toolchain/aarch64-linux-gcc.sh +++ b/toolchain/aarch64-linux-gcc.sh @@ -1,4 +1,3 @@ #!/bin/bash -set -euox pipefail -echo $PWD +set -euo pipefail external/aarch64-none-linux-gnu/bin/aarch64-linux-gcc "$@" diff --git a/toolchain/aarch64/extract_sysroot.py b/toolchain/aarch64/extract_sysroot.py new file mode 100644 index 00000000..1692dcb6 --- /dev/null +++ b/toolchain/aarch64/extract_sysroot.py @@ -0,0 +1,142 @@ +#! /usr/bin/env python3 + +# Note that this is not a bazel target +import subprocess +import tarfile +import re +import tempfile +import os +from pathlib import Path + + +def build_docker_container(): + label = "jetson_sysroot:latest" + DOCKERFILE_CONTENT = """ + FROM nvcr.io/nvidia/l4t-base:35.4.1 + + RUN apt-get update && apt-get install libblas-dev liblapack-dev && rm -rf /var/lib/apt/lists/* + """ + subprocess.run(["docker", "build", "-t", label, '-'], input=DOCKERFILE_CONTENT, text=True) + return label + + +def export_docker_filesystem(tag): + # Launch a container + result = subprocess.run(["docker", "run", "-dt", tag], capture_output=True, text=True) + container_id = result.stdout.strip() + + TAR_FILE = "/tmp/jetson.tar" + with open(TAR_FILE, 'wb') as file_out: + subprocess.run(["docker", "export", container_id], stdout=file_out) + + # Kill the container + subprocess.run(["docker", "kill", container_id]) + return TAR_FILE + + +def clean_up_tar_file(file_path): + paths_to_exclude = [ + "^etc/(?!alternatives).*", + "^run/.*", + "^usr/sbin/.*", + "^usr/share.*", + "^var/.*", + "^usr/include/X11/.*", + "^usr/bin/X11.*", + "^usr/lib/python.*", + "^usr/lib/systemd.*", + "^usr/lib/udev.*", + "^usr/lib/apt.*", + "^usr/lib/aarch64-linux-gnu/perl.*", + "^usr/lib/aarch64-linux-gnu/gtk.*", + "^usr/lib/aarch64-linux-gnu/gstreamer.*", + "^usr/lib/aarch64-linux-gnu/gdk.*", + "^usr/lib/aarch64-linux-gnu/libLLVM.*", + "^usr/lib/aarch64-linux-gnu/libicudata.*", + "^usr/lib/aarch64-linux-gnu/dri/.*", + "^usr/lib/aarch64-linux-gnu/libcodec.*", + "^usr/lib/aarch64-linux-gnu/libavcodec.*", + "^usr/lib/aarch64-linux-gnu/libgtk.*", + "^usr/lib/aarch64-linux-gnu/librsvg.*", + "^usr/lib/python.*", + ] + + f = tarfile.TarFile(file_path, mode='r') + remaining = [] + + size_total = 0 + total = 0 + size_kept = 0 + kept = 0 + for member in f: + total += 1 + size_total += member.size + should_keep = True + # print('to test:', member.path) + for pattern in paths_to_exclude: + match = re.match(pattern, member.path) + # print('on pattern:', pattern, match) + if match is not None: + should_keep = False + break + + if should_keep: + kept += 1 + size_kept += member.size + remaining.append(member) + + for member in remaining: + print(member, member.size / (1024 ** 2)) + + print(f" Files kept / total: {kept} / {total}") + print(f" Megabytes kept / total: {size_kept / 1024 / 1024} / {size_total / 1024 / 1024}") + CLEANED_TAR_FILE = "/tmp/cleaned_jetson.tar" + with tempfile.TemporaryDirectory() as tempdir: + f.extractall(path=tempdir, members=remaining) + f.close() + + # make some symlinks + for obj in ["crt1.o", "crti.o", "crtn.o"]: + src_path = os.path.join(tempdir, f'lib/aarch64-linux-gnu/{obj}') + target_path = os.path.join(tempdir, f'lib/{obj}') + + target_dir = os.path.dirname(target_path) + source_dir = os.path.dirname(src_path) + rel_src = os.path.relpath(src_path, target_dir) + print(f'src: {src_path} target: {target_path} target_dir: {target_dir} rel_src: {rel_src}') + os.symlink(rel_src, target_path) + + # edit /lib/aarch64-linux-gnu/libc.so + libc_path = Path(tempdir) / 'lib/aarch64-linux-gnu/libc.so' + contents = libc_path.read_text() + contents = contents.replace('/usr/lib/aarch64-linux-gnu/', '') + contents = contents.replace('/lib/aarch64-linux-gnu/', '') + libc_path.write_text(contents) + + cleaned = tarfile.TarFile(CLEANED_TAR_FILE, mode='w') + cleaned.add(tempdir, arcname='/') + cleaned.close() + return CLEANED_TAR_FILE + + + +def main(): + # Build the docker container + # print('Building Docker Container') + # tag = build_docker_container() + + # # Export the filesystem as a tar file + # print('Exporting Docker Container') + # file_path = export_docker_filesystem(tag) + + file_path = "/tmp/jetson.tar" + + # Remove unneeded files + print('Removing Unneeded files') + cleaned_file_path = clean_up_tar_file(file_path) + print(cleaned_file_path) + + +if __name__ == "__main__": + main() + diff --git a/toolchain/gcc_aarch64_toolchain_config.bzl b/toolchain/gcc_aarch64_toolchain_config.bzl index 81bf8e21..33f2ae31 100644 --- a/toolchain/gcc_aarch64_toolchain_config.bzl +++ b/toolchain/gcc_aarch64_toolchain_config.bzl @@ -57,9 +57,11 @@ def _impl(ctx): flag_group( flags=[ "--sysroot", - "external/aarch64-none-linux-gnu/aarch64-buildroot-linux-gnu/sysroot", + "external/jetson_sysroot", "-isystem", - "external/aarch64-none-linux-gnu/aarch64-buildroot-linux-gnu/include", + "external/jetson_sysroot/usr/include/aarch64-linux-gnu", + # "-isystem", + # "external/aarch64-none-linux-gnu/aarch64-buildroot-linux-gnu/include", "-isystem", "external/aarch64-none-linux-gnu/aarch64-buildroot-linux-gnu/include/c++/10.2.0", "-isystem", @@ -85,6 +87,9 @@ def _impl(ctx): flags=[ "-lstdc++", "-lm", + "--sysroot", + "external/jetson_sysroot", + "-Lexternal/jetson_sysroot/lib/aarch64-linux-gnu", ], ), ],