Only download necessary toolchains (#16)

Fixes #13.

We now create a repo per 'environment' (os/cpu combo) in addition to the hub repo, and then refer to the downloads of those repos from the hub.

multitool/private/env_repo_template/BUILD.bazel.template

@@ -0,0 +1 @@
+# generated by multitool

multitool/private/env_repo_template/tools/BUILD.bazel.template

@@ -0,0 +1 @@
+# generated by multitool

multitool/private/env_repo_tool_template/BUILD.bazel.template

@@ -0,0 +1,3 @@
+# generated by multitool
+
+exports_files(glob(include=["*_executable"]))

multitool/private/tool_template/BUILD.bazel.template → multitool/private/hub_repo_tool_template/BUILD.bazel.template

@@ -2,8 +2,6 @@
 
 load(":tool.bzl", "tool")
 
-exports_files(glob(include=["*_executable"]))
-
 toolchain_type(
     name = "toolchain_type",
     visibility = ["//:__subpackages__"],

multitool/private/tool_template/tool.bzl.template → multitool/private/hub_repo_tool_template/tool.bzl.template

@@ -15,7 +15,7 @@ tool = rule(executable = True, implementation = _tool_impl, toolchains = [_TOOLC
 def _declare_toolchain(name, os, cpu):
     toolchain_info(
         name = "{name}_{os}_{cpu}_toolchain_info".format(name=name, os=os, cpu=cpu),
-        executable = "//tools/{name}:{os}_{cpu}_executable".format(name=name, os=os, cpu=cpu),
+        executable = "@@{hub_name}.{os}_{cpu}//tools/{name}:{os}_{cpu}_executable".format(name=name, os=os, cpu=cpu),
         os = os,
         cpu = cpu,
     )

multitool/private/multitool.bzl

@@ -1,30 +1,64 @@
-"multitool hub implementation"
-
-_HUB_TEMPLATE = "//multitool/private:hub_repo_template/{filename}.template"
-_TOOL_TEMPLATE = "//multitool/private:tool_template/{filename}.template"
-
-def _render_hub(rctx, filename, substitutions = None):
-    rctx.template(
-        filename,
-        Label(_HUB_TEMPLATE.format(filename = filename)),
-        substitutions = substitutions or {},
-    )
-
-def _render_tool(rctx, tool_name, filename, substitutions = None):
-    rctx.template(
-        "tools/{tool_name}/{filename}".format(tool_name = tool_name, filename = filename),
-        Label(_TOOL_TEMPLATE.format(filename = filename)),
-        substitutions = {
-            "{name}": tool_name,
-        } | (substitutions or {}),
-    )
+"""
+multitool
+
+Multitool takes as input a JSON lockfile and emits the following repos:
+
+ - [hub].[os]_[cpu], for each [os]/[cpu] combo in _SUPPORTED_ENVS:
+     This repository holds os/cpu specific binaries for all tools in the provided
+     lockfile(s) and constructs clean symlinks to their content for inclusion in
+     toolchains defined in the [hub] repo.
+
+     The structure of this repo is, very simply:
+       tools/
+         [tool-name]/
+           BUILD.bazel            (export all *_executable files)
+           [os]_[cpu]_executable  (a downloaded file or a symlink to a file in a
+                                   downloaded and extracted archive)
+
+ - [hub]:
+     This repository holds toolchain definitions for all tools in the provided
+     lockfile(s), as well as an executable tool target that will pick the
+     appropriate toolchain.
+
+     The structure of this repo is:
+       toolchains/
+         BUILD.bazel      (a single file containing all declared toolchains for easy registration)
+       tools/
+         [tool-name]/
+           BUILD.bazel    (declares the toolchain_type and the executable tool target)
+           tool.bzl       (scaffolding for the tool target and toolchain declarations in toolchains/BUILD.bazel)
+       toolchain_info.bzl (common scaffolding for toolchain declarations)
+
+       (additional BUILD.bazel and a WORKSPACE file are included as required by Bazel)
+
+To keep things orderly, we keep all the toolchain Bazel goo in the [hub] repo and only stash
+the binaries in the [hub].[os]_[cpu] repos. It's a conscious decision not to place some fragments
+of the toolchain definitions in the latter repos to make the dependencies run exactly one way:
+[hub] -> [hub].[os]_[cpu].
+
+This implementation depends on rendering a number of templates, which are defined in sibling
+folders and managed by the templates starlark file.
+
+To maintain support both bzlmod and non-bzlmod setups, we provide two entrypoints to the rule:
+ - (bzlmod)     hub       : invoked by the hub tag in extension.bzl
+ - (non-bzlmod) multitool : invoked in WORKSPACE or related macros, and additionally registers toolchains
+"""
+
+load(":templates.bzl", "templates")
+
+_SUPPORTED_ENVS = [
+    ("linux", "arm64"),
+    ("linux", "x86_64"),
+    ("macos", "arm64"),
+    ("macos", "x86_64"),
+]
 
 def _check(condition, message):
     "fails iff condition is False and emits message"
     if not condition:
         fail(message)
 
-def _multitool_hub_impl(rctx):
+def _load_tools(rctx):
     tools = {}
     for lockfile in rctx.attr.lockfiles:
         # TODO: validate no conflicts from multiple hub declarations and/or
@@ -33,15 +67,33 @@ def _multitool_hub_impl(rctx):
         #  (this is also a very naive merge at the tool level)
         tools = tools | json.decode(rctx.read(lockfile))
 
-    loads = []
-    defines = []
-
+    # validation
     for tool_name, tool in tools.items():
-        toolchains = []
+        for binary in tool["binaries"]:
+            _check(
+                binary["os"] in ["linux", "macos"],
+                "{tool_name}: Unknown os '{os}'".format(
+                    tool_name = tool_name,
+                    os = binary["os"],
+                ),
+            )
+            _check(
+                binary["cpu"] in ["x86_64", "arm64"],
+                "{tool_name}: Unknown cpu '{cpu}'".format(
+                    tool_name = tool_name,
+                    cpu = binary["cpu"],
+                ),
+            )
+
+    return tools
 
+def _env_specific_tools_impl(rctx):
+    tools = _load_tools(rctx)
+
+    for tool_name, tool in tools.items():
         for binary in tool["binaries"]:
-            _check(binary["os"] in ["linux", "macos"], "Unknown os '{os}'".format(os = binary["os"]))
-            _check(binary["cpu"] in ["x86_64", "arm64"], "Unknown cpu '{cpu}'".format(cpu = binary["cpu"]))
+            if binary["os"] != rctx.attr.os or binary["cpu"] != rctx.attr.cpu:
+                continue
 
             target_executable = "tools/{tool_name}/{os}_{cpu}_executable".format(
                 tool_name = tool_name,
@@ -104,14 +156,42 @@ def _multitool_hub_impl(rctx):
             else:
                 fail("Unknown 'kind' {kind}".format(kind = binary["kind"]))
 
+            templates.env_tool(rctx, tool_name, "BUILD.bazel")
+
+    templates.env(rctx, "tools/BUILD.bazel")
+    templates.env(rctx, "BUILD.bazel")
+
+_env_specific_tools = repository_rule(
+    attrs = {
+        "lockfiles": attr.label_list(mandatory = True, allow_files = True),
+        "os": attr.string(),
+        "cpu": attr.string(),
+    },
+    implementation = _env_specific_tools_impl,
+)
+
+def _sort_fn(tup):
+    return tup[0]
+
+def _multitool_hub_impl(rctx):
+    tools = _load_tools(rctx)
+
+    loads = []
+    defines = []
+
+    for tool_name, tool in sorted(tools.items(), key = _sort_fn):
+        toolchains = []
+
+        for binary in tool["binaries"]:
             toolchains.append('\n    _declare_toolchain(name="{name}", os="{os}", cpu="{cpu}")'.format(
                 name = tool_name,
                 cpu = binary["cpu"],
                 os = binary["os"],
             ))
 
-        _render_tool(rctx, tool_name, "BUILD.bazel")
-        _render_tool(rctx, tool_name, "tool.bzl", {
+        templates.hub_tool(rctx, tool_name, "BUILD.bazel")
+        templates.hub_tool(rctx, tool_name, "tool.bzl", {
+            "{hub_name}": rctx.attr.name,
             "{toolchains}": "\n".join(toolchains),
         })
 
@@ -122,10 +202,10 @@ def _multitool_hub_impl(rctx):
         ))
         defines.append("declare_{clean_name}_toolchains()".format(clean_name = clean_name))
 
-    _render_hub(rctx, "BUILD.bazel")
-    _render_hub(rctx, "toolchain_info.bzl")
-    _render_hub(rctx, "tools/BUILD.bazel")
-    _render_hub(rctx, "toolchains/BUILD.bazel", {
+    templates.hub(rctx, "BUILD.bazel")
+    templates.hub(rctx, "toolchain_info.bzl")
+    templates.hub(rctx, "tools/BUILD.bazel")
+    templates.hub(rctx, "toolchains/BUILD.bazel", {
         "{loads}": "\n".join(loads),
         "{defines}": "\n".join(defines),
     })
@@ -139,11 +219,16 @@ _multitool_hub = repository_rule(
 
 def hub(name, lockfiles):
     "Create a multitool hub."
+    for env in _SUPPORTED_ENVS:
+        _env_specific_tools(
+            name = "{name}.{os}_{cpu}".format(name = name, os = env[0], cpu = env[1]),
+            lockfiles = lockfiles,
+            os = env[0],
+            cpu = env[1],
+        )
     _multitool_hub(name = name, lockfiles = lockfiles)
 
 def multitool(name, lockfile):
     "(non-bzlmod) Create a multitool hub and register its toolchains."
-
-    _multitool_hub(name = name, lockfiles = [lockfile])
-
-    native.register_toolchains("@multitool//toolchains:all")
+    hub(name, [lockfile])
+    native.register_toolchains("@{name}//toolchains:all".format(name = name))

multitool/private/templates.bzl

@@ -0,0 +1,46 @@
+"multitool templating"
+
+_ENV_TEMPLATE = "//multitool/private:env_repo_template/{filename}.template"
+_ENV_TOOL_TEMPLATE = "//multitool/private:env_repo_tool_template/{filename}.template"
+
+_HUB_TEMPLATE = "//multitool/private:hub_repo_template/{filename}.template"
+_HUB_TOOL_TEMPLATE = "//multitool/private:hub_repo_tool_template/{filename}.template"
+
+def _render_env(rctx, filename, substitutions = None):
+    rctx.template(
+        filename,
+        Label(_ENV_TEMPLATE.format(filename = filename)),
+        substitutions = substitutions or {},
+    )
+
+def _render_env_tool(rctx, tool_name, filename, substitutions = None):
+    rctx.template(
+        "tools/{tool_name}/{filename}".format(tool_name = tool_name, filename = filename),
+        Label(_ENV_TOOL_TEMPLATE.format(filename = filename)),
+        substitutions = {
+            "{name}": tool_name,
+        } | (substitutions or {}),
+    )
+
+def _render_hub(rctx, filename, substitutions = None):
+    rctx.template(
+        filename,
+        Label(_HUB_TEMPLATE.format(filename = filename)),
+        substitutions = substitutions or {},
+    )
+
+def _render_hub_tool(rctx, tool_name, filename, substitutions = None):
+    rctx.template(
+        "tools/{tool_name}/{filename}".format(tool_name = tool_name, filename = filename),
+        Label(_HUB_TOOL_TEMPLATE.format(filename = filename)),
+        substitutions = {
+            "{name}": tool_name,
+        } | (substitutions or {}),
+    )
+
+templates = struct(
+    env = _render_env,
+    env_tool = _render_env_tool,
+    hub = _render_hub,
+    hub_tool = _render_hub_tool,
+)