feat: add synth rules (#6)

* feat: Add synth rules

The rules let running `synth.py` scripts from bazel build. Please check `synth/README.md` (part of this commit) for more details.

* rename `gapic_assemblies` attribute to `srcs`

* add newline

* Update README.md

* Update README.md

Co-authored-by: Alexander Fenster <fenster@google.com>

README.md

@@ -35,7 +35,7 @@ legacy reasons)
 
 ### Requirements
 
-- Bazel version `3.0.0`.
+- Bazel version `3.0.0+`.
 - Linux (may work on other platforms, but this haven't been tested).
 - `gcc`, `make`, `autoconf`, `unzip` tools
 

WORKSPACE

@@ -1,5 +1,13 @@
 workspace(name = "rules_gapic")
 
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_python",
+    strip_prefix = "rules_python-0.1.0",
+    url = "https://github.com/bazelbuild/rules_python/archive/0.1.0.tar.gz",
+)
+
 load("//:repositories.bzl", "rules_gapic_repositories")
 rules_gapic_repositories()
 
@@ -10,3 +18,7 @@ protobuf_deps()
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
 go_rules_dependencies()
 go_register_toolchains()
+
+load("@rules_gapic//synth:synth_repositories.bzl", "synth")
+
+synth()

synth/BUILD.bazel

@@ -0,0 +1,21 @@
+load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
+load("@synth_pip_deps//:requirements.bzl", "requirement")
+load("//synth:gapic_postprocessor.bzl", "postprocessed_gapic_pkg")
+
+py_binary(
+    name = "gapic_postprocessor",
+    srcs = glob(["**/*.py"]),
+    main = "gapic_postprocessor.py",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    imports = ["."],
+    deps = [],
+)
+
+java_binary(
+    name = "google_java_format_binary",
+    jvm_flags = ["-Xmx512m"],
+    main_class = "com.google.googlejavaformat.java.Main",
+    visibility = ["//visibility:public"],
+    runtime_deps = ["@google_java_format_all_deps//jar"],
+)

synth/README.md

@@ -0,0 +1,29 @@
+## GAPIC Postprocessor [Bazel](https://www.bazel.build/) Integration
+
+This directory contains Bazel rules for running `synth.py` postprocessing 
+scripts.
+
+Most of the Python scripts in this directory are modified versions of the 
+scripts from [Synthtool](https://github.com/googleapis/synthtool).
+
+### Usage
+To run `synth.py` for Language API put the following in the `BUILD.bazel` file 
+in the `//google/cloud/language` package:  
+
+```bzl
+load("@rules_gapic//synth:gapic_postprocessor.bzl", "java_synth_pkg")
+
+java_synth_pkg(
+    name = "java_language",
+    synth_script = "synth.py",
+    srcs = [
+        "//google/cloud/language/v1:google-cloud-language-v1-java",
+        "//google/cloud/language/v1beta2:google-cloud-language-v1beta2-java",
+    ],
+)
+```
+
+The `synth.py` script ran by the rule above is a regular `synth.py` script,
+some parts of which (like templates generation) will result in no-op if they 
+are specific to code publishing and are not about postprocessing/correcting the
+code generated by the GAPIC generators.

synth/gapic_postprocessor.bzl

@@ -0,0 +1,93 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def _postprocessed_gapic_pkg_impl(ctx):
+    postprocessor = ctx.executable.postprocessor
+    formatter = ctx.executable.formatter
+    output_pkg = ctx.outputs.pkg
+    synth_script = ctx.attr.synth_script.files.to_list()[0]
+
+    output_dir_name = ctx.label.name
+    tmp_dir_name = "%s_tmp" % output_dir_name;
+    output_dir_path = "%s/%s" % (output_pkg.dirname, output_dir_name)
+    tmp_dir_path = "%s/%s" % (output_pkg.dirname, tmp_dir_name)
+
+    gapic_assemblies = []
+    for gapic_assembly in ctx.attr.srcs:
+        gapic_assemblies.extend(gapic_assembly.files.to_list())
+
+    script = """
+    set -e
+    echo {formatter}
+    mkdir -p {output_dir_path}
+    mkdir -p {tmp_dir_path}
+    for gapic_assembly in {gapic_assemblies}; do
+        tar -xzpf $gapic_assembly -C {tmp_dir_path}
+    done
+    {postprocessor} {output_dir_path} {synth_script} {tmp_dir_path} {formatter}
+    pushd {output_dir_path}/..
+    tar -zchpf {output_pkg_basename} {output_dir_name}
+    popd
+    """.format(
+        gapic_assemblies = " ".join(["'%s'" % d.path for d in gapic_assemblies]),
+        output_dir_path = output_dir_path,
+        output_pkg_basename = output_pkg.basename,
+        postprocessor = postprocessor.path,
+        output_dir_name = output_dir_name,
+        tmp_dir_path = tmp_dir_path,
+        tmp_dir_name = tmp_dir_name,
+        synth_script = synth_script.path,
+        formatter = formatter.path
+    )
+
+    ctx.actions.run_shell(
+        inputs = gapic_assemblies + [synth_script],
+        tools = [postprocessor, formatter],
+        command = script,
+        outputs = [output_pkg],
+    )
+
+postprocessed_gapic_pkg = rule(
+    attrs = {
+        "srcs": attr.label_list(
+            allow_files = True,
+            mandatory = False
+        ),
+        "synth_script": attr.label(
+            mandatory = True,
+            allow_single_file = True
+        ),
+        "postprocessor": attr.label(
+            default = Label("//synth:gapic_postprocessor"),
+            executable = True,
+            cfg = "host",
+        ),
+        "formatter": attr.label(
+            executable = True,
+            cfg = "host",
+        )
+
+    },
+    outputs = {"pkg": "%{name}.tar.gz"},
+    implementation = _postprocessed_gapic_pkg_impl,
+)
+
+def java_synth_pkg(name, synth_script, srcs, visibility = None, **kwargs):
+    postprocessed_gapic_pkg(
+        name = name,
+        synth_script = synth_script,
+        srcs = srcs,
+        formatter = Label("//synth:google_java_format_binary"),
+        visibility = visibility,
+    )

synth/gapic_postprocessor.py

@@ -0,0 +1,14 @@
+import os
+import sys
+
+if __name__ == '__main__':
+    os.environ['PYTHONNOUSERSITE'] = 'True'
+
+    sys.argv[2] = os.path.abspath(sys.argv[2])
+    sys.argv[3] = os.path.abspath(sys.argv[3])
+    sys.argv[4] = os.path.abspath(sys.argv[4])
+    os.chdir(sys.argv[1])
+
+    args = [sys.executable] + sys.argv[2:]
+
+    os.execv(args[0], args)

synth/synth_repositories.bzl

@@ -0,0 +1,33 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:jvm.bzl", "jvm_maven_import_external")
+load("@rules_python//python:pip.bzl", "pip_install")
+
+def synth():
+    _maybe(
+        pip_install,
+        name = "synth_pip_deps",
+        requirements = "@rules_gapic//synth:requirements.txt",
+    )
+
+    _maybe(
+        http_archive,
+        name = "bazel_skylib",
+        strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b",
+        urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
+    )
+
+    _maybe(
+        jvm_maven_import_external,
+        name = "google_java_format_all_deps",
+        artifact = "com.google.googlejavaformat:google-java-format:jar:all-deps:1.7" ,
+        server_urls = ["https://repo.maven.apache.org/maven2/", "http://repo1.maven.org/maven2/"],
+        licenses = ["notice", "reciprocal"]
+    )
+
+def _maybe(repo_rule, name, strip_repo_prefix = "", **kwargs):
+    if not name.startswith(strip_repo_prefix):
+        return
+    repo_name = name[len(strip_repo_prefix):]
+    if repo_name in native.existing_rules():
+        return
+    repo_rule(name = repo_name, **kwargs)

synth/synthtool/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synthtool.transforms import move, replace, dont_overwrite
+
+copy = move
+
+__all__ = ["copy", "move", "replace", "dont_overwrite"]

synth/synthtool/_tracked_paths.py

@@ -0,0 +1,39 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tracked paths.
+
+This is a bit of a hack.
+"""
+
+import pathlib
+
+
+_tracked_paths = []
+
+
+def add(path):
+    _tracked_paths.append(pathlib.Path(path))
+    # Reverse sort the list, so that the deepest paths get matched first.
+    _tracked_paths.sort(key=lambda s: -len(str(s)))
+
+
+def relativize(path):
+    path = pathlib.Path(path)
+    for tracked_path in _tracked_paths:
+        try:
+            return path.relative_to(tracked_path)
+        except ValueError:
+            pass
+    raise ValueError(f"The root for {path} is not tracked.")

synth/synthtool/gcp/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import gapic_bazel
+
+GAPICBazel = gapic_bazel.GAPICBazel
+
+__all__ = (
+    "GAPICBazel",
+)

synth/synthtool/gcp/gapic_bazel.py

@@ -0,0 +1,43 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pathlib import Path
+import sys
+
+class GAPICBazel:
+    def __init__(self):
+        pass
+
+    def py_library(self, **kwargs) -> Path:
+        return self._generate_code(**kwargs)
+
+    def go_library(self, **kwargs) -> Path:
+        return self._generate_code(**kwargs)
+
+    def node_library(self, **kwargs) -> Path:
+        return self._generate_code(**kwargs)
+
+    def csharp_library(self, **kwargs) -> Path:
+        return self._generate_code(**kwargs)
+
+    def php_library(self, **kwargs) -> Path:
+        return self._generate_code(**kwargs)
+
+    def java_library(self, **kwargs) -> Path:
+        return self._generate_code(**kwargs)
+
+    def ruby_library(self, **kwargs) -> Path:
+        return self._generate_code(**kwargs)
+
+    def _generate_code(self, **kwargs):
+        return Path(sys.argv[1])