diff --git a/.github/ci_scripts/integration_test.sh b/.github/ci_scripts/integration_test.sh index cb41617f..d5eb76b9 100755 --- a/.github/ci_scripts/integration_test.sh +++ b/.github/ci_scripts/integration_test.sh @@ -5,9 +5,7 @@ set -o nounset # abort on unbound variable set -o pipefail # don't hide errors within pipes cd example -TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/delete_build_files.sh -TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh -TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh +TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate.sh bazel test ... changes=$(git diff --name-only --diff-filter=ACMRT | xargs) diff --git a/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py b/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py index 014408d5..fcd49923 100644 --- a/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py +++ b/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py @@ -23,12 +23,14 @@ set +e +set -x bazel build {targets} \ --aspects build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl%wheel_scanner_aspect \ --output_groups=+wheel_scanner_out \ --override_repository=external_build_tooling_gen=${{BZL_GEN_BUILD_TOOLS_PATH}} \ --show_result=1000000 2> /tmp/cmd_out RET=$? +set +x if [ "$RET" != "0" ]; then cat /tmp/cmd_out exit $RET @@ -56,12 +58,15 @@ def __transform_target(t): - return "@%s//:pkg" % (t.lstrip("//external:")) + if t.startswith("//external:"): + return "@%s//:pkg" % (t.lstrip("//external:")) + else: + return t def write_command(file, output_idx, command_list): file.write( TEMPLATE.format( - targets=" ".join([t for t in command_list if t.endswith("pkg")]), + targets=" ".join([t for t in command_list]), output_idx=output_idx, target_count=len(command_list), ) diff --git a/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl b/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl index d7cb9f12..bb73622b 100644 --- a/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl +++ b/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl @@ -1,7 +1,4 @@ def _wheel_scanner_impl(target, ctx): - if not target.label.workspace_name.startswith("rules_python"): - return [] - # Make sure the rule has a srcs attribute. out_content = ctx.actions.declare_file("%s_wheel_scanner.json" % (target.label.name)) @@ -21,6 +18,14 @@ def _wheel_scanner_impl(target, ctx): fail("Didn't have workspace prefix") all_py_relative_paths.append(path[len(workspace_root) + 1:]) all_py_files.append(file) + elif ctx.rule.kind == "py_proto_library": + info = target[DefaultInfo] + first_file = info.files.to_list()[0] + parts = first_file.path.split("/bin/", 1) + workspace_root = "./{}/bin".format(parts[0]) + relative_path = parts[1] + all_py_relative_paths.append(relative_path) + all_py_files.append(first_file) input_files = ctx.actions.declare_file("%s_wheel_scanner_input_files.txt" % (target.label.name)) ctx.actions.write( @@ -33,10 +38,11 @@ def _wheel_scanner_impl(target, ctx): args.add("--label-or-repo-path") args.add(str(target.label)) - args.add("--import-path-relative-from") - args.add("%s/" % (target.label.workspace_root)) + if workspace_root != "": + args.add("--import-path-relative-from") + args.add("%s/" % (workspace_root)) args.add("--working-directory") - args.add(target.label.workspace_root) + args.add(workspace_root) args.add("--relative-input-paths") args.add("@%s" % input_files.path) args.add("--output") diff --git a/example/build_tools/lang_support/create_lang_build_files/regenerate.sh b/example/build_tools/lang_support/create_lang_build_files/regenerate.sh new file mode 100755 index 00000000..581a47c7 --- /dev/null +++ b/example/build_tools/lang_support/create_lang_build_files/regenerate.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +if [ -n "${INVOKED_VIA_BAZEL:-}" ]; then + REPO_ROOT="$BUILD_WORKING_DIRECTORY" +else + REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )" +fi + +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/delete_build_files.sh" +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh" +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh" +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_jvm_build_files.sh" diff --git a/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh b/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh index c99d6548..0a72ed86 100755 --- a/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh +++ b/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh @@ -14,7 +14,10 @@ GEN_FLAVOR=python source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh" set -x -bazel query '@pip//...' | grep '@pip' > $TMP_WORKING_STATE/external_targets +bazel query '@pip//...' | grep "@pip.*:pkg" > $TMP_WORKING_STATE/external_targets + +bazel query 'kind("py", com/...)' > /dev/null +cat $OUTPUT_BASE/command.log | grep '//' >> $TMP_WORKING_STATE/external_targets CACHE_KEY="$(generate_cache_key $TMP_WORKING_STATE/external_targets $REPO_ROOT/WORKSPACE $REPO_ROOT/requirements_lock_3_9.txt)" rm -rf $TMP_WORKING_STATE/external_files &> /dev/null || true @@ -23,7 +26,7 @@ rm -rf $TMP_WORKING_STATE/external_files &> /dev/null || true # if [ ! -d $TMP_WORKING_STATE/external_files ]; then # log "cache wasn't ready or populated" bazel run build_tools/bazel_rules/wheel_scanner:py_build_commands -- $TMP_WORKING_STATE/external_targets $TMP_WORKING_STATE/external_targets_commands.sh - chmod +x ${TMP_WORKING_STATE}/external_targets_commands.sh + chmod +x "${TMP_WORKING_STATE}/external_targets_commands.sh" mkdir -p $TMP_WORKING_STATE/external_files if [[ -d $TOOLING_WORKING_DIRECTORY ]]; then BZL_GEN_BUILD_TOOLS_PATH=$TOOLING_WORKING_DIRECTORY ${TMP_WORKING_STATE}/external_targets_commands.sh diff --git a/example/com/example/BUILD.bazel b/example/com/example/BUILD.bazel index cde1ce67..a4790908 100644 --- a/example/com/example/BUILD.bazel +++ b/example/com/example/BUILD.bazel @@ -6,7 +6,7 @@ java_proto_library(name='aa_proto_java', deps=[':aa_proto'], visibility=['//visi py_proto_library(name='aa_proto_py', deps=[':aa_proto'], visibility=['//visibility:public']) # ---- END BZL_GEN_BUILD_GENERATED_CODE ---- no_hash # ---- BEGIN BZL_GEN_BUILD_GENERATED_CODE ---- no_hash -py_library(name='hello', srcs=['hello.py'], deps=['@@rules_python~0.24.0~pip~pip_39_pandas//:pkg'], visibility=['//visibility:public']) +py_library(name='hello', srcs=['hello.py'], deps=['@@//com/example/foo:bb_proto_py', '@@//com/example/foo:cc_proto_py', '@@//com/example:aa_proto_py', '@@rules_python~0.24.0~pip~pip_39_pandas//:pkg'], visibility=['//visibility:public']) # ---- END BZL_GEN_BUILD_GENERATED_CODE ---- no_hash # ---- BEGIN BZL_GEN_BUILD_GENERATED_CODE ---- no_hash py_test(name='hello_test', srcs=['hello_test.py'], deps=['//com/example:hello'], visibility=['//visibility:public']) diff --git a/example/com/example/hello.py b/example/com/example/hello.py index 85632f20..2d4c6fea 100644 --- a/example/com/example/hello.py +++ b/example/com/example/hello.py @@ -1,4 +1,5 @@ import pandas as pd +from com.example.aa_pb2 import A FOO = [""]