From 3ec38bd396a3cf2473cbf227b6ff5631603a9f7c Mon Sep 17 00:00:00 2001 From: eugene yokota Date: Thu, 29 Feb 2024 20:33:03 -0500 Subject: [PATCH] Improve Python indexing example (#227) **Problem** In the current example Python indexing only tracks 3rdparty libraries, but not Protobuf-generated sources. **Solution** This adjusts the scripts to index both 3rdparty pip targets and Protobuf-generated sources. --- .github/ci_scripts/integration_test.sh | 4 +--- .../jar_scanner/py_build_commands.py | 4 +++- .../wheel_scanner/py_build_commands.py | 9 +++++++-- .../wheel_scanner/wheel_scanner.bzl | 18 ++++++++++++------ .../create_lang_build_files/regenerate.sh | 16 ++++++++++++++++ .../regenerate_python_build_files.sh | 7 +++++-- example/com/example/BUILD.bazel | 2 +- example/com/example/hello.py | 1 + 8 files changed, 46 insertions(+), 15 deletions(-) create mode 100755 example/build_tools/lang_support/create_lang_build_files/regenerate.sh diff --git a/.github/ci_scripts/integration_test.sh b/.github/ci_scripts/integration_test.sh index cb41617f..d5eb76b9 100755 --- a/.github/ci_scripts/integration_test.sh +++ b/.github/ci_scripts/integration_test.sh @@ -5,9 +5,7 @@ set -o nounset # abort on unbound variable set -o pipefail # don't hide errors within pipes cd example -TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/delete_build_files.sh -TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh -TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh +TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate.sh bazel test ... changes=$(git diff --name-only --diff-filter=ACMRT | xargs) diff --git a/example/build_tools/bazel_rules/jar_scanner/py_build_commands.py b/example/build_tools/bazel_rules/jar_scanner/py_build_commands.py index fafa73b7..392dba0f 100644 --- a/example/build_tools/bazel_rules/jar_scanner/py_build_commands.py +++ b/example/build_tools/bazel_rules/jar_scanner/py_build_commands.py @@ -21,12 +21,14 @@ START_BATCH=$(date +%s) set +e +set -x bazel build {targets} \ --aspects build_tools/bazel_rules/jar_scanner/rule.bzl%jar_scanner_aspect \ --output_groups=+jar_scanner_out \ - --override_repository=external_build_tooling_gen={bzl_gen_build_path} \ + --override_repository=external_build_tooling_gen=${{BZL_GEN_BUILD_TOOLS_PATH}} \ --show_result=1000000 2> /tmp/cmd_out RET=$? +set +x if [ "$RET" != "0" ]; then cat /tmp/cmd_out exit $RET diff --git a/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py b/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py index 014408d5..fcd49923 100644 --- a/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py +++ b/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py @@ -23,12 +23,14 @@ set +e +set -x bazel build {targets} \ --aspects build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl%wheel_scanner_aspect \ --output_groups=+wheel_scanner_out \ --override_repository=external_build_tooling_gen=${{BZL_GEN_BUILD_TOOLS_PATH}} \ --show_result=1000000 2> /tmp/cmd_out RET=$? +set +x if [ "$RET" != "0" ]; then cat /tmp/cmd_out exit $RET @@ -56,12 +58,15 @@ def __transform_target(t): - return "@%s//:pkg" % (t.lstrip("//external:")) + if t.startswith("//external:"): + return "@%s//:pkg" % (t.lstrip("//external:")) + else: + return t def write_command(file, output_idx, command_list): file.write( TEMPLATE.format( - targets=" ".join([t for t in command_list if t.endswith("pkg")]), + targets=" ".join([t for t in command_list]), output_idx=output_idx, target_count=len(command_list), ) diff --git a/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl b/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl index d7cb9f12..2cd9f73d 100644 --- a/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl +++ b/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl @@ -1,7 +1,4 @@ def _wheel_scanner_impl(target, ctx): - if not target.label.workspace_name.startswith("rules_python"): - return [] - # Make sure the rule has a srcs attribute. out_content = ctx.actions.declare_file("%s_wheel_scanner.json" % (target.label.name)) @@ -21,6 +18,14 @@ def _wheel_scanner_impl(target, ctx): fail("Didn't have workspace prefix") all_py_relative_paths.append(path[len(workspace_root) + 1:]) all_py_files.append(file) + elif ctx.rule.kind == "py_proto_library": + info = target[DefaultInfo] + last_file = info.files.to_list()[-1] + parts = last_file.path.split("/bin/", 1) + workspace_root = "./{}/bin".format(parts[0]) + relative_path = parts[1] + all_py_relative_paths.append(relative_path) + all_py_files.append(last_file) input_files = ctx.actions.declare_file("%s_wheel_scanner_input_files.txt" % (target.label.name)) ctx.actions.write( @@ -33,10 +38,11 @@ def _wheel_scanner_impl(target, ctx): args.add("--label-or-repo-path") args.add(str(target.label)) - args.add("--import-path-relative-from") - args.add("%s/" % (target.label.workspace_root)) + if workspace_root != "": + args.add("--import-path-relative-from") + args.add("%s/" % (workspace_root)) args.add("--working-directory") - args.add(target.label.workspace_root) + args.add(workspace_root) args.add("--relative-input-paths") args.add("@%s" % input_files.path) args.add("--output") diff --git a/example/build_tools/lang_support/create_lang_build_files/regenerate.sh b/example/build_tools/lang_support/create_lang_build_files/regenerate.sh new file mode 100755 index 00000000..581a47c7 --- /dev/null +++ b/example/build_tools/lang_support/create_lang_build_files/regenerate.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +if [ -n "${INVOKED_VIA_BAZEL:-}" ]; then + REPO_ROOT="$BUILD_WORKING_DIRECTORY" +else + REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )" +fi + +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/delete_build_files.sh" +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh" +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh" +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_jvm_build_files.sh" diff --git a/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh b/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh index c99d6548..0a72ed86 100755 --- a/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh +++ b/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh @@ -14,7 +14,10 @@ GEN_FLAVOR=python source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh" set -x -bazel query '@pip//...' | grep '@pip' > $TMP_WORKING_STATE/external_targets +bazel query '@pip//...' | grep "@pip.*:pkg" > $TMP_WORKING_STATE/external_targets + +bazel query 'kind("py", com/...)' > /dev/null +cat $OUTPUT_BASE/command.log | grep '//' >> $TMP_WORKING_STATE/external_targets CACHE_KEY="$(generate_cache_key $TMP_WORKING_STATE/external_targets $REPO_ROOT/WORKSPACE $REPO_ROOT/requirements_lock_3_9.txt)" rm -rf $TMP_WORKING_STATE/external_files &> /dev/null || true @@ -23,7 +26,7 @@ rm -rf $TMP_WORKING_STATE/external_files &> /dev/null || true # if [ ! -d $TMP_WORKING_STATE/external_files ]; then # log "cache wasn't ready or populated" bazel run build_tools/bazel_rules/wheel_scanner:py_build_commands -- $TMP_WORKING_STATE/external_targets $TMP_WORKING_STATE/external_targets_commands.sh - chmod +x ${TMP_WORKING_STATE}/external_targets_commands.sh + chmod +x "${TMP_WORKING_STATE}/external_targets_commands.sh" mkdir -p $TMP_WORKING_STATE/external_files if [[ -d $TOOLING_WORKING_DIRECTORY ]]; then BZL_GEN_BUILD_TOOLS_PATH=$TOOLING_WORKING_DIRECTORY ${TMP_WORKING_STATE}/external_targets_commands.sh diff --git a/example/com/example/BUILD.bazel b/example/com/example/BUILD.bazel index cde1ce67..2548d7e8 100644 --- a/example/com/example/BUILD.bazel +++ b/example/com/example/BUILD.bazel @@ -6,7 +6,7 @@ java_proto_library(name='aa_proto_java', deps=[':aa_proto'], visibility=['//visi py_proto_library(name='aa_proto_py', deps=[':aa_proto'], visibility=['//visibility:public']) # ---- END BZL_GEN_BUILD_GENERATED_CODE ---- no_hash # ---- BEGIN BZL_GEN_BUILD_GENERATED_CODE ---- no_hash -py_library(name='hello', srcs=['hello.py'], deps=['@@rules_python~0.24.0~pip~pip_39_pandas//:pkg'], visibility=['//visibility:public']) +py_library(name='hello', srcs=['hello.py'], deps=['@@//com/example:aa_proto_py', '@@rules_python~0.24.0~pip~pip_39_pandas//:pkg'], visibility=['//visibility:public']) # ---- END BZL_GEN_BUILD_GENERATED_CODE ---- no_hash # ---- BEGIN BZL_GEN_BUILD_GENERATED_CODE ---- no_hash py_test(name='hello_test', srcs=['hello_test.py'], deps=['//com/example:hello'], visibility=['//visibility:public']) diff --git a/example/com/example/hello.py b/example/com/example/hello.py index 85632f20..2d4c6fea 100644 --- a/example/com/example/hello.py +++ b/example/com/example/hello.py @@ -1,4 +1,5 @@ import pandas as pd +from com.example.aa_pb2 import A FOO = [""]