Improve Python indexing example

**Problem**
In the current example Python indexing only tracks
3rdparty libraries, but not Protobuf-generated sources.

**Solution**
This adjusts the scripts to index both 3rdparty pip targets
and Protobuf-generated sources.

.github/ci_scripts/integration_test.sh

@@ -5,9 +5,7 @@ set -o nounset  # abort on unbound variable
 set -o pipefail # don't hide errors within pipes
 
 cd example
-TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/delete_build_files.sh
-TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh
-TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh
+TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate.sh
 bazel test ...
 
 changes=$(git diff --name-only --diff-filter=ACMRT | xargs)

example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py

@@ -23,12 +23,14 @@
 
 
 set +e
+set -x
 bazel build {targets} \
   --aspects build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl%wheel_scanner_aspect \
   --output_groups=+wheel_scanner_out \
   --override_repository=external_build_tooling_gen=${{BZL_GEN_BUILD_TOOLS_PATH}} \
   --show_result=1000000 2> /tmp/cmd_out
 RET=$?
+set +x
 if [ "$RET" != "0" ]; then
     cat /tmp/cmd_out
     exit $RET
@@ -56,12 +58,15 @@
 
 
 def __transform_target(t):
-    return "@%s//:pkg" % (t.lstrip("//external:"))
+    if t.startswith("//external:"):
+        return "@%s//:pkg" % (t.lstrip("//external:"))
+    else:
+        return t
 
 def write_command(file, output_idx, command_list):
     file.write(
         TEMPLATE.format(
-            targets=" ".join([t for t in command_list if t.endswith("pkg")]),
+            targets=" ".join([t for t in command_list]),
             output_idx=output_idx,
             target_count=len(command_list),
         )

example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl

@@ -1,7 +1,4 @@
 def _wheel_scanner_impl(target, ctx):
-    if not target.label.workspace_name.startswith("rules_python"):
-        return []
-
     # Make sure the rule has a srcs attribute.
     out_content = ctx.actions.declare_file("%s_wheel_scanner.json" % (target.label.name))
 
@@ -21,6 +18,14 @@ def _wheel_scanner_impl(target, ctx):
                     fail("Didn't have workspace prefix")
                 all_py_relative_paths.append(path[len(workspace_root) + 1:])
                 all_py_files.append(file)
+    elif ctx.rule.kind == "py_proto_library":
+        info = target[DefaultInfo]
+        first_file = info.files.to_list()[0]
+        parts = first_file.path.split("/bin/", 1)
+        workspace_root = "./{}/bin".format(parts[0])
+        relative_path = parts[1]
+        all_py_relative_paths.append(relative_path)
+        all_py_files.append(first_file)
 
     input_files = ctx.actions.declare_file("%s_wheel_scanner_input_files.txt" % (target.label.name))
     ctx.actions.write(
@@ -33,10 +38,11 @@ def _wheel_scanner_impl(target, ctx):
     args.add("--label-or-repo-path")
     args.add(str(target.label))
 
-    args.add("--import-path-relative-from")
-    args.add("%s/" % (target.label.workspace_root))
+    if workspace_root != "":
+        args.add("--import-path-relative-from")
+        args.add("%s/" % (workspace_root))
     args.add("--working-directory")
-    args.add(target.label.workspace_root)
+    args.add(workspace_root)
     args.add("--relative-input-paths")
     args.add("@%s" % input_files.path)
     args.add("--output")

example/build_tools/lang_support/create_lang_build_files/regenerate.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+
+if [ -n "${INVOKED_VIA_BAZEL:-}" ]; then
+    REPO_ROOT="$BUILD_WORKING_DIRECTORY"
+else
+    REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )"
+fi
+
+source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/delete_build_files.sh"
+source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh"
+source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh"
+source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/regenerate_jvm_build_files.sh"

example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh

@@ -14,7 +14,10 @@ GEN_FLAVOR=python
 source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh"
 set -x
 
-bazel query '@pip//...' | grep '@pip' > $TMP_WORKING_STATE/external_targets
+bazel query '@pip//...' | grep "@pip.*:pkg" > $TMP_WORKING_STATE/external_targets
+
+bazel query 'kind("py", com/...)' > /dev/null
+cat $OUTPUT_BASE/command.log | grep '//' >> $TMP_WORKING_STATE/external_targets
 
 CACHE_KEY="$(generate_cache_key $TMP_WORKING_STATE/external_targets $REPO_ROOT/WORKSPACE $REPO_ROOT/requirements_lock_3_9.txt)"
 rm -rf $TMP_WORKING_STATE/external_files &> /dev/null || true
@@ -23,7 +26,7 @@ rm -rf $TMP_WORKING_STATE/external_files &> /dev/null || true
 # if [ ! -d $TMP_WORKING_STATE/external_files ]; then
     # log "cache wasn't ready or populated"
     bazel run build_tools/bazel_rules/wheel_scanner:py_build_commands -- $TMP_WORKING_STATE/external_targets $TMP_WORKING_STATE/external_targets_commands.sh
-    chmod +x ${TMP_WORKING_STATE}/external_targets_commands.sh
+    chmod +x "${TMP_WORKING_STATE}/external_targets_commands.sh"
     mkdir -p $TMP_WORKING_STATE/external_files
     if [[ -d $TOOLING_WORKING_DIRECTORY ]]; then
         BZL_GEN_BUILD_TOOLS_PATH=$TOOLING_WORKING_DIRECTORY ${TMP_WORKING_STATE}/external_targets_commands.sh

example/com/example/BUILD.bazel

@@ -6,7 +6,7 @@ java_proto_library(name='aa_proto_java', deps=[':aa_proto'], visibility=['//visi
 py_proto_library(name='aa_proto_py', deps=[':aa_proto'], visibility=['//visibility:public'])
 # ---- END BZL_GEN_BUILD_GENERATED_CODE ---- no_hash
 # ---- BEGIN BZL_GEN_BUILD_GENERATED_CODE ---- no_hash
-py_library(name='hello', srcs=['hello.py'], deps=['@@rules_python~0.24.0~pip~pip_39_pandas//:pkg'], visibility=['//visibility:public'])
+py_library(name='hello', srcs=['hello.py'], deps=['@@//com/example/foo:bb_proto_py', '@@//com/example/foo:cc_proto_py', '@@//com/example:aa_proto_py', '@@rules_python~0.24.0~pip~pip_39_pandas//:pkg'], visibility=['//visibility:public'])
 # ---- END BZL_GEN_BUILD_GENERATED_CODE ---- no_hash
 # ---- BEGIN BZL_GEN_BUILD_GENERATED_CODE ---- no_hash
 py_test(name='hello_test', srcs=['hello_test.py'], deps=['//com/example:hello'], visibility=['//visibility:public'])

example/com/example/hello.py

@@ -1,4 +1,5 @@
 import pandas as pd
+from com.example.aa_pb2 import A
 
 FOO = [""]