ci(all): only run continuous tests on changed modules (#3527)

Fixes: #3454 and #3134
[One pager summary](https://docs.google.com/document/d/1ZWLb85kkjcnHwUOqnJCexEx_GNf-UVSEAbdfL8A5NGo/edit?resourcekey=0-LO2alh3JcP1DEWFJLshMUA#heading=h.tdx2ho1d7dsd)

**Changes**
Job | Current | Changes
-- | -- | --
Presubmit | Runs --short tests only<br /> Tests all submodules | No change
Continuous | Runs on all submodules | Runs only the tests in submodules changed by the PR<br /> Runs the root test
Nightly | None | Runs on all submodules<br /> [Optional] can configure a job `nightly/module_name` to only run the tests inside that module, skipping the root test.

**New Jobs**
- [Nightly](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod:cloud-devrel%2Fclient-libraries%2Fgo%2Fgoogle-cloud-go%2Fnightly%2Fmaster?search_pattern=google-cloud-go&include_inactive_projects=false), s.t. all submodules tests still run during the nightly check.
- Temporary [Testing](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod:cloud-devrel%2Fclient-libraries%2Fgo%2Fgoogle-cloud-go%2Fcontinuous%2Ftesting) job which points to a `test-submodules` branch in this repo used to test this new continuous testing logic. 

**Testing this implementation:** 
I tested using `git diff HEAD~1` in this [closed branch](#3513):
✅  I changed a file inside /internal => ci tested [all submodules](https://fusion2.corp.google.com/invocations/3ed3c0c2-db12-4d0e-97cd-5d43210a5eb1/log)
✅  I changed a file inside /spanner => ci tested [only spanner](https://fusion2.corp.google.com/invocations/f0ff1f3d-e43f-4e70-9ad1-45534cb79997/targets/cloud-devrel%2Fclient-libraries%2Fgo%2Fgoogle-cloud-go%2Fcontinuous%2Ftesting/log)
✅  I changed file inside /logging => ci tested [only logging](https://fusion2.corp.google.com/invocations/4b9a5206-720f-4837-9cdb-c4b64b25483e/targets/cloud-devrel%2Fclient-libraries%2Fgo%2Fgoogle-cloud-go%2Fcontinuous%2Ftesting/log)
✅  I changed files in both bigtable & logging => ci tested [2 submodules](https://fusion2.corp.google.com/invocations/0fddcec3-de4d-4710-97ff-9340cffefaf8/targets/cloud-devrel%2Fclient-libraries%2Fgo%2Fgoogle-cloud-go%2Fcontinuous%2Ftesting/log)

**Open questions:** 
- What are other directories in this repo that, upon being changed in a PR, should trigger CI on ALL submodules? Other than `internal`?
- Should we keep the `perf-bench` tests e.g. [pubsub](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod:cloud-devrel%2Fclient-libraries%2Fgo%2Fgoogle-cloud-go%2Fcontinuous%2Fpubsub_perf_bench?search_pattern=google-cloud-go&include_inactive_projects=false) within the continuous job configs, or should I move them over to nightly? We'll save a bit of compute as these tests are already scheduled to run nightly.

internal/kokoro/continuous.sh

@@ -13,10 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+##
+# continuous.sh
+# Runs CI checks for entire repository.
+#
+# Jobs types
+#
+# Continuous: Runs root tests & tests in submodules changed by a PR. Triggered by PR merges.
+# Nightly: Runs root tests & tests in all modules. Triggered nightly.
+# Nightly/$MODULE: Runs tests in a specified module. Triggered nightly.
+##
+
 export GOOGLE_APPLICATION_CREDENTIALS=$KOKORO_KEYSTORE_DIR/72523_go_integration_service_account
 # Removing the GCLOUD_TESTS_GOLANG_PROJECT_ID setting may make some integration
 # tests (like profiler's) silently skipped, so make sure you know what you are
 # doing when changing / removing the next line.
+
 export GCLOUD_TESTS_GOLANG_PROJECT_ID=dulcet-port-762
 export GCLOUD_TESTS_GOLANG_KEY=$GOOGLE_APPLICATION_CREDENTIALS
 export GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID=gcloud-golang-firestore-tests
@@ -53,23 +65,79 @@ try3 go mod download
 go install github.com/jstemmer/go-junit-report
 ./internal/kokoro/vet.sh
 
+# runTests runs all tests in the current directory.
+# If a PATH argument is specified, it runs `go test [PATH]`.
+runDirectoryTests() {
+  go test -race -v -timeout 45m "${1:-./...}" 2>&1 \
+    | tee sponge_log.log
+  # Takes the kokoro output log (raw stdout) and creates a machine-parseable
+  # xUnit XML file.
+  cat sponge_log.log \
+    | go-junit-report -set-exit-code > sponge_log.xml
+  # Add the exit codes together so we exit non-zero if any module fails.
+  exit_code=$(($exit_code + $?))
+}
+
+# testAllModules runs all modules tests.
+testAllModules() {
+  echo "Testing all modules"
+  for i in $(find . -name go.mod); do
+    pushd "$(dirname "$i")" > /dev/null;
+      runDirectoryTests
+    popd > /dev/null;
+  done
+}
+
+# testChangedModules runs tests in changed modules only.
+testChangedModules() {
+  for d in $CHANGED_DIRS; do
+    goDirectories="$(find "$d" -name "*.go" -printf "%h\n" | sort -u)"
+    if [[ -n "$goDirectories" ]]; then
+      for gd in $goDirectories; do
+        pushd "$gd" > /dev/null;
+          runDirectoryTests .
+        popd > /dev/null;
+      done
+    fi
+  done
+}
+
 set +e # Run all tests, don't stop after the first failure.
 exit_code=0
-# Run tests and tee output to log file, to be pushed to GCS as artifact.
-for i in `find . -name go.mod`; do
-  pushd `dirname $i`;
-    go test -race -v -timeout 45m ./... 2>&1 \
-      | tee sponge_log.log
-    # Takes the kokoro output log (raw stdout) and creates a machine-parseable
-    # xUnit XML file.
-    cat sponge_log.log \
-      | go-junit-report -set-exit-code > sponge_log.xml
-    # Add the exit codes together so we exit non-zero if any module fails.
-    exit_code=$(($exit_code + $?))
-  popd;
-done
-
-if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
+
+if [[ $KOKORO_JOB_NAME == *"continuous"* ]]; then
+  # Continuous jobs only run root tests & tests in submodules changed by the PR
+  SIGNIFICANT_CHANGES=$(git --no-pager diff --name-only master..HEAD | grep -Ev '(\.md$|^\.github)' || true)
+  # CHANGED_DIRS is the list of significant top-level directories that changed,
+  # but weren't deleted by the current PR. CHANGED_DIRS will be empty when run on master.
+  CHANGED_DIRS=$(echo "$SIGNIFICANT_CHANGES" | tr ' ' '\n' | grep "/" | cut -d/ -f1 | sort -u | tr '\n' ' ' | xargs ls -d 2>/dev/null || true)
+  # If PR changes affect all submodules, then run all tests
+  if echo "$SIGNIFICANT_CHANGES" | tr ' ' '\n' | grep "^go.mod$" || [[ $CHANGED_DIRS =~ "internal" ]]; then
+    testAllModules
+  else
+    runDirectoryTests . # Always run base tests
+    echo "Running tests only in changed submodules: $CHANGED_DIRS"
+    testChangedModules
+  fi
+elif [[ $KOKORO_JOB_NAME == *"nightly"* ]]; then
+  SUBMODULE_NAME="${KOKORO_JOB_NAME#*"nightly/"}"
+  if [[ -n $SUBMODULE_NAME ]] && [[ -d "./$SUBMODULE_NAME" ]]; then
+    # Only run tests in the submodule designated in the Kokoro job name.
+    # Expected format example: ...google-cloud-go/nightly/logging.
+    runDirectoryTests . # Always run base tests
+    echo "Running tests in one submodule: $SUBMODULE_NAME"
+    pushd $SUBMODULE_NAME > /dev/null;
+      runDirectoryTests
+    popd > /dev/null
+  else
+    # Run all tests if it is a regular nightly job
+    testAllModules
+  fi
+else
+  testAllModules
+fi
+
+if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then
   chmod +x $KOKORO_GFILE_DIR/linux_amd64/buildcop
   $KOKORO_GFILE_DIR/linux_amd64/buildcop -logs_dir=$GOCLOUD_HOME \
     -repo=googleapis/google-cloud-go \