ci: filter presubmit tests to run only changed directories (#5089)

* Filters presubmit tests based on changed directories
* Adds emulator tests to be run when all module tests run, such as during regular nightly jobs

internal/kokoro/continuous.sh

@@ -71,7 +71,7 @@ try3() { eval "$*" || eval "$*" || eval "$*"; }
 try3 go mod download
 ./internal/kokoro/vet.sh
 
-# runTests runs all tests in the current directory.
+# runDirectoryTests runs all tests in the current directory.
 # If a PATH argument is specified, it runs `go test [PATH]`.
 runDirectoryTests() {
   go test -race -v -timeout 45m "${1:-./...}" 2>&1 \
@@ -84,12 +84,27 @@ runDirectoryTests() {
   exit_code=$(($exit_code + $?))
 }
 
-# testAllModules runs all modules tests.
+# runEmulatorTests runs emulator tests in the current directory.
+runEmulatorTests() {
+  if [ -f "emulator_test.sh" ]; then
+    ./emulator_test.sh
+  fi
+  # Takes the kokoro output log (raw stdout) and creates a machine-parseable
+  # xUnit XML file.
+  cat sponge_log.log \
+    | go-junit-report -set-exit-code > sponge_log.xml
+  # Add the exit codes together so we exit non-zero if any module fails.
+  exit_code=$(($exit_code + $?))
+}
+
+# testAllModules runs all modules' tests, including emulator tests.
 testAllModules() {
   echo "Testing all modules"
   for i in $(find . -name go.mod); do
     pushd "$(dirname "$i")" > /dev/null;
       runDirectoryTests
+      # Run integration tests against an emulator.
+      runEmulatorTests
     popd > /dev/null;
   done
 }

internal/kokoro/presubmit.sh

@@ -45,27 +45,44 @@ try3 go mod download
 
 set +e # Run all tests, don't stop after the first failure.
 exit_code=0
-# Run tests and tee output to log file, to be pushed to GCS as artifact.
-for i in `find . -name go.mod`; do
-  pushd `dirname $i`;
-    if [ -z ${RUN_INTEGRATION_TESTS} ]; then
-      go test -race -v -timeout 15m -short ./... 2>&1 \
-      | tee sponge_log.log
-    else
-      go test -race -v -timeout 45m ./... 2>&1 \
-      | tee sponge_log.log
-    fi
-    # Run integration tests against an emulator.
-    if [ -f "emulator_test.sh" ]; then
-      ./emulator_test.sh
-    fi
-    # Takes the kokoro output log (raw stdout) and creates a machine-parseable
-    # xUnit XML file.
-    cat sponge_log.log \
-      | go-junit-report -set-exit-code > sponge_log.xml
-    # Add the exit codes together so we exit non-zero if any module fails.
-    exit_code=$(($exit_code + $?))
-  popd;
+
+# Run tests in the current directory and tee output to log file,
+# to be pushed to GCS as artifact.
+runPresubmitTests() {
+  if [ -z ${RUN_INTEGRATION_TESTS} ]; then
+    go test -race -v -timeout 15m -short ./... 2>&1 \
+    | tee sponge_log.log
+  else
+    go test -race -v -timeout 45m ./... 2>&1 \
+    | tee sponge_log.log
+  fi
+
+  # Run integration tests against an emulator.
+  if [ -f "emulator_test.sh" ]; then
+    ./emulator_test.sh
+  fi
+  # Takes the kokoro output log (raw stdout) and creates a machine-parseable
+  # xUnit XML file.
+  cat sponge_log.log \
+    | go-junit-report -set-exit-code > sponge_log.xml
+  # Add the exit codes together so we exit non-zero if any module fails.
+  exit_code=$(($exit_code + $?))
+}
+
+SIGNIFICANT_CHANGES=$(git --no-pager diff --name-only $KOKORO_GIT_COMMIT^..$KOKORO_GIT_COMMIT \
+  | grep -Ev '(\.md$|^\.github)' || true)
+# CHANGED_DIRS is the list of significant top-level directories that changed,
+# but weren't deleted by the current PR. CHANGED_DIRS will be empty when run on master.
+CHANGED_DIRS=$(echo "$SIGNIFICANT_CHANGES" | tr ' ' '\n' | grep "/" | cut -d/ -f1 | sort -u \
+  | tr '\n' ' ' | xargs ls -d 2>/dev/null || true)
+
+echo "Running tests only in changed submodules: $CHANGED_DIRS"
+for d in $CHANGED_DIRS; do
+  for i in `find "$d" -name go.mod`; do
+    pushd `dirname $i`;
+      runPresubmitTests
+    popd;
+  done
 done
 
 exit $exit_code