ci: add dependency list completeness check (#490)

Source-Author: Stephanie Wang <stephaniewang526@users.noreply.github.com>
Source-Date: Mon Apr 13 18:30:27 2020 -0400
Source-Repo: googleapis/synthtool
Source-Sha: 3df869dd6eb546ef13beeb7a9efa6ee0226afafd
Source-Link: googleapis/synthtool@3df869d

.kokoro/common.sh

@@ -52,3 +52,8 @@ function retry_with_backoff {
 
     return $exit_code
 }
+
+## Helper functionss
+function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
+function msg() { println "$*" >&2; }
+function println() { printf '%s\n' "$(now) $*"; }

.kokoro/dependencies.sh

@@ -36,3 +36,51 @@ retry_with_backoff 3 10 \
     -Dclirr.skip=true
 
 mvn -B dependency:analyze -DfailOnWarning=true
+
+echo "****************** DEPENDENCY LIST COMPLETENESS CHECK *******************"
+## Run dependency list completeness check
+function completenessCheck() {
+  # Output dep list with compile scope generated using the original pom
+  msg "Generating dependency list using original pom..."
+  mvn dependency:list -f pom.xml -Dsort=true | grep '\[INFO]    .*:.*:.*:.*:.*' | grep -v ':test$' >.org-list.txt
+
+  # Output dep list generated using the flattened pom (test scope deps are ommitted)
+  msg "Generating dependency list using flattened pom..."
+  mvn dependency:list -f .flattened-pom.xml -Dsort=true | grep '\[INFO]    .*:.*:.*:.*:.*' >.new-list.txt
+
+  # Compare two dependency lists
+  msg "Comparing dependency lists..."
+  diff .org-list.txt .new-list.txt >.diff.txt
+  if [[ $? == 0 ]]
+    then
+      msg "Success. No diff!"
+  else
+    msg "Diff found. See below: "
+    msg "You can also check .diff.txt file located in $1."
+    cat .diff.txt
+    return 1
+  fi
+}
+
+# Allow failures to continue running the script
+set +e
+
+error_count=0
+for path in $(find -name ".flattened-pom.xml")
+do
+  # Check flattened pom in each dir that contains it for completeness
+  dir=$(dirname "$path")
+  pushd "$dir"
+  completenessCheck "$dir"
+  error_count=$(($error_count + $?))
+  popd
+done
+
+if [[ $error_count == 0 ]]
+then
+  msg "All checks passed."
+  exit 0
+else
+  msg "Errors found. See log statements above."
+  exit 1
+fi

synth.metadata

@@ -11,8 +11,7 @@
       "git": {
         "name": "synthtool",
         "remote": "https://github.com/googleapis/synthtool.git",
-        "sha": "c7e0e517d7f46f77bebd27da2e5afcaa6eee7e25",
-        "log": "c7e0e517d7f46f77bebd27da2e5afcaa6eee7e25\nbuild(java): fix nightly integration test config to run integrations (#465)\n\nThis was only running the units.\nbd69a2aa7b70875f3c988e269706b22fefbef40e\nbuild(java): fix retry_with_backoff when -e option set (#475)\n\n\nd9b173c427bfa0c6cca818233562e7e8841a357c\nfix: record version of working repo in synth.metadata (#473)\n\nPartial revert of b37cf74d12e9a42b9de9e61a4f26133d7cd9c168.\nf73a541770d95a609e5be6bf6b3b220d17cefcbe\nfeat(discogapic): allow local discovery-artifact-manager (#474)\n\n\n8cf0f5d93a70c3dcb0b4999d3152c46d4d9264bf\ndoc: describe the Autosynth & Synthtool protocol (#472)\n\n* doc: describe the Autosynth & Synthtool protocol\n\n* Accommodate review comments.\n980baaa738a1ad8fa02b4fdbd56be075ee77ece5\nfix: pin sphinx to <3.0.0 as new version causes new error (#471)\n\nThe error `toctree contains reference to document changlelog that doesn't have a title: no link will be generated` occurs as of 3.0.0. Pinning to 2.x until we address the docs build issue.\n\nTowards #470\n\nI did this manually for python-datastore https://github.com/googleapis/python-datastore/pull/22\n928b2998ac5023e7c7e254ab935f9ef022455aad\nchore(deps): update dependency com.google.cloud.samples:shared-configuration to v1.0.15 (#466)\n\nCo-authored-by: Jeffrey Rennie <rennie@google.com>\n188f1b1d53181f739b98f8aa5d40cfe99eb90c47\nfix: allow local and external deps to be specified (#469)\n\nModify noxfile.py to allow local and external dependencies for\nsystem tests to be specified.\n1df68ed6735ddce6797d0f83641a731c3c3f75b4\nfix: apache license URL (#468)\n\n\nf4a59efa54808c4b958263de87bc666ce41e415f\nfeat: Add discogapic support for GAPICBazel generation (#459)\n\n* feat: Add discogapic support for GAPICBazel generation\n\n* reformat with black\n\n* Rename source repository variable\n\nCo-authored-by: Jeffrey Rennie <rennie@google.com>\n99820243d348191bc9c634f2b48ddf65096285ed\nfix: update template files for Node.js libraries (#463)\n\n\n3cbe6bcd5623139ac9834c43818424ddca5430cb\nfix(ruby): remove dead troubleshooting link from generated auth guide (#462)\n\n\na003d8655d3ebec2bbbd5fc3898e91e152265c67\ndocs: remove \"install stable\" instructions (#461)\n\nThe package hasn't been released to PyPI in some time\nf5e8c88d9870d8aa4eb43fa0b39f07e02bfbe4df\nchore(python): add license headers to config files; make small tweaks to templates (#458)\n\n\n"
+        "sha": "3df869dd6eb546ef13beeb7a9efa6ee0226afafd"
       }
     }
   ]