ci: changes triggered by synthtool (#274)

* ci: add dependency list completeness check (#490)

Source-Author: Stephanie Wang <stephaniewang526@users.noreply.github.com>
Source-Date: Mon Apr 13 18:30:27 2020 -0400
Source-Repo: googleapis/synthtool
Source-Sha: 3df869dd6eb546ef13beeb7a9efa6ee0226afafd
Source-Link: googleapis/synthtool@3df869d

* build(java): set GOOGLE_CLOUD_PROJECT env for samples/integration tests (#484)

* build(java): set GOOGLE_CLOUD_PROJECT env variable for samples/integration tests

* ci: use java-docs-samples-testing for sample tests

Source-Author: Jeff Ching <chingor@google.com>
Source-Date: Mon Apr 13 16:24:21 2020 -0700
Source-Repo: googleapis/synthtool
Source-Sha: 52638600f387deb98efb5f9c85fec39e82aa9052
Source-Link: googleapis/synthtool@5263860

* chore: resolve merge conflicts

.kokoro/common.sh

@@ -52,3 +52,8 @@ function retry_with_backoff {
 
     return $exit_code
 }
+
+## Helper functionss
+function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
+function msg() { println "$*" >&2; }
+function println() { printf '%s\n' "$(now) $*"; }

.kokoro/dependencies.sh

@@ -36,3 +36,51 @@ retry_with_backoff 3 10 \
     -Dclirr.skip=true
 
 mvn -B dependency:analyze -DfailOnWarning=true
+
+echo "****************** DEPENDENCY LIST COMPLETENESS CHECK *******************"
+## Run dependency list completeness check
+function completenessCheck() {
+  # Output dep list with compile scope generated using the original pom
+  msg "Generating dependency list using original pom..."
+  mvn dependency:list -f pom.xml -Dsort=true | grep '\[INFO]    .*:.*:.*:.*:.*' | grep -v ':test$' >.org-list.txt
+
+  # Output dep list generated using the flattened pom (test scope deps are ommitted)
+  msg "Generating dependency list using flattened pom..."
+  mvn dependency:list -f .flattened-pom.xml -Dsort=true | grep '\[INFO]    .*:.*:.*:.*:.*' >.new-list.txt
+
+  # Compare two dependency lists
+  msg "Comparing dependency lists..."
+  diff .org-list.txt .new-list.txt >.diff.txt
+  if [[ $? == 0 ]]
+    then
+      msg "Success. No diff!"
+  else
+    msg "Diff found. See below: "
+    msg "You can also check .diff.txt file located in $1."
+    cat .diff.txt
+    return 1
+  fi
+}
+
+# Allow failures to continue running the script
+set +e
+
+error_count=0
+for path in $(find -name ".flattened-pom.xml")
+do
+  # Check flattened pom in each dir that contains it for completeness
+  dir=$(dirname "$path")
+  pushd "$dir"
+  completenessCheck "$dir"
+  error_count=$(($error_count + $?))
+  popd
+done
+
+if [[ $error_count == 0 ]]
+then
+  msg "All checks passed."
+  exit 0
+else
+  msg "Errors found. See log statements above."
+  exit 1
+fi

.kokoro/nightly/integration.cfg

@@ -10,20 +10,25 @@ env_vars: {
   key: "JOB_TYPE"
   value: "integration"
 }
-
+# TODO: remove this after we've migrated all tests and scripts
 env_vars: {
   key: "GCLOUD_PROJECT"
   value: "gcloud-devel"
 }
 
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
 env_vars: {
   key: "ENABLE_BUILD_COP"
   value: "true"
 }
 
 env_vars: {
-    key: "GOOGLE_APPLICATION_CREDENTIALS"
-    value: "keystore/73713_java_it_service_account"
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "keystore/73713_java_it_service_account"
 }
 
 before_action {

.kokoro/nightly/samples.cfg

@@ -11,9 +11,15 @@ env_vars: {
   value: "samples"
 }
 
+# TODO: remove this after we've migrated all tests and scripts
 env_vars: {
   key: "GCLOUD_PROJECT"
-  value: "gcloud-devel"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "java-docs-samples-testing"
 }
 
 env_vars: {

.kokoro/presubmit/integration.cfg

@@ -11,14 +11,20 @@ env_vars: {
   value: "integration"
 }
 
+# TODO: remove this after we've migrated all tests and scripts
 env_vars: {
-    key: "GCLOUD_PROJECT"
-    value: "gcloud-devel"
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
 }
 
 env_vars: {
-    key: "GOOGLE_APPLICATION_CREDENTIALS"
-    value: "keystore/73713_java_it_service_account"
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "keystore/73713_java_it_service_account"
 }
 
 before_action {

.kokoro/presubmit/samples.cfg

@@ -11,14 +11,20 @@ env_vars: {
   value: "samples"
 }
 
+# TODO: remove this after we've migrated all tests and scripts
 env_vars: {
-    key: "GCLOUD_PROJECT"
-    value: "gcloud-devel"
+  key: "GCLOUD_PROJECT"
+  value: "java-docs-samples-testing"
 }
 
 env_vars: {
-    key: "GOOGLE_APPLICATION_CREDENTIALS"
-    value: "keystore/73713_java_it_service_account"
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "keystore/73713_java_it_service_account"
 }
 
 before_action {

pom.xml

@@ -70,7 +70,7 @@
     <grpc.version>1.28.1</grpc.version>
     <protobuf.version>3.11.4</protobuf.version>
     <junit.version>4.13</junit.version>
-    <guava.version>29.0-android</guava.version>
+    <guava.version>28.2-android</guava.version>
     <threeten.version>1.4.3</threeten.version>
     <javax.annotations.version>1.3.2</javax.annotations.version>
     <animal-sniffer.version>1.18</animal-sniffer.version>

samples/snippets/pom.xml

@@ -44,7 +44,7 @@
       <dependency>
         <groupId>com.google.cloud</groupId>
         <artifactId>libraries-bom</artifactId>
-        <version>5.1.0</version>
+        <version>4.4.1</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>

synth.metadata

@@ -1,5 +1,4 @@
 {
-  "updateTime": "2020-04-08T22:19:29.769274Z",
   "sources": [
     {
       "git": {
@@ -12,8 +11,8 @@
       "git": {
         "name": "synthtool",
         "remote": "https://github.com/googleapis/synthtool.git",
-        "sha": "c7e0e517d7f46f77bebd27da2e5afcaa6eee7e25",
-        "log": "c7e0e517d7f46f77bebd27da2e5afcaa6eee7e25\nbuild(java): fix nightly integration test config to run integrations (#465)\n\nThis was only running the units.\nbd69a2aa7b70875f3c988e269706b22fefbef40e\nbuild(java): fix retry_with_backoff when -e option set (#475)\n\n\nd9b173c427bfa0c6cca818233562e7e8841a357c\nfix: record version of working repo in synth.metadata (#473)\n\nPartial revert of b37cf74d12e9a42b9de9e61a4f26133d7cd9c168.\nf73a541770d95a609e5be6bf6b3b220d17cefcbe\nfeat(discogapic): allow local discovery-artifact-manager (#474)\n\n\n8cf0f5d93a70c3dcb0b4999d3152c46d4d9264bf\ndoc: describe the Autosynth & Synthtool protocol (#472)\n\n* doc: describe the Autosynth & Synthtool protocol\n\n* Accommodate review comments.\n980baaa738a1ad8fa02b4fdbd56be075ee77ece5\nfix: pin sphinx to <3.0.0 as new version causes new error (#471)\n\nThe error `toctree contains reference to document changlelog that doesn't have a title: no link will be generated` occurs as of 3.0.0. Pinning to 2.x until we address the docs build issue.\n\nTowards #470\n\nI did this manually for python-datastore https://github.com/googleapis/python-datastore/pull/22\n928b2998ac5023e7c7e254ab935f9ef022455aad\nchore(deps): update dependency com.google.cloud.samples:shared-configuration to v1.0.15 (#466)\n\nCo-authored-by: Jeffrey Rennie <rennie@google.com>\n188f1b1d53181f739b98f8aa5d40cfe99eb90c47\nfix: allow local and external deps to be specified (#469)\n\nModify noxfile.py to allow local and external dependencies for\nsystem tests to be specified.\n1df68ed6735ddce6797d0f83641a731c3c3f75b4\nfix: apache license URL (#468)\n\n\nf4a59efa54808c4b958263de87bc666ce41e415f\nfeat: Add discogapic support for GAPICBazel generation (#459)\n\n* feat: Add discogapic support for GAPICBazel generation\n\n* reformat with black\n\n* Rename source repository variable\n\nCo-authored-by: Jeffrey Rennie <rennie@google.com>\n99820243d348191bc9c634f2b48ddf65096285ed\nfix: update template files for Node.js libraries (#463)\n\n\n3cbe6bcd5623139ac9834c43818424ddca5430cb\nfix(ruby): remove dead troubleshooting link from generated auth guide (#462)\n\n\na003d8655d3ebec2bbbd5fc3898e91e152265c67\ndocs: remove \"install stable\" instructions (#461)\n\nThe package hasn't been released to PyPI in some time\nf5e8c88d9870d8aa4eb43fa0b39f07e02bfbe4df\nchore(python): add license headers to config files; make small tweaks to templates (#458)\n\n\n"
+        "sha": "85cbc7f954f4c4c3534f0caafbaed22a8f80f602",
+        "log": "85cbc7f954f4c4c3534f0caafbaed22a8f80f602\nfix: use https vcs install for python-test-utils (#492)\n\n\n52638600f387deb98efb5f9c85fec39e82aa9052\nbuild(java): set GOOGLE_CLOUD_PROJECT env for samples/integration tests (#484)\n\n* build(java): set GOOGLE_CLOUD_PROJECT env variable for samples/integration tests\n\n* ci: use java-docs-samples-testing for sample tests\n3df869dd6eb546ef13beeb7a9efa6ee0226afafd\nci: add dependency list completeness check (#490)\n\n\n682c0c37d1054966ca662a44259e96cc7aea4413\nbuild(nodejs): update lint ignore rules (#488)\n\n\n97c7ccfdceb927db1cbe6f3bb09616aa02bafd89\ndoc: document context-aware commit flags (#481)\n\nAlso, delete obsolete blurb about cleaning up old, dead files.\n\nCo-authored-by: Jeff Ching <chingor@google.com>\n8eff3790f88b50706a0c4b6a20b385f24e9ac4e7\nfeat: common postprocessing for node libraries (#485)\n\nCo-authored-by: Justin Beckwith <justin.beckwith@gmail.com>\n21c3b57ae54ae9db6a3a6b48b31c970c6ab56f19\nbuild(nodejs): remove unused codecov config (#486)\n\n\n6f32150677c9784f3c3a7e1949472bd29c9d72c5\nfix: installs test_utils from its common repo (#480)\n\n\n74ce986d3b5431eb66985e9a00c4eb45295a4020\nfix: stop recording update_time in synth.metadata (#478)\n\n\n7f8e62aa3edd225f76347a16f92e400661fdfb52\nchore(java): release-please only updates non maven versions in README (#476)\n\nPrevent release-please and synthtool from fighting over the released library version. Synthtool updates the install snippets from the samples pom.xml files so the bots fight if they are temporarily out of sync after a release.\nc7e0e517d7f46f77bebd27da2e5afcaa6eee7e25\nbuild(java): fix nightly integration test config to run integrations (#465)\n\nThis was only running the units.\nbd69a2aa7b70875f3c988e269706b22fefbef40e\nbuild(java): fix retry_with_backoff when -e option set (#475)\n\n\nd9b173c427bfa0c6cca818233562e7e8841a357c\nfix: record version of working repo in synth.metadata (#473)\n\nPartial revert of b37cf74d12e9a42b9de9e61a4f26133d7cd9c168.\nf73a541770d95a609e5be6bf6b3b220d17cefcbe\nfeat(discogapic): allow local discovery-artifact-manager (#474)\n\n\n8cf0f5d93a70c3dcb0b4999d3152c46d4d9264bf\ndoc: describe the Autosynth & Synthtool protocol (#472)\n\n* doc: describe the Autosynth & Synthtool protocol\n\n* Accommodate review comments.\n980baaa738a1ad8fa02b4fdbd56be075ee77ece5\nfix: pin sphinx to <3.0.0 as new version causes new error (#471)\n\nThe error `toctree contains reference to document changlelog that doesn't have a title: no link will be generated` occurs as of 3.0.0. Pinning to 2.x until we address the docs build issue.\n\nTowards #470\n\nI did this manually for python-datastore https://github.com/googleapis/python-datastore/pull/22\n928b2998ac5023e7c7e254ab935f9ef022455aad\nchore(deps): update dependency com.google.cloud.samples:shared-configuration to v1.0.15 (#466)\n\nCo-authored-by: Jeffrey Rennie <rennie@google.com>\n188f1b1d53181f739b98f8aa5d40cfe99eb90c47\nfix: allow local and external deps to be specified (#469)\n\nModify noxfile.py to allow local and external dependencies for\nsystem tests to be specified.\n1df68ed6735ddce6797d0f83641a731c3c3f75b4\nfix: apache license URL (#468)\n\n\nf4a59efa54808c4b958263de87bc666ce41e415f\nfeat: Add discogapic support for GAPICBazel generation (#459)\n\n* feat: Add discogapic support for GAPICBazel generation\n\n* reformat with black\n\n* Rename source repository variable\n\nCo-authored-by: Jeffrey Rennie <rennie@google.com>\n99820243d348191bc9c634f2b48ddf65096285ed\nfix: update template files for Node.js libraries (#463)\n\n\n3cbe6bcd5623139ac9834c43818424ddca5430cb\nfix(ruby): remove dead troubleshooting link from generated auth guide (#462)\n\n\na003d8655d3ebec2bbbd5fc3898e91e152265c67\ndocs: remove \"install stable\" instructions (#461)\n\nThe package hasn't been released to PyPI in some time\nf5e8c88d9870d8aa4eb43fa0b39f07e02bfbe4df\nchore(python): add license headers to config files; make small tweaks to templates (#458)\n\n\n"
       }
     }
   ]