Enabled automated, sequenced releases from Travis CI (#1387)

The goal is to release automatically from Travis and enable convenient merging of PRs, even if many PRs are merged concurrently (bug in Shipkit Gradle plugin: mockito/shipkit#395).
- instead of using Shipkit Gradle plugin we keep most CI/CD automation inside Ambry project. This way, it is easier to configure, debug and understand.
- we're using a new Gradle plugin "shipkit-auto-version". This plugin automatically deducts the version based on the most recent tag and the "version.properties" file. It is a very simple plugin, keeps the build logic simple, and keeps the Ambry build easy to maintain in the future.

This PR removes following features (acceptable trade-offs, we can implement it in the future):
- automated release notes generation

.travis.yml

@@ -15,11 +15,11 @@ addons:
     packages:
       - oracle-java8-installer
       - oracle-java8-unlimited-jce-policy
-#don't build tags
 branches:
-  except:
+  except: # don't build tags
     - /^v\d/
+install: skip # skips unnecessary ./gradlew assemble (https://docs.travis-ci.com/user/job-lifecycle/#skipping-the-installation-phase) 
 script:
-  - ./gradlew -s --scan build codeCoverageReport && ./gradlew -s -i --scan ciPerformRelease
+  - ./travis-build.sh
 after_success:
   - bash <(curl -s https://codecov.io/bash)

build.gradle

@@ -11,23 +11,22 @@
 buildscript {
     repositories {
         mavenCentral()
+        mavenLocal()
     }
     apply from: file('gradle/buildscript.gradle'), to: buildscript
 }
 
-plugins {
-    id 'org.shipkit.java' version '2.2.5' apply false
-}
-if (!project.hasProperty('disableShipkit')) {
-    apply plugin: 'org.shipkit.java'
-}
+apply plugin: 'org.shipkit.shipkit-auto-version'
+println "Building version $version"
 
 apply from: file('gradle/license.gradle')
 apply from: file('gradle/environment.gradle')
 apply from: file("gradle/dependency-versions.gradle")
+apply from: file("gradle/ci-release.gradle")
 
 allprojects {
     group = "com.github.ambry"
+    version = rootProject.version //TODO: fix in https://github.com/shipkit/org.shipkit.shipkit-auto-version
 
     apply plugin: 'eclipse'
     apply plugin: 'idea'
@@ -42,6 +41,8 @@ allprojects {
 
 subprojects {
     apply plugin: 'java'
+    apply from: "$rootDir/gradle/java-publishing.gradle"    
+
     sourceCompatibility = 1.8
     targetCompatibility = 1.8
 
@@ -109,22 +110,7 @@ subprojects {
         group = 'verification'
     }
     allTest.dependsOn test
-    allTest.dependsOn intTest
-
-    task testJar(type: Jar, dependsOn: testClasses) {
-        from sourceSets.test.output
-        classifier = 'test'
-    }
-
-    artifacts {
-        archives testJar
-    }
-
-    if (!project.hasProperty('disableShipkit')) {
-        // add test jars to maven publication if using shipkit
-        publishing.publications.javaLibrary.artifact testJar
-    }
-
+    allTest.dependsOn intTest    
     javadoc {
         // TODO audit and fix our javadocs so that we don't need this setting
         // This is mainly for cases where param/throws tags don't have descriptions

gradle/buildscript.gradle

@@ -17,5 +17,7 @@ repositories {
 }
 
 dependencies {
-    classpath "gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.15.0"
+    classpath "gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.15.0"    
+    classpath "org.shipkit:shipkit-auto-version:0.0.22"
+    classpath "com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4"
 }

gradle/ci-release.gradle

@@ -0,0 +1,37 @@
+task bintrayUploadAll {
+    description = "Runs 'bintrayUpload' tasks from all projects"
+    mustRunAfter "gitPush" //git push is easier to rollback so we run it first
+}
+
+allprojects {
+    tasks.matching { it.name == "bintrayUpload" }.all {
+        bintrayUploadAll.dependsOn it
+    }
+}
+
+task ciPerformRelease {
+    description = "Performs the release, intended to be ran on CI"
+    dependsOn "gitTag", "gitPush", "bintrayUploadAll"    
+}
+
+task gitTag {
+    description = "Creates annotated tag 'v$version'"
+
+    doLast {
+        println "Setting Git user and email to Travis CI"
+        exec { commandLine "git", "config", "user.email", "travis@travis-ci.org" }
+        exec { commandLine "git", "config", "user.name", "Travis CI" }
+        println "Creating tag v$version"
+        exec { commandLine "git", "tag", "-a", "-m", "Release $version", "v$version" }
+        println "Created tag v$version"
+    }
+}
+
+task gitPush(type: Exec) {
+    description = "Pushes tags by running 'git push --tags'. Hides secret key from git push output."
+    mustRunAfter "gitTag" //tag first, push later
+
+    doFirst { println "Pushing tag v$version" }
+    commandLine "./gradle/git-push.sh", "v$version"
+    doLast { println "Pushed tag v$version" }
+}

gradle/git-push.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+#Git push is implemented in the script to make sure we are not leaking GH key to the output
+#Expects 'GIT_SECRET' env variable to be in format: user:github_access_token, 
+#for example: mockitoguy:qq43234xc23x23d24d
+
+echo "Running git push without output for security. If it fails make sure that GIT_SECRET env variable is set."
+git push --quiet https://$GIT_SECRET@github.com/linkedin/ambry.git --tags > /dev/null 2>&1
+EXIT_CODE=$?
+echo "'git push --quiet' exit code: $EXIT_CODE"
+exit $EXIT_CODE

gradle/java-publishing.gradle

@@ -0,0 +1,77 @@
+assert plugins.hasPlugin("java")
+
+apply plugin: 'maven-publish'
+apply plugin: 'com.jfrog.bintray'
+
+tasks.withType(Jar) {
+    from "$rootDir/LICENSE"
+    from "$rootDir/NOTICE"
+}
+
+task sourcesJar(type: Jar, dependsOn: classes) {
+    classifier = 'sources'
+    from sourceSets.main.allSource    
+}
+
+task javadocJar(type: Jar, dependsOn: javadoc) {
+    classifier = 'javadoc'
+    from javadoc.destinationDir  
+}
+
+task testJar(type: Jar, dependsOn: testClasses) {
+    from sourceSets.test.output
+    classifier = 'test'
+}
+
+artifacts {
+    archives sourcesJar
+    archives javadocJar
+    archives testJar
+}
+
+publishing {
+    publications {
+        maven(MavenPublication) {
+            from components.java
+
+            artifact javadocJar
+            artifact sourcesJar
+            artifact testJar
+        }
+    }
+}
+
+bintray { //docs: https://github.com/bintray/gradle-bintray-plugin
+    user = System.getenv("BINTRAY_USER")
+    key = System.getenv("BINTRAY_API_KEY")
+
+    publish = true
+    dryRun = project.hasProperty("bintray.dryRun")? true : false //useful for testing
+
+    publications = ['maven'] 
+
+    pkg {
+        repo = 'test-repo'
+        userOrg = 'linkedin'
+        name = 'ambry'
+
+        licenses = ['Apache-2.0']
+        labels = ['blob storage']
+        version {
+            // disable gpg signing to speed up publishing
+            gpg {
+                sign = false
+            }
+            // disable upload to maven central
+            mavenCentralSync {
+                sync = false
+            }
+        }
+    }    
+}
+
+bintrayUpload {
+    doFirst {
+        println "Publishing $jar.baseName to Bintray (dryRun: $dryRun, repo: $repoName, publish: $publish)"
+    }
+}

travis-build.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# exit when any command fails
+set -e
+
+echo "Building and testing artifacts, and creating pom files"
+./gradlew -s --scan build publishToMavenLocal codeCoverageReport
+
+echo "Testing Bintray publication by uploading in dry run mode"
+./gradlew -s -i --scan bintrayUploadAll -Pbintray.dryRun
+
+echo "Pull request: [$TRAVIS_PULL_REQUEST], Travis branch: [$TRAVIS_BRANCH]"
+# release only from master when no pull request build
+if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]
+then    
+    echo "Releasing (tagging, uploading to Bintray)"
+    ./gradlew -s -i --scan ciPerformRelease
+fi

version.properties

@@ -1,4 +1,5 @@
-#Version of the produced binaries. This file is intended to be checked-in.
-#It will be automatically bumped by release automation.
-version=0.3.110
-previousVersion=0.3.109
+# shipkit-auto-version Gradle plugin uses this version spec to deduct the version to build
+# it increments patch version based on most recent tag.
+# You can put explicit version here if needed, e.g. "1.0.0"
+# More information: https://github.com/shipkit/shipkit-auto-version/blob/master/README.md
+version=0.3.*