build.gradle

/*
   Copyright (c) 2012 LinkedIn Corp.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

buildscript {
  repositories {
    jcenter()
  }
  dependencies {
    classpath 'org.jfrog.buildinfo:build-info-extractor-gradle:4.32.0'
    classpath 'org.jacoco:org.jacoco.core:0.8.7'
  }
}

project.ext {
  buildScriptDirPath = "${projectDir.path}/build_script"
  isDefaultEnvironment = !project.hasProperty('overrideBuildEnvironment')
  privateModules = ['d2-benchmark', 'd2-int-test', 'generator-test', 'log-test-config', 'r2-int-test',
                     'r2-perf-test', 'restli-internal-testutils'] as Set
  skipTestsForSubprojects = (project.findProperty('pegasus.skipTestsForSubprojects') ?: '').split(',') as Set
}

File getEnvironmentScript()
{
  final File env = file(isDefaultEnvironment ? 'defaultEnvironment.gradle' : project.overrideBuildEnvironment)
  assert env.isFile() : "The environment script [$env] does not exists or is not a file."
  return env
}

apply from: environmentScript
apply from: "${buildScriptDirPath}/configBuildScript.gradle"

project.ext.externalDependency = [
  'antlr': 'org.antlr:antlr4:4.5',
  'antlrRuntime': 'org.antlr:antlr4-runtime:4.5',
  'avro': 'org.apache.avro:avro:1.9.2',
  'avro_1_6': 'org.apache.avro:avro:1.6.3',
   // avro compatibility layer
  'avroUtil': 'com.linkedin.avroutil1:helper-all:0.2.138',
  'caffeine': 'com.github.ben-manes.caffeine:caffeine:2.7.0',
  'cglib': 'cglib:cglib-nodep:2.2',
  'codemodel': 'com.sun.codemodel:codemodel:2.2',
  'commonsCli': 'commons-cli:commons-cli:1.0',
  'commonsCodec': 'commons-codec:commons-codec:1.3',
  'commonsCompress': 'org.apache.commons:commons-compress:1.2',
  'commonsHttpClient': 'commons-httpclient:commons-httpclient:3.1',
  'commonsIo': 'commons-io:commons-io:2.4',
  'commonsLang': 'commons-lang:commons-lang:2.6',
  'commonsText': 'org.apache.commons:commons-text:1.10.0',
  'disruptor': 'com.lmax:disruptor:3.2.0',
  'easymock': 'org.easymock:easymock:4.0.2',
  'findbugs': 'com.google.code.findbugs:annotations:3.0.0',
  'mockito': 'org.mockito:mockito-all:1.9.5',
  'metricsCore': 'io.dropwizard.metrics:metrics-core:4.2.10',
  'guava': 'com.google.guava:guava:18.0',
  'httpclient': 'org.apache.httpcomponents:httpclient:4.3.1',
  'httpcore': 'org.apache.httpcomponents:httpcore:4.3.1',
  'jacksonCore': 'com.fasterxml.jackson.core:jackson-core:2.10.2',
  'jacksonSmile': 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.10.2',
  'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.10.2',
  'jacksonCoreAsl_1_4': 'org.codehaus.jackson:jackson-core-asl:1.4.2',
  'jacksonCoreAsl_1_8': 'org.codehaus.jackson:jackson-core-asl:1.8.8',
  'javaparser': 'com.github.javaparser:javaparser-symbol-solver-core:3.15.11',
  "javaxActivation": "com.sun.activation:javax.activation:1.2.0",
  'javaxAnnotation': 'javax.annotation:javax.annotation-api:1.3.1',
  'javaxInject': 'javax.inject:javax.inject:1',
  'jdkTools': files("${System.getProperty('java.home')}/../lib/tools.jar"),
  'jettyAlpnServer': 'org.eclipse.jetty:jetty-alpn-server:9.3.21.v20170918',
  'jettyHttp': 'org.eclipse.jetty:jetty-http:9.3.21.v20170918',
  'jettyHttp2Server': 'org.eclipse.jetty.http2:http2-server:9.3.21.v20170918',
  'jettyServlet': 'org.eclipse.jetty:jetty-servlet:9.3.21.v20170918',
  'jettyServer': 'org.eclipse.jetty:jetty-server:9.3.21.v20170918',
  'jettyUtil': 'org.eclipse.jetty:jetty-util:9.3.21.v20170918',
  'jmhCore': 'org.openjdk.jmh:jmh-core:1.21',
  'jmhAnnotations': 'org.openjdk.jmh:jmh-generator-annprocess:1.21',
  'junit': 'junit:junit:4.12',
  'log4j2Api': 'org.apache.logging.log4j:log4j-api:2.0.2',
  'log4j2Core': 'org.apache.logging.log4j:log4j-core:2.0.2',
  'log4jLog4j2': 'org.apache.logging.log4j:log4j-1.2-api:2.0.2',
  'mail': 'javax.mail:mail:1.4.4',
  'netty': 'io.netty:netty-all:4.1.79.Final',
  'objenesis': 'org.objenesis:objenesis:1.2',
  'parseq': 'com.linkedin.parseq:parseq:5.1.2',
  'parseq_tracevis': 'com.linkedin.parseq:parseq-tracevis:5.1.2',
  'parseq_restClient': 'com.linkedin.parseq:parseq-restli-client:5.1.2',
  'parseq_testApi': 'com.linkedin.parseq:parseq-test-api:5.1.2',
  'servletApi': 'javax.servlet:javax.servlet-api:3.1.0',
  'slf4jApi': 'org.slf4j:slf4j-api:1.7.30',
  'slf4jLog4j2': 'org.apache.logging.log4j:log4j-slf4j-impl:2.0.2',
  'snappy': 'org.iq80.snappy:snappy:0.4',
  'xerialSnappy': 'org.xerial.snappy:snappy-java:1.1.7.7',
  'spock': 'org.spockframework:spock-core:1.3-groovy-2.5',
  'testng': 'org.testng:testng:6.13.1',
  'velocity': 'org.apache.velocity:velocity-engine-core:2.2',
  'zero_allocation_hashing': 'net.openhft:zero-allocation-hashing:0.7',
  'zookeeper': 'org.apache.zookeeper:zookeeper:3.6.3',
  'hdrhistogram': 'org.hdrhistogram:HdrHistogram:2.1.9',
  'xchart': 'org.knowm.xchart:xchart:3.2.2',

  // for restli-spring-bridge ONLY, we must keep these dependencies isolated
  'springCore': 'org.springframework:spring-core:3.2.18.RELEASE',
  'springContext': 'org.springframework:spring-context:3.2.18.RELEASE',
  'springWeb': 'org.springframework:spring-web:3.2.18.RELEASE',
  'springBeans': 'org.springframework:spring-beans:3.2.18.RELEASE',

  // for restli-guice-bridge ONLY, we should keep these dependencies isolated
  'guice': 'com.google.inject:guice:3.0',
  'guiceServlet': 'com.google.inject.extensions:guice-servlet:3.0',

  'jsr305': 'com.google.code.findbugs:jsr305:3.0.0',
  "avroSpotBugsPlugin": "com.linkedin.avroutil1:spotbugs-plugin:0.2.56",
  "classgraph": "io.github.classgraph:classgraph:4.8.149",

  // for integrating with xDS service discovery
  'grpcNettyShaded': 'io.grpc:grpc-netty-shaded:1.45.1',
  'grpcProtobuf': 'io.grpc:grpc-protobuf:1.45.1',
  'grpcStub': 'io.grpc:grpc-stub:1.45.1',
  'protoc': 'com.google.protobuf:protoc:3.21.7',
  'protobufJava': 'com.google.protobuf:protobuf-java:3.21.7',
  'protobufJavaUtil': 'com.google.protobuf:protobuf-java-util:3.21.7',
  'envoyApi': 'io.envoyproxy.controlplane:api:0.1.35',
];

if (!project.ext.isDefaultEnvironment)
{
  spec.external.each { overrideDepKey, overrideDepValue ->
    if (project.ext.externalDependency[overrideDepKey] != null)
    {
      project.ext.externalDependency[overrideDepKey] = overrideDepValue
    }
  }
}

allprojects {
  apply plugin: 'idea'

  tasks.withType(JavaCompile).all { JavaCompile compile ->

    if(!project.sourceCompatibility.isJava8Compatible()) // lower bound of java versions we support.
    {
      throw new GradleScriptException("Pegasus required Java 8 or later to build, current version: ${JavaVersion.current()}", null)
    }
    // for all supported versions that we test build, fail the build if any compilation warnings are reported
    compile.options.compilerArgs = ['-Xlint', '-Xlint:-path', '-Xlint:-static']
    // Set the default Java bytecode level to version 8 to ensure backward compatibility
    if (JavaVersion.current() >= JavaVersion.VERSION_11) {
      sourceCompatibility = 8
      targetCompatibility = 8
      // Ideally we should uncomment the line below to set the release option to Java 8 to restrict the use of new APIs after Java 8.
      // However, com.sun.javadoc APIs were moved from tools.jar in Java 8 to JDK internals in Java 11.
      // Therefore, if we set the release option to Java 8, we will not be able to use the APIs in JDK internals when compiling in Java 11.
      // compile.options.compilerArgs.addAll(['--release', '8'])
    }
  }

  tasks.withType(Javadoc)
  {
    if (JavaVersion.current().isJava8Compatible())
    {
      //Disable the crazy super-strict doclint tool in Java 8
      //noinspection SpellCheckingInspection
      options.addStringOption('Xdoclint:none', '-quiet')
    }
  }
}

allprojects {
  plugins.withId('li-spotbugs') {
    dependencies {
      spotbugsPlugins spec.external.avroSpotBugsPlugin
    }
  }
}

idea {
  project {
    languageLevel = '1.8'
  }
}

subprojects {
  apply plugin: 'java'
  apply plugin: 'eclipse'
  apply from: "${buildScriptDirPath}/cleanGenerated.gradle"
  apply plugin: 'jacoco'

  afterEvaluate {
    if (project.plugins.hasPlugin('pegasus')) {
      configurations {
        pluginsRuntime {
          visible = false
        }
      }
      dependencies {
        pluginsRuntime project(':gradle-plugins')
        dataTemplateCompile project(':data')
        restClientCompile project(':restli-client')

        //Providing the required dependencies for the plugin to execute under a configuration
        pegasusPlugin project(':data')
        pegasusPlugin project(':data-avro-generator')
        pegasusPlugin project(':generator')
        pegasusPlugin project(':restli-tools')
        pegasusPlugin externalDependency.javaxAnnotation
      }
    }
  }

  configurations {
    compile {
      transitive = false
    }
    testArtifacts {
      visible = false
    }
    // Exclude a bunch of undesirable, unused dependencies
    // that Log4j 1.2.15 drags in; these break the open source
    // build since they are not available in maven central.
    all*.exclude group: 'javax.jms', module: 'jms'
    all*.exclude group: 'com.sun.jdmk', module: 'jmxtools'
    all*.exclude group: 'com.sun.jmx', module: 'jmxri'
    // Exclude older versions of velocity engine under a different module name
    // so they don't conflict with "velocity-engine-core" module used
    // for versions >=2.0
    all*.exclude group: 'org.apache.velocity', module: 'velocity'
  }

  if (!(it.name in ['data-avro', 'restli-int-test'])) {
    configurations {
      // Prevent Guava from creeping in to avoid incompatibilities in multiple classloader environments.
      compile.resolutionStrategy {
        eachDependency { DependencyResolveDetails details ->
          if (details.requested.group == 'com.google.guava' && details.requested.name == 'guava') {
            throw new GradleException('Cannot directly or transitively depend on Guava.')
          }
        }
      }
    }
  }

  // Default dependencies for all subprojects
  dependencies {
    compile externalDependency.slf4jApi
    // Note this adds a default log4j.xml configuration to all projects testRuntime configuration.
    // However, projects can still override simply by creating their own
    // src/test/resources/log4j.xml, since that will always appear in the testRuntime classpath
    // before this one.
    testRuntime project(':log-test-config')
  }

  if (isDefaultEnvironment) {
    repositories {
      mavenLocal()
      mavenCentral()
      maven {
        url "https://linkedin.jfrog.io/artifactory/open-source"
      }
    }

    task sourcesJar(type: Jar, dependsOn: classes) {
      from sourceSets.main.allSource
      classifier = 'sources'
    }

    task javadocJar(type: Jar, dependsOn: javadoc) {
      from javadoc.destinationDir
      classifier = 'javadoc'
    }

    artifacts {
      archives jar
      archives sourcesJar
      archives javadocJar
    }

    // Only build and publish artifacts for consumer-facing modules
    if (!(it.name in privateModules)) {
      // Configure all publications and the tasks to publish them
      apply from: "${buildScriptDirPath}/publications.gradle"
      apply from: "${buildScriptDirPath}/jfrog.gradle"
    }
  }

  task testJar(type: Jar) {
    from sourceSets.test.output
    classifier = 'tests'
  }

  artifacts {
    testArtifacts testJar
  }

  // Configure the IDEA plugin to (1) add the codegen as source dirs and (2) work around
  // an apparent bug in the plugin which doesn't set the outputDir/testOutputDir as documented
  idea.module {
    // Gradle docs claim the two settings below are the default, but
    // the actual defaults appear to be "out/production/$MODULE_NAME"
    // and "out/test/$MODULE_NAME". Changing it so IDEA and gradle share
    // the class output directory.

    outputDir = sourceSets.main.output.classesDirs.getSingleFile()
    testOutputDir = sourceSets.test.output.classesDirs.getSingleFile()
  }

  test {
    maxHeapSize = '4g'

    useTestNG() {
      excludeGroups 'not_implemented'
      excludeGroups 'integration'
      excludeGroups 'd2integration'
      excludeGroups 'integration_external_product_dependent'
      excludeGroups 'known_issue'
      excludeGroups 'withoutAssertion'
    }
  }

  // Do some special test configuration for the CI environment
  if (System.getenv('CI') == 'true' && System.getenv('GITHUB_ACTIONS') == 'true') {
    afterEvaluate {
      project.tasks.withType(Test).forEach {
        // Exclude tests which are known to be flaky (only TestNG tests, not other tests e.g. Gradle int tests)
        if (it.options instanceof TestNGOptions) {
          it.options.excludeGroups 'ci-flaky'
        }
        // Increase the Rest.li int test timeout
        it.systemProperties['test.httpRequestTimeout'] = '20000'
        // Make the build fail fast in CI (may have unintended consequences locally)
        it.failFast = true
      }
    }
  }

  task asyncTests(type: Test) {
    useTestNG() {
      includeGroups 'async'
    }
  }

  task integrationTests(type: Test) {
    useTestNG() {
      includeGroups 'integration'
      excludeGroups 'd2integration'
      excludeGroups 'integration_external_product_dependent'
    }
  }

  task d2integrationTests(type: Test) {
    useTestNG() {
      includeGroups 'd2integration'
      excludeGroups 'integration'
      excludeGroups 'integration_external_product_dependent'
    }
  }

  //all functional tests which are expected to pass assuming proper environment setup
  task allTests(type: Test) {
    maxHeapSize = '4g'

    useTestNG() {
      excludeGroups 'known_issue'
      excludeGroups 'not_implemented'
    }
  }

  task integrationExternalProductDependentTests(type: Test) {
    useTestNG() {
      includeGroups 'integration_external_product_dependent'
    }
  }

  task notImplementedFeaturesTests(type: Test) {
    useTestNG() {
      includeGroups 'not_implemented'
    }
  }

  task knownIssuesTests(type: Test) {
    useTestNG() {
      includeGroups 'known_issue'
    }
  }

  task allFailingTests(type: Test) {
    useTestNG() {
      includeGroups 'known_issue'
      includeGroups 'not_implemented'
    }
  }

  task testsWithoutAssertion(type: Test) {
    enableAssertions = false

    useTestNG() {
      includeGroups 'withoutAssertion'
    }
  }

  project.tasks.test.dependsOn(asyncTests)
  project.tasks.test.dependsOn(testsWithoutAssertion)

  task cleanBuildDir(type: Exec){
    delete(rootDir.toString()+'build')
    commandLine = ['rm', '-rf', rootDir.toString() + '/build']
    workingDir = file(rootDir.toString())
  }

  task makedir(type:Exec) {
    // gradle mkdir does not create dir with "-p" flag on so this is a generic task that takes -Ddirpath=..... parameter
    // Usage examples:
    // gradle makedir -Ddirpath=abc/xyz
    def dir = System.getProperties()['dirpath'] ?: null
    if (dir != null){
      commandLine = ['mkdir','-p', dir]
      workingDir = file(rootDir.toString())
    }
  }

  // Disable tests for specific subprojects if requested
  if (project.name in rootProject.ext.skipTestsForSubprojects) {
    project.tasks.withType(Test) {
      enabled = false
    }
    logger.lifecycle "Tests for subproject ${project.name} will be skipped."
  }
}

final skippedTests = [:].withDefault {[]}
project.gradle.addListener(new TestListener() {
  @Override
  void beforeSuite(TestDescriptor suite) {}

  @Override
  void afterSuite(TestDescriptor suite, TestResult result) {}

  @Override
  void beforeTest(TestDescriptor testDescriptor) {}

  @Override
  void afterTest(TestDescriptor testDescriptor, TestResult result) {
    // Display test failure messages since these aren't shown by default
    if (result.failures) {
      logger.error "\nFailure message for ${testDescriptor.getClassName()} > ${testDescriptor.getDisplayName()}:"
      result.failures.forEach {
        logger.error "    ${it.getMessage()}"
      }
    }
    // Accumulate info about skipped tests, display at the end of the build as a warning
    if (result.skippedTestCount > 0)
    {
      skippedTests[testDescriptor.className] << testDescriptor.name
    }
  }
})

project.gradle.buildFinished { BuildResult result ->
  if (skippedTests.size() > 0)
  {
    final StringBuilder msgBuilder = new StringBuilder("\nThe following tests were skipped or failed and had to retry.");
    skippedTests.each { className, methods ->
      msgBuilder.append("\n    ${className} > ${methods}")
    }
    logger.warn msgBuilder.toString()
  }
}