build.gradle

/*
 *   Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *   
 *   Licensed under the Apache License, Version 2.0 (the "License").
 *   You may not use this file except in compliance with the License.
 *   A copy of the License is located at
 *   
 *       http://www.apache.org/licenses/LICENSE-2.0
 *   
 *   or in the "license" file accompanying this file. This file is distributed 
 *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 
 *   express or implied. See the License for the specific language governing 
 *   permissions and limitations under the License.
 */

import java.util.concurrent.Callable
import org.elasticsearch.gradle.test.RestIntegTestTask

buildscript {
    ext {
        es_group = "org.elasticsearch"
        es_version = '7.10.2'
    }

    repositories {
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
        jcenter()
    }

    dependencies {
        classpath "${es_group}.gradle:build-tools:${es_version}"
    }
}

plugins {
    id 'nebula.ospackage' version "8.3.0" apply false
    id "com.diffplug.gradle.spotless" version "3.26.1"
    id 'java-library'
    id 'checkstyle'
}

repositories {
    mavenCentral()
    maven { url "https://plugins.gradle.org/m2/" }
    jcenter()
}

ext {
    opendistroVersion = '1.13.0'
    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
}

version = "${opendistroVersion}.1"

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'elasticsearch.esplugin'
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'base'
apply plugin: 'jacoco'
apply plugin: 'eclipse'

allprojects {
    group = 'com.amazon.opendistroforelasticsearch'

    plugins.withId('java') {
        sourceCompatibility = targetCompatibility = "1.8"
    }
}

ext {
    projectSubstitutions = [:]
    licenseFile = rootProject.file('LICENSE.txt')
    noticeFile = rootProject.file('NOTICE.txt')
}

esplugin {
    name 'opendistro-anomaly-detection'
    description 'Amazon opendistro elasticsearch anomaly detector plugin'
    classname 'com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin'
    extendedPlugins = ['lang-painless', 'opendistro-job-scheduler']
}

// Handle case where older versions of esplugin doesn't expose the joda time version it uses
configurations.all {
    if (it.state != Configuration.State.UNRESOLVED) return
    resolutionStrategy {
        force "joda-time:joda-time:${versions.joda}"
        force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
        force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
        force "commons-logging:commons-logging:${versions.commonslogging}"
        force "org.apache.httpcomponents:httpcore:${versions.httpcore}"
        force "commons-codec:commons-codec:${versions.commonscodec}"

        force "org.mockito:mockito-core:3.0.0"
        force "org.objenesis:objenesis:3.0.1"
        force "net.bytebuddy:byte-buddy:1.9.15"
        force "net.bytebuddy:byte-buddy-agent:1.9.15"
        force "com.google.code.gson:gson:2.8.9"
    }
}

configurations {
    testCompile {
        exclude group: 'org.elasticsearch', module: 'securemock'
        exclude group: 'org.hamcrest', module: 'hamcrest-core'
    }
}

tasks.named('forbiddenApisMain').configure {
  // Only enable limited check because AD code has too many violations.
  replaceSignatureFiles 'jdk-signatures'
  signaturesFiles += files('src/forbidden/ad-signatures.txt')
}

tasks.named('forbiddenApisTest').configure {
  // Disable check because AD code has too many violations.
  // For example, we have to allow @Test to be used in test classes not inherited from LuceneTestCase.
  // see https://github.com/elastic/elasticsearch/blob/master/buildSrc/src/main/resources/forbidden/es-test-signatures.txt
  ignoreFailures = true
}

// Allow test cases to be named Tests without having to be inherited from LuceneTestCase.
// see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
testingConventions.enabled = false

licenseHeaders.enabled = true
dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
loggerUsageCheck.enabled = false

// See package README.md for details on using these tasks.
def _numNodes = findProperty('numNodes') as Integer ?: 1

def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
es_tmp_dir.mkdirs()

test {
    include '**/*Tests.class'
    systemProperty 'tests.security.manager', 'false'
}

task integTest(type: RestIntegTestTask) {
    description = "Run tests against a cluster"
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {
    dependsOn "bundlePlugin"
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    // Only rest case can run with remote cluster
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "com.amazon.opendistroforelasticsearch.ad.rest.*IT"
            includeTestsMatching "com.amazon.opendistroforelasticsearch.ad.e2e.*IT"
        }
    }

    if (System.getProperty("https") == null || System.getProperty("https") == "false") {
        filter {
            excludeTestsMatching "com.amazon.opendistroforelasticsearch.ad.rest.SecureADRestIT"
        }
    }

    // The 'doFirst' delays till execution time.
    doFirst {
        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
        // use longer timeouts for requests.
        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
        systemProperty 'cluster.debug', isDebuggingCluster
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
    }

}

testClusters.integTest {
    testDistribution = "OSS"
    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("es.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
            debugPort += 1
        }
    }
    plugin(project.tasks.bundlePlugin.archiveFile)

    plugin(provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    return fileTree("src/test/resources/job-scheduler").getSingleFile()
                }
            }
        }
    }))

    // As of ES 7.7.0 the opendistro-anomaly-detection plugin is being added to the list of plugins for the testCluster during build before
    // the opendistro-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug.
    //
    // Exception in thread "main" java.lang.IllegalArgumentException: Missing plugin [opendistro-job-scheduler], dependency of [opendistro-anomaly-detection]
    //       at org.elasticsearch.plugins.PluginsService.addSortedBundle(PluginsService.java:452)
    //
    // One explanation is that ES build script sort plugins according to the natural ordering of their names.
    // opendistro-anomaly-detection comes before opendistro-job-scheduler.
    //
    // The following is a comparison of different plugin installation order:
    // Before 7.7:
    // ./bin/elasticsearch-plugin install --batch file:opendistro-anomaly-detection.zip file:opendistro-job-scheduler.zip
    //
    // After 7.7:
    // ./bin/elasticsearch-plugin install --batch file:opendistro-job-scheduler.zip file:opendistro-anomaly-detection.zip
    //
    // A temporary hack is to reorder the plugins list after evaluation but prior to task execution when the plugins are installed.
    nodes.each { node ->
        def plugins = node.plugins
        def firstPlugin = plugins.get(0)
        plugins.remove(0)
        plugins.add(firstPlugin)
    }
}

run {
    doFirst {
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    useCluster testClusters.integTest
}

evaluationDependsOnChildren()

task release(type: Copy, group: 'build') {
    dependsOn allprojects*.tasks.build
    from(zipTree(project.tasks.bundlePlugin.outputs.files.getSingleFile()))
    into "build/plugins/opendistro-anomaly-detection"
    includeEmptyDirs = false
    // ES versions < 6.3 have a top-level elasticsearch directory inside the plugin zip which we need to remove
    eachFile { it.path = it.path - "elasticsearch/" }
}

List<String> jacocoExclusions = [
        // code for configuration, settings, etc is excluded from coverage
        'com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin',
        'com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings',

        //TODO: add more test cases later for these package
        'com.amazon.opendistroforelasticsearch.ad.model.*',
        'com.amazon.opendistroforelasticsearch.ad.rest.*',
        'com.amazon.opendistroforelasticsearch.ad.transport.handler.DetectionStateHandler',
        'com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorJobRunner',

        // Class containing just constants.  Don't need to test
        'com.amazon.opendistroforelasticsearch.ad.constant.*',

        // mostly skeleton code.  Tested major logic in restful api tests
        'com.amazon.opendistroforelasticsearch.ad.settings.EnabledSetting',

        'com.amazon.opendistroforelasticsearch.ad.common.exception.FeatureNotAvailableException',
        'com.amazon.opendistroforelasticsearch.ad.common.exception.AnomalyDetectionException',
        'com.amazon.opendistroforelasticsearch.ad.util.ClientUtil',

        'com.amazon.opendistroforelasticsearch.ad.transport.StopDetectorRequest',
        'com.amazon.opendistroforelasticsearch.ad.transport.StopDetectorResponse',
        'com.amazon.opendistroforelasticsearch.ad.transport.StopDetectorTransportAction',
        'com.amazon.opendistroforelasticsearch.ad.transport.DeleteDetectorAction',
        'com.amazon.opendistroforelasticsearch.ad.transport.CronTransportAction',
        'com.amazon.opendistroforelasticsearch.ad.transport.CronRequest',
        'com.amazon.opendistroforelasticsearch.ad.transport.ADStatsNodesAction',
        'com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorRunner',
        'com.amazon.opendistroforelasticsearch.ad.util.ParseUtils',

        // related to transport actions added for security
        'com.amazon.opendistroforelasticsearch.ad.transport.StatsAnomalyDetectorTransportAction',
        'com.amazon.opendistroforelasticsearch.ad.transport.DeleteAnomalyDetectorTransportAction*',
        'com.amazon.opendistroforelasticsearch.ad.transport.SearchAnomalyDetectorTransportAction*',
        'com.amazon.opendistroforelasticsearch.ad.transport.GetAnomalyDetectorTransportAction*',
        'com.amazon.opendistroforelasticsearch.ad.transport.SearchAnomalyResultTransportAction*',
        'com.amazon.opendistroforelasticsearch.ad.transport.SearchAnomalyDetectorInfoTransportAction*',

        // TODO: hc caused coverage to drop
        'com.amazon.opendistroforelasticsearch.ad.indices.AnomalyDetectionIndices',
        'com.amazon.opendistroforelasticsearch.ad.transport.handler.MultiEntityResultHandler',
        'com.amazon.opendistroforelasticsearch.ad.util.ThrowingSupplierWrapper',
]

jacocoTestCoverageVerification {
    violationRules {
        rule {
            element = 'CLASS'
            excludes = jacocoExclusions
            limit {
                counter = 'BRANCH'
                minimum = 0.60
            }
        }
        rule {
            element = 'CLASS'
            excludes = jacocoExclusions
            limit {
                counter = 'LINE'
                value = 'COVEREDRATIO'
                minimum = 0.75
            }
        }
    }
}

jacocoTestReport {
    reports {
        xml.enabled = true
        html.enabled = true
    }
}

check.dependsOn jacocoTestCoverageVerification
jacocoTestCoverageVerification.dependsOn jacocoTestReport

checkstyle {
    toolVersion = '8.20'
}

dependencies {
    compile "org.elasticsearch:elasticsearch:${es_version}"
    compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${versions.elasticsearch}"
    compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:${opendistroVersion}.0"
    compile "com.amazon.opendistroforelasticsearch:common-utils:${opendistroVersion}.0"
    compile group: 'com.google.guava', name: 'guava', version:'29.0-jre'
    compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
    compile group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
    compile group: 'com.yahoo.datasketches', name: 'sketches-core', version: '0.13.4'
    compile group: 'com.yahoo.datasketches', name: 'memory', version: '0.12.2'
    compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
    compile 'software.amazon.randomcutforest:randomcutforest-core:1.0'
    compile 'software.amazon.randomcutforest:randomcutforest-serialization-json:1.0'
    compile "org.elasticsearch.client:elasticsearch-rest-client:${es_version}"

    compile "org.jacoco:org.jacoco.agent:0.8.5"
    compile ("org.jacoco:org.jacoco.ant:0.8.5") {
        exclude group: 'org.ow2.asm', module: 'asm-commons'
        exclude group: 'org.ow2.asm', module: 'asm'
        exclude group: 'org.ow2.asm', module: 'asm-tree'
    }

    testCompile group: 'pl.pragmatists', name: 'JUnitParams', version: '1.1.1'
    testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.0.0'
    testImplementation group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.2'
    testImplementation group: 'org.powermock', name: 'powermock-module-junit4', version: '2.0.2'
    testImplementation group: 'org.powermock', name: 'powermock-module-junit4-common', version: '2.0.2'
    testImplementation group: 'org.powermock', name: 'powermock-core', version: '2.0.2'
    testImplementation group: 'org.powermock', name: 'powermock-api-support', version: '2.0.2'
    testImplementation group: 'org.powermock', name: 'powermock-reflect', version: '2.0.7'
    testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.0.1'
    testImplementation group: 'org.javassist', name: 'javassist', version: '3.27.0-GA'
    testCompile group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.15'
    testCompile group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.9.15'

    checkstyle "com.puppycrawl.tools:checkstyle:${project.checkstyle.toolVersion}"
}

compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"

apply plugin: 'nebula.ospackage'

// This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
afterEvaluate {
    ospackage {
        packageName = "${name}"
        release = isSnapshot ? "0.1" : '1'
        version = "${project.version}" - "-SNAPSHOT"

        into '/usr/share/elasticsearch/plugins'
        from(zipTree(bundlePlugin.archivePath)) {
            into esplugin.name
        }

        user 'root'
        permissionGroup 'root'
        fileMode 0644
        dirMode 0755

        requires('elasticsearch-oss', versions.elasticsearch, EQUAL)
        packager = 'Amazon'
        vendor = 'Amazon'
        os = 'LINUX'
        prefix '/usr'

        license 'ASL-2.0'
        maintainer 'OpenDistro for Elasticsearch Team <opendistro@amazon.com>'
        url 'https://opendistro.github.io/for-elasticsearch/downloads.html'
        summary '''
         Anomaly Detection plugin for OpenDistro for Elasticsearch.
         Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/.
    '''.stripIndent().replace('\n', ' ').trim()
    }

    buildRpm {
        arch = 'NOARCH'
        dependsOn 'assemble'
        finalizedBy 'renameRpm'
        task renameRpm(type: Copy) {
            from("$buildDir/distributions")
            into("$buildDir/distributions")
            include archiveName
            rename archiveName, "${packageName}-${version}.rpm"
            doLast { delete file("$buildDir/distributions/$archiveName") }
        }
    }

    buildDeb {
        arch = 'all'
        dependsOn 'assemble'
        finalizedBy 'renameDeb'
        task renameDeb(type: Copy) {
            from("$buildDir/distributions")
            into("$buildDir/distributions")
            include archiveName
            rename archiveName, "${packageName}-${version}.deb"
            doLast { delete file("$buildDir/distributions/$archiveName") }
        }
    }

    task buildPackages(type: GradleBuild) {
        tasks = ['build', 'buildRpm', 'buildDeb']
    }
}

spotless {
    java {
        removeUnusedImports()
        importOrder 'java', 'javax', 'org', 'com'

        eclipse().configFile rootProject.file('.eclipseformat.xml')
    }
}

// no need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false

tasks.withType(licenseHeaders.class) {
    additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
}