Add support for Java versions higher than 8

h2oai · Oct 6, 2023 · 622acad · 622acad
1 parent 424b2dd
commit 622acad
Show file tree

Hide file tree

Showing 13 changed files with 65 additions and 11 deletions.
diff --git a/api-generation/build.gradle b/api-generation/build.gradle
@@ -15,6 +15,9 @@ dependencies {
 
 shadowJar {
   mergeServiceFiles()
+  manifest {
+    attributes 'Add-Opens': openJavaModules.join(" ")
+  }
 }
 
 build.dependsOn shadowJar

diff --git a/assembly-extensions/build.gradle b/assembly-extensions/build.gradle
@@ -18,6 +18,9 @@ shadowJar {
   mergeServiceFiles()
 
   archiveBaseName = "${archiveBaseName.get()}_${scalaBaseVersion}"
+  manifest {
+    attributes 'Add-Opens': openJavaModules.join(" ")
+  }
 }
 
 artifacts {

diff --git a/assembly-scoring/build.gradle b/assembly-scoring/build.gradle
@@ -17,6 +17,9 @@ jar {
 shadowJar {
   mergeServiceFiles()
   archiveBaseName = "${archiveBaseName.get()}_${scalaBaseVersion}"
+  manifest {
+    attributes 'Add-Opens': openJavaModules.join(" ")
+  }
 }
 
 artifacts {

diff --git a/assembly/build.gradle b/assembly/build.gradle
@@ -66,6 +66,10 @@ shadowJar {
   from "$project.buildDir/reports/" include '**/*'
   exclude 'www/flow/packs/test-*/**'
 
+  manifest {
+    attributes 'Add-Opens': openJavaModules.join(" ")
+  }
+
   archiveBaseName = "${archiveBaseName.get()}_${scalaBaseVersion}"
 }
 

diff --git a/build.gradle b/build.gradle
@@ -199,6 +199,12 @@ configure(subprojects) { project ->
   // All subprojects needs Spark support
   apply from: "$rootDir/gradle/spark.gradle"
 
+  // All tests need Java module system run options
+  apply from: "$rootDir/gradle/javaModuleSystem.gradle"
+  tasks.withType(Test).configureEach {
+    jvmArgs += project.defaultJavaModuleRunOptions
+  }
+
   if (project in docProjects) {
     apply plugin: 'java'
     apply plugin: 'kr.motd.sphinx'

diff --git a/ci/Jenkinsfile-kubernetes b/ci/Jenkinsfile-kubernetes
@@ -119,7 +119,7 @@ def getBuildAndTestStagesForSparkVersion(commons, props, sparkMajorVersion) {
                     def sparklingVersion = getSparklingVersion(props, sparkMajorVersion)
                     def master = "k8s://" + getMinikubeMaster()
 
-                    // Spark home within sparkling_water_tests image the code runs currently in, for a current sparkVersion
+                    // Spark home within sparkling_water_tests_jdk17 image the code runs currently in, for a current sparkVersion
                     def currentSparkHome = "/home/jenkins/spark-${sparkVersion}-bin"
 
                     // Spark home within a built sparkling-water:scala/python/r/external-backend-... image
@@ -652,7 +652,7 @@ def testPythonExternalBackendAutoClientMode(master, version, swImageSparkHome) {
 def withSparklingWaterMinikubeImage(commons, code) {
     def repoUrl = commons.getAWSDockerRepo()
     commons.withAWSDocker {
-        def image = "${repoUrl}/opsh2oai/sparkling_water_tests:" + commons.getDockerImageVersion()
+        def image = "${repoUrl}/opsh2oai/sparkling_water_tests_jdk17:" + commons.getDockerImageVersion()
         def dockerOptions = "--init --privileged --network=host -v /var/run/docker.sock:/var/run/docker.sock"
         groovy.lang.Closure initCode = {
             sh "activate_java_8"

diff --git a/ci/build.gradle b/ci/build.gradle
@@ -21,6 +21,12 @@ task createDockerfile(type: Dockerfile, dependsOn: copyFiles) {
   destFile = outputFile
   from testingBaseImage
 
+  // switching to a newer Java version as the base image uses ancient 8u171
+  // we're copying JDK from another docker image
+  copyFile(new Dockerfile.CopyFile(testingJdkImageJavaPath, testingJdkImageJavaPath).withStage(testingJdkImage))
+  runCommand("rm -rf /usr/lib/jvm/java-8-oracle")
+  environmentVariable('JAVA_HOME', testingJdkImageJavaPath)
+
   runCommand "rm /etc/startup/70_start_slapd"
 
   // Install Terraform
@@ -29,8 +35,7 @@ task createDockerfile(type: Dockerfile, dependsOn: copyFiles) {
                 |   unzip terraform.zip -d /usr/local/bin/ && \\
                 |   rm -f terraform.zip
                """.stripMargin()
-
-  environmentVariable("PATH","/usr/lib/jvm/java-8-oracle/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+  environmentVariable("PATH","${testingJdkImageJavaPath}/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
 
   // Update to R 4.2
   runCommand """\\
@@ -96,8 +101,7 @@ task createDockerfile(type: Dockerfile, dependsOn: copyFiles) {
                rm /home/jenkins/miniconda.sh
                 """
 
-  environmentVariable("PATH", "/home/jenkins/miniconda/bin:/usr/lib/jvm/java-8-oracle/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
-  environmentVariable("JAVA_HOME", "/usr/lib/jvm/java-8-oracle")
+  environmentVariable("PATH", "/home/jenkins/miniconda/bin:\$PATH")
 
   runCommand """\\
                 conda install anaconda-client conda-build conda-verify -y && \\

diff --git a/ci/commons.groovy b/ci/commons.groovy
@@ -63,7 +63,7 @@ def withDocker(image, groovy.lang.Closure code, String dockerOptions = "", groov
 def withSparklingWaterDockerImage(code) {
     def repoUrl = getAWSDockerRepo()
     withAWSDocker {
-        def image = "${repoUrl}/opsh2oai/sparkling_water_tests:" + getDockerImageVersion()
+        def image = "${repoUrl}/opsh2oai/sparkling_water_tests_jdk17:" + getDockerImageVersion()
         def dockerOptions = "--init --privileged"
         groovy.lang.Closure initCode = {
             sh "activate_java_8"

diff --git a/ci/docker/Jenkinsfile-build-docker b/ci/docker/Jenkinsfile-build-docker
@@ -35,7 +35,7 @@ node('mr-0xc10') {
         dir("ci/build/docker") {
             sh """
             docker pull $testingBaseImage
-            docker build --no-cache -t harbor.h2o.ai/opsh2oai/sparkling_water_tests:${nextVersion} -f Dockerfile .
+            docker build --no-cache -t harbor.h2o.ai/opsh2oai/sparkling_water_tests_jdk17:${nextVersion} -f Dockerfile .
            """
         }
     }
@@ -44,8 +44,8 @@ node('mr-0xc10') {
         def repoUrl = commons.getAWSDockerRepo()
         commons.withAWSDocker {
             sh """
-            docker tag harbor.h2o.ai/opsh2oai/sparkling_water_tests:${nextVersion} $repoUrl/opsh2oai/sparkling_water_tests:${nextVersion}
-            docker push $repoUrl/opsh2oai/sparkling_water_tests:${nextVersion}
+            docker tag harbor.h2o.ai/opsh2oai/sparkling_water_tests_jdk17:${nextVersion} $repoUrl/opsh2oai/sparkling_water_tests_jdk17:${nextVersion}
+            docker push $repoUrl/opsh2oai/sparkling_water_tests_jdk17:${nextVersion}
             """
         }
     }

diff --git a/gradle.properties b/gradle.properties
@@ -36,4 +36,6 @@ databricksTestSinceSpark=2.4
 spotlessModern=true
 testH2OBranch=rel-3.42.0
 makeBooklet=false
-testingBaseImage="harbor.h2o.ai/opsh2oai/h2o-3-hadoop-cdh-6.3:84"
+testingBaseImage=harbor.h2o.ai/opsh2oai/h2o-3-hadoop-cdh-6.3:84
+testingJdkImage=eclipse-temurin:17.0.8.1_1-jdk-alpine
+testingJdkImageJavaPath=/opt/java/openjdk
diff --git a/gradle/javaModuleSystem.gradle b/gradle/javaModuleSystem.gradle
@@ -0,0 +1,23 @@
+String jdk8CompatibilityOption = "-XX:+IgnoreUnrecognizedVMOptions"
+List<String> openJavaModules = [
+        "java.base/java.lang",
+        "java.base/java.lang.invoke",
+        "java.base/java.lang.reflect",
+        "java.base/java.io",
+        "java.base/java.net",
+        "java.base/java.nio",
+        "java.base/java.util",
+        "java.base/java.util.concurrent",
+        "java.base/java.util.concurrent.atomic",
+        "java.base/sun.nio.ch",
+        "java.base/sun.nio.cs",
+        "java.base/sun.security.action",
+        "java.base/sun.util.calendar",
+        "java.security.jgss/sun.security.krb5"
+]
+
+List<String> defaultJavaModuleRunOptions =
+        [jdk8CompatibilityOption] + openJavaModules.collect { module -> "--add-opens=${module}=ALL-UNNAMED " }
+
+ext.defaultJavaModuleRunOptions = defaultJavaModuleRunOptions
+ext.openJavaModules = openJavaModules
diff --git a/package/build.gradle b/package/build.gradle
@@ -12,6 +12,9 @@ jar {
 
 shadowJar {
   archiveClassifier = null
+  manifest {
+    attributes 'Add-Opens': openJavaModules.join(" ")
+  }
 }
 
 defineShadowPublication().call()
diff --git a/scoring-package/build.gradle b/scoring-package/build.gradle
@@ -12,6 +12,9 @@ jar {
 
 shadowJar {
   archiveClassifier = null
+  manifest {
+    attributes 'Add-Opens': openJavaModules.join(" ")
+  }
 }
 
 defineShadowPublication().call()