Merge branch 'master' into mn/GH-15748

docker/prisma/Dockerfile.mainjars

@@ -0,0 +1,2 @@
+FROM alpine:latest
+COPY ./h2o-assemblies/main/build/libs/*.jar /tmp/

h2o-algos/src/test/java/hex/kmeans/KMeansTest.java

@@ -37,16 +37,10 @@ private static KMeansModel doSeed( KMeansModel.KMeansParameters parms, long seed
     return kmm;
   }
 
-  //PUBDEV-871: Double-check the training metrics (gathered by computeStatsFillModel) and the scoring logic by scoring on the training set
   private static void checkConsistency(KMeansModel kmm) {
-    //FIXME: TODO: remove this false, and fix the algo! PUBDEV-871
-    if (false) {
       KMeansModel.KMeansParameters parms = kmm._parms;
       Assert.assertTrue((ArrayUtils.sum(kmm._output._size) - parms.train().numRows()) <= 1);
 
-//    Log.info(kmm._output._model_summary);
-//    Log.info(kmm._output._scoring_history);
-//    Log.info(((ModelMetricsClustering)kmm._output._training_metrics).createCentroidStatsTable().toString());
       kmm.score(parms.train()).delete(); //this scores on the training data and appends a ModelMetrics
       ModelMetricsClustering mm = (ModelMetricsClustering) ModelMetrics.getFromDKV(kmm, parms.train());
       Assert.assertTrue(Arrays.equals(mm._size, ((ModelMetricsClustering) kmm._output._training_metrics)._size));
@@ -56,7 +50,6 @@ private static void checkConsistency(KMeansModel kmm) {
       Assert.assertTrue(MathUtils.compare(mm._totss, ((ModelMetricsClustering) kmm._output._training_metrics)._totss, 1e-6, 1e-6));
       Assert.assertTrue(MathUtils.compare(mm._betweenss, ((ModelMetricsClustering) kmm._output._training_metrics)._betweenss, 1e-6, 1e-6));
       Assert.assertTrue(MathUtils.compare(mm._tot_withinss, ((ModelMetricsClustering) kmm._output._training_metrics)._tot_withinss, 1e-6, 1e-6));
-    }
   }
 
   @Test public void testIris() {

h2o-jaas-pam/build.gradle

@@ -5,5 +5,5 @@
 description = "JAAS PAM Module"
 
 dependencies {
-    api "org.kohsuke:libpam4j:1.8"
+    api "org.kohsuke:libpam4j:1.11"
 }

scripts/jenkins/groovy/defineTestStages.groovy

@@ -440,7 +440,7 @@ def call(final pipelineContext) {
     ],
     [
       stageName: 'Py3.6 Medium-large', target: 'test-pyunit-medium-large', pythonVersion: '3.6',
-      timeoutValue: 220, component: pipelineContext.getBuildConfig().COMPONENT_PY
+      timeoutValue: 300, component: pipelineContext.getBuildConfig().COMPONENT_PY
     ],
     [
       stageName: 'R3.3 Medium-large', target: 'test-r-medium-large', rVersion: '3.3.3',
@@ -546,8 +546,8 @@ def call(final pipelineContext) {
     ],
     [
       stageName: 'Py3.9 Medium-large', target: 'test-pyunit-medium-large', pythonVersion: '3.9',
-      timeoutValue: 170, component: pipelineContext.getBuildConfig().COMPONENT_PY
-    ],
+      timeoutValue: 300, component: pipelineContext.getBuildConfig().COMPONENT_PY
+    ],      
     [ // These run with reduced number of file descriptors for early detection of FD leaks
       stageName: 'XGBoost Stress tests', target: 'test-pyunit-xgboost-stress', pythonVersion: '3.6', timeoutValue: 40,
       component: pipelineContext.getBuildConfig().COMPONENT_PY, customDockerArgs: [ '--ulimit nofile=150:150' ]

scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan

@@ -3,15 +3,42 @@
 @Library('test-shared-library') _
 
 def dockerImage
-def branchOrTag
-def steamImage
 
-pipeline {
-    agent { node { label 'linux&&docker' } }
+def setScanningStages(assemblyType, stageIndex) {
+    def assemblyImage
+    stage("${stageIndex}.A. Scan ${assemblyType} jar using Prisma") {
+        script {
+            branchName = "${env.BRANCH_NAME}".replace('/', '-')
+            assemblyImage = "h2o-assemblies/${assemblyType}:${BUILD_NUMBER}-${branchName}"
+
+            sh "docker build . -t ${assemblyImage} -f ./docker/prisma/Dockerfile.${assemblyType}jars"
 
-    parameters {
-        string(name: 'BRANCH_OR_TAG', defaultValue: 'master', description: 'Enter branch or tag you want to scan.')
+            // scan the image
+            prismaCloudScanImage ca: '',
+                    cert: '',
+                    dockerAddress: 'unix:///var/run/docker.sock',
+                    image: "${assemblyImage}",
+                    key: '',
+                    logLevel: 'info',
+                    podmanPath: '',
+                    project: '',
+                    resultsFile: "prisma-${assemblyType}-scan-results.json",
+                    ignoreImageBuildTime: true
+        }
+    }
+    stage("${stageIndex}.B. Export results for ${assemblyType} jar to CSV") {
+        withCredentials([usernamePassword(credentialsId: 'twistlock_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+            sh "curl -k -u \$USERNAME:\$PASSWORD https://mr-0xz1:8083/api/v1/scans/download?search=${assemblyImage} > ${assemblyImage}.csv"
+        }
+        archiveArtifacts artifacts: "${assemblyImage}.csv"
     }
+    stage("${stageIndex}.C. Publish report for ${assemblyType} jar") {
+        prismaCloudPublish resultsFilePattern: "prisma-${assemblyType}-scan-results.json"
+    }
+}
+
+pipeline {
+    agent { node { label 'linux&&docker' } }
 
     options {
         ansiColor('xterm')
@@ -29,54 +56,28 @@ pipeline {
 
             }
         }
-        stage('1. Build jar') {
+        stage('1. Build jars') {
             steps {
                 script{
                     dockerImage.inside(){
                         sh "./gradlew :h2o-assemblies:steam:shadowJar"
+                        sh "./gradlew :h2o-assemblies:main:shadowJar"
                         archiveArtifacts artifacts: "h2o-assemblies/steam/build/libs/*.jar"
+                        archiveArtifacts artifacts: "h2o-assemblies/main/build/libs/*.jar"
                     }
                 }
             }
         }
-        stage('2. Scan jar using Prisma'){
+        stage('2. Steam assembly jar') {
             steps {
-                script{
-                    branchOrTag = "${BRANCH_OR_TAG}".replace('/','-')
-                    steamImage = "h2o-assemblies/steam:${BUILD_NUMBER}-${branchOrTag}"
-
-                    sh "docker build . -t ${steamImage} -f ./docker/prisma/Dockerfile.scanningjars"
-
-                    // scan the image
-                    prismaCloudScanImage ca: '',
-                    cert: '',
-                    dockerAddress: 'unix:///var/run/docker.sock',
-                    image: "${steamImage}",
-                    key: '',
-                    logLevel: 'info',
-                    podmanPath: '',
-                    project: '',
-                    resultsFile: 'prisma-cloud-scan-results.json',
-                    ignoreImageBuildTime:true
-                }
-
+                setScanningStages("steam", 2)
             }
         }
-        stage('3. Export results to CSV'){
-            steps{
-                withCredentials([usernamePassword(credentialsId: 'twistlock_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-                    sh "curl -k -u \$USERNAME:\$PASSWORD https://mr-0xz1:8083/api/v1/scans/download?search=${steamImage} > ${steamImage}.csv"
-                }
-                archiveArtifacts artifacts: "${steamImage}.csv"
-            }
-        }
-        stage('4. Publish report'){
-            steps{
-                prismaCloudPublish resultsFilePattern: 'prisma-cloud-scan-results.json'
+        stage('3. Main assembly jar') {
+            steps {
+                setScanningStages("main", 3)
             }
         }
-
-
     }
     post {
       always {