Merge pull request #9983 from NvTimLiu/release-tmp

Merge branch 'branch-23.12' into main [skip ci]

.github/CODEOWNERS

@@ -1,4 +1,4 @@
-#  Copyright (c) 2020-2022, NVIDIA CORPORATION.
+#  Copyright (c) 2020-2023, NVIDIA CORPORATION.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -10,5 +10,7 @@
 #  limitations under the License.
 
 
-/jenkins/	@jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @zhanga5
-pom.xml		@jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @zhanga5
+/jenkins/  @jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @pxLi
+pom.xml    @jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @pxLi
+/dist/     @jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @pxLi
+/.github/  @jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @pxLi

.github/workflows/auto-merge.yml

@@ -18,7 +18,7 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-    - branch-23.10
+    - branch-23.12
     types: [closed]
 
 jobs:
@@ -29,13 +29,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: branch-23.10 # force to fetch from latest upstream instead of PR ref
+          ref: branch-23.12 # force to fetch from latest upstream instead of PR ref
 
       - name: auto-merge job
         uses: ./.github/workflows/auto-merge
         env:
           OWNER: NVIDIA
           REPO_NAME: spark-rapids
-          HEAD: branch-23.10
-          BASE: branch-23.12
+          HEAD: branch-23.12
+          BASE: branch-24.02
           AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR

.github/workflows/blossom-ci.yml

@@ -59,7 +59,6 @@ jobs:
       hyperbolic2346,\
       gerashegalov,\
       ttnghia,\
-      zhanga5,\
       nvliyuan,\
       res-life,\
       HaoYang670,\
@@ -71,6 +70,7 @@ jobs:
       thirtiseven,\
       winningsix,\
       viadea,\
+      yinqingh,\
       ', format('{0},', github.actor)) && github.event.comment.body == 'build'
     steps:
       - name: Check if comment is issued by authorized person

.github/workflows/mvn-verify-check.yml

@@ -24,10 +24,10 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  COMMON_MVN_FLAGS: >
+  COMMON_MVN_FLAGS: >-
     -Ddist.jar.compress=false
     -DskipTests
-    -Dskip
+    -Dmaven.scaladoc.skip
 
 jobs:
   get-shim-versions-from-dist:
@@ -36,6 +36,7 @@ jobs:
       defaultSparkVersion: ${{ steps.allShimVersionsStep.outputs.defaultSparkVersion }}
       sparkTailVersions: ${{ steps.allShimVersionsStep.outputs.tailVersions }}
       sparkJDKVersions: ${{ steps.allShimVersionsStep.outputs.jdkVersions }}
+      scala213Versions: ${{ steps.allShimVersionsStep.outputs.scala213Versions }}
     steps:
       - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
 
@@ -78,6 +79,23 @@ jobs:
           jdkVersionJsonStr=$(printf {\"include\":[%s]} $jdkVersionArrBody)
           echo "jdkVersions=$jdkVersionJsonStr" >> $GITHUB_OUTPUT
 
+          SCALA_BINARY_VER=2.13
+          . jenkins/version-def.sh
+          svArrBodyNoSnapshot=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":false}" "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS[@]}")
+          svArrBodyNoSnapshot=${svArrBodyNoSnapshot:1}
+          # get private artifact version
+          privateVer=$(mvn help:evaluate -q -pl dist -Dexpression=spark-rapids-private.version -DforceStdout)
+          # do not add empty snapshot versions or when private version is released one (does not include snapshot shims)
+          if [[ ${#SPARK_SHIM_VERSIONS_SNAPSHOTS_ONLY[@]} -gt 0 && $privateVer == *"-SNAPSHOT" ]]; then
+            svArrBodySnapshot=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":true}" "${SPARK_SHIM_VERSIONS_SNAPSHOTS_ONLY[@]}")
+            svArrBodySnapshot=${svArrBodySnapshot:1}
+            svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot,$svArrBodySnapshot)
+          else
+            svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot)
+          fi
+
+          echo "scala213Versions=$svJsonStr" >> $GITHUB_OUTPUT
+
   package-tests:
     needs: get-shim-versions-from-dist
     continue-on-error: ${{ matrix.isSnapshot }}
@@ -106,14 +124,78 @@ jobs:
           export JAVA_HOME=${JAVA_HOME_8_X64}
           export PATH=${JAVA_HOME}/bin:${PATH}
           java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH"
-          # test command
-          mvn -Dmaven.wagon.http.retryHandler.count=3 -B package \
-            -pl integration_tests,tests -am \
-            -P 'individual,pre-merge' \
-            -Dbuildver=${{ matrix.spark-version }} \
-            -Dmaven.scalastyle.skip=true \
-            -Drat.skip=true \
-            ${{ env.COMMON_MVN_FLAGS }}
+          # test command, will retry for 3 times if failed.
+          max_retry=3; delay=30; i=1
+          while true; do
+            mvn -Dmaven.wagon.http.retryHandler.count=3 \
+              -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \
+              -Daether.connector.http.connectionMaxTtl=30 -B package \
+              -pl integration_tests,tests -am -P 'individual,pre-merge' \
+              -Dbuildver=${{ matrix.spark-version }} -Dmaven.scalastyle.skip=true \
+              -Drat.skip=true ${{ env.COMMON_MVN_FLAGS }} && break || {
+            if [[ $i -le $max_retry ]]; then
+              echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
+            else
+              echo "mvn command failed. Exit 1"; exit 1
+            fi
+          }
+          done
+
+  package-tests-scala213:
+    needs: get-shim-versions-from-dist
+    continue-on-error: ${{ matrix.isSnapshot }}
+    strategy:
+      matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.scala213Versions) }}
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
+
+      - name: Setup Java and Maven Env
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 8
+
+      - name: check runtime before tests
+        run: |
+          env | grep JAVA
+          java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH"
+
+      - name: package tests check
+        run: |
+          # https://github.com/NVIDIA/spark-rapids/issues/8847
+          # specify expected versions
+          export JAVA_HOME=${JAVA_HOME_8_X64}
+          export PATH=${JAVA_HOME}/bin:${PATH}
+          java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH"
+          # verify Scala 2.13 build files
+          ./build/make-scala-version-build-files.sh 2.13
+          # verify git status
+          if [ -n "$(echo -n $(git status -s | grep 'scala2.13'))" ]; then
+              git add -N scala2.13/* && git diff 'scala2.13/*'
+              echo "Generated Scala 2.13 build files don't match what's in repository"
+              exit 1
+          fi
+          # change to Scala 2.13 Directory
+          cd scala2.13
+          # test command, will retry for 3 times if failed.
+          max_retry=3; delay=30; i=1
+          while true; do
+            mvn -Dmaven.wagon.http.retryHandler.count=3 \
+              -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \
+              -Daether.connector.http.connectionMaxTtl=30 -B package \
+              -pl integration_tests,tests -am -P 'individual,pre-merge' \
+              -Dbuildver=${{ matrix.spark-version }} -Dmaven.scalastyle.skip=true \
+              -Drat.skip=true ${{ env.COMMON_MVN_FLAGS }} && break || {
+            if [[ $i -le $max_retry ]]; then
+              echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
+            else
+              echo "mvn command failed. Exit 1"; exit 1
+            fi
+          }
+          done
+
 
   verify-all-modules:
     needs: get-shim-versions-from-dist
@@ -141,11 +223,21 @@ jobs:
           export JAVA_HOME=${JAVA_HOME_${{ matrix.java-version }}_X64}
           export PATH=${JAVA_HOME}/bin:${PATH}
           java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH"
-          # test command
-          mvn -Dmaven.wagon.http.retryHandler.count=3 -B verify \
-            -P "individual,pre-merge,jdk${{ matrix.java-version }}" \
-            -Dbuildver=${{ matrix.spark-version }} \
-            ${{ env.COMMON_MVN_FLAGS }}
+          # test command, will retry for 3 times if failed.
+          max_retry=3; delay=30; i=1
+          while true; do
+            mvn -Dmaven.wagon.http.retryHandler.count=$max_retry \
+              -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \
+              -Daether.connector.http.connectionMaxTtl=30 -B verify \
+              -P "individual,pre-merge" -Dbuildver=${{ matrix.spark-version }} \
+              ${{ env.COMMON_MVN_FLAGS }} && break || {
+            if [[ $i -le $max_retry ]]; then
+              echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
+            else
+              echo "mvn command failed. Exit 1"; exit 1
+            fi
+          }
+          done
 
   install-modules:
     needs: get-shim-versions-from-dist
@@ -174,11 +266,22 @@ jobs:
         run: |
           # https://github.com/NVIDIA/spark-rapids/issues/8847
           # specify expected versions
-          export JAVA_HOME=export JAVA_HOME=${JAVA_HOME_11_X64}
+          export JAVA_HOME=${JAVA_HOME_11_X64}
           export PATH=${JAVA_HOME}/bin:${PATH}
           java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH"
-          # test command
-          ./mvnw -Dmaven.wagon.http.retryHandler.count=3 -B install \
-            -P "individual,pre-merge,jdk11" \
-            -Dbuildver=${{ needs.get-shim-versions-from-dist.outputs.defaultSparkVersion }} \
-            ${{ env.COMMON_MVN_FLAGS }}
+          # test command, will retry for 3 times if failed.
+          max_retry=3; delay=30; i=1
+          while true; do
+            ./mvnw -Dmaven.wagon.http.retryHandler.count=3 \
+              -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 \
+              -Daether.connector.http.connectionMaxTtl=30 -B install \
+              -P "individual,pre-merge" \
+              -Dbuildver=${{ needs.get-shim-versions-from-dist.outputs.defaultSparkVersion }} \
+              ${{ env.COMMON_MVN_FLAGS }} && break || {
+            if [[ $i -le $max_retry ]]; then
+              echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
+            else
+              echo "mvn command failed. Exit 1"; exit 1
+            fi
+          }
+          done