hyperledger · macfarla · Apr 16, 2024 · Apr 17, 2024 · Apr 18, 2024 · Apr 23, 2024
diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml
@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+  GRADLE_OPTS: "-Xmx6g"
   total-runners: 16
 
 jobs:
@@ -35,6 +35,8 @@ jobs:
         with:
           distribution: temurin
           java-version: 17
+      - name: Install required packages
+        run:  sudo apt-get install -y xmlstarlet
       - name: get acceptance test report
         uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d
         with:
@@ -47,20 +49,27 @@ jobs:
         uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
         with:
           cache-disabled: true
+      - name: Compile acceptance tests
+        run: ./gradlew :acceptance-tests:tests:testClasses
+      - name: List acceptance tests
+        run: ./gradlew :acceptance-tests:tests:listAcceptanceTestNotPrivacy -Dorg.gradle.caching=true | fgrep org.hyperledger.besu.tests.acceptance > tmp/currentTests.list
       - name: Split tests
-        id: split-tests
-        uses: r7kamura/split-tests-by-timings@9322bd292d9423e2bc5a65bec548901801341e3f
+        run: .github/workflows/splitTestsByTime.sh tmp/junit-xml-reports-downloaded ${{env.total-runners}} ${{ matrix.runner_index }} > testList.txt
+      - name: Upload Timing
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        if: matrix.runner_index == 0
+        with:
+          name: acceptance-tests-timing
+          path: 'tmp/timing.tsv'
+      - name: Upload Lists
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        if: matrix.runner_index == 0
         with:
-          reports: tmp/junit-xml-reports-downloaded
-          glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java'
-          total: ${{env.total-runners}}
-          index: ${{ matrix.runner_index }}
-      - name: write out test list
-        run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt
+          name: acceptance-tests-lists
+          path: 'tmp/*.list'
       - name: format gradle args
-        #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter,
-        #then drop file extension, then insert --tests option between each.
-        run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g'  > gradleArgs.txt
+        # insert --tests option between each.
+        run:  cat testList.txt | sed -e 's/^\| / --tests /g' | tee gradleArgs.txt
       - name: run acceptance tests
         run: ./gradlew acceptanceTestNotPrivacy `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
       - name: cleanup tempfiles

diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml
@@ -13,6 +13,9 @@ jobs:
     runs-on: ubuntu-22.04
     permissions:
       contents: write
+    outputs:
+      tarSha: ${{steps.hashes.outputs.tarSha}}
+      zipSha: ${{steps.hashes.outputs.zipSha}}
     steps:
       - name: checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -32,6 +35,8 @@ jobs:
         id: hashes
         run: |
           cd build/distributions
+          echo "zipSha=$(shasum -a 256 besu*.zip)"
+          echo "tarSha=$(shasum -a 256 besu*.tar.gz)"
           echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT
           echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT
       - name: upload tarball
@@ -73,7 +78,9 @@ jobs:
           besu\bin\besu.bat --version
   publish:
     runs-on: ubuntu-22.04
-    needs: testWindows
+    needs: [artifacts, testWindows]
+    permissions:
+      contents: write
     steps:
       - name: Download archives
         uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe
@@ -89,12 +96,23 @@ jobs:
             build/distributions/besu*.tar.gz
             build/distributions/besu*.zip
           body: |
-            ${{steps.hashes.outputs.tarSha}}
-            ${{steps.hashes.outputs.zipSha}}
-  arifactoryPublish:
+            ${{needs.artifacts.outputs.tarSha}}
+            ${{needs.artifacts.outputs.zipSha}}
+  artifactoryPublish:
     runs-on: ubuntu-22.04
     needs: artifacts
     steps:
+      - name: checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - name: Set up JDK 17
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
       - name: Artifactory Publish
         env:
           ARTIFACTORY_USER: ${{ secrets.BESU_ARTIFACTORY_USER }}

diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -0,0 +1,121 @@
+name: docker develop
+
+on:
+  push:
+    branches:
+      - main
+env:
+  registry: docker.io
+
+jobs:
+  hadolint:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - name: Set up Java
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
+
+      - name: hadoLint
+        run: docker run --rm -i hadolint/hadolint < docker/Dockerfile
+  buildDocker:
+    needs: hadolint
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - ubuntu-22.04
+          - [self-hosted, ARM64]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Prepare
+        id: prep
+        run: |
+          platform=${{ matrix.platform }}
+          if [ "$platform" = 'ubuntu-22.04' ]; then 
+            echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT
+            echo "ARCH=amd64" >> $GITHUB_OUTPUT
+          else
+            echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT
+            echo "ARCH=arm64" >> $GITHUB_OUTPUT
+          fi
+
+          # Get the current date and time in the format YY.MM
+          DATE_TIME=$(date +"%y.%-m")
+          # Get the short SHA of the merge commit
+          SHORT_SHA=${GITHUB_SHA::7}
+          # Construct the build target name
+          BUILD_TARGET_NAME="${DATE_TIME}-develop-${SHORT_SHA}"
+          echo "Build Target Name: $BUILD_TARGET_NAME"
+          # Set the build target name as an environment variable
+          echo "BUILD_TARGET_NAME=${BUILD_TARGET_NAME}" >> $GITHUB_ENV
+
+      - name: Checkout Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - name: Set up Java
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
+      - name: install goss
+        run: |
+          mkdir -p docker/reports
+          curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }}
+      - name: login to ${{ env.registry }}
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ secrets.DOCKER_USER_RW }}
+          password: ${{ secrets.DOCKER_PASSWORD_RW }}
+      - name: build and test docker
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        env:
+          architecture: ${{ steps.prep.outputs.ARCH }}
+        with:
+          cache-disabled: true
+          arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{ env.BUILD_TARGET_NAME}} -Prelease.releaseVersion=develop
+      - name: publish
+        env:
+          architecture: ${{ steps.prep.outputs.ARCH }}
+        run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{ env.BUILD_TARGET_NAME }} -Prelease.releaseVersion=develop
+  multiArch:
+    needs: buildDocker
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - name: Set up Java
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
+      - name: login to ${{ env.registry }}
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ secrets.DOCKER_USER_RW }}
+          password: ${{ secrets.DOCKER_PASSWORD_RW }}
+      - name: multi-arch docker
+        run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{ env.BUILD_TARGET_NAME }} -Prelease.releaseVersion=develop
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -21,16 +21,8 @@ jobs:
         uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
         with:
           cache-disabled: true
-      - name: hadoLint_openj9-jdk_17
-        run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile
-      - name: hadoLint_openjdk_17
-        run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
-      - name: hadoLint_openjdk_17_debug
-        run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile
-      - name: hadoLint_openjdk_latest
-        run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile
-      - name: hadoLint_graalvm
-        run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile
+      - name: hadoLint
+        run: docker run --rm -i hadolint/hadolint < docker/Dockerfile
   buildDocker:
     needs: hadolint
     permissions:

diff --git a/.github/workflows/splitTestsByTime.sh b/.github/workflows/splitTestsByTime.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+REPORTS_DIR="$1"
+SPLIT_COUNT=$2
+SPLIT_INDEX=$3
+
+# extract tests time from Junit XML reports
+find "$REPORTS_DIR" -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v 'sum(//testcase/@time)' '{}'; echo '{}' | sed 's/.*TEST\-\(.*\)\.xml/ \1/'" > tmp/timing.tsv
+
+# Sort times in descending order
+IFS=$'\n' sorted=($(sort -nr tmp/timing.tsv))
+unset IFS
+
+sums=()
+tests=()
+
+# Initialize sums
+for ((i=0; i<SPLIT_COUNT; i++))
+do
+	sums[$i]=0
+done
+
+# add tests to groups trying to balance the sum of execution time of each group
+for line in "${sorted[@]}"; do
+	line_parts=( $line )
+	test_time=${line_parts[0]//./} # convert to millis
+	test_time=${test_time##0} # remove leading zeros
+	test_name=${line_parts[1]}
+
+  # Does the test still exists?
+  if grep -F -q --line-regexp "$test_name" tmp/currentTests.list
+  then
+    # Find index of min sum
+    idx_min_sum=0
+    min_sum=${sums[0]}
+    for ((i=0; i<SPLIT_COUNT; i++))
+    do
+      if [[ ${sums[$i]} -lt $min_sum ]]
+      then
+        idx_min_sum=$i
+        min_sum=${sums[$i]}
+      fi
+    done
+
+    # Add the test to the min sum list
+    min_sum_tests=${tests[$idx_min_sum]}
+    tests[$idx_min_sum]="${min_sum_tests}${test_name},"
+
+    # Update the sums
+    ((sums[idx_min_sum]+=test_time))
+
+    echo "$test_name" >> tmp/processedTests.list
+  fi
+done
+
+# Any new test?
+grep -F --line-regexp -v -f tmp/processedTests.list tmp/currentTests.list > tmp/newTests.list
+idx_new_test=0
+while read -r new_test_name
+do
+  idx_group=$(( idx_new_test % SPLIT_COUNT ))
+  group=${tests[$idx_group]}
+  tests[$idx_group]="${group}${new_test_name},"
+  idx_new_test=$(( idx_new_test + 1 ))
+done < tmp/newTests.list
+
+
+# return the requests index, without quotes to drop the last trailing space
+echo ${tests[$SPLIT_INDEX]//,/ }
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -44,6 +44,8 @@
 - Fix Shanghai/QBFT block import bug when syncing new nodes [#6765](https://github.com/hyperledger/besu/pull/6765)
 - Fix to avoid broadcasting full blob txs, instead of only the tx announcement, to a subset of nodes [#6835](https://github.com/hyperledger/besu/pull/6835)
 - Snap client fixes discovered during snap server testing [#6847](https://github.com/hyperledger/besu/pull/6847)
+- Correctly initialize the txpool as disabled on creation [#6890](https://github.com/hyperledger/besu/pull/6890)
+- Fix worldstate download halt when using snap sync during initial sync [#6981](https://github.com/hyperledger/besu/pull/6981)
 
 ### Download Links
 

diff --git a/acceptance-tests/tests/build.gradle b/acceptance-tests/tests/build.gradle
@@ -258,3 +258,32 @@ task acceptanceTestPermissioning(type: Test) {
 
   doFirst { mkdir "${buildDir}/jvmErrorLogs" }
 }
+
+// temporary solution to get a list of tests
+// Gradle >8.3 has a supported test dry-run option to achieve the same result
+task listAcceptanceTestNotPrivacy {
+  doLast {
+    def testExecutionSpec = tasks.getByName("acceptanceTestNotPrivacy") as Test
+
+    def processor = new org.gradle.api.internal.tasks.testing.TestClassProcessor() {
+        void startProcessing(org.gradle.api.internal.tasks.testing.TestResultProcessor processor) {}
+        void stop() {}
+        void stopNow() {}
+
+        void processTestClass(org.gradle.api.internal.tasks.testing.TestClassRunInfo info) {
+          def splitName = info.getTestClassName().split("\\.");
+          def testClassName = splitName[splitName.length-1];
+          if(testClassName.endsWith("Test") && !testClassName.startsWith("Abstract")) {
+            println(info.getTestClassName())
+          }
+        }
+      }
+
+    def detector = new org.gradle.api.internal.tasks.testing.detection.DefaultTestClassScanner(testExecutionSpec.getCandidateClassFiles(), testExecutionSpec.getTestFramework().getDetector()?.tap {
+      setTestClasses(testExecutionSpec.getTestClassesDirs().getFiles())
+      setTestClasspath(Collections.unmodifiableSet(testExecutionSpec.getClasspath().getFiles()))
+    }, processor)
+
+    detector.run()
+  }
+}
diff --git a/...ge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/...ge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java
@@ -501,10 +501,6 @@ public Optional<BlockHeader> getOrSyncHeadByHash(final Hash headHash, final Hash
           .addArgument(maybeHeadHeader.get()::toLogString)
           .log();
     } else {
-      LOG.atDebug()
-          .setMessage("Appending new head block hash {} to backward sync")
-          .addArgument(headHash::toHexString)
-          .log();
       backwardSyncContext.maybeUpdateTargetHeight(headHash);
       backwardSyncContext
           .syncBackwardsUntil(headHash)

diff --git a/docker/test.sh b/docker/test.sh
@@ -20,7 +20,7 @@ if [[ $i != 0 ]]; then exit $i; fi
 # we test that things listen on the right interface/port, not what interface the advertise
 # hence we dont set p2p-host=0.0.0.0 because this sets what its advertising to devp2p; the important piece is that it defaults to listening on all interfaces
 GOSS_FILES_PATH=$TEST_PATH/01 \
-bash $TEST_PATH/dgoss run $DOCKER_IMAGE \
+bash $TEST_PATH/dgoss run --sysctl net.ipv6.conf.all.disable_ipv6=1 $DOCKER_IMAGE \
 --network=dev \
 --rpc-http-enabled \
 --rpc-ws-enabled \