diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3105324fb9..bffacc76ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ on: branches: - master env: - GO_VERSION: 1.16 + GO_VERSION: 1.19 GOPATH: ${{ github.workspace }}/go WORKING_DIR: ${{ github.workspace }}/go/src/github.com/ethereum/go-ethereum jobs: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c1fe7451ea..6e44887c64 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -6,7 +6,7 @@ on: - '**.md' - .gitignore env: - GO_VERSION: 1.16 + GO_VERSION: 1.19 jobs: lint: name: 'Code linters' @@ -17,8 +17,9 @@ jobs: with: go-version: ${{ env.GO_VERSION }} - name: 'Check out project files' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + fetch-depth: '0' submodules: false - name: 'Prepare environment' run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 677e0ffb06..2229810b19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: tags: - 'v*' env: - GO_VERSION: 1.16 + GO_VERSION: 1.19 GOPATH: ${{ github.workspace }}/go WORKING_DIR: ${{ github.workspace }}/go/src/github.com/ethereum/go-ethereum jobs: diff --git a/.golangci.yml b/.golangci.yml index 395a91fe1b..32b416f190 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,6 +23,8 @@ linters: - unconvert # - unused - varcheck + - typecheck + linters-settings: gofmt: @@ -32,6 +34,7 @@ linters-settings: min-occurrences: 6 # minimum number of occurrences issues: + new-from-rev: c4e9657 exclude-rules: - path: crypto/blake2b/ linters: diff --git a/.travis.yml b/.travis.yml index 7406f31fe7..e08e271f3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jobs: allow_failures: - stage: build os: osx - go: 1.15.x + go: 1.17.x env: - azure-osx - azure-ios @@ -16,7 +16,7 @@ jobs: - stage: lint os: linux dist: bionic - go: 1.16.x + go: 1.18.x env: - lint git: @@ -24,12 +24,48 @@ jobs: script: - go run build/ci.go lint + # These builders create the Docker sub-images for multi-arch push and each + # will attempt to push the multi-arch image if they are the last builder + - stage: build + if: type = push + os: linux + arch: amd64 + dist: bionic + go: 1.18.x + env: + - docker + services: + - docker + git: + submodules: false # avoid cloning ethereum/tests + before_install: + - export DOCKER_CLI_EXPERIMENTAL=enabled + script: + - go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go + + - stage: build + if: type = push + os: linux + arch: arm64 + dist: bionic + go: 1.18.x + env: + - docker + services: + - docker + git: + submodules: false # avoid cloning ethereum/tests + before_install: + - export DOCKER_CLI_EXPERIMENTAL=enabled + script: + - go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go + # This builder does the Ubuntu PPA upload - stage: build if: type = push os: linux dist: bionic - go: 1.16.x + go: 1.18.x env: - ubuntu-ppa - GO111MODULE=on @@ -54,7 +90,7 @@ jobs: os: linux dist: bionic sudo: required - go: 1.16.x + go: 1.18.x env: - azure-linux - GO111MODULE=on @@ -84,36 +120,6 @@ jobs: - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - # This builder does the Linux Azure MIPS xgo uploads - - stage: build - if: type = push - os: linux - dist: bionic - services: - - docker - go: 1.16.x - env: - - azure-linux-mips - - GO111MODULE=on - git: - submodules: false # avoid cloning ethereum/tests - script: - - go run build/ci.go xgo --alltools -- --targets=linux/mips --ldflags '-extldflags "-static"' -v - - for bin in build/bin/*-linux-mips; do mv -f "${bin}" "${bin/-linux-mips/}"; done - - go run build/ci.go archive -arch mips -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - - - go run build/ci.go xgo --alltools -- --targets=linux/mipsle --ldflags '-extldflags "-static"' -v - - for bin in build/bin/*-linux-mipsle; do mv -f "${bin}" "${bin/-linux-mipsle/}"; done - - go run build/ci.go archive -arch mipsle -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - - - go run build/ci.go xgo --alltools -- --targets=linux/mips64 --ldflags '-extldflags "-static"' -v - - for bin in build/bin/*-linux-mips64; do mv -f "${bin}" "${bin/-linux-mips64/}"; done - - go run build/ci.go archive -arch mips64 -type tar -signer LINUX_SIGNING_KEY signify SIGNIFY_KEY -upload gethstore/builds - - - go run build/ci.go xgo --alltools -- --targets=linux/mips64le --ldflags '-extldflags "-static"' -v - - for bin in build/bin/*-linux-mips64le; do mv -f "${bin}" "${bin/-linux-mips64le/}"; done - - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - # This builder does the Android Maven and Azure uploads - stage: build if: type = push @@ -142,7 +148,7 @@ jobs: - sdkmanager "platform-tools" "platforms;android-15" "platforms;android-19" "platforms;android-24" "ndk-bundle" # Install Go to allow building with - - curl https://dl.google.com/go/go1.16.linux-amd64.tar.gz | tar -xz + - curl https://dl.google.com/go/go1.18.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go @@ -156,7 +162,7 @@ jobs: - stage: build if: type = push os: osx - go: 1.16.x + go: 1.18.x env: - azure-osx - azure-ios @@ -188,7 +194,7 @@ jobs: os: linux arch: amd64 dist: bionic - go: 1.16.x + go: 1.18.x env: - GO111MODULE=on script: @@ -199,7 +205,7 @@ jobs: os: linux arch: arm64 dist: bionic - go: 1.16.x + go: 1.18.x env: - GO111MODULE=on script: @@ -208,7 +214,7 @@ jobs: - stage: build os: linux dist: bionic - go: 1.15.x + go: 1.17.x env: - GO111MODULE=on script: @@ -219,7 +225,7 @@ jobs: if: type = cron os: linux dist: bionic - go: 1.16.x + go: 1.18.x env: - azure-purge - GO111MODULE=on @@ -227,3 +233,15 @@ jobs: submodules: false # avoid cloning ethereum/tests script: - go run build/ci.go purge -store gethstore/builds -days 14 + + # This builder executes race tests + - stage: build + if: type = cron + os: linux + dist: bionic + go: 1.18.x + env: + - GO111MODULE=on + script: + - go run build/ci.go test -race -coverage $TEST_PACKAGES + diff --git a/Dockerfile b/Dockerfile index a4678d971f..607dd948a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,21 @@ +# Support setting various labels on the final image +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + # Build Geth in a stock Go builder container -FROM golang:1.16-alpine as builder +FROM golang:1.19-alpine as builder + +RUN apk add --no-cache gcc musl-dev linux-headers git -RUN apk add --no-cache make gcc musl-dev linux-headers git +# Get dependencies - will also be cached if we won't change go.mod/go.sum +COPY go.mod /go-ethereum/ +COPY go.sum /go-ethereum/ +RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && make geth bootnode +RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/geth +RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/bootnode # Pull Geth into a second stage deploy alpine container FROM alpine:latest @@ -15,3 +26,10 @@ COPY --from=builder /go-ethereum/build/bin/bootnode /usr/local/bin/ EXPOSE 8545 8546 30303 30303/udp ENTRYPOINT ["geth"] + +# Add some metadata labels to help programatic image consumption +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + +LABEL commit="$COMMIT" version="$VERSION" buildnum="$BUILDNUM" diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 483afad8c3..044a9a6895 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -1,10 +1,20 @@ +# Support setting various labels on the final image +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + # Build Geth in a stock Go builder container -FROM golang:1.16-alpine as builder +FROM golang:1.19-alpine as builder + +RUN apk add --no-cache gcc musl-dev linux-headers git -RUN apk add --no-cache make gcc musl-dev linux-headers git +# Get dependencies - will also be cached if we won't change go.mod/go.sum +COPY go.mod /go-ethereum/ +COPY go.sum /go-ethereum/ +RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && make all +RUN cd /go-ethereum && go run build/ci.go install -static # Pull all binaries into a second stage deploy alpine container FROM alpine:latest @@ -13,3 +23,10 @@ RUN apk add --no-cache ca-certificates COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/ EXPOSE 8545 8546 30303 30303/udp + +# Add some metadata labels to help programatic image consumption +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + +LABEL commit="$COMMIT" version="$VERSION" buildnum="$BUILDNUM" diff --git a/build/checksums.txt b/build/checksums.txt index 0e33e07f2c..c1490c3329 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,33 +1,37 @@ # This file contains sha256 checksums of optional build dependencies. -b298d29de9236ca47a023e382313bcc2d2eed31dfa706b60a04103ce83a71a25 go1.16.3.src.tar.gz -6bb1cf421f8abc2a9a4e39140b7397cdae6aca3e8d36dcff39a1a77f4f1170ac go1.16.3.darwin-amd64.tar.gz -f4e96bbcd5d2d1942f5b55d9e4ab19564da4fad192012f6d7b0b9b055ba4208f go1.16.3.darwin-arm64.tar.gz -48b2d1481db756c88c18b1f064dbfc3e265ce4a775a23177ca17e25d13a24c5d go1.16.3.linux-386.tar.gz -951a3c7c6ce4e56ad883f97d9db74d3d6d80d5fec77455c6ada6c1f7ac4776d2 go1.16.3.linux-amd64.tar.gz -566b1d6f17d2bc4ad5f81486f0df44f3088c3ed47a3bec4099d8ed9939e90d5d go1.16.3.linux-arm64.tar.gz -0dae30385e3564a557dac7f12a63eedc73543e6da0f6017990e214ce8cc8797c go1.16.3.linux-armv6l.tar.gz -a3c16e1531bf9726f47911c4a9ed7cb665a6207a51c44f10ebad4db63b4bcc5a go1.16.3.windows-386.zip -a4400345135b36cb7942e52bbaf978b66814738b855eeff8de879a09fd99de7f go1.16.3.windows-amd64.zip -31ecd11d497684fa8b0f01ba784590c4c760943665fdc4fe0adaa1405c71736c go1.16.3.freebsd-386.tar.gz -ffbd920b309e62e807457b11d80e8c17fefe3ef6de423aaba4b1e270b2ca4c3d go1.16.3.freebsd-amd64.tar.gz -5eb046bbbbc7fe2591846a4303884cb5a01abb903e3e61e33459affe7874e811 go1.16.3.linux-ppc64le.tar.gz -3e8bd7bde533a73fd6fa75b5288678ef397e76c198cfb26b8ae086035383b1cf go1.16.3.linux-s390x.tar.gz +38f423db4cc834883f2b52344282fa7a39fbb93650dc62a11fdf0be6409bdad6 go1.18.src.tar.gz +70bb4a066997535e346c8bfa3e0dfe250d61100b17ccc5676274642447834969 go1.18.darwin-amd64.tar.gz +9cab6123af9ffade905525d79fc9ee76651e716c85f1f215872b5f2976782480 go1.18.darwin-arm64.tar.gz +e63492d4f38487331518eb4b50e670d853bb8d67e88596269af84bb9aca0b381 go1.18.freebsd-386.tar.gz +01cd67bbc12e659ff236ecebde1806f76452f7ca145c172d5ecdbf4f4803daae go1.18.freebsd-amd64.tar.gz +1c04cf4440b323a66328e0df95d409f955b9b475e58eae235fdd3d1f1cf02f4f go1.18.linux-386.tar.gz +e85278e98f57cdb150fe8409e6e5df5343ecb13cebf03a5d5ff12bd55a80264f go1.18.linux-amd64.tar.gz +7ac7b396a691e588c5fb57687759e6c4db84a2a3bbebb0765f4b38e5b1c5b00e go1.18.linux-arm64.tar.gz +a80fa43d1f4575fb030adbfbaa94acd860c6847820764eecb06c63b7c103612b go1.18.linux-armv6l.tar.gz +070351edac192483c074b38d08ec19251a83f8210765a532a84c3dcf8aec04d8 go1.18.linux-ppc64le.tar.gz +ea265f5e62fcaf941d53f0cdb81222d9668e1672a0d39d992f16ff0e87c0ee6b go1.18.linux-s390x.tar.gz +e23fd2a0509690fe7e63b2b1bcd4c39ed57b46ccde76f35dc0d16ca7fdbc5aaa go1.18.windows-386.zip +65c5c0c709a7ca1b357091b10b795b439d8b50e579d3893edab4c7e9b384f435 go1.18.windows-amd64.zip +1c454eb60c64d481965a165c623ff1ed6cf32d68c6b31f36069c8768d908f093 go1.18.windows-arm64.zip -7e9a47ab540aa3e8472fbf8120d28bed3b9d9cf625b955818e8bc69628d7187c golangci-lint-1.39.0-darwin-amd64.tar.gz -574daa2c9c299b01672a6daeb1873b5f12e413cdb6dc0e30f2ff163956778064 golangci-lint-1.39.0-darwin-arm64.tar.gz -6225f7014987324ab78e9b511f294e3f25be013728283c33918c67c8576d543e golangci-lint-1.39.0-freebsd-386.tar.gz -6b3e76e1e5eaf0159411c8e2727f8d533989d3bb19f10e9caa6e0b9619ee267d golangci-lint-1.39.0-freebsd-amd64.tar.gz -a301cacfff87ed9b00313d95278533c25a4527a06b040a17d969b4b7e1b8a90d golangci-lint-1.39.0-freebsd-armv7.tar.gz -25bfd96a29c3112f508d5e4fc860dbad7afce657233c343acfa20715717d51e7 golangci-lint-1.39.0-freebsd-armv6.tar.gz -9687e4ff15545cfc722b0e46107a94195166a505023b48a316579af25ad09505 golangci-lint-1.39.0-linux-armv7.tar.gz -a7fa7ab2bfc99cbe5e5bcbf5684f5a997f920afbbe2f253d2feb1001d5e3c8b3 golangci-lint-1.39.0-linux-armv6.tar.gz -c8f9634115beddb4ed9129c1f7ecd4c97c99d07aeef33e3707234097eeb51b7b golangci-lint-1.39.0-linux-mips64le.tar.gz -d1234c213b74751f1af413302dde0e9a6d4d29aecef034af7abb07dc1b6e887f golangci-lint-1.39.0-linux-arm64.tar.gz -df25d9267168323b163147acb823ab0215a8a3bb6898a4a9320afdfedde66817 golangci-lint-1.39.0-linux-386.tar.gz -1767e75fba357b7651b1a796d38453558f371c60af805505ec99e166908c04b5 golangci-lint-1.39.0-linux-ppc64le.tar.gz -25fd75bf3186b3d930ecae10185689968fd18fd8fa6f9f555d6beb04348c20f6 golangci-lint-1.39.0-linux-s390x.tar.gz -3a73aa7468087caa62673c8adea99b4e4dff846dc72707222db85f8679b40cbf golangci-lint-1.39.0-linux-amd64.tar.gz -578caceccf81739bda67dbfec52816709d03608c6878888ecdc0e186a094a41b golangci-lint-1.39.0-linux-mips64.tar.gz -494b66ba0e32c8ddf6c4f6b1d05729b110900f6017eda943057e43598c17d7a8 golangci-lint-1.39.0-windows-386.zip -52ec2e13a3cbb47147244dff8cfc35103563deb76e0459133058086fc35fb2c7 golangci-lint-1.39.0-windows-amd64.zip +d4bd25b9814eeaa2134197dd2c7671bb791eae786d42010d9d788af20dee4bfa golangci-lint-1.42.0-darwin-amd64.tar.gz +e56859c04a2ad5390c6a497b1acb1cc9329ecb1010260c6faae9b5a4c35b35ea golangci-lint-1.42.0-darwin-arm64.tar.gz +14d912a3fa856830339472fc4dc341933adf15f37bdb7130bbbfcf960ecf4809 golangci-lint-1.42.0-freebsd-386.tar.gz +337257fccc9baeb5ee1cd7e70c153e9d9f59d3afde46d631659500048afbdf80 golangci-lint-1.42.0-freebsd-amd64.tar.gz +6debcc266b629359fdd8eef4f4abb05a621604079d27016265afb5b4593b0eff golangci-lint-1.42.0-freebsd-armv6.tar.gz +878f0e190169db2ce9dde8cefbd99adc4fe28b90b68686bbfcfcc2085e6d693e golangci-lint-1.42.0-freebsd-armv7.tar.gz +42c78e31faf62b225363eff1b1d2aa74f9dbcb75686c8914aa3e90d6af65cece golangci-lint-1.42.0-linux-386.tar.gz +6937f62f8e2329e94822dc11c10b871ace5557ae1fcc4ee2f9980cd6aecbc159 golangci-lint-1.42.0-linux-amd64.tar.gz +2cf8d23d96cd854a537b355dab2962b960b88a06b615232599f066afd233f246 golangci-lint-1.42.0-linux-arm64.tar.gz +08b003d1ed61367473886defc957af5301066e62338e5d96a319c34dadc4c1d1 golangci-lint-1.42.0-linux-armv6.tar.gz +c7c00ec4845e806a1f32685f5b150219e180bd6d6a9d584be8d27f0c41d7a1bf golangci-lint-1.42.0-linux-armv7.tar.gz +3650fcf29eb3d8ee326d77791a896b15259eb2d5bf77437dc72e7efe5af6bd40 golangci-lint-1.42.0-linux-mips64.tar.gz +f51ae003fdbca4fef78ba73e2eb736a939c8eaa178cd452234213b489da5a420 golangci-lint-1.42.0-linux-mips64le.tar.gz +1b0bb7b8b22cc4ea7da44fd5ad5faaf6111d0677e01cc6f961b62a96537de2c6 golangci-lint-1.42.0-linux-ppc64le.tar.gz +8cb56927eb75e572450efbe0ff0f9cf3f56dc9faa81d9e8d30d6559fc1d06e6d golangci-lint-1.42.0-linux-riscv64.tar.gz +5ac41cd31825a176b21505a371a7b307cd9cdf17df0f35bbb3bf1466f9356ccc golangci-lint-1.42.0-linux-s390x.tar.gz +e1cebd2af621ac4b64c20937df92c3819264f2174c92f51e196db1e64ae097e0 golangci-lint-1.42.0-windows-386.zip +7e70fcde8e87a17cae0455df07d257ebc86669f3968d568e12727fa24bbe9883 golangci-lint-1.42.0-windows-amd64.zip +59da7ce1bda432616bfc28ae663e52c3675adee8d9bf5959fafd657c159576ab golangci-lint-1.42.0-windows-armv6.zip +65f62dda937bfcede0326ac77abe947ce1548931e6e13298ca036cb31f224db5 golangci-lint-1.42.0-windows-armv7.zip diff --git a/build/ci.go b/build/ci.go index 3acd3691f1..974e9d3ff9 100644 --- a/build/ci.go +++ b/build/ci.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build none // +build none /* @@ -32,7 +33,6 @@ Available commands are: nsis -- creates a Windows NSIS installer aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework - xgo [ -alltools ] [ options ] -- cross builds according to options purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore For all commands, -n prevents execution of external programs (dry run mode). @@ -54,6 +54,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "time" @@ -128,20 +129,15 @@ var ( // Distros for which packages are created. // Note: vivid is unsupported because there is no golang-1.6 package for it. - // Note: wily is unsupported because it was officially deprecated on Launchpad. - // Note: yakkety is unsupported because it was officially deprecated on Launchpad. - // Note: zesty is unsupported because it was officially deprecated on Launchpad. - // Note: artful is unsupported because it was officially deprecated on Launchpad. - // Note: cosmic is unsupported because it was officially deprecated on Launchpad. - // Note: disco is unsupported because it was officially deprecated on Launchpad. - // Note: eoan is unsupported because it was officially deprecated on Launchpad. + // Note: the following Ubuntu releases have been officially deprecated on Launchpad: + // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite debDistroGoBoots = map[string]string{ - "trusty": "golang-1.11", - "xenial": "golang-go", - "bionic": "golang-go", - "focal": "golang-go", - "groovy": "golang-go", - "hirsute": "golang-go", + "trusty": "golang-1.11", // EOL: 04/2024 + "xenial": "golang-go", // EOL: 04/2026 + "bionic": "golang-go", // EOL: 04/2028 + "focal": "golang-go", // EOL: 04/2030 + "impish": "golang-go", // EOL: 07/2022 + // "jammy": "golang-go", // EOL: 04/2027 } debGoBootPaths = map[string]string{ @@ -152,7 +148,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.16.3" + dlgoVersion = "1.18" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -182,6 +178,8 @@ func main() { doLint(os.Args[2:]) case "archive": doArchive(os.Args[2:]) + case "docker": + doDocker(os.Args[2:]) case "debsrc": doDebianSource(os.Args[2:]) case "nsis": @@ -190,8 +188,6 @@ func main() { doAndroidArchive(os.Args[2:]) case "xcode": doXCodeFramework(os.Args[2:]) - case "xgo": - doXgo(os.Args[2:]) case "purge": doPurge(os.Args[2:]) default: @@ -203,9 +199,10 @@ func main() { func doInstall(cmdline []string) { var ( - dlgo = flag.Bool("dlgo", false, "Download Go and build with it") - arch = flag.String("arch", "", "Architecture to cross build for") - cc = flag.String("cc", "", "C compiler to cross build with") + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Architecture to cross build for") + cc = flag.String("cc", "", "C compiler to cross build with") + staticlink = flag.Bool("static", false, "Create statically-linked executable") ) flag.CommandLine.Parse(cmdline) @@ -216,9 +213,12 @@ func doInstall(cmdline []string) { tc.Root = build.DownloadGo(csdb, dlgoVersion) } + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + // Configure the build. env := build.Env() - gobuild := tc.Go("build", buildFlags(env)...) + gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, // better disable it. This check isn't the best, it should probably @@ -251,7 +251,7 @@ func doInstall(cmdline []string) { } // buildFlags returns the go tool flags for building. -func buildFlags(env build.Environment) (flags []string) { +func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { var ld []string if env.Commit != "" { ld = append(ld, "-X", "main.gitCommit="+env.Commit) @@ -262,9 +262,24 @@ func buildFlags(env build.Environment) (flags []string) { if runtime.GOOS == "darwin" { ld = append(ld, "-s") } + if runtime.GOOS == "linux" { + // Enforce the stacksize to 8M, which is the case on most platforms apart from + // alpine Linux. + extld := []string{"-Wl,-z,stack-size=0x800000"} + if staticLinking { + extld = append(extld, "-static") + // Under static linking, use of certain glibc features must be + // disabled to avoid shared library dependencies. + buildTags = append(buildTags, "osusergo", "netgo") + } + ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") + } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } + if len(buildTags) > 0 { + flags = append(flags, "-tags", strings.Join(buildTags, ",")) + } return flags } @@ -279,11 +294,12 @@ func doTest(cmdline []string) { cc = flag.String("cc", "", "Sets C compiler binary") coverage = flag.Bool("coverage", false, "Whether to record code coverage") verbose = flag.Bool("v", false, "Whether to log verbosely") + race = flag.Bool("race", false, "Execute the race detector") ) flag.CommandLine.Parse(cmdline) // Configure the toolchain. - tc := &build.GoToolchain{GOARCH: *arch, CC: *cc} + tc := build.GoToolchain{GOARCH: *arch, CC: *cc} if *dlgo { csdb := build.MustLoadChecksums("build/checksums.txt") tc.Root = build.DownloadGo(csdb, dlgoVersion) @@ -292,22 +308,21 @@ func doTest(cmdline []string) { // Test a single package at a time. CI builders are slow // and some tests run into timeouts under load. - gotest.Args = append(gotest.Args, "-p", "1", "--short") + gotest.Args = append(gotest.Args, "-p", "1") if *coverage { gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") } if *verbose { gotest.Args = append(gotest.Args, "-v") } + if *race { + gotest.Args = append(gotest.Args, "-race") + } packages := []string{"./..."} if len(flag.CommandLine.Args()) > 0 { packages = flag.CommandLine.Args() } - // Quorum - // Ignore not Quorum related packages to accelerate build - packages = build.ExpandPackagesNoVendor(tc, packages) - packages = build.IgnorePackages(packages) gotest.Args = append(gotest.Args, packages...) build.MustRun(gotest) } @@ -331,12 +346,21 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.39.0" + const version = "1.42.0" csdb := build.MustLoadChecksums("build/checksums.txt") - base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) - url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base) - archivePath := filepath.Join(cachedir, base+".tar.gz") + arch := runtime.GOARCH + ext := ".tar.gz" + + if runtime.GOOS == "windows" { + ext = ".zip" + } + if arch == "arm" { + arch += "v" + os.Getenv("GOARM") + } + base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) + url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext) + archivePath := filepath.Join(cachedir, base+ext) if err := csdb.DownloadFile(url, archivePath); err != nil { log.Fatal(err) } @@ -436,11 +460,6 @@ func archiveUpload(archive string, blobstore string, signer string, signifyVar s return err } } - if signifyVar != "" { - if err := build.AzureBlobstoreUpload(archive+".sig", filepath.Base(archive+".sig"), auth); err != nil { - return err - } - } } return nil } @@ -456,11 +475,177 @@ func maybeSkipArchive(env build.Environment) { os.Exit(0) } if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { - log.Printf("skipping archive creation because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) + log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag) os.Exit(0) } } +// Builds the docker images and optionally uploads them to Docker Hub. +func doDocker(cmdline []string) { + var ( + image = flag.Bool("image", false, `Whether to build and push an arch specific docker image`) + manifest = flag.String("manifest", "", `Push a multi-arch docker image for the specified architectures (usually "amd64,arm64")`) + upload = flag.String("upload", "", `Where to upload the docker image (usually "ethereum/client-go")`) + ) + flag.CommandLine.Parse(cmdline) + + // Skip building and pushing docker images for PR builds + env := build.Env() + maybeSkipArchive(env) + + // Retrieve the upload credentials and authenticate + user := getenvBase64("DOCKER_HUB_USERNAME") + pass := getenvBase64("DOCKER_HUB_PASSWORD") + + if len(user) > 0 && len(pass) > 0 { + auther := exec.Command("docker", "login", "-u", string(user), "--password-stdin") + auther.Stdin = bytes.NewReader(pass) + build.MustRun(auther) + } + // Retrieve the version infos to build and push to the following paths: + // - ethereum/client-go:latest - Pushes to the master branch, Geth only + // - ethereum/client-go:stable - Version tag publish on GitHub, Geth only + // - ethereum/client-go:alltools-latest - Pushes to the master branch, Geth & tools + // - ethereum/client-go:alltools-stable - Version tag publish on GitHub, Geth & tools + // - ethereum/client-go:release-. - Version tag publish on GitHub, Geth only + // - ethereum/client-go:alltools-release-. - Version tag publish on GitHub, Geth & tools + // - ethereum/client-go:v.. - Version tag publish on GitHub, Geth only + // - ethereum/client-go:alltools-v.. - Version tag publish on GitHub, Geth & tools + var tags []string + + switch { + case env.Branch == "master": + tags = []string{"latest"} + case strings.HasPrefix(env.Tag, "v1."): + tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version} + } + // If architecture specific image builds are requested, build and push them + if *image { + build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".") + build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".") + + // Tag and upload the images to Docker Hub + for _, tag := range tags { + gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH) + toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH) + + // If the image already exists (non version tag), check the build + // number to prevent overwriting a newer commit if concurrent builds + // are running. This is still a tiny bit racey if two published are + // done at the same time, but that's extremely unlikely even on the + // master branch. + for _, img := range []string{gethImage, toolImage} { + if exec.Command("docker", "pull", img).Run() != nil { + continue // Generally the only failure is a missing image, which is good + } + buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() + if err != nil { + log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) + } + buildnum = bytes.TrimSpace(buildnum) + + if len(buildnum) > 0 && len(env.Buildnum) > 0 { + oldnum, err := strconv.Atoi(string(buildnum)) + if err != nil { + log.Fatalf("Failed to parse old image build number: %v", err) + } + newnum, err := strconv.Atoi(env.Buildnum) + if err != nil { + log.Fatalf("Failed to parse current build number: %v", err) + } + if oldnum > newnum { + log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum) + } else { + log.Printf("Updating %s from build %d to %d", img, oldnum, newnum) + } + } + } + build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), gethImage) + build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage) + build.MustRunCommand("docker", "push", gethImage) + build.MustRunCommand("docker", "push", toolImage) + } + } + // If multi-arch image manifest push is requested, assemble it + if len(*manifest) != 0 { + // Since different architectures are pushed by different builders, wait + // until all required images are updated. + var mismatch bool + for i := 0; i < 2; i++ { // 2 attempts, second is race check + mismatch = false // hope there's no mismatch now + + for _, tag := range tags { + for _, arch := range strings.Split(*manifest, ",") { + gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch) + toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch) + + for _, img := range []string{gethImage, toolImage} { + if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil { + log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out) + mismatch = true + break + } + buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() + if err != nil { + log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) + } + buildnum = bytes.TrimSpace(buildnum) + + if string(buildnum) != env.Buildnum { + log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum) + mismatch = true + break + } + } + if mismatch { + break + } + } + if mismatch { + break + } + } + if mismatch { + // Build numbers mismatching, retry in a short time to + // avoid concurrent failes in both publisher images. If + // however the retry failed too, it means the concurrent + // builder is still crunching, let that do the publish. + if i == 0 { + time.Sleep(30 * time.Second) + } + continue + } + break + } + if mismatch { + log.Println("Relinquishing publish to other builder") + return + } + // Assemble and push the Geth manifest image + for _, tag := range tags { + gethImage := fmt.Sprintf("%s:%s", *upload, tag) + + var gethSubImages []string + for _, arch := range strings.Split(*manifest, ",") { + gethSubImages = append(gethSubImages, gethImage+"-"+arch) + } + build.MustRunCommand("docker", append([]string{"manifest", "create", gethImage}, gethSubImages...)...) + build.MustRunCommand("docker", "manifest", "push", gethImage) + } + // Assemble and push the alltools manifest image + for _, tag := range tags { + toolImage := fmt.Sprintf("%s:alltools-%s", *upload, tag) + + var toolSubImages []string + for _, arch := range strings.Split(*manifest, ",") { + toolSubImages = append(toolSubImages, toolImage+"-"+arch) + } + build.MustRunCommand("docker", append([]string{"manifest", "create", toolImage}, toolSubImages...)...) + build.MustRunCommand("docker", "manifest", "push", toolImage) + } + } +} + // Debian Packaging func doDebianSource(cmdline []string) { var ( @@ -1045,48 +1230,6 @@ func newPodMetadata(env build.Environment, archive string) podMetadata { } } -// Cross compilation - -func doXgo(cmdline []string) { - var ( - alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`) - ) - flag.CommandLine.Parse(cmdline) - env := build.Env() - var tc build.GoToolchain - - // Make sure xgo is available for cross compilation - build.MustRun(tc.Install(GOBIN, "github.com/karalabe/xgo@latest")) - - // If all tools building is requested, build everything the builder wants - args := append(buildFlags(env), flag.Args()...) - - if *alltools { - args = append(args, []string{"--dest", GOBIN}...) - for _, res := range allToolsArchiveFiles { - if strings.HasPrefix(res, GOBIN) { - // Binary tool found, cross build it explicitly - args = append(args, "./"+filepath.Join("cmd", filepath.Base(res))) - build.MustRun(xgoTool(args)) - args = args[:len(args)-1] - } - } - return - } - - // Otherwise execute the explicit cross compilation - path := args[len(args)-1] - args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...) - build.MustRun(xgoTool(args)) -} - -func xgoTool(args []string) *exec.Cmd { - cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, []string{"GOBIN=" + GOBIN}...) - return cmd -} - // Binary distribution cleanups func doPurge(cmdline []string) { diff --git a/internal/build/util.go b/internal/build/util.go index cd58fa9a61..fcb1b51fe8 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -29,6 +29,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "text/template" ) @@ -38,7 +39,7 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands") // MustRun executes the given command and exits the host process for // any error. func MustRun(cmd *exec.Cmd) { - fmt.Println(">>>", strings.Join(cmd.Args, " ")) + fmt.Println(">>>", printArgs(cmd.Args)) if !*DryRunFlag { cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout @@ -48,6 +49,20 @@ func MustRun(cmd *exec.Cmd) { } } +func printArgs(args []string) string { + var s strings.Builder + for i, arg := range args { + if i > 0 { + s.WriteByte(' ') + } + if strings.IndexByte(arg, ' ') >= 0 { + arg = strconv.QuoteToASCII(arg) + } + s.WriteString(arg) + } + return s.String() +} + func MustRunCommand(cmd string, args ...string) { MustRun(exec.Command(cmd, args...)) } @@ -121,7 +136,7 @@ func UploadSFTP(identityFile, host, dir string, files []string) error { sftp.Args = append(sftp.Args, "-i", identityFile) } sftp.Args = append(sftp.Args, host) - fmt.Println(">>>", strings.Join(sftp.Args, " ")) + fmt.Println(">>>", printArgs(sftp.Args)) if *DryRunFlag { return nil } diff --git a/mobile/android_test.go b/mobile/android_test.go index c85314c157..fb16b4879e 100644 --- a/mobile/android_test.go +++ b/mobile/android_test.go @@ -155,6 +155,8 @@ public class AndroidTest extends InstrumentationTestCase { // // This method has been adapted from golang.org/x/mobile/bind/java/seq_test.go/runTest func TestAndroid(t *testing.T) { + t.Skip("Android mobile has been nuked in future versions") + // Skip tests on Windows altogether if runtime.GOOS == "windows" { t.Skip("cannot test Android bindings on Windows, skipping") diff --git a/params/version.go b/params/version.go index 6a4403056f..0635f5a1e6 100644 --- a/params/version.go +++ b/params/version.go @@ -28,7 +28,7 @@ const ( QuorumVersionMajor = 22 QuorumVersionMinor = 7 - QuorumVersionPatch = 5 + QuorumVersionPatch = 6 ) // Version holds the textual version string.