Merge pull request #4460 from stacks-network/ci/dry_x64_builds

[CI] Dry x86-64-v3 builds/default images to use glibc

.github/actions/dockerfiles/Dockerfile.alpine-binary

@@ -7,16 +7,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG TARGETVARIANT
-ARG REPO=stacks-network/stacks-core
+ARG REPO
 
-RUN case ${TARGETARCH} in \
-    "amd64")  BIN_ARCH=linux-musl-x64  ;; \
-    "arm64")  BIN_ARCH=linux-musl-arm64  ;; \
-    "arm")  BIN_ARCH=linux-musl-armv7  ;; \
-    "*") exit 1 ;; \
+RUN case ${TARGETPLATFORM} in \
+    linux/amd64/v2) BIN_ARCH=linux-glibc-x64-v2 ;; \
+    linux/amd64*) BIN_ARCH=linux-glibc-x64 ;; \
+    linux/arm64*) BIN_ARCH=linux-glibc-arm64 ;; \
+    linux/arm/v7) BIN_ARCH=linux-glibc-armv7 ;; \
+    *) exit 1 ;; \
     esac \
-    && echo "wget  -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
-    && wget  -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
+    && echo "TARGETPLATFORM: $TARGETPLATFORM" \
+    && echo "BIN_ARCH: $BIN_ARCH" \
+    && echo "wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
+    && wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
     && unzip ${BIN_ARCH}.zip -d /out
 
 FROM --platform=${TARGETPLATFORM} alpine

.github/actions/dockerfiles/Dockerfile.debian-binary

@@ -7,16 +7,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG TARGETVARIANT
-ARG REPO=stacks-network/stacks-core
+ARG REPO
 
-RUN case ${TARGETARCH} in \
-    "amd64")  BIN_ARCH=linux-musl-x64  ;; \
-    "arm64")  BIN_ARCH=linux-musl-arm64  ;; \
-    "arm")  BIN_ARCH=linux-musl-armv7  ;; \
-    "*") exit 1 ;; \
+RUN case ${TARGETPLATFORM} in \
+    linux/amd64/v2) BIN_ARCH=linux-glibc-x64-v2 ;; \
+    linux/amd64*) BIN_ARCH=linux-glibc-x64 ;; \
+    linux/arm64*) BIN_ARCH=linux-glibc-arm64 ;; \
+    linux/arm/v7) BIN_ARCH=linux-glibc-armv7 ;; \
+    *) exit 1 ;; \
     esac \
-    && echo "wget  -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
-    && wget  -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
+    && echo "TARGETPLATFORM: $TARGETPLATFORM" \
+    && echo "BIN_ARCH: $BIN_ARCH" \
+    && echo "wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
+    && wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
     && unzip ${BIN_ARCH}.zip -d /out
 
 FROM --platform=${TARGETPLATFORM} debian:bookworm

.github/actions/dockerfiles/Dockerfile.debian-source

@@ -1,10 +1,13 @@
-FROM rust:bullseye as build
+FROM rust:bookworm as build
 
 ARG STACKS_NODE_VERSION="No Version Info"
 ARG GIT_BRANCH='No Branch Info'
 ARG GIT_COMMIT='No Commit Info'
 ARG BUILD_DIR=/build
 ARG TARGET=x86_64-unknown-linux-gnu
+# Allow us to override the default `--target-cpu` for the given target triplet
+ARG TARGET_CPU
+ENV RUSTFLAGS="${TARGET_CPU:+${RUSTFLAGS} -Ctarget-cpu=${TARGET_CPU}}"
 WORKDIR /src
 
 COPY . .

.github/workflows/create-source-binary.yml

@@ -9,12 +9,6 @@ on:
         description: "Tag name of this release (x.y.z)"
         required: true
         type: string
-      arch:
-        description: "Stringified JSON object listing of platform matrix"
-        required: false
-        type: string
-        default: >-
-          ["linux-glibc-arm64", "linux-glibc-armv7", "linux-musl-arm64", "linux-musl-armv7"]
 
 ## change the display name to the tag being built
 run-name: ${{ inputs.tag }}
@@ -30,41 +24,100 @@ jobs:
   ##  - workflow is building default branch (master)
   artifact:
     if: |
-      inputs.tag != '' && 
+      inputs.tag != '' &&
       github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
     name: Build Binaries
     runs-on: ubuntu-latest
     strategy:
       ## Run a maximum of 10 builds concurrently, using the matrix defined in inputs.arch
       max-parallel: 10
       matrix:
-        platform: ${{ fromJson(inputs.arch) }}
+        arch:
+          - linux-musl
+          - linux-glibc
+          - macos
+          - windows
+        cpu:
+          - arm64
+          - armv7
+          - x86-64 ## defaults to x86-64-v3 variant - intel haswell (2013) and newer
+          # - x86-64-v2 ## intel nehalem (2008) and newer
+          # - x86-64-v3 ## intel haswell (2013) and newer
+          # - x86-64-v4 ## intel skylake (2017) and newer
+        exclude:
+          - arch: windows # excludes windows-arm64
+            cpu: arm64
+          - arch: windows # excludes windows-armv7
+            cpu: armv7
+          - arch: macos # excludes macos-armv7
+            cpu: armv7
+
     steps:
       ## Setup Docker for the builds
       - name: Docker setup
+        id: docker_setup
         uses: stacks-network/actions/docker@main
 
+      - name: Set Local env vars
+        id: set_env
+        run: |
+          case ${{ matrix.cpu }} in
+              x86-64)
+                  ## default x64 builds to use v3 variant. TARGET_CPU is required to build for v3 via RUSTFLAGS
+                  TARGET_CPU="${{ matrix.cpu }}-v3"
+                  DOCKERFILE_CPU="x64"
+                  ARCHIVE_NAME="x64"
+                  ;;
+              x86-64-v2)
+                  ## intel nehalem (2008) and newer
+                  TARGET_CPU="${{ matrix.cpu }}"
+                  DOCKERFILE_CPU="x64"
+                  ARCHIVE_NAME="x64-v2"
+                  ;;
+              x86-64-v3)
+                  ## intel haswell (2013) and newer
+                  TARGET_CPU="${{ matrix.cpu }}"
+                  DOCKERFILE_CPU="x64"
+                  ARCHIVE_NAME="x64-v3"
+                  ;;
+              x86-64-v4)
+                  ## intel skylake (2017) and newer
+                  TARGET_CPU="${{ matrix.cpu }}"
+                  DOCKERFILE_CPU="x64"
+                  ARCHIVE_NAME="x64-v4"
+                  ;;
+              *)
+                  TARGET_CPU=""
+                  DOCKERFILE_CPU="${{ matrix.cpu }}"
+                  ARCHIVE_NAME="${{ matrix.cpu }}"
+                  ;;
+          esac
+          echo "DOCKERFILE=Dockerfile.${{ matrix.arch }}-${DOCKERFILE_CPU}" >> "$GITHUB_ENV"
+          echo "ZIPFILE=${{ matrix.arch }}-${ARCHIVE_NAME}" >> "$GITHUB_ENV"
+          echo "TARGET_CPU=${TARGET_CPU}" >> "$GITHUB_ENV"
+
       ## Build the binaries using defined dockerfiles
-      - name: Build Binary (${{ matrix.platform }})
+      - name: Build Binary (${{ matrix.arch }}_${{ matrix.cpu }})
         id: build_binaries
         uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # 5.0.0
         with:
-          file: build-scripts/Dockerfile.${{ matrix.platform }}
-          outputs: type=local,dest=./release/${{ matrix.platform }}
+          file: build-scripts/${{ env.DOCKERFILE }}
+          outputs: type=local,dest=./release/${{ matrix.arch }}
           build-args: |
             STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }}
-            OS_ARCH=${{ matrix.platform }}
+            OS_ARCH=${{ matrix.arch }}
+            TARGET_CPU=${{ env.TARGET_CPU }}
             GIT_BRANCH=${{ env.GITHUB_REF_SHORT }}
             GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }}
 
       ## Compress the binary artifact
       - name: Compress artifact
         id: compress_artifact
-        run: zip --junk-paths ${{ matrix.platform }} ./release/${{ matrix.platform }}/*
+        run: zip --junk-paths ${{ env.ZIPFILE }} ./release/${{ matrix.arch }}/*
 
       ## Upload the binary artifact to the github action (used in `github-release.yml` to create a release)
       - name: Upload artifact
         id: upload_artifact
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
-          path: ${{ matrix.platform }}.zip
+          path: ${{ env.ZIPFILE }}.zip

.github/workflows/github-release.yml

@@ -36,21 +36,6 @@ jobs:
       tag: ${{ inputs.tag }}
     secrets: inherit
 
-  ## Build x86_64 binaries from source
-  ##
-  ## Runs when the following is true:
-  ##  - tag is provided
-  ##  - workflow is building default branch (master)
-  build-binaries-x64:
-    if: |
-      inputs.tag != '' && 
-      github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
-    name: Build Binaries (x64_64)
-    uses: ./.github/workflows/create-source-binary-x64.yml
-    with:
-      tag: ${{ inputs.tag }}
-    secrets: inherit
-
   ## Runs when the following is true:
   ##  - tag is provided
   ##  - workflow is building default branch (master)
@@ -62,7 +47,6 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - build-binaries
-      - build-binaries-x64
     steps:
       ## Downloads the artifacts built in `create-source-binary.yml`
       - name: Download Artifacts
@@ -111,7 +95,6 @@ jobs:
     uses: ./.github/workflows/image-build-binary.yml
     needs:
       - build-binaries
-      - build-binaries-x64
       - create-release
     with:
       tag: ${{ inputs.tag }}

.github/workflows/image-build-binary.yml

@@ -8,16 +8,11 @@ on:
       tag:
         required: true
         type: string
-        description: "Version tag for alpine images"
-      docker-org:
-        required: false
-        type: string
-        description: "Docker repo org for uploading images (defaults to github org)"
-        default: "${GITHUB_REPOSITORY_OWNER}"
+        description: "Version tag for docker images"
 
 ## Define which docker arch to build for
 env:
-  docker_platforms: "linux/arm64, linux/arm/v7, linux/amd64, linux/amd64/v2, linux/amd64/v3"
+  docker_platforms: "linux/arm64, linux/arm/v7, linux/amd64, linux/amd64/v3"
   docker-org: blockstack
 
 concurrency:
@@ -48,28 +43,39 @@ jobs:
     steps:
       ## Setup Docker for the builds
       - name: Docker setup
+        id: docker_setup
         uses: stacks-network/actions/docker@main
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      ## if the repo owner is not `stacks-network`, default to a docker-org of the repo owner (i.e. github user id)
+      ## this allows forks to run the docker push workflows without having to hardcode a dockerhub org (but it does require docker hub user to match github username)
+      - name: Set Local env vars
+        id: set_env
+        if: |
+          github.repository_owner != 'stacks-network'
+        run: |
+          echo "docker-org=${{ github.repository_owner }}" >> "$GITHUB_ENV"
+
       ## Set docker metatdata
       ## - depending on the matrix.dist, different tags will be enabled
-      ## ex. alpine will have this tag: `type=ref,event=tag,enable=${{ matrix.dist == 'alpine' }}`
+      ## ex. debian will have this tag: `type=ref,event=tag,enable=${{ matrix.dist == 'debian' }}`
       - name: Docker Metadata ( ${{matrix.dist}} )
         id: docker_metadata
         uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 #v5.0.0
         with:
+          ## tag images with current repo name `stacks-core` as well as legacy `stacks-blockchain`
           images: |
             ${{env.docker-org}}/${{ github.event.repository.name }}
             ${{env.docker-org}}/stacks-blockchain
           tags: |
-            type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'alpine' }}
-            type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine'}}
-            type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine' }}
-            type=ref,event=tag,enable=${{ matrix.dist == 'alpine' }}
-            type=raw,value=latest-${{ matrix.dist }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'debian' }}
-            type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'debian' }}
+            type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'debian' }}
+            type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'debian'}}
+            type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' && matrix.dist == 'debian' }}
+            type=ref,event=tag,enable=${{ matrix.dist == 'debian' }}
+            type=raw,value=latest-${{ matrix.dist }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'alpine' }}
+            type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine' }}
 
       ## Build docker image for release
       - name: Build and Push ( ${{matrix.dist}} )