From ddd84492ee3a42d2ebe69cdf025c8ac5ccba03f0 Mon Sep 17 00:00:00 2001 From: Darren Worrall Date: Tue, 16 Jan 2024 16:11:50 +0000 Subject: [PATCH 1/2] Build multi-arch debs in test workflow --- .github/workflows/build-deb.sh | 9 ++------ .github/workflows/tests.yml | 40 ++++++++++++++++++++++++++++++++++ Makefile | 12 +++++----- copydb/debian/control | 2 +- sharding/debian/control | 2 +- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-deb.sh b/.github/workflows/build-deb.sh index 6baa68e1..577ffb26 100755 --- a/.github/workflows/build-deb.sh +++ b/.github/workflows/build-deb.sh @@ -6,8 +6,8 @@ sudo apt-get update sudo apt-get install -y make # We set this here, so it's the same between the copydb and sharding debian -# package. -DATETIME=$(date -u +%Y%m%d%H%M%S) +# package, and between different arch builds +DATETIME=$(date -u +%Y%m%d) git status @@ -15,16 +15,11 @@ make copydb-deb DATETIME=${DATETIME} make sharding-deb DATETIME=${DATETIME} cd build -sha256sum *.deb > ghostferry-$GITHUB_SHA.sha256sum - set +x echo "Debian package built successfully as follows:" ls -l ghostferry* -echo "sha256sum:" -cat ghostferry-$GITHUB_SHA.sha256sum - # Make sure the we didn't release a dirty build by accident if ls | grep -q dirty; then echo "ERROR: source directory is not clean! refused to release. showing git status below:" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ac1ae573..df63c2a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -71,4 +71,44 @@ jobs: - name: Running Ruby tests run: bundle exec ruby test/main.rb + build-debs: + strategy: + matrix: + runner: [ubuntu-latest, shopify-ubuntu-arm64] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v2 + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - name: Building Ghostferry + run: .github/workflows/build-deb.sh + + - name: Upload debs + uses: actions/upload-artifact@v3 + with: + name: debs-${{ github.sha }} + path: build/ghostferry* + + checksum-debs: + runs-on: ubuntu-latest + needs: build-debs + steps: + - name: Fetch uploaded artifacts + uses: actions/download-artifact@v3 + with: + name: debs-${{ github.sha }} + - name: shasum + shell: bash + run: | + sha256sum *.deb > ghostferry-$GITHUB_SHA.sha256sum + echo "sha256sum:" + cat ghostferry-$GITHUB_SHA.sha256sum + - name: Upload checksum + uses: actions/upload-artifact@v3 + with: + name: debs-${{ github.sha }} + path: "*.sha256sum" diff --git a/Makefile b/Makefile index 29416f0d..3c88a1ef 100644 --- a/Makefile +++ b/Makefile @@ -26,15 +26,15 @@ BIN_DIR := usr/bin PROJECTS := copydb sharding PROJECT_DEBS := $(foreach name,$(PROJECTS),$(name)-deb) +PLATFORM := $(shell uname -s | tr A-Z a-z) +ARCH := $(shell uname -m | sed 's/x86_64/amd64/g' | sed 's/aarch64/arm64/g') +GOTESTSUM_URL := "https://github.com/gotestyourself/gotestsum/releases/download/v1.7.0/gotestsum_1.7.0_$(PLATFORM)_$(ARCH).tar.gz" + # Target specific variable, set proj to have a valid value. PROJECT_PKG = ./$(proj)/cmd PROJECT_BIN = ghostferry-$(proj) BIN_TARGET = $(GOBIN)/$(PROJECT_BIN) -DEB_TARGET = $(BUILD_DIR)/$(PROJECT_BIN)_$(VERSION_STR).deb - -PLATFORM := $(shell uname -s | tr A-Z a-z) -ARCH := $(shell uname -m | sed 's/x86_64/amd64/g' | sed 's/aarch64/arm64/g') -GOTESTSUM_URL := "https://github.com/gotestyourself/gotestsum/releases/download/v1.7.0/gotestsum_1.7.0_$(PLATFORM)_$(ARCH).tar.gz" +DEB_TARGET = $(BUILD_DIR)/$(PROJECT_BIN)_$(VERSION_STR)_$(ARCH).deb .PHONY: test test-go test-ruby clean reset-deb-dir $(PROJECTS) $(PROJECT_DEBS) .DEFAULT_GOAL := test @@ -46,7 +46,7 @@ $(PROJECTS): $(GOBIN) $(PROJECT_DEBS): LDFLAGS += -X github.com/Shopify/ghostferry.WebUiBasedir=/$(SHARE_DIR) $(PROJECT_DEBS): reset-deb-dir $(eval proj := $(subst -deb,,$@)) - sed -e "s/{version}/$(VERSION_STR)/" $(proj)/debian/control > $(DEB_PREFIX)/DEBIAN/control + sed -e "s/{version}/$(VERSION_STR)/" -e "s/{arch}/$(ARCH)/" $(proj)/debian/control > $(DEB_PREFIX)/DEBIAN/control cp $(proj)/debian/copyright $(DEB_PREFIX)/DEBIAN/copyright go build -ldflags "$(LDFLAGS)" -o $(DEB_PREFIX)/$(BIN_DIR)/$(PROJECT_BIN) $(PROJECT_PKG) cp -ar webui $(DEB_PREFIX)/$(SHARE_DIR) diff --git a/copydb/debian/control b/copydb/debian/control index 56485312..300d9c1b 100644 --- a/copydb/debian/control +++ b/copydb/debian/control @@ -2,7 +2,7 @@ Package: ghostferry-copydb Version: {version}-1 Section: misc Priority: optional -Architecture: amd64 +Architecture: {arch} Maintainer: Shuhao Wu Description: Migrate MySQL data from one server to another without downtime via ghostferry. diff --git a/sharding/debian/control b/sharding/debian/control index 972a5e23..f37106e1 100644 --- a/sharding/debian/control +++ b/sharding/debian/control @@ -2,7 +2,7 @@ Package: ghostferry-sharding Version: {version} Section: misc Priority: optional -Architecture: amd64 +Architecture: {arch} Maintainer: Shuhao Wu Uploaders: Justin Li ,Hormoz Kheradmand Description: Migrate MySQL sharded data from one server to From d18efa5f660a3575f96fb507adb3fd89552bef93 Mon Sep 17 00:00:00 2001 From: Darren Worrall Date: Fri, 9 Feb 2024 12:49:18 +0000 Subject: [PATCH 2/2] Update release workflow for multi-arch --- .github/workflows/release.yml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 25e6f14f..93c7d6fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,8 +6,11 @@ on: - main jobs: - release: - runs-on: ubuntu-latest + build-debs: + strategy: + matrix: + runner: [ubuntu-latest, shopify-ubuntu-arm64] + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v2 @@ -19,7 +22,28 @@ jobs: - name: Building Ghostferry run: .github/workflows/build-deb.sh + - name: Upload debs + uses: actions/upload-artifact@v3 + with: + name: debs-${{ github.sha }} + path: build/ghostferry* + + release: + runs-on: ubuntu-latest + needs: build-debs + steps: + - name: Fetch uploaded artifacts + uses: actions/download-artifact@v3 + with: + name: debs-${{ github.sha }} + - name: shasum + shell: bash + run: | + sha256sum *.deb > ghostferry-$GITHUB_SHA.sha256sum + echo "sha256sum:" + cat ghostferry-$GITHUB_SHA.sha256sum + - name: Releasing Ghostferry to Github - run: gh release create --target ${GITHUB_REF#refs/heads/} -p release-${GITHUB_SHA::7} build/ghostferry* + run: gh release create --target ${GITHUB_REF#refs/heads/} -p release-${GITHUB_SHA::7} ghostferry* env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}