br: migrate br github actions to tidb (#27598)

.github/workflows/br_compatible_test.yml

@@ -0,0 +1,65 @@
+name: BR / Compatibility Test
+
+on:
+  push:
+    branches:
+      - master
+      - 'release-[0-9].[0-9]*'
+    paths:
+      - 'br/**'
+      - '!**.html'
+      - '!**.md'
+      - '!CNAME'
+      - '!LICENSE'
+      - '!br/docs/**'
+      - '!br/tests/**'
+      - '!br/docker/**'
+  pull_request:
+    branches:
+      - master
+      - 'release-[0-9].[0-9]*'
+    paths:
+      - 'br/**'
+      - '!**.html'
+      - '!**.md'
+      - '!CNAME'
+      - '!LICENSE'
+      - '!br/docs/**'
+      - '!br/tests/**'
+      - '!br/docker/**'
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16
+
+    - name: Generate compatibility test backup data
+      timeout-minutes: 15
+      run: sh br/compatibility/prepare_backup.sh
+
+    - name: Start server
+      run: |
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml rm -s -v
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml build
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml up --remove-orphans -d
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml exec -T control make build_br
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f br/compatibility/backup_cluster.yaml exec -T control br/tests/run_compatible.sh run
+
+    - name: Collect component log
+      if: ${{ failure() }}
+      run: |
+        tar czvf ${{ github.workspace }}/logs.tar.gz /tmp/br/docker/backup_logs/*
+
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: logs
+        path: ${{ github.workspace }}/logs.tar.gz

br/.github/workflows/build.yml → .github/workflows/compile_br.yaml

@@ -1,35 +1,37 @@
-name: build
+name: BR & Lightning 
 on:
   push:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
-    paths-ignore:
-      - '**.html'
-      - '**.md'
-      - 'CNAME'
-      - 'LICENSE'
-      - 'docs/**'
-      - 'tests/**'
-      - 'docker/**'
-      - '.github/workflows/**.yml'
+    paths:
+      - 'br/**'
+      - '!**.html'
+      - '!**.md'
+      - '!CNAME'
+      - '!LICENSE'
+      - '!br/docs/**'
+      - '!br/tests/**'
+      - '!br/docker/**'
   pull_request:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
-    paths-ignore:
-      - '**.html'
-      - '**.md'
-      - 'CNAME'
-      - 'LICENSE'
-      - 'docs/**'
-      - 'tests/**'
-      - 'docker/**'
-      - '.github/workflows/**.yml'
+    paths:
+      - 'br/**'
+      - '!**.html'
+      - '!**.md'
+      - '!CNAME'
+      - '!LICENSE'
+      - '!br/docs/**'
+      - '!br/tests/**'
+      - '!br/docker/**'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   compile:
-    name: ${{ matrix.os }} / ${{ matrix.target}}
+    name: Compile for ${{ matrix.os }} / ${{ matrix.target}}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -51,7 +53,7 @@ jobs:
           go-version: 1.16
 
       - name: Run build
-        run: make build
+        run: make build_tools
 
   compile-freebsd:
     name: Compile for FreeBSD job
@@ -65,4 +67,4 @@ jobs:
           go-version: 1.16
 
       - name: Compile for FreeBSD
-        run: GOOS=freebsd make build
+        run: GOOS=freebsd make build_tools

br/README.md

@@ -123,7 +123,7 @@ bin/br restore table --db test \
 mysql --host 127.0.0.1 -P4000 -E -e "SELECT COUNT(*) FROM test.order_line" -uroot -p
 ```
 
-## Compatible test
+## Compatibility test
 
 See [COMPATBILE_TEST](./COMPATIBLE_TEST.md)
 

br/compatibility/backup_cluster.yaml

@@ -6,13 +6,13 @@ services:
   control:
     image: control:nightly
     build:
-      context: ../
-      dockerfile: ./docker/Dockerfile
+      context: ../../
+      dockerfile: ./br/docker/Dockerfile
     volumes:
       - /tmp/br/docker/backup_data/${TAG}:/data
       - /tmp/br/docker/backup_logs/${TAG}:/tmp
       - /tmp/br/docker/backup_logs/${TAG}:/logs
-      - ../bin:/go/src/github.com/pingcap/br/bin
+      - ../../bin:/go/src/github.com/pingcap/br/bin
       - ./prepare_data:/prepare_data
     command: -c "/usr/bin/tail -f /dev/null"
     environment:

br/compatibility/prepare_backup.sh

@@ -16,7 +16,7 @@
 
 set -eux
 
-. compatibility/get_last_tags.sh
+. br/compatibility/get_last_tags.sh
 
 TAGS="v5.0.0"
 getLatestTags
@@ -27,14 +27,14 @@ i=0
 runBackup() {
   # generate backup data in /tmp/br/docker/backup_data/$TAG/, we can do restore after all data backuped later.
   echo "build $1 cluster"
-  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml build
-  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml up -d
-  trap "TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml down" EXIT
+  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f br/compatibility/backup_cluster.yaml build
+  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f br/compatibility/backup_cluster.yaml up -d
+  trap "TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f br/compatibility/backup_cluster.yaml down" EXIT
   # wait for cluster ready
   sleep 20
   # prepare SQL data
-  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml exec -T control /go/bin/go-ycsb load mysql -P /prepare_data/workload -p mysql.host=tidb -p mysql.port=4000 -p mysql.user=root -p mysql.db=test
-  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml exec -T control make compatibility_test_prepare
+  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f br/compatibility/backup_cluster.yaml exec -T control /go/bin/go-ycsb load mysql -P /prepare_data/workload -p mysql.host=tidb -p mysql.port=4000 -p mysql.user=root -p mysql.db=test
+  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f br/compatibility/backup_cluster.yaml exec -T control br/tests/run_compatible.sh prepare
 }
 
 for tag in $TAGS; do

br/docker/Dockerfile

@@ -1,7 +1,7 @@
 # For loading data to TiDB
 FROM golang:1.16.4-buster as go-ycsb-builder
 WORKDIR /go/src/github.com/pingcap/
-RUN git clone https://github.com/pingcap/go-ycsb.git && \
+RUN git clone https://github.com/pingcap/go-ycsb.git --depth=1 && \
     cd go-ycsb && \
     make
 

br/tests/docker_compatible_gcs/run.sh

@@ -31,15 +31,16 @@ EOF
 )
 
 # save CREDENTIALS to file
-echo $KEY > "tests/$TEST_NAME/config.json"
+echo $KEY > "br/tests/$TEST_NAME/config.json"
 
 # export test CREDENTIALS for gcs oauth
-export GOOGLE_APPLICATION_CREDENTIALS="tests/$TEST_NAME/config.json"
+export GOOGLE_APPLICATION_CREDENTIALS="br/tests/$TEST_NAME/config.json"
 
 # restore backup data one by one
 for TAG in ${TAGS}; do
     echo "restore ${TAG} data starts..."
-    bin/br restore db --db test -s "gcs://$BUCKET/bk${TAG}" --pd $PD_ADDR --gcs.endpoint="http://$GCS_HOST:$GCS_PORT/storage/v1/"
+    # after BR merged into TiDB we need skip version check because the build from tidb is not a release version.
+    bin/br restore db --db test -s "gcs://$BUCKET/bk${TAG}" --pd $PD_ADDR --gcs.endpoint="http://$GCS_HOST:$GCS_PORT/storage/v1/" --check-requirements=false
     row_count=$(run_sql_in_container  "SELECT COUNT(*) FROM test.usertable;" | awk '/COUNT/{print $2}')
     if [ $row_count != $EXPECTED_KVS ]; then
        echo "restore kv count is not as expected(1000) $row_count"

br/tests/docker_compatible_s3/run.sh

@@ -26,7 +26,8 @@ S3_KEY="&access-key=$MINIO_ACCESS_KEY&secret-access-key=$MINIO_SECRET_KEY"
 # restore backup data one by one
 for TAG in ${TAGS}; do
     echo "restore ${TAG} data starts..."
-    bin/br restore db --db test -s "s3://$BUCKET/bk${TAG}?endpoint=http://$S3_ENDPOINT$S3_KEY" --pd $PD_ADDR
+    # after BR merged into TiDB we need skip version check because the build from tidb is not a release version.
+    bin/br restore db --db test -s "s3://$BUCKET/bk${TAG}?endpoint=http://$S3_ENDPOINT$S3_KEY" --pd $PD_ADDR --check-requirements=false
     row_count=$(run_sql_in_container  "SELECT COUNT(*) FROM test.usertable;" | awk '/COUNT/{print $2}')
     if [ $row_count != $EXPECTED_KVS ]; then
        echo "restore kv count is not as expected(1000), obtain $row_count"

br/tests/run_compatible.sh

@@ -32,15 +32,15 @@ TEST_DIR=/tmp/backup_restore_compatibility_test
 mkdir -p "$TEST_DIR"
 rm -f "$TEST_DIR"/*.log &> /dev/null
 
-for script in tests/docker_compatible_*/${1}.sh; do
+for script in br/tests/docker_compatible_*/${1}.sh; do
     echo "*===== Running test $script... =====*"
     TEST_DIR="$TEST_DIR" \
     PD_ADDR="$PD_ADDR" \
     GCS_HOST="$GCS_HOST" \
     GCS_PORT="$GCS_PORT" \
     TAGS="$TAGS" \
     EXPECTED_KVS="$EXPECTED_KVS" \
-    PATH="tests/_utils:bin:$PATH" \
+    PATH="br/tests/_utils:bin:$PATH" \
     TEST_NAME="$(basename "$(dirname "$script")")" \
     BR_LOG_TO_TERM=1 \
     bash "$script"

go.mod

@@ -3,8 +3,7 @@ module github.com/pingcap/tidb
 go 1.16
 
 require (
-	cloud.google.com/go v0.54.0 // indirect
-	cloud.google.com/go/storage v1.6.0
+	cloud.google.com/go/storage v1.16.1
 	github.com/BurntSushi/toml v0.3.1
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect
@@ -30,8 +29,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.3
 	github.com/google/btree v1.0.0
-	github.com/google/go-cmp v0.5.6 // indirect
-	github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/handlers v1.5.1 // indirect
 	github.com/gorilla/mux v1.8.0
@@ -76,26 +74,26 @@ require (
 	github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457
 	github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b
-	go.opencensus.io v0.22.5 // indirect
 	go.uber.org/atomic v1.9.0
 	go.uber.org/automaxprocs v1.4.0
 	go.uber.org/goleak v1.1.10
 	go.uber.org/multierr v1.7.0
 	go.uber.org/zap v1.19.0
 	golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
+	golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069
 	golang.org/x/text v0.3.7
 	golang.org/x/tools v0.1.5
-	google.golang.org/api v0.22.0
-	google.golang.org/grpc v1.29.1
+	google.golang.org/api v0.54.0
+	google.golang.org/grpc v1.40.0
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	modernc.org/mathutil v1.2.2
 	sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
 	sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
 )
 
-// FIXME the official repo has some bug makes br_gcs test failed. https://github.com/googleapis/google-cloud-go/pull/3509
-// replace cloud.google.com/go/storage => github.com/3pointer/google-cloud-go/storage v1.6.1-0.20210108125931-b59bfa0720b2
+// cloud.google.com/go/storage will upgrade grpc to v1.40.0
+// we need keep the replacement until go.etcd.io supports the higher version of grpc.
+replace google.golang.org/grpc => google.golang.org/grpc v1.29.1