From 4cce55a3160bdbbffddba907c14d044052f9e1f6 Mon Sep 17 00:00:00 2001 From: Adam Magaluk Date: Tue, 9 Aug 2022 13:54:32 -0400 Subject: [PATCH 1/4] Github Actions use Google Credentials To avoid an issue with google go sdk leaking goroutines we need to ensure it has a proper credential file which avoids it looking up metadata from the GCE metadata endpoints. See: https://github.com/googleapis/google-cloud-go/issues/5430 --- .github/workflows/docker.yml | 1 + .github/workflows/main.yml | 1 + .github/workflows/pullrequest.yml | 3 ++- .github/workflows/test.yml | 35 ++++++++++++++++++++++++++++++- .gitignore | 3 +++ Makefile | 2 +- 6 files changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e668c14a146..35c34541c7d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,3 +41,4 @@ jobs: secrets: REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }} + ARTIFACT_READ_ONLY_GCP_CREDENTIALS: ${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e36d8915251..1340793cac1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,7 @@ jobs: secrets: REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }} + ARTIFACT_READ_ONLY_GCP_CREDENTIALS: ${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }} appimage: needs: test diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index 9375aeb45ed..c168f8ba8de 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -16,10 +16,11 @@ on: jobs: test: - uses: viamrobotics/rdk/.github/workflows/test.yml@main + uses: viamrobotics/rdk/.github/workflows/test.yml@support-credentials secrets: REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }} + ARTIFACT_READ_ONLY_GCP_CREDENTIALS: ${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }} # This lets people add an "appimage" tag to have appimages built for the PR appimage: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1ed5a8d797c..26a3a6b7a24 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,9 +8,12 @@ on: required: true GIT_ACCESS_TOKEN: required: true + ARTIFACT_READ_ONLY_GCP_CREDENTIALS: + required: true env: GOPRIVATE: "github.com/viamrobotics/*,go.viam.com/*" + GOOGLE_APPLICATION_CREDENTIALS_FILENAME: "google-credentials.json" jobs: build_and_test: @@ -38,6 +41,21 @@ jobs: with: fetch-depth: 2 + - name: Authorize GCP + uses: google-github-actions/auth@v0.4.3 + with: + credentials_json: '${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }}' + create_credentials_file: true + export_environment_variables: true + + - name: Move GCP Credential to a known file + run: | + NEW_GOOGLE_APPLICATION_CREDENTIALS=`dirname ${GOOGLE_APPLICATION_CREDENTIALS}`/${GOOGLE_APPLICATION_CREDENTIALS_FILENAME} + mv ${GOOGLE_APPLICATION_CREDENTIALS} ${NEW_GOOGLE_APPLICATION_CREDENTIALS} + echo "GOOGLE_APPLICATION_CREDENTIALS=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV + echo "GOOGLE_GHA_CREDS_PATH=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV + echo "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV + - name: Configure git for private repos run: | sudo -u testbot bash -lc 'echo "machine github.com login viambot password ${{ secrets.REPO_READ_TOKEN }}" > ~/.netrc' @@ -59,7 +77,7 @@ jobs: - name: Test if: matrix.platform == 'linux/amd64' run: | - sudo -u testbot bash -lc 'make cover test-web' + sudo --preserve-env=GOOGLE_APPLICATION_CREDENTIALS -u testbot bash -lc 'make cover test-web' - name: Code Coverage Summary Report if: matrix.platform == 'linux/amd64' @@ -100,6 +118,21 @@ jobs: with: fetch-depth: 2 + - name: Authorize GCP + uses: google-github-actions/auth@v0.4.3 + with: + credentials_json: '${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }}' + create_credentials_file: true + export_environment_variables: true + + - name: Move GCP Credential to a known file + run: | + NEW_GOOGLE_APPLICATION_CREDENTIALS=`dirname ${GOOGLE_APPLICATION_CREDENTIALS}`/${GOOGLE_APPLICATION_CREDENTIALS_FILENAME} + mv ${GOOGLE_APPLICATION_CREDENTIALS} ${NEW_GOOGLE_APPLICATION_CREDENTIALS} + echo "GOOGLE_APPLICATION_CREDENTIALS=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV + echo "GOOGLE_GHA_CREDS_PATH=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV + echo "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV + - name: Clean run: make clean-all diff --git a/.gitignore b/.gitignore index 4e2459205b3..b5a3ab809a1 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ bin/ # exclude files from `ag` search .ignore + +# exclude credential created during CI +google-credentials.json diff --git a/Makefile b/Makefile index dda823fafce..158f94d9da5 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ test-web: build-web # test.short skips tests requiring external hardware (motors/servos) test-pi: go test -c -o $(BIN_OUTPUT_PATH)/test-pi go.viam.com/rdk/component/board/pi/impl - sudo $(BIN_OUTPUT_PATH)/test-pi -test.short -test.v + sudo --preserve-env=GOOGLE_APPLICATION_CREDENTIALS $(BIN_OUTPUT_PATH)/test-pi -test.short -test.v server: go build $(LDFLAGS) -o $(BIN_OUTPUT_PATH)/server web/cmd/server/main.go From b760966c8e277f079bac6cde87119bbb2ee6acf3 Mon Sep 17 00:00:00 2001 From: Adam Magaluk Date: Tue, 9 Aug 2022 14:19:27 -0400 Subject: [PATCH 2/4] use main for test.yaml --- .github/workflows/pullrequest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index c168f8ba8de..3feb3809d2c 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -16,7 +16,7 @@ on: jobs: test: - uses: viamrobotics/rdk/.github/workflows/test.yml@support-credentials + uses: viamrobotics/rdk/.github/workflows/test.yml@main secrets: REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }} From 77eea0d92fb4f763039d79284f8249cdbb057809 Mon Sep 17 00:00:00 2001 From: Adam Magaluk Date: Tue, 9 Aug 2022 16:01:33 -0400 Subject: [PATCH 3/4] use simpler method to add credential file --- .github/workflows/pullrequest.yml | 2 +- .github/workflows/test.yml | 34 ++++++++----------------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index 3feb3809d2c..c168f8ba8de 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -16,7 +16,7 @@ on: jobs: test: - uses: viamrobotics/rdk/.github/workflows/test.yml@main + uses: viamrobotics/rdk/.github/workflows/test.yml@support-credentials secrets: REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 26a3a6b7a24..661959d0775 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,20 +41,11 @@ jobs: with: fetch-depth: 2 - - name: Authorize GCP - uses: google-github-actions/auth@v0.4.3 - with: - credentials_json: '${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }}' - create_credentials_file: true - export_environment_variables: true - - - name: Move GCP Credential to a known file + - name: Create GCP Credential File from secret run: | - NEW_GOOGLE_APPLICATION_CREDENTIALS=`dirname ${GOOGLE_APPLICATION_CREDENTIALS}`/${GOOGLE_APPLICATION_CREDENTIALS_FILENAME} - mv ${GOOGLE_APPLICATION_CREDENTIALS} ${NEW_GOOGLE_APPLICATION_CREDENTIALS} - echo "GOOGLE_APPLICATION_CREDENTIALS=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV - echo "GOOGLE_GHA_CREDS_PATH=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV - echo "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV + GOOGLE_APPLICATION_CREDENTIALS=`pwd`/${GOOGLE_APPLICATION_CREDENTIALS_FILENAME} + echo "${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }}" >> ${GOOGLE_APPLICATION_CREDENTIALS} + echo "GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV - name: Configure git for private repos run: | @@ -118,20 +109,11 @@ jobs: with: fetch-depth: 2 - - name: Authorize GCP - uses: google-github-actions/auth@v0.4.3 - with: - credentials_json: '${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }}' - create_credentials_file: true - export_environment_variables: true - - - name: Move GCP Credential to a known file + - name: Create GCP Credential File from secret run: | - NEW_GOOGLE_APPLICATION_CREDENTIALS=`dirname ${GOOGLE_APPLICATION_CREDENTIALS}`/${GOOGLE_APPLICATION_CREDENTIALS_FILENAME} - mv ${GOOGLE_APPLICATION_CREDENTIALS} ${NEW_GOOGLE_APPLICATION_CREDENTIALS} - echo "GOOGLE_APPLICATION_CREDENTIALS=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV - echo "GOOGLE_GHA_CREDS_PATH=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV - echo "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${NEW_GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV + GOOGLE_APPLICATION_CREDENTIALS=`pwd`/${GOOGLE_APPLICATION_CREDENTIALS_FILENAME} + echo "${{ secrets.ARTIFACT_READ_ONLY_GCP_CREDENTIALS }}" >> ${GOOGLE_APPLICATION_CREDENTIALS} + echo "GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}" >> $GITHUB_ENV - name: Clean run: make clean-all From fab46f4a9234377b18c743cec0055e09b586f1bd Mon Sep 17 00:00:00 2001 From: Adam Magaluk Date: Tue, 9 Aug 2022 16:11:36 -0400 Subject: [PATCH 4/4] removed tag to branch --- .github/workflows/pullrequest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index c168f8ba8de..3feb3809d2c 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -16,7 +16,7 @@ on: jobs: test: - uses: viamrobotics/rdk/.github/workflows/test.yml@support-credentials + uses: viamrobotics/rdk/.github/workflows/test.yml@main secrets: REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}