From 1c7ed241364bc391037104e3e51abd7a521d0e77 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Thu, 29 Feb 2024 12:17:53 +0100 Subject: [PATCH 01/31] feat: unified dockerfile --- .dockerignore | 6 +++ Dockerfile | 73 ++++++++++++++++++++++++++++++++++++ package.json | 3 +- packages/webapp/package.json | 57 ++++++++++++++-------------- tsconfig.docker.json | 29 ++++++++++++++ 5 files changed, 138 insertions(+), 30 deletions(-) create mode 100644 Dockerfile create mode 100644 tsconfig.docker.json diff --git a/.dockerignore b/.dockerignore index f5adeae5e7..f13304b67b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,3 +12,9 @@ docker-compose.yml **/nango-data dev/ docs-v2/ +tsconfig.tsbuildinfo +integration-templates +tests +assets/ +vite*.ts +packages/cli/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..29b7847884 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,73 @@ +# ------------------ +# New tmp image +# ------------------ +FROM node:18.19.1-bullseye-slim AS tmp + +# Setup the app WORKDIR +WORKDIR /app/tmp + +# Copy and install dependencies separately from the app's code +# To leverage Docker's cache when no dependency has change +COPY packages/frontend/package.json ./packages/frontend/package.json +COPY packages/jobs/package.json ./packages/jobs/package.json +COPY packages/node-client/package.json ./packages/node-client/package.json +COPY packages/persist/package.json ./packages/persist/package.json +COPY packages/runner/package.json ./packages/runner/package.json +COPY packages/server/package.json ./packages/server/package.json +COPY packages/shared/package.json ./packages/shared/package.json +COPY packages/webapp/package.json ./packages/webapp/package.json +COPY package*.json ./ + +# Install every dependencies +RUN true \ + && npm i + +# At this stage we copy back all sources, nothing can be cached anymore +COPY . /app/tmp + +ARG build_env +ARG git_hash +ENV GIT_HASH ${git_hash:-dev} + +# /!\ It's counter intuitive but do not set NODE_ENV=production before building, it will break some modules +# ENV NODE_ENV=production + +# Build +RUN true \ + && npm run ts-build:docker \ + && npm run webapp-build:${build_env:-staging} + +# Clean src +RUN true \ + && rm -rf packages/*/src \ + && rm -rf packages/webapp/public \ + && rm -rf packages/webapp/node_modules + +# Clean dev dependencies +RUN true \ + && npm prune --omit=dev --omit=peer --omit=optional + +# ---- Web ---- +# Resulting new, minimal image +# This image must have the minimum amount of layers +FROM node:18.19.1-bullseye-slim as web + +ENV PORT=8080 +ENV NODE_ENV=production + +# - Bash is just to be able to log inside the image and have a decent shell +RUN true \ + && apt update && apt-get install -y bash ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false + +# Do not use root to run the app +USER node + +WORKDIR /app/nango + +# Code +COPY --from=tmp --chown=node:node /app/tmp /app/nango + +EXPOSE 8080 diff --git a/package.json b/package.json index b25b3fc042..f41e09b319 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "prettier-watch": "onchange './**/*.{ts,tsx}' -- prettier --write {{changed}}", "lint": "eslint . --ext .ts,.tsx", "lint:fix": "eslint . --ext .ts,.tsx --fix", - "ts-build": "tsc -b --clean packages/shared packages/server packages/cli packages/runner packages/jobs packages/persist && tsc -b tsconfig.build.json && npm run postbuild -ws --if-present && tsc -b packages/webapp/tsconfig.json", + "ts-build": "tsc -b tsconfig.build.json && npm run postbuild -ws --if-present", + "ts-build:docker": "tsc -b tsconfig.docker.json", "docker-build": "docker build -f packages/server/Dockerfile -t nango-server:latest .", "webapp-build:hosted": "cd ./packages/webapp && npm run build:hosted && cd ../..", "webapp-build:staging": "cd ./packages/webapp && npm run build:staging && cd ../..", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 16e1f2143a..2e0fd29b85 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -2,7 +2,33 @@ "name": "@nangohq/webapp", "version": "1.0.0", "private": true, - "dependencies": { + "dependencies": {}, + "scripts": { + "start": "DISABLE_ESLINT_PLUGIN=true react-scripts start", + "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build", + "test": "DISABLE_ESLINT_PLUGIN=true react-scripts test", + "eject": "DISABLE_ESLINT_PLUGIN=true react-scripts eject", + "start:hosted": "env-cmd -f .env.hosted npm run start", + "start:staging": "env-cmd -f .env.staging npm run start", + "start:prod": "env-cmd -f .env.prod npm run start", + "build:hosted": "env-cmd -f .env.hosted npm run build", + "build:enterprise": "env-cmd -f .env.enterprise npm run build", + "build:staging": "env-cmd -f .env.staging npm run build", + "build:prod": "env-cmd -f .env.prod npm run build" + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + }, + "devDependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", @@ -32,34 +58,7 @@ "swr": "^2.2.5", "typescript": "^4.9.5", "web-vitals": "^2.1.4", - "zustand": "^4.3.9" - }, - "scripts": { - "start": "DISABLE_ESLINT_PLUGIN=true react-scripts start", - "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build", - "test": "DISABLE_ESLINT_PLUGIN=true react-scripts test", - "eject": "DISABLE_ESLINT_PLUGIN=true react-scripts eject", - "start:hosted": "env-cmd -f .env.hosted npm run start", - "start:staging": "env-cmd -f .env.staging npm run start", - "start:prod": "env-cmd -f .env.prod npm run start", - "build:hosted": "env-cmd -f .env.hosted npm run build", - "build:enterprise": "env-cmd -f .env.enterprise npm run build", - "build:staging": "env-cmd -f .env.staging npm run build", - "build:prod": "env-cmd -f .env.prod npm run build" - }, - "browserslist": { - "production": [ - ">0.2%", - "not dead", - "not op_mini all" - ], - "development": [ - "last 1 chrome version", - "last 1 firefox version", - "last 1 safari version" - ] - }, - "devDependencies": { + "zustand": "^4.3.9", "@tailwindcss/forms": "^0.5.3", "@types/jest": "^27.5.2", "@types/js-cookie": "^3.0.3", diff --git a/tsconfig.docker.json b/tsconfig.docker.json new file mode 100644 index 0000000000..d025108fd4 --- /dev/null +++ b/tsconfig.docker.json @@ -0,0 +1,29 @@ +{ + "files": [], + "references": [ + { + "path": "packages/frontend" + }, + { + "path": "packages/jobs" + }, + { + "path": "packages/node-client" + }, + { + "path": "packages/persist" + }, + { + "path": "packages/runner" + }, + { + "path": "packages/server" + }, + { + "path": "packages/shared" + }, + { + "path": "packages/webapp" + } + ] +} From 8e4e2421faca9913d6adc7f176b0dd0bf35fa198 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 12:36:12 +0100 Subject: [PATCH 02/31] working --- .dockerignore | 1 + Dockerfile | 29 ++++++++++++++++++++++------- package.json | 1 + scripts/build_docker.sh | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 7 deletions(-) create mode 100755 scripts/build_docker.sh diff --git a/.dockerignore b/.dockerignore index f13304b67b..7270c0ab44 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ tests assets/ vite*.ts packages/cli/ +scripts/ diff --git a/Dockerfile b/Dockerfile index 29b7847884..be4e70e4ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,24 +22,34 @@ COPY package*.json ./ RUN true \ && npm i -# At this stage we copy back all sources, nothing can be cached anymore +# At this stage we copy back all sources COPY . /app/tmp -ARG build_env -ARG git_hash -ENV GIT_HASH ${git_hash:-dev} +# Build the backend separately because it can be cached even we change env vars +RUN true \ + && npm run ts-build:docker # /!\ It's counter intuitive but do not set NODE_ENV=production before building, it will break some modules # ENV NODE_ENV=production +ARG image_env +ARG git_hash +ARG posthog_key +ARG sentry_key -# Build +ENV REACT_APP_ENV $image_env +ENV REACT_APP_PUBLIC_GIT_HASH $git_hash +ENV REACT_APP_PUBLIC_POSTHOG_HOST https://app.posthog.com +ENV REACT_APP_PUBLIC_POSTHOG_KEY $posthog_key +ENV REACT_APP_PUBLIC_SENTRY_KEY $sentry_key + +# Build the frontend RUN true \ - && npm run ts-build:docker \ - && npm run webapp-build:${build_env:-staging} + && npm run -w @nangohq/webapp build # Clean src RUN true \ && rm -rf packages/*/src \ + && rm -rf packages/*/lib \ && rm -rf packages/webapp/public \ && rm -rf packages/webapp/node_modules @@ -52,8 +62,13 @@ RUN true \ # This image must have the minimum amount of layers FROM node:18.19.1-bullseye-slim as web +ARG image_env +ARG git_hash + ENV PORT=8080 ENV NODE_ENV=production +ENV IMAGE_ENV $image_env +ENV GIT_HASH $image_env # - Bash is just to be able to log inside the image and have a decent shell RUN true \ diff --git a/package.json b/package.json index f716b522c7..6f6041fee6 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "webapp-build:prod": "cd ./packages/webapp && npm run build:prod && cd ../..", "webapp-build:enterprise": "cd ./packages/webapp && npm run build:enterprise && cd ../..", "webapp-build:watch": "tsc -b -w packages/webapp/tsconfig.json", + "docker-build:unified": "./scripts/build_docker.sh", "shared:build": "cd ./packages/shared && npm run build && cd ../..", "build:hosted": "npm i && npm run ts-build && npm run webapp-build:hosted ", "build:staging": "npm i && npm run ts-build && npm run webapp-build:staging", diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh new file mode 100755 index 0000000000..852b879fc5 --- /dev/null +++ b/scripts/build_docker.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -xe + +ENV=$1 # enterprise | hosted | prod | staging +GIT_HASH=$2 +POSTHOG_KEY=$3 +SENTRY_KEY=$4 + +USAGE="./build_docker.sh GIT_HASH POSTHOG_KEY SENTRY_KEY" + +if [ "$ENV" != "enterprise" ] && [ "$ENV" != "hosted" ] && [ "$ENV" != "prod" ] && [ "$ENV" != "staging" ]; then + echo "$USAGE" + exit +fi + +# Move to here no matter where the file was executed +cd "$(dirname "$0")" + +tags="-t nango:latest -t nango:${ENV}" +if [ $GIT_HASH ]; then + tags+=" -t nango:${ENV}-${GIT_HASH} -t nango:${GIT_HASH}" +fi + +docker build \ + ${tags} \ + --file ../Dockerfile \ + --build-arg image_env=${ENV} \ + --build-arg git_hash=${GIT_HASH} \ + --build-arg posthog_key=${SENTRY_KEY} \ + --build-arg sentry_key=${POSTHOG_KEY} \ + ../ From 34146d589166add79e86229624542ed3f5955ff3 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:25:18 +0100 Subject: [PATCH 03/31] test --- .github/workflows/build-image.yaml | 34 ++++++++++++++++++++++++++++++ scripts/build_docker.sh | 25 ++++++++++++++++++---- 2 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build-image.yaml diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml new file mode 100644 index 0000000000..157cc35926 --- /dev/null +++ b/.github/workflows/build-image.yaml @@ -0,0 +1,34 @@ +name: Build unified Docker image + +on: + push: + branches: + - master + - staging/** + pull_request: + +jobs: + push-container: + runs-on: ubuntu-latest + env: + CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: env.CAN_PUSH == 'true' + with: + username: '${{ secrets.DOCKER_USERNAME }}' + password: '${{ secrets.DOCKER_PASSWORD }}' + + - name: Build image + run: | + ./scripts/build_docker.sh staging ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 852b879fc5..6d251eca0c 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -xe +set -e ENV=$1 # enterprise | hosted | prod | staging GIT_HASH=$2 @@ -8,11 +8,24 @@ POSTHOG_KEY=$3 SENTRY_KEY=$4 USAGE="./build_docker.sh GIT_HASH POSTHOG_KEY SENTRY_KEY" +RED='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' if [ "$ENV" != "enterprise" ] && [ "$ENV" != "hosted" ] && [ "$ENV" != "prod" ] && [ "$ENV" != "staging" ]; then + echo -e "${RED}Please specify an environment${NC}\n" echo "$USAGE" exit fi +if [ -z $GIT_HASH ]; then + echo -e "${YELLOW}GIT_HASH is empty${NC}" +fi +if [ -z $SENTRY_KEY ]; then + echo -e "${YELLOW}SENTRY_KEY is empty${NC}" +fi +if [ -z $POSTHOG_KEY ]; then + echo -e "${YELLOW}POSTHOG_KEY is empty${NC}" +fi # Move to here no matter where the file was executed cd "$(dirname "$0")" @@ -22,11 +35,15 @@ if [ $GIT_HASH ]; then tags+=" -t nango:${ENV}-${GIT_HASH} -t nango:${GIT_HASH}" fi -docker build \ - ${tags} \ - --file ../Dockerfile \ +echo "" +echo -e "Building nango:${ENV}\n" + +docker buildx build \ + --platform linux/amd64 \ --build-arg image_env=${ENV} \ --build-arg git_hash=${GIT_HASH} \ --build-arg posthog_key=${SENTRY_KEY} \ --build-arg sentry_key=${POSTHOG_KEY} \ + --file ../Dockerfile \ + ${tags} \ ../ From 0143bbcf75c1ae6f16e66ae4102cf9de7e42dcca Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:45:32 +0100 Subject: [PATCH 04/31] more --- scripts/build_docker.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 6d251eca0c..73930756b9 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -44,6 +44,8 @@ docker buildx build \ --build-arg git_hash=${GIT_HASH} \ --build-arg posthog_key=${SENTRY_KEY} \ --build-arg sentry_key=${POSTHOG_KEY} \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ --file ../Dockerfile \ - ${tags} \ + $tags \ ../ From 200b921511fee0a554181d9e7aa8aaaf0f212846 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:11:11 +0100 Subject: [PATCH 05/31] working --- .github/workflows/build-image.yaml | 4 ++++ Dockerfile | 2 +- scripts/build_docker.sh | 22 +++++++++++++++++----- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 157cc35926..f2b84520e2 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -32,3 +32,7 @@ jobs: - name: Build image run: | ./scripts/build_docker.sh staging ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Test image + run: | + docker run --rm nango:staging diff --git a/Dockerfile b/Dockerfile index be4e70e4ac..2c537b95d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,7 +68,7 @@ ARG git_hash ENV PORT=8080 ENV NODE_ENV=production ENV IMAGE_ENV $image_env -ENV GIT_HASH $image_env +ENV GIT_HASH $git_hash # - Bash is just to be able to log inside the image and have a decent shell RUN true \ diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 73930756b9..f29b26847f 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -2,16 +2,22 @@ set -e -ENV=$1 # enterprise | hosted | prod | staging -GIT_HASH=$2 -POSTHOG_KEY=$3 -SENTRY_KEY=$4 +ACTION=$1 +ENV=$2 # enterprise | hosted | prod | staging +GIT_HASH=$3 +POSTHOG_KEY=$4 +SENTRY_KEY=$5 -USAGE="./build_docker.sh GIT_HASH POSTHOG_KEY SENTRY_KEY" +USAGE="./build_docker.sh GIT_HASH POSTHOG_KEY SENTRY_KEY" RED='\033[0;31m' YELLOW='\033[0;33m' NC='\033[0m' +if [ "$ACTION" != "push" ] && [ "$ACTION" != "build" ]; then + echo -e "${RED}Please specify an action${NC}\n" + echo "$USAGE" + exit +fi if [ "$ENV" != "enterprise" ] && [ "$ENV" != "hosted" ] && [ "$ENV" != "prod" ] && [ "$ENV" != "staging" ]; then echo -e "${RED}Please specify an environment${NC}\n" echo "$USAGE" @@ -35,6 +41,12 @@ if [ $GIT_HASH ]; then tags+=" -t nango:${ENV}-${GIT_HASH} -t nango:${GIT_HASH}" fi +if [ $ACTION == 'build' ]; then + tags+=" --output=type=docker" +else + tags+=" --output=type=registry" +fi + echo "" echo -e "Building nango:${ENV}\n" From 7b3f08c33fbe1e6d3701a59e68cd242fc44e4b99 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:15:45 +0100 Subject: [PATCH 06/31] push --- .github/workflows/build-image.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index f2b84520e2..109504742f 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -31,8 +31,12 @@ jobs: - name: Build image run: | - ./scripts/build_docker.sh staging ${{ github.event.pull_request.head.sha || github.sha }} + ./scripts/build_docker.sh build staging ${{ github.event.pull_request.head.sha || github.sha }} - name: Test image run: | - docker run --rm nango:staging + docker run --rm nango:staging node packages/runner/dist/app.js + + - name: Push image + run: | + ./scripts/build_docker.sh push staging ${{ github.event.pull_request.head.sha || github.sha }} From 9683845be312319f1077b1a4d6f04499d8ba59c3 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:22:21 +0100 Subject: [PATCH 07/31] do not wait --- .github/workflows/build-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 109504742f..7e3d38095f 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -35,7 +35,7 @@ jobs: - name: Test image run: | - docker run --rm nango:staging node packages/runner/dist/app.js + docker run --rm nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' - name: Push image run: | From ffd9252d89d194ceb0a474141b7a7b37755cca97 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:31:41 +0100 Subject: [PATCH 08/31] error to null --- .github/workflows/build-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 7e3d38095f..19387a3614 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -35,7 +35,7 @@ jobs: - name: Test image run: | - docker run --rm nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' + docker run --rm nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' 2>/dev/null - name: Push image run: | From af429351c46bf480b4cd07a6f547811b946aba53 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:36:11 +0100 Subject: [PATCH 09/31] why is everything so hard --- .github/workflows/build-image.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 19387a3614..7a6cd810f6 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -33,9 +33,9 @@ jobs: run: | ./scripts/build_docker.sh build staging ${{ github.event.pull_request.head.sha || github.sha }} - - name: Test image - run: | - docker run --rm nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' 2>/dev/null + # - name: Test image + # run: | + # docker run --rm nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' 2>/dev/null - name: Push image run: | From 607b40a4d2b8659aae6e6626a183f5d7f39238ab Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:50:36 +0100 Subject: [PATCH 10/31] let's try --- .github/workflows/build-image.yaml | 6 +++++- scripts/build_docker.sh | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 7a6cd810f6..06f1d8ff55 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -29,13 +29,17 @@ jobs: username: '${{ secrets.DOCKER_USERNAME }}' password: '${{ secrets.DOCKER_PASSWORD }}' + # Needed for buildx gha cache to work + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + - name: Build image run: | ./scripts/build_docker.sh build staging ${{ github.event.pull_request.head.sha || github.sha }} # - name: Test image # run: | - # docker run --rm nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' 2>/dev/null + # docker run --rm nangohq/nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' 2>/dev/null - name: Push image run: | diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index f29b26847f..17527f0c01 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -36,9 +36,9 @@ fi # Move to here no matter where the file was executed cd "$(dirname "$0")" -tags="-t nango:latest -t nango:${ENV}" +tags="-t nangohq/nango:latest -t nangohq/nango:${ENV}" if [ $GIT_HASH ]; then - tags+=" -t nango:${ENV}-${GIT_HASH} -t nango:${GIT_HASH}" + tags+=" -t nangohq/nango:${ENV}-${GIT_HASH} -t nangohq/nango:${GIT_HASH}" fi if [ $ACTION == 'build' ]; then @@ -48,7 +48,7 @@ else fi echo "" -echo -e "Building nango:${ENV}\n" +echo -e "Building nangohq/nango:${ENV}\n" docker buildx build \ --platform linux/amd64 \ From a53ceaabe5ed2a8c8706a05b2899c7bdd2155d84 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:15:10 +0100 Subject: [PATCH 11/31] build less --- .github/workflows/build-image.yaml | 62 +++++++++++++++++++++++++++--- scripts/build_docker.sh | 11 +++--- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 06f1d8ff55..0d23bb4c4c 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -8,10 +8,12 @@ on: pull_request: jobs: - push-container: + push-staging-container: runs-on: ubuntu-latest env: CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + steps: - name: Check out uses: actions/checkout@v4 @@ -33,14 +35,64 @@ jobs: - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v2 - - name: Build image + - name: Build image (staging) run: | - ./scripts/build_docker.sh build staging ${{ github.event.pull_request.head.sha || github.sha }} + ./scripts/build_docker.sh build staging ${{ env.SHA }} # - name: Test image # run: | # docker run --rm nangohq/nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' 2>/dev/null - - name: Push image + - name: Push images + if: env.CAN_PUSH == 'true' + run: | + docker push staging-${{ env.SHA }} + + + push-other-container: + if: github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + needs: push-staging-container + env: + CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: env.CAN_PUSH == 'true' + with: + username: '${{ secrets.DOCKER_USERNAME }}' + password: '${{ secrets.DOCKER_PASSWORD }}' + + # Needed for buildx gha cache to work + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + + - name: Build image (prod) + run: | + ./scripts/build_docker.sh build prod ${{ env.SHA }} + + - name: Build image (hosted) + run: | + ./scripts/build_docker.sh build hosted ${{ env.SHA }} + + - name: Build image (enterprise) + run: | + ./scripts/build_docker.sh build enterprise ${{ env.SHA }} + + - name: Push images run: | - ./scripts/build_docker.sh push staging ${{ github.event.pull_request.head.sha || github.sha }} + docker push staging-${{ env.SHA }} + docker push hosted-${{ env.SHA }} + docker push enterperise-${{ env.SHA }} + docker push prod-${{ env.SHA }} diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 17527f0c01..6898f3367f 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -18,14 +18,18 @@ if [ "$ACTION" != "push" ] && [ "$ACTION" != "build" ]; then echo "$USAGE" exit fi + if [ "$ENV" != "enterprise" ] && [ "$ENV" != "hosted" ] && [ "$ENV" != "prod" ] && [ "$ENV" != "staging" ]; then echo -e "${RED}Please specify an environment${NC}\n" echo "$USAGE" exit fi + if [ -z $GIT_HASH ]; then - echo -e "${YELLOW}GIT_HASH is empty${NC}" + echo -e "${RED}GIT_HASH is empty${NC}" + exit fi + if [ -z $SENTRY_KEY ]; then echo -e "${YELLOW}SENTRY_KEY is empty${NC}" fi @@ -36,10 +40,7 @@ fi # Move to here no matter where the file was executed cd "$(dirname "$0")" -tags="-t nangohq/nango:latest -t nangohq/nango:${ENV}" -if [ $GIT_HASH ]; then - tags+=" -t nangohq/nango:${ENV}-${GIT_HASH} -t nangohq/nango:${GIT_HASH}" -fi +tags="-t nangohq/nango:${ENV}-${GIT_HASH}" if [ $ACTION == 'build' ]; then tags+=" --output=type=docker" From 50ceed1971635e14e077a789442b518b19afb120 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:19:57 +0100 Subject: [PATCH 12/31] fail push --- .github/workflows/build-image.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 0d23bb4c4c..9835f06790 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -46,7 +46,7 @@ jobs: - name: Push images if: env.CAN_PUSH == 'true' run: | - docker push staging-${{ env.SHA }} + docker push nangohq/nango:staging-${{ env.SHA }} push-other-container: @@ -57,6 +57,10 @@ jobs: CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' SHA: ${{ github.event.pull_request.head.sha || github.sha }} + concurrency: + group: push-unified-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + steps: - name: Check out uses: actions/checkout@v4 @@ -92,7 +96,6 @@ jobs: - name: Push images run: | - docker push staging-${{ env.SHA }} - docker push hosted-${{ env.SHA }} - docker push enterperise-${{ env.SHA }} - docker push prod-${{ env.SHA }} + docker push nangohq/nango:hosted-${{ env.SHA }} + docker push nangohq/nango:enterperise-${{ env.SHA }} + docker push nangohq/nango:prod-${{ env.SHA }} From 209904cef2ac45708441a1ffdb5aab1789cba389 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:26:57 +0100 Subject: [PATCH 13/31] test with everything --- .github/workflows/build-image.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 9835f06790..471966fa48 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -50,7 +50,6 @@ jobs: push-other-container: - if: github.ref == 'refs/heads/master' runs-on: ubuntu-latest needs: push-staging-container env: From f2b1d85f387270a587dd2b0357b398cc8ba4835e Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:28:50 +0100 Subject: [PATCH 14/31] remove from dockerfile --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2c537b95d0..f7375ef5da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,12 +32,10 @@ RUN true \ # /!\ It's counter intuitive but do not set NODE_ENV=production before building, it will break some modules # ENV NODE_ENV=production ARG image_env -ARG git_hash ARG posthog_key ARG sentry_key ENV REACT_APP_ENV $image_env -ENV REACT_APP_PUBLIC_GIT_HASH $git_hash ENV REACT_APP_PUBLIC_POSTHOG_HOST https://app.posthog.com ENV REACT_APP_PUBLIC_POSTHOG_KEY $posthog_key ENV REACT_APP_PUBLIC_SENTRY_KEY $sentry_key From 5e9c800aa28d929f7427425f55342df9c636a924 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:44:46 +0100 Subject: [PATCH 15/31] use matrix --- .github/workflows/build-image.yaml | 25 +++++++++++-------------- Dockerfile | 15 ++++++++------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 471966fa48..0bc1c4351d 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -51,7 +51,6 @@ jobs: push-other-container: runs-on: ubuntu-latest - needs: push-staging-container env: CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -60,6 +59,14 @@ jobs: group: push-unified-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true + strategy: + matrix: + test-group: + [ + prod, + enterprise + ] + steps: - name: Check out uses: actions/checkout@v4 @@ -81,20 +88,10 @@ jobs: - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v2 - - name: Build image (prod) - run: | - ./scripts/build_docker.sh build prod ${{ env.SHA }} - - - name: Build image (hosted) - run: | - ./scripts/build_docker.sh build hosted ${{ env.SHA }} - - - name: Build image (enterprise) + - name: Build image (${{ matrix.test-group }}) run: | - ./scripts/build_docker.sh build enterprise ${{ env.SHA }} + ./scripts/build_docker.sh build ${{ matrix.test-group }} ${{ env.SHA }} - name: Push images run: | - docker push nangohq/nango:hosted-${{ env.SHA }} - docker push nangohq/nango:enterperise-${{ env.SHA }} - docker push nangohq/nango:prod-${{ env.SHA }} + docker push nangohq/nango:${{ matrix.test-group }}-${{ env.SHA }} diff --git a/Dockerfile b/Dockerfile index f7375ef5da..52bb770623 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,13 +60,6 @@ RUN true \ # This image must have the minimum amount of layers FROM node:18.19.1-bullseye-slim as web -ARG image_env -ARG git_hash - -ENV PORT=8080 -ENV NODE_ENV=production -ENV IMAGE_ENV $image_env -ENV GIT_HASH $git_hash # - Bash is just to be able to log inside the image and have a decent shell RUN true \ @@ -83,4 +76,12 @@ WORKDIR /app/nango # Code COPY --from=tmp --chown=node:node /app/tmp /app/nango +ARG image_env +ARG git_hash + +ENV PORT=8080 +ENV NODE_ENV=production +ENV IMAGE_ENV $image_env +ENV GIT_HASH $git_hash + EXPOSE 8080 From 8bc55981d32985f4a37d5d11702db2cd40f1f638 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:51:14 +0100 Subject: [PATCH 16/31] remove concurrency --- .github/workflows/build-image.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 0bc1c4351d..13ff21b630 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -55,10 +55,6 @@ jobs: CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' SHA: ${{ github.event.pull_request.head.sha || github.sha }} - concurrency: - group: push-unified-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - strategy: matrix: test-group: @@ -93,5 +89,6 @@ jobs: ./scripts/build_docker.sh build ${{ matrix.test-group }} ${{ env.SHA }} - name: Push images + if: env.CAN_PUSH == 'true' run: | docker push nangohq/nango:${{ matrix.test-group }}-${{ env.SHA }} From 35c2b57660d25fbec8227844db30c24f93a075f3 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:22:21 +0100 Subject: [PATCH 17/31] try reusable --- .github/workflows/build-image-reusable.yaml | 60 +++++++++++++ .github/workflows/build-image.yaml | 96 ++++----------------- scripts/build_docker.sh | 4 +- 3 files changed, 76 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/build-image-reusable.yaml diff --git a/.github/workflows/build-image-reusable.yaml b/.github/workflows/build-image-reusable.yaml new file mode 100644 index 0000000000..3c113f23f3 --- /dev/null +++ b/.github/workflows/build-image-reusable.yaml @@ -0,0 +1,60 @@ +name: Build unified Docker image + +on: + workflow_call: + inputs: + if: + description: 'Whether to run this job' + required: false + default: true + type: boolean + name: + required: true + type: string + key_secret_sentry: + required: true + type: string + key_secret_posthog: + required: true + type: string + +jobs: + push-prod-containers: + if: ${{ inputs.if }} + + runs-on: ubuntu-latest + env: + CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: env.CAN_PUSH == 'true' + with: + username: '${{ secrets.DOCKER_USERNAME }}' + password: '${{ secrets.DOCKER_PASSWORD }}' + + # Needed for buildx gha cache to work + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + + - name: Build image (${{ inputs.name }}) + run: | + export SENTRY_KEY=${{ secrets[inputs.key_secret_sentry] }} + export POSTHOG_KEY=${{ secrets[inputs.key_secret_posthog] }} + ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} + + - name: Push image + if: env.CAN_PUSH == 'true' + run: | + docker push nangohq/nango:${{ inputs.name }}-${{ env.SHA }} diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 13ff21b630..219a3f11c8 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -7,88 +7,22 @@ on: - staging/** pull_request: -jobs: - push-staging-container: - runs-on: ubuntu-latest - env: - CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' - SHA: ${{ github.event.pull_request.head.sha || github.sha }} - - steps: - - name: Check out - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - if: env.CAN_PUSH == 'true' - with: - username: '${{ secrets.DOCKER_USERNAME }}' - password: '${{ secrets.DOCKER_PASSWORD }}' - - # Needed for buildx gha cache to work - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - - name: Build image (staging) - run: | - ./scripts/build_docker.sh build staging ${{ env.SHA }} - - # - name: Test image - # run: | - # docker run --rm nangohq/nango:staging timeout 5 sh -c 'node packages/runner/dist/app.js' 2>/dev/null - - - name: Push images - if: env.CAN_PUSH == 'true' - run: | - docker push nangohq/nango:staging-${{ env.SHA }} - - - push-other-container: - runs-on: ubuntu-latest - env: - CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' - SHA: ${{ github.event.pull_request.head.sha || github.sha }} +jobs: + build-image: strategy: matrix: - test-group: - [ - prod, - enterprise - ] - - steps: - - name: Check out - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - if: env.CAN_PUSH == 'true' - with: - username: '${{ secrets.DOCKER_USERNAME }}' - password: '${{ secrets.DOCKER_PASSWORD }}' - - # Needed for buildx gha cache to work - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - - name: Build image (${{ matrix.test-group }}) - run: | - ./scripts/build_docker.sh build ${{ matrix.test-group }} ${{ env.SHA }} + group: + - name: 'staging' + if: github.ref == 'refs/heads/master' + - name: 'prod' + if: github.ref == 'refs/heads/master' + - name: 'enterprise' + if: github.ref == 'refs/heads/master' + + secrets: inherit + uses: ./.github/workflows/build-image-reusable.yml + with: + if: ${{ matrix.group.if }} + name: ${{ matrix.group.foo }} - - name: Push images - if: env.CAN_PUSH == 'true' - run: | - docker push nangohq/nango:${{ matrix.test-group }}-${{ env.SHA }} diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 6898f3367f..02899508d7 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -5,10 +5,8 @@ set -e ACTION=$1 ENV=$2 # enterprise | hosted | prod | staging GIT_HASH=$3 -POSTHOG_KEY=$4 -SENTRY_KEY=$5 -USAGE="./build_docker.sh GIT_HASH POSTHOG_KEY SENTRY_KEY" +USAGE="./build_docker.sh GIT_HASH" RED='\033[0;31m' YELLOW='\033[0;33m' NC='\033[0m' From 2bb699fc7aa539130bc1d16b32646ab5bf2fee31 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:24:15 +0100 Subject: [PATCH 18/31] a --- .github/workflows/build-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 219a3f11c8..4e6ef57d55 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -21,7 +21,7 @@ jobs: if: github.ref == 'refs/heads/master' secrets: inherit - uses: ./.github/workflows/build-image-reusable.yml + uses: ./.github/workflows/build-image-reusable.yaml with: if: ${{ matrix.group.if }} name: ${{ matrix.group.foo }} From cbfecb899f8c8abf311db32f5efea573b7a1c570 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:27:02 +0100 Subject: [PATCH 19/31] missing secret --- .github/workflows/build-image-reusable.yaml | 4 ++-- .github/workflows/build-image.yaml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image-reusable.yaml b/.github/workflows/build-image-reusable.yaml index 3c113f23f3..c47dc275c0 100644 --- a/.github/workflows/build-image-reusable.yaml +++ b/.github/workflows/build-image-reusable.yaml @@ -12,10 +12,10 @@ on: required: true type: string key_secret_sentry: - required: true + required: false type: string key_secret_posthog: - required: true + required: false type: string jobs: diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 4e6ef57d55..1c382eae52 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -15,8 +15,11 @@ jobs: group: - name: 'staging' if: github.ref == 'refs/heads/master' + sentry_key: SENTRY_KEY_staging - name: 'prod' if: github.ref == 'refs/heads/master' + sentry_key: SENTRY_KEY_prod + posthog_key: POSTHOG_KEY_prod - name: 'enterprise' if: github.ref == 'refs/heads/master' @@ -25,4 +28,6 @@ jobs: with: if: ${{ matrix.group.if }} name: ${{ matrix.group.foo }} + key_secret_sentry: ${{ matrix.group.sentry_key }} + key_secret_posthog: ${{ matrix.group.posthog_key }} From c129ec24eb7171ed6ac4e13dc13102cb72ff79c6 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:29:12 +0100 Subject: [PATCH 20/31] fix condition --- .github/workflows/build-image.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 1c382eae52..916d17267e 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -14,14 +14,14 @@ jobs: matrix: group: - name: 'staging' - if: github.ref == 'refs/heads/master' + if: true sentry_key: SENTRY_KEY_staging - name: 'prod' - if: github.ref == 'refs/heads/master' + if: true sentry_key: SENTRY_KEY_prod posthog_key: POSTHOG_KEY_prod - name: 'enterprise' - if: github.ref == 'refs/heads/master' + if: ${{ github.ref == 'refs/heads/master' }} secrets: inherit uses: ./.github/workflows/build-image-reusable.yaml From c74b6df01a92803333aa04cd990dfe2d0a731b7d Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:31:31 +0100 Subject: [PATCH 21/31] fix --- .github/workflows/build-image-reusable.yaml | 2 +- .github/workflows/build-image.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image-reusable.yaml b/.github/workflows/build-image-reusable.yaml index c47dc275c0..2e205c977f 100644 --- a/.github/workflows/build-image-reusable.yaml +++ b/.github/workflows/build-image-reusable.yaml @@ -19,7 +19,7 @@ on: type: string jobs: - push-prod-containers: + build-container: if: ${{ inputs.if }} runs-on: ubuntu-latest diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 916d17267e..c5cd133e37 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -27,7 +27,7 @@ jobs: uses: ./.github/workflows/build-image-reusable.yaml with: if: ${{ matrix.group.if }} - name: ${{ matrix.group.foo }} + name: ${{ matrix.group.name }} key_secret_sentry: ${{ matrix.group.sentry_key }} key_secret_posthog: ${{ matrix.group.posthog_key }} From 532ec922cfdce753c7f49e37cd6de1fe3fc616be Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:56:41 +0100 Subject: [PATCH 22/31] lock --- package-lock.json | 167 ----------------------------------- packages/webapp/package.json | 10 +-- 2 files changed, 3 insertions(+), 174 deletions(-) diff --git a/package-lock.json b/package-lock.json index e7db6df0e9..d3b79e5c51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -65,12 +65,6 @@ "version": "5.1.2", "license": "MIT" }, - "node_modules/@adobe/css-tools": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.3.3.tgz", - "integrity": "sha512-rE0Pygv0sEZ4vBWHlAgJLGDU7Pm8xoO6p3wsEceb7GYAjScrOHpEo8KK/eVkAcnSM+slAEtXjA2JpdjLp4fJQQ==", - "dev": true - }, "node_modules/@ampproject/remapping": { "version": "2.2.1", "dev": true, @@ -4925,22 +4919,6 @@ "version": "11.0.2", "license": "BSD-3-Clause" }, - "node_modules/@headlessui/react": { - "version": "1.7.12", - "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-1.7.12.tgz", - "integrity": "sha512-FhSx5V+Qp0GvbTpaxyS+ymGDDNntCacClWsk/d8Upbr19g3AsPbjfPk4+m2CgJGcuCB5Dz7LpUIOAbvQTyjL2g==", - "dev": true, - "dependencies": { - "client-only": "^0.0.1" - }, - "engines": { - "node": ">=10" - }, - "peerDependencies": { - "react": "^16 || ^17 || ^18", - "react-dom": "^16 || ^17 || ^18" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -7001,41 +6979,6 @@ "dev": true, "peer": true }, - "node_modules/@testing-library/jest-dom": { - "version": "5.17.0", - "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-5.17.0.tgz", - "integrity": "sha512-ynmNeT7asXyH3aSVv4vvX4Rb+0qjOhdNHnO/3vuZNqPmhDpV/+rCSGwQ7bLcmU2cJ4dvoheIO85LQj0IbJHEtg==", - "dev": true, - "dependencies": { - "@adobe/css-tools": "^4.0.1", - "@babel/runtime": "^7.9.2", - "@types/testing-library__jest-dom": "^5.9.1", - "aria-query": "^5.0.0", - "chalk": "^3.0.0", - "css.escape": "^1.5.1", - "dom-accessibility-api": "^0.5.6", - "lodash": "^4.17.15", - "redent": "^3.0.0" - }, - "engines": { - "node": ">=8", - "npm": ">=6", - "yarn": ">=1" - } - }, - "node_modules/@testing-library/jest-dom/node_modules/chalk": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", - "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/@testing-library/react": { "version": "13.4.0", "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-13.4.0.tgz", @@ -7516,48 +7459,6 @@ "@types/istanbul-lib-report": "*" } }, - "node_modules/@types/jest": { - "version": "27.5.2", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-27.5.2.tgz", - "integrity": "sha512-mpT8LJJ4CMeeahobofYWIjFo0xonRS/HfxnVEPMPFSQdGUt1uHCnoPT7Zhb+sjDU2wz0oKV0OLUR0WzrHNgfeA==", - "dev": true, - "dependencies": { - "jest-matcher-utils": "^27.0.0", - "pretty-format": "^27.0.0" - } - }, - "node_modules/@types/jest/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@types/jest/node_modules/pretty-format": { - "version": "27.5.1", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", - "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", - "dev": true, - "dependencies": { - "ansi-regex": "^5.0.1", - "ansi-styles": "^5.0.0", - "react-is": "^17.0.1" - }, - "engines": { - "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" - } - }, - "node_modules/@types/jest/node_modules/react-is": { - "version": "17.0.2", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", - "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", - "dev": true - }, "node_modules/@types/js-yaml": { "version": "4.0.5", "dev": true, @@ -7868,15 +7769,6 @@ "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, - "node_modules/@types/testing-library__jest-dom": { - "version": "5.14.9", - "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.9.tgz", - "integrity": "sha512-FSYhIjFlfOpGSRyVoMBMuS3ws5ehFQODymf3vlI7U1K8c7PHwWwFY7VREfmsuzHSOnoKs/9/Y983ayOs7eRzqw==", - "dev": true, - "dependencies": { - "@types/jest": "*" - } - }, "node_modules/@types/triple-beam": { "version": "1.3.2", "license": "MIT" @@ -11499,12 +11391,6 @@ "url": "https://github.com/sponsors/fb55" } }, - "node_modules/css.escape": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz", - "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==", - "dev": true - }, "node_modules/cssdb": { "version": "7.11.0", "resolved": "https://registry.npmjs.org/cssdb/-/cssdb-7.11.0.tgz", @@ -15700,15 +15586,6 @@ "node": ">=0.8.19" } }, - "node_modules/indent-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/inflight": { "version": "1.0.6", "license": "ISC", @@ -18682,15 +18559,6 @@ "node": ">=6" } }, - "node_modules/min-indent": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", - "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/mini-css-extract-plugin": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.8.0.tgz", @@ -21979,19 +21847,6 @@ "node": ">=6.0.0" } }, - "node_modules/redent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", - "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", - "dev": true, - "dependencies": { - "indent-string": "^4.0.0", - "strip-indent": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/redis": { "version": "4.6.11", "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.11.tgz", @@ -23484,18 +23339,6 @@ "node": ">=6" } }, - "node_modules/strip-indent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", - "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", - "dev": true, - "dependencies": { - "min-indent": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-json-comments": { "version": "3.1.1", "dev": true, @@ -27795,18 +27638,14 @@ "devDependencies": { "@geist-ui/core": "2.3.8", "@geist-ui/icons": "1.0.2", - "@headlessui/react": "1.7.12", "@heroicons/react": "2.0.18", "@mantine/prism": "5.10.5", "@nangohq/frontend": "^0.39.1", "@sentry/react": "7.83.0", "@tailwindcss/forms": "0.5.3", - "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "13.4.0", "@testing-library/user-event": "13.5.0", - "@types/jest": "^27.5.2", "@types/js-cookie": "3.0.3", - "@types/node": "18.7.6", "@types/react": "18.2.58", "@types/react-dom": "18.2.19", "@types/react-helmet": "6.1.8", @@ -27971,12 +27810,6 @@ "integrity": "sha512-Xe7IImK09HP1sv2M/aI+48a20VX+TdRJucfq4vfRVy6nWN8PYPOEnlMRSgxJAgYQIXJVL8dZ4/ilAM7dWNaOww==", "dev": true }, - "packages/webapp/node_modules/@types/node": { - "version": "18.7.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.6.tgz", - "integrity": "sha512-EdxgKRXgYsNITy5mjjXjVE/CS8YENSdhiagGrLqjG0pvA2owgJ6i4l7wy/PFZGC0B1/H20lWKN7ONVDNYDZm7A==", - "dev": true - }, "packages/webapp/node_modules/@types/react-helmet": { "version": "6.1.8", "resolved": "https://registry.npmjs.org/@types/react-helmet/-/react-helmet-6.1.8.tgz", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 52306554a7..2790a76d1c 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -30,21 +30,17 @@ "devDependencies": { "@geist-ui/core": "2.3.8", "@geist-ui/icons": "1.0.2", - "@headlessui/react": "1.7.12", "@heroicons/react": "2.0.18", "@mantine/prism": "5.10.5", "@nangohq/frontend": "^0.39.1", "@sentry/react": "7.83.0", "@tailwindcss/forms": "0.5.3", - "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "13.4.0", "@testing-library/user-event": "13.5.0", - "@types/jest": "^27.5.2", "@types/js-cookie": "3.0.3", - "@types/node": "18.7.6", + "@types/react": "18.2.58", "@types/react-dom": "18.2.19", "@types/react-helmet": "6.1.8", - "@types/react": "18.2.58", "buffer": "6.0.3", "class-variance-authority": "0.5.2", "classnames": "2.3.2", @@ -53,17 +49,17 @@ "env-cmd": "10.1.0", "js-cookie": "3.0.5", "ms": "2.1.3", - "postcss-loader": "8.1.1", "postcss": "8.4.35", + "postcss-loader": "8.1.1", "posthog-js": "1.57.2", "query-string": "8.1.0", + "react": "18.2.0", "react-code-blocks": "0.0.9-0", "react-dom": "18.2.0", "react-helmet": "6.1.0", "react-router-dom": "6.8.2", "react-scripts": "5.0.1", "react-toastify": "9.1.1", - "react": "18.2.0", "swr": "2.2.5", "tailwind-merge": "2.2.1", "tailwindcss": "3.2.7", From d0f7ed247c934da805b0a5aa23b4d0c3fc6f2256 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:55:13 +0100 Subject: [PATCH 23/31] more --- .github/actions/build-image.yaml | 60 +++++++++++++++++++++ .github/workflows/build-image-reusable.yaml | 60 --------------------- .github/workflows/build-image.yaml | 52 +++++++++--------- 3 files changed, 85 insertions(+), 87 deletions(-) create mode 100644 .github/actions/build-image.yaml delete mode 100644 .github/workflows/build-image-reusable.yaml diff --git a/.github/actions/build-image.yaml b/.github/actions/build-image.yaml new file mode 100644 index 0000000000..031ae4943b --- /dev/null +++ b/.github/actions/build-image.yaml @@ -0,0 +1,60 @@ +name: Build unified Docker image + +on: + workflow_call: + inputs: + if: + description: 'Whether to run this job' + required: false + default: true + type: boolean + name: + required: true + type: string + key_secret_sentry: + required: false + type: string + key_secret_posthog: + required: false + type: string + +jobs: + build-container: + if: ${{ inputs.if }} + + runs-on: ubuntu-latest + env: + CAN_PUSH: "${{ secrets.DOCKER_PASSWORD != ' && secrets.DOCKER_USERNAME != ' }}" + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: env.CAN_PUSH == 'true' + with: + username: '${{ secrets.DOCKER_USERNAME }}' + password: '${{ secrets.DOCKER_PASSWORD }}' + + # Needed for buildx gha cache to work + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + + - name: Build image (${{ inputs.name }}) + run: | + export SENTRY_KEY=${{ secrets[inputs.key_secret_sentry] }} + export POSTHOG_KEY=${{ secrets[inputs.key_secret_posthog] }} + ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} + + - name: Push image + if: env.CAN_PUSH == 'true' + run: | + docker push nangohq/nango:${{ inputs.name }}-${{ env.SHA }} diff --git a/.github/workflows/build-image-reusable.yaml b/.github/workflows/build-image-reusable.yaml deleted file mode 100644 index 2e205c977f..0000000000 --- a/.github/workflows/build-image-reusable.yaml +++ /dev/null @@ -1,60 +0,0 @@ -name: Build unified Docker image - -on: - workflow_call: - inputs: - if: - description: 'Whether to run this job' - required: false - default: true - type: boolean - name: - required: true - type: string - key_secret_sentry: - required: false - type: string - key_secret_posthog: - required: false - type: string - -jobs: - build-container: - if: ${{ inputs.if }} - - runs-on: ubuntu-latest - env: - CAN_PUSH: '${{ secrets.DOCKER_PASSWORD != '' && secrets.DOCKER_USERNAME != '' }}' - SHA: ${{ github.event.pull_request.head.sha || github.sha }} - - steps: - - name: Check out - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - if: env.CAN_PUSH == 'true' - with: - username: '${{ secrets.DOCKER_USERNAME }}' - password: '${{ secrets.DOCKER_PASSWORD }}' - - # Needed for buildx gha cache to work - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - - name: Build image (${{ inputs.name }}) - run: | - export SENTRY_KEY=${{ secrets[inputs.key_secret_sentry] }} - export POSTHOG_KEY=${{ secrets[inputs.key_secret_posthog] }} - ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} - - - name: Push image - if: env.CAN_PUSH == 'true' - run: | - docker push nangohq/nango:${{ inputs.name }}-${{ env.SHA }} diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index c5cd133e37..0be646b747 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -1,33 +1,31 @@ name: Build unified Docker image on: - push: - branches: - - master - - staging/** - pull_request: - + push: + branches: + - master + - staging/** + pull_request: jobs: - build-image: - strategy: - matrix: - group: - - name: 'staging' - if: true - sentry_key: SENTRY_KEY_staging - - name: 'prod' - if: true - sentry_key: SENTRY_KEY_prod - posthog_key: POSTHOG_KEY_prod - - name: 'enterprise' - if: ${{ github.ref == 'refs/heads/master' }} - - secrets: inherit - uses: ./.github/workflows/build-image-reusable.yaml - with: - if: ${{ matrix.group.if }} - name: ${{ matrix.group.name }} - key_secret_sentry: ${{ matrix.group.sentry_key }} - key_secret_posthog: ${{ matrix.group.posthog_key }} + build-image: + strategy: + matrix: + group: + - name: 'staging' + if: true + sentry_key: SENTRY_KEY_staging + - name: 'prod' + if: ${{ github.ref == 'refs/heads/master' }} + sentry_key: SENTRY_KEY_prod + posthog_key: POSTHOG_KEY_prod + # - name: 'enterprise' + # if: ${{ github.ref == 'refs/heads/master' }} + secrets: inherit + uses: ./.github/actions/build-image.yaml + with: + if: ${{ matrix.group.if }} + name: ${{ matrix.group.name }} + key_secret_sentry: ${{ matrix.group.sentry_key }} + key_secret_posthog: ${{ matrix.group.posthog_key }} From fa755dd86b4e31ef658c7c09881f13295b972e92 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:56:55 +0100 Subject: [PATCH 24/31] rename --- .github/actions/{build-image.yaml => build-image/action.yaml} | 0 .github/workflows/build-image.yaml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/actions/{build-image.yaml => build-image/action.yaml} (100%) diff --git a/.github/actions/build-image.yaml b/.github/actions/build-image/action.yaml similarity index 100% rename from .github/actions/build-image.yaml rename to .github/actions/build-image/action.yaml diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 0be646b747..dc9a286081 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -23,7 +23,7 @@ jobs: # if: ${{ github.ref == 'refs/heads/master' }} secrets: inherit - uses: ./.github/actions/build-image.yaml + uses: ./.github/actions/build-image/action.yaml with: if: ${{ matrix.group.if }} name: ${{ matrix.group.name }} From ce8b7ba409060bcbbf06c96b26cb16e0d3681284 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:57:54 +0100 Subject: [PATCH 25/31] -__ --- .github/workflows/build-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index dc9a286081..490e720d9f 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -23,7 +23,7 @@ jobs: # if: ${{ github.ref == 'refs/heads/master' }} secrets: inherit - uses: ./.github/actions/build-image/action.yaml + uses: ./.github/actions/build-image with: if: ${{ matrix.group.if }} name: ${{ matrix.group.name }} From 1a892299b81ea8b84ab716b824c1002c83e0e549 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 19 Mar 2024 11:04:15 +0100 Subject: [PATCH 26/31] test --- .github/actions/build-image/action.yaml | 113 ++++++++++++------------ 1 file changed, 55 insertions(+), 58 deletions(-) diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index 031ae4943b..a6749a3892 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -1,60 +1,57 @@ name: Build unified Docker image -on: - workflow_call: - inputs: - if: - description: 'Whether to run this job' - required: false - default: true - type: boolean - name: - required: true - type: string - key_secret_sentry: - required: false - type: string - key_secret_posthog: - required: false - type: string - -jobs: - build-container: - if: ${{ inputs.if }} - - runs-on: ubuntu-latest - env: - CAN_PUSH: "${{ secrets.DOCKER_PASSWORD != ' && secrets.DOCKER_USERNAME != ' }}" - SHA: ${{ github.event.pull_request.head.sha || github.sha }} - - steps: - - name: Check out - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - if: env.CAN_PUSH == 'true' - with: - username: '${{ secrets.DOCKER_USERNAME }}' - password: '${{ secrets.DOCKER_PASSWORD }}' - - # Needed for buildx gha cache to work - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - - name: Build image (${{ inputs.name }}) - run: | - export SENTRY_KEY=${{ secrets[inputs.key_secret_sentry] }} - export POSTHOG_KEY=${{ secrets[inputs.key_secret_posthog] }} - ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} - - - name: Push image - if: env.CAN_PUSH == 'true' - run: | - docker push nangohq/nango:${{ inputs.name }}-${{ env.SHA }} +inputs: + if: + description: 'Whether to run this job' + required: false + default: true + type: boolean + name: + required: true + type: string + key_secret_sentry: + required: false + type: string + key_secret_posthog: + required: false + type: string + +runs: + using: 'composite' + if: ${{ inputs.if }} + + env: + CAN_PUSH: "${{ secrets.DOCKER_PASSWORD != ' && secrets.DOCKER_USERNAME != ' }}" + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: env.CAN_PUSH == 'true' + with: + username: '${{ secrets.DOCKER_USERNAME }}' + password: '${{ secrets.DOCKER_PASSWORD }}' + + # Needed for buildx gha cache to work + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + + - name: Build image (${{ inputs.name }}) + run: | + export SENTRY_KEY=${{ secrets[inputs.key_secret_sentry] }} + export POSTHOG_KEY=${{ secrets[inputs.key_secret_posthog] }} + ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} + + - name: Push image + if: env.CAN_PUSH == 'true' + run: | + docker push nangohq/nango:${{ inputs.name }}-${{ env.SHA }} From 24d0b267f85fe05a52032ae48892b962e73ae8e8 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 19 Mar 2024 11:05:59 +0100 Subject: [PATCH 27/31] revert to something working --- .github/actions/build-image/action.yaml | 57 -------------------- .github/workflows/build-image-reusable.yaml | 60 +++++++++++++++++++++ .github/workflows/build-image.yaml | 10 ++-- 3 files changed, 65 insertions(+), 62 deletions(-) delete mode 100644 .github/actions/build-image/action.yaml create mode 100644 .github/workflows/build-image-reusable.yaml diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml deleted file mode 100644 index a6749a3892..0000000000 --- a/.github/actions/build-image/action.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: Build unified Docker image - -inputs: - if: - description: 'Whether to run this job' - required: false - default: true - type: boolean - name: - required: true - type: string - key_secret_sentry: - required: false - type: string - key_secret_posthog: - required: false - type: string - -runs: - using: 'composite' - if: ${{ inputs.if }} - - env: - CAN_PUSH: "${{ secrets.DOCKER_PASSWORD != ' && secrets.DOCKER_USERNAME != ' }}" - SHA: ${{ github.event.pull_request.head.sha || github.sha }} - - steps: - - name: Check out - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - if: env.CAN_PUSH == 'true' - with: - username: '${{ secrets.DOCKER_USERNAME }}' - password: '${{ secrets.DOCKER_PASSWORD }}' - - # Needed for buildx gha cache to work - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - - name: Build image (${{ inputs.name }}) - run: | - export SENTRY_KEY=${{ secrets[inputs.key_secret_sentry] }} - export POSTHOG_KEY=${{ secrets[inputs.key_secret_posthog] }} - ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} - - - name: Push image - if: env.CAN_PUSH == 'true' - run: | - docker push nangohq/nango:${{ inputs.name }}-${{ env.SHA }} diff --git a/.github/workflows/build-image-reusable.yaml b/.github/workflows/build-image-reusable.yaml new file mode 100644 index 0000000000..031ae4943b --- /dev/null +++ b/.github/workflows/build-image-reusable.yaml @@ -0,0 +1,60 @@ +name: Build unified Docker image + +on: + workflow_call: + inputs: + if: + description: 'Whether to run this job' + required: false + default: true + type: boolean + name: + required: true + type: string + key_secret_sentry: + required: false + type: string + key_secret_posthog: + required: false + type: string + +jobs: + build-container: + if: ${{ inputs.if }} + + runs-on: ubuntu-latest + env: + CAN_PUSH: "${{ secrets.DOCKER_PASSWORD != ' && secrets.DOCKER_USERNAME != ' }}" + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: env.CAN_PUSH == 'true' + with: + username: '${{ secrets.DOCKER_USERNAME }}' + password: '${{ secrets.DOCKER_PASSWORD }}' + + # Needed for buildx gha cache to work + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + + - name: Build image (${{ inputs.name }}) + run: | + export SENTRY_KEY=${{ secrets[inputs.key_secret_sentry] }} + export POSTHOG_KEY=${{ secrets[inputs.key_secret_posthog] }} + ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} + + - name: Push image + if: env.CAN_PUSH == 'true' + run: | + docker push nangohq/nango:${{ inputs.name }}-${{ env.SHA }} diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 490e720d9f..bda5eb7a41 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -1,4 +1,4 @@ -name: Build unified Docker image +name: '[Release] Build unified Docker image' on: push: @@ -16,14 +16,14 @@ jobs: if: true sentry_key: SENTRY_KEY_staging - name: 'prod' - if: ${{ github.ref == 'refs/heads/master' }} + if: true sentry_key: SENTRY_KEY_prod posthog_key: POSTHOG_KEY_prod - # - name: 'enterprise' - # if: ${{ github.ref == 'refs/heads/master' }} + - name: 'enterprise' + if: ${{ github.ref == 'refs/heads/master' }} secrets: inherit - uses: ./.github/actions/build-image + uses: ./.github/workflows/build-image-reusable.yaml with: if: ${{ matrix.group.if }} name: ${{ matrix.group.name }} From 9be501fd42e3b0cf0d0cc388eac9ab65bc6c99ae Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 19 Mar 2024 11:15:22 +0100 Subject: [PATCH 28/31] do not compile other --- .github/workflows/build-image.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index bda5eb7a41..53676bd473 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -16,11 +16,12 @@ jobs: if: true sentry_key: SENTRY_KEY_staging - name: 'prod' - if: true + if: ${{ github.ref == 'refs/heads/master' }} sentry_key: SENTRY_KEY_prod posthog_key: POSTHOG_KEY_prod - - name: 'enterprise' - if: ${{ github.ref == 'refs/heads/master' }} + # Commented for now + # - name: 'enterprise' + # if: ${{ github.ref == 'refs/heads/master' }} secrets: inherit uses: ./.github/workflows/build-image-reusable.yaml From 72189e857658a71d43c8239a18262a77b4fdc124 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Tue, 19 Mar 2024 11:20:27 +0100 Subject: [PATCH 29/31] clean up dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52bb770623..5ce601d52d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,16 +25,17 @@ RUN true \ # At this stage we copy back all sources COPY . /app/tmp -# Build the backend separately because it can be cached even we change env vars +# Build the backend separately because it can be cached even when we change the env vars RUN true \ && npm run ts-build:docker -# /!\ It's counter intuitive but do not set NODE_ENV=production before building, it will break some modules +# /!\ Do not set NODE_ENV=production before building, it will break some modules # ENV NODE_ENV=production ARG image_env ARG posthog_key ARG sentry_key +# TODO: remove the need for this ENV REACT_APP_ENV $image_env ENV REACT_APP_PUBLIC_POSTHOG_HOST https://app.posthog.com ENV REACT_APP_PUBLIC_POSTHOG_KEY $posthog_key @@ -57,7 +58,6 @@ RUN true \ # ---- Web ---- # Resulting new, minimal image -# This image must have the minimum amount of layers FROM node:18.19.1-bullseye-slim as web From a06d05c651b082486d7c98944d3802ee6e6853ae Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:15:50 +0100 Subject: [PATCH 30/31] review --- .github/workflows/build-image-reusable.yaml | 8 ++++---- .github/workflows/build-image.yaml | 6 ++++-- Dockerfile | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-image-reusable.yaml b/.github/workflows/build-image-reusable.yaml index 031ae4943b..1141937bfb 100644 --- a/.github/workflows/build-image-reusable.yaml +++ b/.github/workflows/build-image-reusable.yaml @@ -11,10 +11,10 @@ on: name: required: true type: string - key_secret_sentry: + key_for_sentry_secret: required: false type: string - key_secret_posthog: + key_for_posthog_secret: required: false type: string @@ -50,8 +50,8 @@ jobs: - name: Build image (${{ inputs.name }}) run: | - export SENTRY_KEY=${{ secrets[inputs.key_secret_sentry] }} - export POSTHOG_KEY=${{ secrets[inputs.key_secret_posthog] }} + export SENTRY_KEY=${{ secrets[inputs.key_for_sentry_secret] }} + export POSTHOG_KEY=${{ secrets[inputs.key_for_posthog_secret] }} ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} - name: Push image diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 53676bd473..df37f11b90 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -15,10 +15,12 @@ jobs: - name: 'staging' if: true sentry_key: SENTRY_KEY_staging + - name: 'prod' if: ${{ github.ref == 'refs/heads/master' }} sentry_key: SENTRY_KEY_prod posthog_key: POSTHOG_KEY_prod + # Commented for now # - name: 'enterprise' # if: ${{ github.ref == 'refs/heads/master' }} @@ -28,5 +30,5 @@ jobs: with: if: ${{ matrix.group.if }} name: ${{ matrix.group.name }} - key_secret_sentry: ${{ matrix.group.sentry_key }} - key_secret_posthog: ${{ matrix.group.posthog_key }} + key_for_sentry_secret: ${{ matrix.group.sentry_key }} + key_for_posthog_secret: ${{ matrix.group.posthog_key }} diff --git a/Dockerfile b/Dockerfile index 5ce601d52d..63011a2aa8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ FROM node:18.19.1-bullseye-slim AS tmp WORKDIR /app/tmp # Copy and install dependencies separately from the app's code -# To leverage Docker's cache when no dependency has change +# To leverage Docker's cache when no dependency has changed COPY packages/frontend/package.json ./packages/frontend/package.json COPY packages/jobs/package.json ./packages/jobs/package.json COPY packages/node-client/package.json ./packages/node-client/package.json From 7bc73543b38e25e2976c74220d7e432fd326d8a5 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:57:04 +0100 Subject: [PATCH 31/31] review --- Dockerfile | 4 ++-- scripts/build_docker.sh | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63011a2aa8..dde677e3d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # ------------------ # New tmp image # ------------------ -FROM node:18.19.1-bullseye-slim AS tmp +FROM node:18.19.1-bullseye-slim AS build # Setup the app WORKDIR WORKDIR /app/tmp @@ -74,7 +74,7 @@ USER node WORKDIR /app/nango # Code -COPY --from=tmp --chown=node:node /app/tmp /app/nango +COPY --from=build --chown=node:node /app/tmp /app/nango ARG image_env ARG git_hash diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 02899508d7..0de483022f 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -47,14 +47,14 @@ else fi echo "" -echo -e "Building nangohq/nango:${ENV}\n" +echo -e "Building nangohq/nango:$ENV\n" docker buildx build \ --platform linux/amd64 \ - --build-arg image_env=${ENV} \ - --build-arg git_hash=${GIT_HASH} \ - --build-arg posthog_key=${SENTRY_KEY} \ - --build-arg sentry_key=${POSTHOG_KEY} \ + --build-arg image_env="$ENV" \ + --build-arg git_hash="$GIT_HASH" \ + --build-arg posthog_key="$SENTRY_KEY" \ + --build-arg sentry_key="$POSTHOG_KEY" \ --cache-from type=gha \ --cache-to type=gha,mode=max \ --file ../Dockerfile \