From 2ef8f40665f281a6b85b36f638fc2ff9ca626423 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:47:35 +0100 Subject: [PATCH] chore: unified dockerfile (#1840) > [!NOTE] > This is a proposal, open to debate. ## Context Fixes NAN-596 We currently maintain 4 Dockerfiles, and need to build 7 in total. I feel like it's a lot to maintain. As we grow we will add more images and more folder in packages which can results in more stuff being build for nothing and errors. I would love to reduce this maintenance to 3 images (`staging`, `prod`, `enterprise`) and down to 1 if possible. Right now we need many versions also because we pass dedicated env vars to the Webapp build which is something we could do differently (e.g: serving those via API calls) ## Proposal - Build one unified Docker image `nangohq/nango` It's currently 148MB (compressed) [hub.docker.com](https://hub.docker.com/repository/docker/nangohq/nango/tags?page=1&ordering=last_updated) vs `nangohq/nango` is 145MB. So the difference is already minimal enough that it doesn't really matter. NB: the image is private rn NB: I disabled build for `entreprise` for the moment. I feel like it would be a waste of time and if we manage to get rid of the Env Vars situation, the single image could serve this scenario. ## Migration If we agree the migration path is not urgent. We just need to take the time to do it when we want. E.g for jobs: - changing the image to `nangohq/jobs:production` to `nangohq/nango:prod-` - adding a `Docker Command`, e.g: `node packages/jobs/dist/app.js` - When we deploy, instead of triggering a new deploy on the same tag, we can trigger a deploy with the new image url NB: this should also solve our docker caching issue ### Should we keep the other images? In the long-term I feel like it's not really useful, I think what other projects are doing is perfect with only 2-3 images: - project/image_community - project/image_entreprise - project/image_private (for us) --- .dockerignore | 7 ++ .github/workflows/build-image-reusable.yaml | 60 ++++++++++++++ .github/workflows/build-image.yaml | 34 ++++++++ Dockerfile | 87 +++++++++++++++++++++ package.json | 4 +- scripts/build_docker.sh | 62 +++++++++++++++ tsconfig.docker.json | 29 +++++++ 7 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-image-reusable.yaml create mode 100644 .github/workflows/build-image.yaml create mode 100644 Dockerfile create mode 100755 scripts/build_docker.sh create mode 100644 tsconfig.docker.json diff --git a/.dockerignore b/.dockerignore index f5adeae5e7..7270c0ab44 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,3 +12,10 @@ docker-compose.yml **/nango-data dev/ docs-v2/ +tsconfig.tsbuildinfo +integration-templates +tests +assets/ +vite*.ts +packages/cli/ +scripts/ diff --git a/.github/workflows/build-image-reusable.yaml b/.github/workflows/build-image-reusable.yaml new file mode 100644 index 0000000000..1141937bfb --- /dev/null +++ b/.github/workflows/build-image-reusable.yaml @@ -0,0 +1,60 @@ +name: Build unified Docker image + +on: + workflow_call: + inputs: + if: + description: 'Whether to run this job' + required: false + default: true + type: boolean + name: + required: true + type: string + key_for_sentry_secret: + required: false + type: string + key_for_posthog_secret: + required: false + type: string + +jobs: + build-container: + if: ${{ inputs.if }} + + runs-on: ubuntu-latest + env: + CAN_PUSH: "${{ secrets.DOCKER_PASSWORD != ' && secrets.DOCKER_USERNAME != ' }}" + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: env.CAN_PUSH == 'true' + with: + username: '${{ secrets.DOCKER_USERNAME }}' + password: '${{ secrets.DOCKER_PASSWORD }}' + + # Needed for buildx gha cache to work + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + + - name: Build image (${{ inputs.name }}) + run: | + export SENTRY_KEY=${{ secrets[inputs.key_for_sentry_secret] }} + export POSTHOG_KEY=${{ secrets[inputs.key_for_posthog_secret] }} + ./scripts/build_docker.sh build ${{ inputs.name }} ${{ env.SHA }} + + - name: Push image + if: env.CAN_PUSH == 'true' + run: | + docker push nangohq/nango:${{ inputs.name }}-${{ env.SHA }} diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml new file mode 100644 index 0000000000..df37f11b90 --- /dev/null +++ b/.github/workflows/build-image.yaml @@ -0,0 +1,34 @@ +name: '[Release] Build unified Docker image' + +on: + push: + branches: + - master + - staging/** + pull_request: + +jobs: + build-image: + strategy: + matrix: + group: + - name: 'staging' + if: true + sentry_key: SENTRY_KEY_staging + + - name: 'prod' + if: ${{ github.ref == 'refs/heads/master' }} + sentry_key: SENTRY_KEY_prod + posthog_key: POSTHOG_KEY_prod + + # Commented for now + # - name: 'enterprise' + # if: ${{ github.ref == 'refs/heads/master' }} + + secrets: inherit + uses: ./.github/workflows/build-image-reusable.yaml + with: + if: ${{ matrix.group.if }} + name: ${{ matrix.group.name }} + key_for_sentry_secret: ${{ matrix.group.sentry_key }} + key_for_posthog_secret: ${{ matrix.group.posthog_key }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..dde677e3d3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,87 @@ +# ------------------ +# New tmp image +# ------------------ +FROM node:18.19.1-bullseye-slim AS build + +# Setup the app WORKDIR +WORKDIR /app/tmp + +# Copy and install dependencies separately from the app's code +# To leverage Docker's cache when no dependency has changed +COPY packages/frontend/package.json ./packages/frontend/package.json +COPY packages/jobs/package.json ./packages/jobs/package.json +COPY packages/node-client/package.json ./packages/node-client/package.json +COPY packages/persist/package.json ./packages/persist/package.json +COPY packages/runner/package.json ./packages/runner/package.json +COPY packages/server/package.json ./packages/server/package.json +COPY packages/shared/package.json ./packages/shared/package.json +COPY packages/webapp/package.json ./packages/webapp/package.json +COPY package*.json ./ + +# Install every dependencies +RUN true \ + && npm i + +# At this stage we copy back all sources +COPY . /app/tmp + +# Build the backend separately because it can be cached even when we change the env vars +RUN true \ + && npm run ts-build:docker + +# /!\ Do not set NODE_ENV=production before building, it will break some modules +# ENV NODE_ENV=production +ARG image_env +ARG posthog_key +ARG sentry_key + +# TODO: remove the need for this +ENV REACT_APP_ENV $image_env +ENV REACT_APP_PUBLIC_POSTHOG_HOST https://app.posthog.com +ENV REACT_APP_PUBLIC_POSTHOG_KEY $posthog_key +ENV REACT_APP_PUBLIC_SENTRY_KEY $sentry_key + +# Build the frontend +RUN true \ + && npm run -w @nangohq/webapp build + +# Clean src +RUN true \ + && rm -rf packages/*/src \ + && rm -rf packages/*/lib \ + && rm -rf packages/webapp/public \ + && rm -rf packages/webapp/node_modules + +# Clean dev dependencies +RUN true \ + && npm prune --omit=dev --omit=peer --omit=optional + +# ---- Web ---- +# Resulting new, minimal image +FROM node:18.19.1-bullseye-slim as web + + +# - Bash is just to be able to log inside the image and have a decent shell +RUN true \ + && apt update && apt-get install -y bash ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false + +# Do not use root to run the app +USER node + +WORKDIR /app/nango + +# Code +COPY --from=build --chown=node:node /app/tmp /app/nango + +ARG image_env +ARG git_hash + +ENV PORT=8080 +ENV NODE_ENV=production +ENV IMAGE_ENV $image_env +ENV GIT_HASH $git_hash + +EXPOSE 8080 diff --git a/package.json b/package.json index 3f39791a20..9942d1034a 100644 --- a/package.json +++ b/package.json @@ -19,14 +19,16 @@ "prettier-watch": "onchange './**/*.{ts,tsx}' -- prettier --write {{changed}}", "lint": "eslint . --ext .ts,.tsx", "lint:fix": "eslint . --ext .ts,.tsx --fix", + "ts-build": "tsc -b tsconfig.build.json && npm run postbuild -ws --if-present", + "ts-build:docker": "tsc -b tsconfig.docker.json", "ts-clean": "npx rimraf packages/*/tsconfig.tsbuildinfo packages/*/dist", - "ts-build": "tsc -b --clean packages/shared packages/server packages/cli packages/runner packages/jobs packages/persist && tsc -b tsconfig.build.json && npm run postbuild -ws --if-present && tsc -b packages/webapp/tsconfig.json", "docker-build": "docker build -f packages/server/Dockerfile -t nango-server:latest .", "webapp-build:hosted": "cd ./packages/webapp && npm run build:hosted && cd ../..", "webapp-build:staging": "cd ./packages/webapp && npm run build:staging && cd ../..", "webapp-build:prod": "cd ./packages/webapp && npm run build:prod && cd ../..", "webapp-build:enterprise": "cd ./packages/webapp && npm run build:enterprise && cd ../..", "webapp-build:watch": "tsc -b -w packages/webapp/tsconfig.json", + "docker-build:unified": "./scripts/build_docker.sh", "shared:build": "cd ./packages/shared && npm run build && cd ../..", "build:hosted": "npm i && npm run ts-build && npm run webapp-build:hosted ", "build:staging": "npm i && npm run ts-build && npm run webapp-build:staging", diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh new file mode 100755 index 0000000000..0de483022f --- /dev/null +++ b/scripts/build_docker.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -e + +ACTION=$1 +ENV=$2 # enterprise | hosted | prod | staging +GIT_HASH=$3 + +USAGE="./build_docker.sh GIT_HASH" +RED='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' + +if [ "$ACTION" != "push" ] && [ "$ACTION" != "build" ]; then + echo -e "${RED}Please specify an action${NC}\n" + echo "$USAGE" + exit +fi + +if [ "$ENV" != "enterprise" ] && [ "$ENV" != "hosted" ] && [ "$ENV" != "prod" ] && [ "$ENV" != "staging" ]; then + echo -e "${RED}Please specify an environment${NC}\n" + echo "$USAGE" + exit +fi + +if [ -z $GIT_HASH ]; then + echo -e "${RED}GIT_HASH is empty${NC}" + exit +fi + +if [ -z $SENTRY_KEY ]; then + echo -e "${YELLOW}SENTRY_KEY is empty${NC}" +fi +if [ -z $POSTHOG_KEY ]; then + echo -e "${YELLOW}POSTHOG_KEY is empty${NC}" +fi + +# Move to here no matter where the file was executed +cd "$(dirname "$0")" + +tags="-t nangohq/nango:${ENV}-${GIT_HASH}" + +if [ $ACTION == 'build' ]; then + tags+=" --output=type=docker" +else + tags+=" --output=type=registry" +fi + +echo "" +echo -e "Building nangohq/nango:$ENV\n" + +docker buildx build \ + --platform linux/amd64 \ + --build-arg image_env="$ENV" \ + --build-arg git_hash="$GIT_HASH" \ + --build-arg posthog_key="$SENTRY_KEY" \ + --build-arg sentry_key="$POSTHOG_KEY" \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ + --file ../Dockerfile \ + $tags \ + ../ diff --git a/tsconfig.docker.json b/tsconfig.docker.json new file mode 100644 index 0000000000..d025108fd4 --- /dev/null +++ b/tsconfig.docker.json @@ -0,0 +1,29 @@ +{ + "files": [], + "references": [ + { + "path": "packages/frontend" + }, + { + "path": "packages/jobs" + }, + { + "path": "packages/node-client" + }, + { + "path": "packages/persist" + }, + { + "path": "packages/runner" + }, + { + "path": "packages/server" + }, + { + "path": "packages/shared" + }, + { + "path": "packages/webapp" + } + ] +}