Use environment variables to configure docker compose variable proper…

…ties and build with bake (#22225)

* Remove redundant docker-cache directory

* Use environment variables to configure docker compose variable properties

.circleci/config.yml

@@ -357,25 +357,19 @@ commands:
         type: string
         default: "latest"
     steps:
+      - run:
+          name: Set environment variables
+          command: |
+            echo 'export DOCKER_VERSION=<< parameters.image_tag >>' >> $BASH_ENV
+            echo 'export DOCKER_COMMIT=$CIRCLE_SHA1' >> $BASH_ENV
+            echo 'export VERSION_BUILD_URL=$CIRCLE_BUILD_URL' >> $BASH_ENV
+            echo 'export DOCKER_PUSH=<< parameters.push >>' >> $BASH_ENV
       - run:
           name: Build docker image and push to repo
           command: |
             docker version
             docker login -u "${DOCKERHUB_USER}" -p "${DOCKERHUB_PASS}"
-            make build_docker_image \
-              DOCKER_TAG="app:build" \
-              DOCKER_VERSION=<< parameters.image_tag >> \
-              DOCKER_COMMIT="$CIRCLE_SHA1" \
-              VERSION_BUILD_URL="$CIRCLE_BUILD_URL"
-            docker images
-      - when:
-          condition: << parameters.push >>
-          steps:
-            - run:
-                name: Wait for services to be ready
-                command: |
-                  docker tag app:build "${DOCKERHUB_REPO}":<< parameters.image_tag >>
-                  docker push "${DOCKERHUB_REPO}":<< parameters.image_tag >>
+            make build_docker_image
 
   better_checkout:
     description: circle ci checkout step on steroids
@@ -624,7 +618,7 @@ jobs:
     steps:
       - checkout
       - make_release:
-          image_tag: ${CIRCLE_BRANCH}
+          image_tag: circle-${CIRCLE_BRANCH}
           # explicitly don't push
           push: false
 

.dockerignore

@@ -7,6 +7,4 @@ node_modules/
 storage/
 logs/*
 
-# Don't include the docker cache in the build context or you will get memory leaks
-docker-cache/
-docker-cache-new/
+

.github/actions/build-docker/action.yml

@@ -82,15 +82,10 @@ runs:
           VERSION_BUILD_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
 
     - name: Build Image
-      uses: docker/build-push-action@v5
+      uses: docker/bake-action@v4
       with:
-        context: .
-        platforms: linux/amd64
-        pull: true
+        targets: web
         push: ${{ inputs.push }}
         load: ${{ inputs.push == 'false' }}
-        tags: ${{ steps.meta.outputs.tags }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
-        build-args: |
-            PYTHON_VERSION=${{ inputs.python_version }}
+      env:
+        DOCKER_VERSION: ${{ steps.meta.outputs.version }}

.github/workflows/verify-docker-image.yml

@@ -6,6 +6,48 @@ on:
       - master
 
 jobs:
+  docker_config_check:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - expected: { version: local, push: false }
+            input: { version: '', push: '' }
+          - expected: { version: version, push: true }
+            input: { version: version, push: true }
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check Docker Compose (default)
+        id: default
+        env:
+          DOCKER_VERSION: ${{ matrix.input.version }}
+          DOCKER_PUSH: ${{ matrix.input.push }}
+        shell: bash
+        run: |
+          set -xue
+          config=$(make docker_compose_config)
+
+          # Expect image tag is correct
+          echo $config | grep -q "image: mozilla/addons-server:${{ matrix.expected.version }}"
+
+          # Expect docker push args are correct
+          if [[ ${{ matrix.expected.push }} == "true" ]]; then
+            echo $config | grep -q -- "--push"
+            echo $config | grep -v -q -- "--load"
+          else
+            echo $config | grep -v -q -- "--push"
+            echo $config | grep -q -- "--load"
+          fi
+
+          # Expect docker platform is correct
+          echo $config | grep -q -- "platform: linux/amd64"
+          echo $config | grep -q -- "\"platforms\": \[
+            \"linux/amd64\"
+          \]"
+
+
   verify_docker_image:
     runs-on: ubuntu-latest
 
@@ -16,7 +58,32 @@ jobs:
         id: build
         uses: ./.github/actions/build-docker
 
-      - name: Smoke test
+      - name: Create failure
+        id: failure
+        uses: ./.github/actions/run-docker
+        with:
+          image: ${{ steps.build.outputs.tags }}
+          run: |
+            exit 1
+        continue-on-error: true
+      - name: Verify failure
+        if: always()
+        run: |
+          if [ "${{ steps.failure.outcome }}" -ne "failure" ]; then
+            echo "Expected failure"
+            exit 1
+          fi
+
+      - name: Check (special characters in command)
+        uses: ./.github/actions/run-docker
+        with:
+          image: ${{ steps.build.outputs.tags }}
+          run: |
+            echo 'this is a question?'
+            echo 'a * is born'
+            echo 'wow an array []'
+
+      - name: Manage py check
         uses: ./.github/actions/run-docker
         with:
           image: ${{ steps.build.outputs.tags }}

Makefile-os

@@ -2,14 +2,12 @@ export HOST_UID := $(shell id -u)
 
 export DOCKER_BUILDER=container
 
-DOCKER_TAG := addons-server-test
-DOCKER_PLATFORM := linux/amd64
-DOCKER_PROGRESS := auto
-export DOCKER_COMMIT := $(shell git rev-parse HEAD || echo "commit")
-DOCKER_CACHE_DIR := docker-cache
-
+DOCKER_PROGRESS ?= auto
+DOCKER_PUSH ?= false
+DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit")
+VERSION_BUILD_URL ?= build
+# Exporting these variables make them default values for docker-compose*.yml files
 export DOCKER_VERSION ?= local
-export VERSION_BUILD_URL ?= build
 
 .PHONY: help_redirect
 help_redirect:
@@ -51,35 +49,37 @@ create_docker_builder: ## Create a custom builder for buildkit to efficiently bu
 		--name $(DOCKER_BUILDER) \
 		--driver=docker-container
 
-DOCKER_BUILD_ARGS := -t $(DOCKER_TAG) \
---load \
---platform $(DOCKER_PLATFORM) \
+DOCKER_BUILD_ARGS := \
 --progress=$(DOCKER_PROGRESS) \
 --builder=$(DOCKER_BUILDER) \
---label git.commit=$(DOCKER_COMMIT) \
---cache-to=type=local,dest=$(DOCKER_CACHE_DIR)-new \
-
-DOCKER_CACHE_INDEX = $(wildcard $(DOCKER_CACHE_DIR)/index.json)
 
-ifneq ($(DOCKER_CACHE_INDEX),)
-	DOCKER_BUILD_ARGS += --cache-from=type=local,src=$(DOCKER_CACHE_DIR),mode=max
+ifeq ($(DOCKER_PUSH), true)
+	DOCKER_BUILD_ARGS += --push
+else
+	DOCKER_BUILD_ARGS += --load
 endif
 
 .PHONY: version
 version: ## create version.json file
 	./scripts/version.sh $(DOCKER_VERSION) $(DOCKER_COMMIT) $(VERSION_BUILD_URL)
 
+.PHONY: docker_compose_config
+docker_compose_config: ## Show the docker compose configuration
+	@echo "version: $(DOCKER_VERSION)"
+	@echo "push: $(DOCKER_PUSH)"
+	docker compose config web
+	docker buildx bake web --print
+	echo $(DOCKER_BUILD_ARGS)
+
 .PHONY: build_docker_image
-build_docker_image: create_docker_builder version ## Build the docker image
-	DOCKER_BUILDKIT=1 docker buildx build $(DOCKER_BUILD_ARGS) .
-	rm -rf $(DOCKER_CACHE_DIR)
-	mv $(DOCKER_CACHE_DIR)-new $(DOCKER_CACHE_DIR)
+build_docker_image: create_docker_builder version docker_compose_config ## Build the docker image
+	docker buildx bake web $(DOCKER_BUILD_ARGS) --print
+	docker buildx bake web $(DOCKER_BUILD_ARGS)
 
 .PHONY: clean_docker
 clean_docker: ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this commandUse with caution.
-	docker compose down --rmi all --volumes
-	docker rmi $(DOCKER_TAG) || true
-	rm -rf $(DOCKER_CACHE_DIR)
+	docker compose down --rmi local --volumes --remove-orphans
+	docker buildx prune -af
 	rm -rf ./deps/**
 
 .PHONY: initialize_docker

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "2.4"
-
 x-env-mapping: &env
   environment:
     - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia
@@ -27,7 +25,17 @@ x-env-mapping: &env
 services:
   worker: &worker
     <<: *env
-    image: mozilla/addons-server:latest
+    image: mozilla/addons-server:${DOCKER_VERSION:-local}
+    build:
+      context: .
+      dockerfile: Dockerfile
+      cache_from:
+        - type=gha
+      cache_to:
+        - type=gha,mode=max
+      x-bake:
+        pull: true
+        platforms: linux/amd64
     # We drop down to a different user through supervisord, but starting as
     # root allows us to fix the ownership of files generated at image build
     # time through the ./docker/entrypoint.sh script.
@@ -47,7 +55,6 @@ services:
 
   web:
     <<: *worker
-    platform: linux/amd64
     command:
       - supervisord -n -c /data/olympia/docker/supervisor.conf
   nginx:
@@ -134,3 +141,6 @@ services:
       # exposed using webpack and not by the node app server).
       - 7011:7011
     command: yarn amo:olympia
+
+networks:
+  default: