Development: Restructure docker compose setups (#5915)

.bamboo/E2E-tests-with-flake-detection/cleanup.sh

@@ -1,7 +1,28 @@
 #!/bin/sh
 
-cd src/main/docker/cypress
-
 # HOST_HOSTNAME not really necessary for shutdown but otherwise docker-compose complains
 export HOST_HOSTNAME=$(hostname)
-docker compose -f cypress-E2E-tests.yml -f cypress-E2E-tests-coverage-override.yml down -v
+# show all running docker containers and volumes after the cleanup to detect issues
+echo "SHOW RUNNING Docker containers and volumes:"
+docker ps -a
+docker volume ls
+# first kill ALL containers on the bamboo agent
+echo "KILLING Docker containers, volumes and networks:"
+docker container stop $(docker ps -a -q) || true
+docker container rm $(docker ps -a -q) || true
+docker volume rm $(docker volume ls -q) || true
+
+# then kill remaining project volumes and networks which should be easy removable as not bound to containers anymore
+docker compose -f ./docker/cypress-E2E-tests.yml -f ./docker/cypress-E2E-tests-coverage-override.yml down -v
+
+
+# show all running docker containers and volumes after the cleanup to detect issues
+echo "SHOW RUNNING Docker containers and volumes:"
+docker ps -a
+docker volume ls
+
+# show docker and docker compose version
+echo "VERSIONS:"
+docker compose version || true
+docker-compose version || true
+docker version || true

.bamboo/E2E-tests-with-flake-detection/execute.sh

@@ -4,29 +4,26 @@
 mkdir -p build/libs
 mv ./*.war build/libs/
 
+# pass current host's hostname to the docker container for server.url (see docker compose config file)
+export HOST_HOSTNAME=$(hostname)
+
 # Load git history needed for analysis
 git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
-
 git fetch --unshallow || git fetch --all
 
-docker build . -f ./src/main/docker/Dockerfile -t artemis:coverage-latest
-
-# Start Artemis docker containers with docker-compose
-cd src/main/docker/cypress
-
-# pass current host's hostname to the docker container for server.url (see docker compose config file)
-export HOST_HOSTNAME=$(hostname)
-
-docker compose -f cypress-E2E-tests.yml -f cypress-E2E-tests-coverage-override.yml pull
+cd docker
+docker build .. --build-arg WAR_FILE_STAGE=external_builder -f ./artemis/Dockerfile -t artemis:coverage-latest
+docker compose -f cypress-E2E-tests.yml -f cypress-E2E-tests-coverage-override.yml pull nginx mysql
 docker compose -f cypress-E2E-tests.yml -f cypress-E2E-tests-coverage-override.yml build --no-cache --pull artemis-cypress
 #do not pull the base image artemis:coverage-latest for artemis-app as it's stored locally and built above
-docker compose -f cypress-E2E-tests.yml -f cypress-E2E-tests-coverage-override.yml build --no-cache artemis-app
+docker compose -f cypress-E2E-tests.yml -f cypress-E2E-tests-coverage-override.yml build --build-arg WAR_FILE_STAGE=external_builder --no-cache artemis-app
 docker compose -f cypress-E2E-tests.yml -f cypress-E2E-tests-coverage-override.yml up --exit-code-from artemis-cypress
 exitCode=$?
+cd ..
 echo "Cypress container exit code: $exitCode"
 if [ $exitCode -eq 0 ]
 then
-    touch ../../../../.successful
+    touch .successful
 else
     echo "Not creating success file because the tests failed"
 fi

.bamboo/E2E-tests/cleanup.sh

@@ -1,7 +1,26 @@
 #!/bin/sh
 
-cd src/main/docker/cypress
-
 # HOST_HOSTNAME not really necessary for shutdown but otherwise docker-compose complains
 export HOST_HOSTNAME=$(hostname)
-docker compose -f cypress-E2E-tests.yml down -v
+# show all running docker containers and volumes after the cleanup to detect issues
+echo "SHOW RUNNING Docker containers and volumes:"
+docker ps -a
+docker volume ls
+# first kill ALL containers on the bamboo agent
+echo "KILLING Docker containers, volumes and networks:"
+docker container stop $(docker ps -a -q) || true
+docker container rm $(docker ps -a -q) || true
+docker volume rm $(docker volume ls -q) || true
+
+docker compose -f ./docker/cypress-E2E-tests.yml down -v
+
+# show all running docker containers and volumes after the cleanup to detect issues
+echo "SHOW RUNNING Docker containers and volumes:"
+docker ps -a
+docker volume ls
+
+# show docker and docker compose version
+echo "VERSIONS:"
+docker compose version || true
+docker-compose version || true
+docker version || true

.bamboo/E2E-tests/execute.sh

@@ -4,20 +4,20 @@
 mkdir -p build/libs
 mv ./*.war build/libs/
 
-# Start Artemis docker containers with docker-compose
-cd src/main/docker/cypress
-
 # pass current host's hostname to the docker container for server.url (see docker compose config file)
 export HOST_HOSTNAME=$(hostname)
 
-docker compose -f cypress-E2E-tests.yml pull
-docker compose -f cypress-E2E-tests.yml build --no-cache --pull
+cd docker
+#just pull everything else than artemis-app as we build it later either way
+docker compose -f cypress-E2E-tests.yml pull artemis-cypress mysql nginx
+docker compose -f cypress-E2E-tests.yml build --build-arg WAR_FILE_STAGE=external_builder --no-cache --pull artemis-app
 docker compose -f cypress-E2E-tests.yml up --exit-code-from artemis-cypress
 exitCode=$?
+cd ..
 echo "Cypress container exit code: $exitCode"
 if [ $exitCode -eq 0 ]
 then
-    touch ../../../../.successful
+    touch .successful
 else
     echo "Not creating success file because the tests failed"
 fi

.dockerignore

@@ -0,0 +1,39 @@
+# https://docs.docker.com/engine/reference/builder/#dockerignore-file
+classes/
+generated-sources/
+generated-test-sources/
+h2db/
+maven-archiver/
+maven-status/
+reports/
+surefire-reports/
+test-classes/
+test-results/
+www/
+!*.jar
+!*.war
+
+# exclude hidden directories like .idea, .gradle, .cache, .github
+.*
+# do not exclude .git as it's necessary to insert the commit id into the build
+!.git
+# do not exclude .eslint files as they are required in the build process
+!.eslintignore
+!.eslintrc
+# do not exclude .npmrc as it's required in the build process for setting the correct node options
+!.npmrc
+# exclude node_modules if installed locally
+node_modules
+# exclude build binaries except a pre-built .war file
+build/*
+!build/libs/*.war
+
+# exclude the docker files and the /docker/.docker-data folders
+docker/
+
+# files inside of the root directory not needed
+CITATION.cff
+CODE_OF_CONDUCT.md
+LICENSE
+README.md
+SECURITY.md

.eslintignore

@@ -11,7 +11,7 @@ node_modules/
 out/
 repos/
 repos-download/
-src/main/docker/
+docker/
 src/main/generated/
 src/main/resources/
 uploads/

.github/workflows/build-deploy.yml

@@ -40,6 +40,7 @@ env:
 jobs:
 
   build:
+    name: Build .war artifact
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -73,18 +74,10 @@ jobs:
         asset_content_type: application/x-webarchive
 
   docker:
+    name: Build and Push Docker Image
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'ls1intum/Artemis' }}
-    needs: build
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: Download Artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: Artemis.war
-    - name: Build Docker Image
-      run: docker build --build-arg WAR_PATH=. -t artemis -f src/main/docker/Dockerfile .
-
     - name: Compute Tag
       uses: actions/github-script@v6
       id: compute-tag
@@ -106,22 +99,30 @@ jobs:
             }
           }
           return "FALSE";
-
-    # Push to GitHub Container Registry
+    - uses: actions/checkout@v3
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+    # Build and Push to GitHub Container Registry
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v2
+      if: ${{ steps.compute-tag.outputs.result != 'FALSE' }}
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Build and Push to GitHub Container Registry
+      uses: docker/build-push-action@v3
       if: ${{ steps.compute-tag.outputs.result != 'FALSE' }}
-    - name: Push to GitHub Container Registry
-      env:
-        TAG: ${{ steps.compute-tag.outputs.result }}
-      run: |
-        docker tag artemis ghcr.io/ls1intum/artemis:$TAG
-        docker push ghcr.io/ls1intum/artemis:$TAG
-      if: ${{ steps.compute-tag.outputs.result != 'FALSE' }}
+      with:
+        # TODO: maybe even add arm/v7 for older DevBoards and arm64 for Mac M1 and new DevBoards
+        # TODO: add arm64 support by first building the .war file in a amd64 step then creating the runtime env
+        platforms: amd64
+        file: ./docker/artemis/Dockerfile
+        context: .
+        tags: ghcr.io/ls1intum/artemis:${{ steps.compute-tag.outputs.result }}
+        push: true
 
     # TODO: Push to Docker Hub (develop + tag)
 

.gitignore

@@ -18,7 +18,7 @@ node_modules/
 npm-debug.log.*
 /.awcache/*
 .node-gyp/
-/.cache-loader/*
+/.cache-loader/
 
 ######################
 # SASS
@@ -59,8 +59,9 @@ local.properties
 ######################
 # Intellij
 ######################
-.idea/
-!.idea/runConfigurations/*
+.idea/*
+!.idea/runConfigurations/
+!.idea/icon.svg
 *.iml
 *.iws
 *.ipr
@@ -151,6 +152,11 @@ Desktop.ini
 repos/
 repos-download/
 /templates/
+.bash_history
+.v8flags.*.json
+/src/main/generated
+/coverage
+/exports
 
 ######################
 # Gradle Wrapper
@@ -171,20 +177,26 @@ repos-download/
 ######################
 # Docker
 ######################
-.bash_history
-.v8flags.*.json
-/.cache-loader
-/src/main/generated
-/coverage
-/exports
-/src/main/docker/.env
-/src/main/resources/config/application-local*.yml
-/src/main/resources/id_*
-/src/main/resources/known_hosts
+/docker/.env
+/docker/.docker-data/artemis-data/*
+!/docker/.docker-data/artemis-data/.gitkeep
+/docker/.docker-data/artemis-mysql-data/*
+!/docker/.docker-data/artemis-mysql-data/.gitkeep
+
+######################
+# Cypress
+######################
 /src/test/cypress/screenshots/
 /src/test/cypress/videos/
 /src/test/cypress/build
 
+######################
+# Artemis configs
+######################
+/src/main/resources/config/application-local*.yml
+/src/main/resources/id_*
+/src/main/resources/known_hosts
+
 ######################
 # Coverage Files
 ######################

build.gradle

@@ -61,7 +61,18 @@ spotless {
     java {
         target project.fileTree(project.rootDir) {
             include "**/*.java"
-            exclude "**/src/main/java/de/tum/in/www1/artemis/service/connectors/BambooService.java", "**/src/test/resources/test-data/repository-export/EncodingISO_8559_1.java", "**/node_modules/**", "**/out/**", "**/repos/**", "**/repos-download/**", "**/build/**", "**/src/main/generated/**", "**/src/main/resources/templates/**"
+            exclude(
+                "**/src/main/java/de/tum/in/www1/artemis/service/connectors/BambooService.java",
+                "**/src/test/resources/test-data/repository-export/EncodingISO_8559_1.java",
+                "**/node_modules/**",
+                "**/out/**",
+                "**/repos/**",
+                "**/repos-download/**",
+                "**/build/**",
+                "**/src/main/generated/**",
+                "**/src/main/resources/templates/**",
+                "/docker/**"
+            )
         }
         importOrderFile "artemis-spotless.importorder"
         eclipse("4.19.0").configFile "artemis-spotless-style.xml"
@@ -404,10 +415,6 @@ checkstyle {
     maxErrors = 0
 }
 
-task buildJarForDocker (type: Exec) {
-    commandLine "./gradlew", "build", "-x", "webapp", "-x", "test", "-x", "jacocoTestCoverageVerification"
-}
-
 def isNonStable = { String version ->
     def stableKeyword = ["RELEASE", "FINAL", "GA"].any { it -> version.toUpperCase().contains(it) }
     def regex = /^[0-9,.v-]+(-r)?$/