Support for pre.Dockerfile.* (#3999) fixes #3960, fixes #3988

.github/workflows/master-build.yml

@@ -5,7 +5,7 @@ defaults:
     shell: bash
 on:
   push:
-    branches: [ master, main, 20220714_publish_binaries_on_master_workflow ]
+    branches: [ master, main ]
   release:
     types: [ created ]
 

cmd/ddev/cmd/debug-capabilities.go

@@ -11,7 +11,7 @@ var DebugCapabilitiesCmd = &cobra.Command{
 	Use:   "capabilities",
 	Short: "Show capabilities of this version of ddev",
 	Run: func(cmd *cobra.Command, args []string) {
-		capabilities := []string{"multiple-dockerfiles", "interactive-project-selection", "ddev-get-yaml-interpolation"}
+		capabilities := []string{"multiple-dockerfiles", "interactive-project-selection", "ddev-get-yaml-interpolation", "pre-dockerfile-insertion"}
 		output.UserOut.WithField("raw", capabilities).Print(strings.Join(capabilities, "\n"))
 	},
 }

docs/content/users/extend/customizing-images.md

@@ -36,7 +36,16 @@ For more complex requirements, you can add:
 * `.ddev/db-build/Dockerfile`
 * `.ddev/db-build/Dockerfile.*`
 
-These files' content will be inserted into the constructed Dockerfile for each image. They are inserted *before* most of the rest of the things that are done to build the image, and are done in alpha order, so `Dockerfile` is inserted first, followed by `Dockerfile.*` in alpha order. You can examine the resultant Dockerfile (which should not be changed as it is generated) at `.ddev/.webimageBuild/Dockerfile` and you can force a rebuild with `ddev debug refresh`.
+These files' content will be inserted into the constructed Dockerfile for each image. They are inserted *after* most of the rest of the things that are done to build the image, and are done in alpha order, so `Dockerfile` is inserted first, followed by `Dockerfile.*` in alpha order.
+
+For certain use cases, you might need to add stuff very early on the Dockerfile (i.e. proxy settings, SSL termination, etc.) you can also create:
+
+* `.ddev/web-build/pre.Dockerfile.*`
+* `.ddev/db-build/pre.Dockerfile.*`
+
+These are inserted *before* everything else.
+
+You can examine the resultant Dockerfile (which should not be changed as it is generated) at `.ddev/.webimageBuild/Dockerfile` and you can force a rebuild with `ddev debug refresh`.
 
 Examples of possible Dockerfiles are given in `.ddev/web-build/Dockerfile.example` and `.ddev/db-build/Dockerfile.example` (these examples are created in your project when you `ddev config` the project).
 
@@ -74,10 +83,6 @@ ENV COMPOSER_HOME=""
 
 **Remember that the Dockerfile is building a docker image that will be used later with ddev.** At the time the Dockerfile is executing, your code is not mounted and the container is not running, it's just being built. So for example, an `npm install` in /var/www/html will not do anything useful because the code is not there at image building time.
 
-## HTTP proxy support inside containers and during build
-
-DDEV will automatically recognize systems that have the environment variables HTTP_PROXY, HTTPS_PROXY, and NO_PROXY set to configure proxy behavior. It will then configure generated images to include those values and set them up to be able to use the proxy during build time and at run time.
-
 ### Debugging the Dockerfile build
 
 It can be complicated to figure out what's going on when building a Dockerfile, and even more complicated when you're seeing it go by as part of `ddev start`.

pkg/ddevapp/config.go

@@ -879,61 +879,33 @@ func WriteBuildDockerfile(fullpath string, userDockerfilePath string, extraPacka
 ARG BASE_IMAGE
 FROM $BASE_IMAGE
 `
-	// Provide proxy handling inside container if necessary
-	proxyVars := []string{"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"}
-	useProxy := false
-	for _, proxyVar := range proxyVars {
-		v := os.Getenv(proxyVar)
-		if v != "" {
-			useProxy = true
-			contents = contents + fmt.Sprintf("\nENV %s %s\n", proxyVar, v)
-		}
-	}
-	if useProxy {
-		contents = contents + `
-RUN if [ ! -z "${HTTP_PROXY}" ]; then printf "Acquire {\nHTTP::proxy \"$HTTP_PROXY\";\nHTTPS::proxy \"$HTTPS_PROXY\";\n}\n"  > /etc/apt/apt.conf.d/proxy.conf ; fi`
-	}
-
 	contents = contents + `
 ARG username
 ARG uid
 ARG gid
 RUN (groupadd --gid $gid "$username" || groupadd "$username" || true) && (useradd  -l -m -s "/bin/bash" --gid "$username" --comment '' --uid $uid "$username" || useradd  -l -m -s "/bin/bash" --gid "$username" --comment '' "$username" || useradd  -l -m -s "/bin/bash" --gid "$gid" --comment '' "$username")
 `
-	// If there are user dockerfiles, insert their contents
+	// If there are user pre.Dockerfile* files, insert their contents
 	if userDockerfilePath != "" {
-		files, err := filepath.Glob(userDockerfilePath + "/Dockerfile*")
+		files, err := filepath.Glob(userDockerfilePath + "/pre.Dockerfile*")
 		if err != nil {
 			return err
 		}
 
 		for _, file := range files {
-			// We'll skip the example file
-			if file == userDockerfilePath+"/Dockerfile.example" {
-				continue
-			}
-
 			userContents, err := fileutil.ReadFileIntoString(file)
 			if err != nil {
 				return err
 			}
 
-			// Backward compatible fix, remove unnecessary BASE_IMAGE references
-			re, err := regexp.Compile(`ARG BASE_IMAGE.*\n|FROM \$BASE_IMAGE.*\n`)
-			if err != nil {
-				return err
-			}
-
-			userContents = re.ReplaceAllString(userContents, "")
-			contents = contents + "\n\n### From user file " + file + ":\n" + userContents
+			contents = contents + "\n\n### From user Dockerfile " + file + ":\n" + userContents
 		}
 	}
 
 	if extraPackages != nil {
-
 		contents = contents + `
-### from webimage_extra_packages or dbimage_extra_packages
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confold" --no-install-recommends --no-install-suggests ` + strings.Join(extraPackages, " ") + "\n"
+### DDEV-injected from webimage_extra_packages or dbimage_extra_packages
+RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y -o Dpkg::Options::="--force-confold" --no-install-recommends --no-install-suggests ` + strings.Join(extraPackages, " ") + "\n"
 	}
 
 	// For webimage, update to latest composer.
@@ -957,11 +929,46 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg:
 		// Try composer self-update twice because of troubles with composer downloads
 		// breaking testing.
 		contents = contents + fmt.Sprintf(`
+### DDEV-injected composer update
 RUN export XDEBUG_MODE=off && ( composer self-update %s || composer self-update %s || true )
 `, composerSelfUpdateArg, composerSelfUpdateArg)
 	}
 
-	contents = contents + extraContent
+	if extraContent != "" {
+		contents = contents + fmt.Sprintf(`
+### DDEV-injected extra content
+%s
+`, extraContent)
+	}
+
+	// If there are user dockerfiles, appends their contents
+	if userDockerfilePath != "" {
+		files, err := filepath.Glob(userDockerfilePath + "/Dockerfile*")
+		if err != nil {
+			return err
+		}
+
+		for _, file := range files {
+			// Skip the example file
+			if file == userDockerfilePath+"/Dockerfile.example" {
+				continue
+			}
+
+			userContents, err := fileutil.ReadFileIntoString(file)
+			if err != nil {
+				return err
+			}
+
+			// Backward compatible fix, remove unnecessary BASE_IMAGE references
+			re, err := regexp.Compile(`ARG BASE_IMAGE.*\n|FROM \$BASE_IMAGE.*\n`)
+			if err != nil {
+				return err
+			}
+
+			userContents = re.ReplaceAllString(userContents, "")
+			contents = contents + "\n\n### From user Dockerfile " + file + ":\n" + userContents
+		}
+	}
 
 	// Assets in the web-build directory copied to .webimageBuild so .webimageBuild can be "context"
 	// This actually copies the Dockerfile, but it is then immediately overwritten by WriteImageDockerfile()

pkg/ddevapp/config_test.go

@@ -1128,6 +1128,20 @@ ARG BASE_IMAGE
 FROM $BASE_IMAGE
 RUN touch /var/tmp/`+"added-by-"+item+"-test2.txt"))
 		assert.NoError(err)
+
+		// Testing pre.Dockerfile.*
+		err = WriteImageDockerfile(app.GetConfigPath(item+"-build/pre.Dockerfile.test3"), []byte(`
+RUN touch /var/tmp/`+"added-by-"+item+"-test3.txt"))
+		assert.NoError(err)
+
+		// Testing that pre comes before post, we create a file on pre and remove
+		// it on post
+		err = WriteImageDockerfile(app.GetConfigPath(item+"-build/pre.Dockerfile.test4"), []byte(`
+RUN touch /var/tmp/`+"added-by-"+item+"-test4.txt"))
+		assert.NoError(err)
+		err = WriteImageDockerfile(app.GetConfigPath(item+"-build/Dockerfile.test4"), []byte(`
+RUN rm /var/tmp/`+"added-by-"+item+"-test4.txt"))
+		assert.NoError(err)
 	}
 	// Start and make sure that the packages don't exist already
 	err = app.Start()
@@ -1155,6 +1169,16 @@ RUN touch /var/tmp/`+"added-by-"+item+"-test2.txt"))
 			Cmd:     "ls /var/tmp/added-by-" + item + "-test2.txt",
 		})
 		assert.NoError(err)
+		_, _, err = app.Exec(&ExecOpts{
+			Service: item,
+			Cmd:     "ls /var/tmp/added-by-" + item + "-test3.txt",
+		})
+		assert.NoError(err)
+		_, _, err = app.Exec(&ExecOpts{
+			Service: item,
+			Cmd:     "ls /var/tmp/added-by-" + item + "-test4.txt",
+		})
+		assert.Error(err)
 	}
 
 	err = app.Stop(true, false)