refactor: move projects list into its own yaml file, fixes #5639 (#5651)

.buildkite/testbot_maintenance.sh

@@ -4,7 +4,7 @@ set -eu -o pipefail
 
 os=$(go env GOOS)
 
-rm -rf ~/.ddev/Test* ~/.ddev/global_config.yaml ~/.ddev/homeadditions ~/.ddev/commands ~/.ddev/bin/docker-compose* ~/tmp/ddevtest
+rm -rf ~/.ddev/Test* ~/.ddev/global_config.yaml ~/.ddev/project_list.yaml ~/.ddev/homeadditions ~/.ddev/commands ~/.ddev/bin/docker-compose* ~/tmp/ddevtest
 
 # Latest git won't let you do much in a non-safe directory
 git config --global --add safe.directory '*' || true

.github/workflows/selfhosted-maintenance.sh

@@ -31,7 +31,7 @@ if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then
   set -x
 fi
 
-rm -rf ~/.ddev/Test* ~/.ddev/global_config.yaml ~/.ddev/homeadditions ~/.ddev/commands
+rm -rf ~/.ddev/Test* ~/.ddev/global_config.yaml ~/.ddev/project_list.yaml ~/.ddev/homeadditions ~/.ddev/commands
 
 # Run any testbot maintenance that may need to be done
 echo "running selfhosted-upgrades.sh"

cmd/ddev/cmd/config-global_test.go

@@ -27,6 +27,12 @@ func TestCmdGlobalConfig(t *testing.T) {
 		err := os.Remove(configFile)
 		require.NoError(t, err)
 	}
+	// And no projects file
+	projectsFile := globalconfig.GetProjectListPath()
+	if fileutil.FileExists(projectsFile) {
+		err := os.Remove(projectsFile)
+		require.NoError(t, err)
+	}
 	// We need to make sure that the (corrupted, bogus) global config file is removed
 	// and then read (empty)
 	// nolint: errcheck
@@ -43,6 +49,10 @@ func TestCmdGlobalConfig(t *testing.T) {
 		if err != nil {
 			t.Logf("Unable to remove %v: %v", configFile, err)
 		}
+		err = os.Remove(projectsFile)
+		if err != nil {
+			t.Logf("Unable to remove %v: %v", configFile, err)
+		}
 		err = globalconfig.ReadGlobalConfig()
 		if err != nil {
 			t.Logf("Unable to ReadGlobalConfig: %v", err)

cmd/ddev/cmd/list_test.go

@@ -33,7 +33,7 @@ func TestCmdList(t *testing.T) {
 		globalconfig.DdevGlobalConfig.SimpleFormatting = false
 		_ = globalconfig.WriteGlobalConfig(globalconfig.DdevGlobalConfig)
 	})
-	// This gratuitous ddev start -a repopulates the ~/.ddev/global_config.yaml
+	// This gratuitous ddev start -a repopulates the ~/.ddev/project_list.yaml
 	// project list, which has been damaged by other tests which use
 	// direct app techniques.
 	_, err = exec.RunHostCommand(DdevBin, "start", "-a", "-y")

pkg/ddevapp/ddevapp.go

@@ -2606,7 +2606,7 @@ func deleteServiceVolumes(app *DdevApp) {
 	}
 }
 
-// RemoveGlobalProjectInfo deletes the project from ProjectList
+// RemoveGlobalProjectInfo deletes the project from DdevProjectList
 func (app *DdevApp) RemoveGlobalProjectInfo() {
 	_ = globalconfig.RemoveProjectInfo(app.Name)
 }

pkg/ddevapp/ddevapp_test.go

@@ -3104,7 +3104,7 @@ func TestCleanupWithoutCompose(t *testing.T) {
 	// by ensuring any associated database files get cleaned up as well.
 	err = app.Stop(true, false)
 	assert.NoError(err)
-	assert.Empty(globalconfig.DdevGlobalConfig.ProjectList[app.Name])
+	assert.Empty(globalconfig.DdevProjectList[app.Name])
 	for _, containerType := range []string{"web", "db"} {
 		_, err := constructContainerName(containerType, app)
 		assert.Error(err)
@@ -3925,7 +3925,7 @@ func TestPortSpecifications(t *testing.T) {
 	err = globalconfig.ReadGlobalConfig()
 	require.NoError(t, err)
 	// Since host ports were not explicitly set in nospecApp, they shouldn't be in globalconfig.
-	require.Empty(t, globalconfig.DdevGlobalConfig.ProjectList[nospecApp.Name].UsedHostPorts)
+	require.Empty(t, globalconfig.DdevProjectList[nospecApp.Name].UsedHostPorts)
 
 	err = nospecApp.Start()
 	assert.NoError(err)
@@ -3958,8 +3958,8 @@ func TestPortSpecifications(t *testing.T) {
 	err = specAPP.Stop(false, false)
 	require.NoError(t, err)
 	// Verify that DdevGlobalConfig got updated properly
-	require.NotEmpty(t, globalconfig.DdevGlobalConfig.ProjectList[specAPP.Name])
-	require.NotEmpty(t, globalconfig.DdevGlobalConfig.ProjectList[specAPP.Name].UsedHostPorts)
+	require.NotEmpty(t, globalconfig.DdevProjectList[specAPP.Name])
+	require.NotEmpty(t, globalconfig.DdevProjectList[specAPP.Name].UsedHostPorts)
 
 	// However, if we change change the name to make it appear to be a
 	// different project, we should not be able to config or start
@@ -3979,15 +3979,15 @@ func TestPortSpecifications(t *testing.T) {
 	// Now delete the specAPP and we should be able to use the conflictApp
 	err = specAPP.Stop(true, false)
 	assert.NoError(err)
-	assert.Empty(globalconfig.DdevGlobalConfig.ProjectList[specAPP.Name])
+	assert.Empty(globalconfig.DdevProjectList[specAPP.Name])
 
 	err = conflictApp.WriteConfig()
 	assert.NoError(err)
 	err = conflictApp.Start()
 	assert.NoError(err)
 
-	require.NotEmpty(t, globalconfig.DdevGlobalConfig.ProjectList[conflictApp.Name])
-	require.NotEmpty(t, globalconfig.DdevGlobalConfig.ProjectList[conflictApp.Name].UsedHostPorts)
+	require.NotEmpty(t, globalconfig.DdevProjectList[conflictApp.Name])
+	require.NotEmpty(t, globalconfig.DdevProjectList[conflictApp.Name].UsedHostPorts)
 }
 
 // TestDdevGetProjects exercises GetProjects()

pkg/globalconfig/global_config.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
@@ -24,9 +25,14 @@ import (
 // DdevGlobalConfigName is the name of the global config file.
 const DdevGlobalConfigName = "global_config.yaml"
 
+// DdevProjectListFileName is the name of the global projects file.
+const DdevProjectListFileName = "project_list.yaml"
+
 var (
 	// DdevGlobalConfig is the currently active global configuration struct
 	DdevGlobalConfig GlobalConfig
+	// DdevProjectList is the list of all existing DDEV projects
+	DdevProjectList map[string]*ProjectInfo
 )
 
 type ProjectInfo struct {
@@ -98,17 +104,27 @@ func New() GlobalConfig {
 // Make sure the global configuration has been initialized
 func EnsureGlobalConfig() {
 	DdevGlobalConfig = New()
+	DdevProjectList = make(map[string]*ProjectInfo)
 	err := ReadGlobalConfig()
 	if err != nil {
 		output.UserErr.Fatalf("unable to read global config: %v", err)
 	}
+	err = ReadProjectList()
+	if err != nil {
+		output.UserErr.Fatalf("unable to read global projects list: %v", err)
+	}
 }
 
 // GetGlobalConfigPath gets the path to global config file
 func GetGlobalConfigPath() string {
 	return filepath.Join(GetGlobalDdevDir(), DdevGlobalConfigName)
 }
 
+// GetProjectListPath gets the path to global projects file
+func GetProjectListPath() string {
+	return filepath.Join(GetGlobalDdevDir(), DdevProjectListFileName)
+}
+
 // GetDDEVBinDir returns the directory of the Mutagen config and binary
 func GetDDEVBinDir() string {
 	return filepath.Join(GetGlobalDdevDir(), "bin")
@@ -445,6 +461,82 @@ func WriteGlobalConfig(config GlobalConfig) error {
 	return nil
 }
 
+// ReadProjectList reads the global projects file into DdevProjectList
+// Or creates the file
+func ReadProjectList() error {
+	globalProjectsFile := GetProjectListPath()
+
+	// Can't use fileutil.FileExists() here because of import cycle.
+	if _, err := os.Stat(globalProjectsFile); err != nil {
+		// ~/.ddev doesn't exist and running as root (only ddev hostname could do this)
+		// Then create global projects list.
+		if os.Geteuid() == 0 {
+			logrus.Warning("Not reading global projects file because running with root privileges")
+			return nil
+		}
+		if os.IsNotExist(err) {
+			// write an empty file
+			err := os.WriteFile(GetProjectListPath(), make([]byte, 0), 0644)
+			if err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	}
+
+	source, err := os.ReadFile(globalProjectsFile)
+	if err != nil {
+		return fmt.Errorf("unable to read DDEV global projects file %s: %v", source, err)
+	}
+
+	// ReadConfig config values from file.
+	err = yaml.Unmarshal(source, &DdevProjectList)
+	if err != nil {
+		return err
+	}
+
+	// For backwards compatability we're keeping the project_list in global config
+	// in sync with the list in project_list.yaml. If someone upgrades from an earlier
+	// version the global config will have correct content that isn't in projects.yaml.
+	// For now, treat global config as the source of truth when the two lists differ.
+	if !reflect.DeepEqual(DdevGlobalConfig.ProjectList, DdevProjectList) {
+		DdevProjectList = DdevGlobalConfig.ProjectList
+		err := WriteProjectList(DdevProjectList)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// WriteProjectList writes the global projects list into ~/.ddev.
+func WriteProjectList(projects map[string]*ProjectInfo) error {
+	// Write to global config for backwards compatability.
+	// This allows devs to downgrade to an earlier version without
+	// worrying about copying project info into their global config file.
+	DdevGlobalConfig.ProjectList = projects
+	err := WriteGlobalConfig(DdevGlobalConfig)
+	if err != nil {
+		return err
+	}
+
+	// Prepare projects file content
+	projectsBytes, err := yaml.Marshal(projects)
+	if err != nil {
+		return err
+	}
+
+	// Write to projects file
+	err = os.WriteFile(GetProjectListPath(), projectsBytes, 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // GetGlobalDdevDir returns ~/.ddev, the global caching directory
 func GetGlobalDdevDir() string {
 	userHome, err := os.UserHomeDir()
@@ -499,7 +591,7 @@ func GetValidOmitContainers() []string {
 // HostPostIsAllocated returns the project name that has allocated
 // the port, or empty string.
 func HostPostIsAllocated(port string) string {
-	for project, item := range DdevGlobalConfig.ProjectList {
+	for project, item := range DdevProjectList {
 		if nodeps.ArrayContainsString(item.UsedHostPorts, port) {
 			return project
 		}
@@ -558,38 +650,38 @@ func GetFreePort(localIPAddr string) (string, error) {
 // ReservePorts adds the ProjectInfo if necessary and assigns the reserved ports
 func ReservePorts(projectName string, ports []string) error {
 	// If the project doesn't exist, add it.
-	_, ok := DdevGlobalConfig.ProjectList[projectName]
+	_, ok := DdevProjectList[projectName]
 	if !ok {
-		DdevGlobalConfig.ProjectList[projectName] = &ProjectInfo{}
+		DdevProjectList[projectName] = &ProjectInfo{}
 	}
-	DdevGlobalConfig.ProjectList[projectName].UsedHostPorts = ports
-	err := WriteGlobalConfig(DdevGlobalConfig)
+	DdevProjectList[projectName].UsedHostPorts = ports
+	err := WriteProjectList(DdevProjectList)
 	return err
 }
 
 // SetProjectAppRoot sets the approot in the ProjectInfo of global config
 func SetProjectAppRoot(projectName string, appRoot string) error {
 	// If the project doesn't exist, add it.
-	_, ok := DdevGlobalConfig.ProjectList[projectName]
+	_, ok := DdevProjectList[projectName]
 	if !ok {
-		DdevGlobalConfig.ProjectList[projectName] = &ProjectInfo{}
+		DdevProjectList[projectName] = &ProjectInfo{}
 	}
 	// Can't use fileutil.FileExists because of import cycle.
 	if _, err := os.Stat(appRoot); err != nil {
 		return fmt.Errorf("project %s project root %s does not exist", projectName, appRoot)
 	}
-	if DdevGlobalConfig.ProjectList[projectName].AppRoot != "" && DdevGlobalConfig.ProjectList[projectName].AppRoot != appRoot {
-		return fmt.Errorf("project %s project root is already set to %s, refusing to change it to %s; you can `ddev stop --unlist %s` and start again if the listed project root is in error", projectName, DdevGlobalConfig.ProjectList[projectName].AppRoot, appRoot, projectName)
+	if DdevProjectList[projectName].AppRoot != "" && DdevProjectList[projectName].AppRoot != appRoot {
+		return fmt.Errorf("project %s project root is already set to %s, refusing to change it to %s; you can `ddev stop --unlist %s` and start again if the listed project root is in error", projectName, DdevProjectList[projectName].AppRoot, appRoot, projectName)
 	}
-	DdevGlobalConfig.ProjectList[projectName].AppRoot = appRoot
-	err := WriteGlobalConfig(DdevGlobalConfig)
+	DdevProjectList[projectName].AppRoot = appRoot
+	err := WriteProjectList(DdevProjectList)
 	return err
 }
 
 // GetProject returns a project given name provided,
 // or nil if not found.
 func GetProject(projectName string) *ProjectInfo {
-	project, ok := DdevGlobalConfig.ProjectList[projectName]
+	project, ok := DdevProjectList[projectName]
 	if !ok {
 		return nil
 	}
@@ -598,10 +690,10 @@ func GetProject(projectName string) *ProjectInfo {
 
 // RemoveProjectInfo removes the ProjectInfo line for a project
 func RemoveProjectInfo(projectName string) error {
-	_, ok := DdevGlobalConfig.ProjectList[projectName]
+	_, ok := DdevProjectList[projectName]
 	if ok {
-		delete(DdevGlobalConfig.ProjectList, projectName)
-		err := WriteGlobalConfig(DdevGlobalConfig)
+		delete(DdevProjectList, projectName)
+		err := WriteProjectList(DdevProjectList)
 		if err != nil {
 			return err
 		}
@@ -611,7 +703,7 @@ func RemoveProjectInfo(projectName string) error {
 
 // GetGlobalProjectList returns the global project list map
 func GetGlobalProjectList() map[string]*ProjectInfo {
-	return DdevGlobalConfig.ProjectList
+	return DdevProjectList
 }
 
 // GetCAROOT is a wrapper on global config

pkg/globalconfig/global_config_test.go

@@ -3,6 +3,12 @@ package globalconfig_test
 import (
 	"context"
 	"errors"
+	"net"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+
 	"github.com/ddev/ddev/pkg/dockerutil"
 	"github.com/ddev/ddev/pkg/exec"
 	"github.com/ddev/ddev/pkg/globalconfig"
@@ -11,11 +17,6 @@ import (
 	"github.com/ddev/ddev/pkg/versionconstants"
 	asrt "github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"net"
-	"os"
-	"strconv"
-	"testing"
-	"time"
 )
 
 func init() {
@@ -55,7 +56,7 @@ func TestGetFreePort(t *testing.T) {
 	for try := 0; try < 5; try++ {
 		port, err := globalconfig.GetFreePort(dockerIP)
 		require.NoError(t, err)
-		assert.NotContains(globalconfig.DdevGlobalConfig.ProjectList["TestGetFreePort"].UsedHostPorts, port)
+		assert.NotContains(globalconfig.DdevProjectList["TestGetFreePort"].UsedHostPorts, port)
 
 		// Make sure we can actually use the port.
 		dockerCommand := []string{"run", "--rm", "-p" + dockerIP + ":" + port + ":" + port, versionconstants.BusyboxImage}

pkg/testcommon/testcommon.go

@@ -144,6 +144,7 @@ func (site *TestSite) Prepare() error {
 
 	// Force creation of new global config if none exists.
 	_ = globalconfig.ReadGlobalConfig()
+	_ = globalconfig.ReadProjectList()
 
 	err = app.WriteConfig()
 	if err != nil {