fix(build): cleanup orphan build containers on ctrl-c or gitlab cancel

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>

cmd/werf/common/termination_signals.go

@@ -4,6 +4,8 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+
+	"github.com/werf/werf/pkg/container_backend"
 )
 
 var (
@@ -21,6 +23,8 @@ func EnableTerminationSignalsTrap() {
 	go func() {
 		select {
 		case <-terminationSignalsChan:
+			container_backend.TerminateRunningDockerContainers()
+
 			TerminateWithError("interrupted", 17)
 		case <-disableTerminationSignalsTrapChan:
 			return

pkg/container_backend/docker_running_containers.go

@@ -0,0 +1,58 @@
+package container_backend
+
+import (
+	"context"
+	"sync"
+
+	"github.com/docker/docker/api/types"
+
+	"github.com/werf/logboek"
+	"github.com/werf/werf/pkg/docker"
+)
+
+var (
+	runningDockerContainers []*runningDockerContainer
+	mu                      sync.Mutex
+)
+
+type runningDockerContainer struct {
+	Name string
+	Ctx  context.Context
+}
+
+func RegisterRunningContainer(name string, ctx context.Context) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	runningDockerContainers = append(runningDockerContainers, &runningDockerContainer{
+		Name: name,
+		Ctx:  ctx,
+	})
+}
+
+func UnregisterRunningContainer(name string) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	var res []*runningDockerContainer
+	for _, cont := range runningDockerContainers {
+		if cont.Name != name {
+			res = append(res, cont)
+		}
+	}
+	runningDockerContainers = res
+}
+
+func TerminateRunningDockerContainers() error {
+	mu.Lock()
+	defer mu.Unlock()
+
+	for _, container := range runningDockerContainers {
+		err := docker.ContainerRemove(container.Ctx, container.Name, types.ContainerRemoveOptions{RemoveVolumes: true, Force: true})
+		if err != nil {
+			logboek.Context(container.Ctx).Error().LogF("WARNING: Unable to remove container %q: %w\n", container.Name, err.Error())
+		}
+	}
+
+	return nil
+}

pkg/container_backend/legacy_stage_image.go

@@ -91,7 +91,7 @@ func (i *LegacyStageImage) Build(ctx context.Context, options BuildOptions) erro
 			}
 
 			if err := i.container.rm(ctx); err != nil {
-				return fmt.Errorf("introspect error failed: %w", err)
+				return fmt.Errorf("unable to remove container (original error %s): %w", containerRunErr, err)
 			}
 		}
 

pkg/container_backend/legacy_stage_image_container.go

@@ -267,10 +267,12 @@ func (c *LegacyStageImageContainer) run(ctx context.Context) error {
 		return err
 	}
 
-	if err := docker.CliRun_LiveOutput(ctx, runArgs...); err != nil {
+	RegisterRunningContainer(c.name, ctx)
+	err = docker.CliRun_LiveOutput(ctx, runArgs...)
+	UnregisterRunningContainer(c.name)
+	if err != nil {
 		return fmt.Errorf("container run failed: %w", err)
 	}
-
 	return nil
 }
 
@@ -339,5 +341,12 @@ func (c *LegacyStageImageContainer) commit(ctx context.Context) (string, error)
 func (c *LegacyStageImageContainer) rm(ctx context.Context) error {
 	_ = c.image.ContainerBackend.(*DockerServerBackend)
 
-	return docker.ContainerRemove(ctx, c.name, types.ContainerRemoveOptions{})
+	err := docker.ContainerRemove(ctx, c.name, types.ContainerRemoveOptions{RemoveVolumes: true, Force: true})
+	if err != nil {
+		if strings.Contains(err.Error(), fmt.Sprintf("removal of container %s is already in progress", c.name)) {
+			return nil
+		}
+		return fmt.Errorf("unable to remove container %s: %w", c.name, err)
+	}
+	return nil
 }

pkg/process_exterminator/unix.go

@@ -13,6 +13,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/werf/werf/pkg/container_backend"
 	"github.com/werf/werf/pkg/werf"
 )
 
@@ -45,6 +46,7 @@ func run(parentsPids []int) {
 				fmt.Fprintf(os.Stderr, "Process exterminator error: %s\n", err)
 			}
 
+			container_backend.TerminateRunningDockerContainers()
 			syscall.Kill(ownPid, syscall.SIGINT)
 
 			return