diff --git a/cmd/werf/common/termination_signals.go b/cmd/werf/common/termination_signals.go index ca9b9abbd6..371a504f63 100644 --- a/cmd/werf/common/termination_signals.go +++ b/cmd/werf/common/termination_signals.go @@ -4,6 +4,8 @@ import ( "os" "os/signal" "syscall" + + "github.com/werf/werf/pkg/container_backend" ) var ( @@ -21,6 +23,8 @@ func EnableTerminationSignalsTrap() { go func() { select { case <-terminationSignalsChan: + container_backend.TerminateRunningDockerContainers() + TerminateWithError("interrupted", 17) case <-disableTerminationSignalsTrapChan: return diff --git a/pkg/container_backend/docker_running_containers.go b/pkg/container_backend/docker_running_containers.go new file mode 100644 index 0000000000..21ae1fc55a --- /dev/null +++ b/pkg/container_backend/docker_running_containers.go @@ -0,0 +1,58 @@ +package container_backend + +import ( + "context" + "sync" + + "github.com/docker/docker/api/types" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/docker" +) + +var ( + runningDockerContainers []*runningDockerContainer + mu sync.Mutex +) + +type runningDockerContainer struct { + Name string + Ctx context.Context +} + +func RegisterRunningContainer(name string, ctx context.Context) { + mu.Lock() + defer mu.Unlock() + + runningDockerContainers = append(runningDockerContainers, &runningDockerContainer{ + Name: name, + Ctx: ctx, + }) +} + +func UnregisterRunningContainer(name string) { + mu.Lock() + defer mu.Unlock() + + var res []*runningDockerContainer + for _, cont := range runningDockerContainers { + if cont.Name != name { + res = append(res, cont) + } + } + runningDockerContainers = res +} + +func TerminateRunningDockerContainers() error { + mu.Lock() + defer mu.Unlock() + + for _, container := range runningDockerContainers { + err := docker.ContainerRemove(container.Ctx, container.Name, types.ContainerRemoveOptions{RemoveVolumes: true, Force: true}) + if err != nil { + logboek.Context(container.Ctx).Error().LogF("WARNING: Unable to remove container %q: %w\n", container.Name, err.Error()) + } + } + + return nil +} diff --git a/pkg/container_backend/legacy_stage_image.go b/pkg/container_backend/legacy_stage_image.go index ddfd6db53c..db744e3211 100644 --- a/pkg/container_backend/legacy_stage_image.go +++ b/pkg/container_backend/legacy_stage_image.go @@ -91,7 +91,7 @@ func (i *LegacyStageImage) Build(ctx context.Context, options BuildOptions) erro } if err := i.container.rm(ctx); err != nil { - return fmt.Errorf("introspect error failed: %w", err) + return fmt.Errorf("unable to remove container (original error %s): %w", containerRunErr, err) } } diff --git a/pkg/container_backend/legacy_stage_image_container.go b/pkg/container_backend/legacy_stage_image_container.go index 454ce2ad4d..bf3b47c1fd 100644 --- a/pkg/container_backend/legacy_stage_image_container.go +++ b/pkg/container_backend/legacy_stage_image_container.go @@ -267,10 +267,12 @@ func (c *LegacyStageImageContainer) run(ctx context.Context) error { return err } - if err := docker.CliRun_LiveOutput(ctx, runArgs...); err != nil { + RegisterRunningContainer(c.name, ctx) + err = docker.CliRun_LiveOutput(ctx, runArgs...) + UnregisterRunningContainer(c.name) + if err != nil { return fmt.Errorf("container run failed: %w", err) } - return nil } @@ -339,5 +341,12 @@ func (c *LegacyStageImageContainer) commit(ctx context.Context) (string, error) func (c *LegacyStageImageContainer) rm(ctx context.Context) error { _ = c.image.ContainerBackend.(*DockerServerBackend) - return docker.ContainerRemove(ctx, c.name, types.ContainerRemoveOptions{}) + err := docker.ContainerRemove(ctx, c.name, types.ContainerRemoveOptions{RemoveVolumes: true, Force: true}) + if err != nil { + if strings.Contains(err.Error(), fmt.Sprintf("removal of container %s is already in progress", c.name)) { + return nil + } + return fmt.Errorf("unable to remove container %s: %w", c.name, err) + } + return nil } diff --git a/pkg/process_exterminator/unix.go b/pkg/process_exterminator/unix.go index e57e4e0447..b68134737c 100644 --- a/pkg/process_exterminator/unix.go +++ b/pkg/process_exterminator/unix.go @@ -13,6 +13,7 @@ import ( "syscall" "time" + "github.com/werf/werf/pkg/container_backend" "github.com/werf/werf/pkg/werf" ) @@ -45,6 +46,7 @@ func run(parentsPids []int) { fmt.Fprintf(os.Stderr, "Process exterminator error: %s\n", err) } + container_backend.TerminateRunningDockerContainers() syscall.Kill(ownPid, syscall.SIGINT) return