From 5406416dbc74a05fa70994b0cd5cb9f1ba3a2268 Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Fri, 31 Mar 2023 15:34:26 +0300 Subject: [PATCH] feat(multiarch): implement creation of manifest list for each werf image in multiplatform mode builds Also updated go-containerregistry to v0.14.0 and all related dependencies (including docker/docker and oras-go). Signed-off-by: Timofey Kirillov --- go.mod | 24 +- go.sum | 31 +++ pkg/build/build_phase.go | 287 +++++++++++++------- pkg/build/image/image.go | 21 +- pkg/build/image/image_tree.go | 43 +++ pkg/container_backend/utils.go | 11 +- pkg/docker/main.go | 12 +- pkg/docker_registry/api.go | 108 +++++++- pkg/docker_registry/default.go | 8 +- pkg/docker_registry/gcr.go | 4 +- pkg/docker_registry/interface.go | 6 + pkg/host_cleaning/containers.go | 1 - pkg/logging/main.go | 27 +- pkg/storage/docker_server_stages_storage.go | 26 +- pkg/storage/repo_stages_storage.go | 23 ++ pkg/storage/stages_storage.go | 1 + pkg/util/map.go | 9 +- pkg/util/pair.go | 20 ++ pkg/util/slice.go | 8 + 19 files changed, 521 insertions(+), 149 deletions(-) create mode 100644 pkg/util/pair.go create mode 100644 pkg/util/slice.go diff --git a/go.mod b/go.mod index 4104260b4b..564aed9595 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/alessio/shellescape v1.4.1 github.com/aws/aws-sdk-go v1.44.234 github.com/bmatcuk/doublestar v1.3.4 - github.com/containerd/containerd v1.6.19 + github.com/containerd/containerd v1.7.0 github.com/containers/buildah v1.29.1 github.com/containers/common v0.51.2 github.com/containers/image/v5 v5.24.1 @@ -19,9 +19,9 @@ require ( github.com/deislabs/oras v0.12.0 github.com/djherbis/buffer v1.2.0 github.com/djherbis/nio/v3 v3.0.1 - github.com/docker/cli v20.10.23+incompatible + github.com/docker/cli v23.0.1+incompatible github.com/docker/distribution v2.8.1+incompatible - github.com/docker/docker v20.10.23+incompatible + github.com/docker/docker v23.0.1+incompatible github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 github.com/docker/go-units v0.5.0 github.com/dustin/go-humanize v1.0.1 @@ -31,7 +31,7 @@ require ( github.com/go-openapi/spec v0.20.8 github.com/go-openapi/strfmt v0.21.7 github.com/go-openapi/validate v0.22.1 - github.com/google/go-containerregistry v0.13.0 + github.com/google/go-containerregistry v0.14.0 github.com/google/uuid v1.3.0 github.com/gookit/color v1.5.3 github.com/gophercloud/gophercloud v1.3.0 @@ -83,9 +83,9 @@ require ( k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.90.1 k8s.io/kubectl v0.26.3 - k8s.io/utils v0.0.0-20230209194617-a36077c30491 + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 mvdan.cc/xurls v1.1.0 - oras.land/oras-go v1.2.2 + oras.land/oras-go v1.2.3 sigs.k8s.io/yaml v1.3.0 ) @@ -114,7 +114,7 @@ require ( github.com/chzyer/readline v1.5.1 // indirect github.com/cloudflare/cfssl v1.4.1 // indirect github.com/cloudflare/circl v1.1.0 // indirect - github.com/container-orchestrated-devices/container-device-interface v0.5.3 // indirect + github.com/container-orchestrated-devices/container-device-interface v0.5.4 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/console v1.0.3 // indirect github.com/containerd/continuity v0.3.0 // indirect @@ -141,7 +141,7 @@ require ( github.com/fatih/camelcase v1.0.0 // indirect github.com/fatih/color v1.14.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/fsouza/go-dockerclient v1.9.3 // indirect + github.com/fsouza/go-dockerclient v1.9.6 // indirect github.com/fvbommel/sortorder v1.0.1 // indirect github.com/go-errors/errors v1.0.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect @@ -215,9 +215,11 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect + github.com/moby/patternmatcher v0.5.0 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/sys/mount v0.3.3 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/signal v0.7.0 // indirect github.com/moby/sys/symlink v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -234,7 +236,7 @@ require ( github.com/opencontainers/runc v1.1.5 // indirect github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect github.com/opencontainers/selinux v1.11.0 // indirect - github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df // indirect + github.com/openshift/imagebuilder v1.2.5-0.20230315213933-1693aaac1009 // indirect github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -289,7 +291,7 @@ require ( go.opentelemetry.io/otel/metric v0.37.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/oauth2 v0.5.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/term v0.6.0 // indirect @@ -299,7 +301,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.29.0 // indirect gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect diff --git a/go.sum b/go.sum index 81f967d756..e79ce3d914 100644 --- a/go.sum +++ b/go.sum @@ -381,6 +381,8 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E= github.com/container-orchestrated-devices/container-device-interface v0.5.3 h1:4v6FMaa1Pn8SS0IBwgsvCsno8HRXoQvI87Uj1Zu7Tw4= github.com/container-orchestrated-devices/container-device-interface v0.5.3/go.mod h1:SQohok453ewi9dItvUcO0MrP7K1CEQTxPDNd7OV+nxI= +github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= +github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -428,6 +430,8 @@ github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/containerd v1.6.19 h1:F0qgQPrG0P2JPgwpxWxYavrVeXAG0ezUIB9Z/4FTUAU= github.com/containerd/containerd v1.6.19/go.mod h1:HZCDMn4v/Xl2579/MvtOC2M206i+JJ6VxFWU/NetrGY= +github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXTLTM2Pg= +github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -447,6 +451,7 @@ github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZ github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU= github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= github.com/containerd/fuse-overlayfs-snapshotter v1.0.2/go.mod h1:nRZceC8a7dRm3Ao6cJAwuJWPFiBPaibHiFntRUnzhwU= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= @@ -475,6 +480,7 @@ github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8h github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI= github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/ttrpc v1.2.1 h1:VWv/Rzx023TBLv4WQ+9WPXlBG/s3rsRjY3i9AJ2BJdE= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= @@ -588,6 +594,8 @@ github.com/docker/cli v20.10.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHv github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.23+incompatible h1:qwyha/T3rXk9lfuVcn533cKFc7n/6IzL5GXVAgMVPBg= github.com/docker/cli v20.10.23+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM= +github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.6.0-rc.1.0.20180327202408-83389a148052+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -607,6 +615,8 @@ github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05b github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.23+incompatible h1:1ZQUUYAdh+oylOT85aA2ZcfRp22jmLhoaEcVEfK8dyA= github.com/docker/docker v20.10.23+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY= +github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= @@ -705,6 +715,8 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/fsouza/go-dockerclient v1.7.7/go.mod h1:njNCXvoZj3sLPjf3yO0DPHf1mdLdCPDYPc14GskKA4Y= github.com/fsouza/go-dockerclient v1.9.3 h1:nos0eFmulRVoq9QevZbwJzFS4cFHMoeUTDNC04klEVM= github.com/fsouza/go-dockerclient v1.9.3/go.mod h1:soNpY8X1z9RW5UxuXU+gA94/ESSbiAoWwsiqYa6ofHQ= +github.com/fsouza/go-dockerclient v1.9.6 h1:6yTc8oTLHjz4H3AXAAdO4R+Nh5B7s9Uf5R4oQUtZ9Mg= +github.com/fsouza/go-dockerclient v1.9.6/go.mod h1:Z0GysXoBlysCsdUO2tMK/13R2tvpehDns08Mih0XvKg= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/fullstorydev/grpcurl v1.8.6 h1:WylAwnPauJIofYSHqqMTC1eEfUIzqzevXyogBxnQquo= github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= @@ -1030,6 +1042,8 @@ github.com/google/go-containerregistry v0.0.0-20191010200024-a3d713f9b7f8/go.mod github.com/google/go-containerregistry v0.1.2/go.mod h1:GPivBPgdAyd2SU+vf6EpsgOtWDuPqjW0hJZt4rNdTZ4= github.com/google/go-containerregistry v0.13.0 h1:y1C7Z3e149OJbOPDBxLYR8ITPz8dTKqQwjErKVHJC8k= github.com/google/go-containerregistry v0.13.0/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo= +github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw= +github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= github.com/google/go-intervals v0.0.2 h1:FGrVEiUnTRKR8yE04qzXYaJMtnIYqobR5QbblK3ixcM= @@ -1505,6 +1519,8 @@ github.com/moby/buildkit v0.9.3 h1:0JmMLY45KIKFogJXv4LyWo+KmIMuvhit5TDrwBlxDp0= github.com/moby/buildkit v0.9.3/go.mod h1:5dZQUHg9STw/Fhl4zZiusDJKn8uje/0x952Nce4a8cg= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= +github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mount v0.1.0/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74= @@ -1519,6 +1535,8 @@ github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2J github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= @@ -1670,6 +1688,8 @@ github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaL github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df h1:vf6pdI10F2Tim5a9JKiVVl4/dpNz1OEhz4EnfLdLtiA= github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df/go.mod h1:TRYHe4CH9U6nkDjxjBNM5klrLbJBrRbpJE5SaRwUBsQ= +github.com/openshift/imagebuilder v1.2.5-0.20230315213933-1693aaac1009 h1:xKpnQ1BOGhCtGbmsH16WE07Wrwm5PlyD/Gh0kw6M+bU= +github.com/openshift/imagebuilder v1.2.5-0.20230315213933-1693aaac1009/go.mod h1:9eM1yF1PSe8JaYw4IxcMdTHARvpAQ7q3OBXJM+382Ms= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -2020,6 +2040,7 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.7 h1:aXiFAgRugfJ27UFDsGJ9DB2FvTC73hlVXFSqq5bo9eU= +github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= @@ -2187,6 +2208,7 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCB go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.0-RC1/go.mod h1:cDwRc2Jrh5Gku1peGK8p9rRuX/Uq2OtVmLicjlw2WYU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.0.0-RC1/go.mod h1:OYKzEoxgXFvehW7X12WYT4/a2BlASJK9l7RtG4A91fg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 h1:3jAYbRHQAqzLjd9I4tzxwJ8Pk/N6AqBcF6m1ZHrxG94= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4= @@ -2411,6 +2433,8 @@ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2849,6 +2873,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0= +google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= @@ -2929,6 +2955,7 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -3032,6 +3059,8 @@ k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -3045,6 +3074,8 @@ mvdan.cc/xurls v1.1.0 h1:kj0j2lonKseISJCiq1Tfk+iTv65dDGCl0rTbanXJGGc= mvdan.cc/xurls v1.1.0/go.mod h1:TNWuhvo+IqbUCmtUIb/3LJSQdrzel8loVpgFm0HikbI= oras.land/oras-go v1.2.2 h1:0E9tOHUfrNH7TCDk5KU0jVBEzCqbfdyuVfGmJ7ZeRPE= oras.land/oras-go v1.2.2/go.mod h1:Apa81sKoZPpP7CDciE006tSZ0x3Q3+dOoBcMZ/aNxvw= +oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY= +oras.land/oras-go v1.2.3/go.mod h1:M/uaPdYklze0Vf3AakfarnpoEckvw0ESbRdN8Z1vdJg= pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/letsencrypt v0.0.3 h1:H7xDfhkaFFSYEJlKeq38RwX2jYcnTeHuDQyT+mMNMwM= diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 37f644a7c2..7d8a193d66 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -26,6 +26,7 @@ import ( "github.com/werf/werf/pkg/docker_registry" "github.com/werf/werf/pkg/git_repo" imagePkg "github.com/werf/werf/pkg/image" + "github.com/werf/werf/pkg/logging" "github.com/werf/werf/pkg/stapel" "github.com/werf/werf/pkg/storage" "github.com/werf/werf/pkg/storage/manager" @@ -196,22 +197,175 @@ func (phase *BuildPhase) AfterImages(ctx context.Context) error { targetPlatforms = []string{phase.Conveyor.ContainerBackend.GetDefaultPlatform()} } - for imageName, imagePlatforms := range phase.Conveyor.imagesTree.GetImagePlatformsByName(true) { + for _, desc := range phase.Conveyor.imagesTree.GetImagesByName(true) { + name, images := desc.Unpair() + platforms := util.MapFuncToSlice(images, func(img *image.Image) string { return img.TargetPlatform }) + AssertAllTargetPlatformsPresent: for _, targetPlatform := range targetPlatforms { - for _, imagePlatform := range imagePlatforms { - if targetPlatform == imagePlatform { - logboek.Context(ctx).Debug().LogF("Found image %q built for target platform %q\n", imageName, targetPlatform) + for _, platform := range platforms { + if targetPlatform == platform { + logboek.Context(ctx).Debug().LogF("Found image %q built for target platform %q\n", name, targetPlatform) continue AssertAllTargetPlatformsPresent } } - panic(fmt.Sprintf("there is no image %q built for target platform %q, please report a bug", imageName, targetPlatform)) + panic(fmt.Sprintf("There is no image %q built for target platform %q. Please report a bug.", name, targetPlatform)) + } + + if len(targetPlatforms) != len(platforms) { + panic(fmt.Sprintf("We have built image %q for platforms %v, expected exactly these platforms: %v. Please report a bug.", name, platforms, targetPlatforms)) + } + + if len(targetPlatforms) == 1 { + if err := phase.publishImageMetadata(ctx, name, images[0]); err != nil { + return fmt.Errorf("unable to publish image %q metadata: %w", name, err) + } + } else { + if err := logboek.Context(ctx).LogProcess(logging.ImageLogProcessName(name, false, "")). + Options(func(options types.LogProcessOptionsInterface) { + options.Style(logging.ImageMetadataStyle()) + }). + DoError(func() error { + if err := phase.publishMultiplatformImageMetadata(ctx, name, images); err != nil { + return fmt.Errorf("unable to publish image %q multiplatform metadata: %w", name, err) + } + return nil + }); err != nil { + return err + } } } return phase.createReport(ctx) } +func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string, img *image.Image) error { + if err := phase.addManagedImage(ctx, name); err != nil { + return err + } + + if !phase.BuildPhaseOptions.SkipImageMetadataPublication { + if err := logboek.Context(ctx).Info(). + LogProcess(fmt.Sprintf("Publish image %s git metadata", img.GetName())). + DoError(func() error { return phase.publishImageGitMetadata(ctx, img) }); err != nil { + return err + } + } + + if img.IsArtifact { + return nil + } + if img.IsDockerfileImage && !img.IsDockerfileTargetStage { + return nil + } + + if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { + if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}); err != nil { + return err + } + } + + var customTagStorage storage.StagesStorage + var customTagStage *imagePkg.StageDescription + if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { + customTagStorage = phase.Conveyor.StorageManager.GetFinalStagesStorage() + customTagStage = manager.ConvertStageDescriptionForStagesStorage(img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription(), phase.Conveyor.StorageManager.GetFinalStagesStorage()) + } else { + customTagStorage = phase.Conveyor.StorageManager.GetStagesStorage() + customTagStage = img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription() + } + + if phase.ShouldBeBuiltMode { + if err := phase.checkCustomImageTagsExistence(ctx, img.GetName(), customTagStage, customTagStorage); err != nil { + return err + } + } else { + if err := phase.addCustomImageTags(ctx, img.GetName(), customTagStage, customTagStorage, phase.Conveyor.StorageManager.GetStagesStorage(), phase.CustomTagFuncList); err != nil { + return fmt.Errorf("unable to add custom image tags to stages storage: %w", err) + } + } + + return nil +} + +func calculateMuiltiplatformStageDigest(images []*image.Image) string { + metaStageDeps := util.MapFuncToSlice(images, func(img *image.Image) string { + stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription() + return stageDesc.StageID.String() + }) + return util.Sha3_224Hash(metaStageDeps...) +} + +func getImagesInfoList(images []*image.Image) []*imagePkg.Info { + return util.MapFuncToSlice(images, func(img *image.Image) *imagePkg.Info { + stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription() + return stageDesc.Info + }) +} + +func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context, name string, images []*image.Image) error { + if err := phase.addManagedImage(ctx, name); err != nil { + return err + } + + digest := calculateMuiltiplatformStageDigest(images) + allPlatformsImages := getImagesInfoList(images) + _, uniqueID := phase.Conveyor.StorageManager.GenerateStageUniqueID(digest, nil) + metaStageID := imagePkg.StageID{Digest: digest, UniqueID: uniqueID} + + stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage() + + // FIXME(multiarch): Copy manifest list into final stages storage. + // FIXME(multiarch): Git metadata for artifacts and stages per platform or + + // if !phase.BuildPhaseOptions.SkipImageMetadataPublication { + // if err := logboek.Context(ctx).Info(). + // LogProcess(fmt.Sprintf("Publish image %s git metadata", img.GetName())). + // DoError(func() error { return phase.publishImageGitMetadata(ctx, img) }); err != nil { + // return err + // } + // } + + // if img.IsArtifact { + // return nil + // } + // if img.IsDockerfileImage && !img.IsDockerfileTargetStage { + // return nil + // } + + // if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { + // if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}); err != nil { + // return err + // } + // } + + if len(phase.CustomTagFuncList) == 0 { + platforms := util.MapFuncToSlice(images, func(img *image.Image) string { return img.TargetPlatform }) + + container_backend.LogImageName(ctx, fmt.Sprintf("%s:%s", stagesStorage.Address(), metaStageID)) + container_backend.LogMultiplatformImageInfo(ctx, platforms) + + if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), metaStageID.String(), allPlatformsImages); err != nil { + return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, metaStageID, err) + } + } else { + for _, tagFunc := range phase.CustomTagFuncList { + tag := tagFunc(name, metaStageID.String()) + + if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), tag, allPlatformsImages); err != nil { + return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, metaStageID, err) + } + + // FIXME(multiarch): custom tag registration for cleanup, shall we do it? + // if err := stagesStorage.RegisterStageCustomTag(ctx, stageDesc, tag); err != nil { + // return fmt.Errorf("unable to register stage %s custom tag %s in the primary storage %s: %w", stageDesc.StageID.String(), tag, primaryStagesStorage.String(), err) + // } + } + } + + return nil +} + func (phase *BuildPhase) createReport(ctx context.Context) error { targetPlatforms, err := phase.Conveyor.GetTargetPlatforms() if err != nil { @@ -315,57 +469,13 @@ func (phase *BuildPhase) BeforeImageStages(ctx context.Context, img *image.Image func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *image.Image) error { img.SetLastNonEmptyStage(phase.StagesIterator.PrevNonEmptyStage) img.SetContentDigest(phase.StagesIterator.PrevNonEmptyStage.GetContentDigest()) - - if err := phase.addManagedImage(ctx, img); err != nil { - return err - } - - if !phase.BuildPhaseOptions.SkipImageMetadataPublication { - if err := phase.publishImageMetadata(ctx, img); err != nil { - return err - } - } - - if img.IsArtifact { - return nil - } - if img.IsDockerfileImage && !img.IsDockerfileTargetStage { - return nil - } - - if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { - if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}); err != nil { - return err - } - } - - var customTagStorage storage.StagesStorage - var customTagStage *imagePkg.StageDescription - if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { - customTagStorage = phase.Conveyor.StorageManager.GetFinalStagesStorage() - customTagStage = manager.ConvertStageDescriptionForStagesStorage(img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription(), phase.Conveyor.StorageManager.GetFinalStagesStorage()) - } else { - customTagStorage = phase.Conveyor.StorageManager.GetStagesStorage() - customTagStage = img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription() - } - - if phase.ShouldBeBuiltMode { - if err := phase.checkCustomImageTagsExistence(ctx, img.GetName(), customTagStage, customTagStorage); err != nil { - return err - } - } else { - if err := phase.addCustomImageTags(ctx, img.GetName(), customTagStage, customTagStorage, phase.Conveyor.StorageManager.GetStagesStorage(), phase.CustomTagFuncList); err != nil { - return fmt.Errorf("unable to add custom image tags to stages storage: %w", err) - } - } - return nil } -func (phase *BuildPhase) addManagedImage(ctx context.Context, img *image.Image) error { +func (phase *BuildPhase) addManagedImage(ctx context.Context, name string) error { if phase.ShouldAddManagedImageRecord { stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage() - exist, err := stagesStorage.IsManagedImageExist(ctx, phase.Conveyor.ProjectName(), img.GetName(), storage.WithCache()) + exist, err := stagesStorage.IsManagedImageExist(ctx, phase.Conveyor.ProjectName(), name, storage.WithCache()) if err != nil { return fmt.Errorf("unable to check existence of managed image: %w", err) } @@ -374,61 +484,52 @@ func (phase *BuildPhase) addManagedImage(ctx context.Context, img *image.Image) return nil } - if err := stagesStorage.AddManagedImage(ctx, phase.Conveyor.ProjectName(), img.GetName()); err != nil { - return fmt.Errorf("unable to add image %q to the managed images of project %q: %w", img.GetName(), phase.Conveyor.ProjectName(), err) + if err := stagesStorage.AddManagedImage(ctx, phase.Conveyor.ProjectName(), name); err != nil { + return fmt.Errorf("unable to add image %q to the managed images of project %q: %w", name, phase.Conveyor.ProjectName(), err) } } return nil } -func (phase *BuildPhase) publishImageMetadata(ctx context.Context, img *image.Image) error { - targetPlatforms, err := phase.Conveyor.GetTargetPlatforms() - if err != nil { - return fmt.Errorf("unable to get target platforms: %w", err) - } - if len(targetPlatforms) == 0 { - targetPlatforms = []string{phase.Conveyor.ContainerBackend.GetDefaultPlatform()} - } - targetPlatform := targetPlatforms[0] +func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, img *image.Image) error { + var commits []string - // FIXME(multiarch): publish image metadata for multiarch manifest instead of per-platform-images - if targetPlatform != img.TargetPlatform { - return nil + headCommit := phase.Conveyor.giterminismManager.HeadCommit() + commits = append(commits, headCommit) + + if phase.Conveyor.GetLocalGitRepoVirtualMergeOptions().VirtualMerge { + fromCommit, _, err := git_repo.GetVirtualMergeParents(ctx, phase.Conveyor.giterminismManager.LocalGitRepo(), headCommit) + if err != nil { + return fmt.Errorf("unable to get virtual merge commit %q parents: %w", headCommit, err) + } + + commits = append(commits, fromCommit) } - return logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Processing image %s git metadata", img.GetName())). - DoError(func() error { - var commits []string + stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage() - headCommit := phase.Conveyor.giterminismManager.HeadCommit() - commits = append(commits, headCommit) + info := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info - if phase.Conveyor.GetLocalGitRepoVirtualMergeOptions().VirtualMerge { - fromCommit, _, err := git_repo.GetVirtualMergeParents(ctx, phase.Conveyor.giterminismManager.LocalGitRepo(), headCommit) - if err != nil { - return fmt.Errorf("unable to get virtual merge commit %q parents: %w", headCommit, err) - } + logboek.Context(ctx).Info().LogF("name: %s:%s\n", info.Repository, info.Tag) + logboek.Context(ctx).Info().LogF("commits:\n") - commits = append(commits, fromCommit) - } + for _, commit := range commits { + logboek.Context(ctx).Info().LogF(" %s\n", commit) - for _, commit := range commits { - stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage() - exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID(), storage.WithCache()) - if err != nil { - return fmt.Errorf("unable to get image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err) - } + exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID(), storage.WithCache()) + if err != nil { + return fmt.Errorf("unable to get image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err) + } - if !exist { - if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID()); err != nil { - return fmt.Errorf("unable to put image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err) - } - } + if !exist { + if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID()); err != nil { + return fmt.Errorf("unable to put image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err) } + } + } - return nil - }) + return nil } func (phase *BuildPhase) addCustomImageTags(ctx context.Context, imageName string, stageDesc *imagePkg.StageDescription, stagesStorage storage.StagesStorage, primaryStagesStorage storage.PrimaryStagesStorage, customTagFuncList []imagePkg.CustomTagFunc) error { @@ -529,7 +630,7 @@ func (phase *BuildPhase) onImageStage(ctx context.Context, img *image.Image, stg if foundSuitableStage { logboek.Context(ctx).Default().LogFHighlight("Use cache image for %s\n", stg.LogDetailedName()) - container_backend.LogImageInfo(ctx, stg.GetStageImage().Image, phase.getPrevNonEmptyStageImageSize()) + container_backend.LogImageInfo(ctx, stg.GetStageImage().Image, phase.getPrevNonEmptyStageImageSize(), img.ShouldLogPlatform()) logboek.Context(ctx).LogOptionalLn() @@ -610,7 +711,7 @@ func (phase *BuildPhase) findAndFetchStageFromSecondaryStagesStorage(ctx context stg.SetStageImage(i) logboek.Context(ctx).Default().LogFHighlight("Use cache image for %s\n", stg.LogDetailedName()) - container_backend.LogImageInfo(ctx, stg.GetStageImage().Image, phase.getPrevNonEmptyStageImageSize()) + container_backend.LogImageInfo(ctx, stg.GetStageImage().Image, phase.getPrevNonEmptyStageImageSize(), img.ShouldLogPlatform()) return nil } @@ -816,7 +917,7 @@ func (phase *BuildPhase) buildStage(ctx context.Context, img *image.Image, stg s if err != nil { return } - container_backend.LogImageInfo(ctx, stg.GetStageImage().Image, phase.getPrevNonEmptyStageImageSize()) + container_backend.LogImageInfo(ctx, stg.GetStageImage().Image, phase.getPrevNonEmptyStageImageSize(), img.ShouldLogPlatform()) } if err := logboek.Context(ctx).Default().LogProcess("Building stage %s", stg.LogDetailedName()). diff --git a/pkg/build/image/image.go b/pkg/build/image/image.go index c045b63187..a9a2971fc5 100644 --- a/pkg/build/image/image.go +++ b/pkg/build/image/image.go @@ -117,9 +117,13 @@ func (i *Image) LogName() string { return logging.ImageLogName(i.Name, i.IsArtifact) } +func (i *Image) ShouldLogPlatform() bool { + return i.ForceTargetPlatformLogging || i.TargetPlatform != i.ContainerBackend.GetRuntimePlatform() +} + func (i *Image) LogDetailedName() string { var targetPlatformForLog string - if i.ForceTargetPlatformLogging || i.TargetPlatform != i.ContainerBackend.GetRuntimePlatform() { + if i.ShouldLogPlatform() { targetPlatformForLog = i.TargetPlatform } return logging.ImageLogProcessName(i.Name, i.IsArtifact, targetPlatformForLog) @@ -134,22 +138,11 @@ func (i *Image) LogTagStyle() color.Style { } func ImageLogProcessStyle(isArtifact bool) color.Style { - return imageDefaultStyle(isArtifact) + return logging.ImageDefaultStyle(isArtifact) } func ImageLogTagStyle(isArtifact bool) color.Style { - return imageDefaultStyle(isArtifact) -} - -func imageDefaultStyle(isArtifact bool) color.Style { - var colors []color.Color - if isArtifact { - colors = []color.Color{color.FgCyan, color.Bold} - } else { - colors = []color.Color{color.FgYellow, color.Bold} - } - - return color.New(colors...) + return logging.ImageDefaultStyle(isArtifact) } func (i *Image) SetStages(stages []stage.Interface) { diff --git a/pkg/build/image/image_tree.go b/pkg/build/image/image_tree.go index bec55cb6a6..e2487f0ac8 100644 --- a/pkg/build/image/image_tree.go +++ b/pkg/build/image/image_tree.go @@ -6,6 +6,7 @@ import ( "path" "path/filepath" "reflect" + "sort" "github.com/gookit/color" @@ -16,6 +17,7 @@ import ( "github.com/werf/werf/pkg/git_repo" "github.com/werf/werf/pkg/giterminism_manager" "github.com/werf/werf/pkg/logging" + "github.com/werf/werf/pkg/util" ) type ImagesTree struct { @@ -120,6 +122,47 @@ func (tree *ImagesTree) GetImage(name string) *Image { return nil } +func (tree *ImagesTree) GetImagesByName(finalOnly bool) []*util.Pair[string, []*Image] { + images := make(map[string]map[string]*Image) + var names []string + + appendImage := func(img *Image) { + names = util.UniqAppendString(names, img.Name) + if images[img.Name] == nil { + images[img.Name] = make(map[string]*Image) + } + images[img.Name][img.TargetPlatform] = img + } + + for _, img := range tree.GetImages() { + if finalOnly { + for _, finalImageName := range tree.werfConfig.GetAllImages() { + if finalImageName.GetName() == img.Name { + appendImage(img) + } + } + } else { + appendImage(img) + } + } + + var res []*util.Pair[string, []*Image] + + sort.Strings(names) + for _, name := range names { + platforms := util.MapKeys(images[name]) + sort.Strings(platforms) + + allPlatformsImages := util.NewPair(name, make([]*Image, 0, len(platforms))) + for _, platform := range platforms { + allPlatformsImages.Second = append(allPlatformsImages.Second, images[name][platform]) + } + res = append(res, allPlatformsImages) + } + + return res +} + func (tree *ImagesTree) GetImagePlatformsByName(finalOnly bool) map[string][]string { res := make(map[string][]string) for _, img := range tree.GetImages() { diff --git a/pkg/container_backend/utils.go b/pkg/container_backend/utils.go index 7a95395ec2..1d8fd71ec8 100644 --- a/pkg/container_backend/utils.go +++ b/pkg/container_backend/utils.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strings" "github.com/docker/docker/pkg/stringid" @@ -23,7 +24,7 @@ func LogImageName(ctx context.Context, name string) { logboek.Context(ctx).Default().LogFDetails(logImageInfoFormat, "name", name) } -func LogImageInfo(ctx context.Context, img LegacyImageInterface, prevStageImageSize int64) { +func LogImageInfo(ctx context.Context, img LegacyImageInterface, prevStageImageSize int64, withPlatform bool) { LogImageName(ctx, img.Name()) logboek.Context(ctx).Default().LogFDetails(logImageInfoFormat, "id", stringid.TruncateID(img.GetStageDescription().Info.ID)) @@ -34,6 +35,14 @@ func LogImageInfo(ctx context.Context, img LegacyImageInterface, prevStageImageS } else { logboek.Context(ctx).Default().LogFDetails(logImageInfoFormat, "size", fmt.Sprintf("%s (+%s)", byteCountBinary(img.GetStageDescription().Info.Size), byteCountBinary(img.GetStageDescription().Info.Size-prevStageImageSize))) } + + if withPlatform { + logboek.Context(ctx).Default().LogFDetails(logImageInfoFormat, "platform", img.GetTargetPlatform()) + } +} + +func LogMultiplatformImageInfo(ctx context.Context, platforms []string) { + logboek.Context(ctx).Default().LogFDetails(logImageInfoFormat, "platform", strings.Join(platforms, ",")) } func byteCountBinary(b int64) string { diff --git a/pkg/docker/main.go b/pkg/docker/main.go index 44112e835c..a2b6aa952a 100644 --- a/pkg/docker/main.go +++ b/pkg/docker/main.go @@ -120,11 +120,11 @@ func newDockerCli(opts []command.DockerCliOption) (command.Cli, error) { dockerCertPath = cliconfig.Dir() } - clientOpts.Common.TLS = os.Getenv("DOCKER_TLS") != "" - clientOpts.Common.TLSVerify = os.Getenv("DOCKER_TLS_VERIFY") != "" + clientOpts.TLS = os.Getenv("DOCKER_TLS") != "" + clientOpts.TLSVerify = os.Getenv("DOCKER_TLS_VERIFY") != "" - if clientOpts.Common.TLSVerify { - clientOpts.Common.TLSOptions = &tlsconfig.Options{ + if clientOpts.TLSVerify { + clientOpts.TLSOptions = &tlsconfig.Options{ CAFile: filepath.Join(dockerCertPath, flags.DefaultCaFile), CertFile: filepath.Join(dockerCertPath, flags.DefaultCertFile), KeyFile: filepath.Join(dockerCertPath, flags.DefaultKeyFile), @@ -132,9 +132,9 @@ func newDockerCli(opts []command.DockerCliOption) (command.Cli, error) { } if isDebug { - clientOpts.Common.LogLevel = "debug" + clientOpts.LogLevel = "debug" } else { - clientOpts.Common.LogLevel = "fatal" + clientOpts.LogLevel = "fatal" } if err := newCli.Initialize(clientOpts); err != nil { diff --git a/pkg/docker_registry/api.go b/pkg/docker_registry/api.go index 7da112dbee..0765aec37c 100644 --- a/pkg/docker_registry/api.go +++ b/pkg/docker_registry/api.go @@ -17,10 +17,13 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/werf/logboek" "github.com/werf/werf/pkg/docker_registry/container_registry_extensions" @@ -48,8 +51,8 @@ func (api *api) Tags(ctx context.Context, reference string, _ ...Option) ([]stri return api.tags(ctx, reference) } -func (api *api) tags(_ context.Context, reference string, extraListOptions ...remote.Option) ([]string, error) { - tags, err := api.list(reference, extraListOptions...) +func (api *api) tags(ctx context.Context, reference string, extraListOptions ...remote.Option) ([]string, error) { + tags, err := api.list(ctx, reference, extraListOptions...) if err != nil { if IsStatusNotFoundErr(err) { return []string{}, nil @@ -61,20 +64,20 @@ func (api *api) tags(_ context.Context, reference string, extraListOptions ...re return tags, nil } -func (api *api) tagImage(reference, tag string) error { +func (api *api) tagImage(ctx context.Context, reference, tag string) error { ref, err := name.ParseReference(reference, api.parseReferenceOptions()...) if err != nil { return fmt.Errorf("parsing reference %q: %w", reference, err) } - desc, err := remote.Get(ref, api.defaultRemoteOptions()...) + desc, err := remote.Get(ref, api.defaultRemoteOptions(ctx)...) if err != nil { return fmt.Errorf("getting reference %q: %w", reference, err) } dst := ref.Context().Tag(tag) - return remote.Tag(dst, desc, api.defaultRemoteOptions()...) + return remote.Tag(dst, desc, api.defaultRemoteOptions(ctx)...) } func (api *api) IsRepoImageExists(ctx context.Context, reference string) (bool, error) { @@ -171,14 +174,14 @@ func (api *api) GetRepoImage(_ context.Context, reference string) (*image.Info, return repoImage, nil } -func (api *api) list(reference string, extraListOptions ...remote.Option) ([]string, error) { +func (api *api) list(ctx context.Context, reference string, extraListOptions ...remote.Option) ([]string, error) { repo, err := name.NewRepository(reference, api.newRepositoryOptions()...) if err != nil { return nil, fmt.Errorf("parsing repo %q: %w", reference, err) } listOptions := append( - api.defaultRemoteOptions(), + api.defaultRemoteOptions(ctx), extraListOptions..., ) tags, err := remote.List(repo, listOptions...) @@ -189,13 +192,13 @@ func (api *api) list(reference string, extraListOptions ...remote.Option) ([]str return tags, nil } -func (api *api) deleteImageByReference(reference string) error { +func (api *api) deleteImageByReference(ctx context.Context, reference string) error { r, err := name.ParseReference(reference, api.parseReferenceOptions()...) if err != nil { return fmt.Errorf("parsing reference %q: %w", reference, err) } - if err := remote.Delete(r, api.defaultRemoteOptions()...); err != nil { + if err := remote.Delete(r, api.defaultRemoteOptions(ctx)...); err != nil { return fmt.Errorf("deleting image %q: %w", r, err) } @@ -297,8 +300,12 @@ func (api *api) parseReferenceOptions() []name.Option { return options } -func (api *api) defaultRemoteOptions() []remote.Option { - return []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithTransport(api.getHttpTransport())} +func (api *api) defaultRemoteOptions(ctx context.Context) []remote.Option { + return []remote.Option{ + remote.WithContext(ctx), + remote.WithAuthFromKeychain(authn.DefaultKeychain), + remote.WithTransport(api.getHttpTransport()), + } } func (api *api) getHttpTransport() (transport http.RoundTripper) { @@ -420,7 +427,13 @@ func (api *api) writeToRemote(ctx context.Context, ref name.Reference, img v1.Im c := make(chan v1.Update, 200) - go remote.Write(ref, img, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithProgress(c)) + go remote.Write( + ref, img, + remote.WithAuthFromKeychain(authn.DefaultKeychain), + remote.WithProgress(c), + remote.WithTransport(api.getHttpTransport()), + remote.WithContext(ctx), + ) for upd := range c { switch { @@ -443,7 +456,7 @@ func (api *api) PullImageArchive(ctx context.Context, archiveWriter io.Writer, r return fmt.Errorf("unable to parse reference %q: %w", reference, err) } - desc, err := remote.Get(ref, api.defaultRemoteOptions()...) + desc, err := remote.Get(ref, api.defaultRemoteOptions(ctx)...) if err != nil { return fmt.Errorf("getting reference %q: %w", reference, err) } @@ -472,6 +485,75 @@ func (api *api) PullImageArchive(ctx context.Context, archiveWriter io.Writer, r return nil } +func (api *api) PushManifestList(ctx context.Context, reference string, opts ManifestListOptions) error { + if len(opts.Manifests) == 0 { + panic("unexpected empty manifests list") + } + + manifestListRef, err := name.ParseReference(reference, api.parseReferenceOptions()...) + if err != nil { + return fmt.Errorf("unable to parse reference %q: %w", reference, err) + } + + // FIXME(multiarch): Check manifest list already exists and do not republish in this case. + // FIXME(multiarch): If custom tag, then we should check by digest => we need to get each manifest from registry. + // FIXME(multiarch): If checksum tag, then we could simply check existing. + // FIXME(multiarch): ^^ This behaviour should be controlled by an option. + + var ii v1.ImageIndex + ii = empty.Index + ii = mutate.IndexMediaType(ii, types.DockerManifestList) + + adds := make([]mutate.IndexAddendum, 0, len(opts.Manifests)) + + for _, info := range opts.Manifests { + ref, err := name.ParseReference(info.Name, api.parseReferenceOptions()...) + if err != nil { + return fmt.Errorf("unable to parse reference %q: %w", info.Name, err) + } + // FIXME(multiarch): Optimize: do not get manifest, save v1.Image into the image.Info (optional) + desc, err := remote.Get(ref, api.defaultRemoteOptions(ctx)...) + if err != nil { + return fmt.Errorf("unable to get manifest of %q: %w", info.Name, err) + } + img, err := desc.Image() + if err != nil { + return fmt.Errorf("unable to get image descriptor of %q: %w", info.Name, err) + } + cf, err := img.ConfigFile() + if err != nil { + return fmt.Errorf("unable to get config file of %q: %w", info.Name, err) + } + newDesc, err := partial.Descriptor(img) + if err != nil { + return fmt.Errorf("unable to create image descriptor of %q: %w", info.Name, err) + } + newDesc.Platform = cf.Platform() + + adds = append(adds, mutate.IndexAddendum{ + Add: img, + Descriptor: *newDesc, + }) + } + + ii = mutate.AppendManifests(ii, adds...) + + // TODO(multiarch): research whether we could use digest-only manifest writing mode + // digest, err := ii.Digest() + // if err != nil { + // return fmt.Errorf("unable to calculate manifest list digest: %w", err) + // } + // if _, ok := ref.(name.Digest); ok { + // ref = ref.Context().Digest(digest.String()) + // } + + if err := remote.WriteIndex(manifestListRef, ii, api.defaultRemoteOptions(ctx)...); err != nil { + return fmt.Errorf("unable to write manifest %q: %w", manifestListRef, err) + } + + return nil +} + func ValidateRepositoryReference(reference string) error { reg := regexp.MustCompile(`^` + dockerReference.NameRegexp.String() + `$`) if !reg.MatchString(reference) { diff --git a/pkg/docker_registry/default.go b/pkg/docker_registry/default.go index 25032dcc1d..533532792f 100644 --- a/pkg/docker_registry/default.go +++ b/pkg/docker_registry/default.go @@ -73,12 +73,12 @@ func (r *defaultImplementation) DeleteRepo(_ context.Context, _ string) error { return fmt.Errorf("method is not implemented") } -func (r *defaultImplementation) TagRepoImage(_ context.Context, repoImage *image.Info, tag string) error { - return r.api.tagImage(strings.Join([]string{repoImage.Repository, repoImage.Tag}, ":"), tag) +func (r *defaultImplementation) TagRepoImage(ctx context.Context, repoImage *image.Info, tag string) error { + return r.api.tagImage(ctx, strings.Join([]string{repoImage.Repository, repoImage.Tag}, ":"), tag) } -func (r *defaultImplementation) DeleteRepoImage(_ context.Context, repoImage *image.Info) error { - return r.api.deleteImageByReference(strings.Join([]string{repoImage.Repository, repoImage.RepoDigest}, "@")) +func (r *defaultImplementation) DeleteRepoImage(ctx context.Context, repoImage *image.Info) error { + return r.api.deleteImageByReference(ctx, strings.Join([]string{repoImage.Repository, repoImage.RepoDigest}, "@")) } func (r *defaultImplementation) String() string { diff --git a/pkg/docker_registry/gcr.go b/pkg/docker_registry/gcr.go index 25fb4dba83..0d9b44773d 100644 --- a/pkg/docker_registry/gcr.go +++ b/pkg/docker_registry/gcr.go @@ -30,9 +30,9 @@ func newGcr(options GcrOptions) (*gcr, error) { return gcr, nil } -func (r *gcr) DeleteRepoImage(_ context.Context, repoImage *image.Info) error { +func (r *gcr) DeleteRepoImage(ctx context.Context, repoImage *image.Info) error { reference := strings.Join([]string{repoImage.Repository, repoImage.Tag}, ":") - return r.api.deleteImageByReference(reference) + return r.api.deleteImageByReference(ctx, reference) } func (r *gcr) String() string { diff --git a/pkg/docker_registry/interface.go b/pkg/docker_registry/interface.go index 76dd165fd9..4f98e0ec59 100644 --- a/pkg/docker_registry/interface.go +++ b/pkg/docker_registry/interface.go @@ -23,6 +23,7 @@ type Interface interface { PushImageArchive(ctx context.Context, archiveOpener ArchiveOpener, reference string) error PullImageArchive(ctx context.Context, archiveWriter io.Writer, reference string) error + PushManifestList(ctx context.Context, reference string, opts ManifestListOptions) error String() string @@ -36,3 +37,8 @@ type ApiInterface interface { type ArchiveOpener interface { Open() (io.ReadCloser, error) } + +type ManifestListOptions struct { + PushImageOptions + Manifests []*image.Info +} diff --git a/pkg/host_cleaning/containers.go b/pkg/host_cleaning/containers.go index b5a7d6648b..dd4532bbb0 100644 --- a/pkg/host_cleaning/containers.go +++ b/pkg/host_cleaning/containers.go @@ -32,7 +32,6 @@ func werfContainersByFilterSet(ctx context.Context, filterSet filters.Args) ([]t func containersByFilterSet(ctx context.Context, filterSet filters.Args) ([]types.Container, error) { containersOptions := types.ContainerListOptions{} containersOptions.All = true - containersOptions.Quiet = true containersOptions.Filters = filterSet return docker.Containers(ctx, containersOptions) diff --git a/pkg/logging/main.go b/pkg/logging/main.go index 36da258515..9b97520077 100644 --- a/pkg/logging/main.go +++ b/pkg/logging/main.go @@ -2,11 +2,14 @@ package logging import ( "fmt" + + "github.com/gookit/color" ) var ( - imageNameFormat = "⛵ image %s" - artifactNameFormat = "🛸 artifact %s" + imageNameFormat = "⛵ image %s" + artifactNameFormat = "🛸 artifact %s" + imageMetadataFormat = "⚙ image %s metadata" ) func ImageLogName(name string, isArtifact bool) string { @@ -19,6 +22,10 @@ func ImageLogName(name string, isArtifact bool) string { return name } +func ImageMetadataLogProcess(name string) string { + return fmt.Sprintf(imageMetadataFormat, name) +} + func ImageLogProcessName(name string, isArtifact bool, targetPlatform string) string { appendPlatformFunc := func(name string) string { if targetPlatform == "" { @@ -38,5 +45,21 @@ func ImageLogProcessName(name string, isArtifact bool, targetPlatform string) st func DisablePrettyLog() { imageNameFormat = "image %s" + imageMetadataFormat = "image %s metadata" artifactNameFormat = "artifact %s" } + +func ImageDefaultStyle(isArtifact bool) color.Style { + var colors []color.Color + if isArtifact { + colors = []color.Color{color.FgCyan, color.Bold} + } else { + colors = []color.Color{color.FgYellow, color.Bold} + } + + return color.New(colors...) +} + +func ImageMetadataStyle() color.Style { + return ImageDefaultStyle(false) +} diff --git a/pkg/storage/docker_server_stages_storage.go b/pkg/storage/docker_server_stages_storage.go index 4afab9d584..aec8d992f7 100644 --- a/pkg/storage/docker_server_stages_storage.go +++ b/pkg/storage/docker_server_stages_storage.go @@ -371,6 +371,31 @@ func (storage *DockerServerStagesStorage) PostClientIDRecord(ctx context.Context return nil } +func (storage *DockerServerStagesStorage) PostMultiplatformImage(ctx context.Context, projectName, tag string, allPlatformsImages []*image.Info) error { + logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.PostMultiplatformImage by tag %s for project %s\n", tag, projectName) + + fullImageName := fmt.Sprintf("%s:%s", projectName, tag) + + logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.PostMultiplatformImage full image name: %s\n", fullImageName) + + // opts := docker_registry.ManifestListOptions{ + // PushImageOptions: docker_registry.PushImageOptions{ + // Labels: map[string]string{image.WerfLabel: projectName}, + // }, + // Manifests: allPlatformsImages, + // } + + // if err := storage.DockerRegistry.PushManifestList(ctx, fullImageName, opts); err != nil { + // return fmt.Errorf("unable to push image %s: %w", fullImageName, err) + // } + + // FIXME(multiarch): use container backend to create manifest list + + // logboek.Context(ctx).Info().LogF("Created manifest list %s for project %s\n", fullImageName, projectName) + + return nil +} + type processRelatedContainersOptions struct { skipUsedImages bool rmContainersThatUseImage bool @@ -418,7 +443,6 @@ func processRelatedContainers(ctx context.Context, stageDescriptionList []*image func containerListByFilterSet(ctx context.Context, filterSet filters.Args) ([]types.Container, error) { containersOptions := types.ContainerListOptions{} containersOptions.All = true - containersOptions.Quiet = true containersOptions.Filters = filterSet return docker.Containers(ctx, containersOptions) diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index d8fdcd8ac7..526271fede 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -866,3 +866,26 @@ func (storage *RepoStagesStorage) PostClientIDRecord(ctx context.Context, projec return nil } + +func (storage *RepoStagesStorage) PostMultiplatformImage(ctx context.Context, projectName, tag string, allPlatformsImages []*image.Info) error { + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PostMultiplatformImage by tag %s for project %s\n", tag, projectName) + + fullImageName := fmt.Sprintf("%s:%s", storage.RepoAddress, tag) + + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PostMultiplatformImage full image name: %s\n", fullImageName) + + opts := docker_registry.ManifestListOptions{ + PushImageOptions: docker_registry.PushImageOptions{ + Labels: map[string]string{image.WerfLabel: projectName}, + }, + Manifests: allPlatformsImages, + } + + if err := storage.DockerRegistry.PushManifestList(ctx, fullImageName, opts); err != nil { + return fmt.Errorf("unable to push image %s: %w", fullImageName, err) + } + + logboek.Context(ctx).Info().LogF("Posted manifest list %s for project %s\n", fullImageName, projectName) + + return nil +} diff --git a/pkg/storage/stages_storage.go b/pkg/storage/stages_storage.go index 4c9b26d3b2..4ff2039cbd 100644 --- a/pkg/storage/stages_storage.go +++ b/pkg/storage/stages_storage.go @@ -72,6 +72,7 @@ type StagesStorage interface { GetClientIDRecords(ctx context.Context, projectName string, opts ...Option) ([]*ClientIDRecord, error) PostClientIDRecord(ctx context.Context, projectName string, rec *ClientIDRecord) error + PostMultiplatformImage(ctx context.Context, projectName, tag string, allPlatformsImages []*image.Info) error String() string Address() string diff --git a/pkg/util/map.go b/pkg/util/map.go index 3d362b71d8..c7b98fa4f6 100644 --- a/pkg/util/map.go +++ b/pkg/util/map.go @@ -19,7 +19,14 @@ func MergeMaps[K comparable, V any](src, dest map[K]V) map[K]V { return result } -func SortedStringKeys(m map[string]string) []string { +func MapKeys[M ~map[K]V, K comparable, V any](m M) (res []K) { + for k := range m { + res = append(res, k) + } + return +} + +func SortedStringKeys[T map[string]any](m T) []string { var keys []string for k := range m { keys = append(keys, k) diff --git a/pkg/util/pair.go b/pkg/util/pair.go new file mode 100644 index 0000000000..d44a8835df --- /dev/null +++ b/pkg/util/pair.go @@ -0,0 +1,20 @@ +package util + +import "fmt" + +type Pair[T1, T2 any] struct { + First T1 + Second T2 +} + +func NewPair[T1, T2 any](first T1, second T2) *Pair[T1, T2] { + return &Pair[T1, T2]{First: first, Second: second} +} + +func (p *Pair[T1, T2]) Unpair() (T1, T2) { + return p.First, p.Second +} + +func (p *Pair[T1, T2]) String() string { + return fmt.Sprintf("[%v %v]", p.First, p.Second) +} diff --git a/pkg/util/slice.go b/pkg/util/slice.go new file mode 100644 index 0000000000..5bbc9ffcb8 --- /dev/null +++ b/pkg/util/slice.go @@ -0,0 +1,8 @@ +package util + +func MapFuncToSlice[T, RT any, FT func(T) RT](arr []T, f FT) (res []RT) { + for _, v := range arr { + res = append(res, f(v)) + } + return +}