Skip to content

Commit

Permalink
Move image pull args into ImagePullTypes
Browse files Browse the repository at this point in the history
Signed-off-by: David Son <davbson@amazon.com>
  • Loading branch information
sondavidb committed Mar 29, 2024
1 parent c74ca03 commit 77e11ab
Show file tree
Hide file tree
Showing 7 changed files with 77 additions and 65 deletions.
25 changes: 18 additions & 7 deletions cmd/nerdctl/image_pull.go
Expand Up @@ -20,6 +20,8 @@ import (
"github.com/containerd/nerdctl/v2/pkg/api/types"
"github.com/containerd/nerdctl/v2/pkg/clientutil"
"github.com/containerd/nerdctl/v2/pkg/cmd/image"
"github.com/containerd/nerdctl/v2/pkg/platformutil"
"github.com/containerd/nerdctl/v2/pkg/strutil"
"github.com/spf13/cobra"
)

Expand Down Expand Up @@ -81,10 +83,19 @@ func processPullCommandFlags(cmd *cobra.Command) (types.ImagePullOptions, error)
return types.ImagePullOptions{}, err
}

ociSpecPlatform, err := platformutil.NewOCISpecPlatformSlice(allPlatforms, platform)
if err != nil {
return types.ImagePullOptions{}, err
}

unpackStr, err := cmd.Flags().GetString("unpack")
if err != nil {
return types.ImagePullOptions{}, err
}
unpack, err := strutil.ParseBoolOrAuto(unpackStr)
if err != nil {
return types.ImagePullOptions{}, err
}

quiet, err := cmd.Flags().GetBool("quiet")
if err != nil {
Expand All @@ -105,13 +116,13 @@ func processPullCommandFlags(cmd *cobra.Command) (types.ImagePullOptions, error)
return types.ImagePullOptions{}, err
}
return types.ImagePullOptions{
GOptions: globalOptions,
VerifyOptions: verifyOptions,
AllPlatforms: allPlatforms,
Platform: platform,
Unpack: unpackStr,
Quiet: quiet,
IPFSAddress: ipfsAddressStr,
GOptions: globalOptions,
VerifyOptions: verifyOptions,
OCISpecPlatform: ociSpecPlatform,
Unpack: unpack,
Mode: "always",
Quiet: quiet,
IPFSAddress: ipfsAddressStr,
RFlags: types.RemoteSnapshotterFlags{
SociIndexDigest: sociIndexDigest,
},
Expand Down
15 changes: 9 additions & 6 deletions pkg/api/types/image_types.go
Expand Up @@ -18,6 +18,8 @@ package types

import (
"io"

v1 "github.com/opencontainers/image-spec/specs-go/v1"
)

// ImageListOptions specifies options for `nerdctl image list`.
Expand Down Expand Up @@ -191,12 +193,13 @@ type ImagePullOptions struct {
Stderr io.Writer
GOptions GlobalCommandOptions
VerifyOptions ImageVerifyOptions
// Unpack the image for the current single platform (auto/true/false)
Unpack string
// Pull content for a specific platform
Platform []string
// Pull content for all platforms
AllPlatforms bool
// Unpack the image for the current single platform.
// If nil, it will unpack automatically if only 1 platform is specified.
Unpack *bool
// Content for specific platforms
OCISpecPlatform []v1.Platform
// Pull mode
Mode string
// Suppress verbose output
Quiet bool
// multiaddr of IPFS API (default uses $IPFS_PATH env variable if defined or local directory ~/.ipfs)
Expand Down
17 changes: 13 additions & 4 deletions pkg/cmd/compose/compose.go
Expand Up @@ -110,6 +110,17 @@ func New(client *containerd.Client, globalOptions types.GlobalCommandOptions, op
ocispecPlatforms = []ocispec.Platform{parsed} // no append
}

imgPullOpts := types.ImagePullOptions{
GOptions: globalOptions,
OcispecPlatforms: ocispecPlatforms,

Check failure on line 115 in pkg/cmd/compose/compose.go

View workflow job for this annotation

GitHub Actions / cross (1.21.x)

unknown field OcispecPlatforms in struct literal of type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions

Check failure on line 115 in pkg/cmd/compose/compose.go

View workflow job for this annotation

GitHub Actions / cross (1.22.x)

unknown field OcispecPlatforms in struct literal of type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions

Check failure on line 115 in pkg/cmd/compose/compose.go

View workflow job for this annotation

GitHub Actions / test-integration-docker-compatibility

unknown field OcispecPlatforms in struct literal of type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions

Check failure on line 115 in pkg/cmd/compose/compose.go

View workflow job for this annotation

GitHub Actions / test-integration-windows

unknown field OcispecPlatforms in struct literal of type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions

Check failure on line 115 in pkg/cmd/compose/compose.go

View workflow job for this annotation

GitHub Actions / test-unit

unknown field OcispecPlatforms in struct literal of type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions
Unpack: nil,
Mode: pullMode,
Quiet: quiet,
RFlags: types.RemoteSnapshotterFlags{},
Stdout: stdout,
Stderr: stderr,
}

// IPFS reference
if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(imageName); err == nil {
var ipfsPath string
Expand All @@ -124,8 +135,7 @@ func New(client *containerd.Client, globalOptions types.GlobalCommandOptions, op
}
ipfsPath = dir
}
_, err = ipfs.EnsureImage(ctx, client, stdout, stderr, globalOptions.Snapshotter, scheme, ref,
pullMode, ocispecPlatforms, nil, quiet, ipfsPath, types.RemoteSnapshotterFlags{})
_, err = ipfs.EnsureImage(ctx, client, scheme, ref, ipfsPath, imgPullOpts)
return err
}

Expand All @@ -135,8 +145,7 @@ func New(client *containerd.Client, globalOptions types.GlobalCommandOptions, op
return err
}

_, err = imgutil.EnsureImage(ctx, client, stdout, stderr, globalOptions.Snapshotter, ref,
pullMode, globalOptions.InsecureRegistry, globalOptions.HostsDir, ocispecPlatforms, nil, quiet, types.RemoteSnapshotterFlags{})
_, err = imgutil.EnsureImage(ctx, client, ref, imgPullOpts)
return err
}

Expand Down
7 changes: 6 additions & 1 deletion pkg/cmd/container/create.go
Expand Up @@ -117,7 +117,12 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa
}
rawRef := args[0]

ensuredImage, err = image.EnsureImage(ctx, client, rawRef, ocispecPlatforms, options.Pull, nil, false, options.ImagePullOpt)
options.ImagePullOpt.Mode = options.Pull
options.ImagePullOpt.OcispecPlatforms = ocispecPlatforms

Check failure on line 121 in pkg/cmd/container/create.go

View workflow job for this annotation

GitHub Actions / cross (1.21.x)

options.ImagePullOpt.OcispecPlatforms undefined (type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions has no field or method OcispecPlatforms)

Check failure on line 121 in pkg/cmd/container/create.go

View workflow job for this annotation

GitHub Actions / cross (1.22.x)

options.ImagePullOpt.OcispecPlatforms undefined (type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions has no field or method OcispecPlatforms)

Check failure on line 121 in pkg/cmd/container/create.go

View workflow job for this annotation

GitHub Actions / test-integration-docker-compatibility

options.ImagePullOpt.OcispecPlatforms undefined (type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions has no field or method OcispecPlatforms)

Check failure on line 121 in pkg/cmd/container/create.go

View workflow job for this annotation

GitHub Actions / test-integration-windows

options.ImagePullOpt.OcispecPlatforms undefined (type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions has no field or method OcispecPlatforms)

Check failure on line 121 in pkg/cmd/container/create.go

View workflow job for this annotation

GitHub Actions / test-unit

options.ImagePullOpt.OcispecPlatforms undefined (type "github.com/containerd/nerdctl/v2/pkg/api/types".ImagePullOptions has no field or method OcispecPlatforms)
options.ImagePullOpt.Unpack = nil
options.ImagePullOpt.Quiet = false

ensuredImage, err = image.EnsureImage(ctx, client, rawRef, options.ImagePullOpt)
if err != nil {
return nil, nil, err
}
Expand Down
23 changes: 4 additions & 19 deletions pkg/cmd/image/pull.go
Expand Up @@ -26,26 +26,13 @@ import (
"github.com/containerd/nerdctl/v2/pkg/api/types"
"github.com/containerd/nerdctl/v2/pkg/imgutil"
"github.com/containerd/nerdctl/v2/pkg/ipfs"
"github.com/containerd/nerdctl/v2/pkg/platformutil"
"github.com/containerd/nerdctl/v2/pkg/referenceutil"
"github.com/containerd/nerdctl/v2/pkg/signutil"
"github.com/containerd/nerdctl/v2/pkg/strutil"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
)

// Pull pulls an image specified by `rawRef`.
func Pull(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePullOptions) error {
ocispecPlatforms, err := platformutil.NewOCISpecPlatformSlice(options.AllPlatforms, options.Platform)
if err != nil {
return err
}

unpack, err := strutil.ParseBoolOrAuto(options.Unpack)
if err != nil {
return err
}

_, err = EnsureImage(ctx, client, rawRef, ocispecPlatforms, "always", unpack, options.Quiet, options)
_, err := EnsureImage(ctx, client, rawRef, options)
if err != nil {
return err
}
Expand All @@ -54,7 +41,7 @@ func Pull(ctx context.Context, client *containerd.Client, rawRef string, options
}

// EnsureImage pulls an image either from ipfs or from registry.
func EnsureImage(ctx context.Context, client *containerd.Client, rawRef string, ocispecPlatforms []v1.Platform, pull string, unpack *bool, quiet bool, options types.ImagePullOptions) (*imgutil.EnsuredImage, error) {
func EnsureImage(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePullOptions) (*imgutil.EnsuredImage, error) {
var ensured *imgutil.EnsuredImage

if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(rawRef); err == nil {
Expand All @@ -75,8 +62,7 @@ func EnsureImage(ctx context.Context, client *containerd.Client, rawRef string,
ipfsPath = dir
}

ensured, err = ipfs.EnsureImage(ctx, client, options.Stdout, options.Stderr, options.GOptions.Snapshotter, scheme, ref,
pull, ocispecPlatforms, unpack, quiet, ipfsPath, options.RFlags)
ensured, err = ipfs.EnsureImage(ctx, client, scheme, ref, ipfsPath, options)
if err != nil {
return nil, err
}
Expand All @@ -88,8 +74,7 @@ func EnsureImage(ctx context.Context, client *containerd.Client, rawRef string,
return nil, err
}

ensured, err = imgutil.EnsureImage(ctx, client, options.Stdout, options.Stderr, options.GOptions.Snapshotter, ref,
pull, options.GOptions.InsecureRegistry, options.GOptions.HostsDir, ocispecPlatforms, unpack, quiet, options.RFlags)
ensured, err = imgutil.EnsureImage(ctx, client, ref, options)
if err != nil {
return nil, err
}
Expand Down
46 changes: 22 additions & 24 deletions pkg/imgutil/imgutil.go
Expand Up @@ -20,7 +20,6 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"reflect"

"github.com/containerd/containerd"
Expand Down Expand Up @@ -102,9 +101,8 @@ func GetExistingImage(ctx context.Context, client *containerd.Client, snapshotte
// EnsureImage ensures the image.
//
// # When insecure is set, skips verifying certs, and also falls back to HTTP when the registry does not speak HTTPS
//
// FIXME: this func has too many args
func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr io.Writer, snapshotter, rawRef string, mode PullMode, insecure bool, hostsDirs []string, ocispecPlatforms []ocispec.Platform, unpack *bool, quiet bool, rFlags types.RemoteSnapshotterFlags) (*EnsuredImage, error) {
func EnsureImage(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePullOptions) (*EnsuredImage, error) {
var mode PullMode = options.Mode
switch mode {
case "always", "missing", "never":
// NOP
Expand All @@ -113,8 +111,8 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr
}

// if not `always` pull and given one platform and image found locally, return existing image directly.
if mode != "always" && len(ocispecPlatforms) == 1 {
if res, err := GetExistingImage(ctx, client, snapshotter, rawRef, ocispecPlatforms[0]); err == nil {
if mode != "always" && len(options.OCISpecPlatform) == 1 {
if res, err := GetExistingImage(ctx, client, options.GOptions.Snapshotter, rawRef, options.OCISpecPlatform[0]); err == nil {
return res, nil
} else if !errdefs.IsNotFound(err) {
return nil, err
Expand All @@ -133,30 +131,30 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr
refDomain := refdocker.Domain(named)

var dOpts []dockerconfigresolver.Opt
if insecure {
if options.GOptions.InsecureRegistry {
log.G(ctx).Warnf("skipping verifying HTTPS certs for %q", refDomain)
dOpts = append(dOpts, dockerconfigresolver.WithSkipVerifyCerts(true))
}
dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(hostsDirs))
dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(options.GOptions.HostsDir))
resolver, err := dockerconfigresolver.New(ctx, refDomain, dOpts...)
if err != nil {
return nil, err
}

img, err := PullImage(ctx, client, stdout, stderr, snapshotter, resolver, ref, ocispecPlatforms, unpack, quiet, rFlags)
img, err := PullImage(ctx, client, resolver, ref, options)
if err != nil {
// In some circumstance (e.g. people just use 80 port to support pure http), the error will contain message like "dial tcp <port>: connection refused".
if !errutil.IsErrHTTPResponseToHTTPSClient(err) && !errutil.IsErrConnectionRefused(err) {
return nil, err
}
if insecure {
if options.GOptions.InsecureRegistry {
log.G(ctx).WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain)
dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true))
resolver, err = dockerconfigresolver.New(ctx, refDomain, dOpts...)
if err != nil {
return nil, err
}
return PullImage(ctx, client, stdout, stderr, snapshotter, resolver, ref, ocispecPlatforms, unpack, quiet, rFlags)
return PullImage(ctx, client, resolver, ref, options)
}
log.G(ctx).WithError(err).Errorf("server %q does not seem to support HTTPS", refDomain)
log.G(ctx).Info("Hint: you may want to try --insecure-registry to allow plain HTTP (if you are in a trusted network)")
Expand Down Expand Up @@ -195,7 +193,7 @@ func ResolveDigest(ctx context.Context, rawRef string, insecure bool, hostsDirs
}

// PullImage pulls an image using the specified resolver.
func PullImage(ctx context.Context, client *containerd.Client, stdout, stderr io.Writer, snapshotter string, resolver remotes.Resolver, ref string, ocispecPlatforms []ocispec.Platform, unpack *bool, quiet bool, rFlags types.RemoteSnapshotterFlags) (*EnsuredImage, error) {
func PullImage(ctx context.Context, client *containerd.Client, resolver remotes.Resolver, ref string, options types.ImagePullOptions) (*EnsuredImage, error) {
ctx, done, err := client.WithLease(ctx)
if err != nil {
return nil, err
Expand All @@ -206,34 +204,34 @@ func PullImage(ctx context.Context, client *containerd.Client, stdout, stderr io
config := &pull.Config{
Resolver: resolver,
RemoteOpts: []containerd.RemoteOpt{},
Platforms: ocispecPlatforms, // empty for all-platforms
Platforms: options.OCISpecPlatform, // empty for all-platforms
}
if !quiet {
config.ProgressOutput = stderr
if !options.Quiet {
config.ProgressOutput = options.Stderr
}

// unpack(B) if given 1 platform unless specified by `unpack`
unpackB := len(ocispecPlatforms) == 1
if unpack != nil {
unpackB = *unpack
if unpackB && len(ocispecPlatforms) != 1 {
unpackB := len(options.OCISpecPlatform) == 1
if options.Unpack != nil {
unpackB = *options.Unpack
if unpackB && len(options.OCISpecPlatform) != 1 {
return nil, fmt.Errorf("unpacking requires a single platform to be specified (e.g., --platform=amd64)")
}
}

snOpt := getSnapshotterOpts(snapshotter)
snOpt := getSnapshotterOpts(options.GOptions.Snapshotter)
if unpackB {
log.G(ctx).Debugf("The image will be unpacked for platform %q, snapshotter %q.", ocispecPlatforms[0], snapshotter)
log.G(ctx).Debugf("The image will be unpacked for platform %q, snapshotter %q.", options.OCISpecPlatform[0], options.GOptions.Snapshotter)
imgcryptPayload := imgcrypt.Payload{}
imgcryptUnpackOpt := encryption.WithUnpackConfigApplyOpts(encryption.WithDecryptedUnpack(&imgcryptPayload))
config.RemoteOpts = append(config.RemoteOpts,
containerd.WithPullUnpack,
containerd.WithUnpackOpts([]containerd.UnpackOpt{imgcryptUnpackOpt}))

// different remote snapshotters will update pull.Config separately
snOpt.apply(config, ref, rFlags)
snOpt.apply(config, ref, options.RFlags)
} else {
log.G(ctx).Debugf("The image will not be unpacked. Platforms=%v.", ocispecPlatforms)
log.G(ctx).Debugf("The image will not be unpacked. Platforms=%v.", options.OCISpecPlatform)
}

containerdImage, err = pull.Pull(ctx, client, ref, config)
Expand All @@ -248,7 +246,7 @@ func PullImage(ctx context.Context, client *containerd.Client, stdout, stderr io
Ref: ref,
Image: containerdImage,
ImageConfig: *imgConfig,
Snapshotter: snapshotter,
Snapshotter: options.GOptions.Snapshotter,
Remote: snOpt.isRemote(),
}
return res, nil
Expand Down
9 changes: 5 additions & 4 deletions pkg/ipfs/image.go
Expand Up @@ -41,7 +41,8 @@ import (
const ipfsPathEnv = "IPFS_PATH"

// EnsureImage pull the specified image from IPFS.
func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr io.Writer, snapshotter string, scheme string, ref string, mode imgutil.PullMode, ocispecPlatforms []ocispec.Platform, unpack *bool, quiet bool, ipfsPath string, rFlags types.RemoteSnapshotterFlags) (*imgutil.EnsuredImage, error) {
func EnsureImage(ctx context.Context, client *containerd.Client, scheme, ref, ipfsPath string, options types.ImagePullOptions) (*imgutil.EnsuredImage, error) {
var mode imgutil.PullMode = options.Mode
switch mode {
case "always", "missing", "never":
// NOP
Expand All @@ -56,8 +57,8 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr
}

// if not `always` pull and given one platform and image found locally, return existing image directly.
if mode != "always" && len(ocispecPlatforms) == 1 {
if res, err := imgutil.GetExistingImage(ctx, client, snapshotter, ref, ocispecPlatforms[0]); err == nil {
if mode != "always" && len(options.OCISpecPlatform) == 1 {
if res, err := imgutil.GetExistingImage(ctx, client, options.GOptions.Snapshotter, ref, options.OCISpecPlatform[0]); err == nil {
return res, nil
} else if !errdefs.IsNotFound(err) {
return nil, err
Expand All @@ -74,7 +75,7 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout, stderr
if err != nil {
return nil, err
}
return imgutil.PullImage(ctx, client, stdout, stderr, snapshotter, r, ref, ocispecPlatforms, unpack, quiet, rFlags)
return imgutil.PullImage(ctx, client, r, ref, options)
}

// Push pushes the specified image to IPFS.
Expand Down

0 comments on commit 77e11ab

Please sign in to comment.