feat: Client copy - Issue 670

command/cp.go

@@ -107,12 +107,18 @@ Examples:
 
 	22. Upload a file to S3 with a content-type and content-encoding header
 		 > s5cmd --content-type "text/css" --content-encoding "br" myfile.css.br s3://bucket/
-		 
+
 	23. Download the specific version of a remote object to working directory
 		 > s5cmd {{.HelpName}} --version-id VERSION_ID s3://bucket/prefix/object .
 
-	24. Pass arbitrary metadata to the object during upload or copy 
-		 > s5cmd {{.HelpName}} --metadata "camera=Nixon D750" --metadata "imageSize=6032x4032" flowers.png s3://bucket/prefix/flowers.png 
+	24. Pass arbitrary metadata to the object during upload or copy
+		 > s5cmd {{.HelpName}} --metadata "camera=Nixon D750" --metadata "imageSize=6032x4032" flowers.png s3://bucket/prefix/flowers.png
+
+	25. Copy from S3 bucket to S3 bucket via the local client
+		 > s5cmd {{.HelpName}} --client-copy "s3://bucket/*.gz" s3://target-bucket/prefix/
+
+	26. Copy from S3 bucket to S3 bucket via the local client with custom endpoints and profiles
+		 > s5cmd {{.HelpName}}  --client-copy --source-region-endpoint-url URL  --source-region-profile SRCPROFILE --destination-region-endpoint-url URL  --destination-region-profile DSTPROFILE "s3://bucket/*.gz" s3://target-bucket/prefix/
 `
 
 func NewSharedFlags() []cli.Flag {
@@ -241,6 +247,37 @@ func NewCopyCommandFlags() []cli.Flag {
 			Aliases: []string{"sp"},
 			Usage:   "show a progress bar",
 		},
+		&cli.BoolFlag{
+			Name:    "client-copy",
+			Aliases: []string{"cc"},
+			Usage:   "copies from S3 bucket to S3 bucket through the local client (Advanced Use Case)",
+		},
+		&cli.StringFlag{
+			Name:  "source-region-profile",
+			Usage: "use the specified profile from the credentials file for the source region",
+		},
+		&cli.StringFlag{
+			Name:  "destination-region-profile",
+			Usage: "use the specified profile from the credentials file for the destination region",
+		},
+		&cli.StringFlag{
+			Name:    "source-region-endpoint-url",
+			Usage:   "override default S3 host for custom services for the source region",
+			EnvVars: []string{"S3_ENDPOINT_URL"},
+		},
+		&cli.BoolFlag{
+			Name:  "source-region-no-verify-ssl",
+			Usage: "disable SSL certificate verification for the source region endpoint",
+		},
+		&cli.StringFlag{
+			Name:    "destination-region-endpoint-url",
+			Usage:   "override default S3 host for custom services for the destination region",
+			EnvVars: []string{"S3_ENDPOINT_URL"},
+		},
+		&cli.BoolFlag{
+			Name:  "destination-region-no-verify-ssl",
+			Usage: "disable SSL certificate verification for the destination region endpoint",
+		},
 	}
 	sharedFlags := NewSharedFlags()
 	return append(copyFlags, sharedFlags...)
@@ -307,14 +344,21 @@ type Copy struct {
 	metadata              map[string]string
 	showProgress          bool
 	progressbar           progressbar.ProgressBar
+	clientCopy            bool
 
 	// patterns
 	excludePatterns []*regexp.Regexp
 	includePatterns []*regexp.Regexp
 
 	// region settings
-	srcRegion string
-	dstRegion string
+	srcRegion            string
+	dstRegion            string
+	srcRegionProfile     string
+	dstRegionProfile     string
+	srcRegionEndpoint    string
+	dstRegionEndpoint    string
+	srcRegionNoVerifySSL bool
+	dstRegionNoVerifySSL bool
 
 	// s3 options
 	concurrency int
@@ -384,10 +428,17 @@ func NewCopy(c *cli.Context, deleteSource bool) (*Copy, error) {
 		metadata:              metadata,
 		showProgress:          c.Bool("show-progress"),
 		progressbar:           commandProgressBar,
+		clientCopy:            c.Bool("client-copy"),
 
 		// region settings
-		srcRegion: c.String("source-region"),
-		dstRegion: c.String("destination-region"),
+		srcRegion:            c.String("source-region"),
+		dstRegion:            c.String("destination-region"),
+		srcRegionProfile:     c.String("source-region-profile"),
+		dstRegionProfile:     c.String("destination-region-profile"),
+		srcRegionEndpoint:    c.String("source-region-endpoint-url"),
+		dstRegionEndpoint:    c.String("destination-region-endpoint-url"),
+		srcRegionNoVerifySSL: c.Bool("source-region-no-verify-ssl"),
+		dstRegionNoVerifySSL: c.Bool("destination-region-no-verify-ssl"),
 
 		storageOpts: NewStorageOpts(c),
 	}, nil
@@ -406,6 +457,19 @@ func (c Copy) Run(ctx context.Context) error {
 		c.storageOpts.SetRegion(c.srcRegion)
 	}
 
+	// TODO: Consider putting an override if sourceprofile exists, then it would not be needed later
+	// Otherwise for now we would need a default profile, or we'd need credentials in the chain
+	// Override with a specific source region identity profile if set
+	if c.srcRegionProfile != "" && c.storageOpts.Profile == "" {
+		c.storageOpts.Profile = c.srcRegionProfile
+	}
+
+	// override endpoint if set for the srcRegion only
+	if c.srcRegionEndpoint != "" {
+		c.storageOpts.Endpoint = c.srcRegionEndpoint
+		c.storageOpts.NoVerifySSL = c.srcRegionNoVerifySSL
+	}
+
 	client, err := storage.NewClient(ctx, c.src, c.storageOpts)
 	if err != nil {
 		printError(c.fullCommand, c.op, err)
@@ -513,6 +577,8 @@ func (c Copy) Run(ctx context.Context) error {
 		c.progressbar.IncrementTotalObjects()
 
 		switch {
+		case (srcurl.Type == c.dst.Type) && c.clientCopy: // local->local or remote->remote and client copy
+			task = c.prepareClientCopyTask(ctx, srcurl, c.dst, isBatch, c.metadata)
 		case srcurl.Type == c.dst.Type: // local->local or remote->remote
 			task = c.prepareCopyTask(ctx, srcurl, c.dst, isBatch, c.metadata)
 		case srcurl.IsRemote(): // remote->local
@@ -553,6 +619,81 @@ func (c Copy) prepareCopyTask(
 	}
 }
 
+func (c Copy) prepareClientCopyTask(
+	ctx context.Context,
+	srcurl *url.URL,
+	dsturl *url.URL,
+	isBatch bool,
+	metadata map[string]string,
+) func() error {
+	return func() error {
+		defaultProfile := c.storageOpts.Profile
+		defaultEndpoint := c.storageOpts.Endpoint
+		defaultNoVerifySSL := c.storageOpts.NoVerifySSL
+
+		tempfilelocation := filepath.Join("tmp")
+		templocaldst, err := url.New(tempfilelocation)
+		if err != nil {
+			printError("temp destination", "temp destination", err)
+			return err
+		}
+
+		// set a temporary local file destination for the client copy
+		templocaldsturl, err := prepareLocalDestination(ctx, srcurl, templocaldst, c.flatten, isBatch, c.storageOpts)
+		if err != nil {
+			return err
+		}
+
+		// Override with a specific source region identity profile if set
+		if c.srcRegionProfile != "" {
+			c.storageOpts.Profile = c.srcRegionProfile
+		}
+
+		err = c.doDownload(ctx, srcurl, templocaldsturl)
+		if err != nil {
+			return &errorpkg.Error{
+				Op:  c.op,
+				Src: srcurl,
+				Dst: dsturl,
+				Err: err,
+			}
+		}
+
+		dsturl = prepareRemoteDestination(srcurl, dsturl, c.flatten, isBatch)
+
+		// set to delete local copy after upload to true to clean up local filesystem
+		c.deleteSource = true
+
+		// Override with a specific destination region identity profile if set, reset otherwise
+		if c.dstRegionProfile != "" {
+			c.storageOpts.Profile = c.dstRegionProfile
+		} else {
+			c.storageOpts.Profile = defaultProfile
+		}
+
+		if c.dstRegionEndpoint != "" {
+			c.storageOpts.Endpoint = c.dstRegionEndpoint
+			c.storageOpts.NoVerifySSL = c.dstRegionNoVerifySSL
+		} else { //reset
+			c.storageOpts.Endpoint = defaultEndpoint
+			c.storageOpts.NoVerifySSL = defaultNoVerifySSL
+		}
+
+		err2 := c.doUpload(ctx, templocaldsturl, dsturl, metadata)
+		if err2 != nil {
+			return &errorpkg.Error{
+				Op:  c.op,
+				Src: srcurl,
+				Dst: dsturl,
+				Err: err2,
+			}
+		}
+
+		c.progressbar.IncrementCompletedObjects()
+		return nil
+	}
+}
+
 func (c Copy) prepareDownloadTask(
 	ctx context.Context,
 	srcurl *url.URL,