Merge pull request #382 from nerdalize/feature/s3-private-bucket

Make it possible to configure private S3 buckets

cmd/dataset_upload.go

@@ -51,7 +51,7 @@ func (cmd *DatasetUpload) Execute(args []string) (err error) {
 		return renderServiceError(err, "failed to expand home directory in dataset local path")
 	}
 
-	dir, err = filepath.Abs(args[0])
+	dir, err = filepath.Abs(dir)
 	if err != nil {
 		return renderServiceError(err, "failed to turn local path into absolute path")
 	}
@@ -72,6 +72,7 @@ func (cmd *DatasetUpload) Execute(args []string) (err error) {
 	if !ok {
 		return renderConfigError(fmt.Errorf("unable to use transfer options"), "failed to configure")
 	}
+
 	mgr, sto, sta, err := t.TransferManager(kube)
 	if err != nil {
 		return errors.Wrap(err, "failed to setup transfer manager")

cmd/opts.go

@@ -20,11 +20,12 @@ import (
 
 //TransferOpts hold CLI options for configuring data transfer
 type TransferOpts struct {
-	AWSS3Bucket        string `long:"aws-s3-bucket" description:"AWS S3 Bucket name that will be used for dataset storage" default:"nlz-datasets-dev"`
-	AWSRegion          string `long:"aws-region" description:"AWS region used for dataset storage"`
-	AWSAccessKeyID     string `long:"aws-access-key-id" description:"AWS access key used for auth with the storage backend"`
-	AWSSecretAccessKey string `long:"aws-secret-access-key" description:"AWS secret key for auth with the storage backend"`
-	AWSSessionToken    string `long:"aws-session-token" description:"AWS temporary auth token for the storage backend"`
+	S3Bucket       string `long:"s3-bucket" description:"S3 Bucket name that will be used for dataset storage" default:"nlz-datasets-dev"`
+	AWSRegion      string `long:"aws-region" description:"AWS region used for dataset storage"`
+	S3AccessKey    string `long:"s3-access-key" description:"access key used for auth with the storage backend"`
+	S3SecretKey    string `long:"s3-secret-key" description:"secret key for auth with the storage backend"`
+	S3SessionToken string `long:"s3-session-token" description:"temporary auth token for the storage backend"`
+	S3Prefix       string `long:"s3-prefix" description:"store this dataset under a specific prefix"`
 }
 
 //TransferManager creates a transfermanager using the command line options
@@ -36,8 +37,13 @@ func (opts TransferOpts) TransferManager(kube *svc.Kube) (mgr transfer.Manager,
 	}
 
 	sto = &transferstore.StoreOptions{
-		Type:          transferstore.StoreTypeS3,
-		S3StoreBucket: "nlz-datasets-dev",
+		Type:             transferstore.StoreTypeS3,
+		S3StoreBucket:    opts.S3Bucket,
+		S3StoreAWSRegion: opts.AWSRegion,
+		S3StoreAccessKey: opts.S3AccessKey,
+		S3StoreSecretKey: opts.S3SecretKey,
+		S3SessionToken:   opts.S3SessionToken,
+		S3StorePrefix:    opts.S3Prefix,
 	}
 	sta = &transferarchiver.ArchiverOptions{
 		Type: transferarchiver.ArchiverTypeTar,

pkg/transfer/store/s3_store.go

@@ -36,6 +36,7 @@ type S3Store struct {
 
 	sess *session.Session
 	dwn  s3manageriface.DownloaderAPI
+	upl  s3manageriface.UploaderAPI
 	api  s3iface.S3API
 }
 
@@ -71,12 +72,13 @@ func NewS3Store(cfg StoreOptions) (store *S3Store, err error) {
 		s3api.Handlers.Sign.Clear()
 		s3api.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
 		//we delibrately don't add actual signing middleware for anonymous access
+	} else {
+		store.upl = s3manager.NewUploaderWithClient(s3api)
 	}
 
-	s3dwn := s3manager.NewDownloaderWithClient(s3api)
-	s3dwn.PartSize = 1024 * 1024 * 1024 * 5 //5Gib
+	store.dwn = s3manager.NewDownloaderWithClient(s3api)
+	// s3dwn.PartSize = 1024 * 1024 * 1024 * 5 //5Gib @TODO, test if this is necessary
 
-	store.dwn = s3dwn
 	store.api = s3api
 
 	return store, nil
@@ -114,20 +116,32 @@ func (store *S3Store) Get(ctx context.Context, k string, w io.WriterAt) (err err
 			}
 		}
 
-		return errors.Wrapf(err, "failed to download object")
+		return errors.Wrapf(err, "failed to multi-part download object")
 	}
 
 	return nil
 }
 
 //Put an object into the store at key 'k' by reading from 'r'
 func (store *S3Store) Put(ctx context.Context, k string, r io.ReadSeeker) (err error) {
+	if store.upl != nil {
+		if _, err := store.upl.UploadWithContext(ctx, &s3manager.UploadInput{
+			Body:   r,
+			Bucket: aws.String(store.bucket),
+			Key:    aws.String(k),
+		}); err != nil {
+			return errors.Wrap(err, "failed to multi-part upload object")
+		}
+
+		return nil
+	}
+
 	if _, err := store.api.PutObjectWithContext(ctx, &s3.PutObjectInput{
 		Body:   r,
 		Bucket: aws.String(store.bucket),
 		Key:    aws.String(k),
 	}); err != nil {
-		return errors.Wrap(err, "failed to download object")
+		return errors.Wrap(err, "failed to upload object")
 	}
 
 	return nil

spec.json

@@ -162,8 +162,8 @@
 			"options": {
 				"Advanced Options": [
 					{
-						"long_name": "aws-s3-bucket",
-						"description": "AWS S3 Bucket name that will be used for dataset storage",
+						"long_name": "s3-bucket",
+						"description": "S3 Bucket name that will be used for dataset storage",
 						"default_value": [
 							"nlz-datasets-dev"
 						],
@@ -176,20 +176,26 @@
 						"choices": null
 					},
 					{
-						"long_name": "aws-access-key-id",
-						"description": "AWS access key used for auth with the storage backend",
+						"long_name": "s3-access-key",
+						"description": "access key used for auth with the storage backend",
 						"default_value": null,
 						"choices": null
 					},
 					{
-						"long_name": "aws-secret-access-key",
-						"description": "AWS secret key for auth with the storage backend",
+						"long_name": "s3-secret-key",
+						"description": "secret key for auth with the storage backend",
 						"default_value": null,
 						"choices": null
 					},
 					{
-						"long_name": "aws-session-token",
-						"description": "AWS temporary auth token for the storage backend",
+						"long_name": "s3-session-token",
+						"description": "temporary auth token for the storage backend",
+						"default_value": null,
+						"choices": null
+					},
+					{
+						"long_name": "s3-prefix",
+						"description": "store this dataset under a specific prefix",
 						"default_value": null,
 						"choices": null
 					}
@@ -340,8 +346,8 @@
 			"options": {
 				"Advanced Options": [
 					{
-						"long_name": "aws-s3-bucket",
-						"description": "AWS S3 Bucket name that will be used for dataset storage",
+						"long_name": "s3-bucket",
+						"description": "S3 Bucket name that will be used for dataset storage",
 						"default_value": [
 							"nlz-datasets-dev"
 						],
@@ -354,20 +360,26 @@
 						"choices": null
 					},
 					{
-						"long_name": "aws-access-key-id",
-						"description": "AWS access key used for auth with the storage backend",
+						"long_name": "s3-access-key",
+						"description": "access key used for auth with the storage backend",
+						"default_value": null,
+						"choices": null
+					},
+					{
+						"long_name": "s3-secret-key",
+						"description": "secret key for auth with the storage backend",
 						"default_value": null,
 						"choices": null
 					},
 					{
-						"long_name": "aws-secret-access-key",
-						"description": "AWS secret key for auth with the storage backend",
+						"long_name": "s3-session-token",
+						"description": "temporary auth token for the storage backend",
 						"default_value": null,
 						"choices": null
 					},
 					{
-						"long_name": "aws-session-token",
-						"description": "AWS temporary auth token for the storage backend",
+						"long_name": "s3-prefix",
+						"description": "store this dataset under a specific prefix",
 						"default_value": null,
 						"choices": null
 					}