-
Notifications
You must be signed in to change notification settings - Fork 7
/
upload.go
108 lines (89 loc) · 2.65 KB
/
upload.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
package command
import (
"fmt"
"io/ioutil"
"os"
"path"
"github.com/Sirupsen/logrus"
"github.com/jessevdk/go-flags"
"github.com/mitchellh/cli"
"github.com/nerdalize/nerd/nerd/aws"
"github.com/pkg/errors"
)
const (
//DatasetFilename is the filename of the file that contains the dataset ID in the data folder.
DatasetFilename = ".dataset"
//DatasetPermissions are the permissions for DatasetFilename
DatasetPermissions = 0644
)
//UploadOpts describes command options
type UploadOpts struct {
NerdOpts
}
//Upload command
type Upload struct {
*command
opts *UploadOpts
parser *flags.Parser
}
//UploadFactory returns a factory method for the join command
func UploadFactory() func() (cmd cli.Command, err error) {
cmd := &Upload{
command: &command{
help: "",
synopsis: "Upload a dataset to cloud storage.\nOptionally you can specify a dataset-ID to append files to that dataset. This is also useful to continue an upload in case a previous try failed.",
parser: flags.NewNamedParser("nerd upload <path> [dataset-ID]", flags.Default),
ui: &cli.BasicUi{
Reader: os.Stdin,
Writer: os.Stderr,
},
},
opts: &UploadOpts{},
}
cmd.runFunc = cmd.DoRun
_, err := cmd.command.parser.AddGroup("options", "options", cmd.opts)
if err != nil {
panic(err)
}
return func() (cli.Command, error) {
return cmd, nil
}
}
//DoRun is called by run and allows an error to be returned
func (cmd *Upload) DoRun(args []string) (err error) {
if len(args) < 1 {
return fmt.Errorf("not enough arguments, see --help")
}
dataPath := args[0]
nerdclient, err := NewClient(cmd.ui)
if err != nil {
HandleError(err, cmd.opts.VerboseOutput)
}
ds, err := nerdclient.CreateDataset()
if err != nil {
HandleError(err, cmd.opts.VerboseOutput)
}
client, err := aws.NewDataClient(&aws.DataClientConfig{
Credentials: aws.NewNerdalizeCredentials(nerdclient),
Bucket: ds.Bucket,
})
if err != nil {
HandleError(errors.Wrap(err, "could not create data client"), cmd.opts.VerboseOutput)
}
fi, err := os.Stat(dataPath)
if err != nil {
HandleError(errors.Errorf("argument '%v' is not a valid file or directory", dataPath), cmd.opts.VerboseOutput)
} else if !fi.IsDir() {
HandleError(errors.Errorf("provided path '%s' is not a directory", dataPath), cmd.opts.VerboseOutput)
}
err = ioutil.WriteFile(path.Join(dataPath, DatasetFilename), []byte(ds.DatasetID), DatasetPermissions)
if err != nil {
HandleError(err, cmd.opts.VerboseOutput)
}
err = client.UploadDir(dataPath, ds.Root, &stdoutkw{}, 64)
if err != nil {
HandleError(err, cmd.opts.VerboseOutput)
}
logrus.Infof("Uploaded data to dataset with ID: %v", ds.DatasetID)
return nil
}