-
Notifications
You must be signed in to change notification settings - Fork 7
/
workload_work.go
122 lines (106 loc) · 3.63 KB
/
workload_work.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
package command
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"log"
"os"
"os/signal"
"syscall"
"github.com/mitchellh/cli"
nerdaws "github.com/nerdalize/nerd/nerd/aws"
v1datatransfer "github.com/nerdalize/nerd/nerd/service/datatransfer/v1"
"github.com/nerdalize/nerd/nerd/service/working/v1"
"github.com/pkg/errors"
)
//WorkloadWorkOpts describes command options
type WorkloadWorkOpts struct {
EntrypointJSONB64 string `long:"entrypoint-json-base64" default:"W10=" description:"work entrypoint, first json and then base64 encoded"`
CmdJSONB64 string `long:"cmd-json-base64" default:"W10=" description:"work cmd, first json and then base64 encoded"`
OutputDir string `long:"output-dir" default:"" default-mask:"" description:"when set, data in --output-dir will be uploaded after each task run"`
}
//WorkloadWork command
type WorkloadWork struct {
*command
opts *WorkloadWorkOpts
}
//WorkloadWorkFactory returns a factory method for the join command
func WorkloadWorkFactory() (cli.Command, error) {
opts := &WorkloadWorkOpts{}
comm, err := newCommand("nerd workload work <workload-id>", "start working tasks of a queue locally", "", opts)
if err != nil {
return nil, errors.Wrap(err, "failed to create command")
}
cmd := &WorkloadWork{
command: comm,
opts: opts,
}
cmd.runFunc = cmd.DoRun
return cmd, nil
}
//DoRun is called by run and allows an error to be returned
func (cmd *WorkloadWork) DoRun(args []string) (err error) {
if len(args) < 1 {
return fmt.Errorf("not enough arguments, see --help")
}
var entrypoint, command []string
entrJSONString, err := base64.StdEncoding.DecodeString(cmd.opts.EntrypointJSONB64)
if err != nil {
HandleError(errors.Wrapf(err, "failed to base64 decode entrypoint '%v'", cmd.opts.EntrypointJSONB64))
}
commandJSONString, err := base64.StdEncoding.DecodeString(cmd.opts.CmdJSONB64)
if err != nil {
HandleError(errors.Wrapf(err, "failed to base64 decode cmd '%v'", cmd.opts.CmdJSONB64))
}
err = json.Unmarshal(entrJSONString, &entrypoint)
if err != nil {
HandleError(errors.Wrapf(err, "failed to decode entrypoint '%s'", entrJSONString))
}
err = json.Unmarshal(commandJSONString, &command)
if err != nil {
HandleError(errors.Wrapf(err, "failed to decode cmd '%s'", commandJSONString))
}
bclient, err := NewClient(cmd.config, cmd.session)
if err != nil {
HandleError(errors.Wrap(err, "failed to create client"))
}
ss, err := cmd.session.Read()
if err != nil {
HandleError(err)
}
creds := nerdaws.NewNerdalizeCredentials(bclient, ss.Project.Name)
qops, err := nerdaws.NewQueueClient(creds, ss.Project.AWSRegion)
if err != nil {
HandleError(err)
}
logger := log.New(os.Stderr, "worker/", log.Lshortfile)
conf := v1working.DefaultConf()
var worker *v1working.Worker
if cmd.opts.OutputDir != "" {
dataOps, err := nerdaws.NewDataClient(
nerdaws.NewNerdalizeCredentials(bclient, ss.Project.Name),
ss.Project.AWSRegion,
)
if err != nil {
HandleError(errors.Wrap(err, "could not create aws dataops client"))
}
uploadConf := &v1datatransfer.UploadConfig{
BatchClient: bclient,
DataOps: dataOps,
LocalDir: cmd.opts.OutputDir,
ProjectID: ss.Project.Name,
Concurrency: 64,
}
worker = v1working.NewWorker(logger, bclient, qops, ss.Project.Name, args[0], entrypoint, command, uploadConf, conf)
} else {
worker = v1working.NewWorker(logger, bclient, qops, ss.Project.Name, args[0], entrypoint, command, nil, conf)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go worker.Start(ctx)
exitCh := make(chan os.Signal, 1)
signal.Notify(exitCh, os.Interrupt, syscall.SIGTERM)
<-exitCh
return nil
}