Skip to content

Commit

Permalink
Implemented #6 chunked uploads
Browse files Browse the repository at this point in the history
  • Loading branch information
pierotofy committed Oct 11, 2019
1 parent 6873ca3 commit ef42976
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 5 deletions.
7 changes: 6 additions & 1 deletion internal/odm/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,12 @@ func (n Node) TaskDownload(uuid string, asset string, outputFile string) error {
bar.Prefix("[" + asset + "]")
}

writer := io.MultiWriter(out, bar)
var writer io.Writer
if bar != nil {
writer = io.MultiWriter(out, bar)
} else {
writer = out
}

written, err := io.Copy(writer, resp.Body)
if written == 0 {
Expand Down
10 changes: 6 additions & 4 deletions internal/odm/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package odm
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
Expand Down Expand Up @@ -141,7 +140,7 @@ func uploadWorker(id int, node Node, uuid string, barPool *pb.Pool, filesToProce
}

for f := range filesToProcess {
results <- fileUploadResult{f.filename, node.TaskNewUpload(f.filename, uuid, bar), f.retries + 1}
results <- fileUploadResult{f.filename, node.TaskNewUpload(f.filename, uuid, bar), f.retries}
}
}

Expand Down Expand Up @@ -194,20 +193,23 @@ func chunkedUpload(node Node, files []string, jsonOptions []byte, parallelUpload
if fur.err != nil {
if fur.retries < MaxRetries {
// Retry
fmt.Println("RETRY: " + fur.filename + " " + string(fur.retries))
filesToProcess <- fileUpload{fur.filename, fur.retries + 1}
} else {
logger.Error(errors.New("Cannot upload " + fur.filename + ", exceeded max retries (" + string(MaxRetries) + ")"))
}
} else {
filesLeft--
mainBar.Set(len(files) - filesLeft)
if mainBar != nil {
mainBar.Set(len(files) - filesLeft)
}
}
}
close(filesToProcess)

if barPool != nil {
barPool.Stop()
}
if mainBar != nil {
mainBar.Finish()
}

Expand Down

0 comments on commit ef42976

Please sign in to comment.