[SYSINFRA-1673] reduce reflow cpu usage in scheduler.run (grailbio/gr…

…ail!902)

reduce reflow cpu/mem usage

Various changes to reduce reflow cpu/mem usage:
- Use a higher (`50`) than default (`2`) number of idle conns per reflowlet.
- Make sure to close `http` calls to release connections
- Don't update status in `repository/manager` for non-console
  use-cases (eg: `pi-evaluator`), and don't set a default status.

Co-authored-by:
Approved-by: Approved-by: Jim Clune <jclune@grailbio.com>

GitLab URL: https://gitlab.com/grailbio/grail/-/merge_requests/902

fbshipit-source-id: 4af52c5

pool/client/client.go

@@ -418,10 +418,10 @@ func (o *clientExec) RemoteLogs(ctx context.Context, stdout bool) (reflow.Remote
 		which = "stdoutloc"
 	}
 	call, code, err := o.makeLogsCall(ctx, which, false)
+	defer func() { _ = call.Close() }()
 	if err != nil {
 		return reflow.RemoteLogs{}, errors.E("logslocation", o.URI(), which, err)
 	}
-	defer func() { _ = call.Close() }()
 	if code != http.StatusOK {
 		return reflow.RemoteLogs{}, call.Error()
 	}

pool/server/server.go

@@ -332,13 +332,13 @@ func (n execNode) logNode(loc, stdout, stderr bool, follow string) rest.Node {
 			call.Error(err)
 			return
 		}
+		defer func() { _ = rc.Close() }()
 		_, err = io.Copy(&rest.StreamingCall{Call: call}, rc)
 		if err != nil {
 			call.Error(err)
 			return
 		}
-		rc.Close()
-		call.Write(http.StatusOK, bytes.NewReader(nil))
+		_ = call.Write(http.StatusOK, bytes.NewReader(nil))
 	})
 }
 

repository/manager.go

@@ -320,6 +320,10 @@ func (m *Manager) transfer(ctx context.Context, dst, src reflow.Repository, file
 }
 
 func (m *Manager) updateStats(src, dst reflow.Repository, status transferStatus, stat stat) {
+	// If there's no status object to report to, then don't bother with updating stats.
+	if m.Status == nil {
+		return
+	}
 	k := key(src) + key(dst)
 	m.mu.Lock()
 	t := m.tasks[k]

runtime/scheduler.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/grailbio/base/status"
 	"github.com/grailbio/infra"
 	"github.com/grailbio/reflow"
 	"github.com/grailbio/reflow/errors"
@@ -46,7 +45,7 @@ func newScheduler(config infra.Config, logger *log.Logger) (*sched.Scheduler, er
 		return nil, err
 	}
 	transferer := &repository.Manager{
-		Status:           new(status.Status).Group("transfers"),
+		Status:           nil,
 		PendingTransfers: repository.NewLimits(limit),
 		Stat:             repository.NewLimits(statLimit),
 		Log:              logger,