Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[DONT MERGE] [FD-892] instrument index reader - 2.8.4 #9910

Closed
wants to merge 1 commit into from
Closed

[DONT MERGE] [FD-892] instrument index reader - 2.8.4 #9910

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
74 changes: 63 additions & 11 deletions src/internal/storage/fileset/index/reader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,14 @@ import (
"bytes"
"context"
fmt "fmt"
"io"

"github.com/docker/go-units"
"github.com/pachyderm/pachyderm/v2/src/internal/errors"
"github.com/pachyderm/pachyderm/v2/src/internal/errutil"
"github.com/pachyderm/pachyderm/v2/src/internal/miscutil"
"github.com/pachyderm/pachyderm/v2/src/internal/pbutil"
"github.com/pachyderm/pachyderm/v2/src/internal/storage/chunk"
"google.golang.org/protobuf/proto"
"io"
)

const (
Expand All @@ -24,26 +23,59 @@ const (
DefaultShardSizeThreshold = units.GB
)

type edge struct {
from string
to string
reason string
}

func (e *edge) String() string {
return e.from + ", " + e.to + ", reason: " + e.reason
}

func (idx *Index) toEdgeStr() string {
if idx.File != nil {
return "{type:file path:" + idx.Path + "}"
}
return fmt.Sprintf("{type:range path:%s chunk:%x}", idx.Path, idx.Range.ChunkRef.Ref.Id)
}

func toEdge(from, to *Index, reason string) edge {
return edge{
from: from.toEdgeStr(),
to: to.toEdgeStr(),
reason: reason,
}
}

// Reader is used for reading a multilevel index.
type Reader struct {
chunks *chunk.Storage
cache *Cache
filter *pathFilter
topIdx *Index
datum string
shardConfig *ShardConfig
chunks *chunk.Storage
cache *Cache
filter *pathFilter
topIdx *Index
datum string
shardConfig *ShardConfig
prevIdx *Index
traversedIdx []edge
skippedIdx []edge
}

// NewReader creates a new Reader.
func NewReader(chunks *chunk.Storage, cache *Cache, topIdx *Index, opts ...Option) *Reader {
traversedEdges := make([]edge, 0)
skippedEdges := make([]edge, 0)
r := &Reader{
chunks: chunks,
cache: cache,
topIdx: topIdx,
chunks: chunks,
cache: cache,
topIdx: topIdx,
prevIdx: topIdx,
shardConfig: &ShardConfig{
NumFiles: DefaultShardNumThreshold,
SizeBytes: DefaultShardSizeThreshold,
},
traversedIdx: traversedEdges,
skippedIdx: skippedEdges,
}
for _, opt := range opts {
opt(r)
Expand All @@ -58,23 +90,42 @@ func (r *Reader) Iterate(ctx context.Context, cb func(*Index) error) error {
}
traverseCb := func(idx *Index) (bool, error) {
if atEnd(idx.Path, r.filter) {
r.skippedIdx = append(r.skippedIdx, toEdge(r.prevIdx, idx, "past end"))
return false, errutil.ErrBreak
}
if idx.File != nil {
if !atStart(idx.Path, r.filter) || !(r.datum == "" || r.datum == idx.File.Datum) {
r.skippedIdx = append(r.skippedIdx, toEdge(r.prevIdx, idx, "file before start / datum mismatch"))
return false, nil
}
r.traversedIdx = append(r.traversedIdx, toEdge(r.prevIdx, idx, "traversed file"))
return false, cb(idx)
}
if !atStart(idx.Range.LastPath, r.filter) {
r.skippedIdx = append(r.skippedIdx, toEdge(r.prevIdx, idx, "range before start"))
return false, nil
}
r.traversedIdx = append(r.traversedIdx, toEdge(r.prevIdx, idx, "traversed range"))
return true, nil
}
_, err := r.traverse(ctx, r.topIdx, []byte{}, traverseCb)
if errors.Is(err, errutil.ErrBreak) {
err = nil
}
traversedEdges := ""
for _, edge := range r.traversedIdx {
traversedEdges += edge.String() + "\n"
}
skippedEdges := ""
for _, edge := range r.skippedIdx {
skippedEdges += edge.String() + "\n"
}
if len(skippedEdges)+len(traversedEdges) > 0 {
fmt.Println("traversed indices:", len(r.traversedIdx), "skipped indices:", len(r.skippedIdx))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be log.Info(pctx.TODO(), ...). This messes up anything else that relies on the logs being JSON.

fmt.Println(traversedEdges)
fmt.Println(skippedEdges)
}

return err
}

Expand All @@ -95,6 +146,7 @@ func (r *Reader) traverse(ctx context.Context, idx *Index, prependBytes []byte,
if err := r.getChunk(ctx, idx, buf); err != nil {
return nil, err
}
r.prevIdx = idx
pbr := pbutil.NewReader(buf)
nextPrependBytes := []byte{}
for {
Expand Down