-
Notifications
You must be signed in to change notification settings - Fork 0
/
integrity.go
151 lines (130 loc) · 3.74 KB
/
integrity.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
package main
import (
"context"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"time"
"github.com/peterbourgon/ff/v3/ffcli"
"github.com/jtolio/jam/manifest"
"github.com/jtolio/jam/session"
"github.com/jtolio/jam/streams"
"github.com/jtolio/jam/utils"
)
var (
integrityFlags = flag.NewFlagSet("", flag.ExitOnError)
integrityFlagShowUnneeded = integrityFlags.Bool("show-unneeded", false, "if true, show unneeded blobs")
integrityFlagSkipBlobEnd = integrityFlags.Bool("skip-blob-end", false, "if true, skip trying to read the known end of each blob")
cmdIntegrity = &ffcli.Command{
Name: "integrity",
ShortHelp: "integrity check. for full effect, disable caching and enable read\n\tcomparison",
ShortUsage: fmt.Sprintf("%s [opts] integrity [opts]", os.Args[0]),
FlagSet: integrityFlags,
Exec: Integrity,
}
)
func Integrity(ctx context.Context, args []string) error {
if len(args) != 0 {
return flag.ErrHelp
}
utils.L(ctx).Debugf("loading backend and hash db")
mgr, backend, hashes, mgrClose, err := getManager(ctx)
if err != nil {
return err
}
defer mgrClose()
blobs := map[string]bool{}
blobLastRange := map[string]*manifest.Range{}
missing := map[string]bool{}
bad := map[string]bool{}
utils.L(ctx).Debugf("confirming that a blob exists for every hash")
err = backend.List(ctx, streams.BlobPrefix,
func(ctx context.Context, path string) error {
blobs[path] = true
return nil
})
if err != nil {
return err
}
err = hashes.Iterate(ctx, func(ctx context.Context, hash, hashset string, stream *manifest.Stream) error {
for _, r := range stream.Ranges {
blobPath := streams.BlobPath(r.Blob())
if lastRange, exists := blobLastRange[blobPath]; r.Length > 0 && (!exists || lastRange.Offset < r.Offset) {
blobLastRange[blobPath] = r
}
if !blobs[blobPath] {
if !missing[r.Blob()] {
missing[r.Blob()] = true
fmt.Printf("missing blob: %s\n", r.Blob())
}
if !bad[hashset] {
bad[hashset] = true
fmt.Printf("from hash set: %s\n", hashset)
}
}
}
return nil
})
if err != nil {
return err
}
utils.L(ctx).Debugf("no dangling hashes")
if *integrityFlagShowUnneeded {
for path := range blobs {
if _, exists := blobLastRange[path]; !exists {
fmt.Printf("blob unnecessary: %s\n", path)
}
}
}
utils.L(ctx).Debugf("making sure a hash for every listed path in every snapshot exists")
// check to make sure a hash for every listed path exists
err = mgr.ListSnapshots(ctx, func(ctx context.Context, timestamp time.Time) error {
utils.L(ctx).Debugf("checking snapshot %v: %v",
timestamp.UnixNano(),
timestamp.Local().Format("2006-01-02 03:04:05 pm"))
snap, err := mgr.OpenSnapshot(ctx, timestamp)
if err != nil {
return err
}
defer snap.Close()
return snap.List(ctx, "", true, func(ctx context.Context, entry *session.ListEntry) error {
if entry.Meta.Type != manifest.Metadata_FILE {
return nil
}
stream, err := entry.Stream(ctx)
if err != nil {
return err
}
return stream.Close()
})
})
if err != nil {
return err
}
utils.L(ctx).Debugf("no dangling paths")
// check to make sure none of the blobs are truncated
if !*integrityFlagSkipBlobEnd {
utils.L(ctx).Debugf("checking to make sure the last byte of each blob is readable")
for path, r := range blobLastRange {
utils.L(ctx).Debugf("checking end of %q, %d", path, r.Offset+r.Length)
rc, err := streams.OpenRange(ctx, backend, r, r.Length-1)
if err != nil {
return err
}
// authenticated encryption will throw an error if the data is bad
_, err = io.Copy(ioutil.Discard, rc)
if err != nil {
rc.Close()
return err
}
err = rc.Close()
if err != nil {
return err
}
}
utils.L(ctx).Debugf("looks good")
}
return nil
}