Skip to content

Commit

Permalink
Switch to snappy framing
Browse files Browse the repository at this point in the history
  • Loading branch information
olegbespalov committed Jun 20, 2023
1 parent 4a62131 commit 79ac093
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 9 deletions.
36 changes: 30 additions & 6 deletions output/cloud/expv2/metrics_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (mc *metricsClient) push(referenceID string, samples *pbcloud.MetricSet) er
}

req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("Content-Encoding", "snappy")
req.Header.Set("Content-Encoding", "x-snappy-framed")
req.Header.Set("K6-Metrics-Protocol-Version", "2.0")

err = mc.httpClient.Do(req, nil)
Expand All @@ -81,13 +81,37 @@ func newRequestBody(data *pbcloud.MetricSet) ([]byte, error) {
if err != nil {
return nil, fmt.Errorf("encoding metrics as Protobuf write request failed: %w", err)
}
// TODO: use the framing format
// https://github.com/google/snappy/blob/main/framing_format.txt
// It can be done replacing the encode with
// https://pkg.go.dev/github.com/klauspost/compress/snappy#NewBufferedWriter

if snappy.MaxEncodedLen(len(b)) < 0 {
return nil, fmt.Errorf("the Protobuf message is too large to be handled by Snappy encoder; "+
"size: %d, limit: %d", len(b), 0xffffffff)
}
return snappy.Encode(nil, b), nil

b, err = encodeWithSnappyFraming(b)
if err != nil {
return nil, fmt.Errorf("metrics request compression failed: %w", err)
}

return b, nil
}

// encodeWithSnappyFraming encodes the given data with Snappy framing.
// See https://github.com/google/snappy/blob/main/framing_format.txt
func encodeWithSnappyFraming(data []byte) ([]byte, error) {
var buf bytes.Buffer
writer := snappy.NewBufferedWriter(&buf)

_, err := writer.Write(data)
if err != nil {
_ = writer.Close()

return nil, err
}

err = writer.Close()
if err != nil {
return nil, err
}

return buf.Bytes(), nil
}
13 changes: 10 additions & 3 deletions output/cloud/expv2/metrics_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ func TestMetricsClientPush(t *testing.T) {
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, "Token fake-token", r.Header.Get("Authorization"))
assert.Contains(t, r.Header.Get("User-Agent"), "k6cloud/v0.4")
assert.Equal(t, "application/x-protobuf", r.Header.Get("Content-Type"))
assert.Equal(t, "snappy", r.Header.Get("Content-Encoding"))
assert.Equal(t, "x-snappy-framed", r.Header.Get("Content-Encoding"))
assert.Equal(t, "2.0", r.Header.Get("K6-Metrics-Protocol-Version"))
b, err := io.ReadAll(r.Body)
require.NoError(t, err)
Expand All @@ -39,7 +38,15 @@ func TestMetricsClientPush(t *testing.T) {
mc, err := newMetricsClient(c)
require.NoError(t, err)

mset := pbcloud.MetricSet{}
mset := pbcloud.MetricSet{
Metrics: []*pbcloud.Metric{
{
Name: "http_reqs",
Type: pbcloud.MetricType_METRIC_TYPE_COUNTER,
TimeSeries: []*pbcloud.TimeSeries{},
},
},
}
err = mc.push("test-ref-id", &mset)
require.NoError(t, err)
assert.Equal(t, 1, reqs)
Expand Down

0 comments on commit 79ac093

Please sign in to comment.