From d86e24ac9df43a029276df8a812be7d31f41e1cf Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 26 Feb 2021 17:17:47 +0200 Subject: [PATCH] Add a way to mark http requests as failed This is done through running a callback on every request before emitting the metrics. Currently only a built-in metric looking at good statuses is possible, but a possibility for future JS based callbacks is left open. The implementation specifically makes it hard to figure out anything about the returned callback from JS and tries not to change any other code so it makes it easier for future implementation, but instead tries to do the bare minimum without imposing any limitations on the future work. Additionally because it turned out to be easy, setting the callback to null will make the http library to neither tag requests with `expected_response` nor emit the new `http_req_failed` metric, essentially giving users a way to go back to the previous behaviour. part of #1828 --- core/engine.go | 14 + core/local/local_test.go | 13 +- js/modules/k6/http/http.go | 4 + js/modules/k6/http/request.go | 21 +- js/modules/k6/http/request_test.go | 52 +- js/modules/k6/http/response_callback.go | 117 ++++ js/modules/k6/http/response_callback_test.go | 562 +++++++++++++++++++ js/modules/k6/http/response_test.go | 1 + js/runner.go | 3 +- lib/metrics/metrics.go | 3 +- lib/netext/httpext/request.go | 49 +- lib/netext/httpext/tracer.go | 8 +- lib/netext/httpext/transport.go | 49 +- lib/netext/httpext/transport_test.go | 86 +++ stats/cloud/bench_test.go | 30 + stats/cloud/cloud_easyjson.go | 66 ++- stats/cloud/data.go | 56 +- stats/cloud/data_test.go | 102 +++- stats/system_tag.go | 3 +- stats/system_tag_set_gen.go | 48 +- stats/units.go | 8 + 21 files changed, 1200 insertions(+), 95 deletions(-) create mode 100644 js/modules/k6/http/response_callback.go create mode 100644 js/modules/k6/http/response_callback_test.go create mode 100644 lib/netext/httpext/transport_test.go diff --git a/core/engine.go b/core/engine.go index daf5bed3505..1c527357281 100644 --- a/core/engine.go +++ b/core/engine.go @@ -108,6 +108,20 @@ func NewEngine( e.submetrics[parent] = append(e.submetrics[parent], sm) } + // TODO: refactor this out of here when https://github.com/loadimpact/k6/issues/1832 lands and + // there is a better way to enable a metric with tag + if opts.SystemTags.Has(stats.TagExpectedResponse) { + for _, name := range []string{ + "http_req_duration{expected_response:true}", + } { + if _, ok := e.thresholds[name]; ok { + continue + } + parent, sm := stats.NewSubmetric(name) + e.submetrics[parent] = append(e.submetrics[parent], sm) + } + } + return e, nil } diff --git a/core/local/local_test.go b/core/local/local_test.go index 382dd1a0028..c86f687b8cd 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -329,12 +329,13 @@ func TestExecutionSchedulerSystemTags(t *testing.T) { }() expCommonTrailTags := stats.IntoSampleTags(&map[string]string{ - "group": "", - "method": "GET", - "name": sr("HTTPBIN_IP_URL/"), - "url": sr("HTTPBIN_IP_URL/"), - "proto": "HTTP/1.1", - "status": "200", + "group": "", + "method": "GET", + "name": sr("HTTPBIN_IP_URL/"), + "url": sr("HTTPBIN_IP_URL/"), + "proto": "HTTP/1.1", + "status": "200", + "expected_response": "true", }) expTrailPVUTagsRaw := expCommonTrailTags.CloneTags() expTrailPVUTagsRaw["scenario"] = "per_vu_test" diff --git a/js/modules/k6/http/http.go b/js/modules/k6/http/http.go index d0be75b8ec5..83aa592734d 100644 --- a/js/modules/k6/http/http.go +++ b/js/modules/k6/http/http.go @@ -73,6 +73,8 @@ func (g *GlobalHTTP) NewModuleInstancePerVU() interface{} { // this here needs t OCSP_REASON_REMOVE_FROM_CRL: netext.OCSP_REASON_REMOVE_FROM_CRL, OCSP_REASON_PRIVILEGE_WITHDRAWN: netext.OCSP_REASON_PRIVILEGE_WITHDRAWN, OCSP_REASON_AA_COMPROMISE: netext.OCSP_REASON_AA_COMPROMISE, + + responseCallback: defaultExpectedStatuses.match, } } @@ -97,6 +99,8 @@ type HTTP struct { OCSP_REASON_REMOVE_FROM_CRL string `js:"OCSP_REASON_REMOVE_FROM_CRL"` OCSP_REASON_PRIVILEGE_WITHDRAWN string `js:"OCSP_REASON_PRIVILEGE_WITHDRAWN"` OCSP_REASON_AA_COMPROMISE string `js:"OCSP_REASON_AA_COMPROMISE"` + + responseCallback func(int) bool } func (*HTTP) XCookieJar(ctx *context.Context) *HTTPCookieJar { diff --git a/js/modules/k6/http/request.go b/js/modules/k6/http/request.go index 91fc9a73c88..22f18ab9dac 100644 --- a/js/modules/k6/http/request.go +++ b/js/modules/k6/http/request.go @@ -134,12 +134,14 @@ func (h *HTTP) parseRequest( URL: reqURL.GetURL(), Header: make(http.Header), }, - Timeout: 60 * time.Second, - Throw: state.Options.Throw.Bool, - Redirects: state.Options.MaxRedirects, - Cookies: make(map[string]*httpext.HTTPRequestCookie), - Tags: make(map[string]string), + Timeout: 60 * time.Second, + Throw: state.Options.Throw.Bool, + Redirects: state.Options.MaxRedirects, + Cookies: make(map[string]*httpext.HTTPRequestCookie), + Tags: make(map[string]string), + ResponseCallback: h.responseCallback, } + if state.Options.DiscardResponseBodies.Bool { result.ResponseType = httpext.ResponseTypeNone } else { @@ -349,6 +351,15 @@ func (h *HTTP) parseRequest( return nil, err } result.ResponseType = responseType + case "responseCallback": + v := params.Get(k).Export() + if v == nil { + result.ResponseCallback = nil + } else if c, ok := v.(*expectedStatuses); ok { + result.ResponseCallback = c.match + } else { + return nil, fmt.Errorf("unsupported responseCallback") + } } } } diff --git a/js/modules/k6/http/request_test.go b/js/modules/k6/http/request_test.go index 2c2d4af1a1c..878bf1f9005 100644 --- a/js/modules/k6/http/request_test.go +++ b/js/modules/k6/http/request_test.go @@ -81,6 +81,7 @@ func TestRunES6String(t *testing.T) { }) } +// TODO replace this with the Single version func assertRequestMetricsEmitted(t *testing.T, sampleContainers []stats.SampleContainer, method, url, name string, status int, group string) { if name == "" { name = url @@ -130,6 +131,29 @@ func assertRequestMetricsEmitted(t *testing.T, sampleContainers []stats.SampleCo assert.True(t, seenReceiving, "url %s didn't emit Receiving", url) } +func assertRequestMetricsEmittedSingle(t *testing.T, sampleContainer stats.SampleContainer, expectedTags map[string]string, metrics []*stats.Metric, callback func(sample stats.Sample)) { + t.Helper() + + metricMap := make(map[string]bool, len(metrics)) + for _, m := range metrics { + metricMap[m.Name] = false + } + for _, sample := range sampleContainer.GetSamples() { + tags := sample.Tags.CloneTags() + v, ok := metricMap[sample.Metric.Name] + assert.True(t, ok, "unexpected metric %s", sample.Metric.Name) + assert.False(t, v, "second metric %s", sample.Metric.Name) + metricMap[sample.Metric.Name] = true + assert.EqualValues(t, expectedTags, tags, "%s", tags) + if callback != nil { + callback(sample) + } + } + for k, v := range metricMap { + assert.True(t, v, "didn't emit %s", k) + } +} + func newRuntime( t testing.TB, ) (*httpmultibin.HTTPMultiBin, *lib.State, chan stats.SampleContainer, *goja.Runtime, *context.Context) { @@ -1945,26 +1969,28 @@ func TestRedirectMetricTags(t *testing.T) { checkTags := func(sc stats.SampleContainer, expTags map[string]string) { allSamples := sc.GetSamples() - assert.Len(t, allSamples, 8) + assert.Len(t, allSamples, 9) for _, s := range allSamples { assert.Equal(t, expTags, s.Tags.CloneTags()) } } expPOSTtags := map[string]string{ - "group": "", - "method": "POST", - "url": sr("HTTPBIN_URL/redirect/post"), - "name": sr("HTTPBIN_URL/redirect/post"), - "status": "301", - "proto": "HTTP/1.1", + "group": "", + "method": "POST", + "url": sr("HTTPBIN_URL/redirect/post"), + "name": sr("HTTPBIN_URL/redirect/post"), + "status": "301", + "proto": "HTTP/1.1", + "expected_response": "true", } expGETtags := map[string]string{ - "group": "", - "method": "GET", - "url": sr("HTTPBIN_URL/get"), - "name": sr("HTTPBIN_URL/get"), - "status": "200", - "proto": "HTTP/1.1", + "group": "", + "method": "GET", + "url": sr("HTTPBIN_URL/get"), + "name": sr("HTTPBIN_URL/get"), + "status": "200", + "proto": "HTTP/1.1", + "expected_response": "true", } checkTags(<-samples, expPOSTtags) checkTags(<-samples, expGETtags) diff --git a/js/modules/k6/http/response_callback.go b/js/modules/k6/http/response_callback.go new file mode 100644 index 00000000000..cdcb7f74663 --- /dev/null +++ b/js/modules/k6/http/response_callback.go @@ -0,0 +1,117 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2021 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package http + +import ( + "context" + "errors" + "fmt" + + "github.com/dop251/goja" + "github.com/loadimpact/k6/js/common" +) + +//nolint:gochecknoglobals +var defaultExpectedStatuses = expectedStatuses{ + minmax: [][2]int{{200, 399}}, +} + +// expectedStatuses is specifically totally unexported so it can't be used for anything else but +// SetResponseCallback and nothing can be done from the js side to modify it or make an instance of +// it except using ExpectedStatuses +type expectedStatuses struct { + minmax [][2]int + exact []int +} + +func (e expectedStatuses) match(status int) bool { + for _, v := range e.exact { + if v == status { + return true + } + } + + for _, v := range e.minmax { + if v[0] <= status && status <= v[1] { + return true + } + } + return false +} + +// ExpectedStatuses returns expectedStatuses object based on the provided arguments. +// The arguments must be either integers or object of `{min: , max: }` +// kind. The "integer"ness is checked by the Number.isInteger. +func (*HTTP) ExpectedStatuses(ctx context.Context, args ...goja.Value) *expectedStatuses { //nolint: golint + rt := common.GetRuntime(ctx) + + if len(args) == 0 { + common.Throw(rt, errors.New("no arguments")) + } + var result expectedStatuses + + jsIsInt, _ := goja.AssertFunction(rt.GlobalObject().Get("Number").ToObject(rt).Get("isInteger")) + isInt := func(a goja.Value) bool { + v, err := jsIsInt(goja.Undefined(), a) + return err == nil && v.ToBoolean() + } + + errMsg := "argument number %d to expectedStatuses was neither an integer nor an object like {min:100, max:329}" + for i, arg := range args { + o := arg.ToObject(rt) + if o == nil { + common.Throw(rt, fmt.Errorf(errMsg, i+1)) + } + + if isInt(arg) { + result.exact = append(result.exact, int(o.ToInteger())) + } else { + min := o.Get("min") + max := o.Get("max") + if min == nil || max == nil { + common.Throw(rt, fmt.Errorf(errMsg, i+1)) + } + if !(isInt(min) && isInt(max)) { + common.Throw(rt, fmt.Errorf("both min and max need to be integers for argument number %d", i+1)) + } + + result.minmax = append(result.minmax, [2]int{int(min.ToInteger()), int(max.ToInteger())}) + } + } + return &result +} + +// SetResponseCallback sets the responseCallback to the value provided. Supported values are +// expectedStatuses object or a `null` which means that metrics shouldn't be tagged as failed and +// `http_req_failed` should not be emitted - the behaviour previous to this +func (h *HTTP) SetResponseCallback(ctx context.Context, val goja.Value) { + if val != nil && !goja.IsNull(val) { + // This is done this way as ExportTo exports functions to empty structs without an error + if es, ok := val.Export().(*expectedStatuses); ok { + h.responseCallback = es.match + } else { + //nolint:golint + common.Throw(common.GetRuntime(ctx), fmt.Errorf("unsupported argument, expected http.expectedStatuses")) + } + } else { + h.responseCallback = nil + } +} diff --git a/js/modules/k6/http/response_callback_test.go b/js/modules/k6/http/response_callback_test.go new file mode 100644 index 00000000000..735fc7ff05b --- /dev/null +++ b/js/modules/k6/http/response_callback_test.go @@ -0,0 +1,562 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2021 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package http + +import ( + "context" + "fmt" + "sort" + "testing" + + "github.com/dop251/goja" + "github.com/loadimpact/k6/js/common" + "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/metrics" + "github.com/loadimpact/k6/stats" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExpectedStatuses(t *testing.T) { + t.Parallel() + rt := goja.New() + rt.SetFieldNameMapper(common.FieldNameMapper{}) + ctx := context.Background() + + ctx = common.WithRuntime(ctx, rt) + rt.Set("http", common.Bind(rt, new(GlobalHTTP).NewModuleInstancePerVU(), &ctx)) + cases := map[string]struct { + code, err string + expected expectedStatuses + }{ + "good example": { + expected: expectedStatuses{exact: []int{200, 300}, minmax: [][2]int{{200, 300}}}, + code: `(http.expectedStatuses(200, 300, {min: 200, max:300}))`, + }, + + "strange example": { + expected: expectedStatuses{exact: []int{200, 300}, minmax: [][2]int{{200, 300}}}, + code: `(http.expectedStatuses(200, 300, {min: 200, max:300, other: "attribute"}))`, + }, + + "string status code": { + code: `(http.expectedStatuses(200, "300", {min: 200, max:300}))`, + err: "argument number 2 to expectedStatuses was neither an integer nor an object like {min:100, max:329}", + }, + + "string max status code": { + code: `(http.expectedStatuses(200, 300, {min: 200, max:"300"}))`, + err: "both min and max need to be integers for argument number 3", + }, + "float status code": { + err: "argument number 2 to expectedStatuses was neither an integer nor an object like {min:100, max:329}", + code: `(http.expectedStatuses(200, 300.5, {min: 200, max:300}))`, + }, + + "float max status code": { + err: "both min and max need to be integers for argument number 3", + code: `(http.expectedStatuses(200, 300, {min: 200, max:300.5}))`, + }, + "no arguments": { + code: `(http.expectedStatuses())`, + err: "no arguments", + }, + } + + for name, testCase := range cases { + name, testCase := name, testCase + t.Run(name, func(t *testing.T) { + val, err := rt.RunString(testCase.code) + if testCase.err == "" { + require.NoError(t, err) + got := new(expectedStatuses) + err = rt.ExportTo(val, &got) + require.NoError(t, err) + require.Equal(t, testCase.expected, *got) + return // the t.Run + } + + require.Error(t, err) + exc := err.(*goja.Exception) + require.Contains(t, exc.Error(), testCase.err) + }) + } +} + +type expectedSample struct { + tags map[string]string + metrics []*stats.Metric +} + +func TestResponseCallbackInAction(t *testing.T) { + t.Parallel() + tb, _, samples, rt, ctx := newRuntime(t) + defer tb.Cleanup() + sr := tb.Replacer.Replace + httpModule := new(GlobalHTTP).NewModuleInstancePerVU().(*HTTP) + rt.Set("http", common.Bind(rt, httpModule, ctx)) + + HTTPMetricsWithoutFailed := []*stats.Metric{ + metrics.HTTPReqs, + metrics.HTTPReqBlocked, + metrics.HTTPReqConnecting, + metrics.HTTPReqDuration, + metrics.HTTPReqReceiving, + metrics.HTTPReqWaiting, + metrics.HTTPReqSending, + metrics.HTTPReqTLSHandshaking, + } + + allHTTPMetrics := append(HTTPMetricsWithoutFailed, metrics.HTTPReqFailed) + + testCases := map[string]struct { + code string + expectedSamples []expectedSample + }{ + "basic": { + code: `http.request("GET", "HTTPBIN_URL/redirect/1");`, + expectedSamples: []expectedSample{ + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/redirect/1"), + "name": sr("HTTPBIN_URL/redirect/1"), + "status": "302", + "group": "", + "expected_response": "true", + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/get"), + "name": sr("HTTPBIN_URL/get"), + "status": "200", + "group": "", + "expected_response": "true", + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + }, + }, + "overwrite per request": { + code: ` + http.setResponseCallback(http.expectedStatuses(200)); + res = http.request("GET", "HTTPBIN_URL/redirect/1"); + `, + expectedSamples: []expectedSample{ + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/redirect/1"), + "name": sr("HTTPBIN_URL/redirect/1"), + "status": "302", + "group": "", + "expected_response": "false", // this is on purpose + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/get"), + "name": sr("HTTPBIN_URL/get"), + "status": "200", + "group": "", + "expected_response": "true", + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + }, + }, + + "global overwrite": { + code: `http.request("GET", "HTTPBIN_URL/redirect/1", null, {responseCallback: http.expectedStatuses(200)});`, + expectedSamples: []expectedSample{ + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/redirect/1"), + "name": sr("HTTPBIN_URL/redirect/1"), + "status": "302", + "group": "", + "expected_response": "false", // this is on purpose + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/get"), + "name": sr("HTTPBIN_URL/get"), + "status": "200", + "group": "", + "expected_response": "true", + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + }, + }, + "per request overwrite with null": { + code: `http.request("GET", "HTTPBIN_URL/redirect/1", null, {responseCallback: null});`, + expectedSamples: []expectedSample{ + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/redirect/1"), + "name": sr("HTTPBIN_URL/redirect/1"), + "status": "302", + "group": "", + "proto": "HTTP/1.1", + }, + metrics: HTTPMetricsWithoutFailed, + }, + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/get"), + "name": sr("HTTPBIN_URL/get"), + "status": "200", + "group": "", + "proto": "HTTP/1.1", + }, + metrics: HTTPMetricsWithoutFailed, + }, + }, + }, + "global overwrite with null": { + code: ` + http.setResponseCallback(null); + res = http.request("GET", "HTTPBIN_URL/redirect/1"); + `, + expectedSamples: []expectedSample{ + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/redirect/1"), + "name": sr("HTTPBIN_URL/redirect/1"), + "status": "302", + "group": "", + "proto": "HTTP/1.1", + }, + metrics: HTTPMetricsWithoutFailed, + }, + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/get"), + "name": sr("HTTPBIN_URL/get"), + "status": "200", + "group": "", + "proto": "HTTP/1.1", + }, + metrics: HTTPMetricsWithoutFailed, + }, + }, + }, + } + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + httpModule.responseCallback = defaultExpectedStatuses.match + + _, err := rt.RunString(sr(testCase.code)) + assert.NoError(t, err) + bufSamples := stats.GetBufferedSamples(samples) + + reqsCount := 0 + for _, container := range bufSamples { + for _, sample := range container.GetSamples() { + if sample.Metric.Name == "http_reqs" { + reqsCount++ + } + } + } + + require.Equal(t, len(testCase.expectedSamples), reqsCount) + + for i, expectedSample := range testCase.expectedSamples { + assertRequestMetricsEmittedSingle(t, bufSamples[i], expectedSample.tags, expectedSample.metrics, nil) + } + }) + } +} + +func TestResponseCallbackBatch(t *testing.T) { + t.Parallel() + tb, _, samples, rt, ctx := newRuntime(t) + defer tb.Cleanup() + sr := tb.Replacer.Replace + httpModule := new(GlobalHTTP).NewModuleInstancePerVU().(*HTTP) + rt.Set("http", common.Bind(rt, httpModule, ctx)) + + HTTPMetricsWithoutFailed := []*stats.Metric{ + metrics.HTTPReqs, + metrics.HTTPReqBlocked, + metrics.HTTPReqConnecting, + metrics.HTTPReqDuration, + metrics.HTTPReqReceiving, + metrics.HTTPReqWaiting, + metrics.HTTPReqSending, + metrics.HTTPReqTLSHandshaking, + } + + allHTTPMetrics := append(HTTPMetricsWithoutFailed, metrics.HTTPReqFailed) + // IMPORTANT: the tests here depend on the fact that the url they hit can be ordered in the same + // order as the expectedSamples even if they are made concurrently + testCases := map[string]struct { + code string + expectedSamples []expectedSample + }{ + "basic": { + code: ` + http.batch([["GET", "HTTPBIN_URL/status/200", null, {responseCallback: null}], + ["GET", "HTTPBIN_URL/status/201"], + ["GET", "HTTPBIN_URL/status/202", null, {responseCallback: http.expectedStatuses(4)}], + ["GET", "HTTPBIN_URL/status/405", null, {responseCallback: http.expectedStatuses(405)}], + ]);`, + expectedSamples: []expectedSample{ + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/status/200"), + "name": sr("HTTPBIN_URL/status/200"), + "status": "200", + "group": "", + "proto": "HTTP/1.1", + }, + metrics: HTTPMetricsWithoutFailed, + }, + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/status/201"), + "name": sr("HTTPBIN_URL/status/201"), + "status": "201", + "group": "", + "expected_response": "true", + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/status/202"), + "name": sr("HTTPBIN_URL/status/202"), + "status": "202", + "group": "", + "expected_response": "false", + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + { + tags: map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/status/405"), + "name": sr("HTTPBIN_URL/status/405"), + "status": "405", + "error_code": "1405", + "group": "", + "expected_response": "true", + "proto": "HTTP/1.1", + }, + metrics: allHTTPMetrics, + }, + }, + }, + } + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + httpModule.responseCallback = defaultExpectedStatuses.match + + _, err := rt.RunString(sr(testCase.code)) + assert.NoError(t, err) + bufSamples := stats.GetBufferedSamples(samples) + + reqsCount := 0 + for _, container := range bufSamples { + for _, sample := range container.GetSamples() { + if sample.Metric.Name == "http_reqs" { + reqsCount++ + } + } + } + sort.Slice(bufSamples, func(i, j int) bool { + iURL, _ := bufSamples[i].GetSamples()[0].Tags.Get("url") + jURL, _ := bufSamples[j].GetSamples()[0].Tags.Get("url") + return iURL < jURL + }) + + require.Equal(t, len(testCase.expectedSamples), reqsCount) + + for i, expectedSample := range testCase.expectedSamples { + assertRequestMetricsEmittedSingle(t, bufSamples[i], expectedSample.tags, expectedSample.metrics, nil) + } + }) + } +} + +func TestResponseCallbackInActionWithoutPassedTag(t *testing.T) { + t.Parallel() + tb, state, samples, rt, ctx := newRuntime(t) + defer tb.Cleanup() + sr := tb.Replacer.Replace + allHTTPMetrics := []*stats.Metric{ + metrics.HTTPReqs, + metrics.HTTPReqFailed, + metrics.HTTPReqBlocked, + metrics.HTTPReqConnecting, + metrics.HTTPReqDuration, + metrics.HTTPReqReceiving, + metrics.HTTPReqSending, + metrics.HTTPReqWaiting, + metrics.HTTPReqTLSHandshaking, + } + deleteSystemTag(state, stats.TagExpectedResponse.String()) + httpModule := new(GlobalHTTP).NewModuleInstancePerVU().(*HTTP) + rt.Set("http", common.Bind(rt, httpModule, ctx)) + + _, err := rt.RunString(sr(`http.request("GET", "HTTPBIN_URL/redirect/1", null, {responseCallback: http.expectedStatuses(200)});`)) + assert.NoError(t, err) + bufSamples := stats.GetBufferedSamples(samples) + + reqsCount := 0 + for _, container := range bufSamples { + for _, sample := range container.GetSamples() { + if sample.Metric.Name == "http_reqs" { + reqsCount++ + } + } + } + + require.Equal(t, 2, reqsCount) + + tags := map[string]string{ + "method": "GET", + "url": sr("HTTPBIN_URL/redirect/1"), + "name": sr("HTTPBIN_URL/redirect/1"), + "status": "302", + "group": "", + "proto": "HTTP/1.1", + } + assertRequestMetricsEmittedSingle(t, bufSamples[0], tags, allHTTPMetrics, func(sample stats.Sample) { + if sample.Metric.Name == metrics.HTTPReqFailed.Name { + require.EqualValues(t, sample.Value, 1) + } + }) + tags["url"] = sr("HTTPBIN_URL/get") + tags["name"] = tags["url"] + tags["status"] = "200" + assertRequestMetricsEmittedSingle(t, bufSamples[1], tags, allHTTPMetrics, func(sample stats.Sample) { + if sample.Metric.Name == metrics.HTTPReqFailed.Name { + require.EqualValues(t, sample.Value, 0) + } + }) +} + +func TestDigestWithResponseCallback(t *testing.T) { + t.Parallel() + tb, _, samples, rt, ctx := newRuntime(t) + defer tb.Cleanup() + + httpModule := new(GlobalHTTP).NewModuleInstancePerVU().(*HTTP) + rt.Set("http", common.Bind(rt, httpModule, ctx)) + + urlWithCreds := tb.Replacer.Replace( + "http://testuser:testpwd@HTTPBIN_IP:HTTPBIN_PORT/digest-auth/auth/testuser/testpwd", + ) + + allHTTPMetrics := []*stats.Metric{ + metrics.HTTPReqs, + metrics.HTTPReqFailed, + metrics.HTTPReqBlocked, + metrics.HTTPReqConnecting, + metrics.HTTPReqDuration, + metrics.HTTPReqReceiving, + metrics.HTTPReqSending, + metrics.HTTPReqWaiting, + metrics.HTTPReqTLSHandshaking, + } + _, err := rt.RunString(fmt.Sprintf(` + var res = http.get(%q, { auth: "digest" }); + if (res.status !== 200) { throw new Error("wrong status: " + res.status); } + if (res.error_code !== 0) { throw new Error("wrong error code: " + res.error_code); } + `, urlWithCreds)) + require.NoError(t, err) + bufSamples := stats.GetBufferedSamples(samples) + + reqsCount := 0 + for _, container := range bufSamples { + for _, sample := range container.GetSamples() { + if sample.Metric.Name == "http_reqs" { + reqsCount++ + } + } + } + + require.Equal(t, 2, reqsCount) + + urlRaw := tb.Replacer.Replace( + "http://HTTPBIN_IP:HTTPBIN_PORT/digest-auth/auth/testuser/testpwd") + + tags := map[string]string{ + "method": "GET", + "url": urlRaw, + "name": urlRaw, + "status": "401", + "group": "", + "proto": "HTTP/1.1", + "expected_response": "true", + "error_code": "1401", + } + assertRequestMetricsEmittedSingle(t, bufSamples[0], tags, allHTTPMetrics, func(sample stats.Sample) { + if sample.Metric.Name == metrics.HTTPReqFailed.Name { + require.EqualValues(t, sample.Value, 0) + } + }) + tags["status"] = "200" + delete(tags, "error_code") + assertRequestMetricsEmittedSingle(t, bufSamples[1], tags, allHTTPMetrics, func(sample stats.Sample) { + if sample.Metric.Name == metrics.HTTPReqFailed.Name { + require.EqualValues(t, sample.Value, 0) + } + }) +} + +func deleteSystemTag(state *lib.State, tag string) { + enabledTags := state.Options.SystemTags.Map() + delete(enabledTags, tag) + tagsList := make([]string, 0, len(enabledTags)) + for k := range enabledTags { + tagsList = append(tagsList, k) + } + state.Options.SystemTags = stats.ToSystemTagSet(tagsList) +} diff --git a/js/modules/k6/http/response_test.go b/js/modules/k6/http/response_test.go index dbce89e0a5b..6b346e11754 100644 --- a/js/modules/k6/http/response_test.go +++ b/js/modules/k6/http/response_test.go @@ -55,6 +55,7 @@ const testGetFormHTML = ` ` + const jsonData = `{"glossary": { "friends": [ {"first": "Dale", "last": "Murphy", "age": 44}, diff --git a/js/runner.go b/js/runner.go index 1d1afbe3a7d..9651c3e32b9 100644 --- a/js/runner.go +++ b/js/runner.go @@ -179,7 +179,7 @@ func (r *Runner) newVU(id int64, samplesOut chan<- stats.SampleContainer) (*VU, } tlsConfig := &tls.Config{ - InsecureSkipVerify: r.Bundle.Options.InsecureSkipTLSVerify.Bool, + InsecureSkipVerify: r.Bundle.Options.InsecureSkipTLSVerify.Bool, //nolint:gosec CipherSuites: cipherSuites, MinVersion: uint16(tlsVersions.Min), MaxVersion: uint16(tlsVersions.Max), @@ -244,6 +244,7 @@ func (r *Runner) newVU(id int64, samplesOut chan<- stats.SampleContainer) (*VU, return vu, nil } +// Setup runs the setup function if there is one and sets the setupData to the returned value func (r *Runner) Setup(ctx context.Context, out chan<- stats.SampleContainer) error { setupCtx, setupCancel := context.WithTimeout(ctx, r.getTimeoutFor(consts.SetupFn)) defer setupCancel() diff --git a/lib/metrics/metrics.go b/lib/metrics/metrics.go index cb7eda3b0b2..a47aa156eba 100644 --- a/lib/metrics/metrics.go +++ b/lib/metrics/metrics.go @@ -24,7 +24,7 @@ import ( "github.com/loadimpact/k6/stats" ) -//TODO: refactor this, using non thread-safe global variables seems like a bad idea for various reasons... +// TODO: refactor this, using non thread-safe global variables seems like a bad idea for various reasons... //nolint:gochecknoglobals var ( @@ -42,6 +42,7 @@ var ( // HTTP-related. HTTPReqs = stats.New("http_reqs", stats.Counter) + HTTPReqFailed = stats.New("http_req_failed", stats.Rate) HTTPReqDuration = stats.New("http_req_duration", stats.Trend, stats.Time) HTTPReqBlocked = stats.New("http_req_blocked", stats.Trend, stats.Time) HTTPReqConnecting = stats.New("http_req_connecting", stats.Trend, stats.Time) diff --git a/lib/netext/httpext/request.go b/lib/netext/httpext/request.go index 5642f0235d5..c527fe0b572 100644 --- a/lib/netext/httpext/request.go +++ b/lib/netext/httpext/request.go @@ -103,18 +103,19 @@ type Request struct { // ParsedHTTPRequest a represantion of a request after it has been parsed from a user script type ParsedHTTPRequest struct { - URL *URL - Body *bytes.Buffer - Req *http.Request - Timeout time.Duration - Auth string - Throw bool - ResponseType ResponseType - Compressions []CompressionType - Redirects null.Int - ActiveJar *cookiejar.Jar - Cookies map[string]*HTTPRequestCookie - Tags map[string]string + URL *URL + Body *bytes.Buffer + Req *http.Request + Timeout time.Duration + Auth string + Throw bool + ResponseType ResponseType + ResponseCallback func(int) bool + Compressions []CompressionType + Redirects null.Int + ActiveJar *cookiejar.Jar + Cookies map[string]*HTTPRequestCookie + Tags map[string]string } // Matches non-compliant io.Closer implementations (e.g. zstd.Decoder) @@ -139,7 +140,7 @@ func (r readCloser) Close() error { } func stdCookiesToHTTPRequestCookies(cookies []*http.Cookie) map[string][]*HTTPRequestCookie { - var result = make(map[string][]*HTTPRequestCookie, len(cookies)) + result := make(map[string][]*HTTPRequestCookie, len(cookies)) for _, cookie := range cookies { result[cookie.Name] = append(result[cookie.Name], &HTTPRequestCookie{Name: cookie.Name, Value: cookie.Value}) @@ -249,7 +250,7 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error } } - tracerTransport := newTransport(ctx, state, tags) + tracerTransport := newTransport(ctx, state, tags, preq.ResponseCallback) var transport http.RoundTripper = tracerTransport // Combine tags with common log fields @@ -269,8 +270,26 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error } if preq.Auth == "digest" { + // Until digest authentication is refactored, the first response will always + // be a 401 error, so we expect that. + if tracerTransport.responseCallback != nil { + originalResponseCallback := tracerTransport.responseCallback + tracerTransport.responseCallback = func(status int) bool { + tracerTransport.responseCallback = originalResponseCallback + return status == 401 + } + } transport = digestTransport{originalTransport: transport} } else if preq.Auth == "ntlm" { + // The first response of NTLM auth may be a 401 error. + if tracerTransport.responseCallback != nil { + originalResponseCallback := tracerTransport.responseCallback + tracerTransport.responseCallback = func(status int) bool { + tracerTransport.responseCallback = originalResponseCallback + // ntlm is connection-level based so we could've already authorized the connection and to now reuse it + return status == 401 || originalResponseCallback(status) + } + } transport = ntlmssp.Negotiator{RoundTripper: transport} } @@ -381,7 +400,7 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error // SetRequestCookies sets the cookies of the requests getting those cookies both from the jar and // from the reqCookies map. The Replace field of the HTTPRequestCookie will be taken into account func SetRequestCookies(req *http.Request, jar *cookiejar.Jar, reqCookies map[string]*HTTPRequestCookie) { - var replacedCookies = make(map[string]struct{}) + replacedCookies := make(map[string]struct{}) for key, reqCookie := range reqCookies { req.AddCookie(&http.Cookie{Name: key, Value: reqCookie.Value}) if reqCookie.Replace { diff --git a/lib/netext/httpext/tracer.go b/lib/netext/httpext/tracer.go index 60c02b548c7..a40968e760a 100644 --- a/lib/netext/httpext/tracer.go +++ b/lib/netext/httpext/tracer.go @@ -29,6 +29,7 @@ import ( "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/stats" + "gopkg.in/guregu/null.v3" ) // A Trail represents detailed information about an HTTP request. @@ -53,6 +54,7 @@ type Trail struct { ConnReused bool ConnRemoteAddr net.Addr + Failed null.Bool // Populated by SaveSamples() Tags *stats.SampleTags Samples []stats.Sample @@ -61,17 +63,17 @@ type Trail struct { // SaveSamples populates the Trail's sample slice so they're accesible via GetSamples() func (tr *Trail) SaveSamples(tags *stats.SampleTags) { tr.Tags = tags - tr.Samples = []stats.Sample{ + tr.Samples = make([]stats.Sample, 0, 9) // this is with 1 more for a possible HTTPReqFailed + tr.Samples = append(tr.Samples, []stats.Sample{ {Metric: metrics.HTTPReqs, Time: tr.EndTime, Tags: tags, Value: 1}, {Metric: metrics.HTTPReqDuration, Time: tr.EndTime, Tags: tags, Value: stats.D(tr.Duration)}, - {Metric: metrics.HTTPReqBlocked, Time: tr.EndTime, Tags: tags, Value: stats.D(tr.Blocked)}, {Metric: metrics.HTTPReqConnecting, Time: tr.EndTime, Tags: tags, Value: stats.D(tr.Connecting)}, {Metric: metrics.HTTPReqTLSHandshaking, Time: tr.EndTime, Tags: tags, Value: stats.D(tr.TLSHandshaking)}, {Metric: metrics.HTTPReqSending, Time: tr.EndTime, Tags: tags, Value: stats.D(tr.Sending)}, {Metric: metrics.HTTPReqWaiting, Time: tr.EndTime, Tags: tags, Value: stats.D(tr.Waiting)}, {Metric: metrics.HTTPReqReceiving, Time: tr.EndTime, Tags: tags, Value: stats.D(tr.Receiving)}, - } + }...) } // GetSamples implements the stats.SampleContainer interface. diff --git a/lib/netext/httpext/transport.go b/lib/netext/httpext/transport.go index bc754125f8b..6c77648b056 100644 --- a/lib/netext/httpext/transport.go +++ b/lib/netext/httpext/transport.go @@ -29,6 +29,7 @@ import ( "sync" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/netext" "github.com/loadimpact/k6/stats" ) @@ -36,9 +37,10 @@ import ( // transport is an implementation of http.RoundTripper that will measure and emit // different metrics for each roundtrip type transport struct { - ctx context.Context - state *lib.State - tags map[string]string + ctx context.Context + state *lib.State + tags map[string]string + responseCallback func(int) bool lastRequest *unfinishedRequest lastRequestLock *sync.Mutex @@ -76,17 +78,20 @@ func newTransport( ctx context.Context, state *lib.State, tags map[string]string, + responseCallback func(int) bool, ) *transport { return &transport{ - ctx: ctx, - state: state, - tags: tags, - lastRequestLock: new(sync.Mutex), + ctx: ctx, + state: state, + tags: tags, + responseCallback: responseCallback, + lastRequestLock: new(sync.Mutex), } } // Helper method to finish the tracer trail, assemble the tag values and emits // the metric samples for the supplied unfinished request. +//nolint:nestif,funlen func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRequest { trail := unfReq.tracer.Done() @@ -101,7 +106,6 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe } enabledTags := t.state.Options.SystemTags - urlEnabled := enabledTags.Has(stats.TagURL) var setName bool if _, ok := tags["name"]; !ok && enabledTags.Has(stats.TagName) { @@ -164,8 +168,35 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe tags["ip"] = ip } } + var failed float64 + if t.responseCallback != nil { + var statusCode int + if unfReq.response != nil { + statusCode = unfReq.response.StatusCode + } + expected := t.responseCallback(statusCode) + if !expected { + failed = 1 + } + + if enabledTags.Has(stats.TagExpectedResponse) { + tags[stats.TagExpectedResponse.String()] = strconv.FormatBool(expected) + } + } - trail.SaveSamples(stats.IntoSampleTags(&tags)) + finalTags := stats.IntoSampleTags(&tags) + trail.SaveSamples(finalTags) + if t.responseCallback != nil { + trail.Failed.SetValid(true) + if failed == 1 { + trail.Failed.Bool = true + } + trail.Samples = append(trail.Samples, + stats.Sample{ + Metric: metrics.HTTPReqFailed, Time: trail.EndTime, Tags: finalTags, Value: failed, + }, + ) + } stats.PushIfNotDone(t.ctx, t.state.Samples, trail) return result diff --git a/lib/netext/httpext/transport_test.go b/lib/netext/httpext/transport_test.go new file mode 100644 index 00000000000..76f8585f547 --- /dev/null +++ b/lib/netext/httpext/transport_test.go @@ -0,0 +1,86 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2021 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package httpext + +import ( + "context" + "net/http" + "net/url" + "testing" + + "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/stats" + "github.com/sirupsen/logrus" +) + +func BenchmarkMeasureAndEmitMetrics(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + samples := make(chan stats.SampleContainer, 10) + defer close(samples) + go func() { + for range samples { + } + }() + logger := logrus.New() + logger.Level = logrus.DebugLevel + + state := &lib.State{ + Options: lib.Options{ + RunTags: &stats.SampleTags{}, + SystemTags: &stats.DefaultSystemTagSet, + }, + Samples: samples, + Logger: logger, + } + t := transport{ + state: state, + ctx: ctx, + } + + b.ResetTimer() + unfRequest := &unfinishedRequest{ + tracer: &Tracer{}, + response: &http.Response{ + StatusCode: 200, + }, + request: &http.Request{ + URL: &url.URL{ + Host: "example.com", + Scheme: "https", + }, + }, + } + + b.Run("no responseCallback", func(b *testing.B) { + for i := 0; i < b.N; i++ { + t.measureAndEmitMetrics(unfRequest) + } + }) + + t.responseCallback = func(n int) bool { return true } + + b.Run("responseCallback", func(b *testing.B) { + for i := 0; i < b.N; i++ { + t.measureAndEmitMetrics(unfRequest) + } + }) +} diff --git a/stats/cloud/bench_test.go b/stats/cloud/bench_test.go index 27a674feb46..bd125189ad8 100644 --- a/stats/cloud/bench_test.go +++ b/stats/cloud/bench_test.go @@ -330,3 +330,33 @@ func BenchmarkHTTPPush(b *testing.B) { }) } } + +func BenchmarkNewSampleFromTrail(b *testing.B) { + tags := generateTags(1, 200) + now := time.Now() + trail := &httpext.Trail{ + Blocked: 200 * 100 * time.Millisecond, + Connecting: 200 * 200 * time.Millisecond, + TLSHandshaking: 200 * 300 * time.Millisecond, + Sending: 200 * 400 * time.Millisecond, + Waiting: 500 * time.Millisecond, + Receiving: 600 * time.Millisecond, + EndTime: now, + ConnDuration: 500 * time.Millisecond, + Duration: 150 * 1500 * time.Millisecond, + Tags: tags, + } + + b.Run("no failed", func(b *testing.B) { + for s := 0; s < b.N; s++ { + _ = NewSampleFromTrail(trail) + } + }) + trail.Failed = null.BoolFrom(true) + + b.Run("failed", func(b *testing.B) { + for s := 0; s < b.N; s++ { + _ = NewSampleFromTrail(trail) + } + }) +} diff --git a/stats/cloud/cloud_easyjson.go b/stats/cloud/cloud_easyjson.go index 7f412efc1fa..1a06e49c462 100644 --- a/stats/cloud/cloud_easyjson.go +++ b/stats/cloud/cloud_easyjson.go @@ -382,6 +382,7 @@ func easyjson9def2ecdDecode(in *jlexer.Lexer, out *struct { Sending AggregatedMetric `json:"http_req_sending"` Waiting AggregatedMetric `json:"http_req_waiting"` Receiving AggregatedMetric `json:"http_req_receiving"` + Failed AggregatedRate `json:"http_req_failed,omitempty"` }) { isTopLevel := in.IsStart() if in.IsNull() { @@ -415,6 +416,8 @@ func easyjson9def2ecdDecode(in *jlexer.Lexer, out *struct { easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud4(in, &out.Waiting) case "http_req_receiving": easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud4(in, &out.Receiving) + case "http_req_failed": + easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(in, &out.Failed) default: in.SkipRecursive() } @@ -433,6 +436,7 @@ func easyjson9def2ecdEncode(out *jwriter.Writer, in struct { Sending AggregatedMetric `json:"http_req_sending"` Waiting AggregatedMetric `json:"http_req_waiting"` Receiving AggregatedMetric `json:"http_req_receiving"` + Failed AggregatedRate `json:"http_req_failed,omitempty"` }) { out.RawByte('{') first := true @@ -472,6 +476,60 @@ func easyjson9def2ecdEncode(out *jwriter.Writer, in struct { out.RawString(prefix) easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud4(out, in.Receiving) } + if (in.Failed).IsDefined() { + const prefix string = ",\"http_req_failed\":" + out.RawString(prefix) + easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(out, in.Failed) + } + out.RawByte('}') +} +func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(in *jlexer.Lexer, out *AggregatedRate) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "count": + out.Count = float64(in.Float64()) + case "nz_count": + out.NzCount = float64(in.Float64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(out *jwriter.Writer, in AggregatedRate) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"count\":" + out.RawString(prefix[1:]) + out.Float64(float64(in.Count)) + } + { + const prefix string = ",\"nz_count\":" + out.RawString(prefix) + out.Float64(float64(in.NzCount)) + } out.RawByte('}') } func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud4(in *jlexer.Lexer, out *AggregatedMetric) { @@ -530,7 +588,7 @@ func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud4(out *jwriter.Writer, } out.RawByte('}') } -func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(in *jlexer.Lexer, out *Sample) { +func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud6(in *jlexer.Lexer, out *Sample) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -571,7 +629,7 @@ func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(in *jlexer.Lexer, ou in.Consumed() } } -func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(out *jwriter.Writer, in Sample) { +func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud6(out *jwriter.Writer, in Sample) { out.RawByte('{') first := true _ = first @@ -601,10 +659,10 @@ func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(out *jwriter.Writer, // MarshalEasyJSON supports easyjson.Marshaler interface func (v Sample) MarshalEasyJSON(w *jwriter.Writer) { - easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(w, v) + easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud6(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *Sample) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(l, v) + easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud6(l, v) } diff --git a/stats/cloud/data.go b/stats/cloud/data.go index 0a63372128b..ee6ce33e8f8 100644 --- a/stats/cloud/data.go +++ b/stats/cloud/data.go @@ -110,23 +110,30 @@ type SampleDataMap struct { // NewSampleFromTrail just creates a ready-to-send Sample instance // directly from a httpext.Trail. func NewSampleFromTrail(trail *httpext.Trail) *Sample { + length := 8 + if trail.Failed.Valid { + length++ + } + + values := make(map[string]float64, length) + values[metrics.HTTPReqs.Name] = 1 + values[metrics.HTTPReqDuration.Name] = stats.D(trail.Duration) + values[metrics.HTTPReqBlocked.Name] = stats.D(trail.Blocked) + values[metrics.HTTPReqConnecting.Name] = stats.D(trail.Connecting) + values[metrics.HTTPReqTLSHandshaking.Name] = stats.D(trail.TLSHandshaking) + values[metrics.HTTPReqSending.Name] = stats.D(trail.Sending) + values[metrics.HTTPReqWaiting.Name] = stats.D(trail.Waiting) + values[metrics.HTTPReqReceiving.Name] = stats.D(trail.Receiving) + if trail.Failed.Valid { // this is done so the adding of 1 map element doesn't reexpand the map as this is a hotpath + values[metrics.HTTPReqFailed.Name] = stats.B(trail.Failed.Bool) + } return &Sample{ Type: DataTypeMap, Metric: "http_req_li_all", Data: &SampleDataMap{ - Time: toMicroSecond(trail.GetTime()), - Tags: trail.GetTags(), - Values: map[string]float64{ - metrics.HTTPReqs.Name: 1, - metrics.HTTPReqDuration.Name: stats.D(trail.Duration), - - metrics.HTTPReqBlocked.Name: stats.D(trail.Blocked), - metrics.HTTPReqConnecting.Name: stats.D(trail.Connecting), - metrics.HTTPReqTLSHandshaking.Name: stats.D(trail.TLSHandshaking), - metrics.HTTPReqSending.Name: stats.D(trail.Sending), - metrics.HTTPReqWaiting.Name: stats.D(trail.Waiting), - metrics.HTTPReqReceiving.Name: stats.D(trail.Receiving), - }, + Time: toMicroSecond(trail.GetTime()), + Tags: trail.GetTags(), + Values: values, }, } } @@ -146,6 +153,7 @@ type SampleDataAggregatedHTTPReqs struct { Sending AggregatedMetric `json:"http_req_sending"` Waiting AggregatedMetric `json:"http_req_waiting"` Receiving AggregatedMetric `json:"http_req_receiving"` + Failed AggregatedRate `json:"http_req_failed,omitempty"` } `json:"values"` } @@ -159,6 +167,9 @@ func (sdagg *SampleDataAggregatedHTTPReqs) Add(trail *httpext.Trail) { sdagg.Values.Sending.Add(trail.Sending) sdagg.Values.Waiting.Add(trail.Waiting) sdagg.Values.Receiving.Add(trail.Receiving) + if trail.Failed.Valid { + sdagg.Values.Failed.Add(trail.Failed.Bool) + } } // CalcAverages calculates and sets all `Avg` properties in the `Values` struct @@ -173,6 +184,25 @@ func (sdagg *SampleDataAggregatedHTTPReqs) CalcAverages() { sdagg.Values.Receiving.Calc(count) } +// AggregatedRate is an aggregation of a Rate metric +type AggregatedRate struct { + Count float64 `json:"count"` + NzCount float64 `json:"nz_count"` +} + +// Add a boolean to the aggregated rate +func (ar *AggregatedRate) Add(b bool) { + ar.Count++ + if b { + ar.NzCount++ + } +} + +// IsDefined implements easyjson.Optional +func (ar AggregatedRate) IsDefined() bool { + return ar.Count != 0 +} + // AggregatedMetric is used to store aggregated information for a // particular metric in an SampleDataAggregatedMap. type AggregatedMetric struct { diff --git a/stats/cloud/data_test.go b/stats/cloud/data_test.go index d9fd5a58fe7..02c1c63535f 100644 --- a/stats/cloud/data_test.go +++ b/stats/cloud/data_test.go @@ -29,6 +29,7 @@ import ( "github.com/mailru/easyjson" "github.com/stretchr/testify/assert" + "gopkg.in/guregu/null.v3" "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/netext/httpext" @@ -87,6 +88,105 @@ func TestSampleMarshaling(t *testing.T) { }), fmt.Sprintf(`{"type":"Points","metric":"http_req_li_all","data":{"time":"%d","type":"counter","values":{"http_req_blocked":0.001,"http_req_connecting":0.002,"http_req_duration":0.123,"http_req_receiving":0.006,"http_req_sending":0.004,"http_req_tls_handshaking":0.003,"http_req_waiting":0.005,"http_reqs":1}}}`, exptoMicroSecond), }, + { + NewSampleFromTrail(&httpext.Trail{ + EndTime: now, + Duration: 123000, + Blocked: 1000, + Connecting: 2000, + TLSHandshaking: 3000, + Sending: 4000, + Waiting: 5000, + Receiving: 6000, + Failed: null.NewBool(false, true), + }), + fmt.Sprintf(`{"type":"Points","metric":"http_req_li_all","data":{"time":"%d","type":"counter","values":{"http_req_blocked":0.001,"http_req_connecting":0.002,"http_req_duration":0.123,"http_req_failed":0,"http_req_receiving":0.006,"http_req_sending":0.004,"http_req_tls_handshaking":0.003,"http_req_waiting":0.005,"http_reqs":1}}}`, exptoMicroSecond), + }, + { + func() *Sample { + aggrData := &SampleDataAggregatedHTTPReqs{ + Time: exptoMicroSecond, + Type: "aggregated_trend", + Tags: stats.IntoSampleTags(&map[string]string{"test": "mest"}), + } + aggrData.Add( + &httpext.Trail{ + EndTime: now, + Duration: 123000, + Blocked: 1000, + Connecting: 2000, + TLSHandshaking: 3000, + Sending: 4000, + Waiting: 5000, + Receiving: 6000, + }, + ) + + aggrData.Add( + &httpext.Trail{ + EndTime: now, + Duration: 13000, + Blocked: 3000, + Connecting: 1000, + TLSHandshaking: 4000, + Sending: 5000, + Waiting: 8000, + Receiving: 8000, + }, + ) + aggrData.CalcAverages() + + return &Sample{ + Type: DataTypeAggregatedHTTPReqs, + Metric: "http_req_li_all", + Data: aggrData, + } + }(), + fmt.Sprintf(`{"type":"AggregatedPoints","metric":"http_req_li_all","data":{"time":"%d","type":"aggregated_trend","count":2,"tags":{"test":"mest"},"values":{"http_req_duration":{"min":0.013,"max":0.123,"avg":0.068},"http_req_blocked":{"min":0.001,"max":0.003,"avg":0.002},"http_req_connecting":{"min":0.001,"max":0.002,"avg":0.0015},"http_req_tls_handshaking":{"min":0.003,"max":0.004,"avg":0.0035},"http_req_sending":{"min":0.004,"max":0.005,"avg":0.0045},"http_req_waiting":{"min":0.005,"max":0.008,"avg":0.0065},"http_req_receiving":{"min":0.006,"max":0.008,"avg":0.007}}}}`, exptoMicroSecond), + }, + { + func() *Sample { + aggrData := &SampleDataAggregatedHTTPReqs{ + Time: exptoMicroSecond, + Type: "aggregated_trend", + Tags: stats.IntoSampleTags(&map[string]string{"test": "mest"}), + } + aggrData.Add( + &httpext.Trail{ + EndTime: now, + Duration: 123000, + Blocked: 1000, + Connecting: 2000, + TLSHandshaking: 3000, + Sending: 4000, + Waiting: 5000, + Receiving: 6000, + Failed: null.BoolFrom(false), + }, + ) + + aggrData.Add( + &httpext.Trail{ + EndTime: now, + Duration: 13000, + Blocked: 3000, + Connecting: 1000, + TLSHandshaking: 4000, + Sending: 5000, + Waiting: 8000, + Receiving: 8000, + }, + ) + aggrData.CalcAverages() + + return &Sample{ + Type: DataTypeAggregatedHTTPReqs, + Metric: "http_req_li_all", + Data: aggrData, + } + }(), + fmt.Sprintf(`{"type":"AggregatedPoints","metric":"http_req_li_all","data":{"time":"%d","type":"aggregated_trend","count":2,"tags":{"test":"mest"},"values":{"http_req_duration":{"min":0.013,"max":0.123,"avg":0.068},"http_req_blocked":{"min":0.001,"max":0.003,"avg":0.002},"http_req_connecting":{"min":0.001,"max":0.002,"avg":0.0015},"http_req_tls_handshaking":{"min":0.003,"max":0.004,"avg":0.0035},"http_req_sending":{"min":0.004,"max":0.005,"avg":0.0045},"http_req_waiting":{"min":0.005,"max":0.008,"avg":0.0065},"http_req_receiving":{"min":0.006,"max":0.008,"avg":0.007},"http_req_failed":{"count":1,"nz_count":0}}}}`, exptoMicroSecond), + }, } for _, tc := range testCases { @@ -140,6 +240,7 @@ func getDurations(count int, min, multiplier float64) durations { } return data } + func BenchmarkDurationBounds(b *testing.B) { iqrRadius := 0.25 // If it's something different, the Q in IQR won't make much sense... iqrLowerCoef := 1.5 @@ -223,7 +324,6 @@ func TestQuickSelectAndBounds(t *testing.T) { assert.Equal(t, dataForSort[k], data.quickSelect(k)) }) } - }) } } diff --git a/stats/system_tag.go b/stats/system_tag.go index c549e074509..1cf8ee0a96f 100644 --- a/stats/system_tag.go +++ b/stats/system_tag.go @@ -97,6 +97,7 @@ const ( TagTLSVersion TagScenario TagService + TagExpectedResponse // System tags not enabled by default. TagIter @@ -109,7 +110,7 @@ const ( // Other tags that are not enabled by default include: iter, vu, ocsp_status, ip //nolint:gochecknoglobals var DefaultSystemTagSet = TagProto | TagSubproto | TagStatus | TagMethod | TagURL | TagName | TagGroup | - TagCheck | TagCheck | TagError | TagErrorCode | TagTLSVersion | TagScenario | TagService + TagCheck | TagCheck | TagError | TagErrorCode | TagTLSVersion | TagScenario | TagService | TagExpectedResponse // Add adds a tag to tag set. func (i *SystemTagSet) Add(tag SystemTagSet) { diff --git a/stats/system_tag_set_gen.go b/stats/system_tag_set_gen.go index a8ccc3d223d..b65604d01ec 100644 --- a/stats/system_tag_set_gen.go +++ b/stats/system_tag_set_gen.go @@ -7,26 +7,27 @@ import ( "fmt" ) -const _SystemTagSetName = "protosubprotostatusmethodurlnamegroupcheckerrorerror_codetls_versionscenarioserviceitervuocsp_statusip" +const _SystemTagSetName = "protosubprotostatusmethodurlnamegroupcheckerrorerror_codetls_versionscenarioserviceexpected_responseitervuocsp_statusip" var _SystemTagSetMap = map[SystemTagSet]string{ - 1: _SystemTagSetName[0:5], - 2: _SystemTagSetName[5:13], - 4: _SystemTagSetName[13:19], - 8: _SystemTagSetName[19:25], - 16: _SystemTagSetName[25:28], - 32: _SystemTagSetName[28:32], - 64: _SystemTagSetName[32:37], - 128: _SystemTagSetName[37:42], - 256: _SystemTagSetName[42:47], - 512: _SystemTagSetName[47:57], - 1024: _SystemTagSetName[57:68], - 2048: _SystemTagSetName[68:76], - 4096: _SystemTagSetName[76:83], - 8192: _SystemTagSetName[83:87], - 16384: _SystemTagSetName[87:89], - 32768: _SystemTagSetName[89:100], - 65536: _SystemTagSetName[100:102], + 1: _SystemTagSetName[0:5], + 2: _SystemTagSetName[5:13], + 4: _SystemTagSetName[13:19], + 8: _SystemTagSetName[19:25], + 16: _SystemTagSetName[25:28], + 32: _SystemTagSetName[28:32], + 64: _SystemTagSetName[32:37], + 128: _SystemTagSetName[37:42], + 256: _SystemTagSetName[42:47], + 512: _SystemTagSetName[47:57], + 1024: _SystemTagSetName[57:68], + 2048: _SystemTagSetName[68:76], + 4096: _SystemTagSetName[76:83], + 8192: _SystemTagSetName[83:100], + 16384: _SystemTagSetName[100:104], + 32768: _SystemTagSetName[104:106], + 65536: _SystemTagSetName[106:117], + 131072: _SystemTagSetName[117:119], } func (i SystemTagSet) String() string { @@ -36,7 +37,7 @@ func (i SystemTagSet) String() string { return fmt.Sprintf("SystemTagSet(%d)", i) } -var _SystemTagSetValues = []SystemTagSet{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536} +var _SystemTagSetValues = []SystemTagSet{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072} var _SystemTagSetNameToValueMap = map[string]SystemTagSet{ _SystemTagSetName[0:5]: 1, @@ -52,10 +53,11 @@ var _SystemTagSetNameToValueMap = map[string]SystemTagSet{ _SystemTagSetName[57:68]: 1024, _SystemTagSetName[68:76]: 2048, _SystemTagSetName[76:83]: 4096, - _SystemTagSetName[83:87]: 8192, - _SystemTagSetName[87:89]: 16384, - _SystemTagSetName[89:100]: 32768, - _SystemTagSetName[100:102]: 65536, + _SystemTagSetName[83:100]: 8192, + _SystemTagSetName[100:104]: 16384, + _SystemTagSetName[104:106]: 32768, + _SystemTagSetName[106:117]: 65536, + _SystemTagSetName[117:119]: 131072, } // SystemTagSetString retrieves an enum value from the enum constants string name. diff --git a/stats/units.go b/stats/units.go index 4380f2bc6e9..e414f8d20d7 100644 --- a/stats/units.go +++ b/stats/units.go @@ -37,3 +37,11 @@ func D(d time.Duration) float64 { func ToD(d float64) time.Duration { return time.Duration(d * float64(timeUnit)) } + +// B formats a boolean value for emission. +func B(b bool) float64 { + if b { + return 1 + } + return 0 +}