From 81cccd996555d146a9323a4adfb6928ddfcabf9d Mon Sep 17 00:00:00 2001 From: oleiade Date: Wed, 17 Nov 2021 09:07:33 +0100 Subject: [PATCH] Remove the JS runtime from threshold calculations In this commit we replace the previously existing thresholds condition evaluation, which was depending on Goja's Runtime, with a new pure-Go one. Thresholds are now parsed, and evaluated in o, and no JS rutimes are involved in the process anymore. It is built upong the thresholds parser, and parser combinators library introduced in previous commits. --- core/engine_test.go | 29 +-- stats/thresholds.go | 181 ++++++++++------- stats/thresholds_test.go | 411 +++++++++++++++++++++++++++++---------- 3 files changed, 441 insertions(+), 180 deletions(-) diff --git a/core/engine_test.go b/core/engine_test.go index c9c60166029..7a4dc20d095 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -258,7 +258,7 @@ func TestEngine_processSamples(t *testing.T) { }) t.Run("submetric", func(t *testing.T) { t.Parallel() - ths, err := stats.NewThresholds([]string{`1+1==2`}) + ths, err := stats.NewThresholds([]string{`value<2`}) assert.NoError(t, err) e, _, wait := newTestEngine(t, nil, nil, nil, lib.Options{ @@ -286,7 +286,10 @@ func TestEngineThresholdsWillAbort(t *testing.T) { t.Parallel() metric := stats.New("my_metric", stats.Gauge) - ths, err := stats.NewThresholds([]string{"1+1==3"}) + // The incoming samples for the metric set it to 1.25. Considering + // the metric is of type Gauge, value > 1.25 should always fail, and + // trigger an abort. + ths, err := stats.NewThresholds([]string{"value>1.25"}) assert.NoError(t, err) ths.Thresholds[0].AbortOnFail = true @@ -305,7 +308,11 @@ func TestEngineAbortedByThresholds(t *testing.T) { t.Parallel() metric := stats.New("my_metric", stats.Gauge) - ths, err := stats.NewThresholds([]string{"1+1==3"}) + // The MiniRunner sets the value of the metric to 1.25. Considering + // the metric is of type Gauge, value > 1.25 should always fail, and + // trigger an abort. + // **N.B**: a threshold returning an error, won't trigger an abort. + ths, err := stats.NewThresholds([]string{"value>1.25"}) assert.NoError(t, err) ths.Thresholds[0].AbortOnFail = true @@ -343,14 +350,14 @@ func TestEngine_processThresholds(t *testing.T) { ths map[string][]string abort bool }{ - "passing": {true, map[string][]string{"my_metric": {"1+1==2"}}, false}, - "failing": {false, map[string][]string{"my_metric": {"1+1==3"}}, false}, - "aborting": {false, map[string][]string{"my_metric": {"1+1==3"}}, true}, - - "submetric,match,passing": {true, map[string][]string{"my_metric{a:1}": {"1+1==2"}}, false}, - "submetric,match,failing": {false, map[string][]string{"my_metric{a:1}": {"1+1==3"}}, false}, - "submetric,nomatch,passing": {true, map[string][]string{"my_metric{a:2}": {"1+1==2"}}, false}, - "submetric,nomatch,failing": {true, map[string][]string{"my_metric{a:2}": {"1+1==3"}}, false}, + "passing": {true, map[string][]string{"my_metric": {"value<2"}}, false}, + "failing": {false, map[string][]string{"my_metric": {"value>1.25"}}, false}, + "aborting": {false, map[string][]string{"my_metric": {"value>1.25"}}, true}, + + "submetric,match,passing": {true, map[string][]string{"my_metric{a:1}": {"value<2"}}, false}, + "submetric,match,failing": {false, map[string][]string{"my_metric{a:1}": {"value>1.25"}}, false}, + "submetric,nomatch,passing": {true, map[string][]string{"my_metric{a:2}": {"value<2"}}, false}, + "submetric,nomatch,failing": {true, map[string][]string{"my_metric{a:2}": {"value>1.25"}}, false}, } for name, data := range testdata { diff --git a/stats/thresholds.go b/stats/thresholds.go index bb3d58c4ebc..1b7cbaea53f 100644 --- a/stats/thresholds.go +++ b/stats/thresholds.go @@ -17,54 +17,35 @@ * along with this program. If not, see . * */ - package stats import ( "bytes" "encoding/json" "fmt" + "strings" "time" - "github.com/dop251/goja" - "go.k6.io/k6/lib/types" ) -const jsEnvSrc = ` -function p(pct) { - return __sink__.P(pct/100.0); -}; -` - -var jsEnv *goja.Program - -func init() { - pgm, err := goja.Compile("__env__", jsEnvSrc, true) - if err != nil { - panic(err) - } - jsEnv = pgm -} - // Threshold is a representation of a single threshold for a single metric type Threshold struct { // Source is the text based source of the threshold Source string - // LastFailed is a makrer if the last testing of this threshold failed + // LastFailed is a marker if the last testing of this threshold failed LastFailed bool // AbortOnFail marks if a given threshold fails that the whole test should be aborted AbortOnFail bool // AbortGracePeriod is a the minimum amount of time a test should be running before a failing // this threshold will abort the test AbortGracePeriod types.NullDuration - - pgm *goja.Program - rt *goja.Runtime + // parsed is the threshold expression parsed from the Source + parsed *thresholdExpression } -func newThreshold(src string, newThreshold *goja.Runtime, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) { - pgm, err := goja.Compile("__threshold__", src, true) +func newThreshold(src string, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) { + parsedExpression, err := parseThresholdExpression(src) if err != nil { return nil, err } @@ -73,23 +54,57 @@ func newThreshold(src string, newThreshold *goja.Runtime, abortOnFail bool, grac Source: src, AbortOnFail: abortOnFail, AbortGracePeriod: gracePeriod, - pgm: pgm, - rt: newThreshold, + parsed: parsedExpression, }, nil } -func (t Threshold) runNoTaint() (bool, error) { - v, err := t.rt.RunProgram(t.pgm) - if err != nil { - return false, err +func (t *Threshold) runNoTaint(sinks map[string]float64) (bool, error) { + // Extract the sink value for the aggregation method used in the threshold + // expression + lhs, ok := sinks[t.parsed.AggregationMethod] + if !ok { + return false, fmt.Errorf("unable to apply threshold %s over metrics; reason: "+ + "no metric supporting the %s aggregation method found", + t.Source, + t.parsed.AggregationMethod) } - return v.ToBoolean(), nil + + // Apply the threshold expression operator to the left and + // right hand side values + var passes bool + switch t.parsed.Operator { + case ">": + passes = lhs > t.parsed.Value + case ">=": + passes = lhs >= t.parsed.Value + case "<=": + passes = lhs <= t.parsed.Value + case "<": + passes = lhs < t.parsed.Value + case "==", "===": + // Considering a sink always maps to float64 values, + // strictly equal is equivalent to loosely equal + passes = lhs == t.parsed.Value + case "!=": + passes = lhs != t.parsed.Value + default: + // The parseThresholdExpression function should ensure that no invalid + // operator gets through, but let's protect our future selves anyhow. + return false, fmt.Errorf("unable to apply threshold %s over metrics; "+ + "reason: %s is an invalid operator", + t.Source, + t.parsed.Operator, + ) + } + + // Perform the actual threshold verification + return passes, nil } -func (t *Threshold) run() (bool, error) { - b, err := t.runNoTaint() - t.LastFailed = !b - return b, err +func (t *Threshold) run(sinks map[string]float64) (bool, error) { + passes, err := t.runNoTaint(sinks) + t.LastFailed = !passes + return passes, err } type thresholdConfig struct { @@ -98,11 +113,11 @@ type thresholdConfig struct { AbortGracePeriod types.NullDuration `json:"delayAbortEval"` } -//used internally for JSON marshalling +// used internally for JSON marshalling type rawThresholdConfig thresholdConfig func (tc *thresholdConfig) UnmarshalJSON(data []byte) error { - //shortcircuit unmarshalling for simple string format + // shortcircuit unmarshalling for simple string format if err := json.Unmarshal(data, &tc.Threshold); err == nil { return nil } @@ -122,9 +137,9 @@ func (tc thresholdConfig) MarshalJSON() ([]byte, error) { // Thresholds is the combination of all Thresholds for a given metric type Thresholds struct { - Runtime *goja.Runtime Thresholds []*Threshold Abort bool + sinked map[string]float64 } // NewThresholds returns Thresholds objects representing the provided source strings @@ -138,60 +153,88 @@ func NewThresholds(sources []string) (Thresholds, error) { } func newThresholdsWithConfig(configs []thresholdConfig) (Thresholds, error) { - rt := goja.New() - if _, err := rt.RunProgram(jsEnv); err != nil { - return Thresholds{}, fmt.Errorf("threshold builtin error: %w", err) - } + thresholds := make([]*Threshold, len(configs)) + sinked := make(map[string]float64) - ts := make([]*Threshold, len(configs)) for i, config := range configs { - t, err := newThreshold(config.Threshold, rt, config.AbortOnFail, config.AbortGracePeriod) + t, err := newThreshold(config.Threshold, config.AbortOnFail, config.AbortGracePeriod) if err != nil { return Thresholds{}, fmt.Errorf("threshold %d error: %w", i, err) } - ts[i] = t + thresholds[i] = t } - return Thresholds{rt, ts, false}, nil + return Thresholds{thresholds, false, sinked}, nil } -func (ts *Thresholds) updateVM(sink Sink, t time.Duration) error { - ts.Runtime.Set("__sink__", sink) - f := sink.Format(t) - for k, v := range f { - ts.Runtime.Set(k, v) - } - return nil -} - -func (ts *Thresholds) runAll(t time.Duration) (bool, error) { - succ := true - for i, th := range ts.Thresholds { - b, err := th.run() +func (ts *Thresholds) runAll(timeSpentInTest time.Duration) (bool, error) { + succeeded := true + for i, threshold := range ts.Thresholds { + b, err := threshold.run(ts.sinked) if err != nil { return false, fmt.Errorf("threshold %d run error: %w", i, err) } + if !b { - succ = false + succeeded = false - if ts.Abort || !th.AbortOnFail { + if ts.Abort || !threshold.AbortOnFail { continue } - ts.Abort = !th.AbortGracePeriod.Valid || - th.AbortGracePeriod.Duration < types.Duration(t) + ts.Abort = !threshold.AbortGracePeriod.Valid || + threshold.AbortGracePeriod.Duration < types.Duration(timeSpentInTest) } } - return succ, nil + + return succeeded, nil } // Run processes all the thresholds with the provided Sink at the provided time and returns if any // of them fails -func (ts *Thresholds) Run(sink Sink, t time.Duration) (bool, error) { - if err := ts.updateVM(sink, t); err != nil { - return false, err +func (ts *Thresholds) Run(sink Sink, duration time.Duration) (bool, error) { + // Initialize the sinks store + ts.sinked = make(map[string]float64) + + // FIXME: Remove this comment as soon as the stats.Sink does not expose Format anymore. + // + // As of December 2021, this block reproduces the behavior of the + // stats.Sink.Format behavior. As we intend to try to get away from it, + // we instead implement the behavior directly here. + // + // For more details, see https://github.com/grafana/k6/issues/2320 + switch sinkImpl := sink.(type) { + case *CounterSink: + ts.sinked["count"] = sinkImpl.Value + ts.sinked["rate"] = sinkImpl.Value / (float64(duration) / float64(time.Second)) + case *GaugeSink: + ts.sinked["value"] = sinkImpl.Value + case *TrendSink: + ts.sinked["min"] = sinkImpl.Min + ts.sinked["max"] = sinkImpl.Max + ts.sinked["avg"] = sinkImpl.Avg + ts.sinked["med"] = sinkImpl.Med + + // Parse the percentile thresholds and insert them in + // the sinks mapping. + for _, threshold := range ts.Thresholds { + if !strings.HasPrefix(threshold.parsed.AggregationMethod, "p(") { + continue + } + + ts.sinked[threshold.parsed.AggregationMethod] = sinkImpl.P(threshold.parsed.AggregationValue.Float64 / 100) + } + case *RateSink: + ts.sinked["rate"] = float64(sinkImpl.Trues) / float64(sinkImpl.Total) + case DummySink: + for k, v := range sinkImpl { + ts.sinked[k] = v + } + default: + return false, fmt.Errorf("unable to run Thresholds; reason: unknown sink type") } - return ts.runAll(t) + + return ts.runAll(duration) } // UnmarshalJSON is implementation of json.Unmarshaler diff --git a/stats/thresholds_test.go b/stats/thresholds_test.go index 4d06dd0f05f..9e381bf75fb 100644 --- a/stats/thresholds_test.go +++ b/stats/thresholds_test.go @@ -25,76 +25,257 @@ import ( "testing" "time" - "github.com/dop251/goja" "github.com/stretchr/testify/assert" - + "github.com/stretchr/testify/require" "go.k6.io/k6/lib/types" + "gopkg.in/guregu/null.v3" ) func TestNewThreshold(t *testing.T) { - src := `1+1==2` - rt := goja.New() + t.Parallel() + + src := `rate<0.01` abortOnFail := false gracePeriod := types.NullDurationFrom(2 * time.Second) - th, err := newThreshold(src, rt, abortOnFail, gracePeriod) + wantParsed := &thresholdExpression{tokenRate, null.Float{}, tokenLess, 0.01} + + gotThreshold, err := newThreshold(src, abortOnFail, gracePeriod) + assert.NoError(t, err) + assert.Equal(t, src, gotThreshold.Source) + assert.False(t, gotThreshold.LastFailed) + assert.Equal(t, abortOnFail, gotThreshold.AbortOnFail) + assert.Equal(t, gracePeriod, gotThreshold.AbortGracePeriod) + assert.Equal(t, wantParsed, gotThreshold.parsed) +} + +func TestNewThreshold_InvalidThresholdConditionExpression(t *testing.T) { + t.Parallel() + + src := "1+1==2" + abortOnFail := false + gracePeriod := types.NullDurationFrom(2 * time.Second) - assert.Equal(t, src, th.Source) - assert.False(t, th.LastFailed) - assert.NotNil(t, th.pgm) - assert.Equal(t, rt, th.rt) - assert.Equal(t, abortOnFail, th.AbortOnFail) - assert.Equal(t, gracePeriod, th.AbortGracePeriod) + gotThreshold, err := newThreshold(src, abortOnFail, gracePeriod) + + assert.Error(t, err, "instantiating a threshold with an invalid expression should fail") + assert.Nil(t, gotThreshold, "instantiating a threshold with an invalid expression should return a nil Threshold") +} + +func TestThreshold_runNoTaint(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + parsed *thresholdExpression + abortGracePeriod types.NullDuration + sinks map[string]float64 + wantOk bool + wantErr bool + }{ + { + name: "valid expression using the > operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 1}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the > operator over passing threshold and defined abort grace period", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + abortGracePeriod: types.NullDurationFrom(2 * time.Second), + sinks: map[string]float64{"rate": 1}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the >= operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreaterEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.01}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the <= operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLessEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.01}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the < operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLess, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.00001}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the == operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLooselyEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.01}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the === operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenStrictlyEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.01}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using != operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenBangEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.02}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression over failing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.00001}, + wantOk: false, + wantErr: false, + }, + { + name: "valid expression over non-existing sink", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"med": 27.2}, + wantOk: false, + wantErr: true, + }, + { + // The ParseThresholdCondition constructor should ensure that no invalid + // operator gets through, but let's protect our future selves anyhow. + name: "invalid expression operator", + parsed: &thresholdExpression{tokenRate, null.Float{}, "&", 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.00001}, + wantOk: false, + wantErr: true, + }, + } + for _, testCase := range tests { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + threshold := &Threshold{ + LastFailed: false, + AbortOnFail: false, + AbortGracePeriod: testCase.abortGracePeriod, + parsed: testCase.parsed, + } + + gotOk, gotErr := threshold.runNoTaint(testCase.sinks) + + assert.Equal(t, + testCase.wantErr, + gotErr != nil, + "Threshold.runNoTaint() error = %v, wantErr %v", gotErr, testCase.wantErr, + ) + + assert.Equal(t, + testCase.wantOk, + gotOk, + "Threshold.runNoTaint() gotOk = %v, want %v", gotOk, testCase.wantOk, + ) + }) + } +} + +func BenchmarkRunNoTaint(b *testing.B) { + threshold := &Threshold{ + Source: "rate>0.01", + LastFailed: false, + AbortOnFail: false, + AbortGracePeriod: types.NullDurationFrom(2 * time.Second), + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + } + + sinks := map[string]float64{"rate": 1} + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + threshold.runNoTaint(sinks) // nolint + } } func TestThresholdRun(t *testing.T) { + t.Parallel() + t.Run("true", func(t *testing.T) { - th, err := newThreshold(`1+1==2`, goja.New(), false, types.NullDuration{}) + t.Parallel() + + sinks := map[string]float64{"rate": 0.0001} + threshold, err := newThreshold(`rate<0.01`, false, types.NullDuration{}) assert.NoError(t, err) t.Run("no taint", func(t *testing.T) { - b, err := th.runNoTaint() + b, err := threshold.runNoTaint(sinks) assert.NoError(t, err) assert.True(t, b) - assert.False(t, th.LastFailed) + assert.False(t, threshold.LastFailed) }) t.Run("taint", func(t *testing.T) { - b, err := th.run() + t.Parallel() + + b, err := threshold.run(sinks) assert.NoError(t, err) assert.True(t, b) - assert.False(t, th.LastFailed) + assert.False(t, threshold.LastFailed) }) }) t.Run("false", func(t *testing.T) { - th, err := newThreshold(`1+1==4`, goja.New(), false, types.NullDuration{}) + t.Parallel() + + sinks := map[string]float64{"rate": 1} + threshold, err := newThreshold(`rate<0.01`, false, types.NullDuration{}) assert.NoError(t, err) t.Run("no taint", func(t *testing.T) { - b, err := th.runNoTaint() + b, err := threshold.runNoTaint(sinks) assert.NoError(t, err) assert.False(t, b) - assert.False(t, th.LastFailed) + assert.False(t, threshold.LastFailed) }) t.Run("taint", func(t *testing.T) { - b, err := th.run() + b, err := threshold.run(sinks) assert.NoError(t, err) assert.False(t, b) - assert.True(t, th.LastFailed) + assert.True(t, threshold.LastFailed) }) }) } func TestNewThresholds(t *testing.T) { + t.Parallel() + t.Run("empty", func(t *testing.T) { + t.Parallel() + ts, err := NewThresholds([]string{}) assert.NoError(t, err) assert.Len(t, ts.Thresholds, 0) }) t.Run("two", func(t *testing.T) { - sources := []string{`1+1==2`, `1+1==4`} + t.Parallel() + + sources := []string{`rate<0.01`, `p(95)<200`} ts, err := NewThresholds(sources) assert.NoError(t, err) assert.Len(t, ts.Thresholds, 2) @@ -102,22 +283,26 @@ func TestNewThresholds(t *testing.T) { assert.Equal(t, sources[i], th.Source) assert.False(t, th.LastFailed) assert.False(t, th.AbortOnFail) - assert.NotNil(t, th.pgm) - assert.Equal(t, ts.Runtime, th.rt) } }) } func TestNewThresholdsWithConfig(t *testing.T) { + t.Parallel() + t.Run("empty", func(t *testing.T) { + t.Parallel() + ts, err := NewThresholds([]string{}) assert.NoError(t, err) assert.Len(t, ts.Thresholds, 0) }) t.Run("two", func(t *testing.T) { + t.Parallel() + configs := []thresholdConfig{ - {`1+1==2`, false, types.NullDuration{}}, - {`1+1==4`, true, types.NullDuration{}}, + {`rate<0.01`, false, types.NullDuration{}}, + {`p(95)<200`, true, types.NullDuration{}}, } ts, err := newThresholdsWithConfig(configs) assert.NoError(t, err) @@ -126,53 +311,47 @@ func TestNewThresholdsWithConfig(t *testing.T) { assert.Equal(t, configs[i].Threshold, th.Source) assert.False(t, th.LastFailed) assert.Equal(t, configs[i].AbortOnFail, th.AbortOnFail) - assert.NotNil(t, th.pgm) - assert.Equal(t, ts.Runtime, th.rt) } }) } -func TestThresholdsUpdateVM(t *testing.T) { - ts, err := NewThresholds(nil) - assert.NoError(t, err) - assert.NoError(t, ts.updateVM(DummySink{"a": 1234.5}, 0)) - assert.Equal(t, 1234.5, ts.Runtime.Get("a").ToFloat()) -} - func TestThresholdsRunAll(t *testing.T) { + t.Parallel() + zero := types.NullDuration{} oneSec := types.NullDurationFrom(time.Second) twoSec := types.NullDurationFrom(2 * time.Second) testdata := map[string]struct { - succ bool - err bool - abort bool - grace types.NullDuration - srcs []string + succeeded bool + err bool + abort bool + grace types.NullDuration + sources []string }{ - "one passing": {true, false, false, zero, []string{`1+1==2`}}, - "one failing": {false, false, false, zero, []string{`1+1==4`}}, - "two passing": {true, false, false, zero, []string{`1+1==2`, `2+2==4`}}, - "two failing": {false, false, false, zero, []string{`1+1==4`, `2+2==2`}}, - "two mixed": {false, false, false, zero, []string{`1+1==2`, `1+1==4`}}, - "one erroring": {false, true, false, zero, []string{`throw new Error('?!');`}}, - "one aborting": {false, false, true, zero, []string{`1+1==4`}}, - "abort with grace period": {false, false, true, oneSec, []string{`1+1==4`}}, - "no abort with grace period": {false, false, true, twoSec, []string{`1+1==4`}}, + "one passing": {true, false, false, zero, []string{`rate<0.01`}}, + "one failing": {false, false, false, zero, []string{`p(95)<200`}}, + "two passing": {true, false, false, zero, []string{`rate<0.1`, `rate<0.01`}}, + "two failing": {false, false, false, zero, []string{`p(95)<200`, `rate<0.1`}}, + "two mixed": {false, false, false, zero, []string{`rate<0.01`, `p(95)<200`}}, + "one aborting": {false, false, true, zero, []string{`p(95)<200`}}, + "abort with grace period": {false, false, true, oneSec, []string{`p(95)<200`}}, + "no abort with grace period": {false, false, true, twoSec, []string{`p(95)<200`}}, } for name, data := range testdata { t.Run(name, func(t *testing.T) { - ts, err := NewThresholds(data.srcs) - assert.Nil(t, err) - ts.Thresholds[0].AbortOnFail = data.abort - ts.Thresholds[0].AbortGracePeriod = data.grace + t.Parallel() + + thresholds, err := NewThresholds(data.sources) + thresholds.sinked = map[string]float64{"rate": 0.0001, "p(95)": 500} + thresholds.Thresholds[0].AbortOnFail = data.abort + thresholds.Thresholds[0].AbortGracePeriod = data.grace runDuration := 1500 * time.Millisecond assert.NoError(t, err) - b, err := ts.runAll(runDuration) + succeeded, err := thresholds.runAll(runDuration) if data.err { assert.Error(t, err) @@ -180,48 +359,74 @@ func TestThresholdsRunAll(t *testing.T) { assert.NoError(t, err) } - if data.succ { - assert.True(t, b) + if data.succeeded { + assert.True(t, succeeded) } else { - assert.False(t, b) + assert.False(t, succeeded) } if data.abort && data.grace.Duration < types.Duration(runDuration) { - assert.True(t, ts.Abort) + assert.True(t, thresholds.Abort) } else { - assert.False(t, ts.Abort) + assert.False(t, thresholds.Abort) } }) } } -func TestThresholdsRun(t *testing.T) { - ts, err := NewThresholds([]string{"a>0"}) - assert.NoError(t, err) +func TestThresholds_Run(t *testing.T) { + t.Parallel() - t.Run("error", func(t *testing.T) { - b, err := ts.Run(DummySink{}, 0) - assert.Error(t, err) - assert.False(t, b) - }) + type args struct { + sink Sink + duration time.Duration + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + "Running thresholds of existing sink", + args{DummySink{"p(95)": 1234.5}, 0}, + true, + false, + }, + { + "Running thresholds of existing sink but failing threshold", + args{DummySink{"p(95)": 3000}, 0}, + false, + false, + }, + { + "Running threshold on non existing sink fails", + args{DummySink{"dummy": 0}, 0}, + false, + true, + }, + } + for _, testCase := range tests { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() - t.Run("pass", func(t *testing.T) { - b, err := ts.Run(DummySink{"a": 1234.5}, 0) - assert.NoError(t, err) - assert.True(t, b) - }) + thresholds, err := NewThresholds([]string{"p(95)<2000"}) + require.NoError(t, err, "Initializing new thresholds should not fail") - t.Run("fail", func(t *testing.T) { - b, err := ts.Run(DummySink{"a": 0}, 0) - assert.NoError(t, err) - assert.False(t, b) - }) + gotOk, gotErr := thresholds.Run(testCase.args.sink, testCase.args.duration) + assert.Equal(t, gotErr != nil, testCase.wantErr, "Thresholds.Run() error = %v, wantErr %v", gotErr, testCase.wantErr) + assert.Equal(t, gotOk, testCase.want, "Thresholds.Run() = %v, want %v", gotOk, testCase.want) + }) + } } func TestThresholdsJSON(t *testing.T) { - var testdata = []struct { + t.Parallel() + + testdata := []struct { JSON string - srcs []string + sources []string abortOnFail bool gracePeriod types.NullDuration outputJSON string @@ -234,8 +439,8 @@ func TestThresholdsJSON(t *testing.T) { "", }, { - `["1+1==2"]`, - []string{"1+1==2"}, + `["rate<0.01"]`, + []string{"rate<0.01"}, false, types.NullDuration{}, "", @@ -248,55 +453,59 @@ func TestThresholdsJSON(t *testing.T) { `["rate<0.01"]`, }, { - `["1+1==2","1+1==3"]`, - []string{"1+1==2", "1+1==3"}, + `["rate<0.01","p(95)<200"]`, + []string{"rate<0.01", "p(95)<200"}, false, types.NullDuration{}, "", }, { - `[{"threshold":"1+1==2"}]`, - []string{"1+1==2"}, + `[{"threshold":"rate<0.01"}]`, + []string{"rate<0.01"}, false, types.NullDuration{}, - `["1+1==2"]`, + `["rate<0.01"]`, }, { - `[{"threshold":"1+1==2","abortOnFail":true,"delayAbortEval":null}]`, - []string{"1+1==2"}, + `[{"threshold":"rate<0.01","abortOnFail":true,"delayAbortEval":null}]`, + []string{"rate<0.01"}, true, types.NullDuration{}, "", }, { - `[{"threshold":"1+1==2","abortOnFail":true,"delayAbortEval":"2s"}]`, - []string{"1+1==2"}, + `[{"threshold":"rate<0.01","abortOnFail":true,"delayAbortEval":"2s"}]`, + []string{"rate<0.01"}, true, types.NullDurationFrom(2 * time.Second), "", }, { - `[{"threshold":"1+1==2","abortOnFail":false}]`, - []string{"1+1==2"}, + `[{"threshold":"rate<0.01","abortOnFail":false}]`, + []string{"rate<0.01"}, false, types.NullDuration{}, - `["1+1==2"]`, + `["rate<0.01"]`, }, { - `[{"threshold":"1+1==2"}, "1+1==3"]`, - []string{"1+1==2", "1+1==3"}, + `[{"threshold":"rate<0.01"}, "p(95)<200"]`, + []string{"rate<0.01", "p(95)<200"}, false, types.NullDuration{}, - `["1+1==2","1+1==3"]`, + `["rate<0.01","p(95)<200"]`, }, } for _, data := range testdata { + data := data + t.Run(data.JSON, func(t *testing.T) { + t.Parallel() + var ts Thresholds assert.NoError(t, json.Unmarshal([]byte(data.JSON), &ts)) - assert.Equal(t, len(data.srcs), len(ts.Thresholds)) - for i, src := range data.srcs { + assert.Equal(t, len(data.sources), len(ts.Thresholds)) + for i, src := range data.sources { assert.Equal(t, src, ts.Thresholds[i].Source) assert.Equal(t, data.abortOnFail, ts.Thresholds[i].AbortOnFail) assert.Equal(t, data.gracePeriod, ts.Thresholds[i].AbortGracePeriod) @@ -315,18 +524,20 @@ func TestThresholdsJSON(t *testing.T) { } t.Run("bad JSON", func(t *testing.T) { + t.Parallel() + var ts Thresholds assert.Error(t, json.Unmarshal([]byte("42"), &ts)) assert.Nil(t, ts.Thresholds) - assert.Nil(t, ts.Runtime) assert.False(t, ts.Abort) }) t.Run("bad source", func(t *testing.T) { + t.Parallel() + var ts Thresholds assert.Error(t, json.Unmarshal([]byte(`["="]`), &ts)) assert.Nil(t, ts.Thresholds) - assert.Nil(t, ts.Runtime) assert.False(t, ts.Abort) }) }