New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add e2e tests for metrics #11307
Add e2e tests for metrics #11307
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
/* | ||
* This file is part of the KubeVirt project | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
* | ||
* Copyright The Kubevirt Authors | ||
* | ||
*/ | ||
|
||
package libmonitoring | ||
|
||
import ( | ||
"fmt" | ||
|
||
"github.com/machadovilaca/operator-observability/pkg/operatormetrics" | ||
"github.com/onsi/gomega/format" | ||
) | ||
|
||
const ( | ||
prometheusMetricNameLabel = "__name__" | ||
prometheusHistogramBucketSuffix = "_bucket" | ||
) | ||
|
||
type MetricMatcher struct { | ||
Metric operatormetrics.Metric | ||
Labels map[string]string | ||
} | ||
|
||
func (matcher *MetricMatcher) FailureMessage(actual interface{}) (message string) { | ||
msg := format.Message(actual, "to contain metric", matcher.Metric.GetOpts().Name) | ||
|
||
if matcher.Labels != nil { | ||
msg += fmt.Sprintf(" with labels %v", matcher.Labels) | ||
} | ||
|
||
return msg | ||
} | ||
|
||
func (matcher *MetricMatcher) NegatedFailureMessage(actual interface{}) (message string) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why we need both functions? FailureMessage and NegatedFailureMessage? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is a implementation of a gomega interface, so it's mandatory to have both, to handle 2 scenarios FailureMessage is used when you expect to find something and it is not present, for example, expected [1,2] to contain 3 NegatedFailureMessage is used when you expect to not find something, but it is there, for example, expected [A,B] not to contain A There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. +1 thanks! |
||
msg := format.Message(actual, "not to contain metric", matcher.Metric.GetOpts().Name) | ||
|
||
if matcher.Labels != nil { | ||
msg += fmt.Sprintf(" with labels %v", matcher.Labels) | ||
} | ||
|
||
return msg | ||
} | ||
|
||
func (matcher *MetricMatcher) Match(actual interface{}) (success bool, err error) { | ||
actualMetric, ok := actual.(promResult) | ||
if !ok { | ||
return false, fmt.Errorf("metric matcher requires a libmonitoring.PromResult") | ||
} | ||
|
||
actualName, ok := actualMetric.Metric[prometheusMetricNameLabel] | ||
if !ok { | ||
return false, fmt.Errorf("metric matcher requires a map with %s key", prometheusMetricNameLabel) | ||
} | ||
|
||
nameToMatch := matcher.Metric.GetOpts().Name | ||
if matcher.Metric.GetType() == operatormetrics.HistogramType || matcher.Metric.GetType() == operatormetrics.HistogramVecType { | ||
nameToMatch = nameToMatch + prometheusHistogramBucketSuffix | ||
} | ||
|
||
if actualName != nameToMatch { | ||
return false, nil | ||
} | ||
|
||
for k, v := range matcher.Labels { | ||
actualValue, ok := actualMetric.Metric[k] | ||
if !ok { | ||
return false, nil | ||
} | ||
if actualValue != v { | ||
return false, nil | ||
} | ||
} | ||
|
||
return true, nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,155 @@ | ||
/* | ||
* This file is part of the KubeVirt project | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
* | ||
* Copyright The Kubevirt Authors | ||
* | ||
*/ | ||
|
||
package monitoring | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. missing header There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. added |
||
|
||
import ( | ||
"context" | ||
"strconv" | ||
"time" | ||
|
||
. "github.com/onsi/ginkgo/v2" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. please sort the import to groups, I see onsi here and in the second group as well There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think here it makes sense to keep them apart, these 2 are dot imports, and we have this structure on all tests, and are always splitted from the rest even |
||
. "github.com/onsi/gomega" | ||
|
||
"github.com/machadovilaca/operator-observability/pkg/operatormetrics" | ||
"github.com/onsi/gomega/types" | ||
"github.com/prometheus/client_golang/prometheus" | ||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
v1 "kubevirt.io/api/core/v1" | ||
"kubevirt.io/client-go/kubecli" | ||
|
||
"kubevirt.io/kubevirt/pkg/libvmi" | ||
virtapi "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api" | ||
virtcontroller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller" | ||
virtoperator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator" | ||
|
||
"kubevirt.io/kubevirt/tests/decorators" | ||
"kubevirt.io/kubevirt/tests/framework/kubevirt" | ||
"kubevirt.io/kubevirt/tests/libmonitoring" | ||
"kubevirt.io/kubevirt/tests/libvmifact" | ||
"kubevirt.io/kubevirt/tests/libwait" | ||
"kubevirt.io/kubevirt/tests/testsuite" | ||
) | ||
|
||
var _ = Describe("[sig-monitoring]Metrics", decorators.SigMonitoring, func() { | ||
var virtClient kubecli.KubevirtClient | ||
var metrics *libmonitoring.QueryRequestResult | ||
|
||
BeforeEach(func() { | ||
virtClient = kubevirt.Client() | ||
setupVM(virtClient) | ||
metrics = fetchPrometheusMetrics(virtClient) | ||
}) | ||
|
||
Context("Prometheus metrics", func() { | ||
var excludedMetrics = map[string]bool{ | ||
// virt-api | ||
// can later be added in pre-existing feature tests | ||
"kubevirt_portforward_active_tunnels": true, | ||
"kubevirt_usbredir_active_connections": true, | ||
"kubevirt_vnc_active_connections": true, | ||
"kubevirt_console_active_connections": true, | ||
|
||
// virt-controller | ||
// needs a migration - ignoring since already tested in - VM Monitoring, VM migration metrics | ||
"kubevirt_vmi_migration_phase_transition_time_from_creation_seconds": true, | ||
"kubevirt_vmi_migrations_in_pending_phase": true, | ||
"kubevirt_vmi_migrations_in_scheduling_phase": true, | ||
"kubevirt_vmi_migrations_in_running_phase": true, | ||
"kubevirt_vmi_migration_succeeded": true, | ||
"kubevirt_vmi_migration_failed": true, | ||
} | ||
|
||
It("should contain virt components metrics", func() { | ||
err := virtoperator.SetupMetrics() | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
err = virtapi.SetupMetrics() | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
err = virtcontroller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
for _, metric := range operatormetrics.ListMetrics() { | ||
if excludedMetrics[metric.GetOpts().Name] { | ||
continue | ||
} | ||
|
||
Expect(metrics.Data.Result).To(ContainElement(gomegaContainsMetricMatcher(metric, nil))) | ||
} | ||
}) | ||
|
||
It("should have kubevirt_vmi_phase_transition_time_seconds buckets correctly configured", func() { | ||
buckets := virtcontroller.PhaseTransitionTimeBuckets() | ||
|
||
for _, bucket := range buckets { | ||
labels := map[string]string{"le": strconv.FormatFloat(bucket, 'f', -1, 64)} | ||
|
||
metric := operatormetrics.NewHistogram( | ||
operatormetrics.MetricOpts{Name: "kubevirt_vmi_phase_transition_time_from_deletion_seconds"}, | ||
prometheus.HistogramOpts{}, | ||
) | ||
|
||
Expect(metrics.Data.Result).To(ContainElement(gomegaContainsMetricMatcher(metric, labels))) | ||
} | ||
}) | ||
}) | ||
}) | ||
|
||
func fetchPrometheusMetrics(virtClient kubecli.KubevirtClient) *libmonitoring.QueryRequestResult { | ||
metrics, err := libmonitoring.QueryRange(virtClient, "{__name__=~\"kubevirt_.*\"}", time.Now().Add(-1*time.Minute), time.Now(), 15*time.Second) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
Expect(metrics.Status).To(Equal("success")) | ||
Expect(metrics.Data.ResultType).To(Equal("matrix")) | ||
Expect(metrics.Data.Result).ToNot(BeEmpty(), "No metrics found") | ||
|
||
return metrics | ||
} | ||
|
||
func setupVM(virtClient kubecli.KubevirtClient) { | ||
vm := createRunningVM(virtClient) | ||
libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", 1) | ||
|
||
By("Deleting the VirtualMachine") | ||
err := virtClient.VirtualMachine(vm.Namespace).Delete(context.Background(), vm.Name, &metav1.DeleteOptions{}) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", -1) | ||
} | ||
|
||
func createRunningVM(virtClient kubecli.KubevirtClient) *v1.VirtualMachine { | ||
vmi := libvmifact.NewGuestless(libvmi.WithNamespace(testsuite.GetTestNamespace(nil))) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we don't need to specify There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Oh yes, the libvmi.wait needs it. |
||
vm := libvmi.NewVirtualMachine(vmi, libvmi.WithRunning()) | ||
vm, err := virtClient.VirtualMachine(testsuite.GetTestNamespace(vm)).Create(context.Background(), vm) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
Eventually(func() bool { | ||
vm, err := virtClient.VirtualMachine(testsuite.GetTestNamespace(vm)).Get(context.Background(), vm.Name, &metav1.GetOptions{}) | ||
Expect(err).ToNot(HaveOccurred()) | ||
return vm.Status.Ready | ||
}, 300*time.Second, 1*time.Second).Should(BeTrue()) | ||
Comment on lines
+143
to
+147
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Isn't the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. and perform a vm.Get after it? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The VMI still takes a few seconds to be created after you create the VM. And |
||
libwait.WaitForSuccessfulVMIStart(vmi) | ||
machadovilaca marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
return vm | ||
} | ||
|
||
func gomegaContainsMetricMatcher(metric operatormetrics.Metric, labels map[string]string) types.GomegaMatcher { | ||
return &libmonitoring.MetricMatcher{Metric: metric, Labels: labels} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
missing header
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
added