From 72bf912fbe2dfdc0b08f1bb1a0a3bcd34e9c6e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vila=C3=A7a?= Date: Tue, 20 Feb 2024 13:51:39 +0000 Subject: [PATCH] Add e2e tests for metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: João Vilaça --- tests/libmonitoring/BUILD.bazel | 3 + tests/libmonitoring/metric_matcher.go | 39 ++++++++ tests/libmonitoring/prometheus.go | 4 +- tests/monitoring/BUILD.bazel | 6 ++ tests/monitoring/metrics.go | 124 ++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 tests/libmonitoring/metric_matcher.go create mode 100644 tests/monitoring/metrics.go diff --git a/tests/libmonitoring/BUILD.bazel b/tests/libmonitoring/BUILD.bazel index 941fb8ce9104..9df38eb235cb 100644 --- a/tests/libmonitoring/BUILD.bazel +++ b/tests/libmonitoring/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "metric_matcher.go", "prometheus.go", "scaling.go", ], @@ -15,8 +16,10 @@ go_library( "//tests/flags:go_default_library", "//tests/framework/checks:go_default_library", "//tests/framework/kubevirt:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", "//vendor/github.com/onsi/ginkgo/v2:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/onsi/gomega/format:go_default_library", "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library", "//vendor/github.com/prometheus/client_golang/api/prometheus/v1:go_default_library", "//vendor/k8s.io/api/autoscaling/v1:go_default_library", diff --git a/tests/libmonitoring/metric_matcher.go b/tests/libmonitoring/metric_matcher.go new file mode 100644 index 000000000000..7885a746b8ef --- /dev/null +++ b/tests/libmonitoring/metric_matcher.go @@ -0,0 +1,39 @@ +package libmonitoring + +import ( + "fmt" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/onsi/gomega/format" +) + +type MetricMatcher struct { + Metric operatormetrics.Metric +} + +func (matcher *MetricMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to contain metric", matcher.Metric.GetOpts().Name) +} + +func (matcher *MetricMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to contain metric", matcher.Metric.GetOpts().Name) +} + +func (matcher *MetricMatcher) Match(actual interface{}) (success bool, err error) { + actualMetric, ok := actual.(promResult) + if !ok { + return false, fmt.Errorf("metric matcher requires a libmonitoring.PromResult") + } + + actualName, ok := actualMetric.Metric["__name__"] + if !ok { + return false, fmt.Errorf("metric matcher requires a map with __name__ key") + } + + nameToMatch := matcher.Metric.GetOpts().Name + if matcher.Metric.GetType() == operatormetrics.HistogramType || matcher.Metric.GetType() == operatormetrics.HistogramVecType { + nameToMatch = nameToMatch + "_bucket" + } + + return actualName == nameToMatch, nil +} diff --git a/tests/libmonitoring/prometheus.go b/tests/libmonitoring/prometheus.go index 304309cba377..03637fac4a67 100644 --- a/tests/libmonitoring/prometheus.go +++ b/tests/libmonitoring/prometheus.go @@ -84,7 +84,7 @@ func WaitForMetricValueWithLabels(client kubecli.KubevirtClient, metric string, } func GetMetricValueWithLabels(cli kubecli.KubevirtClient, query string, labels map[string]string) (float64, error) { - result, err := fetchMetric(cli, query) + result, err := FetchMetric(cli, query) if err != nil { return -1, err } @@ -130,7 +130,7 @@ func labelsMatch(pr promResult, labels map[string]string) bool { return true } -func fetchMetric(cli kubecli.KubevirtClient, query string) (*QueryRequestResult, error) { +func FetchMetric(cli kubecli.KubevirtClient, query string) (*QueryRequestResult, error) { bodyBytes := DoPrometheusHTTPRequest(cli, fmt.Sprintf("/query?query=%s", query)) var result QueryRequestResult diff --git a/tests/monitoring/BUILD.bazel b/tests/monitoring/BUILD.bazel index 8a198c6db98c..254a40530e6b 100644 --- a/tests/monitoring/BUILD.bazel +++ b/tests/monitoring/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "component_monitoring.go", + "metrics.go", "monitoring.go", "vm_monitoring.go", ], @@ -11,6 +12,10 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/apimachinery/patch:go_default_library", + "//pkg/monitoring/metrics/virt-api:go_default_library", + "//pkg/monitoring/metrics/virt-controller:go_default_library", + "//pkg/monitoring/metrics/virt-operator:go_default_library", + "//pkg/pointer:go_default_library", "//pkg/virtctl/pause:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/kubecli:go_default_library", @@ -30,6 +35,7 @@ go_library( "//tests/libwait:go_default_library", "//tests/testsuite:go_default_library", "//tests/util:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", "//vendor/github.com/onsi/ginkgo/v2:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/onsi/gomega/types:go_default_library", diff --git a/tests/monitoring/metrics.go b/tests/monitoring/metrics.go new file mode 100644 index 000000000000..b1164ca4f9bd --- /dev/null +++ b/tests/monitoring/metrics.go @@ -0,0 +1,124 @@ +package monitoring + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/onsi/gomega/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "kubevirt.io/api/core/v1" + "kubevirt.io/client-go/kubecli" + + virtapi "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api" + virtcontroller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller" + virtoperator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator" + "kubevirt.io/kubevirt/pkg/pointer" + + "kubevirt.io/kubevirt/tests/decorators" + "kubevirt.io/kubevirt/tests/framework/kubevirt" + "kubevirt.io/kubevirt/tests/libmonitoring" + "kubevirt.io/kubevirt/tests/libvmi" + "kubevirt.io/kubevirt/tests/testsuite" +) + +var _ = FDescribe("[Serial][sig-monitoring]Metrics", Serial, decorators.SigMonitoring, func() { + var virtClient kubecli.KubevirtClient + var metrics *libmonitoring.QueryRequestResult + var err error + + setupVM := func() { + vmi := libvmi.NewCirros( + libvmi.WithInterface(libvmi.InterfaceDeviceWithMasqueradeBinding()), + libvmi.WithNetwork(v1.DefaultPodNetwork()), + ) + vmi.Namespace = testsuite.GetTestNamespace(nil) + + By("Creating a running VirtualMachine") + vm := libvmi.NewVirtualMachine(vmi) + vm.Spec.Running = pointer.P(true) + _, err = virtClient.VirtualMachine(vm.Namespace).Create(context.Background(), vm) + Expect(err).ToNot(HaveOccurred()) + + libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", 1) + + By("Deleting the VirtualMachine") + err = virtClient.VirtualMachine(vm.Namespace).Delete(context.Background(), vm.Name, &metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + + libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", -1) + } + + BeforeEach(func() { + virtClient = kubevirt.Client() + + setupVM() + + metrics, err = libmonitoring.FetchMetric(virtClient, "{__name__=~\"kubevirt_.*\"}") + Expect(err).ToNot(HaveOccurred()) + + Expect(metrics.Status).To(Equal("success")) + Expect(metrics.Data.ResultType).To(Equal("vector")) + Expect(metrics.Data.Result).ToNot(BeEmpty(), "No metrics found") + }) + + containsMetric := func(metric operatormetrics.Metric) types.GomegaMatcher { + return &libmonitoring.MetricMatcher{Metric: metric} + } + + Context("Prometheus metrics", func() { + var excludedMetrics = map[string]bool{ + // virt-api + // can later be added in pre-existing feature tests + "kubevirt_portforward_active_tunnels": true, + "kubevirt_usbredir_active_connections": true, + "kubevirt_vnc_active_connections": true, + + // virt-controller + // needs a migration - ignoring since already tested in - VM Monitoring, VM migration metrics + "kubevirt_vmi_migration_phase_transition_time_from_creation_seconds": true, + "kubevirt_vmi_migrations_in_pending_phase": true, + "kubevirt_vmi_migrations_in_scheduling_phase": true, + "kubevirt_vmi_migrations_in_running_phase": true, + "kubevirt_vmi_migration_succeeded": true, + "kubevirt_vmi_migration_failed": true, + } + + It("should contain virt-operator metrics", func() { + err = virtoperator.SetupMetrics() + Expect(err).ToNot(HaveOccurred()) + + for _, metric := range virtoperator.ListMetrics() { + Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric))) + } + }) + + It("should contain virt-api metrics", func() { + err = virtapi.SetupMetrics() + Expect(err).ToNot(HaveOccurred()) + + for _, metric := range virtapi.ListMetrics() { + if excludedMetrics[metric.GetOpts().Name] { + continue + } + + Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric))) + } + }) + + It("should contain virt-controller metrics", func() { + err = virtcontroller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil) + Expect(err).ToNot(HaveOccurred()) + + for _, metric := range virtcontroller.ListMetrics() { + if excludedMetrics[metric.GetOpts().Name] { + continue + } + + Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric))) + } + }) + }) +})