Skip to content

Commit

Permalink
Add e2e tests for metrics
Browse files Browse the repository at this point in the history
Signed-off-by: João Vilaça <jvilaca@redhat.com>
  • Loading branch information
machadovilaca committed Feb 20, 2024
1 parent 34d4b39 commit 72bf912
Show file tree
Hide file tree
Showing 5 changed files with 174 additions and 2 deletions.
3 changes: 3 additions & 0 deletions tests/libmonitoring/BUILD.bazel
Expand Up @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"metric_matcher.go",
"prometheus.go",
"scaling.go",
],
Expand All @@ -15,8 +16,10 @@ go_library(
"//tests/flags:go_default_library",
"//tests/framework/checks:go_default_library",
"//tests/framework/kubevirt:go_default_library",
"//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
"//vendor/github.com/onsi/ginkgo/v2:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/onsi/gomega/format:go_default_library",
"//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library",
"//vendor/github.com/prometheus/client_golang/api/prometheus/v1:go_default_library",
"//vendor/k8s.io/api/autoscaling/v1:go_default_library",
Expand Down
39 changes: 39 additions & 0 deletions tests/libmonitoring/metric_matcher.go
@@ -0,0 +1,39 @@
package libmonitoring

import (
"fmt"

"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
"github.com/onsi/gomega/format"
)

type MetricMatcher struct {
Metric operatormetrics.Metric
}

func (matcher *MetricMatcher) FailureMessage(actual interface{}) (message string) {
return format.Message(actual, "to contain metric", matcher.Metric.GetOpts().Name)
}

func (matcher *MetricMatcher) NegatedFailureMessage(actual interface{}) (message string) {
return format.Message(actual, "not to contain metric", matcher.Metric.GetOpts().Name)
}

func (matcher *MetricMatcher) Match(actual interface{}) (success bool, err error) {
actualMetric, ok := actual.(promResult)
if !ok {
return false, fmt.Errorf("metric matcher requires a libmonitoring.PromResult")
}

actualName, ok := actualMetric.Metric["__name__"]
if !ok {
return false, fmt.Errorf("metric matcher requires a map with __name__ key")
}

nameToMatch := matcher.Metric.GetOpts().Name
if matcher.Metric.GetType() == operatormetrics.HistogramType || matcher.Metric.GetType() == operatormetrics.HistogramVecType {
nameToMatch = nameToMatch + "_bucket"
}

return actualName == nameToMatch, nil
}
4 changes: 2 additions & 2 deletions tests/libmonitoring/prometheus.go
Expand Up @@ -84,7 +84,7 @@ func WaitForMetricValueWithLabels(client kubecli.KubevirtClient, metric string,
}

func GetMetricValueWithLabels(cli kubecli.KubevirtClient, query string, labels map[string]string) (float64, error) {
result, err := fetchMetric(cli, query)
result, err := FetchMetric(cli, query)
if err != nil {
return -1, err
}
Expand Down Expand Up @@ -130,7 +130,7 @@ func labelsMatch(pr promResult, labels map[string]string) bool {
return true
}

func fetchMetric(cli kubecli.KubevirtClient, query string) (*QueryRequestResult, error) {
func FetchMetric(cli kubecli.KubevirtClient, query string) (*QueryRequestResult, error) {
bodyBytes := DoPrometheusHTTPRequest(cli, fmt.Sprintf("/query?query=%s", query))

var result QueryRequestResult
Expand Down
6 changes: 6 additions & 0 deletions tests/monitoring/BUILD.bazel
Expand Up @@ -4,13 +4,18 @@ go_library(
name = "go_default_library",
srcs = [
"component_monitoring.go",
"metrics.go",
"monitoring.go",
"vm_monitoring.go",
],
importpath = "kubevirt.io/kubevirt/tests/monitoring",
visibility = ["//visibility:public"],
deps = [
"//pkg/apimachinery/patch:go_default_library",
"//pkg/monitoring/metrics/virt-api:go_default_library",
"//pkg/monitoring/metrics/virt-controller:go_default_library",
"//pkg/monitoring/metrics/virt-operator:go_default_library",
"//pkg/pointer:go_default_library",
"//pkg/virtctl/pause:go_default_library",
"//staging/src/kubevirt.io/api/core/v1:go_default_library",
"//staging/src/kubevirt.io/client-go/kubecli:go_default_library",
Expand All @@ -30,6 +35,7 @@ go_library(
"//tests/libwait:go_default_library",
"//tests/testsuite:go_default_library",
"//tests/util:go_default_library",
"//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
"//vendor/github.com/onsi/ginkgo/v2:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/onsi/gomega/types:go_default_library",
Expand Down
124 changes: 124 additions & 0 deletions tests/monitoring/metrics.go
@@ -0,0 +1,124 @@
package monitoring

import (
"context"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
"github.com/onsi/gomega/types"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "kubevirt.io/api/core/v1"
"kubevirt.io/client-go/kubecli"

virtapi "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api"
virtcontroller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller"
virtoperator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator"
"kubevirt.io/kubevirt/pkg/pointer"

"kubevirt.io/kubevirt/tests/decorators"
"kubevirt.io/kubevirt/tests/framework/kubevirt"
"kubevirt.io/kubevirt/tests/libmonitoring"
"kubevirt.io/kubevirt/tests/libvmi"
"kubevirt.io/kubevirt/tests/testsuite"
)

var _ = FDescribe("[Serial][sig-monitoring]Metrics", Serial, decorators.SigMonitoring, func() {
var virtClient kubecli.KubevirtClient
var metrics *libmonitoring.QueryRequestResult
var err error

setupVM := func() {
vmi := libvmi.NewCirros(
libvmi.WithInterface(libvmi.InterfaceDeviceWithMasqueradeBinding()),
libvmi.WithNetwork(v1.DefaultPodNetwork()),
)
vmi.Namespace = testsuite.GetTestNamespace(nil)

By("Creating a running VirtualMachine")
vm := libvmi.NewVirtualMachine(vmi)
vm.Spec.Running = pointer.P(true)
_, err = virtClient.VirtualMachine(vm.Namespace).Create(context.Background(), vm)
Expect(err).ToNot(HaveOccurred())

libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", 1)

By("Deleting the VirtualMachine")
err = virtClient.VirtualMachine(vm.Namespace).Delete(context.Background(), vm.Name, &metav1.DeleteOptions{})
Expect(err).ToNot(HaveOccurred())

libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", -1)
}

BeforeEach(func() {
virtClient = kubevirt.Client()

setupVM()

metrics, err = libmonitoring.FetchMetric(virtClient, "{__name__=~\"kubevirt_.*\"}")
Expect(err).ToNot(HaveOccurred())

Expect(metrics.Status).To(Equal("success"))
Expect(metrics.Data.ResultType).To(Equal("vector"))
Expect(metrics.Data.Result).ToNot(BeEmpty(), "No metrics found")
})

containsMetric := func(metric operatormetrics.Metric) types.GomegaMatcher {
return &libmonitoring.MetricMatcher{Metric: metric}
}

Context("Prometheus metrics", func() {
var excludedMetrics = map[string]bool{
// virt-api
// can later be added in pre-existing feature tests
"kubevirt_portforward_active_tunnels": true,
"kubevirt_usbredir_active_connections": true,
"kubevirt_vnc_active_connections": true,

// virt-controller
// needs a migration - ignoring since already tested in - VM Monitoring, VM migration metrics
"kubevirt_vmi_migration_phase_transition_time_from_creation_seconds": true,
"kubevirt_vmi_migrations_in_pending_phase": true,
"kubevirt_vmi_migrations_in_scheduling_phase": true,
"kubevirt_vmi_migrations_in_running_phase": true,
"kubevirt_vmi_migration_succeeded": true,
"kubevirt_vmi_migration_failed": true,
}

It("should contain virt-operator metrics", func() {
err = virtoperator.SetupMetrics()
Expect(err).ToNot(HaveOccurred())

for _, metric := range virtoperator.ListMetrics() {
Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric)))
}
})

It("should contain virt-api metrics", func() {
err = virtapi.SetupMetrics()
Expect(err).ToNot(HaveOccurred())

for _, metric := range virtapi.ListMetrics() {
if excludedMetrics[metric.GetOpts().Name] {
continue
}

Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric)))
}
})

It("should contain virt-controller metrics", func() {
err = virtcontroller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil)
Expect(err).ToNot(HaveOccurred())

for _, metric := range virtcontroller.ListMetrics() {
if excludedMetrics[metric.GetOpts().Name] {
continue
}

Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric)))
}
})
})
})

0 comments on commit 72bf912

Please sign in to comment.