Add e2e tests for metrics

pkg/monitoring/metrics/virt-controller/metrics.go

@@ -85,7 +85,7 @@ func ListMetrics() []operatormetrics.Metric {
 	return operatormetrics.ListMetrics()
 }
 
-func phaseTransitionTimeBuckets() []float64 {
+func PhaseTransitionTimeBuckets() []float64 {
 	return []float64{
 		(0.5 * time.Second.Seconds()),
 		(1 * time.Second.Seconds()),

pkg/monitoring/metrics/virt-controller/migration_metrics.go

@@ -44,7 +44,7 @@ var (
 			Help: "Histogram of VM migration phase transitions duration from creation time in seconds.",
 		},
 		prometheus.HistogramOpts{
-			Buckets: phaseTransitionTimeBuckets(),
+			Buckets: PhaseTransitionTimeBuckets(),
 		},
 		[]string{
 			// phase of the vmi migration

pkg/monitoring/metrics/virt-controller/perfscale_metrics.go

@@ -48,7 +48,7 @@ var (
 			Help: "Histogram of VM phase transitions duration between different phases in seconds.",
 		},
 		prometheus.HistogramOpts{
-			Buckets: phaseTransitionTimeBuckets(),
+			Buckets: PhaseTransitionTimeBuckets(),
 		},
 		[]string{
 			// phase of the vmi
@@ -64,7 +64,7 @@ var (
 			Help: "Histogram of VM phase transitions duration from creation time in seconds.",
 		},
 		prometheus.HistogramOpts{
-			Buckets: phaseTransitionTimeBuckets(),
+			Buckets: PhaseTransitionTimeBuckets(),
 		},
 		[]string{
 			// phase of the vmi
@@ -78,7 +78,7 @@ var (
 			Help: "Histogram of VM phase transitions duration from deletion time in seconds.",
 		},
 		prometheus.HistogramOpts{
-			Buckets: phaseTransitionTimeBuckets(),
+			Buckets: PhaseTransitionTimeBuckets(),
 		},
 		[]string{
 			// phase of the vmi

tests/libmonitoring/BUILD.bazel

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_library(
     name = "go_default_library",
     srcs = [
+        "metric_matcher.go",
         "prometheus.go",
         "scaling.go",
     ],
@@ -14,8 +15,10 @@ go_library(
         "//tests/exec:go_default_library",
         "//tests/flags:go_default_library",
         "//tests/framework/checks:go_default_library",
+        "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
         "//vendor/github.com/onsi/ginkgo/v2:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/github.com/onsi/gomega/format:go_default_library",
         "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library",
         "//vendor/github.com/prometheus/client_golang/api/prometheus/v1:go_default_library",
         "//vendor/k8s.io/api/autoscaling/v1:go_default_library",

tests/libmonitoring/metric_matcher.go

@@ -0,0 +1,90 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright The Kubevirt Authors
+ *
+ */
+
+package libmonitoring
+
+import (
+	"fmt"
+
+	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+	"github.com/onsi/gomega/format"
+)
+
+const (
+	prometheusMetricNameLabel       = "__name__"
+	prometheusHistogramBucketSuffix = "_bucket"
+)
+
+type MetricMatcher struct {
+	Metric operatormetrics.Metric
+	Labels map[string]string
+}
+
+func (matcher *MetricMatcher) FailureMessage(actual interface{}) (message string) {
+	msg := format.Message(actual, "to contain metric", matcher.Metric.GetOpts().Name)
+
+	if matcher.Labels != nil {
+		msg += fmt.Sprintf(" with labels %v", matcher.Labels)
+	}
+
+	return msg
+}
+
+func (matcher *MetricMatcher) NegatedFailureMessage(actual interface{}) (message string) {
+	msg := format.Message(actual, "not to contain metric", matcher.Metric.GetOpts().Name)
+
+	if matcher.Labels != nil {
+		msg += fmt.Sprintf(" with labels %v", matcher.Labels)
+	}
+
+	return msg
+}
+
+func (matcher *MetricMatcher) Match(actual interface{}) (success bool, err error) {
+	actualMetric, ok := actual.(promResult)
+	if !ok {
+		return false, fmt.Errorf("metric matcher requires a libmonitoring.PromResult")
+	}
+
+	actualName, ok := actualMetric.Metric[prometheusMetricNameLabel]
+	if !ok {
+		return false, fmt.Errorf("metric matcher requires a map with %s key", prometheusMetricNameLabel)
+	}
+
+	nameToMatch := matcher.Metric.GetOpts().Name
+	if matcher.Metric.GetType() == operatormetrics.HistogramType || matcher.Metric.GetType() == operatormetrics.HistogramVecType {
+		nameToMatch = nameToMatch + prometheusHistogramBucketSuffix
+	}
+
+	if actualName != nameToMatch {
+		return false, nil
+	}
+
+	for k, v := range matcher.Labels {
+		actualValue, ok := actualMetric.Metric[k]
+		if !ok {
+			return false, nil
+		}
+		if actualValue != v {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}

tests/libmonitoring/prometheus.go

@@ -145,6 +145,22 @@ func fetchMetric(cli kubecli.KubevirtClient, query string) (*QueryRequestResult,
 	return &result, nil
 }
 
+func QueryRange(cli kubecli.KubevirtClient, query string, start time.Time, end time.Time, step time.Duration) (*QueryRequestResult, error) {
+	bodyBytes := DoPrometheusHTTPRequest(cli, fmt.Sprintf("/query_range?query=%s&start=%d&end=%d&step=%d", query, start.Unix(), end.Unix(), int(step.Seconds())))
+
+	var result QueryRequestResult
+	err := json.Unmarshal(bodyBytes, &result)
+	if err != nil {
+		return nil, err
+	}
+
+	if result.Status != "success" {
+		return nil, fmt.Errorf("api request failed. result: %v", result)
+	}
+
+	return &result, nil
+}
+
 func DoPrometheusHTTPRequest(cli kubecli.KubevirtClient, endpoint string) []byte {
 
 	monitoringNs := getMonitoringNs(cli)

tests/monitoring/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "component_monitoring.go",
+        "metrics.go",
         "monitoring.go",
         "vm_monitoring.go",
     ],
@@ -12,6 +13,9 @@ go_library(
     deps = [
         "//pkg/apimachinery/patch:go_default_library",
         "//pkg/libvmi:go_default_library",
+        "//pkg/monitoring/metrics/virt-api:go_default_library",
+        "//pkg/monitoring/metrics/virt-controller:go_default_library",
+        "//pkg/monitoring/metrics/virt-operator:go_default_library",
         "//pkg/virtctl/pause:go_default_library",
         "//staging/src/kubevirt.io/api/core/v1:go_default_library",
         "//staging/src/kubevirt.io/client-go/kubecli:go_default_library",
@@ -32,10 +36,12 @@ go_library(
         "//tests/libwait:go_default_library",
         "//tests/testsuite:go_default_library",
         "//tests/util:go_default_library",
+        "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
         "//vendor/github.com/onsi/ginkgo/v2:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/onsi/gomega/types:go_default_library",
         "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/api/apps/v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/rbac/v1:go_default_library",

tests/monitoring/metrics.go

@@ -0,0 +1,155 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright The Kubevirt Authors
+ *
+ */
+
+package monitoring
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+	"github.com/onsi/gomega/types"
+	"github.com/prometheus/client_golang/prometheus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "kubevirt.io/api/core/v1"
+	"kubevirt.io/client-go/kubecli"
+
+	"kubevirt.io/kubevirt/pkg/libvmi"
+	virtapi "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api"
+	virtcontroller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller"
+	virtoperator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator"
+
+	"kubevirt.io/kubevirt/tests/decorators"
+	"kubevirt.io/kubevirt/tests/framework/kubevirt"
+	"kubevirt.io/kubevirt/tests/libmonitoring"
+	"kubevirt.io/kubevirt/tests/libvmifact"
+	"kubevirt.io/kubevirt/tests/libwait"
+	"kubevirt.io/kubevirt/tests/testsuite"
+)
+
+var _ = Describe("[sig-monitoring]Metrics", decorators.SigMonitoring, func() {
+	var virtClient kubecli.KubevirtClient
+	var metrics *libmonitoring.QueryRequestResult
+
+	BeforeEach(func() {
+		virtClient = kubevirt.Client()
+		setupVM(virtClient)
+		metrics = fetchPrometheusMetrics(virtClient)
+	})
+
+	Context("Prometheus metrics", func() {
+		var excludedMetrics = map[string]bool{
+			// virt-api
+			// can later be added in pre-existing feature tests
+			"kubevirt_portforward_active_tunnels":  true,
+			"kubevirt_usbredir_active_connections": true,
+			"kubevirt_vnc_active_connections":      true,
+			"kubevirt_console_active_connections":  true,
+
+			// virt-controller
+			// needs a migration - ignoring since already tested in - VM Monitoring, VM migration metrics
+			"kubevirt_vmi_migration_phase_transition_time_from_creation_seconds": true,
+			"kubevirt_vmi_migrations_in_pending_phase":                           true,
+			"kubevirt_vmi_migrations_in_scheduling_phase":                        true,
+			"kubevirt_vmi_migrations_in_running_phase":                           true,
+			"kubevirt_vmi_migration_succeeded":                                   true,
+			"kubevirt_vmi_migration_failed":                                      true,
+		}
+
+		It("should contain virt components metrics", func() {
+			err := virtoperator.SetupMetrics()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = virtapi.SetupMetrics()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = virtcontroller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, metric := range operatormetrics.ListMetrics() {
+				if excludedMetrics[metric.GetOpts().Name] {
+					continue
+				}
+
+				Expect(metrics.Data.Result).To(ContainElement(gomegaContainsMetricMatcher(metric, nil)))
+			}
+		})
+
+		It("should have kubevirt_vmi_phase_transition_time_seconds buckets correctly configured", func() {
+			buckets := virtcontroller.PhaseTransitionTimeBuckets()
+
+			for _, bucket := range buckets {
+				labels := map[string]string{"le": strconv.FormatFloat(bucket, 'f', -1, 64)}
+
+				metric := operatormetrics.NewHistogram(
+					operatormetrics.MetricOpts{Name: "kubevirt_vmi_phase_transition_time_from_deletion_seconds"},
+					prometheus.HistogramOpts{},
+				)
+
+				Expect(metrics.Data.Result).To(ContainElement(gomegaContainsMetricMatcher(metric, labels)))
+			}
+		})
+	})
+})
+
+func fetchPrometheusMetrics(virtClient kubecli.KubevirtClient) *libmonitoring.QueryRequestResult {
+	metrics, err := libmonitoring.QueryRange(virtClient, "{__name__=~\"kubevirt_.*\"}", time.Now().Add(-1*time.Minute), time.Now(), 15*time.Second)
+	Expect(err).ToNot(HaveOccurred())
+
+	Expect(metrics.Status).To(Equal("success"))
+	Expect(metrics.Data.ResultType).To(Equal("matrix"))
+	Expect(metrics.Data.Result).ToNot(BeEmpty(), "No metrics found")
+
+	return metrics
+}
+
+func setupVM(virtClient kubecli.KubevirtClient) {
+	vm := createRunningVM(virtClient)
+	libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", 1)
+
+	By("Deleting the VirtualMachine")
+	err := virtClient.VirtualMachine(vm.Namespace).Delete(context.Background(), vm.Name, &metav1.DeleteOptions{})
+	Expect(err).ToNot(HaveOccurred())
+
+	libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", -1)
+}
+
+func createRunningVM(virtClient kubecli.KubevirtClient) *v1.VirtualMachine {
+	vmi := libvmifact.NewGuestless(libvmi.WithNamespace(testsuite.GetTestNamespace(nil)))
+	vm := libvmi.NewVirtualMachine(vmi, libvmi.WithRunning())
+	vm, err := virtClient.VirtualMachine(testsuite.GetTestNamespace(vm)).Create(context.Background(), vm)
+	Expect(err).ToNot(HaveOccurred())
+
+	Eventually(func() bool {
+		vm, err := virtClient.VirtualMachine(testsuite.GetTestNamespace(vm)).Get(context.Background(), vm.Name, &metav1.GetOptions{})
+		Expect(err).ToNot(HaveOccurred())
+		return vm.Status.Ready
+	}, 300*time.Second, 1*time.Second).Should(BeTrue())
+	libwait.WaitForSuccessfulVMIStart(vmi)
+
+	return vm
+}
+
+func gomegaContainsMetricMatcher(metric operatormetrics.Metric, labels map[string]string) types.GomegaMatcher {
+	return &libmonitoring.MetricMatcher{Metric: metric, Labels: labels}
+}