diff --git a/pkg/monitoring/metrics/virt-controller/metrics.go b/pkg/monitoring/metrics/virt-controller/metrics.go index a3e9e59217b0..57c25fb2930f 100644 --- a/pkg/monitoring/metrics/virt-controller/metrics.go +++ b/pkg/monitoring/metrics/virt-controller/metrics.go @@ -85,7 +85,7 @@ func ListMetrics() []operatormetrics.Metric { return operatormetrics.ListMetrics() } -func phaseTransitionTimeBuckets() []float64 { +func PhaseTransitionTimeBuckets() []float64 { return []float64{ (0.5 * time.Second.Seconds()), (1 * time.Second.Seconds()), diff --git a/pkg/monitoring/metrics/virt-controller/migration_metrics.go b/pkg/monitoring/metrics/virt-controller/migration_metrics.go index 8a7a510ed5eb..adacd4883371 100644 --- a/pkg/monitoring/metrics/virt-controller/migration_metrics.go +++ b/pkg/monitoring/metrics/virt-controller/migration_metrics.go @@ -44,7 +44,7 @@ var ( Help: "Histogram of VM migration phase transitions duration from creation time in seconds.", }, prometheus.HistogramOpts{ - Buckets: phaseTransitionTimeBuckets(), + Buckets: PhaseTransitionTimeBuckets(), }, []string{ // phase of the vmi migration diff --git a/pkg/monitoring/metrics/virt-controller/perfscale_metrics.go b/pkg/monitoring/metrics/virt-controller/perfscale_metrics.go index 677c30413de9..75a09d0ad96a 100644 --- a/pkg/monitoring/metrics/virt-controller/perfscale_metrics.go +++ b/pkg/monitoring/metrics/virt-controller/perfscale_metrics.go @@ -48,7 +48,7 @@ var ( Help: "Histogram of VM phase transitions duration between different phases in seconds.", }, prometheus.HistogramOpts{ - Buckets: phaseTransitionTimeBuckets(), + Buckets: PhaseTransitionTimeBuckets(), }, []string{ // phase of the vmi @@ -64,7 +64,7 @@ var ( Help: "Histogram of VM phase transitions duration from creation time in seconds.", }, prometheus.HistogramOpts{ - Buckets: phaseTransitionTimeBuckets(), + Buckets: PhaseTransitionTimeBuckets(), }, []string{ // phase of the vmi @@ -78,7 +78,7 @@ var ( Help: "Histogram of VM phase transitions duration from deletion time in seconds.", }, prometheus.HistogramOpts{ - Buckets: phaseTransitionTimeBuckets(), + Buckets: PhaseTransitionTimeBuckets(), }, []string{ // phase of the vmi diff --git a/tests/libmonitoring/BUILD.bazel b/tests/libmonitoring/BUILD.bazel index 941fb8ce9104..9df38eb235cb 100644 --- a/tests/libmonitoring/BUILD.bazel +++ b/tests/libmonitoring/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "metric_matcher.go", "prometheus.go", "scaling.go", ], @@ -15,8 +16,10 @@ go_library( "//tests/flags:go_default_library", "//tests/framework/checks:go_default_library", "//tests/framework/kubevirt:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", "//vendor/github.com/onsi/ginkgo/v2:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/onsi/gomega/format:go_default_library", "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library", "//vendor/github.com/prometheus/client_golang/api/prometheus/v1:go_default_library", "//vendor/k8s.io/api/autoscaling/v1:go_default_library", diff --git a/tests/libmonitoring/metric_matcher.go b/tests/libmonitoring/metric_matcher.go new file mode 100644 index 000000000000..1fb4593c3e00 --- /dev/null +++ b/tests/libmonitoring/metric_matcher.go @@ -0,0 +1,70 @@ +package libmonitoring + +import ( + "fmt" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/onsi/gomega/format" +) + +type MetricMatcher struct { + Metric operatormetrics.Metric + Labels map[string]string +} + +func (matcher *MetricMatcher) FailureMessage(actual interface{}) (message string) { + msg := format.Message(actual, "to contain metric", matcher.Metric.GetOpts().Name) + + if matcher.Labels != nil { + msg += fmt.Sprintf(" with labels %v", matcher.Labels) + } + + return msg +} + +func (matcher *MetricMatcher) NegatedFailureMessage(actual interface{}) (message string) { + msg := format.Message(actual, "not to contain metric", matcher.Metric.GetOpts().Name) + + if matcher.Labels != nil { + msg += fmt.Sprintf(" with labels %v", matcher.Labels) + } + + return msg +} + +func (matcher *MetricMatcher) Match(actual interface{}) (success bool, err error) { + actualMetric, ok := actual.(promResult) + if !ok { + return false, fmt.Errorf("metric matcher requires a libmonitoring.PromResult") + } + + actualName, ok := actualMetric.Metric["__name__"] + if !ok { + return false, fmt.Errorf("metric matcher requires a map with __name__ key") + } + + nameToMatch := matcher.Metric.GetOpts().Name + if matcher.Metric.GetType() == operatormetrics.HistogramType || matcher.Metric.GetType() == operatormetrics.HistogramVecType { + nameToMatch = nameToMatch + "_bucket" + } + + if actualName != nameToMatch { + return false, nil + } + + if matcher.Labels == nil { + return true, nil + } + + for k, v := range matcher.Labels { + actualValue, ok := actualMetric.Metric[k] + if !ok { + return false, nil + } + if actualValue != v { + return false, nil + } + } + + return true, nil +} diff --git a/tests/libmonitoring/prometheus.go b/tests/libmonitoring/prometheus.go index 304309cba377..6ec8250c673f 100644 --- a/tests/libmonitoring/prometheus.go +++ b/tests/libmonitoring/prometheus.go @@ -146,6 +146,22 @@ func fetchMetric(cli kubecli.KubevirtClient, query string) (*QueryRequestResult, return &result, nil } +func QueryRange(cli kubecli.KubevirtClient, query string, start time.Time, end time.Time, step time.Duration) (*QueryRequestResult, error) { + bodyBytes := DoPrometheusHTTPRequest(cli, fmt.Sprintf("/query_range?query=%s&start=%d&end=%d&step=%d", query, start.Unix(), end.Unix(), int(step.Seconds()))) + + var result QueryRequestResult + err := json.Unmarshal(bodyBytes, &result) + if err != nil { + return nil, err + } + + if result.Status != "success" { + return nil, fmt.Errorf("api request failed. result: %v", result) + } + + return &result, nil +} + func DoPrometheusHTTPRequest(cli kubecli.KubevirtClient, endpoint string) []byte { monitoringNs := getMonitoringNs(cli) diff --git a/tests/monitoring/BUILD.bazel b/tests/monitoring/BUILD.bazel index 8a198c6db98c..98d292da5082 100644 --- a/tests/monitoring/BUILD.bazel +++ b/tests/monitoring/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "component_monitoring.go", + "metrics.go", "monitoring.go", "vm_monitoring.go", ], @@ -11,6 +12,9 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/apimachinery/patch:go_default_library", + "//pkg/monitoring/metrics/virt-api:go_default_library", + "//pkg/monitoring/metrics/virt-controller:go_default_library", + "//pkg/monitoring/metrics/virt-operator:go_default_library", "//pkg/virtctl/pause:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/kubecli:go_default_library", @@ -30,6 +34,7 @@ go_library( "//tests/libwait:go_default_library", "//tests/testsuite:go_default_library", "//tests/util:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", "//vendor/github.com/onsi/ginkgo/v2:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/onsi/gomega/types:go_default_library", diff --git a/tests/monitoring/metrics.go b/tests/monitoring/metrics.go new file mode 100644 index 000000000000..ccfb42d8b1c5 --- /dev/null +++ b/tests/monitoring/metrics.go @@ -0,0 +1,151 @@ +package monitoring + +import ( + "context" + "strconv" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/onsi/gomega/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "kubevirt.io/api/core/v1" + "kubevirt.io/client-go/kubecli" + + virtapi "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api" + virtcontroller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller" + virtoperator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator" + + "kubevirt.io/kubevirt/tests/decorators" + "kubevirt.io/kubevirt/tests/framework/kubevirt" + "kubevirt.io/kubevirt/tests/libmonitoring" + "kubevirt.io/kubevirt/tests/libvmi" + "kubevirt.io/kubevirt/tests/libwait" + "kubevirt.io/kubevirt/tests/testsuite" +) + +var _ = Describe("[Serial][sig-monitoring]Metrics", Serial, decorators.SigMonitoring, func() { + var virtClient kubecli.KubevirtClient + var metrics *libmonitoring.QueryRequestResult + var err error + + setupVM := func() { + vmi := libvmi.NewCirros( + libvmi.WithInterface(libvmi.InterfaceDeviceWithMasqueradeBinding()), + libvmi.WithNetwork(v1.DefaultPodNetwork()), + ) + vmi.Namespace = testsuite.GetTestNamespace(nil) + + By("Creating a running VirtualMachine") + vm := libvmi.NewVirtualMachine(vmi, libvmi.WithRunning()) + _, err = virtClient.VirtualMachine(vm.Namespace).Create(context.Background(), vm) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + vmi, err = kubevirt.Client().VirtualMachineInstance(vm.Namespace).Get(context.Background(), vm.Name, &metav1.GetOptions{}) + return err + }, 120*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + libwait.WaitForSuccessfulVMIStart(vmi) + + libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", 1) + + By("Deleting the VirtualMachine") + err = virtClient.VirtualMachine(vm.Namespace).Delete(context.Background(), vm.Name, &metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + + libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", -1) + } + + BeforeEach(func() { + virtClient = kubevirt.Client() + + setupVM() + + metrics, err = libmonitoring.QueryRange(virtClient, "{__name__=~\"kubevirt_.*\"}", time.Now().Add(-1*time.Minute), time.Now(), 15*time.Second) + Expect(err).ToNot(HaveOccurred()) + + Expect(metrics.Status).To(Equal("success")) + Expect(metrics.Data.ResultType).To(Equal("matrix")) + Expect(metrics.Data.Result).ToNot(BeEmpty(), "No metrics found") + }) + + containsMetric := func(metric operatormetrics.Metric, labels map[string]string) types.GomegaMatcher { + return &libmonitoring.MetricMatcher{Metric: metric, Labels: labels} + } + + Context("Prometheus metrics", func() { + var excludedMetrics = map[string]bool{ + // virt-api + // can later be added in pre-existing feature tests + "kubevirt_portforward_active_tunnels": true, + "kubevirt_usbredir_active_connections": true, + "kubevirt_vnc_active_connections": true, + "kubevirt_console_active_connections": true, + + // virt-controller + // needs a migration - ignoring since already tested in - VM Monitoring, VM migration metrics + "kubevirt_vmi_migration_phase_transition_time_from_creation_seconds": true, + "kubevirt_vmi_migrations_in_pending_phase": true, + "kubevirt_vmi_migrations_in_scheduling_phase": true, + "kubevirt_vmi_migrations_in_running_phase": true, + "kubevirt_vmi_migration_succeeded": true, + "kubevirt_vmi_migration_failed": true, + } + + It("should contain virt-operator metrics", func() { + err = virtoperator.SetupMetrics() + Expect(err).ToNot(HaveOccurred()) + + for _, metric := range virtoperator.ListMetrics() { + Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric, nil))) + } + }) + + It("should contain virt-api metrics", func() { + err = virtapi.SetupMetrics() + Expect(err).ToNot(HaveOccurred()) + + for _, metric := range virtapi.ListMetrics() { + if excludedMetrics[metric.GetOpts().Name] { + continue + } + + Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric, nil))) + } + }) + + It("should contain virt-controller metrics", func() { + err = virtcontroller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil) + Expect(err).ToNot(HaveOccurred()) + + for _, metric := range virtcontroller.ListMetrics() { + if excludedMetrics[metric.GetOpts().Name] { + continue + } + + Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric, nil))) + } + }) + + Context("perfscale histogram metrics", func() { + It("should have kubevirt_vmi_phase_transition_time_seconds buckets correctly configured", func() { + buckets := virtcontroller.PhaseTransitionTimeBuckets() + + for _, bucket := range buckets { + labels := map[string]string{"le": strconv.FormatFloat(bucket, 'f', -1, 64)} + + GinkgoLogr.Info("%+v", labels) + + metric := operatormetrics.NewHistogram( + operatormetrics.MetricOpts{Name: "kubevirt_vmi_phase_transition_time_from_deletion_seconds"}, + operatormetrics.HistogramOpts{}, + ) + + Expect(metrics.Data.Result).To(ContainElement(containsMetric(metric, labels))) + } + }) + }) + }) +})