diff --git a/pkg/monitoring/metrics/virt-controller/metrics.go b/pkg/monitoring/metrics/virt-controller/metrics.go index a3e9e59217b0..57c25fb2930f 100644 --- a/pkg/monitoring/metrics/virt-controller/metrics.go +++ b/pkg/monitoring/metrics/virt-controller/metrics.go @@ -85,7 +85,7 @@ func ListMetrics() []operatormetrics.Metric { return operatormetrics.ListMetrics() } -func phaseTransitionTimeBuckets() []float64 { +func PhaseTransitionTimeBuckets() []float64 { return []float64{ (0.5 * time.Second.Seconds()), (1 * time.Second.Seconds()), diff --git a/pkg/monitoring/metrics/virt-controller/migration_metrics.go b/pkg/monitoring/metrics/virt-controller/migration_metrics.go index 8a7a510ed5eb..adacd4883371 100644 --- a/pkg/monitoring/metrics/virt-controller/migration_metrics.go +++ b/pkg/monitoring/metrics/virt-controller/migration_metrics.go @@ -44,7 +44,7 @@ var ( Help: "Histogram of VM migration phase transitions duration from creation time in seconds.", }, prometheus.HistogramOpts{ - Buckets: phaseTransitionTimeBuckets(), + Buckets: PhaseTransitionTimeBuckets(), }, []string{ // phase of the vmi migration diff --git a/pkg/monitoring/metrics/virt-controller/perfscale_metrics.go b/pkg/monitoring/metrics/virt-controller/perfscale_metrics.go index 677c30413de9..75a09d0ad96a 100644 --- a/pkg/monitoring/metrics/virt-controller/perfscale_metrics.go +++ b/pkg/monitoring/metrics/virt-controller/perfscale_metrics.go @@ -48,7 +48,7 @@ var ( Help: "Histogram of VM phase transitions duration between different phases in seconds.", }, prometheus.HistogramOpts{ - Buckets: phaseTransitionTimeBuckets(), + Buckets: PhaseTransitionTimeBuckets(), }, []string{ // phase of the vmi @@ -64,7 +64,7 @@ var ( Help: "Histogram of VM phase transitions duration from creation time in seconds.", }, prometheus.HistogramOpts{ - Buckets: phaseTransitionTimeBuckets(), + Buckets: PhaseTransitionTimeBuckets(), }, []string{ // phase of the vmi @@ -78,7 +78,7 @@ var ( Help: "Histogram of VM phase transitions duration from deletion time in seconds.", }, prometheus.HistogramOpts{ - Buckets: phaseTransitionTimeBuckets(), + Buckets: PhaseTransitionTimeBuckets(), }, []string{ // phase of the vmi diff --git a/tests/libmonitoring/BUILD.bazel b/tests/libmonitoring/BUILD.bazel index 39209570fc9f..3813d8f39a8f 100644 --- a/tests/libmonitoring/BUILD.bazel +++ b/tests/libmonitoring/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "metric_matcher.go", "prometheus.go", "scaling.go", ], @@ -14,8 +15,10 @@ go_library( "//tests/exec:go_default_library", "//tests/flags:go_default_library", "//tests/framework/checks:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", "//vendor/github.com/onsi/ginkgo/v2:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/onsi/gomega/format:go_default_library", "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library", "//vendor/github.com/prometheus/client_golang/api/prometheus/v1:go_default_library", "//vendor/k8s.io/api/autoscaling/v1:go_default_library", diff --git a/tests/libmonitoring/metric_matcher.go b/tests/libmonitoring/metric_matcher.go new file mode 100644 index 000000000000..e73ee725f452 --- /dev/null +++ b/tests/libmonitoring/metric_matcher.go @@ -0,0 +1,90 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright The Kubevirt Authors + * + */ + +package libmonitoring + +import ( + "fmt" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/onsi/gomega/format" +) + +const ( + prometheusMetricNameLabel = "__name__" + prometheusHistogramBucketSuffix = "_bucket" +) + +type MetricMatcher struct { + Metric operatormetrics.Metric + Labels map[string]string +} + +func (matcher *MetricMatcher) FailureMessage(actual interface{}) (message string) { + msg := format.Message(actual, "to contain metric", matcher.Metric.GetOpts().Name) + + if matcher.Labels != nil { + msg += fmt.Sprintf(" with labels %v", matcher.Labels) + } + + return msg +} + +func (matcher *MetricMatcher) NegatedFailureMessage(actual interface{}) (message string) { + msg := format.Message(actual, "not to contain metric", matcher.Metric.GetOpts().Name) + + if matcher.Labels != nil { + msg += fmt.Sprintf(" with labels %v", matcher.Labels) + } + + return msg +} + +func (matcher *MetricMatcher) Match(actual interface{}) (success bool, err error) { + actualMetric, ok := actual.(promResult) + if !ok { + return false, fmt.Errorf("metric matcher requires a libmonitoring.PromResult") + } + + actualName, ok := actualMetric.Metric[prometheusMetricNameLabel] + if !ok { + return false, fmt.Errorf("metric matcher requires a map with %s key", prometheusMetricNameLabel) + } + + nameToMatch := matcher.Metric.GetOpts().Name + if matcher.Metric.GetType() == operatormetrics.HistogramType || matcher.Metric.GetType() == operatormetrics.HistogramVecType { + nameToMatch = nameToMatch + prometheusHistogramBucketSuffix + } + + if actualName != nameToMatch { + return false, nil + } + + for k, v := range matcher.Labels { + actualValue, ok := actualMetric.Metric[k] + if !ok { + return false, nil + } + if actualValue != v { + return false, nil + } + } + + return true, nil +} diff --git a/tests/libmonitoring/prometheus.go b/tests/libmonitoring/prometheus.go index 8fa6dc3bf7bf..9cc442c217f8 100644 --- a/tests/libmonitoring/prometheus.go +++ b/tests/libmonitoring/prometheus.go @@ -145,6 +145,22 @@ func fetchMetric(cli kubecli.KubevirtClient, query string) (*QueryRequestResult, return &result, nil } +func QueryRange(cli kubecli.KubevirtClient, query string, start time.Time, end time.Time, step time.Duration) (*QueryRequestResult, error) { + bodyBytes := DoPrometheusHTTPRequest(cli, fmt.Sprintf("/query_range?query=%s&start=%d&end=%d&step=%d", query, start.Unix(), end.Unix(), int(step.Seconds()))) + + var result QueryRequestResult + err := json.Unmarshal(bodyBytes, &result) + if err != nil { + return nil, err + } + + if result.Status != "success" { + return nil, fmt.Errorf("api request failed. result: %v", result) + } + + return &result, nil +} + func DoPrometheusHTTPRequest(cli kubecli.KubevirtClient, endpoint string) []byte { monitoringNs := getMonitoringNs(cli) diff --git a/tests/monitoring/BUILD.bazel b/tests/monitoring/BUILD.bazel index 05ba23dc12aa..546ea098e84e 100644 --- a/tests/monitoring/BUILD.bazel +++ b/tests/monitoring/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "component_monitoring.go", + "metrics.go", "monitoring.go", "vm_monitoring.go", ], @@ -12,6 +13,9 @@ go_library( deps = [ "//pkg/apimachinery/patch:go_default_library", "//pkg/libvmi:go_default_library", + "//pkg/monitoring/metrics/virt-api:go_default_library", + "//pkg/monitoring/metrics/virt-controller:go_default_library", + "//pkg/monitoring/metrics/virt-operator:go_default_library", "//pkg/virtctl/pause:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/kubecli:go_default_library", @@ -32,10 +36,12 @@ go_library( "//tests/libwait:go_default_library", "//tests/testsuite:go_default_library", "//tests/util:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", "//vendor/github.com/onsi/ginkgo/v2:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/onsi/gomega/types:go_default_library", "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/api/apps/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/rbac/v1:go_default_library", diff --git a/tests/monitoring/metrics.go b/tests/monitoring/metrics.go new file mode 100644 index 000000000000..b13295b479cf --- /dev/null +++ b/tests/monitoring/metrics.go @@ -0,0 +1,155 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright The Kubevirt Authors + * + */ + +package monitoring + +import ( + "context" + "strconv" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/onsi/gomega/types" + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "kubevirt.io/api/core/v1" + "kubevirt.io/client-go/kubecli" + + "kubevirt.io/kubevirt/pkg/libvmi" + virtapi "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api" + virtcontroller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller" + virtoperator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator" + + "kubevirt.io/kubevirt/tests/decorators" + "kubevirt.io/kubevirt/tests/framework/kubevirt" + "kubevirt.io/kubevirt/tests/libmonitoring" + "kubevirt.io/kubevirt/tests/libvmifact" + "kubevirt.io/kubevirt/tests/libwait" + "kubevirt.io/kubevirt/tests/testsuite" +) + +var _ = Describe("[sig-monitoring]Metrics", decorators.SigMonitoring, func() { + var virtClient kubecli.KubevirtClient + var metrics *libmonitoring.QueryRequestResult + + BeforeEach(func() { + virtClient = kubevirt.Client() + setupVM(virtClient) + metrics = fetchPrometheusMetrics(virtClient) + }) + + Context("Prometheus metrics", func() { + var excludedMetrics = map[string]bool{ + // virt-api + // can later be added in pre-existing feature tests + "kubevirt_portforward_active_tunnels": true, + "kubevirt_usbredir_active_connections": true, + "kubevirt_vnc_active_connections": true, + "kubevirt_console_active_connections": true, + + // virt-controller + // needs a migration - ignoring since already tested in - VM Monitoring, VM migration metrics + "kubevirt_vmi_migration_phase_transition_time_from_creation_seconds": true, + "kubevirt_vmi_migrations_in_pending_phase": true, + "kubevirt_vmi_migrations_in_scheduling_phase": true, + "kubevirt_vmi_migrations_in_running_phase": true, + "kubevirt_vmi_migration_succeeded": true, + "kubevirt_vmi_migration_failed": true, + } + + It("should contain virt components metrics", func() { + err := virtoperator.SetupMetrics() + Expect(err).ToNot(HaveOccurred()) + + err = virtapi.SetupMetrics() + Expect(err).ToNot(HaveOccurred()) + + err = virtcontroller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil) + Expect(err).ToNot(HaveOccurred()) + + for _, metric := range operatormetrics.ListMetrics() { + if excludedMetrics[metric.GetOpts().Name] { + continue + } + + Expect(metrics.Data.Result).To(ContainElement(gomegaContainsMetricMatcher(metric, nil))) + } + }) + + It("should have kubevirt_vmi_phase_transition_time_seconds buckets correctly configured", func() { + buckets := virtcontroller.PhaseTransitionTimeBuckets() + + for _, bucket := range buckets { + labels := map[string]string{"le": strconv.FormatFloat(bucket, 'f', -1, 64)} + + metric := operatormetrics.NewHistogram( + operatormetrics.MetricOpts{Name: "kubevirt_vmi_phase_transition_time_from_deletion_seconds"}, + prometheus.HistogramOpts{}, + ) + + Expect(metrics.Data.Result).To(ContainElement(gomegaContainsMetricMatcher(metric, labels))) + } + }) + }) +}) + +func fetchPrometheusMetrics(virtClient kubecli.KubevirtClient) *libmonitoring.QueryRequestResult { + metrics, err := libmonitoring.QueryRange(virtClient, "{__name__=~\"kubevirt_.*\"}", time.Now().Add(-1*time.Minute), time.Now(), 15*time.Second) + Expect(err).ToNot(HaveOccurred()) + + Expect(metrics.Status).To(Equal("success")) + Expect(metrics.Data.ResultType).To(Equal("matrix")) + Expect(metrics.Data.Result).ToNot(BeEmpty(), "No metrics found") + + return metrics +} + +func setupVM(virtClient kubecli.KubevirtClient) { + vm := createRunningVM(virtClient) + libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", 1) + + By("Deleting the VirtualMachine") + err := virtClient.VirtualMachine(vm.Namespace).Delete(context.Background(), vm.Name, &metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + + libmonitoring.WaitForMetricValue(virtClient, "kubevirt_number_of_vms", -1) +} + +func createRunningVM(virtClient kubecli.KubevirtClient) *v1.VirtualMachine { + vmi := libvmifact.NewGuestless(libvmi.WithNamespace(testsuite.GetTestNamespace(nil))) + vm := libvmi.NewVirtualMachine(vmi, libvmi.WithRunning()) + vm, err := virtClient.VirtualMachine(testsuite.GetTestNamespace(vm)).Create(context.Background(), vm) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + vm, err := virtClient.VirtualMachine(testsuite.GetTestNamespace(vm)).Get(context.Background(), vm.Name, &metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return vm.Status.Ready + }, 300*time.Second, 1*time.Second).Should(BeTrue()) + libwait.WaitForSuccessfulVMIStart(vmi) + + return vm +} + +func gomegaContainsMetricMatcher(metric operatormetrics.Metric, labels map[string]string) types.GomegaMatcher { + return &libmonitoring.MetricMatcher{Metric: metric, Labels: labels} +}