From 8b83a1ecc4e8f50f3c3c28c1e08c65a1dc5cd20a Mon Sep 17 00:00:00 2001 From: machadovilaca Date: Mon, 25 Mar 2024 12:33:22 +0000 Subject: [PATCH 1/5] Move virt-handler prometheus handler to pkg/monitoring/metrics/virt-handler Signed-off-by: machadovilaca --- .../domainstats/prometheus/prometheus.go | 14 ------- .../metrics/virt-handler/handler/BUILD.bazel | 12 ++++++ .../metrics/virt-handler/handler/handler.go | 38 +++++++++++++++++++ 3 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 pkg/monitoring/metrics/virt-handler/handler/BUILD.bazel create mode 100644 pkg/monitoring/metrics/virt-handler/handler/handler.go diff --git a/pkg/monitoring/domainstats/prometheus/prometheus.go b/pkg/monitoring/domainstats/prometheus/prometheus.go index 9785660bd9cf..528e877b2895 100644 --- a/pkg/monitoring/domainstats/prometheus/prometheus.go +++ b/pkg/monitoring/domainstats/prometheus/prometheus.go @@ -21,7 +21,6 @@ package prometheus import ( "fmt" - "net/http" "strings" "time" @@ -30,8 +29,6 @@ import ( vms "kubevirt.io/kubevirt/pkg/monitoring/domainstats" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - k6tv1 "kubevirt.io/api/core/v1" "kubevirt.io/client-go/kubecli" "kubevirt.io/client-go/log" @@ -685,17 +682,6 @@ func (ps *prometheusScraper) Report(socketFile string, vmi *k6tv1.VirtualMachine vmiMetrics.updateMetrics(vmi, vmStats) } -func Handler(MaxRequestsInFlight int) http.Handler { - return promhttp.InstrumentMetricHandler( - prometheus.DefaultRegisterer, - promhttp.HandlerFor( - prometheus.DefaultGatherer, - promhttp.HandlerOpts{ - MaxRequestsInFlight: MaxRequestsInFlight, - }), - ) -} - type vmiMetrics struct { k8sLabels []string k8sLabelValues []string diff --git a/pkg/monitoring/metrics/virt-handler/handler/BUILD.bazel b/pkg/monitoring/metrics/virt-handler/handler/BUILD.bazel new file mode 100644 index 000000000000..bcd9cd506a3f --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/handler/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["handler.go"], + importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/handler", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", + ], +) diff --git a/pkg/monitoring/metrics/virt-handler/handler/handler.go b/pkg/monitoring/metrics/virt-handler/handler/handler.go new file mode 100644 index 000000000000..4d05039ba532 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/handler/handler.go @@ -0,0 +1,38 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package handler + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func Handler(MaxRequestsInFlight int) http.Handler { + return promhttp.InstrumentMetricHandler( + prometheus.DefaultRegisterer, + promhttp.HandlerFor( + prometheus.DefaultGatherer, + promhttp.HandlerOpts{ + MaxRequestsInFlight: MaxRequestsInFlight, + }), + ) +} From f42232c6a82d44620f3ed8c0295e85aba6cf9eeb Mon Sep 17 00:00:00 2001 From: machadovilaca Date: Mon, 25 Mar 2024 12:34:15 +0000 Subject: [PATCH 2/5] Move virt-handler version metric Signed-off-by: machadovilaca --- .../domainstats/prometheus/prometheus.go | 20 --------- .../metrics/virt-handler/metrics.go | 12 ++++- .../metrics/virt-handler/version_metrics.go | 44 +++++++++++++++++++ 3 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 pkg/monitoring/metrics/virt-handler/version_metrics.go diff --git a/pkg/monitoring/domainstats/prometheus/prometheus.go b/pkg/monitoring/domainstats/prometheus/prometheus.go index 528e877b2895..98e7de721881 100644 --- a/pkg/monitoring/domainstats/prometheus/prometheus.go +++ b/pkg/monitoring/domainstats/prometheus/prometheus.go @@ -32,7 +32,6 @@ import ( k6tv1 "kubevirt.io/api/core/v1" "kubevirt.io/client-go/kubecli" "kubevirt.io/client-go/log" - "kubevirt.io/client-go/version" cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client" "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" @@ -53,14 +52,6 @@ var ( // Preffixes used when transforming K8s metadata into metric labels labelPrefix = "kubernetes_vmi_label_" - - // see https://www.robustperception.io/exposing-the-software-version-to-prometheus - versionDesc = prometheus.NewDesc( - "kubevirt_info", - "Version information", - []string{"goversion", "kubeversion"}, - nil, - ) ) func tryToPushMetric(desc *prometheus.Desc, mv prometheus.Metric, err error, ch chan<- prometheus.Metric) { @@ -552,15 +543,6 @@ func (metrics *vmiMetrics) updateFilesystem(vmFSStats k6tv1.VirtualMachineInstan } } -func updateVersion(ch chan<- prometheus.Metric) { - verinfo := version.Get() - ch <- prometheus.MustNewConstMetric( - versionDesc, prometheus.GaugeValue, - 1.0, - verinfo.GoVersion, verinfo.GitVersion, - ) -} - type DomainStatsCollector struct { virtShareDir string nodeName string @@ -588,8 +570,6 @@ func (co *DomainStatsCollector) Describe(_ chan<- *prometheus.Desc) { // Note that Collect could be called concurrently func (co *DomainStatsCollector) Collect(ch chan<- prometheus.Metric) { - updateVersion(ch) - cachedObjs := co.vmiInformer.GetIndexer().List() if len(cachedObjs) == 0 { log.Log.V(4).Infof("No VMIs detected") diff --git a/pkg/monitoring/metrics/virt-handler/metrics.go b/pkg/monitoring/metrics/virt-handler/metrics.go index 8ef0e023711e..0544d966bf5d 100644 --- a/pkg/monitoring/metrics/virt-handler/metrics.go +++ b/pkg/monitoring/metrics/virt-handler/metrics.go @@ -23,6 +23,7 @@ import ( "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats" ) func SetupMetrics() error { @@ -30,7 +31,16 @@ func SetupMetrics() error { return err } - return client.SetupMetrics() + if err := client.SetupMetrics(); err != nil { + return err + } + + if err := operatormetrics.RegisterMetrics(versionMetrics); err != nil { + return err + } + SetVersionInfo() + + return operatormetrics.RegisterCollector(domainstats.Collector) } func ListMetrics() []operatormetrics.Metric { diff --git a/pkg/monitoring/metrics/virt-handler/version_metrics.go b/pkg/monitoring/metrics/virt-handler/version_metrics.go new file mode 100644 index 000000000000..3d57a26e37e3 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/version_metrics.go @@ -0,0 +1,44 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package virt_handler + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "kubevirt.io/client-go/version" +) + +var ( + versionMetrics = []operatormetrics.Metric{ + versionInfo, + } + + versionInfo = operatormetrics.NewGaugeVec( + operatormetrics.MetricOpts{ + Name: "kubevirt_info", + Help: "Version information.", + }, + []string{"goversion", "kubeversion"}, + ) +) + +func SetVersionInfo() { + info := version.Get() + versionInfo.WithLabelValues(info.GoVersion, info.GitVersion).Set(1) +} From bdc3f9adae4dcefdb3193a0a196c79a8accaabd0 Mon Sep 17 00:00:00 2001 From: machadovilaca Date: Mon, 25 Mar 2024 12:36:08 +0000 Subject: [PATCH 3/5] Refactor domainstats metrics Signed-off-by: machadovilaca --- cmd/virt-handler/BUILD.bazel | 2 +- .../domainstats/prometheus/BUILD.bazel | 41 - .../domainstats/prometheus/prometheus.go | 739 ---------- .../prometheus/prometheus_suite_test.go | 11 - .../domainstats/prometheus/prometheus_test.go | 1277 ----------------- .../metrics/virt-handler/BUILD.bazel | 7 +- .../virt-handler/domainstats/BUILD.bazel | 42 + .../virt-handler/domainstats/block_metrics.go | 152 ++ .../domainstats/block_metrics_test.go | 91 ++ .../virt-handler/domainstats/collector.go | 89 ++ .../domainstats/collector_test.go | 121 ++ .../virt-handler/domainstats/cpu_metrics.go | 86 ++ .../domainstats/cpu_metrics_test.go | 74 + .../virt-handler/domainstats/domainstats.go | 93 ++ .../domainstats/domainstats_suite_test.go | 63 + .../domainstats/domainstats_test.go | 79 + .../domainstats/filesystem_metrics.go | 64 + .../domainstats/filesystem_metrics_test.go | 70 + .../domainstats/memory_metrics.go | 177 +++ .../domainstats/memory_metrics_test.go | 100 ++ .../domainstats/migration_metrics.go | 93 ++ .../domainstats/migration_metrics_test.go | 77 + .../domainstats/network_metrics.go | 161 +++ .../domainstats/network_metrics_test.go | 91 ++ .../domainstats/node_cpu_affinity_metrics.go | 57 + .../node_cpu_affinity_metrics_test.go | 67 + .../virt-handler/domainstats/scrapper.go | 116 ++ .../domainstats/unit_converter.go | 13 + .../virt-handler/domainstats/vcpu_metrics.go | 110 ++ .../domainstats/vcpu_metrics_test.go | 78 + .../metrics/virt-handler/metrics.go | 1 + 31 files changed, 2172 insertions(+), 2070 deletions(-) delete mode 100644 pkg/monitoring/domainstats/prometheus/BUILD.bazel delete mode 100644 pkg/monitoring/domainstats/prometheus/prometheus.go delete mode 100644 pkg/monitoring/domainstats/prometheus/prometheus_suite_test.go delete mode 100644 pkg/monitoring/domainstats/prometheus/prometheus_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/BUILD.bazel create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/block_metrics.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/block_metrics_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/collector.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/collector_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/cpu_metrics.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/cpu_metrics_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/domainstats.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/domainstats_suite_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/domainstats_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/filesystem_metrics.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/filesystem_metrics_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/network_metrics.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/network_metrics_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/node_cpu_affinity_metrics.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/node_cpu_affinity_metrics_test.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/scrapper.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/unit_converter.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/vcpu_metrics.go create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/vcpu_metrics_test.go diff --git a/cmd/virt-handler/BUILD.bazel b/cmd/virt-handler/BUILD.bazel index cff502536586..980562235c90 100644 --- a/cmd/virt-handler/BUILD.bazel +++ b/cmd/virt-handler/BUILD.bazel @@ -11,8 +11,8 @@ go_library( "//pkg/controller:go_default_library", "//pkg/healthz:go_default_library", "//pkg/monitoring/domainstats/downwardmetrics:go_default_library", - "//pkg/monitoring/domainstats/prometheus:go_default_library", "//pkg/monitoring/metrics/virt-handler:go_default_library", + "//pkg/monitoring/metrics/virt-handler/handler:go_default_library", "//pkg/monitoring/profiler:go_default_library", "//pkg/safepath:go_default_library", "//pkg/service:go_default_library", diff --git a/pkg/monitoring/domainstats/prometheus/BUILD.bazel b/pkg/monitoring/domainstats/prometheus/BUILD.bazel deleted file mode 100644 index 19f09446a167..000000000000 --- a/pkg/monitoring/domainstats/prometheus/BUILD.bazel +++ /dev/null @@ -1,41 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["prometheus.go"], - importpath = "kubevirt.io/kubevirt/pkg/monitoring/domainstats/prometheus", - visibility = ["//visibility:public"], - deps = [ - "//pkg/monitoring/domainstats:go_default_library", - "//pkg/virt-handler/cmd-client:go_default_library", - "//pkg/virt-launcher/virtwrap/stats:go_default_library", - "//staging/src/kubevirt.io/api/core/v1:go_default_library", - "//staging/src/kubevirt.io/client-go/kubecli:go_default_library", - "//staging/src/kubevirt.io/client-go/log:go_default_library", - "//staging/src/kubevirt.io/client-go/version:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", - "//vendor/k8s.io/client-go/tools/cache:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "prometheus_suite_test.go", - "prometheus_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//pkg/pointer:go_default_library", - "//pkg/virt-launcher/virtwrap/stats:go_default_library", - "//staging/src/kubevirt.io/api/core/v1:go_default_library", - "//staging/src/kubevirt.io/client-go/testutils:go_default_library", - "//vendor/github.com/onsi/ginkgo/v2:go_default_library", - "//vendor/github.com/onsi/gomega:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/github.com/prometheus/client_model/go:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/libvirt.org/go/libvirt:go_default_library", - ], -) diff --git a/pkg/monitoring/domainstats/prometheus/prometheus.go b/pkg/monitoring/domainstats/prometheus/prometheus.go deleted file mode 100644 index 98e7de721881..000000000000 --- a/pkg/monitoring/domainstats/prometheus/prometheus.go +++ /dev/null @@ -1,739 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2018 Red Hat, Inc. - * - */ - -package prometheus - -import ( - "fmt" - "strings" - "time" - - "k8s.io/client-go/tools/cache" - - vms "kubevirt.io/kubevirt/pkg/monitoring/domainstats" - - "github.com/prometheus/client_golang/prometheus" - k6tv1 "kubevirt.io/api/core/v1" - "kubevirt.io/client-go/kubecli" - "kubevirt.io/client-go/log" - - cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client" - "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" -) - -const ( - PrometheusCollectionTimeout = vms.CollectionTimeout - MigrateVmiDataRemainingMetricName = "kubevirt_vmi_migration_data_remaining_bytes" - MigrateVmiDataProcessedMetricName = "kubevirt_vmi_migration_data_processed_bytes" - MigrateVmiDirtyMemoryRateMetricName = "kubevirt_vmi_migration_dirty_memory_rate_bytes" - MigrateVmiMemoryTransferRateMetricName = "kubevirt_vmi_migration_disk_transfer_rate_bytes" -) - -var ( - - // Formatter used to sanitize k8s metadata into metric labels - labelFormatter = strings.NewReplacer(".", "_", "/", "_", "-", "_") - - // Preffixes used when transforming K8s metadata into metric labels - labelPrefix = "kubernetes_vmi_label_" -) - -func tryToPushMetric(desc *prometheus.Desc, mv prometheus.Metric, err error, ch chan<- prometheus.Metric) { - if err != nil { - log.Log.Warningf("Error creating the new const metric for %s: %s", desc, err) - return - } - ch <- mv -} - -func (metrics *vmiMetrics) updateMigrateInfo(jobInfo *stats.DomainJobInfo) { - if jobInfo.DataRemainingSet { - metrics.pushCommonMetric( - MigrateVmiDataRemainingMetricName, - "Number of bytes that still need to be transferred", - prometheus.GaugeValue, - float64(jobInfo.DataRemaining), - ) - } - - if jobInfo.DataProcessedSet { - metrics.pushCommonMetric( - MigrateVmiDataProcessedMetricName, - "Number of bytes transferred from the beginning of the job.", - prometheus.GaugeValue, - float64(jobInfo.DataProcessed), - ) - } - - if jobInfo.MemDirtyRateSet { - metrics.pushCommonMetric( - MigrateVmiDirtyMemoryRateMetricName, - "Number of memory pages dirtied by the guest per second.", - prometheus.GaugeValue, - float64(jobInfo.MemDirtyRate), - ) - } - - if jobInfo.MemoryBpsSet { - metrics.pushCommonMetric( - MigrateVmiMemoryTransferRateMetricName, - "Network throughput used while migrating memory in Bytes per second.", - prometheus.GaugeValue, - float64(jobInfo.MemoryBps), - ) - } -} - -func (metrics *vmiMetrics) updateMemory(mem *stats.DomainStatsMemory) { - if mem.RSSSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_resident_bytes", - "Resident set size of the process running the domain.", - prometheus.GaugeValue, - float64(mem.RSS)*1024, - ) - } - - if mem.AvailableSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_available_bytes", - "amount of usable memory as seen by the domain. This value may not be accurate if a balloon driver is in use or if the guest OS does not initialize all assigned pages", - prometheus.GaugeValue, - float64(mem.Available)*1024, - ) - } - - if mem.UnusedSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_unused_bytes", - "The amount of memory left completely unused by the system. Memory that is available but used for reclaimable caches should NOT be reported as free.", - prometheus.GaugeValue, - float64(mem.Unused)*1024, - ) - } - - if mem.CachedSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_cached_bytes", - "The amount of memory that is being used to cache I/O and is available to be reclaimed, corresponds to the sum of `Buffers` + `Cached` + `SwapCached` in `/proc/meminfo`.", - prometheus.GaugeValue, - float64(mem.Cached)*1024, - ) - } - - if mem.SwapInSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_swap_in_traffic_bytes", - "The total amount of data read from swap space of the guest in bytes.", - prometheus.GaugeValue, - float64(mem.SwapIn)*1024, - ) - } - - if mem.SwapOutSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_swap_out_traffic_bytes", - "The total amount of memory written out to swap space of the guest in bytes.", - prometheus.GaugeValue, - float64(mem.SwapOut)*1024, - ) - } - - if mem.MajorFaultSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_pgmajfault_total", - "The number of page faults when disk IO was required. Page faults occur when a process makes a valid access to virtual memory that is not available. When servicing the page fault, if disk IO is required, it is considered as major fault.", - prometheus.CounterValue, - float64(mem.MajorFault), - ) - } - - if mem.MinorFaultSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_pgminfault_total", - "The number of other page faults, when disk IO was not required. Page faults occur when a process makes a valid access to virtual memory that is not available. When servicing the page fault, if disk IO is NOT required, it is considered as minor fault.", - prometheus.CounterValue, - float64(mem.MinorFault), - ) - } - - if mem.ActualBalloonSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_actual_balloon_bytes", - "Current balloon size in bytes.", - prometheus.GaugeValue, - float64(mem.ActualBalloon)*1024, - ) - } - - if mem.UsableSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_usable_bytes", - "The amount of memory which can be reclaimed by balloon without pushing the guest system to swap, corresponds to 'Available' in /proc/meminfo", - prometheus.GaugeValue, - float64(mem.Usable)*1024, - ) - } - - if mem.TotalSet { - metrics.pushCommonMetric( - "kubevirt_vmi_memory_domain_bytes", - "The amount of memory in bytes allocated to the domain. The `memory` value in domain xml file.", - prometheus.GaugeValue, - float64(mem.Total)*1024, - ) - } -} - -func (metrics *vmiMetrics) updateCPUAffinity(cpuMap [][]bool) { - affinityCount := 0.0 - - for vidx := 0; vidx < len(cpuMap); vidx++ { - for cidx := 0; cidx < len(cpuMap[vidx]); cidx++ { - if cpuMap[vidx][cidx] { - affinityCount++ - } - } - } - - metrics.pushCustomMetric( - "kubevirt_vmi_node_cpu_affinity", - "Number of VMI CPU affinities to node physical cores.", - prometheus.GaugeValue, affinityCount, - nil, nil, - ) -} - -func nanosecondsToSeconds(ns uint64) float64 { - return float64(ns) / 1000000000 -} - -func (metrics *vmiMetrics) updateCPU(vmi *k6tv1.VirtualMachineInstance, domainCPUStats *stats.DomainStatsCPU) { - if !domainCPUStats.TimeSet && !domainCPUStats.UserSet && !domainCPUStats.SystemSet { - log.Log.Warningf("No domain CPU stats is set for %s VMI.", vmi.Name) - } - - if domainCPUStats.TimeSet { - metrics.pushCommonMetric( - "kubevirt_vmi_cpu_usage_seconds_total", - "Total CPU time spent in all modes (sum of both vcpu and hypervisor usage).", - prometheus.CounterValue, - nanosecondsToSeconds(domainCPUStats.Time), - ) - } - - if domainCPUStats.UserSet { - metrics.pushCommonMetric( - "kubevirt_vmi_cpu_user_usage_seconds_total", - "Total CPU time spent in user mode.", - prometheus.CounterValue, - nanosecondsToSeconds(domainCPUStats.User), - ) - } - - if domainCPUStats.SystemSet { - metrics.pushCommonMetric( - "kubevirt_vmi_cpu_system_usage_seconds_total", - "Total CPU time spent in system mode.", - prometheus.CounterValue, - nanosecondsToSeconds(domainCPUStats.System), - ) - } -} - -func (metrics *vmiMetrics) updateVcpu(vcpuStats []stats.DomainStatsVcpu) { - for vcpuIdx, vcpu := range vcpuStats { - stringVcpuIdx := fmt.Sprintf("%d", vcpuIdx) - - if vcpu.StateSet && vcpu.TimeSet { - metrics.pushCustomMetric( - "kubevirt_vmi_vcpu_seconds_total", - "Total amount of time spent in each state by each vcpu (cpu_time excluding hypervisor time). Where `id` is the vcpu identifier and `state` can be one of the following: [`OFFLINE`, `RUNNING`, `BLOCKED`].", - prometheus.CounterValue, - float64(vcpu.Time/1000000), - []string{"id", "state"}, - []string{stringVcpuIdx, humanReadableState(vcpu.State)}, - ) - } - - if vcpu.WaitSet { - metrics.pushCustomMetric( - "kubevirt_vmi_vcpu_wait_seconds_total", - "Amount of time spent by each vcpu while waiting on I/O.", - prometheus.CounterValue, - float64(vcpu.Wait)/float64(1000000), - []string{"id"}, - []string{stringVcpuIdx}, - ) - } - if vcpu.DelaySet { - metrics.pushCustomMetric( - "kubevirt_vmi_vcpu_delay_seconds_total", - "Amount of time spent by each vcpu waiting in the queue instead of running.", - prometheus.CounterValue, - float64(vcpu.Delay)/float64(1000000000), - []string{"id"}, - []string{stringVcpuIdx}, - ) - } - } -} - -func (metrics *vmiMetrics) updateBlock(blkStats []stats.DomainStatsBlock) { - for blockIdx, block := range blkStats { - if !block.NameSet { - log.Log.Warningf("Name not set for block device#%d", blockIdx) - continue - } - - blkLabels := []string{"drive"} - blkLabelValues := []string{block.Name} - - if block.Alias != "" { - blkLabelValues[0] = block.Alias - } - - if block.RdReqsSet { - metrics.pushCustomMetric( - "kubevirt_vmi_storage_iops_read_total", - "Total number of I/O read operations.", - prometheus.CounterValue, - float64(block.RdReqs), - blkLabels, - blkLabelValues, - ) - } - - if block.WrReqsSet { - metrics.pushCustomMetric( - "kubevirt_vmi_storage_iops_write_total", - "Total number of I/O write operations.", - prometheus.CounterValue, - float64(block.WrReqs), - blkLabels, - blkLabelValues, - ) - } - - if block.RdBytesSet { - metrics.pushCustomMetric( - "kubevirt_vmi_storage_read_traffic_bytes_total", - "Total number of bytes read from storage.", - prometheus.CounterValue, - float64(block.RdBytes), - blkLabels, - blkLabelValues, - ) - } - - if block.WrBytesSet { - metrics.pushCustomMetric( - "kubevirt_vmi_storage_write_traffic_bytes_total", - "Total number of written bytes.", - prometheus.CounterValue, - float64(block.WrBytes), - blkLabels, - blkLabelValues, - ) - } - - if block.RdTimesSet { - metrics.pushCustomMetric( - "kubevirt_vmi_storage_read_times_seconds_total", - "Total time spent on read operations.", - prometheus.CounterValue, - float64(block.RdTimes)/1000000000, - blkLabels, - blkLabelValues, - ) - } - - if block.WrTimesSet { - metrics.pushCustomMetric( - "kubevirt_vmi_storage_write_times_seconds_total", - "Total time spent on write operations.", - prometheus.CounterValue, - float64(block.WrTimes)/1000000000, - blkLabels, - blkLabelValues, - ) - } - - if block.FlReqsSet { - metrics.pushCustomMetric( - "kubevirt_vmi_storage_flush_requests_total", - "Total storage flush requests.", - prometheus.CounterValue, - float64(block.FlReqs), - blkLabels, - blkLabelValues, - ) - } - - if block.FlTimesSet { - metrics.pushCustomMetric( - "kubevirt_vmi_storage_flush_times_seconds_total", - "Total time spent on cache flushing.", - prometheus.CounterValue, - float64(block.FlTimes)/1000000000, - blkLabels, - blkLabelValues, - ) - } - } -} - -func (metrics *vmiMetrics) updateNetwork(netStats []stats.DomainStatsNet) { - for _, net := range netStats { - if !net.NameSet { - continue - } - - ifaceLabel := net.Name - if net.AliasSet { - ifaceLabel = net.Alias - } - - netLabels := []string{"interface"} - netLabelValues := []string{ifaceLabel} - - if net.RxBytesSet || net.TxBytesSet { - desc := metrics.newPrometheusDesc( - "kubevirt_vmi_network_traffic_bytes_total", - "deprecated.", - []string{"interface", "type"}, - ) - - if net.RxBytesSet { - metrics.pushPrometheusMetric(desc, prometheus.CounterValue, float64(net.RxBytes), []string{net.Name, "rx"}) - metrics.pushCustomMetric( - "kubevirt_vmi_network_receive_bytes_total", - "Total network traffic received in bytes.", - prometheus.CounterValue, - float64(net.RxBytes), - netLabels, - netLabelValues, - ) - } - - if net.TxBytesSet { - metrics.pushPrometheusMetric(desc, prometheus.CounterValue, float64(net.TxBytes), []string{net.Name, "tx"}) - metrics.pushCustomMetric( - "kubevirt_vmi_network_transmit_bytes_total", - "Total network traffic transmitted in bytes.", - prometheus.CounterValue, - float64(net.TxBytes), - netLabels, - netLabelValues, - ) - } - } - - if net.RxPktsSet { - metrics.pushCustomMetric( - "kubevirt_vmi_network_receive_packets_total", - "Total network traffic received packets.", - prometheus.CounterValue, - float64(net.RxPkts), - netLabels, - netLabelValues, - ) - } - - if net.TxPktsSet { - metrics.pushCustomMetric( - "kubevirt_vmi_network_transmit_packets_total", - "Total network traffic transmitted packets.", - prometheus.CounterValue, - float64(net.TxPkts), - netLabels, - netLabelValues, - ) - } - - if net.RxErrsSet { - metrics.pushCustomMetric( - "kubevirt_vmi_network_receive_errors_total", - "Total network received error packets.", - prometheus.CounterValue, - float64(net.RxErrs), - netLabels, - netLabelValues, - ) - } - - if net.TxErrsSet { - metrics.pushCustomMetric( - "kubevirt_vmi_network_transmit_errors_total", - "Total network transmitted error packets.", - prometheus.CounterValue, - float64(net.TxErrs), - netLabels, - netLabelValues, - ) - } - - if net.RxDropSet { - metrics.pushCustomMetric( - "kubevirt_vmi_network_receive_packets_dropped_total", - "The total number of rx packets dropped on vNIC interfaces.", - prometheus.CounterValue, - float64(net.RxDrop), - netLabels, - netLabelValues, - ) - } - - if net.TxDropSet { - metrics.pushCustomMetric( - "kubevirt_vmi_network_transmit_packets_dropped_total", - "The total number of tx packets dropped on vNIC interfaces.", - prometheus.CounterValue, - float64(net.TxDrop), - netLabels, - netLabelValues, - ) - } - } -} - -func (metrics *vmiMetrics) updateFilesystem(vmFSStats k6tv1.VirtualMachineInstanceFileSystemList) { - if len(vmFSStats.Items) == 0 { - return - } - - fsLabels := []string{"disk_name", "mount_point", "file_system_type"} - - for _, fsStat := range vmFSStats.Items { - fsLabelValues := []string{fsStat.DiskName, fsStat.MountPoint, fsStat.FileSystemType} - - metrics.pushCustomMetric( - "kubevirt_vmi_filesystem_capacity_bytes", - "Total VM filesystem capacity in bytes.", - prometheus.GaugeValue, - float64(fsStat.TotalBytes), - fsLabels, - fsLabelValues, - ) - - metrics.pushCustomMetric( - "kubevirt_vmi_filesystem_used_bytes", - "Used VM filesystem capacity in bytes.", - prometheus.GaugeValue, - float64(fsStat.UsedBytes), - fsLabels, - fsLabelValues, - ) - } -} - -type DomainStatsCollector struct { - virtShareDir string - nodeName string - concCollector *vms.ConcurrentCollector - vmiInformer cache.SharedIndexInformer -} - -// aggregates to virt-launcher -func SetupDomainStatsCollector(virtCli kubecli.KubevirtClient, virtShareDir, nodeName string, MaxRequestsInFlight int, vmiInformer cache.SharedIndexInformer) *DomainStatsCollector { - log.Log.Infof("Starting domain stats collector: node name=%v", nodeName) - co := &DomainStatsCollector{ - virtShareDir: virtShareDir, - nodeName: nodeName, - concCollector: vms.NewConcurrentCollector(MaxRequestsInFlight), - vmiInformer: vmiInformer, - } - - prometheus.MustRegister(co) - return co -} - -func (co *DomainStatsCollector) Describe(_ chan<- *prometheus.Desc) { - // TODO: Use DescribeByCollect? -} - -// Note that Collect could be called concurrently -func (co *DomainStatsCollector) Collect(ch chan<- prometheus.Metric) { - cachedObjs := co.vmiInformer.GetIndexer().List() - if len(cachedObjs) == 0 { - log.Log.V(4).Infof("No VMIs detected") - return - } - - vmis := make([]*k6tv1.VirtualMachineInstance, len(cachedObjs)) - - for i, obj := range cachedObjs { - vmis[i] = obj.(*k6tv1.VirtualMachineInstance) - } - - scraper := &prometheusScraper{ch: ch} - co.concCollector.Collect(vmis, scraper, PrometheusCollectionTimeout) - return -} - -func NewPrometheusScraper(ch chan<- prometheus.Metric) *prometheusScraper { - return &prometheusScraper{ch: ch} -} - -type prometheusScraper struct { - ch chan<- prometheus.Metric -} - -type VirtualMachineInstanceStats struct { - DomainStats *stats.DomainStats - FsStats k6tv1.VirtualMachineInstanceFileSystemList -} - -func (ps *prometheusScraper) Scrape(socketFile string, vmi *k6tv1.VirtualMachineInstance) { - ts := time.Now() - cli, err := cmdclient.NewClient(socketFile) - if err != nil { - log.Log.Reason(err).Error("failed to connect to cmd client socket") - // Ignore failure to connect to client. - // These are all local connections via unix socket. - // A failure to connect means there's nothing on the other - // end listening. - return - } - defer cli.Close() - - vmStats := &VirtualMachineInstanceStats{} - var exists bool - - vmStats.DomainStats, exists, err = cli.GetDomainStats() - if err != nil { - log.Log.Reason(err).Errorf("failed to update domain stats from socket %s", socketFile) - return - } - if !exists || vmStats.DomainStats.Name == "" { - log.Log.V(2).Infof("disappearing VM on %s, ignored", socketFile) // VM may be shutting down - return - } - - vmStats.FsStats, err = cli.GetFilesystems() - if err != nil { - log.Log.Reason(err).Errorf("failed to update filesystem stats from socket %s", socketFile) - return - } - - // GetDomainStats() may hang for a long time. - // If it wakes up past the timeout, there is no point in send back any metric. - // In the best case the information is stale, in the worst case the information is stale *and* - // the reporting channel is already closed, leading to a possible panic - see below - elapsed := time.Now().Sub(ts) - if elapsed > vms.StatsMaxAge { - log.Log.Infof("took too long (%v) to collect stats from %s: ignored", elapsed, socketFile) - return - } - - ps.Report(socketFile, vmi, vmStats) -} - -func (ps *prometheusScraper) Report(socketFile string, vmi *k6tv1.VirtualMachineInstance, vmStats *VirtualMachineInstanceStats) { - // statsMaxAge is an estimation - and there is no better way to do that. So it is possible that - // GetDomainStats() takes enough time to lag behind, but not enough to trigger the statsMaxAge check. - // In this case the next functions will end up writing on a closed channel. This will panic. - // It is actually OK in this case to abort the goroutine that panicked -that's what we want anyway, - // and the very reason we collect in throwaway goroutines. We need however to avoid dump stacktraces in the logs. - // Since this is a known failure condition, let's handle it explicitly. - defer func() { - if err := recover(); err != nil { - log.Log.Warningf("collector goroutine panicked for VM %s: %s", socketFile, err) - } - }() - - vmiMetrics := newVmiMetrics(vmi, ps.ch) - vmiMetrics.updateMetrics(vmi, vmStats) -} - -type vmiMetrics struct { - k8sLabels []string - k8sLabelValues []string - vmi *k6tv1.VirtualMachineInstance - ch chan<- prometheus.Metric -} - -func (metrics *vmiMetrics) updateMetrics(vmi *k6tv1.VirtualMachineInstance, vmStats *VirtualMachineInstanceStats) { - metrics.updateKubernetesLabels() - - metrics.updateMemory(vmStats.DomainStats.Memory) - metrics.updateCPU(vmi, vmStats.DomainStats.Cpu) - metrics.updateVcpu(vmStats.DomainStats.Vcpu) - metrics.updateBlock(vmStats.DomainStats.Block) - metrics.updateNetwork(vmStats.DomainStats.Net) - - if vmStats.DomainStats.CPUMapSet { - metrics.updateCPUAffinity(vmStats.DomainStats.CPUMap) - } - metrics.updateMigrateInfo(vmStats.DomainStats.MigrateDomainJobInfo) - metrics.updateFilesystem(vmStats.FsStats) -} - -func (metrics *vmiMetrics) newPrometheusDesc(name string, help string, customLabels []string) *prometheus.Desc { - labels := []string{"node", "namespace", "name"} // Common labels - labels = append(labels, customLabels...) - labels = append(labels, metrics.k8sLabels...) - return prometheus.NewDesc(name, help, labels, nil) -} - -func (metrics *vmiMetrics) pushPrometheusMetric(desc *prometheus.Desc, valueType prometheus.ValueType, value float64, customLabelValues []string) { - labelValues := []string{metrics.vmi.Status.NodeName, metrics.vmi.Namespace, metrics.vmi.Name} - labelValues = append(labelValues, customLabelValues...) - labelValues = append(labelValues, metrics.k8sLabelValues...) - mv, err := prometheus.NewConstMetric(desc, valueType, value, labelValues...) - tryToPushMetric(desc, mv, err, metrics.ch) -} - -func (metrics *vmiMetrics) pushCommonMetric(name string, help string, valueType prometheus.ValueType, value float64) { - metrics.pushCustomMetric(name, help, valueType, value, nil, nil) -} - -func (metrics *vmiMetrics) pushCustomMetric(name string, help string, valueType prometheus.ValueType, value float64, customLabels []string, customLabelValues []string) { - desc := metrics.newPrometheusDesc(name, help, customLabels) - metrics.pushPrometheusMetric(desc, valueType, value, customLabelValues) -} - -func (metrics *vmiMetrics) updateKubernetesLabels() { - for label, val := range metrics.vmi.Labels { - metrics.k8sLabels = append(metrics.k8sLabels, labelPrefix+labelFormatter.Replace(label)) - metrics.k8sLabelValues = append(metrics.k8sLabelValues, val) - } -} - -func newVmiMetrics(vmi *k6tv1.VirtualMachineInstance, ch chan<- prometheus.Metric) *vmiMetrics { - return &vmiMetrics{ - vmi: vmi, - k8sLabels: []string{}, - k8sLabelValues: []string{}, - ch: ch, - } -} - -func humanReadableState(state int) string { - switch state { - case stats.VCPUOffline: - return "offline" - case stats.VCPUBlocked: - return "blocked" - case stats.VCPURunning: - return "running" - default: - return "unknown" - } -} diff --git a/pkg/monitoring/domainstats/prometheus/prometheus_suite_test.go b/pkg/monitoring/domainstats/prometheus/prometheus_suite_test.go deleted file mode 100644 index 08cfb3a0661d..000000000000 --- a/pkg/monitoring/domainstats/prometheus/prometheus_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package prometheus - -import ( - "testing" - - "kubevirt.io/client-go/testutils" -) - -func TestPrometheus(t *testing.T) { - testutils.KubeVirtTestSuiteSetup(t) -} diff --git a/pkg/monitoring/domainstats/prometheus/prometheus_test.go b/pkg/monitoring/domainstats/prometheus/prometheus_test.go deleted file mode 100644 index 8bf26d799ead..000000000000 --- a/pkg/monitoring/domainstats/prometheus/prometheus_test.go +++ /dev/null @@ -1,1277 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2018 Red Hat, Inc. - * - */ - -package prometheus - -import ( - "kubevirt.io/kubevirt/pkg/pointer" - - "github.com/prometheus/client_golang/prometheus" - io_prometheus_client "github.com/prometheus/client_model/go" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "libvirt.org/go/libvirt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - k6tv1 "kubevirt.io/api/core/v1" - - "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" -) - -var _ = BeforeSuite(func() { -}) - -var _ = Describe("Prometheus", func() { - newVmStats := func(domainStats *stats.DomainStats, fsStats *k6tv1.VirtualMachineInstanceFileSystemList) *VirtualMachineInstanceStats { - if fsStats == nil { - fsStats = &k6tv1.VirtualMachineInstanceFileSystemList{ - Items: []k6tv1.VirtualMachineInstanceFileSystem{}, - } - } - - return &VirtualMachineInstanceStats{ - DomainStats: domainStats, - FsStats: *fsStats, - } - } - - Context("on blocked source", func() { - It("should handle closed reporting socket", func() { - ch := make(chan prometheus.Metric) - close(ch) - - ps := prometheusScraper{ch: ch} - - testReportPanic := func() { - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - // trigger write on a socket. We need a value set - any value - RSS: 1024, - RSSSet: true, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - } - Expect(testReportPanic).ToNot(Panic()) - }) - }) - - Context("on handling push", func() { - It("should send rss", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - // trigger write on a socket. We need a value set - any value - RSS: 1024, - RSSSet: true, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_resident_bytes")) - }) - - It("should send available memory", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - AvailableSet: true, - Available: 1, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_available_bytes")) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should send unused memory", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - UnusedSet: true, - Unused: 1, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_unused_bytes")) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should send cached memory", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - CachedSet: true, - Cached: 1, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_cached_bytes")) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should handle swapin", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - SwapInSet: true, - SwapIn: 1, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_swap_in_traffic_bytes")) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should handle swapout", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - SwapOutSet: true, - SwapOut: 1, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_swap_out_traffic_bytes")) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should handle major page faults metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - MajorFaultSet: true, - MajorFault: 1024, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_pgmajfault_total")) - Expect(dto.Counter.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should handle minor page faults metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - MinorFaultSet: true, - MinorFault: 1024, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_pgminfault_total")) - Expect(dto.Counter.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should handle actual balloon metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - ActualBalloonSet: true, - ActualBalloon: 1, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_actual_balloon_bytes")) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should handle the usable metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - UsableSet: true, - Usable: 1, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_usable_bytes")) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - It("should handle the total memory metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - TotalSet: true, - Total: 1, - }, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_memory_domain_bytes")) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1024))) - }) - - DescribeTable("Assert vmi migration metrics", - func(metricName string, migrateDomainJobInfoStats *stats.DomainJobInfo) { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - MigrateDomainJobInfo: migrateDomainJobInfoStats, - } - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring(metricName)) - Expect(dto.Gauge.GetValue()).To(BeEquivalentTo(float64(1))) - }, - Entry("should handle Data_Processed metrics for VMs", - MigrateVmiDataProcessedMetricName, - &stats.DomainJobInfo{ - DataProcessedSet: true, - DataProcessed: 1, - }), - Entry("should handle Data_Remaining metrics for VMs", - MigrateVmiDataRemainingMetricName, - &stats.DomainJobInfo{ - DataRemainingSet: true, - DataRemaining: 1, - }), - Entry("should handle MemDirtyRate metrics for VMs", - MigrateVmiDirtyMemoryRateMetricName, - &stats.DomainJobInfo{ - MemDirtyRateSet: true, - MemDirtyRate: 1, - }), - Entry("should handle MemoryBps metrics for VMs", - MigrateVmiMemoryTransferRateMetricName, - &stats.DomainJobInfo{ - MemoryBpsSet: true, - MemoryBps: 1, - }), - ) - - It("should handle vcpu metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Vcpu: []stats.DomainStatsVcpu{ - { - StateSet: true, - State: 1, - TimeSet: true, - Time: 2000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_vcpu_seconds_total")) - }) - - It("should not expose vcpu metrics for invalid DomainStats", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Vcpu: []stats.DomainStatsVcpu{ - { - // vcpu State is not set! - TimeSet: true, - Time: 2000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - // metrics about invalid stats never get pushed into the channel - Eventually(ch).Should(BeEmpty()) - }) - - It("should expose vcpu state as a human readable string", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Vcpu: []stats.DomainStatsVcpu{ - { - StateSet: true, - State: int(libvirt.VCPU_RUNNING), - TimeSet: true, - Time: 2000, - }, - }, - } - - metric := &io_prometheus_client.Metric{} - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - result.Write(metric) - - Expect(result).ToNot(BeNil()) - for _, label := range metric.GetLabel() { - if label.GetName() == "state" { - Expect(label.GetValue()).To(BeEquivalentTo("running")) - } - } - - domainStats = &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Vcpu: []stats.DomainStatsVcpu{ - { - StateSet: true, - State: int(libvirt.VCPU_BLOCKED), - TimeSet: true, - Time: 2000, - }, - }, - } - - metric = &io_prometheus_client.Metric{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result = <-ch - result.Write(metric) - - Expect(result).ToNot(BeNil()) - for _, label := range metric.GetLabel() { - if label.GetName() == "state" { - Expect(label.GetValue()).To(BeEquivalentTo("blocked")) - } - } - - domainStats = &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Vcpu: []stats.DomainStatsVcpu{ - { - StateSet: true, - State: int(libvirt.VCPU_OFFLINE), - TimeSet: true, - Time: 2000, - }, - }, - } - - metric = &io_prometheus_client.Metric{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result = <-ch - result.Write(metric) - - Expect(result).ToNot(BeNil()) - for _, label := range metric.GetLabel() { - if label.GetName() == "state" { - Expect(label.GetValue()).To(BeEquivalentTo("offline")) - } - } - }) - - It("should handle block read iops metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - RdReqsSet: true, - RdReqs: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_storage_iops_read_total")) - }) - - It("should handle block write iops metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - WrReqsSet: true, - WrReqs: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_storage_iops_write_total")) - }) - - It("should handle block read bytes metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - RdBytesSet: true, - RdBytes: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_storage_read_traffic_bytes_total")) - }) - - It("should handle block write bytes metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - WrBytesSet: true, - WrBytes: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_storage_write_traffic_bytes_total")) - }) - - It("should handle block read time metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - RdTimesSet: true, - RdTimes: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_storage_read_times_seconds_total")) - }) - - It("should handle block write time metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - WrTimesSet: true, - WrTimes: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_storage_write_times_seconds_total")) - }) - - It("should handle block flush requests metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - FlReqsSet: true, - FlReqs: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_storage_flush_requests_total")) - }) - - It("should handle block flush times metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - FlTimesSet: true, - FlTimes: 1000000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_storage_flush_times_seconds_total")) - }) - - It("should use alias when alias is not empty", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - Alias: "disk0", - FlTimesSet: true, - FlTimes: 1000000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - - dto := &io_prometheus_client.Metric{} - err := result.Write(dto) - Expect(err).ShouldNot(HaveOccurred()) - expectedLabelPair := &io_prometheus_client.LabelPair{ - Name: pointer.P("drive"), - Value: pointer.P("disk0"), - } - Expect(dto.GetLabel()).To(ContainElement(expectedLabelPair)) - }) - - It("should use the name when alias is empty", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - NameSet: true, - Name: "vda", - Alias: "", - FlTimesSet: true, - FlTimes: 1000000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - - dto := &io_prometheus_client.Metric{} - err := result.Write(dto) - Expect(err).ShouldNot(HaveOccurred()) - expectedLabelPair := &io_prometheus_client.LabelPair{ - Name: pointer.P("drive"), - Value: pointer.P("vda"), - } - Expect(dto.GetLabel()).To(ContainElement(expectedLabelPair)) - }) - - It("should not expose nameless block metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Block: []stats.DomainStatsBlock{ - { - RdReqsSet: true, - RdReqs: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - Eventually(ch).Should(BeEmpty()) - }) - - It("should handle network rx traffic bytes metrics", func() { - ch := make(chan prometheus.Metric, 2) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - NameSet: true, - Name: "vnet0", - RxBytesSet: true, - RxBytes: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_traffic_bytes_total")) - - result = <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_receive_bytes_total")) - }) - - It("should handle network tx traffic bytes metrics", func() { - ch := make(chan prometheus.Metric, 2) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - NameSet: true, - Name: "vnet0", - TxBytesSet: true, - TxBytes: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_traffic_bytes_total")) - - result = <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_transmit_bytes_total")) - }) - - It("should handle network rx packets metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - NameSet: true, - Name: "vnet0", - RxPktsSet: true, - RxPkts: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_receive_packets_total")) - }) - - It("should handle network tx traffic packets metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - NameSet: true, - Name: "vnet0", - TxPktsSet: true, - TxPkts: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_transmit_packets_total")) - }) - - It("should handle network rx errors metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - NameSet: true, - Name: "vnet0", - RxErrsSet: true, - RxErrs: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_receive_errors_total")) - }) - - It("should handle network tx traffic error metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - NameSet: true, - Name: "vnet0", - TxErrsSet: true, - TxErrs: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_transmit_errors_total")) - }) - - It("should handle network rx drop metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - NameSet: true, - Name: "vnet0", - RxDropSet: true, - RxDrop: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_receive_packets_dropped_total")) - }) - - It("should handle network tx drop metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - NameSet: true, - Name: "vnet0", - TxDropSet: true, - TxDrop: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_network_transmit_packets_dropped_total")) - }) - - It("should not expose nameless network interface metrics", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{ - { - TxErrsSet: true, - TxErrs: 1000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - Eventually(ch).Should(BeEmpty()) - }) - - It("should add kubernetes metadata labels", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{ - RSS: 1024, - RSSSet: true, - }, - } - - vmi := k6tv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "kubevirt.io/nodeName": "node01", - }, - }, - } - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubernetes_vmi_label_kubevirt_io_nodeName")) - }) - - It("should expose vcpu wait metric", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{}, - Vcpu: []stats.DomainStatsVcpu{ - { - WaitSet: true, - Wait: 6, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_vcpu_wait_seconds_total")) - }) - - It("should expose vcpu delay metric", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{}, - Vcpu: []stats.DomainStatsVcpu{ - { - DelaySet: true, - Delay: 800000000, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_vcpu_delay_seconds_total")) - }) - - It("should expose vcpu to cpu pinning metric", func() { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{}, - Vcpu: []stats.DomainStatsVcpu{}, - CPUMapSet: true, - CPUMap: [][]bool{{true, false, true}}, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_node_cpu_affinity")) - Expect(dto.GetGauge().GetValue()).To(Equal(float64(2))) - }) - - It("should expose filesystem metrics", func() { - ch := make(chan prometheus.Metric, 2) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: &stats.DomainStatsCPU{}, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{}, - MigrateDomainJobInfo: &stats.DomainJobInfo{}, - } - - fsStats := &k6tv1.VirtualMachineInstanceFileSystemList{ - Items: []k6tv1.VirtualMachineInstanceFileSystem{ - { - DiskName: "disk1", - TotalBytes: 1000, - UsedBytes: 10, - }, - }, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, fsStats)) - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_filesystem_capacity_bytes")) - result = <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_filesystem_used_bytes")) - Expect(ch).To(BeEmpty()) - }) - - DescribeTable("CPU metrics", func(metricName string, MetricValue int, cpuStats *stats.DomainStatsCPU) { - ch := make(chan prometheus.Metric, 1) - defer close(ch) - - ps := prometheusScraper{ch: ch} - - domainStats := &stats.DomainStats{ - Cpu: cpuStats, - Memory: &stats.DomainStatsMemory{}, - Net: []stats.DomainStatsNet{}, - Vcpu: []stats.DomainStatsVcpu{}, - } - - vmi := k6tv1.VirtualMachineInstance{} - ps.Report("test", &vmi, newVmStats(domainStats, nil)) - - result := <-ch - Expect(result).ToNot(BeNil()) - Expect(result.Desc().String()).To(ContainSubstring(metricName)) - - dto := &io_prometheus_client.Metric{} - result.Write(dto) - - Expect(dto.GetCounter().GetValue()).To(Equal(float64(MetricValue))) - - }, - Entry("Total CPU time spent in all modes (sum of both vcpu and hypervisor usage)", "kubevirt_vmi_cpu_usage_seconds_total", 123, &stats.DomainStatsCPU{ - TimeSet: true, - Time: 123000000000}, - ), - Entry("Total CPU time spent in user mode", "kubevirt_vmi_cpu_user_usage_seconds_total", 456, &stats.DomainStatsCPU{ - UserSet: true, - User: 456000000000}, - ), - Entry("Total CPU time spent in system mode", "kubevirt_vmi_cpu_system_usage_seconds_total", 789, &stats.DomainStatsCPU{ - SystemSet: true, - System: 789000000000}, - )) - }) -}) diff --git a/pkg/monitoring/metrics/virt-handler/BUILD.bazel b/pkg/monitoring/metrics/virt-handler/BUILD.bazel index 8f0e3cd74c52..3b984bbd8023 100644 --- a/pkg/monitoring/metrics/virt-handler/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-handler/BUILD.bazel @@ -2,12 +2,17 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["metrics.go"], + srcs = [ + "metrics.go", + "version_metrics.go", + ], importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler", visibility = ["//visibility:public"], deps = [ "//pkg/monitoring/metrics/common/client:go_default_library", "//pkg/monitoring/metrics/common/workqueue:go_default_library", + "//pkg/monitoring/metrics/virt-handler/domainstats:go_default_library", + "//staging/src/kubevirt.io/client-go/version:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", ], ) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/BUILD.bazel b/pkg/monitoring/metrics/virt-handler/domainstats/BUILD.bazel new file mode 100644 index 000000000000..d1403a20d9cc --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/BUILD.bazel @@ -0,0 +1,42 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "block_metrics.go", + "collector.go", + "cpu_metrics.go", + "domainstats.go", + "filesystem_metrics.go", + "memory_metrics.go", + "migration_metrics.go", + "network_metrics.go", + "node_cpu_affinity_metrics.go", + "vcpuMetrics.go", + ], + importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats", + visibility = ["//visibility:public"], + deps = [ + "//pkg/virt-launcher/virtwrap/stats:go_default_library", + "//staging/src/kubevirt.io/api/core/v1:go_default_library", + "//staging/src/kubevirt.io/client-go/log:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "domainstats_suite_test.go", + "domainstats_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//staging/src/kubevirt.io/api/core/v1:go_default_library", + "//staging/src/kubevirt.io/client-go/testutils:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + "//vendor/github.com/onsi/ginkgo/v2:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/block_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/block_metrics.go new file mode 100644 index 000000000000..aff4f1aaaca8 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/block_metrics.go @@ -0,0 +1,152 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "kubevirt.io/client-go/log" +) + +var ( + storageIopsRead = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_storage_iops_read_total", + Help: "Total number of I/O read operations.", + }, + ) + + storageIopsWrite = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_storage_iops_write_total", + Help: "Total number of I/O write operations.", + }, + ) + + storageReadTrafficBytes = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_storage_read_traffic_bytes_total", + Help: "Total number of bytes read from storage.", + }, + ) + + storageWriteTrafficBytes = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_storage_write_traffic_bytes_total", + Help: "Total number of written bytes.", + }, + ) + + storageReadTimesSeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_storage_read_times_seconds_total", + Help: "Total time spent on read operations.", + }, + ) + + storageWriteTimesSeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_storage_write_times_seconds_total", + Help: "Total time spent on write operations.", + }, + ) + + storageFlushRequests = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_storage_flush_requests_total", + Help: "Total storage flush requests.", + }, + ) + + storageFlushTimesSeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_storage_flush_times_seconds_total", + Help: "Total time spent on cache flushing.", + }, + ) +) + +type blockMetrics struct{} + +func (blockMetrics) Describe() []operatormetrics.Metric { + return []operatormetrics.Metric{ + storageIopsRead, + storageIopsWrite, + storageReadTrafficBytes, + storageWriteTrafficBytes, + storageReadTimesSeconds, + storageWriteTimesSeconds, + storageFlushRequests, + storageFlushTimesSeconds, + } +} + +func (blockMetrics) Collect(vmiReport *VirtualMachineInstanceReport) []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + + if vmiReport.vmiStats.DomainStats == nil || vmiReport.vmiStats.DomainStats.Block == nil { + return crs + } + + for blockIdx, block := range vmiReport.vmiStats.DomainStats.Block { + if !block.NameSet { + log.Log.Warningf("Name not set for block device#%d", blockIdx) + continue + } + + blkLabels := map[string]string{"drive": block.Name} + if block.Alias != "" { + blkLabels["drive"] = block.Alias + } + + if block.RdReqsSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(storageIopsRead, float64(block.RdReqs), blkLabels)) + } + + if block.WrReqsSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(storageIopsWrite, float64(block.WrReqs), blkLabels)) + } + + if block.RdBytesSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(storageReadTrafficBytes, float64(block.RdBytes), blkLabels)) + } + + if block.WrBytesSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(storageWriteTrafficBytes, float64(block.WrBytes), blkLabels)) + } + + if block.RdTimesSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(storageReadTimesSeconds, nanosecondsToSeconds(block.RdTimes), blkLabels)) + } + + if block.WrTimesSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(storageWriteTimesSeconds, nanosecondsToSeconds(block.WrTimes), blkLabels)) + } + + if block.FlReqsSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(storageFlushRequests, float64(block.FlReqs), blkLabels)) + } + + if block.FlTimesSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(storageFlushTimesSeconds, nanosecondsToSeconds(block.FlTimes), blkLabels)) + } + } + + return crs +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/block_metrics_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/block_metrics_test.go new file mode 100644 index 000000000000..9c5239d21741 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/block_metrics_test.go @@ -0,0 +1,91 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var _ = Describe("block metrics", func() { + Context("on Collect", func() { + vmi := &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + } + + vmiStats := &VirtualMachineInstanceStats{ + DomainStats: &stats.DomainStats{ + Block: []stats.DomainStatsBlock{ + { + NameSet: true, + Name: "vda", + RdReqsSet: true, + RdReqs: 1, + WrReqsSet: true, + WrReqs: 2, + RdBytesSet: true, + RdBytes: 3, + WrBytesSet: true, + WrBytes: 4, + RdTimesSet: true, + RdTimes: 5, + WrTimesSet: true, + WrTimes: 6, + FlReqsSet: true, + FlReqs: 7, + FlTimesSet: true, + FlTimes: 8, + }, + }, + }, + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + + DescribeTable("should collect metrics values", func(metric operatormetrics.Metric, expectedValue float64) { + crs := blockMetrics{}.Collect(vmiReport) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(metric, expectedValue))) + }, + Entry("kubevirt_vmi_storage_iops_read_total", storageIopsRead, 1.0), + Entry("kubevirt_vmi_storage_iops_write_total", storageIopsWrite, 2.0), + Entry("kubevirt_vmi_storage_read_traffic_bytes_total", storageReadTrafficBytes, 3.0), + Entry("kubevirt_vmi_storage_write_traffic_bytes_total", storageWriteTrafficBytes, 4.0), + Entry("kubevirt_vmi_storage_read_times_seconds_total", storageReadTimesSeconds, nanosecondsToSeconds(5)), + Entry("kubevirt_vmi_storage_write_times_seconds_total", storageWriteTimesSeconds, nanosecondsToSeconds(6)), + Entry("kubevirt_vmi_storage_flush_requests_total", storageFlushRequests, 7.0), + Entry("kubevirt_vmi_storage_flush_times_seconds_total", storageFlushTimesSeconds, nanosecondsToSeconds(8)), + ) + + It("result should be empty if stat not populated or set is false", func() { + vmiStats.DomainStats.Block[0].NameSet = false + crs := blockMetrics{}.Collect(vmiReport) + Expect(crs).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/collector.go b/pkg/monitoring/metrics/virt-handler/domainstats/collector.go new file mode 100644 index 000000000000..69f702549c42 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/collector.go @@ -0,0 +1,89 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + */ + +package domainstats + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + k6tv1 "kubevirt.io/api/core/v1" +) + +type resourceMetrics interface { + Describe() []operatormetrics.Metric + Collect(report *VirtualMachineInstanceReport) []operatormetrics.CollectorResult +} + +var ( + rms = []resourceMetrics{ + memoryMetrics{}, + cpuMetrics{}, + vcpuMetrics{}, + blockMetrics{}, + networkMetrics{}, + cpuAffinityMetrics{}, + migrationMetrics{}, + filesystemMetrics{}, + } + + Collector = operatormetrics.Collector{ + Metrics: domainStatsMetrics(rms...), + CollectCallback: domainStatsCollectorCallback, + } +) + +func domainStatsMetrics(rms ...resourceMetrics) []operatormetrics.Metric { + var metrics []operatormetrics.Metric + + for _, rm := range rms { + metrics = append(metrics, rm.Describe()...) + } + + return metrics +} + +func domainStatsCollectorCallback() []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + var vmis []k6tv1.VirtualMachineInstance + + for _, vmi := range vmis { + crs = append(crs, collect(&vmi)...) + } + + return crs +} + +func collect(vmi *k6tv1.VirtualMachineInstance) []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + + vmiStats := scrape(vmi) + + if vmiStats == nil { + return crs + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + for _, rm := range rms { + crs = append(crs, rm.Collect(vmiReport)...) + } + + return crs +} + +func scrape(vmi *k6tv1.VirtualMachineInstance) *VirtualMachineInstanceStats { + return nil +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/collector_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/collector_test.go new file mode 100644 index 000000000000..9f4472e07173 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/collector_test.go @@ -0,0 +1,121 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats/collector" + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var _ = Describe("domain stats collector", func() { + Context("Collect", func() { + vmis := []*k6tv1.VirtualMachineInstance{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + }, + } + + vmiStats := []*VirtualMachineInstanceStats{ + { + DomainStats: &stats.DomainStats{ + Memory: &stats.DomainStatsMemory{ + RSSSet: true, + RSS: 1, + }, + }, + }, + { + DomainStats: &stats.DomainStats{ + Memory: &stats.DomainStatsMemory{ + RSSSet: true, + RSS: 2, + }, + }, + }, + } + + It("should collect metrics for each vmi", func() { + concCollector := fakeCollector{ + vmis: vmis, + vmiStats: vmiStats, + } + crs := execCollector(concCollector, vmis) + + Expect(crs).To(HaveLen(2)) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(memoryResident, kibibytesToBytes(1)))) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(memoryResident, kibibytesToBytes(2)))) + }) + }) +}) + +type fakeCollector struct { + collector.Collector + + vmis []*k6tv1.VirtualMachineInstance + vmiStats []*VirtualMachineInstanceStats +} + +func (fc fakeCollector) Collect(_ []*k6tv1.VirtualMachineInstance, scraper collector.MetricsScraper, _ time.Duration) ([]string, bool) { + var busyScrapers sync.WaitGroup + + for i, vmi := range fc.vmis { + busyScrapers.Add(1) + go fakeCollect(scraper, &busyScrapers, vmi, fc.vmiStats[i]) + } + + c := make(chan struct{}) + go func() { + busyScrapers.Wait() + c <- struct{}{} + }() + select { + case <-c: + scraper.Complete() + case <-time.After(collector.CollectionTimeout): + Fail("timeout") + } + + return nil, true +} + +func fakeCollect(scraper collector.MetricsScraper, wg *sync.WaitGroup, vmi *k6tv1.VirtualMachineInstance, vmiStats *VirtualMachineInstanceStats) { + defer wg.Done() + + dScraper := scraper.(*domainstatsScraper) + dScraper.report(vmi, vmiStats) +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/cpu_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/cpu_metrics.go new file mode 100644 index 000000000000..03db7af98e49 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/cpu_metrics.go @@ -0,0 +1,86 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "kubevirt.io/client-go/log" +) + +var ( + cpuUsageSeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_cpu_usage_seconds_total", + Help: "Total CPU time spent in all modes (sum of both vcpu and hypervisor usage).", + }, + ) + + cpuUserUsageSeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_cpu_user_usage_seconds_total", + Help: "Total CPU time spent in user mode.", + }, + ) + + cpuSystemUsageSeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_cpu_system_usage_seconds_total", + Help: "Total CPU time spent in system mode.", + }, + ) +) + +type cpuMetrics struct{} + +func (cpuMetrics) Describe() []operatormetrics.Metric { + return []operatormetrics.Metric{ + cpuUsageSeconds, + cpuUserUsageSeconds, + cpuSystemUsageSeconds, + } +} + +func (cpuMetrics) Collect(vmiReport *VirtualMachineInstanceReport) []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + + if vmiReport.vmiStats.DomainStats == nil || vmiReport.vmiStats.DomainStats.Cpu == nil { + return crs + } + + cpu := vmiReport.vmiStats.DomainStats.Cpu + + if !cpu.TimeSet && !cpu.UserSet && !cpu.SystemSet { + log.Log.Warningf("No domain CPU stats is set for %s VMI.", vmiReport.vmi.Name) + } + + if cpu.TimeSet { + crs = append(crs, vmiReport.newCollectorResult(cpuUsageSeconds, nanosecondsToSeconds(cpu.Time))) + } + + if cpu.UserSet { + crs = append(crs, vmiReport.newCollectorResult(cpuUserUsageSeconds, nanosecondsToSeconds(cpu.User))) + } + + if cpu.SystemSet { + crs = append(crs, vmiReport.newCollectorResult(cpuSystemUsageSeconds, nanosecondsToSeconds(cpu.System))) + } + + return crs +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/cpu_metrics_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/cpu_metrics_test.go new file mode 100644 index 000000000000..c2792e9cbabd --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/cpu_metrics_test.go @@ -0,0 +1,74 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var _ = Describe("cpu metrics", func() { + Context("on Collect", func() { + vmi := &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + } + + vmiStats := &VirtualMachineInstanceStats{ + DomainStats: &stats.DomainStats{ + Cpu: &stats.DomainStatsCPU{ + TimeSet: true, + Time: 1, + UserSet: true, + User: 2, + SystemSet: true, + System: 3, + }, + }, + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + + DescribeTable("should collect metrics values", func(metric operatormetrics.Metric, expectedValue float64) { + crs := cpuMetrics{}.Collect(vmiReport) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(metric, expectedValue))) + }, + Entry("kubevirt_vmi_cpu_usage_seconds_total", cpuUsageSeconds, nanosecondsToSeconds(1)), + Entry("kubevirt_vmi_cpu_user_usage_seconds_total", cpuUserUsageSeconds, nanosecondsToSeconds(2)), + Entry("kubevirt_vmi_cpu_system_usage_seconds_total", cpuSystemUsageSeconds, nanosecondsToSeconds(3)), + ) + + It("result should be empty if stat not populated or set is false", func() { + vmiStats.DomainStats.Cpu = &stats.DomainStatsCPU{ + TimeSet: false, + } + crs := cpuMetrics{}.Collect(vmiReport) + Expect(crs).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/domainstats.go b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats.go new file mode 100644 index 000000000000..ed9ed29c02b7 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats.go @@ -0,0 +1,93 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "strings" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var ( + // Formatter used to sanitize k8s metadata into metric labels + labelFormatter = strings.NewReplacer(".", "_", "/", "_", "-", "_") + + // Preffixes used when transforming K8s metadata into metric labels + labelPrefix = "kubernetes_vmi_label_" +) + +type VirtualMachineInstanceReport struct { + vmi *k6tv1.VirtualMachineInstance + vmiStats *VirtualMachineInstanceStats + runtimeLabels map[string]string +} + +type VirtualMachineInstanceStats struct { + DomainStats *stats.DomainStats + FsStats k6tv1.VirtualMachineInstanceFileSystemList +} + +func newVirtualMachineInstanceReport(vmi *k6tv1.VirtualMachineInstance, vmiStats *VirtualMachineInstanceStats) *VirtualMachineInstanceReport { + vmiReport := &VirtualMachineInstanceReport{ + vmi: vmi, + vmiStats: vmiStats, + } + vmiReport.buildRuntimeLabels() + + return vmiReport +} + +func (vmiReport *VirtualMachineInstanceReport) buildRuntimeLabels() { + vmiReport.runtimeLabels = map[string]string{} + + for label, val := range vmiReport.vmi.Labels { + key := labelPrefix + labelFormatter.Replace(label) + vmiReport.runtimeLabels[key] = val + } +} + +func (vmiReport *VirtualMachineInstanceReport) newCollectorResult(metric operatormetrics.Metric, value float64) operatormetrics.CollectorResult { + return vmiReport.newCollectorResultWithLabels(metric, value, nil) +} + +func (vmiReport *VirtualMachineInstanceReport) newCollectorResultWithLabels(metric operatormetrics.Metric, value float64, additionalLabels map[string]string) operatormetrics.CollectorResult { + vmiLabels := map[string]string{ + "node": vmiReport.vmi.Status.NodeName, + "namespace": vmiReport.vmi.Namespace, + "name": vmiReport.vmi.Name, + } + + for k, v := range vmiReport.runtimeLabels { + vmiLabels[k] = v + } + + for k, v := range additionalLabels { + vmiLabels[k] = v + } + + return operatormetrics.CollectorResult{ + Metric: metric, + ConstLabels: vmiLabels, + Value: value, + } +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_suite_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_suite_test.go new file mode 100644 index 000000000000..5c2d7d693f25 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_suite_test.go @@ -0,0 +1,63 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/types" + "testing" + + "kubevirt.io/client-go/testutils" +) + +func TestDomainstats(t *testing.T) { + testutils.KubeVirtTestSuiteSetup(t) +} + +func gomegaContainsMetricMatcher(metric operatormetrics.Metric, expectedValue float64) types.GomegaMatcher { + return &metricMatcher{ + Metric: metric, + ExpectedValue: expectedValue, + } +} + +type metricMatcher struct { + Metric operatormetrics.Metric + ExpectedValue float64 +} + +func (matcher *metricMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to contain metric", matcher.Metric.GetOpts().Name, "with value", matcher.ExpectedValue) +} + +func (matcher *metricMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to contain metric", matcher.Metric.GetOpts().Name, "with value", matcher.ExpectedValue) +} + +func (matcher *metricMatcher) Match(actual interface{}) (success bool, err error) { + cr := actual.(operatormetrics.CollectorResult) + if cr.Metric.GetOpts().Name == matcher.Metric.GetOpts().Name { + if cr.Value == matcher.ExpectedValue { + return true, nil + } + } + return false, nil +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_test.go new file mode 100644 index 000000000000..a50ce4815419 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_test.go @@ -0,0 +1,79 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + k6tv1 "kubevirt.io/api/core/v1" +) + +var _ = Describe("domainstats", func() { + Context("collector functions", func() { + var metric = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "test-metric-1", + Help: "test help 1", + }, + ) + + var vmiReport = &VirtualMachineInstanceReport{ + vmi: &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + Status: k6tv1.VirtualMachineInstanceStatus{ + NodeName: "test-node-1", + }, + }, + } + + It("newCollectorResultWithLabels should return a CollectorResult with the correct values", func() { + cr := vmiReport.newCollectorResultWithLabels(metric, 5, map[string]string{"test-label-1": "test-value-1"}) + + Expect(cr.Metric.GetOpts().Name).To(Equal("test-metric-1")) + Expect(cr.Metric.GetOpts().Help).To(Equal("test help 1")) + Expect(cr.Value).To(Equal(5.0)) + + Expect(cr.ConstLabels).To(HaveKeyWithValue("node", "test-node-1")) + Expect(cr.ConstLabels).To(HaveKeyWithValue("namespace", "test-ns-1")) + Expect(cr.ConstLabels).To(HaveKeyWithValue("name", "test-vmi-1")) + + Expect(cr.ConstLabels).To(HaveKeyWithValue("test-label-1", "test-value-1")) + }) + + It("newCollectorResult should return a CollectorResult with the correct values", func() { + cr := vmiReport.newCollectorResult(metric, 5) + + Expect(cr.Metric.GetOpts().Name).To(Equal("test-metric-1")) + Expect(cr.Metric.GetOpts().Help).To(Equal("test help 1")) + Expect(cr.Value).To(Equal(5.0)) + + Expect(cr.ConstLabels).To(HaveKeyWithValue("node", "test-node-1")) + Expect(cr.ConstLabels).To(HaveKeyWithValue("namespace", "test-ns-1")) + Expect(cr.ConstLabels).To(HaveKeyWithValue("name", "test-vmi-1")) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/filesystem_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/filesystem_metrics.go new file mode 100644 index 000000000000..6190b8675314 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/filesystem_metrics.go @@ -0,0 +1,64 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + +var ( + filesystemCapacityBytes = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_filesystem_capacity_bytes", + Help: "Total VM filesystem capacity in bytes.", + }, + ) + + filesystemUsedBytes = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_filesystem_used_bytes", + Help: "Used VM filesystem capacity in bytes.", + }, + ) +) + +type filesystemMetrics struct{} + +func (filesystemMetrics) Describe() []operatormetrics.Metric { + return []operatormetrics.Metric{ + filesystemCapacityBytes, + filesystemUsedBytes, + } +} + +func (filesystemMetrics) Collect(vmiReport *VirtualMachineInstanceReport) []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + + for _, fsStat := range vmiReport.vmiStats.FsStats.Items { + fsLabels := map[string]string{ + "disk_name": fsStat.DiskName, + "mount_point": fsStat.MountPoint, + "file_system_type": fsStat.FileSystemType, + } + + crs = append(crs, vmiReport.newCollectorResultWithLabels(filesystemCapacityBytes, float64(fsStat.TotalBytes), fsLabels)) + crs = append(crs, vmiReport.newCollectorResultWithLabels(filesystemUsedBytes, float64(fsStat.UsedBytes), fsLabels)) + } + + return crs +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/filesystem_metrics_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/filesystem_metrics_test.go new file mode 100644 index 000000000000..5122da1f0d8e --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/filesystem_metrics_test.go @@ -0,0 +1,70 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" +) + +var _ = Describe("filesystem metrics", func() { + Context("on Collect", func() { + vmi := &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + } + + vmiStats := &VirtualMachineInstanceStats{ + FsStats: k6tv1.VirtualMachineInstanceFileSystemList{ + Items: []k6tv1.VirtualMachineInstanceFileSystem{ + { + DiskName: "test-disk-1", + MountPoint: "/", + FileSystemType: "ext4", + TotalBytes: 1, + UsedBytes: 2, + }, + }, + }, + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + + DescribeTable("should collect metrics values", func(metric operatormetrics.Metric, expectedValue float64) { + crs := filesystemMetrics{}.Collect(vmiReport) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(metric, expectedValue))) + }, + Entry("kubevirt_vmi_filesystem_capacity_bytes", filesystemCapacityBytes, 1.0), + Entry("kubevirt_vmi_filesystem_used_bytes", filesystemUsedBytes, 2.0), + ) + + It("result should be empty if stat not populated or set is false", func() { + vmiStats.FsStats.Items = []k6tv1.VirtualMachineInstanceFileSystem{} + crs := filesystemMetrics{}.Collect(vmiReport) + Expect(crs).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics.go new file mode 100644 index 000000000000..a9967f6f7cae --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics.go @@ -0,0 +1,177 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" +) + +var ( + memoryResident = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_resident_bytes", + Help: "Resident set size of the process running the domain.", + }, + ) + + memoryAvailable = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_available_bytes", + Help: "Amount of usable memory as seen by the domain. This value may not be accurate if a balloon driver is in use or if the guest OS does not initialize all assigned pages", + }, + ) + + memoryUnused = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_unused_bytes", + Help: "The amount of memory left completely unused by the system. Memory that is available but used for reclaimable caches should NOT be reported as free.", + }, + ) + + memoryCached = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_cached_bytes", + Help: "The amount of memory that is being used to cache I/O and is available to be reclaimed, corresponds to the sum of `Buffers` + `Cached` + `SwapCached` in `/proc/meminfo`.", + }, + ) + + memorySwapInTrafficBytes = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_swap_in_traffic_bytes", + Help: "The total amount of data read from swap space of the guest in bytes.", + }, + ) + + memorySwapOutTrafficBytes = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_swap_out_traffic_bytes", + Help: "The total amount of memory written out to swap space of the guest in bytes.", + }, + ) + + memoryPgmajfaultTotal = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_pgmajfault_total", + Help: "The number of page faults when disk IO was required. Page faults occur when a process makes a valid access to virtual memory that is not available. When servicing the page fault, if disk IO is required, it is considered as major fault.", + }, + ) + + memoryPgminfaultTotal = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_pgminfault_total", + Help: "The number of other page faults, when disk IO was not required. Page faults occur when a process makes a valid access to virtual memory that is not available. When servicing the page fault, if disk IO is NOT required, it is considered as minor fault.", + }, + ) + + memoryActualBallon = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_actual_balloon_bytes", + Help: "Current balloon size in bytes.", + }, + ) + + memoryUsableBytes = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_usable_bytes", + Help: "The amount of memory which can be reclaimed by balloon without pushing the guest system to swap, corresponds to 'Available' in /proc/meminfo.", + }, + ) + + memoryDomainBytes = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_memory_domain_bytes", + Help: "The amount of memory in bytes allocated to the domain. The `memory` value in domain xml file.", + }, + ) +) + +type memoryMetrics struct{} + +func (memoryMetrics) Describe() []operatormetrics.Metric { + return []operatormetrics.Metric{ + memoryResident, + memoryAvailable, + memoryUnused, + memoryCached, + memorySwapInTrafficBytes, + memorySwapOutTrafficBytes, + memoryPgmajfaultTotal, + memoryPgminfaultTotal, + memoryActualBallon, + memoryUsableBytes, + memoryDomainBytes, + } +} + +func (memoryMetrics) Collect(vmiReport *VirtualMachineInstanceReport) []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + + if vmiReport.vmiStats.DomainStats == nil || vmiReport.vmiStats.DomainStats.Memory == nil { + return crs + } + + mem := vmiReport.vmiStats.DomainStats.Memory + + if mem.RSSSet { + crs = append(crs, vmiReport.newCollectorResult(memoryResident, kibibytesToBytes(mem.RSS))) + } + + if mem.AvailableSet { + crs = append(crs, vmiReport.newCollectorResult(memoryAvailable, kibibytesToBytes(mem.Available))) + } + + if mem.UnusedSet { + crs = append(crs, vmiReport.newCollectorResult(memoryUnused, kibibytesToBytes(mem.Unused))) + } + + if mem.CachedSet { + crs = append(crs, vmiReport.newCollectorResult(memoryCached, kibibytesToBytes(mem.Cached))) + } + + if mem.SwapInSet { + crs = append(crs, vmiReport.newCollectorResult(memorySwapInTrafficBytes, kibibytesToBytes(mem.SwapIn))) + } + + if mem.SwapOutSet { + crs = append(crs, vmiReport.newCollectorResult(memorySwapOutTrafficBytes, kibibytesToBytes(mem.SwapOut))) + } + + if mem.MajorFaultSet { + crs = append(crs, vmiReport.newCollectorResult(memoryPgmajfaultTotal, float64(mem.MajorFault))) + } + + if mem.MinorFaultSet { + crs = append(crs, vmiReport.newCollectorResult(memoryPgminfaultTotal, float64(mem.MinorFault))) + } + + if mem.ActualBalloonSet { + crs = append(crs, vmiReport.newCollectorResult(memoryActualBallon, kibibytesToBytes(mem.ActualBalloon))) + } + + if mem.UsableSet { + crs = append(crs, vmiReport.newCollectorResult(memoryUsableBytes, kibibytesToBytes(mem.Usable))) + } + + if mem.TotalSet { + crs = append(crs, vmiReport.newCollectorResult(memoryDomainBytes, kibibytesToBytes(mem.Total))) + } + + return crs +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics_test.go new file mode 100644 index 000000000000..9aefb74df64b --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics_test.go @@ -0,0 +1,100 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var _ = Describe("memory metrics", func() { + Context("on Collect", func() { + vmi := &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + } + + vmiStats := &VirtualMachineInstanceStats{ + DomainStats: &stats.DomainStats{ + Memory: &stats.DomainStatsMemory{ + RSSSet: true, + RSS: 1, + AvailableSet: true, + Available: 2, + UnusedSet: true, + Unused: 3, + CachedSet: true, + Cached: 4, + SwapInSet: true, + SwapIn: 5, + SwapOutSet: true, + SwapOut: 6, + MajorFaultSet: true, + MajorFault: 7, + MinorFaultSet: true, + MinorFault: 8, + ActualBalloonSet: true, + ActualBalloon: 9, + UsableSet: true, + Usable: 10, + TotalSet: true, + Total: 11, + }, + }, + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + + DescribeTable("should collect metrics values", func(metric operatormetrics.Metric, expectedValue float64) { + crs := memoryMetrics{}.Collect(vmiReport) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(metric, expectedValue))) + }, + Entry("kubevirt_vmi_memory_resident_bytes", memoryResident, kibibytesToBytes(1)), + Entry("kubevirt_vmi_memory_available_bytes", memoryAvailable, kibibytesToBytes(2)), + Entry("kubevirt_vmi_memory_unused_bytes", memoryUnused, kibibytesToBytes(3)), + Entry("kubevirt_vmi_memory_cached_bytes", memoryCached, kibibytesToBytes(4)), + Entry("kubevirt_vmi_memory_swap_in_traffic_bytes", memorySwapInTrafficBytes, kibibytesToBytes(5)), + Entry("kubevirt_vmi_memory_swap_out_traffic_bytes", memorySwapOutTrafficBytes, kibibytesToBytes(6)), + Entry("kubevirt_vmi_memory_pgmajfault_total", memoryPgmajfaultTotal, 7.0), + Entry("kubevirt_vmi_memory_pgminfault_total", memoryPgminfaultTotal, 8.0), + Entry("kubevirt_vmi_memory_actual_ballon_bytes", memoryActualBallon, kibibytesToBytes(9)), + Entry("kubevirt_vmi_memory_usable_bytes", memoryUsableBytes, kibibytesToBytes(10)), + Entry("kubevirt_vmi_memory_domain_bytes", memoryDomainBytes, kibibytesToBytes(11)), + ) + + It("result should be empty if stat not populated or set is false", func() { + vmiStats.DomainStats.Memory = &stats.DomainStatsMemory{ + RSSSet: false, + MinorFaultSet: false, + TotalSet: false, + } + crs := memoryMetrics{}.Collect(vmiReport) + Expect(crs).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics.go new file mode 100644 index 000000000000..bccd55bf3190 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics.go @@ -0,0 +1,93 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" +) + +var ( + migrateVMIDataRemaining = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_migration_data_remaining_bytes", + Help: "Number of bytes that still need to be transferred", + }, + ) + + migrateVMIDataProcessed = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_migration_data_processed_bytes", + Help: "Number of bytes transferred from the beginning of the job.", + }, + ) + + migrateVmiDirtyMemoryRate = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_migration_dirty_memory_rate_bytes", + Help: "Number of memory pages dirtied by the guest per second.", + }, + ) + + migrateVmiMemoryTransferRate = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_migration_disk_transfer_rate_bytes", + Help: "Network throughput used while migrating memory in Bytes per second.", + }, + ) +) + +type migrationMetrics struct{} + +func (migrationMetrics) Describe() []operatormetrics.Metric { + return []operatormetrics.Metric{ + migrateVMIDataRemaining, + migrateVMIDataProcessed, + migrateVmiDirtyMemoryRate, + migrateVmiMemoryTransferRate, + } +} + +func (migrationMetrics) Collect(vmiReport *VirtualMachineInstanceReport) []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + + if vmiReport.vmiStats.DomainStats == nil || vmiReport.vmiStats.DomainStats.MigrateDomainJobInfo == nil { + return crs + } + + jobInfo := vmiReport.vmiStats.DomainStats.MigrateDomainJobInfo + + if jobInfo.DataRemainingSet { + crs = append(crs, vmiReport.newCollectorResult(migrateVMIDataRemaining, float64(jobInfo.DataRemaining))) + } + + if jobInfo.DataProcessedSet { + crs = append(crs, vmiReport.newCollectorResult(migrateVMIDataProcessed, float64(jobInfo.DataProcessed))) + } + + if jobInfo.MemDirtyRateSet { + crs = append(crs, vmiReport.newCollectorResult(migrateVmiDirtyMemoryRate, float64(jobInfo.MemDirtyRate))) + } + + if jobInfo.MemoryBpsSet { + crs = append(crs, vmiReport.newCollectorResult(migrateVmiMemoryTransferRate, float64(jobInfo.MemoryBps))) + } + + return crs +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics_test.go new file mode 100644 index 000000000000..b0437caee60d --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics_test.go @@ -0,0 +1,77 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var _ = Describe("migration metrics", func() { + Context("on Collect", func() { + vmi := &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + } + + vmiStats := &VirtualMachineInstanceStats{ + DomainStats: &stats.DomainStats{ + MigrateDomainJobInfo: &stats.DomainJobInfo{ + DataRemainingSet: true, + DataRemaining: 1, + DataProcessedSet: true, + DataProcessed: 2, + MemDirtyRateSet: true, + MemDirtyRate: 3, + MemoryBpsSet: true, + MemoryBps: 4, + }, + }, + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + + DescribeTable("should collect metrics values", func(metric operatormetrics.Metric, expectedValue float64) { + crs := migrationMetrics{}.Collect(vmiReport) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(metric, expectedValue))) + }, + Entry("kubevirt_vmi_migration_data_remaining_bytes", migrateVMIDataRemaining, 1.0), + Entry("kubevirt_vmi_migration_data_processed_bytes", migrateVMIDataProcessed, 2.0), + Entry("kubevirt_vmi_migration_dirty_memory_rate_bytes", migrateVmiDirtyMemoryRate, 3.0), + Entry("kubevirt_vmi_migration_disk_transfer_rate_bytes", migrateVmiMemoryTransferRate, 4.0), + ) + + It("result should be empty if stat not populated or set is false", func() { + vmiStats.DomainStats.MigrateDomainJobInfo = &stats.DomainJobInfo{ + DataRemainingSet: false, + } + crs := migrationMetrics{}.Collect(vmiReport) + Expect(crs).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/network_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/network_metrics.go new file mode 100644 index 000000000000..4bc753f7356f --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/network_metrics.go @@ -0,0 +1,161 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + +var ( + networkTrafficBytesDeprecated = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_traffic_bytes_total", + Help: "[Deprecated] Total number of bytes sent and received.", + }, + ) + + networkReceiveBytes = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_receive_bytes_total", + Help: "Total network traffic received in bytes.", + }, + ) + + networkTransmitBytes = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_transmit_bytes_total", + Help: "Total network traffic transmitted in bytes.", + }, + ) + + networkReceivePackets = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_receive_packets_total", + Help: "Total network traffic received packets.", + }, + ) + + networkTransmitPackets = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_transmit_packets_total", + Help: "Total network traffic transmitted packets.", + }, + ) + + networkReceiveErrors = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_receive_errors_total", + Help: "Total network received error packets.", + }, + ) + + networkTransmitErrors = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_transmit_errors_total", + Help: "Total network transmitted error packets.", + }, + ) + + networkReceivePacketsDropped = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_receive_packets_dropped_total", + Help: "The total number of rx packets dropped on vNIC interfaces.", + }, + ) + + networkTransmitPacketsDropped = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_network_transmit_packets_dropped_total", + Help: "The total number of tx packets dropped on vNIC interfaces.", + }, + ) +) + +type networkMetrics struct{} + +func (networkMetrics) Describe() []operatormetrics.Metric { + return []operatormetrics.Metric{ + networkTrafficBytesDeprecated, + networkReceiveBytes, + networkTransmitBytes, + networkReceivePackets, + networkTransmitPackets, + networkReceiveErrors, + networkTransmitErrors, + networkReceivePacketsDropped, + networkTransmitPacketsDropped, + } +} + +func (networkMetrics) Collect(vmiReport *VirtualMachineInstanceReport) []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + + if vmiReport.vmiStats.DomainStats == nil || vmiReport.vmiStats.DomainStats.Net == nil { + return crs + } + + for _, net := range vmiReport.vmiStats.DomainStats.Net { + if !net.NameSet { + continue + } + + iface := net.Name + if net.AliasSet { + iface = net.Alias + } + netLabels := map[string]string{"interface": iface} + + if net.RxBytesSet { + deprecatedLabels := map[string]string{"interface": iface, "type": "rx"} + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkTrafficBytesDeprecated, float64(net.RxBytes), deprecatedLabels)) + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkReceiveBytes, float64(net.RxBytes), netLabels)) + } + + if net.TxBytesSet { + deprecatedLabels := map[string]string{"interface": iface, "type": "tx"} + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkTrafficBytesDeprecated, float64(net.TxBytes), deprecatedLabels)) + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkTransmitBytes, float64(net.TxBytes), netLabels)) + } + + if net.RxPktsSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkReceivePackets, float64(net.RxPkts), netLabels)) + } + + if net.TxPktsSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkTransmitPackets, float64(net.TxPkts), netLabels)) + } + + if net.RxErrsSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkReceiveErrors, float64(net.RxErrs), netLabels)) + } + + if net.TxErrsSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkTransmitErrors, float64(net.TxErrs), netLabels)) + } + + if net.RxDropSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkReceivePacketsDropped, float64(net.RxDrop), netLabels)) + } + + if net.TxDropSet { + crs = append(crs, vmiReport.newCollectorResultWithLabels(networkTransmitPacketsDropped, float64(net.TxDrop), netLabels)) + } + } + + return crs +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/network_metrics_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/network_metrics_test.go new file mode 100644 index 000000000000..427914451c6f --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/network_metrics_test.go @@ -0,0 +1,91 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var _ = Describe("network metrics", func() { + Context("on Collect", func() { + vmi := &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + } + + vmiStats := &VirtualMachineInstanceStats{ + DomainStats: &stats.DomainStats{ + Net: []stats.DomainStatsNet{ + { + NameSet: true, + Name: "vnet0", + RxBytesSet: true, + RxBytes: 1, + TxBytesSet: true, + TxBytes: 2, + RxPktsSet: true, + RxPkts: 3, + TxPktsSet: true, + TxPkts: 4, + RxErrsSet: true, + RxErrs: 5, + TxErrsSet: true, + TxErrs: 6, + RxDropSet: true, + RxDrop: 7, + TxDropSet: true, + TxDrop: 8, + }, + }, + }, + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + + DescribeTable("should collect metrics values", func(metric operatormetrics.Metric, expectedValue float64) { + crs := networkMetrics{}.Collect(vmiReport) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(metric, expectedValue))) + }, + Entry("kubevirt_vmi_network_receive_bytes_total", networkReceiveBytes, 1.0), + Entry("kubevirt_vmi_network_transmit_bytes_total", networkTransmitBytes, 2.0), + Entry("kubevirt_vmi_network_receive_packets_total", networkReceivePackets, 3.0), + Entry("kubevirt_vmi_network_transmit_packets_total", networkTransmitPackets, 4.0), + Entry("kubevirt_vmi_network_receive_errors_total", networkReceiveErrors, 5.0), + Entry("kubevirt_vmi_network_transmit_errors_total", networkTransmitErrors, 6.0), + Entry("kubevirt_vmi_network_receive_packets_dropped_total", networkReceivePacketsDropped, 7.0), + Entry("kubevirt_vmi_network_transmit_packets_dropped_total", networkTransmitPacketsDropped, 8.0), + ) + + It("result should be empty if stat not populated or set is false", func() { + vmiStats.DomainStats.Net[0].NameSet = false + crs := networkMetrics{}.Collect(vmiReport) + Expect(crs).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/node_cpu_affinity_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/node_cpu_affinity_metrics.go new file mode 100644 index 000000000000..382abe2f63ac --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/node_cpu_affinity_metrics.go @@ -0,0 +1,57 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + +var ( + nodeCpuAffinity = operatormetrics.NewGauge( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_node_cpu_affinity", + Help: "Number of VMI CPU affinities to node physical cores.", + }, + ) +) + +type cpuAffinityMetrics struct{} + +func (cpuAffinityMetrics) Describe() []operatormetrics.Metric { + return []operatormetrics.Metric{nodeCpuAffinity} +} + +func (cpuAffinityMetrics) Collect(vmiReport *VirtualMachineInstanceReport) []operatormetrics.CollectorResult { + if vmiReport.vmiStats.DomainStats == nil || !vmiReport.vmiStats.DomainStats.CPUMapSet { + return []operatormetrics.CollectorResult{} + } + + affinityCount := 0.0 + + for vidx := 0; vidx < len(vmiReport.vmiStats.DomainStats.CPUMap); vidx++ { + for cidx := 0; cidx < len(vmiReport.vmiStats.DomainStats.CPUMap[vidx]); cidx++ { + if vmiReport.vmiStats.DomainStats.CPUMap[vidx][cidx] { + affinityCount++ + } + } + } + + return []operatormetrics.CollectorResult{ + vmiReport.newCollectorResult(nodeCpuAffinity, affinityCount), + } +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/node_cpu_affinity_metrics_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/node_cpu_affinity_metrics_test.go new file mode 100644 index 000000000000..68e6d2da9df6 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/node_cpu_affinity_metrics_test.go @@ -0,0 +1,67 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var _ = Describe("node cpu affinity metrics", func() { + Context("on Collect", func() { + vmi := &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + } + + vmiStats := &VirtualMachineInstanceStats{ + DomainStats: &stats.DomainStats{ + CPUMapSet: true, + CPUMap: [][]bool{ + {true, false, true}, + {false, true, false}, + }, + }, + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + + DescribeTable("should collect metrics values", func(metric operatormetrics.Metric, expectedValue float64) { + crs := cpuAffinityMetrics{}.Collect(vmiReport) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(metric, expectedValue))) + }, + Entry("kubevirt_vmi_node_cpu_affinity", nodeCpuAffinity, 3.0), + ) + + It("result should be empty if stat not populated or set is false", func() { + vmiStats.DomainStats.CPUMapSet = false + crs := cpuAffinityMetrics{}.Collect(vmiReport) + Expect(crs).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/scrapper.go b/pkg/monitoring/metrics/virt-handler/domainstats/scrapper.go new file mode 100644 index 000000000000..40436d0739aa --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/scrapper.go @@ -0,0 +1,116 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "fmt" + "time" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + k6tv1 "kubevirt.io/api/core/v1" + "kubevirt.io/client-go/log" + + "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats/collector" + cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client" +) + +type domainstatsScraper struct { + ch chan operatormetrics.CollectorResult +} + +func (d domainstatsScraper) Scrape(socketFile string, vmi *k6tv1.VirtualMachineInstance) { + ts := time.Now() + + exists, vmStats, err := gatherMetrics(socketFile) + if err != nil { + log.Log.Reason(err).Errorf("failed to scrape metrics from %s", socketFile) + return + } + + if !exists || vmStats.DomainStats.Name == "" { + log.Log.V(2).Infof("disappearing VM on %s, ignored", socketFile) // VM may be shutting down + return + } + + // GetDomainStats() may hang for a long time. + // If it wakes up past the timeout, there is no point in send back any metric. + // In the best case the information is stale, in the worst case the information is stale *and* + // the reporting channel is already closed, leading to a possible panic - see below + elapsed := time.Now().Sub(ts) + if elapsed > collector.StatsMaxAge { + log.Log.Infof("took too long (%v) to collect stats from %s: ignored", elapsed, socketFile) + return + } + + d.report(vmi, vmStats) +} + +func (d domainstatsScraper) Complete() { + close(d.ch) +} + +func (d domainstatsScraper) report(vmi *k6tv1.VirtualMachineInstance, vmStats *VirtualMachineInstanceStats) { + // statsMaxAge is an estimation - and there is no better way to do that. So it is possible that + // GetDomainStats() takes enough time to lag behind, but not enough to trigger the statsMaxAge check. + // In this case the next functions will end up writing on a closed channel. This will panic. + // It is actually OK in this case to abort the goroutine that panicked -that's what we want anyway, + // and the very reason we collect in throwaway goroutines. We need however to avoid dump stacktraces in the logs. + // Since this is a known failure condition, let's handle it explicitly. + defer func() { + if err := recover(); err != nil { + log.Log.Warningf("collector goroutine panicked for VM %s: %v", vmi.Name, err) + } + }() + + vmiReport := newVirtualMachineInstanceReport(vmi, vmStats) + for _, rm := range rms { + crs := rm.Collect(vmiReport) + for _, cr := range crs { + d.ch <- cr + } + } +} + +func gatherMetrics(socketFile string) (bool, *VirtualMachineInstanceStats, error) { + cli, err := cmdclient.NewClient(socketFile) + if err != nil { + // Ignore failure to connect to client. + // These are all local connections via unix socket. + // A failure to connect means there's nothing on the other + // end listening. + return false, nil, fmt.Errorf("failed to connect to cmd client socket: %w", err) + } + defer cli.Close() + + vmStats := &VirtualMachineInstanceStats{} + var exists bool + + vmStats.DomainStats, exists, err = cli.GetDomainStats() + if err != nil { + return false, nil, fmt.Errorf("failed to update domain stats from socket %s: %w", socketFile, err) + } + + vmStats.FsStats, err = cli.GetFilesystems() + if err != nil { + return false, nil, fmt.Errorf("failed to update filesystem stats from socket %s: %w", socketFile, err) + } + + return exists, vmStats, nil +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/unit_converter.go b/pkg/monitoring/metrics/virt-handler/domainstats/unit_converter.go new file mode 100644 index 000000000000..0085d701758d --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/unit_converter.go @@ -0,0 +1,13 @@ +package domainstats + +func nanosecondsToSeconds(ns uint64) float64 { + return float64(ns) / 1000000000 +} + +func microsecondsToSeconds(us uint64) float64 { + return float64(us) / 1000000 +} + +func kibibytesToBytes(kibibytes uint64) float64 { + return float64(kibibytes) * 1024 +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/vcpu_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/vcpu_metrics.go new file mode 100644 index 000000000000..f98011fc4ae0 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/vcpu_metrics.go @@ -0,0 +1,110 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + "fmt" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var ( + vcpuSeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_vcpu_seconds_total", + Help: "Total amount of time spent in each state by each vcpu (cpu_time excluding hypervisor time). Where `id` is the vcpu identifier and `state` can be one of the following: [`OFFLINE`, `RUNNING`, `BLOCKED`].", + }, + ) + + vcpuWaitSeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_vcpu_wait_seconds_total", + Help: "Amount of time spent by each vcpu while waiting on I/O.", + }, + ) + + vcpuDelaySeconds = operatormetrics.NewCounter( + operatormetrics.MetricOpts{ + Name: "kubevirt_vmi_vcpu_delay_seconds_total", + Help: "Amount of time spent by each vcpu waiting in the queue instead of running.", + }, + ) +) + +type vcpuMetrics struct{} + +func (vcpuMetrics) Describe() []operatormetrics.Metric { + return []operatormetrics.Metric{ + vcpuSeconds, + vcpuWaitSeconds, + vcpuDelaySeconds, + } +} + +func (vcpuMetrics) Collect(vmiReport *VirtualMachineInstanceReport) []operatormetrics.CollectorResult { + var crs []operatormetrics.CollectorResult + + if vmiReport.vmiStats.DomainStats == nil || vmiReport.vmiStats.DomainStats.Vcpu == nil { + return crs + } + + for vcpuIdx, vcpu := range vmiReport.vmiStats.DomainStats.Vcpu { + stringVcpuIdx := fmt.Sprintf("%d", vcpuIdx) + + if vcpu.TimeSet { + additionalLabels := map[string]string{ + "id": stringVcpuIdx, + "state": humanReadableState(vcpu.State), + } + crs = append(crs, vmiReport.newCollectorResultWithLabels(vcpuSeconds, microsecondsToSeconds(vcpu.Time), additionalLabels)) + } + + if vcpu.WaitSet { + additionalLabels := map[string]string{ + "id": stringVcpuIdx, + } + crs = append(crs, vmiReport.newCollectorResultWithLabels(vcpuWaitSeconds, nanosecondsToSeconds(vcpu.Wait), additionalLabels)) + } + + if vcpu.DelaySet { + additionalLabels := map[string]string{ + "id": stringVcpuIdx, + } + crs = append(crs, vmiReport.newCollectorResultWithLabels(vcpuDelaySeconds, nanosecondsToSeconds(vcpu.Delay), additionalLabels)) + } + } + + return crs +} + +func humanReadableState(state int) string { + switch state { + case stats.VCPUOffline: + return "offline" + case stats.VCPUBlocked: + return "blocked" + case stats.VCPURunning: + return "running" + default: + return "unknown" + } +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/vcpu_metrics_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/vcpu_metrics_test.go new file mode 100644 index 000000000000..e6fa066cf1db --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/vcpu_metrics_test.go @@ -0,0 +1,78 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package domainstats + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k6tv1 "kubevirt.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" +) + +var _ = Describe("vcpu metrics", func() { + Context("on Collect", func() { + vmi := &k6tv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vmi-1", + Namespace: "test-ns-1", + }, + } + + vmiStats := &VirtualMachineInstanceStats{ + DomainStats: &stats.DomainStats{ + Vcpu: []stats.DomainStatsVcpu{ + { + TimeSet: true, + Time: 1, + WaitSet: true, + Wait: 2, + DelaySet: true, + Delay: 3, + }, + }, + }, + } + + vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) + + DescribeTable("should collect metrics values", func(metric operatormetrics.Metric, expectedValue float64) { + crs := vcpuMetrics{}.Collect(vmiReport) + Expect(crs).To(ContainElement(gomegaContainsMetricMatcher(metric, expectedValue))) + }, + Entry("kubevirt_vmi_vcpu_seconds_total", vcpuSeconds, microsecondsToSeconds(1)), + Entry("kubevirt_vmi_vcpu_wait_seconds_total", vcpuWaitSeconds, nanosecondsToSeconds(2)), + Entry("kubevirt_vmi_vcpu_delay_seconds_total", vcpuDelaySeconds, nanosecondsToSeconds(3)), + ) + + It("result should be empty if stat not populated or set is false", func() { + vmiStats.DomainStats.Vcpu = []stats.DomainStatsVcpu{ + { + TimeSet: false, + }, + } + crs := vcpuMetrics{}.Collect(vmiReport) + Expect(crs).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/monitoring/metrics/virt-handler/metrics.go b/pkg/monitoring/metrics/virt-handler/metrics.go index 0544d966bf5d..45055cd65a4f 100644 --- a/pkg/monitoring/metrics/virt-handler/metrics.go +++ b/pkg/monitoring/metrics/virt-handler/metrics.go @@ -20,6 +20,7 @@ package virt_handler import ( "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "k8s.io/client-go/tools/cache" "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" From 68afcdc11fab3d127cf28c128190d091f1c0f8d6 Mon Sep 17 00:00:00 2001 From: machadovilaca Date: Mon, 25 Mar 2024 17:48:20 +0000 Subject: [PATCH 4/5] Refactor domainstats collector Signed-off-by: machadovilaca --- cmd/virt-handler/virt-handler.go | 13 ++-- .../domainstats/collector_suite_test.go | 11 --- .../domainstats/downwardmetrics/BUILD.bazel | 2 +- .../domainstats/downwardmetrics/scraper.go | 12 ++-- pkg/monitoring/domainstats/scraper.go | 7 -- .../metrics/virt-handler/BUILD.bazel | 1 + .../virt-handler/domainstats/BUILD.bazel | 20 +++++- .../virt-handler/domainstats/collector.go | 71 ++++++++++++------- .../domainstats/collector}/BUILD.bazel | 2 +- .../domainstats/collector}/collector.go | 14 ++-- .../collector/collector_suite_test.go | 30 ++++++++ .../domainstats/collector}/collector_test.go | 9 +-- .../domainstats/collector/scraper.go | 27 +++++++ .../domainstats/domainstats_suite_test.go | 3 +- .../domainstats/domainstats_test.go | 1 - .../domainstats/memory_metrics.go | 8 +-- .../domainstats/migration_metrics.go | 8 +-- .../metrics/virt-handler/metrics.go | 3 +- 18 files changed, 164 insertions(+), 78 deletions(-) delete mode 100644 pkg/monitoring/domainstats/collector_suite_test.go delete mode 100644 pkg/monitoring/domainstats/scraper.go rename pkg/monitoring/{domainstats => metrics/virt-handler/domainstats/collector}/BUILD.bazel (90%) rename pkg/monitoring/{domainstats => metrics/virt-handler/domainstats/collector}/collector.go (90%) create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/collector/collector_suite_test.go rename pkg/monitoring/{domainstats => metrics/virt-handler/domainstats/collector}/collector_test.go (96%) create mode 100644 pkg/monitoring/metrics/virt-handler/domainstats/collector/scraper.go diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go index 1611ecb2a0c0..70cd91501cad 100644 --- a/cmd/virt-handler/virt-handler.go +++ b/cmd/virt-handler/virt-handler.go @@ -71,8 +71,8 @@ import ( "kubevirt.io/kubevirt/pkg/certificates/bootstrap" containerdisk "kubevirt.io/kubevirt/pkg/container-disk" "kubevirt.io/kubevirt/pkg/controller" - promdomain "kubevirt.io/kubevirt/pkg/monitoring/domainstats/prometheus" // import for prometheus metrics metrics "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler" + metricshandler "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/handler" "kubevirt.io/kubevirt/pkg/monitoring/profiler" "kubevirt.io/kubevirt/pkg/service" "kubevirt.io/kubevirt/pkg/util" @@ -201,10 +201,6 @@ func (app *virtHandlerApp) Run() { logger.V(1).Infof("hostname %s", app.HostOverride) var err error - if err := metrics.SetupMetrics(); err != nil { - panic(fmt.Errorf("failed to set up metrics: %v", err)) - } - // Copy container-disk binary targetFile := filepath.Join(app.VirtLibDir, "/init/usr/bin/container-disk") err = os.MkdirAll(filepath.Dir(targetFile), os.ModePerm) @@ -375,7 +371,10 @@ func (app *virtHandlerApp) Run() { app.VirtShareDir, ) - promdomain.SetupDomainStatsCollector(app.virtCli, app.VirtShareDir, app.HostOverride, app.MaxRequestsInFlight, vmiSourceInformer) + if err := metrics.SetupMetrics(app.VirtShareDir, app.HostOverride, app.MaxRequestsInFlight, vmiSourceInformer); err != nil { + panic(err) + } + if err := downwardmetrics.RunDownwardMetricsCollector(context.Background(), app.HostOverride, vmiSourceInformer, podIsolationDetector); err != nil { panic(fmt.Errorf("failed to set up the downwardMetrics collector: %v", err)) } @@ -543,7 +542,7 @@ func (app *virtHandlerApp) runPrometheusServer(errCh chan error) { mux.Add(webService) log.Log.V(1).Infof("metrics: max concurrent requests=%d", app.MaxRequestsInFlight) - mux.Handle("/metrics", promdomain.Handler(app.MaxRequestsInFlight)) + mux.Handle("/metrics", metricshandler.Handler(app.MaxRequestsInFlight)) server := http.Server{ Addr: app.ServiceListen.Address(), Handler: mux, diff --git a/pkg/monitoring/domainstats/collector_suite_test.go b/pkg/monitoring/domainstats/collector_suite_test.go deleted file mode 100644 index a430dcb83140..000000000000 --- a/pkg/monitoring/domainstats/collector_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package vms - -import ( - "testing" - - "kubevirt.io/client-go/testutils" -) - -func TestPrometheus(t *testing.T) { - testutils.KubeVirtTestSuiteSetup(t) -} diff --git a/pkg/monitoring/domainstats/downwardmetrics/BUILD.bazel b/pkg/monitoring/domainstats/downwardmetrics/BUILD.bazel index 9fcbe7cdbb09..032cb2f782af 100644 --- a/pkg/monitoring/domainstats/downwardmetrics/BUILD.bazel +++ b/pkg/monitoring/domainstats/downwardmetrics/BUILD.bazel @@ -13,7 +13,7 @@ go_library( "//pkg/downwardmetrics/vhostmd:go_default_library", "//pkg/downwardmetrics/vhostmd/api:go_default_library", "//pkg/downwardmetrics/vhostmd/metrics:go_default_library", - "//pkg/monitoring/domainstats:go_default_library", + "//pkg/monitoring/metrics/virt-handler/domainstats/collector:go_default_library", "//pkg/virt-handler/cmd-client:go_default_library", "//pkg/virt-handler/isolation:go_default_library", "//pkg/virt-launcher/virtwrap/stats:go_default_library", diff --git a/pkg/monitoring/domainstats/downwardmetrics/scraper.go b/pkg/monitoring/domainstats/downwardmetrics/scraper.go index 20a735183861..ffff5690cfc2 100644 --- a/pkg/monitoring/domainstats/downwardmetrics/scraper.go +++ b/pkg/monitoring/domainstats/downwardmetrics/scraper.go @@ -14,14 +14,14 @@ import ( "kubevirt.io/kubevirt/pkg/downwardmetrics/vhostmd" "kubevirt.io/kubevirt/pkg/downwardmetrics/vhostmd/api" metricspkg "kubevirt.io/kubevirt/pkg/downwardmetrics/vhostmd/metrics" - vms "kubevirt.io/kubevirt/pkg/monitoring/domainstats" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats/collector" cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client" "kubevirt.io/kubevirt/pkg/virt-handler/isolation" "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" ) const DownwardmetricsRefreshDuration = 5 * time.Second -const DownwardmetricsCollectionTimeout = vms.CollectionTimeout +const DownwardmetricsCollectionTimeout = collector.CollectionTimeout const qemuVersionUnknown = "qemu-unknown" type StaticHostMetrics struct { @@ -35,6 +35,8 @@ type Scraper struct { reporter *DownwardMetricsReporter } +func (s *Scraper) Complete() {} + func (s *Scraper) Scrape(socketFile string, vmi *k6sv1.VirtualMachineInstance) { if !vmi.IsRunning() || !downwardmetrics.HasDownwardMetricDisk(vmi) { return @@ -100,7 +102,7 @@ func (r *DownwardMetricsReporter) Report(socketFile string) (*api.Metrics, error // In the best case the information is stale, in the worst case the information is stale *and* // the reporting channel is already closed, leading to a possible panic - see below elapsed := time.Now().Sub(ts) - if elapsed > vms.StatsMaxAge { + if elapsed > collector.StatsMaxAge { log.Log.Infof("took too long (%v) to collect stats from %s: ignored", elapsed, socketFile) return nil, fmt.Errorf("took too long (%v) to collect stats from %s: ignored", elapsed, socketFile) } @@ -142,7 +144,7 @@ func guestMemoryMetrics(vmStats *stats.DomainStats) []api.Metric { } type Collector struct { - concCollector *vms.ConcurrentCollector + concCollector *collector.ConcurrentCollector } func NewReporter(nodeName string) *DownwardMetricsReporter { @@ -161,7 +163,7 @@ func RunDownwardMetricsCollector(context context.Context, nodeName string, vmiIn isolation: isolation, reporter: NewReporter(nodeName), } - collector := vms.NewConcurrentCollector(1) + collector := collector.NewConcurrentCollector(1) go func() { ticker := time.NewTicker(DownwardmetricsRefreshDuration) diff --git a/pkg/monitoring/domainstats/scraper.go b/pkg/monitoring/domainstats/scraper.go deleted file mode 100644 index 3fccd5ee128f..000000000000 --- a/pkg/monitoring/domainstats/scraper.go +++ /dev/null @@ -1,7 +0,0 @@ -package vms - -import k6tv1 "kubevirt.io/api/core/v1" - -type MetricsScraper interface { - Scrape(key string, vmi *k6tv1.VirtualMachineInstance) -} diff --git a/pkg/monitoring/metrics/virt-handler/BUILD.bazel b/pkg/monitoring/metrics/virt-handler/BUILD.bazel index 3b984bbd8023..de097692417b 100644 --- a/pkg/monitoring/metrics/virt-handler/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-handler/BUILD.bazel @@ -14,5 +14,6 @@ go_library( "//pkg/monitoring/metrics/virt-handler/domainstats:go_default_library", "//staging/src/kubevirt.io/client-go/version:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + "//vendor/k8s.io/client-go/tools/cache:go_default_library", ], ) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/BUILD.bazel b/pkg/monitoring/metrics/virt-handler/domainstats/BUILD.bazel index d1403a20d9cc..9b51e977cba4 100644 --- a/pkg/monitoring/metrics/virt-handler/domainstats/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-handler/domainstats/BUILD.bazel @@ -12,31 +12,49 @@ go_library( "migration_metrics.go", "network_metrics.go", "node_cpu_affinity_metrics.go", - "vcpuMetrics.go", + "scrapper.go", + "unit_converter.go", + "vcpu_metrics.go", ], importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats", visibility = ["//visibility:public"], deps = [ + "//pkg/monitoring/metrics/virt-handler/domainstats/collector:go_default_library", + "//pkg/virt-handler/cmd-client:go_default_library", "//pkg/virt-launcher/virtwrap/stats:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/log:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + "//vendor/k8s.io/client-go/tools/cache:go_default_library", ], ) go_test( name = "go_default_test", srcs = [ + "block_metrics_test.go", + "collector_test.go", + "cpu_metrics_test.go", "domainstats_suite_test.go", "domainstats_test.go", + "filesystem_metrics_test.go", + "memory_metrics_test.go", + "migration_metrics_test.go", + "network_metrics_test.go", + "node_cpu_affinity_metrics_test.go", + "vcpu_metrics_test.go", ], embed = [":go_default_library"], deps = [ + "//pkg/monitoring/metrics/virt-handler/domainstats/collector:go_default_library", + "//pkg/virt-launcher/virtwrap/stats:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/testutils:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", "//vendor/github.com/onsi/ginkgo/v2:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/onsi/gomega/format:go_default_library", + "//vendor/github.com/onsi/gomega/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", ], ) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/collector.go b/pkg/monitoring/metrics/virt-handler/domainstats/collector.go index 69f702549c42..410a705dd4cf 100644 --- a/pkg/monitoring/metrics/virt-handler/domainstats/collector.go +++ b/pkg/monitoring/metrics/virt-handler/domainstats/collector.go @@ -20,13 +20,16 @@ package domainstats import ( "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "k8s.io/client-go/tools/cache" k6tv1 "kubevirt.io/api/core/v1" + "kubevirt.io/client-go/log" + + "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats/collector" ) -type resourceMetrics interface { - Describe() []operatormetrics.Metric - Collect(report *VirtualMachineInstanceReport) []operatormetrics.CollectorResult -} +const ( + PrometheusCollectionTimeout = collector.CollectionTimeout +) var ( rms = []resourceMetrics{ @@ -44,8 +47,31 @@ var ( Metrics: domainStatsMetrics(rms...), CollectCallback: domainStatsCollectorCallback, } + + settings *collectorSettings ) +type resourceMetrics interface { + Describe() []operatormetrics.Metric + Collect(report *VirtualMachineInstanceReport) []operatormetrics.CollectorResult +} + +type collectorSettings struct { + virtShareDir string + nodeName string + maxRequestsInFlight int + vmiInformer cache.SharedIndexInformer +} + +func SetupDomainStatsCollector(virtShareDir, nodeName string, maxRequestsInFlight int, vmiInformer cache.SharedIndexInformer) { + settings = &collectorSettings{ + virtShareDir: virtShareDir, + nodeName: nodeName, + maxRequestsInFlight: maxRequestsInFlight, + vmiInformer: vmiInformer, + } +} + func domainStatsMetrics(rms ...resourceMetrics) []operatormetrics.Metric { var metrics []operatormetrics.Metric @@ -57,33 +83,30 @@ func domainStatsMetrics(rms ...resourceMetrics) []operatormetrics.Metric { } func domainStatsCollectorCallback() []operatormetrics.CollectorResult { - var crs []operatormetrics.CollectorResult - var vmis []k6tv1.VirtualMachineInstance - - for _, vmi := range vmis { - crs = append(crs, collect(&vmi)...) + cachedObjs := settings.vmiInformer.GetIndexer().List() + if len(cachedObjs) == 0 { + log.Log.V(4).Infof("No VMIs detected") + return []operatormetrics.CollectorResult{} } - return crs -} + vmis := make([]*k6tv1.VirtualMachineInstance, len(cachedObjs)) -func collect(vmi *k6tv1.VirtualMachineInstance) []operatormetrics.CollectorResult { - var crs []operatormetrics.CollectorResult + for i, obj := range cachedObjs { + vmis[i] = obj.(*k6tv1.VirtualMachineInstance) + } - vmiStats := scrape(vmi) + concCollector := collector.NewConcurrentCollector(settings.maxRequestsInFlight) + return execCollector(concCollector, vmis) +} - if vmiStats == nil { - return crs - } +func execCollector(concCollector collector.Collector, vmis []*k6tv1.VirtualMachineInstance) []operatormetrics.CollectorResult { + scraper := &domainstatsScraper{ch: make(chan operatormetrics.CollectorResult, len(vmis)*len(rms))} + go concCollector.Collect(vmis, scraper, PrometheusCollectionTimeout) - vmiReport := newVirtualMachineInstanceReport(vmi, vmiStats) - for _, rm := range rms { - crs = append(crs, rm.Collect(vmiReport)...) + var crs []operatormetrics.CollectorResult + for cr := range scraper.ch { + crs = append(crs, cr) } return crs } - -func scrape(vmi *k6tv1.VirtualMachineInstance) *VirtualMachineInstanceStats { - return nil -} diff --git a/pkg/monitoring/domainstats/BUILD.bazel b/pkg/monitoring/metrics/virt-handler/domainstats/collector/BUILD.bazel similarity index 90% rename from pkg/monitoring/domainstats/BUILD.bazel rename to pkg/monitoring/metrics/virt-handler/domainstats/collector/BUILD.bazel index ea65182766d1..d70f3bfc4704 100644 --- a/pkg/monitoring/domainstats/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-handler/domainstats/collector/BUILD.bazel @@ -6,7 +6,7 @@ go_library( "collector.go", "scraper.go", ], - importpath = "kubevirt.io/kubevirt/pkg/monitoring/domainstats", + importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats/collector", visibility = ["//visibility:public"], deps = [ "//pkg/virt-handler/cmd-client:go_default_library", diff --git a/pkg/monitoring/domainstats/collector.go b/pkg/monitoring/metrics/virt-handler/domainstats/collector/collector.go similarity index 90% rename from pkg/monitoring/domainstats/collector.go rename to pkg/monitoring/metrics/virt-handler/domainstats/collector/collector.go index 91d78f6c013a..dcb1013810b3 100644 --- a/pkg/monitoring/domainstats/collector.go +++ b/pkg/monitoring/metrics/virt-handler/domainstats/collector/collector.go @@ -17,7 +17,7 @@ * */ -package vms +package collector import ( "sync" @@ -35,7 +35,6 @@ const StatsMaxAge = CollectionTimeout + 2*time.Second // "a bit more" than timeo type vmiSocketMap map[string]*k6tv1.VirtualMachineInstance type Collector interface { - Scrape(key string, vmi *k6tv1.VirtualMachineInstance) Collect(vmis []*k6tv1.VirtualMachineInstance, scraper MetricsScraper, timeout time.Duration) (skipped []string, completed bool) } @@ -46,11 +45,15 @@ type ConcurrentCollector struct { socketMapper func(vmis []*k6tv1.VirtualMachineInstance) vmiSocketMap } -func NewConcurrentCollector(MaxRequestsPerKey int) *ConcurrentCollector { +func NewConcurrentCollector(MaxRequestsPerKey int) Collector { + return NewConcurrentCollectorWithMapper(MaxRequestsPerKey, newvmiSocketMapFromVMIs) +} + +func NewConcurrentCollectorWithMapper(MaxRequestsPerKey int, mapper func(vmis []*k6tv1.VirtualMachineInstance) vmiSocketMap) Collector { return &ConcurrentCollector{ clientsPerKey: make(map[string]int), maxClientsPerKey: MaxRequestsPerKey, - socketMapper: newvmiSocketMapFromVMIs, + socketMapper: mapper, } } @@ -59,7 +62,7 @@ func (cc *ConcurrentCollector) Collect(vmis []*k6tv1.VirtualMachineInstance, scr log.Log.V(3).Infof("Collecting VM metrics from %d sources", len(socketToVMIs)) var busyScrapers sync.WaitGroup - skipped := []string{} + var skipped []string for key, vmi := range socketToVMIs { reserved := cc.reserveKey(key) if !reserved { @@ -88,6 +91,7 @@ func (cc *ConcurrentCollector) Collect(vmis []*k6tv1.VirtualMachineInstance, scr } log.Log.V(4).Infof("Collection completed") + scraper.Complete() return skipped, completed } diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/collector/collector_suite_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/collector/collector_suite_test.go new file mode 100644 index 000000000000..420762a2f48e --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/collector/collector_suite_test.go @@ -0,0 +1,30 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package collector + +import ( + "testing" + + "kubevirt.io/client-go/testutils" +) + +func TestPrometheus(t *testing.T) { + testutils.KubeVirtTestSuiteSetup(t) +} diff --git a/pkg/monitoring/domainstats/collector_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/collector/collector_test.go similarity index 96% rename from pkg/monitoring/domainstats/collector_test.go rename to pkg/monitoring/metrics/virt-handler/domainstats/collector/collector_test.go index ffeb800ca382..9f85b4b7bae4 100644 --- a/pkg/monitoring/domainstats/collector_test.go +++ b/pkg/monitoring/metrics/virt-handler/domainstats/collector/collector_test.go @@ -17,7 +17,7 @@ * */ -package vms +package collector import ( "time" @@ -32,7 +32,7 @@ import ( var _ = Describe("Collector", func() { var vmis []*k6tv1.VirtualMachineInstance - var newCollector = func(retries int) *ConcurrentCollector { + var newCollector = func(retries int) Collector { fakeMapper := func(vmi []*k6tv1.VirtualMachineInstance) vmiSocketMap { m := vmiSocketMap{} for _, vmi := range vmi { @@ -40,8 +40,7 @@ var _ = Describe("Collector", func() { } return m } - collector := NewConcurrentCollector(retries) - collector.socketMapper = fakeMapper + collector := NewConcurrentCollectorWithMapper(retries, fakeMapper) return collector } @@ -173,3 +172,5 @@ func (fs *fakeScraper) Scrape(key string, _ *k6tv1.VirtualMachineInstance) { fs.ready[key] <- true } } + +func (fs *fakeScraper) Complete() {} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/collector/scraper.go b/pkg/monitoring/metrics/virt-handler/domainstats/collector/scraper.go new file mode 100644 index 000000000000..697c70878357 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/domainstats/collector/scraper.go @@ -0,0 +1,27 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package collector + +import k6tv1 "kubevirt.io/api/core/v1" + +type MetricsScraper interface { + Scrape(key string, vmi *k6tv1.VirtualMachineInstance) + Complete() +} diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_suite_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_suite_test.go index 5c2d7d693f25..a423c85aa368 100644 --- a/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_suite_test.go +++ b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_suite_test.go @@ -20,10 +20,11 @@ package domainstats import ( + "testing" + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" "github.com/onsi/gomega/format" "github.com/onsi/gomega/types" - "testing" "kubevirt.io/client-go/testutils" ) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_test.go b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_test.go index a50ce4815419..8457b678b93d 100644 --- a/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_test.go +++ b/pkg/monitoring/metrics/virt-handler/domainstats/domainstats_test.go @@ -25,7 +25,6 @@ import ( "github.com/machadovilaca/operator-observability/pkg/operatormetrics" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k6tv1 "kubevirt.io/api/core/v1" ) diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics.go index a9967f6f7cae..df550f978767 100644 --- a/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics.go +++ b/pkg/monitoring/metrics/virt-handler/domainstats/memory_metrics.go @@ -19,9 +19,7 @@ package domainstats -import ( - "github.com/machadovilaca/operator-observability/pkg/operatormetrics" -) +import "github.com/machadovilaca/operator-observability/pkg/operatormetrics" var ( memoryResident = operatormetrics.NewGauge( @@ -66,14 +64,14 @@ var ( }, ) - memoryPgmajfaultTotal = operatormetrics.NewGauge( + memoryPgmajfaultTotal = operatormetrics.NewCounter( operatormetrics.MetricOpts{ Name: "kubevirt_vmi_memory_pgmajfault_total", Help: "The number of page faults when disk IO was required. Page faults occur when a process makes a valid access to virtual memory that is not available. When servicing the page fault, if disk IO is required, it is considered as major fault.", }, ) - memoryPgminfaultTotal = operatormetrics.NewGauge( + memoryPgminfaultTotal = operatormetrics.NewCounter( operatormetrics.MetricOpts{ Name: "kubevirt_vmi_memory_pgminfault_total", Help: "The number of other page faults, when disk IO was not required. Page faults occur when a process makes a valid access to virtual memory that is not available. When servicing the page fault, if disk IO is NOT required, it is considered as minor fault.", diff --git a/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics.go b/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics.go index bccd55bf3190..fd9c20315b32 100644 --- a/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics.go +++ b/pkg/monitoring/metrics/virt-handler/domainstats/migration_metrics.go @@ -27,28 +27,28 @@ var ( migrateVMIDataRemaining = operatormetrics.NewGauge( operatormetrics.MetricOpts{ Name: "kubevirt_vmi_migration_data_remaining_bytes", - Help: "Number of bytes that still need to be transferred", + Help: "The remaining guest OS data to be migrated to the new VM.", }, ) migrateVMIDataProcessed = operatormetrics.NewGauge( operatormetrics.MetricOpts{ Name: "kubevirt_vmi_migration_data_processed_bytes", - Help: "Number of bytes transferred from the beginning of the job.", + Help: "The total Guest OS data processed and migrated to the new VM.", }, ) migrateVmiDirtyMemoryRate = operatormetrics.NewGauge( operatormetrics.MetricOpts{ Name: "kubevirt_vmi_migration_dirty_memory_rate_bytes", - Help: "Number of memory pages dirtied by the guest per second.", + Help: "The rate of memory being dirty in the Guest OS.", }, ) migrateVmiMemoryTransferRate = operatormetrics.NewGauge( operatormetrics.MetricOpts{ Name: "kubevirt_vmi_migration_disk_transfer_rate_bytes", - Help: "Network throughput used while migrating memory in Bytes per second.", + Help: "The rate at which the memory is being transferred.", }, ) ) diff --git a/pkg/monitoring/metrics/virt-handler/metrics.go b/pkg/monitoring/metrics/virt-handler/metrics.go index 45055cd65a4f..102367fa62ad 100644 --- a/pkg/monitoring/metrics/virt-handler/metrics.go +++ b/pkg/monitoring/metrics/virt-handler/metrics.go @@ -27,7 +27,7 @@ import ( "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler/domainstats" ) -func SetupMetrics() error { +func SetupMetrics(virtShareDir, nodeName string, MaxRequestsInFlight int, vmiInformer cache.SharedIndexInformer) error { if err := workqueue.SetupMetrics(); err != nil { return err } @@ -41,6 +41,7 @@ func SetupMetrics() error { } SetVersionInfo() + domainstats.SetupDomainStatsCollector(virtShareDir, nodeName, MaxRequestsInFlight, vmiInformer) return operatormetrics.RegisterCollector(domainstats.Collector) } From 9a1a931c55d6636bf2ebf55ecc27b716844d07c7 Mon Sep 17 00:00:00 2001 From: machadovilaca Date: Mon, 25 Mar 2024 17:48:45 +0000 Subject: [PATCH 5/5] Update metrics documentation generator Signed-off-by: machadovilaca --- docs/metrics.md | 24 +- hack/generate.sh | 3 +- tools/doc-generator/BUILD.bazel | 14 +- tools/doc-generator/doc-generator.go | 272 +++--------------- tools/doc-generator/fakeDomainCollector.go | 91 ------ .../pkg/docs/BUILD.bazel | 17 ++ .../operator-observability/pkg/docs/alerts.go | 96 +++++++ .../pkg/docs/metrics.go | 111 +++++++ vendor/modules.txt | 1 + 9 files changed, 274 insertions(+), 355 deletions(-) delete mode 100644 tools/doc-generator/fakeDomainCollector.go create mode 100644 vendor/github.com/machadovilaca/operator-observability/pkg/docs/BUILD.bazel create mode 100644 vendor/github.com/machadovilaca/operator-observability/pkg/docs/alerts.go create mode 100644 vendor/github.com/machadovilaca/operator-observability/pkg/docs/metrics.go diff --git a/docs/metrics.md b/docs/metrics.md index 2f2bc8d4a023..adf0d566f7e9 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,16 +1,4 @@ - - # KubeVirt metrics -This document aims to help users that are not familiar with all metrics exposed by different KubeVirt components. -All metrics documented here are auto-generated by the utility tool `tools/doc-generator` and reflects exactly what is being exposed. - -## KubeVirt Metrics List -### kubevirt_info -Version information. ### kubevirt_allocatable_nodes The number of allocatable nodes in the cluster. Type: Gauge. @@ -24,6 +12,9 @@ Indicates whether the Software Emulation is enabled in the configuration. Type: ### kubevirt_console_active_connections Amount of active Console connections, broken down by namespace and vmi name. Type: Gauge. +### kubevirt_info +Version information. Type: Gauge. + ### kubevirt_memory_delta_from_requested_bytes The delta between the pod with highest memory working set or rss and its requested memory for each container, virt-controller, virt-handler, virt-api and virt-operator. Type: Gauge. @@ -154,7 +145,7 @@ The total amount of memory written out to swap space of the guest in bytes. Type The amount of memory left completely unused by the system. Memory that is available but used for reclaimable caches should NOT be reported as free. Type: Gauge. ### kubevirt_vmi_memory_usable_bytes -The amount of memory which can be reclaimed by balloon without pushing the guest system to swap, corresponds to 'Available' in /proc/meminfo Type: Gauge. +The amount of memory which can be reclaimed by balloon without pushing the guest system to swap, corresponds to 'Available' in /proc/meminfo. Type: Gauge. ### kubevirt_vmi_memory_used_bytes Amount of `used` memory as seen by the domain. Type: Gauge. @@ -202,7 +193,7 @@ The total number of rx packets dropped on vNIC interfaces. Type: Counter. Total network traffic received packets. Type: Counter. ### kubevirt_vmi_network_traffic_bytes_total -Deprecated. Type: Counter. +[Deprecated] Total number of bytes sent and received. Type: Counter. ### kubevirt_vmi_network_transmit_bytes_total Total network traffic transmitted in bytes. Type: Counter. @@ -283,6 +274,7 @@ Returns the labels of the persistent volume claims that are used for restoring v Amount of active VNC connections, broken down by namespace and vmi name. Type: Gauge. ## Developing new metrics -After developing new metrics or changing old ones, please run `make generate` to regenerate this document. -If you feel that the new metric doesn't follow these rules, please change `doc-generator` with your needs. +All metrics documented here are auto-generated and reflect exactly what is being +exposed. After developing new metrics or changing old ones please regenerate +this document. diff --git a/hack/generate.sh b/hack/generate.sh index 936312d7c68d..49eb7c12005d 100755 --- a/hack/generate.sh +++ b/hack/generate.sh @@ -139,8 +139,7 @@ ${KUBEVIRT_DIR}/tools/openapispec/openapispec --dump-api-spec-path ${KUBEVIRT_DI (cd ${KUBEVIRT_DIR}/tools/doc-generator/ && go_build) ( cd ${KUBEVIRT_DIR}/docs - ${KUBEVIRT_DIR}/tools/doc-generator/doc-generator - mv newmetrics.md metrics.md + ${KUBEVIRT_DIR}/tools/doc-generator/doc-generator >metrics.md ) rm -f ${KUBEVIRT_DIR}/manifests/generated/* diff --git a/tools/doc-generator/BUILD.bazel b/tools/doc-generator/BUILD.bazel index 2bc5961a7da1..7bf72f091258 100644 --- a/tools/doc-generator/BUILD.bazel +++ b/tools/doc-generator/BUILD.bazel @@ -2,27 +2,17 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", - srcs = [ - "doc-generator.go", - "fakeDomainCollector.go", - ], + srcs = ["doc-generator.go"], importpath = "kubevirt.io/kubevirt/tools/doc-generator", visibility = ["//visibility:private"], deps = [ - "//pkg/monitoring/domainstats/prometheus:go_default_library", "//pkg/monitoring/metrics/virt-api:go_default_library", "//pkg/monitoring/metrics/virt-controller:go_default_library", "//pkg/monitoring/metrics/virt-handler:go_default_library", "//pkg/monitoring/metrics/virt-operator:go_default_library", "//pkg/monitoring/rules:go_default_library", - "//pkg/virt-controller/watch:go_default_library", - "//pkg/virt-launcher/virtwrap/stats:go_default_library", - "//pkg/virt-launcher/virtwrap/statsconv:go_default_library", - "//pkg/virt-launcher/virtwrap/statsconv/util:go_default_library", - "//staging/src/kubevirt.io/api/core/v1:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/docs:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/libvirt.org/go/libvirt:go_default_library", ], ) diff --git a/tools/doc-generator/doc-generator.go b/tools/doc-generator/doc-generator.go index 4b95e6602d1b..ebd50ef6eb4b 100644 --- a/tools/doc-generator/doc-generator.go +++ b/tools/doc-generator/doc-generator.go @@ -1,265 +1,69 @@ package main import ( - "bufio" "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "sort" - "strings" "github.com/machadovilaca/operator-observability/pkg/operatormetrics" - domainstats "kubevirt.io/kubevirt/pkg/monitoring/domainstats/prometheus" // import for prometheus metrics - virt_api "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api" - virt_controller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller" - virt_handler "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler" - virt_operator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator" + virtapi "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api" + virtcontroller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller" + virthandler "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler" + virtoperator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator" "kubevirt.io/kubevirt/pkg/monitoring/rules" - _ "kubevirt.io/kubevirt/pkg/virt-controller/watch" -) - -// constant parts of the file -const ( - genFileComment = `` - title = "# KubeVirt metrics\n" - background = "This document aims to help users that are not familiar with all metrics exposed by different KubeVirt components.\n" + - "All metrics documented here are auto-generated by the utility tool `tools/doc-generator` and reflects exactly what is being exposed.\n\n" - - KVSpecificMetrics = "## KubeVirt Metrics List\n" + - "### kubevirt_info\n" + - "Version information.\n\n" - - opening = genFileComment + "\n\n" + - title + - background + - KVSpecificMetrics - // footer - footerHeading = "## Developing new metrics\n" - footerContent = "After developing new metrics or changing old ones, please run `make generate` to regenerate this document.\n\n" + - "If you feel that the new metric doesn't follow these rules, please change `doc-generator` with your needs.\n" - - footer = footerHeading + footerContent + "github.com/machadovilaca/operator-observability/pkg/docs" ) -func main() { - handler := domainstats.Handler(1) - RegisterFakeDomainCollector() - - req, err := http.NewRequest(http.MethodGet, "/metrics", nil) - checkError(err) - - recorder := httptest.NewRecorder() - - handler.ServeHTTP(recorder, req) - - metrics := getMetricsNotIncludeInEndpointByDefault() - - if status := recorder.Code; status == http.StatusOK { - err := parseVirtMetrics(recorder.Body, &metrics) - checkError(err) - - } else { - panic(fmt.Errorf("got HTTP status code of %d from /metrics", recorder.Code)) - } - writeToFile(metrics) -} - -func writeToFile(metrics metricList) { - newFile, err := os.Create("newmetrics.md") - checkError(err) - defer newFile.Close() +const tpl = `# KubeVirt metrics - fmt.Fprint(newFile, opening) - metrics.writeToFile(newFile) +{{- range . }} - fmt.Fprint(newFile, footer) +{{ $deprecatedVersion := "" -}} +{{- with index .ExtraFields "DeprecatedVersion" -}} + {{- $deprecatedVersion = printf " in %s" . -}} +{{- end -}} -} - -type metric struct { - name string - description string - mType string -} - -func (m metric) writeToFile(newFile io.WriteCloser) { - fmt.Fprintln(newFile, "###", m.name) - fmt.Fprintln(newFile, m.description, "Type:", m.mType+".") - fmt.Fprintln(newFile) -} - -type metricList []metric +{{- $stabilityLevel := "" -}} +{{- if and (.ExtraFields.StabilityLevel) (ne .ExtraFields.StabilityLevel "STABLE") -}} + {{- $stabilityLevel = printf "[%s%s] " .ExtraFields.StabilityLevel $deprecatedVersion -}} +{{- end -}} -// Len implements sort.Interface.Len -func (m metricList) Len() int { - return len(m) -} +### {{ .Name }} +{{ print $stabilityLevel }}{{ .Help }} Type: {{ .Type -}}. -// Less implements sort.Interface.Less -func (m metricList) Less(i, j int) bool { - return m[i].name < m[j].name -} - -// Swap implements sort.Interface.Swap -func (m metricList) Swap(i, j int) { - m[i], m[j] = m[j], m[i] -} - -func (m metricList) writeToFile(newFile io.WriteCloser) { - for _, met := range m { - met.writeToFile(newFile) - } -} - -func getMetricsNotIncludeInEndpointByDefault() metricList { - metrics := metricList{ - { - name: domainstats.MigrateVmiDataProcessedMetricName, - description: "The total Guest OS data processed and migrated to the new VM.", - mType: "Gauge", - }, - { - name: domainstats.MigrateVmiDataRemainingMetricName, - description: "The remaining guest OS data to be migrated to the new VM.", - mType: "Gauge", - }, - { - name: domainstats.MigrateVmiDirtyMemoryRateMetricName, - description: "The rate of memory being dirty in the Guest OS.", - mType: "Gauge", - }, - { - name: domainstats.MigrateVmiMemoryTransferRateMetricName, - description: "The rate at which the memory is being transferred.", - mType: "Gauge", - }, - { - name: "kubevirt_vmi_phase_count", - description: "Sum of VMIs per phase and node. `phase` can be one of the following: [`Pending`, `Scheduling`, `Scheduled`, `Running`, `Succeeded`, `Failed`, `Unknown`].", - mType: "Gauge", - }, - { - name: "kubevirt_vmi_non_evictable", - description: "Indication for a VirtualMachine that its eviction strategy is set to Live Migration but is not migratable.", - mType: "Gauge", - }, - } +{{- end }} - metrics = append(metrics, getVirtControllerMetrics()...) - metrics = append(metrics, getComponentMetrics(virt_api.SetupMetrics, virt_api.ListMetrics)...) - metrics = append(metrics, getComponentMetrics(virt_operator.SetupMetrics, virt_operator.ListMetrics)...) - metrics = append(metrics, getComponentMetrics(virt_handler.SetupMetrics, virt_handler.ListMetrics)...) +## Developing new metrics - err := rules.SetupRules("") - checkError(err) +All metrics documented here are auto-generated and reflect exactly what is being +exposed. After developing new metrics or changing old ones please regenerate +this document. +` - for _, rule := range rules.ListRecordingRules() { - metrics = append(metrics, metric{ - name: rule.GetOpts().Name, - description: rule.GetOpts().Help, - mType: string(rule.GetType()), - }) - } - - return metrics -} - -func getVirtControllerMetrics() metricList { - err := virt_controller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil) - checkError(err) - return listComponentMetrics(virt_controller.ListMetrics) -} - -func getComponentMetrics(setup func() error, list func() []operatormetrics.Metric) metricList { - err := setup() - checkError(err) - return listComponentMetrics(list) -} - -func listComponentMetrics(list func() []operatormetrics.Metric) metricList { - metrics := metricList{} - - for _, m := range list() { - metrics = append(metrics, newMetric(m)) - } - - err := operatormetrics.CleanRegistry() - checkError(err) - - return metrics -} - -func newMetric(om operatormetrics.Metric) metric { - return metric{ - name: om.GetOpts().Name, - description: om.GetOpts().Help, - mType: strings.Replace(string(om.GetType()), "Vec", "", 1), +func main() { + if err := virtcontroller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil); err != nil { + panic(err) } -} - -func parseMetricDesc(line string) (string, string) { - split := strings.Split(line, " ") - name := split[2] - split[3] = strings.Title(split[3]) - description := strings.Join(split[3:], " ") - return name, description -} -func parseMetricType(scan *bufio.Scanner, name string) string { - for scan.Scan() { - typeLine := scan.Text() - if strings.HasPrefix(typeLine, "# TYPE ") { - split := strings.Split(typeLine, " ") - if split[2] == name { - return strings.Title(split[3]) - } - } + if err := virtapi.SetupMetrics(); err != nil { + panic(err) } - return "" -} - -const filter = "kubevirt_" -func parseVirtMetrics(r io.Reader, metrics *metricList) error { - scan := bufio.NewScanner(r) - for scan.Scan() { - helpLine := scan.Text() - if strings.HasPrefix(helpLine, "# HELP ") { - if strings.Contains(helpLine, filter) { - metName, metDesc := parseMetricDesc(helpLine) - metType := parseMetricType(scan, metName) - *metrics = append(*metrics, metric{name: metName, description: metDesc, mType: metType}) - } - } + if err := virtoperator.SetupMetrics(); err != nil { + panic(err) } - if scan.Err() != nil { - return fmt.Errorf("failed to parse metrics from prometheus endpoint, %w", scan.Err()) + if err := virthandler.SetupMetrics("", "", 0, nil); err != nil { + panic(err) } - sort.Sort(metrics) - - // remove duplicates - for i := 0; i < len(*metrics)-1; i++ { - if (*metrics)[i].name == (*metrics)[i+1].name { - *metrics = append((*metrics)[:i], (*metrics)[i+1:]...) - i-- - } + if err := rules.SetupRules(""); err != nil { + panic(err) } - return nil -} + metricsList := operatormetrics.ListMetrics() + rulesList := rules.ListRecordingRules() -func checkError(err error) { - if err != nil { - panic(err) - } + docsString := docs.BuildMetricsDocsWithCustomTemplate(metricsList, rulesList, tpl) + fmt.Print(docsString) } diff --git a/tools/doc-generator/fakeDomainCollector.go b/tools/doc-generator/fakeDomainCollector.go deleted file mode 100644 index 391f13c95ac4..000000000000 --- a/tools/doc-generator/fakeDomainCollector.go +++ /dev/null @@ -1,91 +0,0 @@ -package main - -import ( - "github.com/prometheus/client_golang/prometheus" - "libvirt.org/go/libvirt" - - domainstats "kubevirt.io/kubevirt/pkg/monitoring/domainstats/prometheus" - - k6tv1 "kubevirt.io/api/core/v1" - - "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats" - "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/statsconv" - "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/statsconv/util" -) - -type fakeDomainCollector struct { -} - -func (fc fakeDomainCollector) Describe(_ chan<- *prometheus.Desc) { -} - -// Collect needs to report all metrics to see it in docs -func (fc fakeDomainCollector) Collect(ch chan<- prometheus.Metric) { - ps := domainstats.NewPrometheusScraper(ch) - - libstatst, err := util.LoadStats() - if err != nil { - panic(err) - } - - in := &libstatst[0] - inMem := []libvirt.DomainMemoryStat{} - inDomInfo := &libvirt.DomainInfo{} - jobInfo := stats.DomainJobInfo{} - out := stats.DomainStats{} - fs := k6tv1.VirtualMachineInstanceFileSystemList{} - ident := statsconv.DomainIdentifier(&fakeDomainIdentifier{}) - devAliasMap := make(map[string]string) - - if err = statsconv.Convert_libvirt_DomainStats_to_stats_DomainStats(ident, in, inMem, inDomInfo, devAliasMap, &jobInfo, &out); err != nil { - panic(err) - } - - out.Memory.ActualBalloonSet = true - out.Memory.UnusedSet = true - out.Memory.CachedSet = true - out.Memory.AvailableSet = true - out.Memory.RSSSet = true - out.Memory.SwapInSet = true - out.Memory.SwapOutSet = true - out.Memory.UsableSet = true - out.Memory.MinorFaultSet = true - out.Memory.MajorFaultSet = true - out.CPUMapSet = true - out.Cpu.SystemSet = true - out.Cpu.UserSet = true - out.Cpu.TimeSet = true - - fs.Items = []k6tv1.VirtualMachineInstanceFileSystem{ - { - DiskName: "disk1", - MountPoint: "/", - FileSystemType: "EXT4", - TotalBytes: 1000, - UsedBytes: 10, - }, - } - - vmi := k6tv1.VirtualMachineInstance{ - Status: k6tv1.VirtualMachineInstanceStatus{ - Phase: k6tv1.Running, - NodeName: "test", - }, - } - ps.Report("test", &vmi, &domainstats.VirtualMachineInstanceStats{DomainStats: &out, FsStats: fs}) -} - -type fakeDomainIdentifier struct { -} - -func (*fakeDomainIdentifier) GetName() (string, error) { - return "test", nil -} - -func (*fakeDomainIdentifier) GetUUIDString() (string, error) { - return "uuid", nil -} - -func RegisterFakeDomainCollector() { - prometheus.MustRegister(fakeDomainCollector{}) -} diff --git a/vendor/github.com/machadovilaca/operator-observability/pkg/docs/BUILD.bazel b/vendor/github.com/machadovilaca/operator-observability/pkg/docs/BUILD.bazel new file mode 100644 index 000000000000..a3465bcd41b4 --- /dev/null +++ b/vendor/github.com/machadovilaca/operator-observability/pkg/docs/BUILD.bazel @@ -0,0 +1,17 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "alerts.go", + "metrics.go", + ], + importmap = "kubevirt.io/kubevirt/vendor/github.com/machadovilaca/operator-observability/pkg/docs", + importpath = "github.com/machadovilaca/operator-observability/pkg/docs", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules:go_default_library", + "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library", + ], +) diff --git a/vendor/github.com/machadovilaca/operator-observability/pkg/docs/alerts.go b/vendor/github.com/machadovilaca/operator-observability/pkg/docs/alerts.go new file mode 100644 index 000000000000..e5d65f922532 --- /dev/null +++ b/vendor/github.com/machadovilaca/operator-observability/pkg/docs/alerts.go @@ -0,0 +1,96 @@ +package docs + +import ( + "bytes" + "log" + "sort" + "text/template" + + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" +) + +const defaultAlertsTemplate = `# Operator Alerts + +{{- range . }} + +### {{.Name}} +**Summary:** {{ index .Annotations "summary" }}. + +**Description:** {{ index .Annotations "description" }}. + +**Severity:** {{ index .Labels "severity" }}. +{{- if .For }} + +**For:** {{ .For }}. +{{- end -}} +{{- end }} + +## Developing new alerts + +All alerts documented here are auto-generated and reflect exactly what is being +exposed. After developing new alerts or changing old ones please regenerate +this document. +` + +type alertDocs struct { + Name string + Expr string + For string + Annotations map[string]string + Labels map[string]string +} + +// BuildAlertsDocsWithCustomTemplate returns a string with the documentation +// for the given alerts, using the given template. +func BuildAlertsDocsWithCustomTemplate( + alerts []promv1.Rule, + tplString string, +) string { + + tpl, err := template.New("alerts").Parse(tplString) + if err != nil { + log.Fatalln(err) + } + + var allDocs []alertDocs + + if alerts != nil { + allDocs = append(allDocs, buildAlertsDocs(alerts)...) + } + + buf := bytes.NewBufferString("") + err = tpl.Execute(buf, allDocs) + if err != nil { + log.Fatalln(err) + } + + return buf.String() +} + +// BuildAlertsDocs returns a string with the documentation for the given +// metrics. +func BuildAlertsDocs(alerts []promv1.Rule) string { + return BuildAlertsDocsWithCustomTemplate(alerts, defaultAlertsTemplate) +} + +func buildAlertsDocs(alerts []promv1.Rule) []alertDocs { + alertsDocs := make([]alertDocs, len(alerts)) + for i, alert := range alerts { + alertsDocs[i] = alertDocs{ + Name: alert.Alert, + Expr: alert.Expr.String(), + For: string(*alert.For), + Annotations: alert.Annotations, + Labels: alert.Labels, + } + } + sortAlertsDocs(alertsDocs) + + return alertsDocs +} + +func sortAlertsDocs(alertsDocs []alertDocs) { + sort.Slice(alertsDocs, func(i, j int) bool { + return alertsDocs[i].Name < alertsDocs[j].Name + }) +} diff --git a/vendor/github.com/machadovilaca/operator-observability/pkg/docs/metrics.go b/vendor/github.com/machadovilaca/operator-observability/pkg/docs/metrics.go new file mode 100644 index 000000000000..c61db1346467 --- /dev/null +++ b/vendor/github.com/machadovilaca/operator-observability/pkg/docs/metrics.go @@ -0,0 +1,111 @@ +package docs + +import ( + "bytes" + "log" + "sort" + "strings" + "text/template" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/machadovilaca/operator-observability/pkg/operatorrules" +) + +const defaultMetricsTemplate = `# Operator Metrics + +{{- range . }} + +### {{.Name}} +{{.Help}}. + +Type: {{.Type}}. +{{- end }} + +## Developing new metrics + +All metrics documented here are auto-generated and reflect exactly what is being +exposed. After developing new metrics or changing old ones please regenerate +this document. +` + +type metricDocs struct { + Name string + Help string + Type string + ExtraFields map[string]string +} + +type docOptions interface { + GetOpts() operatormetrics.MetricOpts + GetType() operatormetrics.MetricType +} + +// BuildMetricsDocsWithCustomTemplate returns a string with the documentation +// for the given metrics, using the given template. +func BuildMetricsDocsWithCustomTemplate( + metrics []operatormetrics.Metric, + recordingRules []operatorrules.RecordingRule, + tplString string, +) string { + + tpl, err := template.New("metrics").Parse(tplString) + if err != nil { + log.Fatalln(err) + } + + var allDocs []metricDocs + + if metrics != nil { + allDocs = append(allDocs, buildMetricsDocs(metrics)...) + } + + if recordingRules != nil { + allDocs = append(allDocs, buildMetricsDocs(recordingRules)...) + } + + sortMetricsDocs(allDocs) + + buf := bytes.NewBufferString("") + err = tpl.Execute(buf, allDocs) + if err != nil { + log.Fatalln(err) + } + + return buf.String() +} + +// BuildMetricsDocs returns a string with the documentation for the given +// metrics. +func BuildMetricsDocs(metrics []operatormetrics.Metric, recordingRules []operatorrules.RecordingRule) string { + return BuildMetricsDocsWithCustomTemplate(metrics, recordingRules, defaultMetricsTemplate) +} + +func buildMetricsDocs[T docOptions](items []T) []metricDocs { + uniqueNames := make(map[string]struct{}) + var metricsDocs []metricDocs + + for _, metric := range items { + metricOpts := metric.GetOpts() + if _, exists := uniqueNames[metricOpts.Name]; !exists { + uniqueNames[metricOpts.Name] = struct{}{} + metricsDocs = append(metricsDocs, metricDocs{ + Name: metricOpts.Name, + Help: metricOpts.Help, + Type: getAndConvertMetricType(metric.GetType()), + ExtraFields: metricOpts.ExtraFields, + }) + } + } + + return metricsDocs +} + +func sortMetricsDocs(metricsDocs []metricDocs) { + sort.Slice(metricsDocs, func(i, j int) bool { + return metricsDocs[i].Name < metricsDocs[j].Name + }) +} + +func getAndConvertMetricType(metricType operatormetrics.MetricType) string { + return strings.ReplaceAll(string(metricType), "Vec", "") +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d7abaf723934..369b2b3b34f9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -243,6 +243,7 @@ github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1 github.com/kubevirt/monitoring/pkg/metrics/parser # github.com/machadovilaca/operator-observability v0.0.20 ## explicit; go 1.21 +github.com/machadovilaca/operator-observability/pkg/docs github.com/machadovilaca/operator-observability/pkg/operatormetrics github.com/machadovilaca/operator-observability/pkg/operatorrules # github.com/mailru/easyjson v0.7.7