diff --git a/docs/metrics.md b/docs/metrics.md index 722bbbb02908..364c0aebc380 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -24,6 +24,9 @@ Indicates whether the Software Emulation is enabled in the configuration. Type: ### kubevirt_console_active_connections Amount of active Console connections, broken down by namespace and vmi name. Type: Gauge. +### kubevirt_memory_delta_from_requested_bytes +The delta between the pod with highest memory working set or rss and its requested memory for each container, virt-controller, virt-handler, virt-api and virt-operator. Type: Gauge. + ### kubevirt_nodes_with_kvm The number of nodes in the cluster that have the devices.kubevirt.io/kvm resource available. Type: Gauge. diff --git a/go.mod b/go.mod index 5b3831af33f3..9a5efb8367da 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a - github.com/machadovilaca/operator-observability v0.0.14 + github.com/machadovilaca/operator-observability v0.0.19 github.com/mdlayher/vsock v1.2.1 github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed diff --git a/go.sum b/go.sum index 7d61ac04e817..9a04db62fc9d 100644 --- a/go.sum +++ b/go.sum @@ -581,8 +581,8 @@ github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyP github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a h1:cdX+oxWw1lJDS3EchP+7Oz1XbErk4r7ffVJu1b1MKgI= github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a/go.mod h1:qGj2agzgwQ27nYhP3xhLs+IBzE5+ALNUg8bDfMcwPqo= -github.com/machadovilaca/operator-observability v0.0.14 h1:tS/GKvQRKvpD7pRauS1ulw0AN2V0j2mobg+mFWBt5LE= -github.com/machadovilaca/operator-observability v0.0.14/go.mod h1:e4Z3VhOXb9InkmSh00JjqBBijE+iD+YMzynBpKB3+gE= +github.com/machadovilaca/operator-observability v0.0.19 h1:v1rIrdaZ65iaUvKyBd2g29Aklwr0X9IsYukXTxTW6lo= +github.com/machadovilaca/operator-observability v0.0.19/go.mod h1:e4Z3VhOXb9InkmSh00JjqBBijE+iD+YMzynBpKB3+gE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= diff --git a/pkg/monitoring/rules/recordingrules/BUILD.bazel b/pkg/monitoring/rules/recordingrules/BUILD.bazel index c043ba966ebe..67aaad1cecbb 100644 --- a/pkg/monitoring/rules/recordingrules/BUILD.bazel +++ b/pkg/monitoring/rules/recordingrules/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "api.go", "nodes.go", + "operator.go", "recordingrules.go", "virt.go", "vm.go", diff --git a/pkg/monitoring/rules/recordingrules/operator.go b/pkg/monitoring/rules/recordingrules/operator.go new file mode 100644 index 000000000000..2bfe2c7bcb39 --- /dev/null +++ b/pkg/monitoring/rules/recordingrules/operator.go @@ -0,0 +1,48 @@ +/* +Copyright The KubeVirt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recordingrules + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/machadovilaca/operator-observability/pkg/operatorrules" + "k8s.io/apimachinery/pkg/util/intstr" +) + +var operatorRecordingRules = []operatorrules.RecordingRule{ + { + MetricsOpts: operatormetrics.MetricOpts{ + Name: "kubevirt_memory_delta_from_requested_bytes", + Help: "The delta between the pod with highest memory working set or rss and its requested memory for each container, virt-controller, virt-handler, virt-api and virt-operator.", + ConstLabels: map[string]string{ + "reason": "memory_working_set", + }, + }, + MetricType: operatormetrics.GaugeType, + Expr: intstr.FromString("topk by(container)(1,max by(container, namespace, node)(container_memory_working_set_bytes{container=~\"virt-controller|virt-api|virt-handler|virt-operator\"} - on(pod) group_left(node) (kube_pod_container_resource_requests{ container=~\"virt-controller|virt-api|virt-handler|virt-operator\",resource=\"memory\"})))"), + }, + { + MetricsOpts: operatormetrics.MetricOpts{ + Name: "kubevirt_memory_delta_from_requested_bytes", + Help: "The delta between the pod with highest memory working set or rss and its requested memory for each container, virt-controller, virt-handler, virt-api and virt-operator.", + ConstLabels: map[string]string{ + "reason": "memory_rss", + }, + }, + MetricType: operatormetrics.GaugeType, + Expr: intstr.FromString("topk by(container)(1,max by(container, namespace, node)(container_memory_rss{container=~\"virt-controller|virt-api|virt-handler|virt-operator\"} - on(pod) group_left(node) (kube_pod_container_resource_requests{ container=~\"virt-controller|virt-api|virt-handler|virt-operator\",resource=\"memory\"})))"), + }, +} diff --git a/pkg/monitoring/rules/recordingrules/recordingrules.go b/pkg/monitoring/rules/recordingrules/recordingrules.go index c3f65e2be1e2..a1b8f4ff3001 100644 --- a/pkg/monitoring/rules/recordingrules/recordingrules.go +++ b/pkg/monitoring/rules/recordingrules/recordingrules.go @@ -6,6 +6,7 @@ func Register(namespace string) error { return operatorrules.RegisterRecordingRules( apiRecordingRules, nodesRecordingRules, + operatorRecordingRules, virtRecordingRules(namespace), vmRecordingRules, vmiRecordingRules, diff --git a/vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics/collector.go b/vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics/collector.go index bcf0e5ee46d6..35015f51a0d3 100644 --- a/vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics/collector.go +++ b/vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics/collector.go @@ -20,9 +20,10 @@ type Collector struct { } type CollectorResult struct { - Metric Metric - Labels []string - Value float64 + Metric Metric + Labels []string + ConstLabels map[string]string + Value float64 } func (c Collector) hash() string { @@ -73,11 +74,19 @@ func collectValue(ch chan<- prometheus.Metric, metric Metric, cr CollectorResult return fmt.Errorf("encountered unsupported type for collector %v", metric.GetType()) } + labels := map[string]string{} + for k, v := range cr.ConstLabels { + labels[k] = v + } + for k, v := range metric.GetOpts().ConstLabels { + labels[k] = v + } + desc := prometheus.NewDesc( metric.GetOpts().Name, metric.GetOpts().Help, metric.GetOpts().labels, - metric.GetOpts().ConstLabels, + labels, ) cm, err := prometheus.NewConstMetric(desc, mType, cr.Value, cr.Labels...) diff --git a/vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules/prometheusrules.go b/vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules/prometheusrules.go index 97b55d07793f..9f54c7b96b58 100644 --- a/vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules/prometheusrules.go +++ b/vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules/prometheusrules.go @@ -62,6 +62,7 @@ func buildRecordingRulesRules() []promv1.Rule { rules = append(rules, promv1.Rule{ Record: recordingRule.MetricsOpts.Name, Expr: recordingRule.Expr, + Labels: recordingRule.MetricsOpts.ConstLabels, }) } diff --git a/vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules/registry.go b/vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules/registry.go index cc72af21d215..fdef00604f2a 100644 --- a/vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules/registry.go +++ b/vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules/registry.go @@ -25,7 +25,8 @@ func newRegistry() operatorRegisterer { func RegisterRecordingRules(recordingRules ...[]RecordingRule) error { for _, recordingRuleList := range recordingRules { for _, recordingRule := range recordingRuleList { - operatorRegistry.registeredRecordingRules[recordingRule.MetricsOpts.Name] = recordingRule + key := recordingRule.MetricsOpts.Name + ":" + recordingRule.Expr.String() + operatorRegistry.registeredRecordingRules[key] = recordingRule } } @@ -51,7 +52,10 @@ func ListRecordingRules() []RecordingRule { } slices.SortFunc(rules, func(a, b RecordingRule) int { - return cmp.Compare(a.GetOpts().Name, b.GetOpts().Name) + aKey := a.GetOpts().Name + ":" + a.Expr.String() + bKey := b.GetOpts().Name + ":" + b.Expr.String() + + return cmp.Compare(aKey, bKey) }) return rules diff --git a/vendor/modules.txt b/vendor/modules.txt index 8fe830361e97..efa69b9afbe7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -241,7 +241,7 @@ github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1 # github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a ## explicit; go 1.20 github.com/kubevirt/monitoring/pkg/metrics/parser -# github.com/machadovilaca/operator-observability v0.0.14 +# github.com/machadovilaca/operator-observability v0.0.19 ## explicit; go 1.21 github.com/machadovilaca/operator-observability/pkg/operatormetrics github.com/machadovilaca/operator-observability/pkg/operatorrules