[release-1.2] Add rule to collect issues with the pods

docs/metrics.md

@@ -24,6 +24,9 @@ Indicates whether the Software Emulation is enabled in the configuration. Type:
 ### kubevirt_console_active_connections
 Amount of active Console connections, broken down by namespace and vmi name. Type: Gauge.
 
+### kubevirt_memory_delta_from_requested_bytes
+The delta between the pod with highest memory working set or rss and its requested memory for each container, virt-controller, virt-handler, virt-api and virt-operator. Type: Gauge.
+
 ### kubevirt_nodes_with_kvm
 The number of nodes in the cluster that have the devices.kubevirt.io/kvm resource available. Type: Gauge.
 

pkg/monitoring/rules/recordingrules/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "api.go",
         "nodes.go",
+        "operator.go",
         "recordingrules.go",
         "virt.go",
         "vm.go",

pkg/monitoring/rules/recordingrules/operator.go

@@ -0,0 +1,45 @@
+/*
+Copyright The KubeVirt Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package recordingrules
+
+import (
+	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+	"github.com/machadovilaca/operator-observability/pkg/operatorrules"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+var operatorRecordingRules = []operatorrules.RecordingRule{
+	{
+		MetricsOpts: operatormetrics.MetricOpts{
+			Name: "kubevirt_memory_delta_from_requested_bytes",
+			Help: "The delta between the pod with highest memory working set or rss and its requested memory for each container, virt-controller, virt-handler, virt-api and virt-operator.",
+			ConstLabels: map[string]string{
+				"reason": "memory_working_set",
+			},
+		},
+		MetricType: operatormetrics.GaugeType,
+		Expr:       intstr.FromString("topk by(container)(1,max by(container, namespace, node)(container_memory_working_set_bytes{container=~\"virt-controller|virt-api|virt-handler|virt-operator\"}  - on(pod) group_left(node) (kube_pod_container_resource_requests{ container=~\"virt-controller|virt-api|virt-handler|virt-operator\",resource=\"memory\"})))"),
+	},
+	{
+		MetricsOpts: operatormetrics.MetricOpts{
+			Name: "kubevirt_memory_delta_from_requested_bytes",
+			Help: "The delta between the pod with highest memory working set or rss and its requested memory for each container, virt-controller, virt-handler, virt-api and virt-operator.",
+			ConstLabels: map[string]string{
+				"reason": "memory_rss",
+			},
+		},
+		MetricType: operatormetrics.GaugeType,
+		Expr:       intstr.FromString("topk by(container)(1,max by(container, namespace, node)(container_memory_rss{container=~\"virt-controller|virt-api|virt-handler|virt-operator\"}  - on(pod) group_left(node) (kube_pod_container_resource_requests{ container=~\"virt-controller|virt-api|virt-handler|virt-operator\",resource=\"memory\"})))"),
+	},
+}

pkg/monitoring/rules/recordingrules/recordingrules.go

@@ -6,6 +6,7 @@ func Register(namespace string) error {
 	return operatorrules.RegisterRecordingRules(
 		apiRecordingRules,
 		nodesRecordingRules,
+		operatorRecordingRules,
 		virtRecordingRules(namespace),
 		vmRecordingRules,
 		vmiRecordingRules,