Skip to content

Commit

Permalink
Add memory metrics
Browse files Browse the repository at this point in the history
Signed-off-by: alitman <alitman@alitman-thinkpadp1gen4i.tlv.csb>
  • Loading branch information
avlitman committed Mar 21, 2024
1 parent 9c3a52d commit 429fb25
Show file tree
Hide file tree
Showing 9 changed files with 77 additions and 4 deletions.
9 changes: 9 additions & 0 deletions docs/metrics.md
Expand Up @@ -12,6 +12,9 @@ All metrics documented here are auto-generated by the utility tool `tools/doc-ge
### kubevirt_info
Version information.

### cnv_abnormal
This rule holds issues with the pods for each container, e.g. memory exceeded. Type: Gauge.

### kubevirt_allocatable_nodes
The number of allocatable nodes in the cluster. Type: Gauge.

Expand All @@ -33,6 +36,9 @@ The number of VMs in the cluster by namespace. Type: Gauge.
### kubevirt_portforward_active_tunnels
Amount of active portforward tunnels, broken down by namespace and vmi name. Type: Gauge.

### kubevirt_rss_memory_exceeded
The pod with the highest exceeded memory for each container based on the rss. Type: Gauge.

### kubevirt_usbredir_active_connections
Amount of active USB redirection connections, broken down by namespace and vmi name. Type: Gauge.

Expand Down Expand Up @@ -267,6 +273,9 @@ Returns the labels of the persistent volume claims that are used for restoring v
### kubevirt_vnc_active_connections
Amount of active VNC connections, broken down by namespace and vmi name. Type: Gauge.

### kubevirt_working_set_memory_exceeded
The pod with the highest exceeded memory for each container based on the working set. Type: Gauge.

## Developing new metrics
After developing new metrics or changing old ones, please run `make generate` to regenerate this document.

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Expand Up @@ -32,7 +32,7 @@ require (
github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a
github.com/machadovilaca/operator-observability v0.0.14
github.com/machadovilaca/operator-observability v0.0.17
github.com/mdlayher/vsock v1.2.1
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Expand Up @@ -581,8 +581,8 @@ github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyP
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a h1:cdX+oxWw1lJDS3EchP+7Oz1XbErk4r7ffVJu1b1MKgI=
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a/go.mod h1:qGj2agzgwQ27nYhP3xhLs+IBzE5+ALNUg8bDfMcwPqo=
github.com/machadovilaca/operator-observability v0.0.14 h1:tS/GKvQRKvpD7pRauS1ulw0AN2V0j2mobg+mFWBt5LE=
github.com/machadovilaca/operator-observability v0.0.14/go.mod h1:e4Z3VhOXb9InkmSh00JjqBBijE+iD+YMzynBpKB3+gE=
github.com/machadovilaca/operator-observability v0.0.17 h1:XHL+YHkCv5WN5MKMR+/ld5pOnRnLVzxVX2b5OvSfhzs=
github.com/machadovilaca/operator-observability v0.0.17/go.mod h1:e4Z3VhOXb9InkmSh00JjqBBijE+iD+YMzynBpKB3+gE=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
Expand Down
1 change: 1 addition & 0 deletions pkg/monitoring/rules/recordingrules/BUILD.bazel
Expand Up @@ -5,6 +5,7 @@ go_library(
srcs = [
"api.go",
"nodes.go",
"operator.go",
"recordingrules.go",
"virt.go",
"vm.go",
Expand Down
60 changes: 60 additions & 0 deletions pkg/monitoring/rules/recordingrules/operator.go
@@ -0,0 +1,60 @@
/*
Copyright 2024 The KubeVirt Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package recordingrules

import (
"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
"github.com/machadovilaca/operator-observability/pkg/operatorrules"
"k8s.io/apimachinery/pkg/util/intstr"
)

const (
abnormalLabelKey = "reason"
)

var operatorRecordingRules = []operatorrules.RecordingRule{
{
MetricsOpts: operatormetrics.MetricOpts{
Name: "kubevirt_working_set_memory_exceeded",
Help: "The pod with the highest exceeded memory for each container based on the working set.",
ConstLabels: map[string]string{
abnormalLabelKey: "WorkingSetMemoryExceeded",
},
},
MetricType: operatormetrics.GaugeType,
Expr: intstr.FromString("max by(container, namespace)(container_memory_working_set_bytes{container=~\"virt-controller|virt-api|virt-handler|virt-operator\"} - on(pod) group_left(node) (kube_pod_container_resource_requests{container=~\"virt-controller|virt-api|virt-handler|virt-operator\",resource=\"memory\"}))"),
},
{
MetricsOpts: operatormetrics.MetricOpts{
Name: "kubevirt_rss_memory_exceeded",
Help: "The pod with the highest exceeded memory for each container based on the rss.",
ConstLabels: map[string]string{
abnormalLabelKey: "RSSMemoryExceeded",
},
},
MetricType: operatormetrics.GaugeType,
Expr: intstr.FromString("max by(container, namespace)(container_memory_rss{container=~\"virt-controller|virt-api|virt-handler|virt-operator\"} - on(pod) group_left(node) (kube_pod_container_resource_requests{container=~\"virt-controller|virt-api|virt-handler|virt-operator\",resource=\"memory\"}))"),
},
{
MetricsOpts: operatormetrics.MetricOpts{
Name: "cnv_abnormal",
Help: "This rule holds issues with the pods for each container, e.g. memory exceeded.",
},
MetricType: operatormetrics.GaugeType,
Expr: intstr.FromString("kubevirt_working_set_memory_exceeded or kubevirt_rss_memory_exceeded"),
},
}
1 change: 1 addition & 0 deletions pkg/monitoring/rules/recordingrules/recordingrules.go
Expand Up @@ -6,6 +6,7 @@ func Register(namespace string) error {
return operatorrules.RegisterRecordingRules(
apiRecordingRules,
nodesRecordingRules,
operatorRecordingRules,
virtRecordingRules(namespace),
vmRecordingRules,
vmiRecordingRules,
Expand Down
1 change: 1 addition & 0 deletions tools/prom-metrics-collector/metrics_collector.go
Expand Up @@ -34,6 +34,7 @@ import (
// should be ignored.
var excludedMetrics = map[string]struct{}{
"kubevirt_vmi_phase_count": struct{}{},
"cnv_abnormal": struct{}{},
}

// Extract the name, help, and type from the metrics doc file
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion vendor/modules.txt
Expand Up @@ -241,7 +241,7 @@ github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1
# github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a
## explicit; go 1.20
github.com/kubevirt/monitoring/pkg/metrics/parser
# github.com/machadovilaca/operator-observability v0.0.14
# github.com/machadovilaca/operator-observability v0.0.17
## explicit; go 1.21
github.com/machadovilaca/operator-observability/pkg/operatormetrics
github.com/machadovilaca/operator-observability/pkg/operatorrules
Expand Down

0 comments on commit 429fb25

Please sign in to comment.