Skip to content

Commit

Permalink
Merge pull request #11140 from assafad/remove-lambda
Browse files Browse the repository at this point in the history 
tests: Create libmonitoring library
  • Loading branch information
@kubevirt-bot
kubevirt-bot committed Feb 19, 2024
2 parents 1e75a49 + b92c6b9 commit 34d4b39
Show file tree
Hide file tree
Showing 12 changed files with 133 additions and 108 deletions.
1 change: 1 addition & 0 deletions tests/infrastructure/BUILD.bazel
View file
Expand Up @@ -37,6 +37,7 @@ go_library(
"//tests/framework/kubevirt:go_default_library",
"//tests/framework/matcher:go_default_library",
"//tests/libinfra:go_default_library",
"//tests/libmonitoring:go_default_library",
"//tests/libnet:go_default_library",
"//tests/libnode:go_default_library",
"//tests/libreplicaset:go_default_library",
Expand Down
19 changes: 8 additions & 11 deletions tests/infrastructure/prometheus.go
View file
Expand Up @@ -58,6 +58,7 @@ import (
"kubevirt.io/kubevirt/tests"
"kubevirt.io/kubevirt/tests/console"
"kubevirt.io/kubevirt/tests/flags"
"kubevirt.io/kubevirt/tests/libmonitoring"
"kubevirt.io/kubevirt/tests/libnet"
)

Expand Down Expand Up @@ -193,11 +194,10 @@ var _ = DescribeInfra("[rfe_id:3187][crit:medium][vendor:cnv-qe@redhat.com][leve
virtClient kubecli.KubevirtClient
err error

preparedVMIs []*v1.VirtualMachineInstance
pod *k8sv1.Pod
handlerMetricIPs []string
controllerMetricIPs []string
getKubevirtVMMetrics func(string) string
preparedVMIs []*v1.VirtualMachineInstance
pod *k8sv1.Pod
handlerMetricIPs []string
controllerMetricIPs []string
)

// collect metrics whose key contains the given string, expects non-empty result
Expand All @@ -207,7 +207,7 @@ var _ = DescribeInfra("[rfe_id:3187][crit:medium][vendor:cnv-qe@redhat.com][leve
var lines []string

Eventually(func() map[string]float64 {
out := getKubevirtVMMetrics(ip)
out := libmonitoring.GetKubevirtVMMetrics(pod, ip)
lines = libinfra.TakeMetricsWithPrefix(out, metricSubstring)
metrics, err = libinfra.ParseMetricsToMap(lines)
Expect(err).ToNot(HaveOccurred())
Expand Down Expand Up @@ -290,9 +290,6 @@ var _ = DescribeInfra("[rfe_id:3187][crit:medium][vendor:cnv-qe@redhat.com][leve
for _, ip := range pod.Status.PodIPs {
handlerMetricIPs = append(handlerMetricIPs, ip.IP)
}

// returns metrics from the node the VMI(s) runs on
getKubevirtVMMetrics = tests.GetKubevirtVMMetricsFunc(&virtClient, pod)
})

It("[test_id:4136] should find one leading virt-controller and two ready", func() {
Expand Down Expand Up @@ -373,7 +370,7 @@ var _ = DescribeInfra("[rfe_id:3187][crit:medium][vendor:cnv-qe@redhat.com][leve

errorsChan := make(chan error)
By("Scraping the Prometheus endpoint")
metricsURL := tests.PrepareMetricsURL(ip, 8443)
metricsURL := libmonitoring.PrepareMetricsURL(ip, 8443)
for ix := 0; ix < concurrency; ix++ {
go func(ix int) {
req, _ := http.NewRequest("GET", metricsURL, nil)
Expand Down Expand Up @@ -402,7 +399,7 @@ var _ = DescribeInfra("[rfe_id:3187][crit:medium][vendor:cnv-qe@redhat.com][leve

By("Scraping the Prometheus endpoint")
Eventually(func() string {
out := getKubevirtVMMetrics(ip)
out := libmonitoring.GetKubevirtVMMetrics(pod, ip)
lines := libinfra.TakeMetricsWithPrefix(out, "kubevirt")
return strings.Join(lines, "\n")
}, 30*time.Second, 2*time.Second).Should(ContainSubstring("kubevirt"))
Expand Down
1 change: 1 addition & 0 deletions tests/libmigration/BUILD.bazel
View file
Expand Up @@ -14,6 +14,7 @@ go_library(
"//tests/framework/kubevirt:go_default_library",
"//tests/framework/matcher:go_default_library",
"//tests/libinfra:go_default_library",
"//tests/libmonitoring:go_default_library",
"//tests/libnode:go_default_library",
"//tests/util:go_default_library",
"//vendor/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1:go_default_library",
Expand Down
4 changes: 2 additions & 2 deletions tests/libmigration/migration.go
View file
Expand Up @@ -23,6 +23,7 @@ import (

"kubevirt.io/kubevirt/tests/flags"
"kubevirt.io/kubevirt/tests/framework/matcher"
"kubevirt.io/kubevirt/tests/libmonitoring"
"kubevirt.io/kubevirt/tests/util"

. "github.com/onsi/ginkgo/v2"
Expand Down Expand Up @@ -544,9 +545,8 @@ func RunMigrationAndCollectMigrationMetrics(vmi *v1.VirtualMachineInstance, migr
return nil
}

getKubevirtVMMetricsFunc := tests.GetKubevirtVMMetricsFunc(&virtClient, pod)
Eventually(func() error {
out := getKubevirtVMMetricsFunc(ip)
out := libmonitoring.GetKubevirtVMMetrics(pod, ip)
for _, metricName := range migrationMetrics {
lines := libinfra.TakeMetricsWithPrefix(out, metricName)
metrics, err := libinfra.ParseMetricsToMap(lines)
Expand Down
27 changes: 27 additions & 0 deletions tests/libmonitoring/BUILD.bazel
View file
@@ -0,0 +1,27 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = [
"prometheus.go",
"scaling.go",
],
importpath = "kubevirt.io/kubevirt/tests/libmonitoring",
visibility = ["//visibility:public"],
deps = [
"//staging/src/kubevirt.io/client-go/kubecli:go_default_library",
"//tests/clientcmd:go_default_library",
"//tests/exec:go_default_library",
"//tests/flags:go_default_library",
"//tests/framework/checks:go_default_library",
"//tests/framework/kubevirt:go_default_library",
"//vendor/github.com/onsi/ginkgo/v2:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library",
"//vendor/github.com/prometheus/client_golang/api/prometheus/v1:go_default_library",
"//vendor/k8s.io/api/autoscaling/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
],
)
58 changes: 40 additions & 18 deletions tests/monitoring/prometheus_utils.go → tests/libmonitoring/prometheus.go
View file
@@ -1,4 +1,4 @@
package monitoring
package libmonitoring

import (
"context"
Expand All @@ -7,6 +7,7 @@ import (
"fmt"
"io"
"math/rand"
"net"
"net/http"
"os/exec"
"regexp"
Expand All @@ -26,8 +27,10 @@ import (
"kubevirt.io/client-go/kubecli"

"kubevirt.io/kubevirt/tests/clientcmd"
execute "kubevirt.io/kubevirt/tests/exec"
"kubevirt.io/kubevirt/tests/flags"
"kubevirt.io/kubevirt/tests/framework/checks"
"kubevirt.io/kubevirt/tests/framework/kubevirt"
)

type AlertRequestResult struct {
Expand All @@ -50,7 +53,7 @@ type promResult struct {
Value []interface{} `json:"value"`
}

func getAlerts(cli kubecli.KubevirtClient) ([]prometheusv1.Alert, error) {
func GetAlerts(cli kubecli.KubevirtClient) ([]prometheusv1.Alert, error) {
bodyBytes := DoPrometheusHTTPRequest(cli, "/alerts")

var result AlertRequestResult
Expand All @@ -66,21 +69,21 @@ func getAlerts(cli kubecli.KubevirtClient) ([]prometheusv1.Alert, error) {
return result.Alerts.Alerts, nil
}

func waitForMetricValue(client kubecli.KubevirtClient, metric string, expectedValue float64) {
waitForMetricValueWithLabels(client, metric, expectedValue, nil, 2)
func WaitForMetricValue(client kubecli.KubevirtClient, metric string, expectedValue float64) {
WaitForMetricValueWithLabels(client, metric, expectedValue, nil, 2)
}

func waitForMetricValueWithLabels(client kubecli.KubevirtClient, metric string, expectedValue float64, labels map[string]string, offset int) {
func WaitForMetricValueWithLabels(client kubecli.KubevirtClient, metric string, expectedValue float64, labels map[string]string, offset int) {
EventuallyWithOffset(offset, func() float64 {
i, err := getMetricValueWithLabels(client, metric, labels)
i, err := GetMetricValueWithLabels(client, metric, labels)
if err != nil {
return -1
}
return i
}, 3*time.Minute, 1*time.Second).Should(BeNumerically("==", expectedValue))
}

func getMetricValueWithLabels(cli kubecli.KubevirtClient, query string, labels map[string]string) (float64, error) {
func GetMetricValueWithLabels(cli kubecli.KubevirtClient, query string, labels map[string]string) (float64, error) {
result, err := fetchMetric(cli, query)
if err != nil {
return -1, err
Expand Down Expand Up @@ -288,8 +291,8 @@ func KillPortForwardCommand(portForwardCmd *exec.Cmd) error {
return err
}

func checkAlertExists(virtClient kubecli.KubevirtClient, alertName string) bool {
currentAlerts, err := getAlerts(virtClient)
func CheckAlertExists(virtClient kubecli.KubevirtClient, alertName string) bool {
currentAlerts, err := GetAlerts(virtClient)
if err != nil {
return false
}
Expand All @@ -301,20 +304,20 @@ func checkAlertExists(virtClient kubecli.KubevirtClient, alertName string) bool
return false
}

func verifyAlertExistWithCustomTime(virtClient kubecli.KubevirtClient, alertName string, timeout time.Duration) {
func VerifyAlertExistWithCustomTime(virtClient kubecli.KubevirtClient, alertName string, timeout time.Duration) {
EventuallyWithOffset(1, func() bool {
return checkAlertExists(virtClient, alertName)
return CheckAlertExists(virtClient, alertName)
}, timeout, 10*time.Second).Should(BeTrue(), "Alert %s should exist", alertName)
}

func verifyAlertExist(virtClient kubecli.KubevirtClient, alertName string) {
verifyAlertExistWithCustomTime(virtClient, alertName, 120*time.Second)
func VerifyAlertExist(virtClient kubecli.KubevirtClient, alertName string) {
VerifyAlertExistWithCustomTime(virtClient, alertName, 120*time.Second)
}

func waitUntilAlertDoesNotExistWithCustomTime(virtClient kubecli.KubevirtClient, timeout time.Duration, alertNames ...string) {
func WaitUntilAlertDoesNotExistWithCustomTime(virtClient kubecli.KubevirtClient, timeout time.Duration, alertNames ...string) {
presentAlert := EventuallyWithOffset(1, func() string {
for _, name := range alertNames {
if checkAlertExists(virtClient, name) {
if CheckAlertExists(virtClient, name) {
return name
}
}
Expand All @@ -324,11 +327,11 @@ func waitUntilAlertDoesNotExistWithCustomTime(virtClient kubecli.KubevirtClient,
presentAlert.Should(BeEmpty(), "Alert %v should not exist", presentAlert)
}

func waitUntilAlertDoesNotExist(virtClient kubecli.KubevirtClient, alertNames ...string) {
waitUntilAlertDoesNotExistWithCustomTime(virtClient, 5*time.Minute, alertNames...)
func WaitUntilAlertDoesNotExist(virtClient kubecli.KubevirtClient, alertNames ...string) {
WaitUntilAlertDoesNotExistWithCustomTime(virtClient, 5*time.Minute, alertNames...)
}

func reduceAlertPendingTime(virtClient kubecli.KubevirtClient) {
func ReduceAlertPendingTime(virtClient kubecli.KubevirtClient) {
newRules := getPrometheusAlerts(virtClient)
var re = regexp.MustCompile("\\[\\d+m\\]")

Expand Down Expand Up @@ -389,3 +392,22 @@ func getPrometheusAlerts(virtClient kubecli.KubevirtClient) promv1.PrometheusRul

return newRules
}

func GetKubevirtVMMetrics(pod *k8sv1.Pod, ip string) string {
metricsURL := PrepareMetricsURL(ip, 8443)
stdout, _, err := execute.ExecuteCommandOnPodWithResults(kubevirt.Client(),
pod,
"virt-handler",
[]string{
"curl",
"-L",
"-k",
metricsURL,
})
Expect(err).ToNot(HaveOccurred())
return stdout
}

func PrepareMetricsURL(ip string, port int) string {
return fmt.Sprintf("https://%s/metrics", net.JoinHostPort(ip, strconv.Itoa(port)))
}
2 changes: 1 addition & 1 deletion tests/monitoring/scaling_utils.go → tests/libmonitoring/scaling.go
View file
@@ -1,4 +1,4 @@
package monitoring
package libmonitoring

import (
"context"
Expand Down
6 changes: 1 addition & 5 deletions tests/monitoring/BUILD.bazel
View file
Expand Up @@ -5,8 +5,6 @@ go_library(
srcs = [
"component_monitoring.go",
"monitoring.go",
"prometheus_utils.go",
"scaling_utils.go",
"vm_monitoring.go",
],
importpath = "kubevirt.io/kubevirt/tests/monitoring",
Expand All @@ -25,6 +23,7 @@ go_library(
"//tests/framework/kubevirt:go_default_library",
"//tests/framework/matcher:go_default_library",
"//tests/libmigration:go_default_library",
"//tests/libmonitoring:go_default_library",
"//tests/libnet:go_default_library",
"//tests/libnode:go_default_library",
"//tests/libvmi:go_default_library",
Expand All @@ -35,16 +34,13 @@ go_library(
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/onsi/gomega/types:go_default_library",
"//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library",
"//vendor/github.com/prometheus/client_golang/api/prometheus/v1:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/autoscaling/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
],
)

0 comments on commit 34d4b39

Please sign in to comment.