diff --git a/cmd/virt-api/BUILD.bazel b/cmd/virt-api/BUILD.bazel index 1454bac3e561..d519484c4706 100644 --- a/cmd/virt-api/BUILD.bazel +++ b/cmd/virt-api/BUILD.bazel @@ -6,9 +6,6 @@ go_library( importpath = "kubevirt.io/kubevirt/cmd/virt-api", visibility = ["//visibility:private"], deps = [ - "//pkg/monitoring/client/prometheus:go_default_library", - "//pkg/monitoring/reflector/prometheus:go_default_library", - "//pkg/monitoring/workqueue/prometheus:go_default_library", "//pkg/service:go_default_library", "//pkg/virt-api:go_default_library", "//staging/src/kubevirt.io/client-go/log:go_default_library", diff --git a/cmd/virt-api/virt-api.go b/cmd/virt-api/virt-api.go index 390aa4a00f13..a83332d81ba3 100644 --- a/cmd/virt-api/virt-api.go +++ b/cmd/virt-api/virt-api.go @@ -22,9 +22,6 @@ package main import ( klog "kubevirt.io/client-go/log" - _ "kubevirt.io/kubevirt/pkg/monitoring/client/prometheus" // import for prometheus metrics - _ "kubevirt.io/kubevirt/pkg/monitoring/reflector/prometheus" // import for prometheus metrics - _ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics "kubevirt.io/kubevirt/pkg/service" virt_api "kubevirt.io/kubevirt/pkg/virt-api" ) diff --git a/cmd/virt-controller/BUILD.bazel b/cmd/virt-controller/BUILD.bazel index 9ff3f0604afa..fb326c5322ae 100644 --- a/cmd/virt-controller/BUILD.bazel +++ b/cmd/virt-controller/BUILD.bazel @@ -6,9 +6,6 @@ go_library( importpath = "kubevirt.io/kubevirt/cmd/virt-controller", visibility = ["//visibility:private"], deps = [ - "//pkg/monitoring/client/prometheus:go_default_library", - "//pkg/monitoring/reflector/prometheus:go_default_library", - "//pkg/monitoring/workqueue/prometheus:go_default_library", "//pkg/virt-controller/watch:go_default_library", ], ) diff --git a/cmd/virt-controller/virt-controller.go b/cmd/virt-controller/virt-controller.go index 03115f65db57..9d225846da75 100644 --- a/cmd/virt-controller/virt-controller.go +++ b/cmd/virt-controller/virt-controller.go @@ -20,9 +20,6 @@ package main import ( - _ "kubevirt.io/kubevirt/pkg/monitoring/client/prometheus" // import for prometheus metrics - _ "kubevirt.io/kubevirt/pkg/monitoring/reflector/prometheus" // import for prometheus metrics - _ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics "kubevirt.io/kubevirt/pkg/virt-controller/watch" ) diff --git a/cmd/virt-handler/BUILD.bazel b/cmd/virt-handler/BUILD.bazel index f60c00cc22ce..cff502536586 100644 --- a/cmd/virt-handler/BUILD.bazel +++ b/cmd/virt-handler/BUILD.bazel @@ -10,12 +10,10 @@ go_library( "//pkg/container-disk:go_default_library", "//pkg/controller:go_default_library", "//pkg/healthz:go_default_library", - "//pkg/monitoring/client/prometheus:go_default_library", "//pkg/monitoring/domainstats/downwardmetrics:go_default_library", "//pkg/monitoring/domainstats/prometheus:go_default_library", + "//pkg/monitoring/metrics/virt-handler:go_default_library", "//pkg/monitoring/profiler:go_default_library", - "//pkg/monitoring/reflector/prometheus:go_default_library", - "//pkg/monitoring/workqueue/prometheus:go_default_library", "//pkg/safepath:go_default_library", "//pkg/service:go_default_library", "//pkg/util:go_default_library", diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go index 65633316844e..1611ecb2a0c0 100644 --- a/cmd/virt-handler/virt-handler.go +++ b/cmd/virt-handler/virt-handler.go @@ -71,11 +71,9 @@ import ( "kubevirt.io/kubevirt/pkg/certificates/bootstrap" containerdisk "kubevirt.io/kubevirt/pkg/container-disk" "kubevirt.io/kubevirt/pkg/controller" - _ "kubevirt.io/kubevirt/pkg/monitoring/client/prometheus" // import for prometheus metrics promdomain "kubevirt.io/kubevirt/pkg/monitoring/domainstats/prometheus" // import for prometheus metrics + metrics "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler" "kubevirt.io/kubevirt/pkg/monitoring/profiler" - _ "kubevirt.io/kubevirt/pkg/monitoring/reflector/prometheus" // import for prometheus metrics - _ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics "kubevirt.io/kubevirt/pkg/service" "kubevirt.io/kubevirt/pkg/util" virtconfig "kubevirt.io/kubevirt/pkg/virt-config" @@ -203,6 +201,10 @@ func (app *virtHandlerApp) Run() { logger.V(1).Infof("hostname %s", app.HostOverride) var err error + if err := metrics.SetupMetrics(); err != nil { + panic(fmt.Errorf("failed to set up metrics: %v", err)) + } + // Copy container-disk binary targetFile := filepath.Join(app.VirtLibDir, "/init/usr/bin/container-disk") err = os.MkdirAll(filepath.Dir(targetFile), os.ModePerm) diff --git a/cmd/virt-operator/BUILD.bazel b/cmd/virt-operator/BUILD.bazel index 4c3d3a5b3a16..19a71abe5773 100644 --- a/cmd/virt-operator/BUILD.bazel +++ b/cmd/virt-operator/BUILD.bazel @@ -6,9 +6,6 @@ go_library( importpath = "kubevirt.io/kubevirt/cmd/virt-operator", visibility = ["//visibility:private"], deps = [ - "//pkg/monitoring/client/prometheus:go_default_library", - "//pkg/monitoring/reflector/prometheus:go_default_library", - "//pkg/monitoring/workqueue/prometheus:go_default_library", "//pkg/virt-operator:go_default_library", ], ) diff --git a/cmd/virt-operator/virt-operator.go b/cmd/virt-operator/virt-operator.go index c90583c48d45..c6462c785890 100644 --- a/cmd/virt-operator/virt-operator.go +++ b/cmd/virt-operator/virt-operator.go @@ -20,9 +20,6 @@ package main import ( - _ "kubevirt.io/kubevirt/pkg/monitoring/client/prometheus" // import for prometheus metrics - _ "kubevirt.io/kubevirt/pkg/monitoring/reflector/prometheus" // import for prometheus metrics - _ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics virt_operator "kubevirt.io/kubevirt/pkg/virt-operator" ) diff --git a/docs/metrics.md b/docs/metrics.md index 8d93ec480d0e..daced8eedcb3 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -270,6 +270,15 @@ Returns the labels of the persistent volume claims that are used for restoring v ### kubevirt_vnc_active_connections Amount of active VNC connections, broken down by namespace and vmi name. Type: Gauge. +### rest_client_rate_limiter_duration_seconds +Client side rate limiter latency in seconds. Broken down by verb and URL. Type: Histogram. + +### rest_client_request_latency_seconds +Request latency in seconds. Broken down by verb and URL. Type: Histogram. + +### rest_client_requests_total +Number of HTTP requests, partitioned by status code, method, and host. Type: Counter. + ## Developing new metrics After developing new metrics or changing old ones, please run `make generate` to regenerate this document. diff --git a/pkg/monitoring/client/prometheus/prometheus.go b/pkg/monitoring/client/prometheus/prometheus.go deleted file mode 100644 index e9d5a25e63f5..000000000000 --- a/pkg/monitoring/client/prometheus/prometheus.go +++ /dev/null @@ -1,201 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package prometheus creates and registers prometheus metrics with -// rest clients. To use this package, you just have to import it. -package prometheus - -import ( - "context" - "fmt" - "net/http" - "net/url" - "regexp" - "strconv" - "strings" - "time" - - "github.com/prometheus/client_golang/prometheus" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/metrics" - - "kubevirt.io/client-go/kubecli" -) - -var ( - // requestLatency is a Prometheus Summary metric type partitioned by - // "verb" and "url" labels. It is used for the rest client latency metrics. - requestLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "rest_client_request_latency_seconds", - Help: "Request latency in seconds. Broken down by verb and URL.", - // Buckets based on Kubernetes apiserver_request_duration_seconds. - // See https://github.com/kubernetes/kubernetes/pull/73638 for the discussion. - Buckets: []float64{0.005, 0.025, 0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3, - 4, 5, 6, 8, 10, 15, 20, 30, 45, 60}, - }, - []string{"verb", "url"}, - ) - - rateLimiterLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "rest_client_rate_limiter_duration_seconds", - Help: "Client side rate limiter latency in seconds. Broken down by verb and URL.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), - }, - []string{"verb", "url"}, - ) - - requestResult = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "rest_client_requests_total", - Help: "Number of HTTP requests, partitioned by status code, method, and host.", - }, - []string{"code", "method", "host", "resource", "verb"}, - ) - - resourceParsingRegexs = []*regexp.Regexp{} -) - -func init() { - - resPat := `[A-Za-z0-9.\-]*` - - // watch core k8s apis - resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/api/%s/watch/namespaces/%s/(?P%s)`, resPat, resPat, resPat))) - resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/api/%s/watch/(?P%s)`, resPat, resPat))) - - // watch custom resource apis - resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/apis/%s/%s/watch/namespaces/%s/(?P%s)`, resPat, resPat, resPat, resPat))) - resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/apis/%s/%s/watch/(?P%s)`, resPat, resPat, resPat))) - - // namespaced core k8 apis and namespaced custom apis - resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/api/%s/namespaces/%s/(?P%s)`, resPat, resPat, resPat))) - resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/apis/%s/%s/namespaces/%s/(?P%s)`, resPat, resPat, resPat, resPat))) - - // globally scoped core k8s apis and globally scoped custom apis - resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/api/%s/(?P%s)`, resPat, resPat))) - resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/apis/%s/%s/(?P%s)`, resPat, resPat, resPat))) - - kubecli.RegisterRestConfigHook(addHTTPRoundTripClientMonitoring) - - prometheus.MustRegister(requestLatency) - prometheus.MustRegister(requestResult) - prometheus.MustRegister(rateLimiterLatency) - metrics.Register(metrics.RegisterOpts{ - RequestLatency: &latencyAdapter{requestLatency}, - RateLimiterLatency: &latencyAdapter{rateLimiterLatency}, - }) -} - -type latencyAdapter struct { - m *prometheus.HistogramVec -} - -func (l *latencyAdapter) Observe(ctx context.Context, verb string, u url.URL, latency time.Duration) { - l.m.WithLabelValues(verb, u.String()).Observe(latency.Seconds()) -} - -type rtWrapper struct { - origRoundTripper http.RoundTripper -} - -func parseURLResourceOperation(request *http.Request) (resource string, verb string) { - method := request.Method - - resource = "" - verb = "" - - if request.URL.Path == "" || method == "" { - return - } - - for _, r := range resourceParsingRegexs { - if resource != "" { - break - } - match := r.FindStringSubmatch(request.URL.Path) - if len(match) > 1 { - resource = match[1] - } - } - - if resource == "" { - return - } - - switch method { - case "GET": - verb = "GET" - if strings.Contains(request.URL.Path, "/watch/") { - verb = "WATCH" - } else if strings.HasSuffix(request.URL.Path, resource) { - // If the resource is the last element in the url, then - // we're asking to list all resources of that type instead - // of getting an individual resource - verb = "LIST" - } - case "PUT": - verb = "UPDATE" - case "PATCH": - verb = "PATCH" - case "POST": - verb = "CREATE" - case "DELETE": - verb = "DELETE" - } - - return resource, verb -} - -func (r *rtWrapper) RoundTrip(request *http.Request) (response *http.Response, err error) { - var status string - var resource string - var verb string - var host string - - response, err = r.origRoundTripper.RoundTrip(request) - if err != nil { - status = "" - } else { - status = strconv.Itoa(response.StatusCode) - } - - host = "none" - if request.URL != nil { - host = request.URL.Host - } - - resource, verb = parseURLResourceOperation(request) - if verb == "" { - verb = "none" - } - if resource == "" { - resource = "none" - } - requestResult.WithLabelValues(status, request.Method, host, resource, verb).Add(1) - - return response, err -} - -func addHTTPRoundTripClientMonitoring(config *rest.Config) { - fn := func(rt http.RoundTripper) http.RoundTripper { - return &rtWrapper{ - origRoundTripper: rt, - } - } - config.Wrap(fn) -} diff --git a/pkg/monitoring/client/prometheus/BUILD.bazel b/pkg/monitoring/metrics/common/client/BUILD.bazel similarity index 68% rename from pkg/monitoring/client/prometheus/BUILD.bazel rename to pkg/monitoring/metrics/common/client/BUILD.bazel index fa50b946ab95..1b07d3984ad8 100644 --- a/pkg/monitoring/client/prometheus/BUILD.bazel +++ b/pkg/monitoring/metrics/common/client/BUILD.bazel @@ -2,11 +2,17 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["prometheus.go"], - importpath = "kubevirt.io/kubevirt/pkg/monitoring/client/prometheus", + srcs = [ + "latency_adapter.go", + "metrics.go", + "rest_metrics.go", + "rt_wrapper.go", + ], + importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client", visibility = ["//visibility:public"], deps = [ "//staging/src/kubevirt.io/client-go/kubecli:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/tools/metrics:go_default_library", @@ -16,8 +22,8 @@ go_library( go_test( name = "go_default_test", srcs = [ - "prometheus_suite_test.go", - "prometheus_test.go", + "client_suite_test.go", + "client_test.go", ], embed = [":go_default_library"], deps = [ diff --git a/pkg/monitoring/client/prometheus/prometheus_suite_test.go b/pkg/monitoring/metrics/common/client/client_suite_test.go similarity index 87% rename from pkg/monitoring/client/prometheus/prometheus_suite_test.go rename to pkg/monitoring/metrics/common/client/client_suite_test.go index 08cfb3a0661d..cbe89ca5b98e 100644 --- a/pkg/monitoring/client/prometheus/prometheus_suite_test.go +++ b/pkg/monitoring/metrics/common/client/client_suite_test.go @@ -1,4 +1,4 @@ -package prometheus +package client import ( "testing" diff --git a/pkg/monitoring/client/prometheus/prometheus_test.go b/pkg/monitoring/metrics/common/client/client_test.go similarity index 97% rename from pkg/monitoring/client/prometheus/prometheus_test.go rename to pkg/monitoring/metrics/common/client/client_test.go index d5488a318f5e..935ccb9362b6 100644 --- a/pkg/monitoring/client/prometheus/prometheus_test.go +++ b/pkg/monitoring/metrics/common/client/client_test.go @@ -17,7 +17,7 @@ * */ -package prometheus +package client import ( "net/http" @@ -28,6 +28,8 @@ import ( ) var _ = BeforeSuite(func() { + err := SetupMetrics() + Expect(err).ToNot(HaveOccurred()) }) var _ = Describe("URL Parsing", func() { diff --git a/pkg/monitoring/metrics/common/client/latency_adapter.go b/pkg/monitoring/metrics/common/client/latency_adapter.go new file mode 100644 index 000000000000..b73d3db195f8 --- /dev/null +++ b/pkg/monitoring/metrics/common/client/latency_adapter.go @@ -0,0 +1,35 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + */ + +package client + +import ( + "context" + "net/url" + "time" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" +) + +type latencyAdapter struct { + m *operatormetrics.HistogramVec +} + +func (l *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { + l.m.WithLabelValues(verb, u.String()).Observe(latency.Seconds()) +} diff --git a/pkg/monitoring/metrics/common/client/metrics.go b/pkg/monitoring/metrics/common/client/metrics.go new file mode 100644 index 000000000000..235e21fe2585 --- /dev/null +++ b/pkg/monitoring/metrics/common/client/metrics.go @@ -0,0 +1,75 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + */ + +package client + +import ( + "fmt" + "net/http" + "regexp" + + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/metrics" + "kubevirt.io/client-go/kubecli" +) + +var resourceParsingRegexs []*regexp.Regexp + +func SetupMetrics() error { + setupResourcesToWatch() + kubecli.RegisterRestConfigHook(addHTTPRoundTripClientMonitoring) + + metrics.Register(metrics.RegisterOpts{ + RequestLatency: &latencyAdapter{requestLatency}, + RateLimiterLatency: &latencyAdapter{rateLimiterLatency}, + }) + + return operatormetrics.RegisterMetrics( + restMetrics, + ) +} + +func setupResourcesToWatch() { + resPat := `[A-Za-z0-9.\-]*` + + // watch core k8s apis + resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/api/%s/watch/namespaces/%s/(?P%s)`, resPat, resPat, resPat))) + resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/api/%s/watch/(?P%s)`, resPat, resPat))) + + // watch custom resource apis + resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/apis/%s/%s/watch/namespaces/%s/(?P%s)`, resPat, resPat, resPat, resPat))) + resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/apis/%s/%s/watch/(?P%s)`, resPat, resPat, resPat))) + + // namespaced core k8 apis and namespaced custom apis + resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/api/%s/namespaces/%s/(?P%s)`, resPat, resPat, resPat))) + resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/apis/%s/%s/namespaces/%s/(?P%s)`, resPat, resPat, resPat, resPat))) + + // globally scoped core k8s apis and globally scoped custom apis + resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/api/%s/(?P%s)`, resPat, resPat))) + resourceParsingRegexs = append(resourceParsingRegexs, regexp.MustCompile(fmt.Sprintf(`/apis/%s/%s/(?P%s)`, resPat, resPat, resPat))) +} + +func addHTTPRoundTripClientMonitoring(config *rest.Config) { + fn := func(rt http.RoundTripper) http.RoundTripper { + return &rtWrapper{ + origRoundTripper: rt, + } + } + config.Wrap(fn) +} diff --git a/pkg/monitoring/metrics/common/client/rest_metrics.go b/pkg/monitoring/metrics/common/client/rest_metrics.go new file mode 100644 index 000000000000..4e7310f11080 --- /dev/null +++ b/pkg/monitoring/metrics/common/client/rest_metrics.go @@ -0,0 +1,69 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + */ + +package client + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + restMetrics = []operatormetrics.Metric{ + requestLatency, + rateLimiterLatency, + requestResult, + } + + // requestLatency is a Prometheus Summary metric type partitioned by + // "verb" and "url" labels. It is used for the rest client latency metrics. + requestLatency = operatormetrics.NewHistogramVec( + operatormetrics.MetricOpts{ + Name: "rest_client_request_latency_seconds", + Help: "Request latency in seconds. Broken down by verb and URL.", + }, + prometheus.HistogramOpts{ + // Buckets based on Kubernetes apiserver_request_duration_seconds. + // See https://github.com/kubernetes/kubernetes/pull/73638 for the discussion. + Buckets: []float64{ + 0.005, 0.025, 0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3, + 4, 5, 6, 8, 10, 15, 20, 30, 45, 60, + }, + }, + []string{"verb", "url"}, + ) + + rateLimiterLatency = operatormetrics.NewHistogramVec( + operatormetrics.MetricOpts{ + Name: "rest_client_rate_limiter_duration_seconds", + Help: "Client side rate limiter latency in seconds. Broken down by verb and URL.", + }, + prometheus.HistogramOpts{ + Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), + }, + []string{"verb", "url"}, + ) + + requestResult = operatormetrics.NewCounterVec( + operatormetrics.MetricOpts{ + Name: "rest_client_requests_total", + Help: "Number of HTTP requests, partitioned by status code, method, and host.", + }, + []string{"code", "method", "host", "resource", "verb"}, + ) +) diff --git a/pkg/monitoring/metrics/common/client/rt_wrapper.go b/pkg/monitoring/metrics/common/client/rt_wrapper.go new file mode 100644 index 000000000000..23bfba0b3bf6 --- /dev/null +++ b/pkg/monitoring/metrics/common/client/rt_wrapper.go @@ -0,0 +1,107 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + */ + +package client + +import ( + "net/http" + "strconv" + "strings" +) + +type rtWrapper struct { + origRoundTripper http.RoundTripper +} + +func (r *rtWrapper) RoundTrip(request *http.Request) (response *http.Response, err error) { + var status string + var resource string + var verb string + var host string + + response, err = r.origRoundTripper.RoundTrip(request) + if err != nil { + status = "" + } else { + status = strconv.Itoa(response.StatusCode) + } + + host = "none" + if request.URL != nil { + host = request.URL.Host + } + + resource, verb = parseURLResourceOperation(request) + if verb == "" { + verb = "none" + } + if resource == "" { + resource = "none" + } + requestResult.WithLabelValues(status, request.Method, host, resource, verb).Add(1) + + return response, err +} + +func parseURLResourceOperation(request *http.Request) (resource string, verb string) { + method := request.Method + + resource = "" + verb = "" + + if request.URL.Path == "" || method == "" { + return + } + + for _, r := range resourceParsingRegexs { + if resource != "" { + break + } + match := r.FindStringSubmatch(request.URL.Path) + if len(match) > 1 { + resource = match[1] + } + } + + if resource == "" { + return + } + + switch method { + case "GET": + verb = "GET" + if strings.Contains(request.URL.Path, "/watch/") { + verb = "WATCH" + } else if strings.HasSuffix(request.URL.Path, resource) { + // If the resource is the last element in the url, then + // we're asking to list all resources of that type instead + // of getting an individual resource + verb = "LIST" + } + case "PUT": + verb = "UPDATE" + case "PATCH": + verb = "PATCH" + case "POST": + verb = "CREATE" + case "DELETE": + verb = "DELETE" + } + + return resource, verb +} diff --git a/pkg/monitoring/metrics/common/workqueue/BUILD.bazel b/pkg/monitoring/metrics/common/workqueue/BUILD.bazel new file mode 100644 index 000000000000..f28a3130b9bd --- /dev/null +++ b/pkg/monitoring/metrics/common/workqueue/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["metrics.go"], + importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", + ], +) diff --git a/pkg/monitoring/metrics/common/workqueue/metrics.go b/pkg/monitoring/metrics/common/workqueue/metrics.go new file mode 100644 index 000000000000..01580f3640ad --- /dev/null +++ b/pkg/monitoring/metrics/common/workqueue/metrics.go @@ -0,0 +1,122 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + */ + +package workqueue + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/prometheus/client_golang/prometheus" + k8sworkqueue "k8s.io/client-go/util/workqueue" +) + +func SetupMetrics() error { + k8sworkqueue.SetProvider(prometheusMetricsProvider{}) + return nil +} + +type prometheusMetricsProvider struct{} + +func (_ prometheusMetricsProvider) NewDepthMetric(name string) k8sworkqueue.GaugeMetric { + depth := operatormetrics.NewGauge(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_depth", + Help: "Current depth of workqueue", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{depth}) + + return depth +} + +func (_ prometheusMetricsProvider) NewAddsMetric(name string) k8sworkqueue.CounterMetric { + adds := operatormetrics.NewCounter(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_adds_total", + Help: "Total number of adds handled by workqueue", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{adds}) + + return adds +} + +func (_ prometheusMetricsProvider) NewLatencyMetric(name string) k8sworkqueue.HistogramMetric { + latency := operatormetrics.NewHistogram( + operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_queue_duration_seconds", + Help: "How long an item stays in workqueue before being requested.", + ConstLabels: prometheus.Labels{"name": name}, + }, + prometheus.HistogramOpts{ + Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), + }, + ) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{latency}) + + return latency +} + +func (_ prometheusMetricsProvider) NewWorkDurationMetric(name string) k8sworkqueue.HistogramMetric { + workDuration := operatormetrics.NewHistogram( + operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_work_duration_seconds", + Help: "How long in seconds processing an item from workqueue takes.", + ConstLabels: prometheus.Labels{"name": name}, + }, + prometheus.HistogramOpts{ + Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), + }, + ) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{workDuration}) + + return workDuration +} + +func (_ prometheusMetricsProvider) NewRetriesMetric(name string) k8sworkqueue.CounterMetric { + retries := operatormetrics.NewCounter(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_retries_total", + Help: "Total number of retries handled by workqueue", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{retries}) + + return retries +} + +func (_ prometheusMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) k8sworkqueue.SettableGaugeMetric { + longestRunningProcessor := operatormetrics.NewGauge(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_longest_running_processor_seconds", + Help: "How many seconds has the longest running processor for workqueue been running.", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{longestRunningProcessor}) + + return longestRunningProcessor +} + +func (_ prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) k8sworkqueue.SettableGaugeMetric { + unfinishedWork := operatormetrics.NewGauge(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_unfinished_work_seconds", + Help: "How many seconds of work has done that is in progress and hasn't " + + "been observed by work_duration. Large values indicate stuck " + + "threads. One can deduce the number of stuck threads by observing " + + "the rate at which this increases.", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{unfinishedWork}) + + return unfinishedWork +} diff --git a/pkg/monitoring/metrics/virt-api/BUILD.bazel b/pkg/monitoring/metrics/virt-api/BUILD.bazel index 634a3e2daeea..5324455f8b60 100644 --- a/pkg/monitoring/metrics/virt-api/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-api/BUILD.bazel @@ -10,6 +10,8 @@ go_library( importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api", visibility = ["//visibility:public"], deps = [ + "//pkg/monitoring/metrics/common/client:go_default_library", + "//pkg/monitoring/metrics/common/workqueue:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", ], diff --git a/pkg/monitoring/metrics/virt-api/metrics.go b/pkg/monitoring/metrics/virt-api/metrics.go index 3e5ae28399e9..348d0f257ba8 100644 --- a/pkg/monitoring/metrics/virt-api/metrics.go +++ b/pkg/monitoring/metrics/virt-api/metrics.go @@ -19,9 +19,22 @@ package virt_api -import "github.com/machadovilaca/operator-observability/pkg/operatormetrics" +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" +) func SetupMetrics() error { + if err := client.SetupMetrics(); err != nil { + return err + } + + if err := workqueue.SetupMetrics(); err != nil { + return err + } + return operatormetrics.RegisterMetrics( connectionMetrics, vmMetrics, diff --git a/pkg/monitoring/metrics/virt-controller/BUILD.bazel b/pkg/monitoring/metrics/virt-controller/BUILD.bazel index 8938417b0769..46d50d155146 100644 --- a/pkg/monitoring/metrics/virt-controller/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-controller/BUILD.bazel @@ -15,6 +15,8 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/controller:go_default_library", + "//pkg/monitoring/metrics/common/client:go_default_library", + "//pkg/monitoring/metrics/common/workqueue:go_default_library", "//pkg/util/migrations:go_default_library", "//pkg/virt-config:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", diff --git a/pkg/monitoring/metrics/virt-controller/metrics.go b/pkg/monitoring/metrics/virt-controller/metrics.go index 57c25fb2930f..0c528518490c 100644 --- a/pkg/monitoring/metrics/virt-controller/metrics.go +++ b/pkg/monitoring/metrics/virt-controller/metrics.go @@ -27,6 +27,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" virtconfig "kubevirt.io/kubevirt/pkg/virt-config" ) @@ -66,6 +68,14 @@ func SetupMetrics( vmiMigrationInformer = vmiMigration clusterConfig = virtClusterConfig + if err := client.SetupMetrics(); err != nil { + return err + } + + if err := workqueue.SetupMetrics(); err != nil { + return err + } + if err := operatormetrics.RegisterMetrics(metrics...); err != nil { return err } diff --git a/pkg/monitoring/metrics/virt-handler/BUILD.bazel b/pkg/monitoring/metrics/virt-handler/BUILD.bazel new file mode 100644 index 000000000000..8f0e3cd74c52 --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["metrics.go"], + importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler", + visibility = ["//visibility:public"], + deps = [ + "//pkg/monitoring/metrics/common/client:go_default_library", + "//pkg/monitoring/metrics/common/workqueue:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + ], +) diff --git a/pkg/monitoring/metrics/virt-handler/metrics.go b/pkg/monitoring/metrics/virt-handler/metrics.go new file mode 100644 index 000000000000..8ef0e023711e --- /dev/null +++ b/pkg/monitoring/metrics/virt-handler/metrics.go @@ -0,0 +1,38 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + */ + +package virt_handler + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" +) + +func SetupMetrics() error { + if err := workqueue.SetupMetrics(); err != nil { + return err + } + + return client.SetupMetrics() +} + +func ListMetrics() []operatormetrics.Metric { + return operatormetrics.ListMetrics() +} diff --git a/pkg/monitoring/metrics/virt-operator/BUILD.bazel b/pkg/monitoring/metrics/virt-operator/BUILD.bazel index 41eb8d85dfbc..04b6c0fc2f4e 100644 --- a/pkg/monitoring/metrics/virt-operator/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-operator/BUILD.bazel @@ -9,5 +9,9 @@ go_library( ], importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library"], + deps = [ + "//pkg/monitoring/metrics/common/client:go_default_library", + "//pkg/monitoring/metrics/common/workqueue:go_default_library", + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + ], ) diff --git a/pkg/monitoring/metrics/virt-operator/metrics.go b/pkg/monitoring/metrics/virt-operator/metrics.go index 7c295b9ea99b..bb0c1a09ac06 100644 --- a/pkg/monitoring/metrics/virt-operator/metrics.go +++ b/pkg/monitoring/metrics/virt-operator/metrics.go @@ -19,9 +19,22 @@ package virt_operator -import "github.com/machadovilaca/operator-observability/pkg/operatormetrics" +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" +) func SetupMetrics() error { + if err := client.SetupMetrics(); err != nil { + return err + } + + if err := workqueue.SetupMetrics(); err != nil { + return err + } + return operatormetrics.RegisterMetrics( configurationMetrics, operatorMetrics, diff --git a/pkg/monitoring/reflector/prometheus/BUILD.bazel b/pkg/monitoring/reflector/prometheus/BUILD.bazel deleted file mode 100644 index d078e30b246c..000000000000 --- a/pkg/monitoring/reflector/prometheus/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["prometheus.go"], - importpath = "kubevirt.io/kubevirt/pkg/monitoring/reflector/prometheus", - visibility = ["//visibility:public"], - deps = [ - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/k8s.io/client-go/tools/cache:go_default_library", - ], -) diff --git a/pkg/monitoring/reflector/prometheus/prometheus.go b/pkg/monitoring/reflector/prometheus/prometheus.go deleted file mode 100644 index a16b5eee82fb..000000000000 --- a/pkg/monitoring/reflector/prometheus/prometheus.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package prometheus sets the cache DefaultMetricsFactory to produce -// prometheus metrics. To use this package, you just have to import it. -package prometheus - -import ( - "github.com/prometheus/client_golang/prometheus" - "k8s.io/client-go/tools/cache" -) - -const reflectorSubsystem = "reflector" - -var ( - listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "lists_total", - Help: "Total number of API lists done by the reflectors", - }, []string{"name"}) - - listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "list_duration_seconds", - Help: "How long an API list takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "items_per_list", - Help: "How many items an API list returns to the reflectors", - }, []string{"name"}) - - watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "watches_total", - Help: "Total number of API watches done by the reflectors", - }, []string{"name"}) - - shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: reflectorSubsystem, - Name: "short_watches_total", - Help: "Total number of short API watches done by the reflectors", - }, []string{"name"}) - - watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "watch_duration_seconds", - Help: "How long an API watch takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: reflectorSubsystem, - Name: "items_per_watch", - Help: "How many items an API watch returns to the reflectors", - }, []string{"name"}) - - lastResourceVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: reflectorSubsystem, - Name: "last_resource_version", - Help: "Last resource version seen for the reflectors", - }, []string{"name"}) -) - -func init() { - prometheus.MustRegister(listsTotal) - prometheus.MustRegister(listsDuration) - prometheus.MustRegister(itemsPerList) - prometheus.MustRegister(watchesTotal) - prometheus.MustRegister(shortWatchesTotal) - prometheus.MustRegister(watchDuration) - prometheus.MustRegister(itemsPerWatch) - prometheus.MustRegister(lastResourceVersion) - - cache.SetReflectorMetricsProvider(prometheusMetricsProvider{}) -} - -type prometheusMetricsProvider struct{} - -func (prometheusMetricsProvider) NewListsMetric(name string) cache.CounterMetric { - return listsTotal.WithLabelValues(name) -} - -// use summary to get averages and percentiles -func (prometheusMetricsProvider) NewListDurationMetric(name string) cache.SummaryMetric { - return listsDuration.WithLabelValues(name) -} - -// use summary to get averages and percentiles -func (prometheusMetricsProvider) NewItemsInListMetric(name string) cache.SummaryMetric { - return itemsPerList.WithLabelValues(name) -} - -func (prometheusMetricsProvider) NewWatchesMetric(name string) cache.CounterMetric { - return watchesTotal.WithLabelValues(name) -} - -func (prometheusMetricsProvider) NewShortWatchesMetric(name string) cache.CounterMetric { - return shortWatchesTotal.WithLabelValues(name) -} - -// use summary to get averages and percentiles -func (prometheusMetricsProvider) NewWatchDurationMetric(name string) cache.SummaryMetric { - return watchDuration.WithLabelValues(name) -} - -// use summary to get averages and percentiles -func (prometheusMetricsProvider) NewItemsInWatchMetric(name string) cache.SummaryMetric { - return itemsPerWatch.WithLabelValues(name) -} - -func (prometheusMetricsProvider) NewLastResourceVersionMetric(name string) cache.GaugeMetric { - return lastResourceVersion.WithLabelValues(name) -} diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go index 3b62c26897d5..32d0eb5ae53f 100644 --- a/pkg/virt-api/api.go +++ b/pkg/virt-api/api.go @@ -154,6 +154,10 @@ func NewVirtApi() VirtApi { } func (app *virtAPIApp) Execute() { + if err := metrics.SetupMetrics(); err != nil { + panic(err) + } + app.reloadableRateLimiter = ratelimiter.NewReloadableRateLimiter(flowcontrol.NewTokenBucketRateLimiter(virtconfig.DefaultVirtAPIQPS, virtconfig.DefaultVirtAPIBurst)) app.reloadableWebhookRateLimiter = ratelimiter.NewReloadableRateLimiter(flowcontrol.NewTokenBucketRateLimiter(virtconfig.DefaultVirtWebhookClientQPS, virtconfig.DefaultVirtWebhookClientBurst)) @@ -188,12 +192,6 @@ func (app *virtAPIApp) Execute() { app.ConfigureOpenAPIService() app.reInitChan = make(chan string, 10) - // setup monitoring - err = metrics.SetupMetrics() - if err != nil { - panic(err) - } - app.Run() } diff --git a/pkg/virt-operator/application.go b/pkg/virt-operator/application.go index 04bf8d861bae..94046bce79f0 100644 --- a/pkg/virt-operator/application.go +++ b/pkg/virt-operator/application.go @@ -119,6 +119,10 @@ func Execute() { log.InitializeLogging(VirtOperator) + if err := metrics.SetupMetrics(); err != nil { + golog.Fatalf("Error setting up metrics: %v", err) + } + host, err := os.Hostname() if err != nil { golog.Fatalf("unable to get hostname: %v", err) @@ -298,11 +302,6 @@ func Execute() { app.clusterConfig.SetConfigModifiedCallback(app.shouldChangeLogVerbosity) app.clusterConfig.SetConfigModifiedCallback(app.shouldUpdateConfigurationMetrics) - // Setup monitoring - if err := metrics.SetupMetrics(); err != nil { - golog.Fatalf("Error setting up metrics: %v", err) - } - go app.Run() <-app.reInitChan } diff --git a/tests/monitoring/metrics.go b/tests/monitoring/metrics.go index b13295b479cf..0c47c3c63f4e 100644 --- a/tests/monitoring/metrics.go +++ b/tests/monitoring/metrics.go @@ -74,6 +74,12 @@ var _ = Describe("[sig-monitoring]Metrics", decorators.SigMonitoring, func() { "kubevirt_vmi_migrations_in_running_phase": true, "kubevirt_vmi_migration_succeeded": true, "kubevirt_vmi_migration_failed": true, + + // name do not follow the convention to be prefixed with 'kubevirt_' + // TODO: @machadovilaca - refactor the metric names + "rest_client_request_latency_seconds": true, + "rest_client_rate_limiter_duration_seconds": true, + "rest_client_requests_total": true, } It("should contain virt components metrics", func() { diff --git a/tools/doc-generator/BUILD.bazel b/tools/doc-generator/BUILD.bazel index 3cfed2320a19..2bc5961a7da1 100644 --- a/tools/doc-generator/BUILD.bazel +++ b/tools/doc-generator/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "//pkg/monitoring/domainstats/prometheus:go_default_library", "//pkg/monitoring/metrics/virt-api:go_default_library", "//pkg/monitoring/metrics/virt-controller:go_default_library", + "//pkg/monitoring/metrics/virt-handler:go_default_library", "//pkg/monitoring/metrics/virt-operator:go_default_library", "//pkg/monitoring/rules:go_default_library", "//pkg/virt-controller/watch:go_default_library", diff --git a/tools/doc-generator/doc-generator.go b/tools/doc-generator/doc-generator.go index 2874586c2d5e..4b95e6602d1b 100644 --- a/tools/doc-generator/doc-generator.go +++ b/tools/doc-generator/doc-generator.go @@ -15,6 +15,7 @@ import ( domainstats "kubevirt.io/kubevirt/pkg/monitoring/domainstats/prometheus" // import for prometheus metrics virt_api "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-api" virt_controller "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-controller" + virt_handler "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler" virt_operator "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-operator" "kubevirt.io/kubevirt/pkg/monitoring/rules" _ "kubevirt.io/kubevirt/pkg/virt-controller/watch" @@ -152,25 +153,12 @@ func getMetricsNotIncludeInEndpointByDefault() metricList { }, } - err := virt_controller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil) - checkError(err) - for _, m := range virt_controller.ListMetrics() { - metrics = append(metrics, newMetric(m)) - } - - err = virt_api.SetupMetrics() - checkError(err) - for _, m := range virt_api.ListMetrics() { - metrics = append(metrics, newMetric(m)) - } + metrics = append(metrics, getVirtControllerMetrics()...) + metrics = append(metrics, getComponentMetrics(virt_api.SetupMetrics, virt_api.ListMetrics)...) + metrics = append(metrics, getComponentMetrics(virt_operator.SetupMetrics, virt_operator.ListMetrics)...) + metrics = append(metrics, getComponentMetrics(virt_handler.SetupMetrics, virt_handler.ListMetrics)...) - err = virt_operator.SetupMetrics() - checkError(err) - for _, m := range virt_operator.ListMetrics() { - metrics = append(metrics, newMetric(m)) - } - - err = rules.SetupRules("") + err := rules.SetupRules("") checkError(err) for _, rule := range rules.ListRecordingRules() { @@ -184,6 +172,31 @@ func getMetricsNotIncludeInEndpointByDefault() metricList { return metrics } +func getVirtControllerMetrics() metricList { + err := virt_controller.SetupMetrics(nil, nil, nil, nil, nil, nil, nil, nil) + checkError(err) + return listComponentMetrics(virt_controller.ListMetrics) +} + +func getComponentMetrics(setup func() error, list func() []operatormetrics.Metric) metricList { + err := setup() + checkError(err) + return listComponentMetrics(list) +} + +func listComponentMetrics(list func() []operatormetrics.Metric) metricList { + metrics := metricList{} + + for _, m := range list() { + metrics = append(metrics, newMetric(m)) + } + + err := operatormetrics.CleanRegistry() + checkError(err) + + return metrics +} + func newMetric(om operatormetrics.Metric) metric { return metric{ name: om.GetOpts().Name, diff --git a/tools/prom-metrics-collector/metrics_collector.go b/tools/prom-metrics-collector/metrics_collector.go index 2987de02b7e6..7d57021646d1 100644 --- a/tools/prom-metrics-collector/metrics_collector.go +++ b/tools/prom-metrics-collector/metrics_collector.go @@ -33,7 +33,10 @@ import ( // https://sdk.operatorframework.io/docs/best-practices/observability-best-practices/#metrics-guidelines // should be ignored. var excludedMetrics = map[string]struct{}{ - "kubevirt_vmi_phase_count": struct{}{}, + "kubevirt_vmi_phase_count": {}, + "rest_client_rate_limiter_duration_seconds": {}, + "rest_client_request_latency_seconds": {}, + "rest_client_requests_total": {}, } // Extract the name, help, and type from the metrics doc file