Add cluster webhook metrics in metrics-v3

endpoint: /minio/metrics/v3/cluster/webhook
metrics:
- failed_messages (counter)
- online (gauge)
- queue_length (gauge)
- total_messages (counter)

cmd/metrics-v2.go

@@ -294,11 +294,6 @@ const (
 	kmsRequestsError   = "request_error"
 	kmsRequestsFail    = "request_failure"
 	kmsUptime          = "uptime"
-
-	webhookOnline         = "online"
-	webhookQueueLength    = "queue_length"
-	webhookTotalMessages  = "total_messages"
-	webhookFailedMessages = "failed_messages"
 )
 
 const (

cmd/metrics-v3-cluster-webhook.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+
+	"github.com/minio/minio/internal/logger"
+)
+
+const (
+	webhookOnline         = "online"
+	webhookQueueLength    = "queue_length"
+	webhookTotalMessages  = "total_messages"
+	webhookFailedMessages = "failed_messages"
+	nameL                 = "name"
+	endpointL             = "endpoint"
+)
+
+var (
+	allWebhookLabels = []string{nameL, endpointL}
+	webhookOnlineMD  = NewGaugeMD(webhookOnline,
+		"Is the webhook online?",
+		allWebhookLabels...)
+	webhookFailedMessagesMD = NewCounterMD(webhookFailedMessages,
+		"Number of messages that failed to send",
+		allWebhookLabels...)
+	webhookQueueLengthMD = NewGaugeMD(webhookQueueLength,
+		"Webhook queue length",
+		allWebhookLabels...)
+	webhookTotalMessagesMD = NewCounterMD(webhookTotalMessages,
+		"Total number of messages sent to this target",
+		allWebhookLabels...)
+)
+
+// loadClusterWebhookMetrics - `MetricsLoaderFn` for cluster webhook
+// such as failed messages and total messages.
+func loadClusterWebhookMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
+	tgts := append(logger.SystemTargets(), logger.AuditTargets()...)
+	for _, t := range tgts {
+		isOnline := 0
+		if t.IsOnline(ctx) {
+			isOnline = 1
+		}
+		labels := []string{nameL, t.String(), endpointL, t.Endpoint()}
+		m.Set(webhookOnline, float64(isOnline), labels...)
+		m.Set(webhookFailedMessages, float64(t.Stats().FailedMessages), labels...)
+		m.Set(webhookQueueLength, float64(t.Stats().QueueLength), labels...)
+		m.Set(webhookTotalMessages, float64(t.Stats().TotalMessages), labels...)
+	}
+
+	return nil
+}

cmd/metrics-v3.go

@@ -46,6 +46,7 @@ const (
 	clusterErasureSetCollectorPath   collectorPath = "/cluster/erasure-set"
 	clusterAuditCollectorPath        collectorPath = "/cluster/audit"
 	clusterNotificationCollectorPath collectorPath = "/cluster/notification"
+	clusterWebhookCollectorPath      collectorPath = "/cluster/webhook"
 )
 
 const (
@@ -254,6 +255,16 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		loadClusterNotificationMetrics,
 	)
 
+	clusterWebhookMG := NewMetricsGroup(clusterWebhookCollectorPath,
+		[]MetricDescriptor{
+			webhookFailedMessagesMD,
+			webhookOnlineMD,
+			webhookQueueLengthMD,
+			webhookTotalMessagesMD,
+		},
+		loadClusterWebhookMetrics,
+	)
+
 	allMetricGroups := []*MetricsGroup{
 		apiRequestsMG,
 		apiBucketMG,
@@ -269,6 +280,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		clusterErasureSetMG,
 		clusterAuditMG,
 		clusterNotificationMG,
+		clusterWebhookMG,
 	}
 
 	// Bucket metrics are special, they always include the bucket label. These

docs/metrics/v3.md

@@ -184,6 +184,15 @@ The standard metrics groups for ProcessCollector and GoCollector are not shown b
 | `minio_cluster_audit_target_queue_length` | `gauge`   | Number of unsent messages in queue for target            | `target_id` |
 | `minio_cluster_audit_total_messages`      | `counter` | Total number of messages sent since start                | `target_id` |
 
+### `/cluster/webhook`
+
+| Name                                    | Type      | Help                                         | Labels          |
+|-----------------------------------------|-----------|----------------------------------------------|-----------------|
+| `minio_cluster_webhook_failed_messages` | `counter` | Number of messages that failed to send       | `name,endpoint` |
+| `minio_cluster_webhook_online`          | `gauge`   | Is the webhook online?                       | `name,endpoint` |
+| `minio_cluster_webhook_queue_length`    | `gauge`   | Webhook queue length                         | `name,endpoint` |
+| `minio_cluster_webhook_total_message`   | `counter` | Total number of messages sent to this target | `name,endpoint` |
+
 ### `/cluster/usage/objects`
 
 | Name                                                     | Type    | Help                                                           | Labels  |