Merge pull request #85 from chenchun/metrics

Add metrics

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/onsi/ginkgo v1.10.3
 	github.com/onsi/gomega v1.7.1
+	github.com/prometheus/client_golang v0.9.2
 	github.com/spf13/pflag v1.0.5
 	github.com/vishvananda/netlink v1.0.0
 	github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f

go.sum

@@ -25,6 +25,7 @@ github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:
 github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/bazelbuild/bazel-gazelle v0.0.0-20181012220611-c728ce9f663e/go.mod h1:uHBSeeATKpVazAACZBDPL/Nk/UhQDDsJWDlqYJo8/Us=
 github.com/bazelbuild/buildtools v0.0.0-20180226164855-80c7f0d45d7e/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
@@ -223,6 +224,7 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-shellwords v0.0.0-20180605041737-f8471b0a71de/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mesos/mesos-go v0.0.9/go.mod h1:kPYCMQ9gsOXVAle1OsoY4I1+9kPu8GHkf88aV59fDr4=
 github.com/mholt/caddy v0.0.0-20180213163048-2de495001514/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
@@ -275,9 +277,13 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/pquerna/ffjson v0.0.0-20180717144149-af8b230fcd20/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
+github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=

pkg/ipam/floatingip/ipam.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"tkestack.io/galaxy/pkg/api/galaxy/constant"
 )
@@ -63,6 +64,8 @@ type IPAM interface {
 	NodeSubnetsByKey(key string) (sets.String, error)
 	// Name returns IPAM's name.
 	Name() string
+	// implements metrics Collector interface
+	prometheus.Collector
 }
 
 // FloatingIPInfo is floatingIP information

pkg/ipam/floatingip/ipam_crd.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/tools/cache"
 	glog "k8s.io/klog"
@@ -59,9 +60,11 @@ type crdIpam struct {
 	ipType      Type
 	//caches for FloatingIP crd, both stores allocated FloatingIPs and unallocated FloatingIPs
 	cacheLock *sync.RWMutex
-	// key is FloatingIP name (ip typed as uint32)
+	// key is ip string
 	allocatedFIPs   map[string]*FloatingIP
 	unallocatedFIPs map[string]*FloatingIP
+
+	ipCounterDesc *prometheus.Desc
 }
 
 // NewCrdIPAM init IPAM struct.
@@ -72,6 +75,8 @@ func NewCrdIPAM(fipClient crd_clientset.Interface, ipType Type, informer crdInfo
 		cacheLock:       new(sync.RWMutex),
 		allocatedFIPs:   make(map[string]*FloatingIP),
 		unallocatedFIPs: make(map[string]*FloatingIP),
+		ipCounterDesc: prometheus.NewDesc("galaxy_ip_counter", "Galaxy floating ip counter",
+			[]string{"type", "subnet", "first_ip"}, nil),
 	}
 	// manually creating and fip to reserve it
 	if informer != nil {
@@ -257,9 +262,13 @@ func (ci *crdIpam) Release(key string, ip net.IP) error {
 
 // First returns the first matched IP by key.
 func (ci *crdIpam) First(key string) (*FloatingIPInfo, error) {
-	fip, err := ci.findFloatingIPByKey(key)
-	if err != nil {
-		return nil, err
+	ci.cacheLock.RLock()
+	defer ci.cacheLock.RUnlock()
+	var fip *FloatingIP
+	for _, spec := range ci.allocatedFIPs {
+		if spec.Key == key {
+			fip = spec
+		}
 	}
 	if fip == nil {
 		return nil, nil
@@ -317,6 +326,8 @@ func (ci *crdIpam) ByPrefix(prefix string) ([]FloatingIP, error) {
 }
 
 func (ci *crdIpam) NodeSubnet(nodeIP net.IP) *net.IPNet {
+	ci.cacheLock.RLock()
+	defer ci.cacheLock.RUnlock()
 	for j := range ci.FloatingIPs {
 		nodeSubnets := ci.FloatingIPs[j].NodeSubnets
 		for k := range nodeSubnets {
@@ -463,17 +474,6 @@ func (ci *crdIpam) syncCacheAfterDel(released *FloatingIP) {
 	return
 }
 
-func (ci *crdIpam) findFloatingIPByKey(key string) (*FloatingIP, error) {
-	ci.cacheLock.RLock()
-	defer ci.cacheLock.RUnlock()
-	for _, spec := range ci.allocatedFIPs {
-		if spec.Key == key {
-			return spec, nil
-		}
-	}
-	return nil, nil
-}
-
 func (ci *crdIpam) filterAllocatedSubnet(key string) sets.String {
 	//key would not be empty
 	subnetSet := sets.NewString()
@@ -548,3 +548,45 @@ func (ci *crdIpam) ReleaseIPs(ipToKey map[string]string) (map[string]string, map
 	}
 	return deleted, undeleted, nil
 }
+
+// Describe sends metrics description to ch
+func (ci *crdIpam) Describe(ch chan<- *prometheus.Desc) {
+	ch <- ci.ipCounterDesc
+}
+
+// Collect sends metrics to ch
+func (ci *crdIpam) Collect(ch chan<- prometheus.Metric) {
+	allocated, unallocated := map[string]*FloatingIP{}, map[string]*FloatingIP{}
+	ci.cacheLock.RLock()
+	pools := make([]*FloatingIPPool, len(ci.FloatingIPs))
+	for ipStr, fip := range ci.allocatedFIPs {
+		allocated[ipStr] = fip
+	}
+	for ipStr, fip := range ci.unallocatedFIPs {
+		unallocated[ipStr] = fip
+	}
+	for i := range ci.FloatingIPs {
+		pools[i] = ci.FloatingIPs[i]
+	}
+	ci.cacheLock.RUnlock()
+	for _, pool := range pools {
+		subnetStr := pool.IPNet().String()
+		var firstIP string
+		var allocatedNum float64
+		for _, ipr := range pool.IPRanges {
+			firstIP = ipr.First.String()
+			break
+		}
+		for _, fip := range allocated {
+			if !pool.Contains(fip.IP) {
+				continue
+			}
+			allocatedNum += 1
+		}
+		// since subnetStr may be the same for different pools, add a first ip tag
+		ch <- prometheus.MustNewConstMetric(ci.ipCounterDesc, prometheus.GaugeValue, allocatedNum,
+			"allocated", subnetStr, firstIP)
+		ch <- prometheus.MustNewConstMetric(ci.ipCounterDesc, prometheus.GaugeValue, float64(pool.Size()),
+			"total", subnetStr, firstIP)
+	}
+}

pkg/ipam/metrics/metrics.go

@@ -0,0 +1,26 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	ScheduleLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "galaxy_schedule_latency",
+			Help:    "Galaxy schedule latency in seconds",
+			Buckets: prometheus.ExponentialBuckets(0.1, 2, 7),
+		}, []string{"func"})
+
+	CloudProviderLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "galaxy_cloud_provider_latency",
+			Help:    "Galaxy cloud provider latency in seconds",
+			Buckets: prometheus.ExponentialBuckets(0.1, 2, 7),
+		}, []string{"func"})
+)
+
+// MustRegister registers all metrics
+func MustRegister() {
+	prometheus.MustRegister(ScheduleLatency, CloudProviderLatency)
+}

pkg/ipam/schedulerplugin/cloudprovider.go

@@ -18,17 +18,21 @@ package schedulerplugin
 
 import (
 	"fmt"
+	"time"
 
 	glog "k8s.io/klog"
 	"tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc"
+	"tkestack.io/galaxy/pkg/ipam/metrics"
 )
 
 // cloudProviderAssignIP send assign ip req to cloud provider
 func (p *FloatingIPPlugin) cloudProviderAssignIP(req *rpc.AssignIPRequest) error {
 	if p.cloudProvider == nil {
 		return nil
 	}
+	start := time.Now()
 	reply, err := p.cloudProvider.AssignIP(req)
+	metrics.CloudProviderLatency.WithLabelValues("assign").Observe(time.Since(start).Seconds())
 	if err != nil {
 		return fmt.Errorf("cloud provider AssignIP reply err %v", err)
 	}
@@ -47,7 +51,9 @@ func (p *FloatingIPPlugin) cloudProviderUnAssignIP(req *rpc.UnAssignIPRequest) e
 	if p.cloudProvider == nil {
 		return nil
 	}
+	start := time.Now()
 	reply, err := p.cloudProvider.UnAssignIP(req)
+	metrics.CloudProviderLatency.WithLabelValues("unassign").Observe(time.Since(start).Seconds())
 	if err != nil {
 		return fmt.Errorf("cloud provider UnAssignIP reply err %v", err)
 	}

pkg/ipam/schedulerplugin/floatingip_plugin.go

@@ -36,6 +36,7 @@ import (
 	"tkestack.io/galaxy/pkg/ipam/cloudprovider"
 	"tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc"
 	"tkestack.io/galaxy/pkg/ipam/floatingip"
+	"tkestack.io/galaxy/pkg/ipam/metrics"
 	"tkestack.io/galaxy/pkg/ipam/schedulerplugin/util"
 )
 
@@ -148,6 +149,7 @@ func (p *FloatingIPPlugin) updateConfigMap() (bool, error) {
 // If the given pod doesn't want floating IP, none failedNodes returns
 func (p *FloatingIPPlugin) Filter(pod *corev1.Pod, nodes []corev1.Node) ([]corev1.Node, schedulerapi.FailedNodesMap,
 	error) {
+	start := time.Now()
 	failedNodesMap := schedulerapi.FailedNodesMap{}
 	if !p.hasResourceName(&pod.Spec) {
 		return nodes, failedNodesMap, nil
@@ -178,6 +180,7 @@ func (p *FloatingIPPlugin) Filter(pod *corev1.Pod, nodes []corev1.Node) ([]corev
 		}
 		glog.V(5).Infof("filtered nodes %v failed nodes %v", nodeNames, failedNodesMap)
 	}
+	metrics.ScheduleLatency.WithLabelValues("filter").Observe(time.Since(start).Seconds())
 	return filteredNodes, failedNodesMap, nil
 }
 
@@ -320,6 +323,7 @@ func (p *FloatingIPPlugin) allocateIP(key string, nodeName string, pod *corev1.P
 
 // Bind binds a new floatingip or reuse an old one to pod
 func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error {
+	start := time.Now()
 	pod, err := p.PluginFactoryArgs.PodLister.Pods(args.PodNamespace).Get(args.PodName)
 	if err != nil {
 		return fmt.Errorf("failed to find pod %s: %w", util.Join(args.PodName, args.PodNamespace), err)
@@ -375,6 +379,7 @@ func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error {
 		// If fails to update, depending on resync to update
 		return fmt.Errorf("update pod %s: %w", keyObj.KeyInDB, err1)
 	}
+	metrics.ScheduleLatency.WithLabelValues("bind").Observe(time.Since(start).Seconds())
 	return nil
 }
 

pkg/ipam/server/server.go

@@ -27,6 +27,8 @@ import (
 
 	"github.com/emicklei/go-restful"
 	"github.com/emicklei/go-restful-swagger12"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	corev1 "k8s.io/api/core/v1"
 	extensionClient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -47,6 +49,7 @@ import (
 	"tkestack.io/galaxy/pkg/ipam/client/clientset/versioned"
 	crdInformer "tkestack.io/galaxy/pkg/ipam/client/informers/externalversions"
 	"tkestack.io/galaxy/pkg/ipam/crd"
+	"tkestack.io/galaxy/pkg/ipam/metrics"
 	"tkestack.io/galaxy/pkg/ipam/schedulerplugin"
 	"tkestack.io/galaxy/pkg/ipam/server/options"
 	"tkestack.io/galaxy/pkg/utils/httputil"
@@ -345,6 +348,10 @@ func (s *Server) startAPIServer() {
 		Writes(httputil.Resp{Code: http.StatusOK}))
 
 	restful.Add(ws)
+	// register prometheus metrics
+	prometheus.MustRegister(s.plugin.GetIpam())
+	metrics.MustRegister()
+	restful.DefaultContainer.Handle("/metrics", promhttp.Handler())
 	addSwaggerUISupport(restful.DefaultContainer)
 	if err := http.ListenAndServe(fmt.Sprintf("%s:%d", s.Bind, s.APIPort), nil); err != nil {
 		glog.Fatalf("unable to listen: %v.", err)