Merge pull request #14 from maier/master

v1.0.0-beta.1

CHANGELOG.md

@@ -1,3 +1,12 @@
+# v1.0.0-beta.1
+
+* UPDATE - stream tag only release
+* add: stream tag support
+* upd: log id in tag `log_id`
+* add: support `tags` attribute to metric stanza in configuration
+* add: support interpolation of named regex patterns in `tags` attribute
+* upd: depdencies (cgm)
+
 # v0.6.0
 
 * Merge pull request #12 from yargevad/agent-interval

README.md

@@ -2,6 +2,8 @@
 
 A small utility for extracting metrics from log files and forwarding to Circonus.
 
+> Note: In v1 all metrics use stream tags, for non-stream tagged metrics use the v0 branch/releases.
+
 ## Install
 
 1. `mkdir -p /opt/circonus/{sbin,etc,etc/log.d}`
@@ -10,11 +12,10 @@ A small utility for extracting metrics from log files and forwarding to Circonus
 1. Create config in `/opt/circonus/etc` and log configs in `/opt/circonus/etc/log.d`
 
 ## Options
-```
-$ /opt/circonus/sbin/circonus-logwatchd -h
-```
 
-```
+```sh
+/opt/circonus/sbin/circonus-logwatchd -h
+
 Flags:
       --api-app string              [ENV: CLW_API_APP] Circonus API Token app (default "circonus-logwatch")
       --api-ca-file string          [ENV: CLW_API_CA_FILE] Circonus API CA certificate file (optional, Inside API server not using public certs)
@@ -56,6 +57,7 @@ Flags:
 Create a JSON, YAML, or TOML config in `/opt/circonus/etc/circonus-logwatch.(json|yaml|toml)`. Or, use environment variables and/or command line parameters.
 
 YAML with a StatsD destination (send metrics to local circonus-agent statsd listener)
+
 ```yaml
 ---
 app_stat_port: "33284"
@@ -113,6 +115,7 @@ JSON with a Check destination (send metrics directly to a Circonus check, a chec
 ```
 
 TOML with an Agent destination (send metrics to local circonus-agent)
+
 ```toml
 app_stat_port = "33284"
 debug = false
@@ -148,6 +151,7 @@ Create one config (JSON, YAML, or TOML) in `--log-conf-dir` for each distinct lo
 1. `metrics` a list of:
     1. `match` regular expression to identify lines and optionally extract named subexpressions for value and metric name
     1. `name` a static string to use as the metric name or a template for naming the metric if named subexpressions were used in match regex
+    1. `tags` comma separated list of k:v pairs, templating can be used accessing named subexpressions (e.g. `foo:bar,yabba:dabba` or `foo:{{.id}},bar:baz`)
     1. `type` what type of metric (all numbers are 64bit)
         * `c` counter int
         * `g` gauge int or float
@@ -156,11 +160,12 @@ Create one config (JSON, YAML, or TOML) in `--log-conf-dir` for each distinct lo
         * `s` set (ala statsd set metrics) unique string to count
         * `t` text string
 
-> NOTE:
-> * any metric which does not have a `type` will be treated as a counter.
-> * any metric which does not have a subexpression named '*Value*' (case insensitive) will be treated as a counter.
-> * named subexpressions can be used in the name template with the following syntax `{{.id}}` where `id` is the name given to a named subexpression in the match regex
-> * metric names will be prefixed with the log `id`
+### Log configuration notes
+
+* any metric which does not have a `type` will be treated as a counter.
+* any metric which does not have a subexpression named '*Value*' (case insensitive) will be treated as a counter.
+* named subexpressions can be used in the name template and tag list with the following syntax `{{.id}}` where `id` is the name given to a named subexpression in the match regex
+* metrics will have a stream tag added for the log `id` (e.g. for a log with an id of "foo" the tag would be `log_id:foo`)
 
 ## Manual build
 

etc/log.d/apache.yaml

@@ -13,9 +13,11 @@ metrics:
     type: h
   # latency histogram by request path
   - match: ' (?P<path>/[^ ]*) HTTP.+tm:(?P<value>[0-9.]+)'
-    name: '{{.path}}`latency'
+    name: 'latency'
+    tags: 'path:{{.path}}'
     type: h
   # latency histogram by request path and specific request methods
   - match: '(?P<method>(GET|POST|PUT)) (?P<path>/[^ ]*) HTTP.+tm:(?P<value>[0-9.]+)'
-    name: '{{.path}}`{{.method}}`latency'
+    name: 'latency'
+    tags: 'path:{{.path}},method:{{.method}}'
     type: h

etc/log.d/system.yaml

@@ -5,5 +5,6 @@ log_file: /var/log/system.log
 metrics:
     # Static name
     - match: '(?P<app>[A-Za-z]+)\[[0-9]+\]:\s+Unlocked user...'
-      name: '{{.app}}`unlocked`user'
+      name: 'unlocked_user'
+      tags: 'app:{{.app}},foo:bar'
       type: c

go.mod

@@ -4,8 +4,7 @@ go 1.14
 
 require (
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
-	github.com/circonus-labs/circonus-gometrics/v3 v3.0.0
-	github.com/hashicorp/go-retryablehttp v0.6.6 // indirect
+	github.com/circonus-labs/circonus-gometrics/v3 v3.0.1
 	github.com/hpcloud/tail v1.0.0
 	github.com/maier/go-appstats v0.2.0
 	github.com/pelletier/go-toml v1.8.0

go.sum

@@ -29,12 +29,12 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/circonus-labs/circonus-gometrics/v3 v3.0.0 h1:5hbWwgfrYaSNCe+eKPGo457QiCe4T5+CfvY+bFGZBNw=
-github.com/circonus-labs/circonus-gometrics/v3 v3.0.0/go.mod h1:1K33fx/wP96fC1uc7ZEp3BoLafJhL0kpjejIXpQHyLM=
-github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA=
-github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
-github.com/circonus-labs/go-apiclient v0.6.6 h1:jAlCy1sCFkdPunczQahfV17g6qwhpCNqTdl9biRnUt4=
-github.com/circonus-labs/go-apiclient v0.6.6/go.mod h1:4LZCGA0xxWyajgjb/mMIoKjnyR4dAx+UmrhBaE0lOio=
+github.com/circonus-labs/circonus-gometrics/v3 v3.0.1 h1:p87LDpIhpCbskOthl6yzfT9Qcgfiwk49CKerZg9mpUU=
+github.com/circonus-labs/circonus-gometrics/v3 v3.0.1/go.mod h1:hbHb81YGFfRAgDZHE8J5kws/aDP1D40tkaTnhVfnqSw=
+github.com/circonus-labs/circonusllhist v0.1.4 h1:G5qJPuD16akpIXMUR7KcfBvrQOVm95+qyqUm+SEAZks=
+github.com/circonus-labs/circonusllhist v0.1.4/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
+github.com/circonus-labs/go-apiclient v0.7.6 h1:Pr69I76ReDKRKe/yb9o0lKphRQ/a6Jr8XLh7M4GZrPY=
+github.com/circonus-labs/go-apiclient v0.7.6/go.mod h1:RP/BcaTRf8MlHaMGCSuSDPGPQqyMeBxaAdwNv5CM/eQ=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -227,8 +227,8 @@ github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8=
-github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
+github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

internal/configs/main.go → internal/configs/configs.go

@@ -30,10 +30,12 @@ type Metric struct {
 	Matcher    *regexp.Regexp
 	MatchParts []string
 	Namer      *template.Template
+	Tagger     *template.Template
 	ValueKey   string
 	Match      string `json:"match" yaml:"match" toml:"match"`
 	Name       string `json:"name" yaml:"name" toml:"name"`
 	Type       string `json:"type" yaml:"type" toml:"type"`
+	Tags       string `json:"tags" toml:"tags" yaml:"tags"`
 }
 
 // Config defines a log to watch
@@ -183,41 +185,72 @@ func validMetricRules(logID string, logger zerolog.Logger, rules []*Metric) bool
 			}
 		}
 
-		// template does not contain template interpolation code, treat it as a literal metric name
-		if !strings.Contains(rule.Name, "{{.") {
-			continue
+		// name contains template interpolation code
+		if strings.Contains(rule.Name, "{{.") {
+			if len(rule.MatchParts) < 2 {
+				logger.Warn().
+					Str("log_id", logID).
+					Int("id_id", ruleID).
+					Str("match", rule.Match).
+					Str("name", rule.Name).
+					Msg("'name' expects matches, match has no named subexpressions, skipping config")
+				return false
+			}
+			templateID := fmt.Sprintf("%s:M%d-name", logID, ruleID)
+			namer, err := template.New(templateID).Parse(rule.Name)
+			if err != nil {
+				logger.Warn().
+					Err(err).
+					Str("log_id", logID).
+					Int("id_id", ruleID).
+					Str("name", rule.Name).
+					Msg("name template parse failed, skipping config")
+				return false
+			}
+			if namer == nil {
+				logger.Warn().
+					Str("log_id", logID).
+					Int("id_id", ruleID).
+					Str("name", rule.Name).
+					Msg("name template parse resulted in nil value, skipping config")
+				return false
+			}
+			rule.Namer = namer
 		}
 
-		if len(rule.MatchParts) < 2 {
-			logger.Warn().
-				Str("log_id", logID).
-				Int("id_id", ruleID).
-				Str("match", rule.Match).
-				Str("name", rule.Name).
-				Msg("'name' expects matches, match has no named subexpressions, skipping config")
-			return false
+		// tags contains template interpolation code
+		if strings.Contains(rule.Tags, "{{.") {
+			if len(rule.MatchParts) < 2 {
+				logger.Warn().
+					Str("log_id", logID).
+					Int("id_id", ruleID).
+					Str("match", rule.Match).
+					Str("tags", rule.Tags).
+					Msg("'tags' expects matches, match has no named subexpressions, skipping config")
+				return false
+			}
+			templateID := fmt.Sprintf("%s:M%d-tags", logID, ruleID)
+			tagger, err := template.New(templateID).Parse(rule.Tags)
+			if err != nil {
+				logger.Warn().
+					Err(err).
+					Str("log_id", logID).
+					Int("id_id", ruleID).
+					Str("tags", rule.Tags).
+					Msg("tags template parse failed, skipping config")
+				return false
+			}
+			if tagger == nil {
+				logger.Warn().
+					Str("log_id", logID).
+					Int("id_id", ruleID).
+					Str("tags", rule.Tags).
+					Msg("tags template parse resulted in nil value, skipping config")
+				return false
+			}
+			rule.Tagger = tagger
 		}
 
-		templateID := fmt.Sprintf("%s:M%d", logID, ruleID)
-		namer, err := template.New(templateID).Parse(rule.Name)
-		if err != nil {
-			logger.Warn().
-				Err(err).
-				Str("log_id", logID).
-				Int("id_id", ruleID).
-				Str("name", rule.Name).
-				Msg("name template parse failed, skipping config")
-			return false
-		}
-		if namer == nil {
-			logger.Warn().
-				Str("log_id", logID).
-				Int("id_id", ruleID).
-				Str("name", rule.Name).
-				Msg("name template parse resulted in nil value, skipping config")
-			return false
-		}
-		rule.Namer = namer
 	}
 
 	return true