crng-ooo-analyzer (WIP)

cfg/cfg.go

@@ -1,8 +1,13 @@
 package cfg
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
 	"time"
 
+	"github.com/BurntSushi/toml"
 	"github.com/grafana/carbon-relay-ng/validate"
 	m20 "github.com/metrics20/go-metrics20/carbon20"
 )
@@ -33,7 +38,7 @@ type Config struct {
 	Rewriter                []Rewriter
 }
 
-func NewConfig() Config {
+func New() Config {
 	return Config{
 		Plain_read_timeout: Duration{
 			2 * time.Minute,
@@ -46,6 +51,40 @@ func NewConfig() Config {
 	}
 }
 
+func NewFromFile(path string) (Config, toml.MetaData, error) {
+	config := New()
+	var meta toml.MetaData
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return config, meta, fmt.Errorf("Couldn't read config file %q: %s", path, err.Error())
+	}
+
+	dataStr := os.Expand(string(data), expandVars)
+	meta, err = toml.Decode(dataStr, &config)
+	if err != nil {
+		return config, meta, fmt.Errorf("Invalid config file %q: %s", path, err.Error())
+	}
+	return config, meta, nil
+}
+
+func expandVars(in string) (out string) {
+	switch in {
+	case "HOST":
+		hostname, _ := os.Hostname()
+		// in case hostname is an fqdn or has dots, only take first part
+		parts := strings.SplitN(hostname, ".", 2)
+		return parts[0]
+	case "GRAFANA_NET_ADDR":
+		return os.Getenv("GRAFANA_NET_ADDR")
+	case "GRAFANA_NET_API_KEY":
+		return os.Getenv("GRAFANA_NET_API_KEY")
+	case "GRAFANA_NET_USER_ID":
+		return os.Getenv("GRAFANA_NET_USER_ID")
+	default:
+		return "$" + in
+	}
+}
+
 type Duration struct {
 	time.Duration
 }

cmd/carbon-relay-ng/carbon-relay-ng.go

@@ -6,7 +6,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net"
 	_ "net/http/pprof"
 	"os"
@@ -15,7 +14,6 @@ import (
 	"runtime/pprof"
 	"syscall"
 
-	"github.com/BurntSushi/toml"
 	"github.com/Dieterbe/go-metrics"
 	"github.com/grafana/carbon-relay-ng/aggregator"
 	"github.com/grafana/carbon-relay-ng/badmetrics"
@@ -37,7 +35,6 @@ import (
 
 var (
 	config_file      string
-	config           = cfg.NewConfig()
 	to_dispatch      = make(chan []byte)
 	inputs           []input.Plugin
 	shutdownTimeout  = time.Second * 30 // how long to wait for shutdown
@@ -59,34 +56,6 @@ func usage() {
 	flag.PrintDefaults()
 }
 
-func readConfigFile(config_file string) string {
-	data, err := ioutil.ReadFile(config_file)
-	if err != nil {
-		log.Fatalf("Couldn't read config file %q: %s", config_file, err.Error())
-	}
-
-	return os.Expand(string(data), expandVars)
-
-}
-
-func expandVars(in string) (out string) {
-	switch in {
-	case "HOST":
-		hostname, _ := os.Hostname()
-		// in case hostname is an fqdn or has dots, only take first part
-		parts := strings.SplitN(hostname, ".", 2)
-		return parts[0]
-	case "GRAFANA_NET_ADDR":
-		return os.Getenv("GRAFANA_NET_ADDR")
-	case "GRAFANA_NET_API_KEY":
-		return os.Getenv("GRAFANA_NET_API_KEY")
-	case "GRAFANA_NET_USER_ID":
-		return os.Getenv("GRAFANA_NET_USER_ID")
-	default:
-		return "$" + in
-	}
-}
-
 func main() {
 
 	flag.Usage = usage
@@ -104,10 +73,9 @@ func main() {
 		config_file = val
 	}
 
-	config_str := readConfigFile(config_file)
-	meta, err := toml.Decode(config_str, &config)
+	config, meta, err := cfg.NewFromFile(config_file)
 	if err != nil {
-		log.Fatalf("Invalid config file %q: %s", config_file, err.Error())
+		log.Fatal(err)
 	}
 	//runtime.SetBlockProfileRate(1) // to enable block profiling. in my experience, adds 35% overhead.
 

cmd/crng-ooo-analyzer/main.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	_ "net/http/pprof"
+	"os"
+	"regexp"
+	"runtime"
+
+	"github.com/grafana/carbon-relay-ng/cfg"
+	"github.com/grafana/carbon-relay-ng/logger"
+	log "github.com/sirupsen/logrus"
+)
+
+var Version = "unknown"
+
+func usage() {
+	header := `Usage:
+        crng-ooo-analyzer version
+        crng-ooo-analyzer <path-to-config>
+	`
+	fmt.Fprintln(os.Stderr, header)
+	flag.PrintDefaults()
+}
+
+func main() {
+
+	flag.Usage = usage
+	flag.Parse()
+
+	config_file := "/etc/carbon-relay-ng.ini"
+	if 1 == flag.NArg() {
+		val := flag.Arg(0)
+		if val == "version" {
+			fmt.Printf("crng-ooo-analyzer %s (built with %s)\n", Version, runtime.Version())
+			return
+		}
+		config_file = val
+	}
+
+	config, _, err := cfg.NewFromFile(config_file)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	formatter := &logger.TextFormatter{}
+	formatter.TimestampFormat = "2006-01-02 15:04:05.000"
+	log.SetFormatter(formatter)
+	lvl, err := log.ParseLevel(config.Log_level)
+	if err != nil {
+		log.Fatalf("failed to parse log-level %q: %s", config.Log_level, err.Error())
+	}
+	log.SetLevel(lvl)
+
+	var revRe []*regexp.Regexp
+
+	for _, agg := range config.Aggregation {
+		r, err := reverseRegex(agg)
+		if err != nil {
+			log.Fatal(err)
+		}
+		revRe = append(revRe, r)
+	}
+
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		fmt.Printf("%s", scanner.Text())
+		for i, rev := range revRe {
+			if rev.MatchString(scanner.Text()) {
+				fmt.Printf(" %d", i)
+			}
+		}
+		fmt.Println()
+	}
+
+	if scanner.Err() != nil {
+		log.Fatal(scanner.Err())
+	}
+}

cmd/crng-ooo-analyzer/regexp.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"regexp/syntax"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/grafana/carbon-relay-ng/cfg"
+)
+
+type group struct {
+	Num  int
+	Name string
+	Patt string
+}
+
+// groups returns the matching groups from a parsed regex
+func groups(re *syntax.Regexp) []group {
+	var g []group
+	for _, s := range re.Sub {
+		if s.Op == syntax.OpCapture {
+			g = append(g, group{
+				Num:  s.Cap,
+				Name: s.Name,
+				Patt: s.Sub0[0].String(),
+			})
+		}
+	}
+	return g
+}
+
+// reverseRegex returns the "reverse regex" of an aggregation:
+// the format of an aggregator turned into a regex by replacing the groups with the capture groups from the aggregations regex.
+//
+// to know whether a metrickey may be the output of an aggregation, we must:
+// * fill in the sub-patterns from the groups into the format, aka the reverseRegex
+// * verify whether the metric key matches formatRegex
+// note: as we don't know what the input metric was, we cannot match anything against the original regex
+// for example:
+// consider this aggregation:
+// regex: '^stats.([^\.]+)\.(bar)\.[^\.]+*\.(?P<path>[a-zA-Z]+).count'
+// format: 'aggstats.$1.$2.$path.count'
+// let's say we encounter a metric with the name 'aggstats.foo.foo.somepath.count'
+// we want to know if it could possibly have been generated through this aggregation
+// we rephrase the question by expanding the format:
+// * replace literal '.' with '\.'
+// * fill in the groups by their patterns
+// result: aggstats\.[^\.]+\.bar\.[a-zA-Z]+\.count
+// does the metric match this regex? If so, than it *may* have been generated by this aggregation
+func reverseRegex(agg cfg.Aggregation) (*regexp.Regexp, error) {
+	re, err := syntax.Parse(agg.Regex, syntax.Perl)
+	if err != nil {
+		return nil, err
+	}
+	groups := groups(re)
+	escFormat := strings.Replace(agg.Format, ".", `\.`, -1)
+
+	formatRegex, err := expand(nil, escFormat, groups)
+	if err != nil {
+		return nil, err
+	}
+
+	return regexp.Compile(string(formatRegex))
+}
+
+// expand parses all the leading variables
+// based on stdlib regexp.Regexp.Expand
+func expand(dst []byte, template string, groups []group) ([]byte, error) {
+	for len(template) > 0 {
+		i := strings.Index(template, "$")
+		if i < 0 {
+			break
+		}
+		dst = append(dst, template[:i]...)
+		template = template[i:]
+		if len(template) > 1 && template[1] == '$' {
+			// Treat $$ as $.
+			dst = append(dst, '$')
+			template = template[2:]
+			continue
+		}
+		name, num, rest, ok := extract(template)
+		if !ok {
+			fmt.Println("Malformed variable syntax. treat $ as raw text")
+			dst = append(dst, '$')
+			template = template[1:]
+			continue
+		}
+		template = rest
+		var found bool
+		if num >= 0 {
+			for _, g := range groups {
+				if g.Num == num {
+					dst = append(dst, []byte(g.Patt)...)
+					found = true
+					break
+				}
+			}
+		} else {
+			for _, g := range groups {
+				if g.Name == name {
+					dst = append(dst, []byte(g.Patt)...)
+					found = true
+					break
+				}
+			}
+		}
+		if !found {
+			return dst, fmt.Errorf("could not find variable %s/%d", name, num)
+		}
+	}
+	dst = append(dst, template...)
+	return dst, nil
+}
+
+// extract returns the name from a leading "$name" or "${name}" in str.
+// If it is a number, extract returns num set to that number; otherwise num = -1.
+// taken from on stdlib regexp.extract()
+func extract(str string) (name string, num int, rest string, ok bool) {
+	if len(str) < 2 || str[0] != '$' {
+		return
+	}
+	brace := false
+	if str[1] == '{' {
+		brace = true
+		str = str[2:]
+	} else {
+		str = str[1:]
+	}
+	i := 0
+	for i < len(str) {
+		rune, size := utf8.DecodeRuneInString(str[i:])
+		if !unicode.IsLetter(rune) && !unicode.IsDigit(rune) && rune != '_' {
+			break
+		}
+		i += size
+	}
+	if i == 0 {
+		// empty name is not okay
+		return
+	}
+	name = str[:i]
+	if brace {
+		if i >= len(str) || str[i] != '}' {
+			// missing closing brace
+			return
+		}
+		i++
+	}
+
+	// Parse number.
+	num = 0
+	for i := 0; i < len(name); i++ {
+		if name[i] < '0' || '9' < name[i] || num >= 1e8 {
+			num = -1
+			break
+		}
+		num = num*10 + int(name[i]) - '0'
+	}
+	// Disallow leading zeros.
+	if name[0] == '0' && len(name) > 1 {
+		num = -1
+	}
+
+	rest = str[i:]
+	ok = true
+	return
+
+}