Merge pull request #30 from Stratoscale/traceback

Traceback

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.9.3 as server
+FROM golang:1.10 as server
 WORKDIR /go/src/github.com/Stratoscale/logserver
 #
 RUN apt update && apt install libsystemd-dev

example/log1/mancala.stratolog

@@ -1,3 +1,4 @@
 {"process": 36154, "args": ["<disk: hostname=stratonode1.node.strato, ID=dce9381a-cada-434d-a1ba-4e351f4afcbb, path=/dev/sdc, type=mancala>", 0, 1], "module": "distributor", "funcName": "_updateDisksInTableStates", "exc_text": null, "name": "mancala.management.distributor.distributor", "thread": 140189066323712, "created": 1514211785.447942, "threadName": "DistributorThread", "filename": "distributor.py", "levelno": 20, "processName": "MainProcess", "pathname": "/usr/share/stratostorage/mancala_management_service.egg/mancala/management/distributor/distributor.py", "lineno": 162, "msg": "data disk %s was found in distributionID:%s table version:%s, setting inTable=True", "levelname": "INFO"}
 {"process": 36154, "args": ["<disk: hostname=stratonode2.node.strato, ID=2d03c436-c197-464f-9ad0-d861e650cd61, path=/dev/sdc, type=mancala>", 0, 1], "module": "distributor", "funcName": "_updateDisksInTableStates", "exc_text": null, "name": "mancala.management.distributor.distributor", "thread": 140189066323712, "created": 1514211785.448366, "threadName": "DistributorThread", "filename": "distributor.py", "levelno": 20, "processName": "MainProcess", "pathname": "/usr/share/stratostorage/mancala_management_service.egg/mancala/management/distributor/distributor.py", "lineno": 162, "msg": "data disk %s was found in distributionID:%s table version:%s, setting inTable=True", "levelname": "INFO"}
 {"process": 36154, "args": ["<disk: hostname=stratonode0.node.strato, ID=f3d510c7-1185-4942-b349-0de055165f78, path=/dev/sdc, type=mancala>", 0, 1], "module": "distributor", "funcName": "_updateDisksInTableStates", "exc_text": null, "name": "mancala.management.distributor.distributor", "thread": 140189066323712, "created": 1514211785.448693, "threadName": "DistributorThread", "filename": "distributor.py", "levelno": 20, "processName": "MainProcess", "pathname": "/usr/share/stratostorage/mancala_management_service.egg/mancala/management/distributor/distributor.py", "lineno": 162, "msg": "data disk %s was found in distributionID:%s table version:%s, setting inTable=True", "levelname": "INFO"}
+{"process": 36154, "exc_text": "Traceback (most recent call last):\n  File \"a.py\", line 4, in <module>\n    a()\n  File \"a.py\", line 2, in \n    raise Exception()\nException", "name": "mancala.management.distributor.distributor", "thread": 140189066323712, "created": 1514211785.448693, "threadName": "DistributorThread", "filename": "distributor.py", "levelno": 20, "processName": "MainProcess", "pathname": "/usr/share/stratostorage/mancala_management_service.egg/mancala/management/distributor/distributor.py", "lineno": 162, "msg": "Failed", "levelname": "ERROR"}

example/logserver.json

@@ -34,7 +34,8 @@
         "args": "args",
 	    "thread": "threadName",
 	    "path": "pathname",
-	    "lineno": "lineno"
+	    "lineno": "lineno",
+        "traceback": "exc_text"
       },
       "time_formats": [
         "unix_float"
@@ -55,10 +56,10 @@
     "open_journal": "/journal"
   },
   "global": {
-    "exclude_extensions" : [
+    "exclude_extensions": [
       ".bin"
     ],
-    "exclude_dirs" : [
+    "exclude_dirs": [
       "lttng"
     ]
   }

main_test.go

@@ -90,6 +90,18 @@ func TestHandler(t *testing.T) {
 							LineNo:   162,
 							Path:     "/usr/share/stratostorage/mancala_management_service.egg/mancala/management/distributor/distributor.py",
 						},
+						{
+							Msg:      "Failed\nTraceback (most recent call last):\n  File \"a.py\", line 4, in <module>\n    a()\n  File \"a.py\", line 2, in \n    raise Exception()\nException",
+							Level:    "ERROR",
+							Time:     mustParseTime("2017-12-25T16:23:05+02:00"),
+							FS:       "node1",
+							FileName: "mancala.stratolog",
+							Line:     4,
+							Offset:   2097,
+							Thread:   "DistributorThread",
+							LineNo:   162,
+							Path:     "/usr/share/stratostorage/mancala_management_service.egg/mancala/management/distributor/distributor.py",
+						},
 					},
 				},
 				{
@@ -269,7 +281,7 @@ func TestHandler(t *testing.T) {
 							Key:       "mancala.stratolog",
 							Path:      engine.Path{"mancala.stratolog"},
 							IsDir:     false,
-							Instances: []engine.FileInstance{{Size: 2100, FS: "node1"}},
+							Instances: []engine.FileInstance{{Size: 2672, FS: "node1"}},
 						},
 						{
 							Key:   "service1.log",

parse/parse.go

@@ -20,18 +20,21 @@ const (
 	KeyThreadName = "thread"
 	KeyPathName   = "path"
 	KeyLineNo     = "lineno"
+	KeyTraceback  = "traceback"
 )
 
 // noParserAfter determines how many line should be parsed before choosing a no-parser
 // for a specific file
-const noParserAfter = 1000
+const noParserAfter = 200
 
 type Config struct {
 	Glob        string            `json:"glob"`
 	JsonMapping map[string]string `json:"json_mapping"`
 	Regexp      string            `json:"regexp"`
 	TimeFormats []string          `json:"time_formats"`
-	AppendArgs  bool              `json:"append_args"`
+	// For JSON mapping
+	// Add key=val to message with all unused key values of json
+	AppendArgs bool `json:"append_args"`
 }
 
 type Parse []parser
@@ -162,33 +165,38 @@ func (p *parser) parseJson(line []byte) *Log {
 		log.injectArgs(j[jsonKey])
 	}
 
-	if p.AppendArgs {
-		log.Msg += argsToMessage(j)
+	// append traceback
+	if jsonKey, ok := p.JsonMapping[KeyTraceback]; ok {
+		if tb, ok := j[jsonKey]; ok {
+			if tbStr, ok := tb.(string); ok {
+				log.Msg = log.Msg + "\n" + tbStr
+			}
+			delete(j, jsonKey)
+		}
 	}
 
 	if jsonKey, ok := p.JsonMapping[KeyThreadName]; ok {
-		if log.Thread, ok = j[jsonKey].(string); !ok {
-			return nil
-		}
+		log.Thread, _ = j[jsonKey].(string)
 		delete(j, jsonKey)
 	}
 
 	if jsonKey, ok := p.JsonMapping[KeyPathName]; ok {
-		if log.Path, ok = j[jsonKey].(string); !ok {
-			return nil
-		}
+		log.Path, _ = j[jsonKey].(string)
 		delete(j, jsonKey)
 	}
 
 	if jsonKey, ok := p.JsonMapping[KeyLineNo]; ok {
-		var floatLineNo float64
-		if floatLineNo, ok = j[jsonKey].(float64); !ok {
-			return nil
+		if ln, ok := j[jsonKey].(float64); ok {
+			log.LineNo = int(ln)
 		}
-		log.LineNo = int(floatLineNo)
 		delete(j, jsonKey)
 	}
 
+	// append all additional not used key-values
+	if p.AppendArgs {
+		log.Msg += argsToMessage(j)
+	}
+
 	return log
 }