Merge pull request #189 from kitabisa/development

Prep v2 (Beta)

.github/workflows/pr.yaml

@@ -15,6 +15,8 @@ on:
       - master
       - development
   pull_request:
+    branches-ignore:
+      - dependabot/**
 
 jobs:
   checks:

.github/workflows/review.yaml

@@ -1,13 +1,18 @@
 name: Review  
 on: 
+  push:
+    branches:
+      - master
   pull_request:
     branches:
-      - development 
+      - master
+      - development
 
 jobs:
   review:
     name: Code Review  
-    runs-on: ubuntu-latest  
+    runs-on: ubuntu-latest
+    if: startsWith(github.head_ref, 'dependabot/') == false
     steps:  
     - name: Check out code  
       uses: actions/checkout@v2.3.4 
@@ -16,4 +21,15 @@ jobs:
       uses: kitabisa/sonarqube-action@development 
       with: 
         host: ${{ secrets.SONARQUBE_HOST }} 
-        login: ${{ secrets.SONARQUBE_TOKEN }}
+        login: ${{ secrets.SONARQUBE_TOKEN }}
+
+    - name: Run Semgrep
+      uses: returntocorp/semgrep-action@v1
+      with:
+        config: |
+          p/ci
+          p/owasp-top-ten
+          p/golang
+          p/command-injection
+          p/security-audit
+          p/secrets

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to this project should be documented in this file.
 
+### v2.0.0-beta
+
+- Add supporting RAW requests from CVEs templates
+- Add Zinc logs engine
+- Refactoring configuration structures
+- Remove `-o/--output` & `--json` flags
+- Add custom threat rules
+
 ### v1.2.2
 
 - Add utility for get datasets

README.md

@@ -31,6 +31,12 @@
 
 [![teler](https://user-images.githubusercontent.com/25837540/97096468-f8ccaa00-1696-11eb-8830-0d3a7be45a2d.gif)](#)
 
+> ### :warning: Important notes
+> If you upgrade from prior to v2 frontwards there will be some **break changes** that affect configuration files. 
+> Appropriate adaptations can refer to [teler.example.yaml](https://github.com/kitabisa/teler/blob/master/teler.example.yaml) file.
+>
+> See the exact changes in the [CHANGELOG.md](#changes) file.
+
 ## Table of Contents
 - [Features](#features)
 - [Why teler?](#why-teler)
@@ -51,13 +57,17 @@
 
 * **Monitoring**: We've our own metrics if you want to monitor threats easily, and we use Prometheus for that.
 
+* **Logging**: is also provided in file form or sends detected threats to the Zinc logs search engine.
+
 * **Latest resources**: Collections is continuously up-to-date.
 
 * **Minimal configuration**: You can just run it against your log file, write the log format and let
   teler analyze the log and show you alerts!
 
 * **Flexible log formats**: teler allows any custom log format string! It all depends on how you write the log format in configuration file.
 
+* **Custom threat rules**: Want to reach a wider range of threats instead of engine-based _(default)_ rules? You can customize threat rules!
+
 * **Incremental log processing**: Need data persistence rather than [buffer stream](https://linux.die.net/man/1/stdbuf)?
   teler has the ability to process logs incrementally through the on-disk persistence options.
 
@@ -128,7 +138,7 @@ All external resources used in this teler are **NOT** provided by us. See all pe
 
 ## Pronunciation
 
-/télér/ bagaimana bisa seorang pemuda itu teler hanya dengan meminum 1 sloki ciu _(?)_
+[`jv_id`](https://www.localeplanet.com/java/jv-ID/index.html) • **/télér/** — bagaimana bisa seorang pemuda itu teler hanya dengan meminum sloki ciu _(?)_
 
 ## Changes
 

common/options.go

@@ -13,8 +13,7 @@ type Options struct {
 	Stdin       bool             // Stdin specifies whether stdin input was given to the process
 	Version     bool             // Version check of teler flag
 	Input       string           // Parse log from data persistence rather than buffer stream
-	Output      string           // Save detected threats to file
-	OutFile     *os.File         // Write log output into file
+	Output      *os.File         // Write log output into file
 	Configs     *parsers.Configs // Get teler configuration interface
 	JSON        bool             // Display threats in the terminal as JSON format
 	RmCache     bool             // To remove all cached resources on local

internal/alert/telegram.go

@@ -2,8 +2,8 @@ package alert
 
 import (
 	"bytes"
+	"html/template"
 	"strconv"
-	"text/template"
 
 	telegramBot "github.com/go-telegram-bot-api/telegram-bot-api"
 	"ktbs.dev/teler/pkg/errors"
@@ -31,12 +31,12 @@ func toTelegram(token string, chatID string, log map[string]string) {
 func telegramMessage(log map[string]string) string {
 	var buffer bytes.Buffer
 
-	template, err := template.ParseFiles("internal/alert/template/telegram.tmpl")
+	tpl, err := template.ParseFiles("internal/alert/template/telegram.tmpl")
 	if err != nil {
 		errors.Exit(err.Error())
 	}
 
-	err = template.Execute(&buffer, log)
+	err = tpl.Execute(&buffer, log)
 	if err != nil {
 		errors.Exit(err.Error())
 	}

internal/runner/logs.go

@@ -0,0 +1,48 @@
+package runner
+
+import (
+	"fmt"
+	"reflect"
+
+	"ktbs.dev/teler/common"
+	"ktbs.dev/teler/pkg/errors"
+	"ktbs.dev/teler/pkg/logs"
+)
+
+func log(options *common.Options, data map[string]string) {
+	m := options.Configs.Logs
+	v := reflect.ValueOf(m)
+	t := v.Type()
+
+	for i := 0; i < v.NumField(); i++ {
+		if v.Field(i).FieldByName("Active").Bool() {
+			switch t.Field(i).Name {
+			case "File":
+				toFile(options, data)
+			case "Zinc":
+				toZinc(options, data)
+			}
+		}
+	}
+}
+
+func toFile(options *common.Options, data map[string]string) {
+	err := logs.File(options, data)
+	if err != nil {
+		errors.Show(err.Error())
+	}
+}
+
+func toZinc(options *common.Options, data map[string]string) {
+	zinc := options.Configs.Logs.Zinc
+	base := "http"
+	if zinc.SSL {
+		base += "s"
+	}
+	base += fmt.Sprint("://", zinc.Host, ":", zinc.Port)
+
+	err := logs.Zinc(base, zinc.Index, zinc.Base64Auth, data)
+	if err != nil {
+		errors.Show(err.Error())
+	}
+}

internal/runner/metrics.go

@@ -0,0 +1,60 @@
+package runner
+
+import (
+	"fmt"
+	"net/http"
+	"reflect"
+	"strconv"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"ktbs.dev/teler/common"
+	"ktbs.dev/teler/pkg/errors"
+	"ktbs.dev/teler/pkg/metrics"
+)
+
+func metric(options *common.Options) {
+	m := options.Configs.Metrics
+	v := reflect.ValueOf(m)
+	t := v.Type()
+
+	for i := 0; i < v.NumField(); i++ {
+		if v.Field(i).FieldByName("Active").Bool() {
+			switch t.Field(i).Name {
+			case "Prometheus":
+				startPrometheus(options)
+			}
+		}
+	}
+}
+
+func startPrometheus(options *common.Options) {
+	p := options.Configs.Metrics.Prometheus
+
+	if p.Host == "" {
+		p.Host = "127.0.0.1"
+	}
+
+	if p.Port == 0 {
+		p.Port = 9090
+	}
+
+	if p.Endpoint == "" {
+		p.Endpoint = "/metrics"
+	}
+
+	s := fmt.Sprint(p.Host, ":", strconv.Itoa(p.Port))
+	e := p.Endpoint
+
+	go func() {
+		http.Handle(e, promhttp.Handler())
+
+		err := http.ListenAndServe(s, nil) // nosemgrep: go.lang.security.audit.net.use-tls.use-tls
+		if err != nil {
+			errors.Exit(err.Error())
+		}
+	}()
+
+	metrics.Prometheus()
+	gologger.Info().Msgf(fmt.Sprint("Listening metrics on http://", s, e))
+}

internal/runner/options.go

@@ -25,9 +25,6 @@ func ParseOptions() *common.Options {
 	flag.IntVar(&options.Concurrency, "x", 20, "")
 	flag.IntVar(&options.Concurrency, "concurrent", 20, "")
 
-	flag.StringVar(&options.Output, "o", "", "")
-	flag.StringVar(&options.Output, "output", "", "")
-
 	flag.BoolVar(&options.Version, "v", false, "")
 	flag.BoolVar(&options.Version, "version", false, "")
 
@@ -47,8 +44,6 @@ func ParseOptions() *common.Options {
 			"  -c, --config <FILE>         teler configuration file",
 			"  -i, --input <FILE>          Analyze logs from data persistence rather than buffer stream",
 			"  -x, --concurrent <i>        Set the concurrency level to analyze logs (default: 20)",
-			"  -o, --output <FILE>         Save detected threats to file",
-			"      --json                  Display threats in the terminal as JSON format",
 			"      --rm-cache              Removes all cached resources",
 			"  -v, --version               Show current teler version",
 			"",

internal/runner/runner.go

@@ -1,18 +1,13 @@
 package runner
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"os/signal"
-	"regexp"
 
-	"github.com/acarl005/stripansi"
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/gologger"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/remeh/sizedwaitgroup"
 	"github.com/satyrius/gonx"
 	"ktbs.dev/teler/common"
@@ -22,31 +17,14 @@ import (
 	"ktbs.dev/teler/pkg/teler"
 )
 
-func removeLBR(s string) string {
-	re := regexp.MustCompile(`\x{000D}\x{000A}|[\x{000A}\x{000B}\x{000C}\x{000D}\x{0085}\x{2028}\x{2029}]`)
-	return re.ReplaceAllString(s, ``)
-}
-
 // New read & pass stdin log
 func New(options *common.Options) {
-	var input *os.File
-	var out string
-	var pass int
-
-	metric, promserve, promendpoint := prometheus(options)
-	if metric {
-		go func() {
-			http.Handle(promendpoint, promhttp.Handler())
+	var (
+		input *os.File
+		pass  int
+	)
 
-			err := http.ListenAndServe(promserve, nil)
-			if err != nil {
-				errors.Exit(err.Error())
-			}
-		}()
-
-		metrics.Init()
-		gologger.Info().Msgf("Listening metrics on http://" + promserve + promendpoint)
-	}
+	go metric(options)
 
 	jobs := make(chan *gonx.Entry)
 	gologger.Info().Msg("Analyzing...")
@@ -64,48 +42,25 @@ func New(options *common.Options) {
 	con := options.Concurrency
 	swg := sizedwaitgroup.New(con)
 	go func() {
-		for log := range jobs {
+		for job := range jobs {
 			swg.Add()
 			go func(line *gonx.Entry) {
 				defer swg.Done()
 
 				threat, obj := teler.Analyze(options, line)
-
 				if threat {
-					if metric {
-						metrics.GetThreatTotal.WithLabelValues(obj["category"]).Inc()
-					}
-
-					if options.JSON {
-						json, err := json.Marshal(obj)
-						if err != nil {
-							errors.Exit(err.Error())
-						}
-						out = fmt.Sprintf("%s\n", json)
-					} else {
-						out = fmt.Sprintf("[%s] [%s] [%s] %s\n",
-							aurora.Cyan(obj["time_local"]),
-							aurora.Green(obj["remote_addr"]),
-							aurora.Yellow(obj["category"]),
-							aurora.Red(obj[obj["element"]]),
-						)
-					}
-
-					fmt.Print(out)
-
-					if options.Output != "" {
-						if !options.JSON {
-							out = stripansi.Strip(out)
-						}
-
-						if _, write := options.OutFile.WriteString(out); write != nil {
-							errors.Show(write.Error())
-						}
-					}
+					fmt.Printf("[%s] [%s] [%s] %s\n",
+						aurora.Cyan(obj["time_local"]),
+						aurora.Green(obj["remote_addr"]),
+						aurora.Yellow(obj["category"]),
+						aurora.Red(obj[obj["element"]]),
+					)
 
 					alert.New(options, common.Version, obj)
+					log(options, obj)
+					metrics.PrometheusInsert(obj)
 				}
-			}(log)
+			}(job)
 		}
 	}()
 

internal/runner/utils.go

@@ -0,0 +1,8 @@
+package runner
+
+import "regexp"
+
+func removeLBR(s string) string {
+	re := regexp.MustCompile(`\x{000D}\x{000A}|[\x{000A}\x{000B}\x{000C}\x{000D}\x{0085}\x{2028}\x{2029}]`)
+	return re.ReplaceAllString(s, ``)
+}