Merge pull request #12 from swisscom/master

merge back updates

.circleci/config.yml

@@ -10,7 +10,7 @@ jobs:
   build:
     docker:
     # primary image for building app
-    - image: circleci/golang:1.15
+    - image: cimg/go:1.16-node
 
     # service images available at `host: localhost`
     - image: circleci/postgres:12-alpine
@@ -24,15 +24,13 @@ jobs:
         MYSQL_DATABASE: mysql
         MYSQL_ROOT_PASSWORD: my-secret-pw
 
-    - image: circleci/mongo:3.6
+    - image: circleci/mongo:4.4
       environment:
         MONGO_INITDB_ROOT_USERNAME: mongoadmin
         MONGO_INITDB_ROOT_PASSWORD: super-secret
 
     - image: circleci/redis:5.0
 
-    working_directory: /go/src/github.com/swisscom/backman
-
     steps:
     - checkout
 
@@ -41,6 +39,12 @@ jobs:
 
     # build binary
     - run: make build
+    - run:
+        name: install binary
+        command: |
+          chmod +x /home/circleci/project/backman
+          sudo cp /home/circleci/project/backman /usr/local/bin/backman
+          sudo chmod +x /usr/local/bin/backman
 
     # minIO server
     - run:
@@ -56,6 +60,13 @@ jobs:
           MINIO_ACCESS_KEY: 6d611e2d-330b-4e52-a27c-59064d6e8a62
           MINIO_SECRET_KEY: eW9sbywgeW91IGhhdmUganVzdCBiZWVuIHRyb2xsZWQh
 
+    # killall
+    - run:
+        name: install killall
+        command: |
+          sudo apt-get update
+          sudo apt-get -y install psmisc
+
     # jq
     - run:
         name: install jq
@@ -78,23 +89,20 @@ jobs:
     - run:
         name: install MongoDB client
         command: |
-          sudo apt-get install -y libcurl4
-          wget https://repo.mongodb.org/apt/debian/dists/stretch/mongodb-org/3.6/main/binary-amd64/mongodb-org-server_3.6.14_amd64.deb -qO mongodb-org-server_3.6.14_amd64.deb
-          sudo dpkg -i --ignore-depends=libcurl3 mongodb-org-server_3.6.14_amd64.deb
-          wget https://repo.mongodb.org/apt/debian/dists/stretch/mongodb-org/3.6/main/binary-amd64/mongodb-org-shell_3.6.14_amd64.deb -qO mongodb-org-shell_3.6.14_amd64.deb
-          sudo dpkg -i --ignore-depends=libcurl3 mongodb-org-shell_3.6.14_amd64.deb
-          wget https://repo.mongodb.org/apt/debian/dists/stretch/mongodb-org/4.2/main/binary-amd64/mongodb-org-tools_4.2.0_amd64.deb -qO mongodb-org-tools_4.2.0_amd64.deb
-          sudo dpkg -i --ignore-depends=libcurl3 mongodb-org-tools_4.2.0_amd64.deb
+          wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | sudo apt-key add -
+          echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/5.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-5.0.list
+          sudo apt-get update
+          sudo apt-get install -y mongodb-database-tools=100.5.0 mongodb-org-tools=5.0.2 mongodb-org-shell=5.0.2
     - run: make mongodb-test
 
-    # mysql integration tests
-    - run:
-        name: install MySQL client
-        command: sudo apt install -y mariadb-client
-    - run: make mysql-test
-
     # redis integration tests
     - run:
         name: install Redis client
         command: sudo apt install -y redis-tools
     - run: make redis-test
+
+    # mysql integration tests
+    - run:
+        name: install MySQL client
+        command: sudo apt install -y mariadb-client
+    - run: make mysql-test

Dockerfile

@@ -6,8 +6,8 @@ RUN echo "debconf debconf/frontend select noninteractive" | debconf-set-selectio
   apt-get -y $package_args update && \
   apt-get -y $package_args dist-upgrade && \
   apt-get -y $package_args install curl ca-certificates gnupg tzdata git
-RUN curl --location --output go.tar.gz "https://golang.org/dl/go1.16.3.linux-amd64.tar.gz" && \
-  echo "951a3c7c6ce4e56ad883f97d9db74d3d6d80d5fec77455c6ada6c1f7ac4776d2  go.tar.gz" | sha256sum -c  && \
+RUN curl --location --output go.tar.gz "https://golang.org/dl/go1.16.7.linux-amd64.tar.gz" && \
+  echo "7fe7a73f55ba3e2285da36f8b085e5c0159e9564ef5f63ee0ed6b818ade8ef04  go.tar.gz" | sha256sum -c  && \
   tar -C /usr/local -xzf go.tar.gz && \
   rm go.tar.gz 
 
@@ -28,12 +28,12 @@ RUN echo "debconf debconf/frontend select noninteractive" | debconf-set-selectio
   apt-get -y $package_args install curl ca-certificates gnupg tzdata
 
 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
-  curl https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - && \
+  curl https://www.mongodb.org/static/pgp/server-5.0.asc | apt-key add - && \
   echo "deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \
-  echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" > /etc/apt/sources.list.d/mongodb-org-4.4.list
+  echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/5.0 multiverse" > /etc/apt/sources.list.d/mongodb-org-5.0.list
 RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash -
 RUN apt-get -y $package_args update && \
-  apt-get -y $package_args install mysql-client postgresql-client-12 mongodb-org-tools=4.4.5 mongodb-org-shell=4.4.5 redis-tools nodejs openssh-server bash vim && \
+  apt-get -y $package_args install mysql-client postgresql-client-12 mongodb-database-tools=100.5.0 mongodb-org-tools=5.0.2 mongodb-org-shell=5.0.2 redis-tools nodejs openssh-server bash vim && \
   apt-get clean && \
   find /usr/share/doc/*/* ! -name copyright | xargs rm -rf && \
   rm -rf \

Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := run
 SHELL := /bin/bash
-APP ?= $(shell basename $$(pwd))
+APP = backman
 COMMIT_SHA = $(shell git rev-parse --short HEAD)
 DOCKER_TAG = latest
 
@@ -13,12 +13,12 @@ help:
 .PHONY: run
 ## run: runs main.go with the race detector
 run:
-	source .env; source .env_*; go run -race main.go
+	source .env; source .env_*; source .env_teams; go run -race main.go
 
 .PHONY: gin
 ## gin: runs main.go via gin (hot reloading)
 gin:
-	source .env; source .env_*; gin --all --immediate run main.go
+	source .env; source .env_*; source .env_teams;  gin --all --immediate run main.go
 
 .PHONY: build
 ## build: builds the application

README.md

@@ -49,6 +49,12 @@ These here are the default values backman will use if not configured via JSON:
 	"disable_web": false,
 	"disable_metrics": false,
 	"unprotected_metrics": false,
+	"notifications": {
+		"teams": {
+			"webhook": "https://example.webhook.office.com/webhookb2/deadbeef/IncomingWebhook/beefdead/deadbeef",
+			"events": ["backup-success", "backup-failed"]
+		}
+	},
 	"s3": {
 		"service_label": "dynstrg",
 		"encryption_key":"a_super_strong_key"
@@ -85,6 +91,8 @@ Possible JSON properties:
 - `disable_web`: optional, disable web interface and api
 - `disable_metrics`: optional, disable Prometheus metrics endpoint
 - `unprotected_metrics`: optional, disable HTTP basic auth protection for Prometheus metrics endpoint
+- `notifications.teams.webhook`: optional, setting a webhook URL will enable MS Teams notifications about backups
+- `notifications.teams.events`: optional, list of events to send a Teams notification for. Can be *backup-started*, *backup-success*, *backup-failed*. Sends a notification for all events if empty.
 - `s3.disable_ssl`: optional, S3 client connections will use HTTP instead of HTTPS
 - `s3.skip_ssl_verification`: optional, S3 client will still use HTTPS but skips certificate verification
 - `s3.service_label`: optional, defines which service label backman will look for to find the S3-compatible object storage

apt.yml

@@ -1,13 +1,14 @@
 ---
 keys:
 - https://www.postgresql.org/media/keys/ACCC4CF8.asc
-- https://www.mongodb.org/static/pgp/server-4.4.asc
+- https://www.mongodb.org/static/pgp/server-5.0.asc
 repos:
 - deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main
-- deb https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse
+- deb https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/5.0 multiverse
 packages:
 - mysql-client
 - postgresql-client-12
-- mongodb-org-tools=4.4.0
-- mongodb-org-shell=4.4.0
+- mongodb-org-shell=5.0.2
+- mongodb-org-tools=5.0.2
+- mongodb-database-tools=100.5.0
 - redis-tools

config/config.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 	"sync"
 	"time"
 )
@@ -20,9 +21,10 @@ type Config struct {
 	LoggingTimestamp   bool   `json:"logging_timestamp"`
 	Username           string
 	Password           string
-	DisableWeb         bool `json:"disable_web"`
-	DisableMetrics     bool `json:"disable_metrics"`
-	UnprotectedMetrics bool `json:"unprotected_metrics"`
+	DisableWeb         bool               `json:"disable_web"`
+	DisableMetrics     bool               `json:"disable_metrics"`
+	UnprotectedMetrics bool               `json:"unprotected_metrics"`
+	Notifications      NotificationConfig `json:"notifications"`
 	S3                 S3Config
 	Services           map[string]ServiceConfig
 	Foreground         bool
@@ -52,6 +54,15 @@ type ServiceConfig struct {
 	RestoreOptions          []string `json:"restore_options"`
 }
 
+type NotificationConfig struct {
+	Teams TeamsNotificationConfig `json:"teams,omitempty"`
+}
+
+type TeamsNotificationConfig struct {
+	Webhook string   `json:"webhook"`
+	Events  []string `json:"events"`
+}
+
 type TimeoutDuration struct {
 	time.Duration
 }
@@ -132,6 +143,12 @@ func Get() *Config {
 			if envConfig.UnprotectedMetrics {
 				config.UnprotectedMetrics = envConfig.UnprotectedMetrics
 			}
+			if len(envConfig.Notifications.Teams.Webhook) > 0 {
+				config.Notifications.Teams.Webhook = envConfig.Notifications.Teams.Webhook
+			}
+			if len(envConfig.Notifications.Teams.Events) > 0 {
+				config.Notifications.Teams.Events = envConfig.Notifications.Teams.Events
+			}
 			if envConfig.S3.DisableSSL {
 				config.S3.DisableSSL = envConfig.S3.DisableSSL
 			}
@@ -195,16 +212,32 @@ func Get() *Config {
 		}
 
 		// use username & password from env if defined
-		if len(os.Getenv("BACKMAN_USERNAME")) > 0 {
-			config.Username = os.Getenv("BACKMAN_USERNAME")
+		if os.Getenv(BackmanUsername) != "" {
+			config.Username = os.Getenv(BackmanUsername)
 		}
-		if len(os.Getenv("BACKMAN_PASSWORD")) > 0 {
-			config.Password = os.Getenv("BACKMAN_PASSWORD")
+		if os.Getenv(BackmanPassword) != "" {
+			config.Password = os.Getenv(BackmanPassword)
 		}
 
 		// use s3 encryption key from env if defined
-		if len(os.Getenv("BACKMAN_ENCRYPTION_KEY")) > 0 {
-			config.S3.EncryptionKey = os.Getenv("BACKMAN_ENCRYPTION_KEY")
+		if os.Getenv(BackmanEncryptionKey) != "" {
+			config.S3.EncryptionKey = os.Getenv(BackmanEncryptionKey)
+		}
+
+		// use teams webhook url from env if defined
+		if os.Getenv(BackmanTeamsWebhook) != "" {
+			config.Notifications.Teams.Webhook = os.Getenv(BackmanTeamsWebhook)
+		}
+
+		// use teams events configuration from env if defined
+		if os.Getenv(BackmanTeamsEvents) != "" {
+			var events []string
+			eventsString := os.Getenv(BackmanTeamsEvents)
+			if eventsString != "" {
+				events = strings.Split(eventsString, ",")
+			}
+
+			config.Notifications.Teams.Events = events
 		}
 	})
 	return &config

config/const.go

@@ -0,0 +1,7 @@
+package config
+
+const BackmanUsername = "BACKMAN_USERNAME"
+const BackmanPassword = "BACKMAN_PASSWORD"
+const BackmanEncryptionKey = "BACKMAN_ENCRYPTION_KEY"
+const BackmanTeamsWebhook = "BACKMAN_TEAMS_WEBHOOK"
+const BackmanTeamsEvents = "BACKMAN_TEAMS_EVENT"

go.mod

@@ -3,6 +3,7 @@ module github.com/swisscom/backman
 go 1.16
 
 require (
+	github.com/atc0005/go-teams-notify/v2 v2.6.0
 	github.com/cloudfoundry-community/go-cfenv v1.18.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -19,7 +20,6 @@ require (
 	github.com/prometheus/common v0.20.0 // indirect
 	github.com/robfig/cron v1.2.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.6.1 // indirect
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1 // indirect
 	golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 // indirect

go.sum

@@ -17,6 +17,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
+github.com/atc0005/go-teams-notify/v2 v2.6.0 h1:YegKDWbjlatR0fP2yHsQYXzTcUGNJXhm1/OiCgbyysc=
+github.com/atc0005/go-teams-notify/v2 v2.6.0/go.mod h1:xo6GejLDHn3tWBA181F8LrllIL0xC1uRsRxq7YNXaaY=
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
@@ -319,8 +321,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

main.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/swisscom/backman/config"
 	"github.com/swisscom/backman/log"
+	"github.com/swisscom/backman/notifications"
 	"github.com/swisscom/backman/router"
 	"github.com/swisscom/backman/scheduler"
 	"github.com/swisscom/backman/service"
@@ -18,6 +19,9 @@ func main() {
 	// init services
 	service.Get()
 
+	// init notifications manager
+	notifications.Manager()
+
 	// check if an immediate backup/restore should run, in non-background mode. otherwise continue and start scheduler
 	if runNow() {
 		return

manifest.yml

@@ -23,7 +23,7 @@ applications:
 
   # ### push either as docker image
   docker:
-    image: jamesclonk/backman:1.28.0 # choose version from https://hub.docker.com/r/jamesclonk/backman/tags, or 'latest'
+    image: jamesclonk/backman:1.28.1 # choose version from https://hub.docker.com/r/jamesclonk/backman/tags, or 'latest'
   # ### or as buildpack/src
   # buildpacks:
   # - https://github.com/cloudfoundry/apt-buildpack

notifications/events/events.go

@@ -0,0 +1,9 @@
+package events
+
+type Event string
+
+const (
+	BackupStarted Event = "backup-started"
+	BackupSuccess Event = "backup-success"
+	BackupFailed  Event = "backup-failed"
+)