diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..cbcae0d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,66 @@ +name: Release + +on: + push: + branches: + - main + +jobs: + verify: + name: Test + runs-on: ubuntu-latest + services: + postgres: + image: postgres:12-alpine + env: + POSTGRES_DB: postmand + POSTGRES_USER: test + POSTGRES_PASSWORD: test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: 1.16 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Set cache + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Get dependencies + run: go mod download + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + args: -E gosec + + - name: Run Tests + env: + POSTMAND_TEST_DATABASE_URL: "postgres://test:test@localhost:5432/postmand?sslmode=disable" + run: make db-test-migrate && go test -covermode=count -coverprofile=count.out -v ./... + + release-please: + needs: verify + runs-on: ubuntu-latest + steps: + - uses: GoogleCloudPlatform/release-please-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + release-type: simple + package-name: postmand diff --git a/.gitignore b/.gitignore index 62e64fd..401e50f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,3 @@ migrate.linux-amd64 # dot env .env - -# goreleaser -dist/ diff --git a/.goreleaser.yml b/.goreleaser.yml deleted file mode 100644 index 285173e..0000000 --- a/.goreleaser.yml +++ /dev/null @@ -1,29 +0,0 @@ -before: - hooks: - - go mod tidy -builds: - - env: - - CGO_ENABLED=0 - goos: - - linux - - windows - - darwin - main: ./cmd/postmand/main.go - binary: postmand -archives: - - replacements: - darwin: Darwin - linux: Linux - windows: Windows - 386: i386 - amd64: x86_64 -checksum: - name_template: "checksums.txt" -snapshot: - name_template: "{{ .Tag }}-next" -changelog: - sort: asc - filters: - exclude: - - "^docs:" - - "^test:" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/Makefile b/Makefile index 13b1124..6e9f4e7 100644 --- a/Makefile +++ b/Makefile @@ -26,4 +26,10 @@ db-migrate: download-golang-migrate-binary db-test-migrate: download-golang-migrate-binary ./migrate.$(PLATFORM)-amd64 -source file://db/migrations -database ${POSTMAND_TEST_DATABASE_URL} up -.PHONY: lint test mock download-golang-migrate-binary db-migrate db-test-migrate +run-server: + go run cmd/postmand/main.go server + +run-worker: + go run cmd/postmand/main.go worker + +.PHONY: lint test mock download-golang-migrate-binary db-migrate db-test-migrate run-server run-worker diff --git a/README.md b/README.md index f75a99d..39cd730 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,238 @@ # postmand +[![Build Status](https://github.com/allisson/postmand/workflows/release/badge.svg)](https://github.com/allisson/postmand/actions) +[![Go Report Card](https://goreportcard.com/badge/github.com/allisson/postmand)](https://goreportcard.com/report/github.com/allisson/postmand) +[![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/allisson/postmand) +[![Docker Image](https://img.shields.io/docker/cloud/build/allisson/postmand)](https://hub.docker.com/r/allisson/postmand) + Simple webhook delivery system powered by Golang and PostgreSQL. + +## Features + +- Simple rest api with only three endpoints (webhooks/deliveries/delivery-attempts). +- Select the status codes that are considered valid for a delivery. +- Control the maximum amount of delivery attempts and delay between these attempts (min and max backoff). +- Locks control of worker deliveries using PostgreSQL SELECT FOR UPDATE SKIP LOCKED. +- Sending the X-Hub-Signature header if the webhook is configured with a secret token. +- Simplicity, it does the minimum necessary, it will not have authentication/permission scheme among other things, the idea is to use it internally in the cloud and not leave exposed. + +## Quickstart + +Let's start with the basic concepts, we have three main entities that we must know to start: + +- Webhook: The configuration of the webhook. +- Delivery: The content sent to a webhook. +- Delivery Attempt: An attempt to deliver the content to the webhook. + +### Run the server + +To run the server it is necessary to have a database available from postgresql, in this example we will consider that we have a database called postmand running in localhost with user and password equal to user. + +#### Docker + +```bash +docker run --rm --env POSTMAND_DATABASE_URL='postgres://user:password@host.docker.internal:5432/postmand?sslmode=disable' allisson/postmand migrate # create database schema +``` + +```bash +docker run -p 8000:8000 -p 8001:8001 --env POSTMAND_DATABASE_URL='postgres://user:password@host.docker.internal:5432/postmand?sslmode=disable' allisson/postmand server # run the server +``` + +#### Local + +```bash +git clone https://github.com/allisson/postmand +cd postmand +cp local.env .env # and edit .env +make run-migrate # create database schema +make run-server # run the server +``` + +### Create a new webhook + +The fields delivery_attempt_timeout/retry_min_backoff/retry_max_backoff are in seconds. + +```bash +curl --location --request POST 'http://localhost:8000/v1/webhooks' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "Httpbin Post", + "url": "https://httpbin.org/post", + "content_type": "application/json", + "valid_status_codes": [ + 200, + 201 + ], + "secret_token": "my-secret-token", + "active": true, + "max_delivery_attempts": 5, + "delivery_attempt_timeout": 1, + "retry_min_backoff": 10, + "retry_max_backoff": 60 +}' +``` + +```javascript +{ + "id":"a6e9a525-ac5a-488c-b118-bd7327ce6d8d", + "name":"Httpbin Post", + "url":"https://httpbin.org/post", + "content_type":"application/json", + "valid_status_codes":[ + 200, + 201 + ], + "secret_token":"my-secret-token", + "active":true, + "max_delivery_attempts":5, + "delivery_attempt_timeout":1, + "retry_min_backoff":10, + "retry_max_backoff":60, + "created_at":"2021-03-08T20:41:25.433671Z", + "updated_at":"2021-03-08T20:41:25.433671Z" +} +``` + +### Create a new delivery + +```bash +curl --location --request POST 'http://localhost:8000/v1/deliveries' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "webhook_id": "a6e9a525-ac5a-488c-b118-bd7327ce6d8d", + "payload": "{\"success\": true}" +}' +``` + +```javascript +{ + "id":"bc76122c-e56b-45c7-8dc3-b80a861191d5", + "webhook_id":"a6e9a525-ac5a-488c-b118-bd7327ce6d8d", + "payload":"{\"success\": true}", + "scheduled_at":"2021-03-08T20:43:49.986771Z", + "delivery_attempts":0, + "status":"pending", + "created_at":"2021-03-08T20:43:49.986771Z", + "updated_at":"2021-03-08T20:43:49.986771Z" +} +``` + +### Run the worker + +The worker is responsible to delivery content to the webhooks. + +#### Docker + +```bash +docker run --env POSTMAND_DATABASE_URL='postgres://user:pass@host.docker.internal:5432/postmand?sslmode=disable' allisson/postmand worker +{"level":"info","ts":1615236411.115703,"caller":"service/worker.go:74","msg":"worker-started"} +{"level":"info","ts":1615236411.1158803,"caller":"http/server.go:60","msg":"http-server-listen-and-server"} +{"level":"info","ts":1615236411.687701,"caller":"service/worker.go:42","msg":"worker-delivery-attempt-created","id":"d72719d6-5a79-4df7-a2c2-2029ab0e1848","webhook_id":"a6e9a525-ac5a-488c-b118-bd7327ce6d8d","delivery_id":"bc76122c-e56b-45c7-8dc3-b80a861191d5","response_status_code":200,"execution_duration":547,"success":true} +``` + +#### Local + +```bash +make run-worker +go run cmd/postmand/main.go worker +{"level":"info","ts":1615236411.115703,"caller":"service/worker.go:74","msg":"worker-started"} +{"level":"info","ts":1615236411.1158803,"caller":"http/server.go:60","msg":"http-server-listen-and-server"} +{"level":"info","ts":1615236411.687701,"caller":"service/worker.go:42","msg":"worker-delivery-attempt-created","id":"d72719d6-5a79-4df7-a2c2-2029ab0e1848","webhook_id":"a6e9a525-ac5a-488c-b118-bd7327ce6d8d","delivery_id":"bc76122c-e56b-45c7-8dc3-b80a861191d5","response_status_code":200,"execution_duration":547,"success":true} +``` + +### Get deliveries + +```bash +curl --location --request GET 'http://localhost:8000/v1/deliveries?webhook_id=a6e9a525-ac5a-488c-b118-bd7327ce6d8d' +``` + +```javascript +{ + "deliveries":[ + { + "id":"bc76122c-e56b-45c7-8dc3-b80a861191d5", + "webhook_id":"a6e9a525-ac5a-488c-b118-bd7327ce6d8d", + "payload":"{\"success\": true}", + "scheduled_at":"2021-03-08T20:43:49.986771Z", + "delivery_attempts":1, + "status":"succeeded", + "created_at":"2021-03-08T20:43:49.986771Z", + "updated_at":"2021-03-08T20:46:51.674623Z" + } + ], + "limit":50, + "offset":0 +} +``` + +### Get delivery + +```bash +curl --location --request GET 'http://localhost:8000/v1/deliveries/bc76122c-e56b-45c7-8dc3-b80a861191d5' +``` + +```javascript +{ + "id":"bc76122c-e56b-45c7-8dc3-b80a861191d5", + "webhook_id":"a6e9a525-ac5a-488c-b118-bd7327ce6d8d", + "payload":"{\"success\": true}", + "scheduled_at":"2021-03-08T20:43:49.986771Z", + "delivery_attempts":1, + "status":"succeeded", + "created_at":"2021-03-08T20:43:49.986771Z", + "updated_at":"2021-03-08T20:46:51.674623Z" +} +``` + +### Get delivery attempts + +```bash +curl --location --request GET 'http://localhost:8000/v1/delivery-attempts?delivery_id=bc76122c-e56b-45c7-8dc3-b80a861191d5' +``` + +```javascript +{ + "delivery_attempts":[ + { + "id":"d72719d6-5a79-4df7-a2c2-2029ab0e1848", + "webhook_id":"a6e9a525-ac5a-488c-b118-bd7327ce6d8d", + "delivery_id":"bc76122c-e56b-45c7-8dc3-b80a861191d5", + "raw_request":"POST /post HTTP/1.1\r\nHost: httpbin.org\r\nContent-Type: application/json\r\nX-Hub-Signature: 3fc5d4b8ff4efb404be24faf543667d29902d6a1306bd0c1ef2084497300cee9\r\n\r\n{\"success\": true}", + "raw_response":"HTTP/2.0 200 OK\r\nContent-Length: 538\r\nAccess-Control-Allow-Credentials: true\r\nAccess-Control-Allow-Origin: *\r\nContent-Type: application/json\r\nDate: Mon, 08 Mar 2021 20:46:51 GMT\r\nServer: gunicorn/19.9.0\r\n\r\n{\n \"args\": {}, \n \"data\": \"{\\\"success\\\": true}\", \n \"files\": {}, \n \"form\": {}, \n \"headers\": {\n \"Accept-Encoding\": \"gzip\", \n \"Content-Length\": \"17\", \n \"Content-Type\": \"application/json\", \n \"Host\": \"httpbin.org\", \n \"User-Agent\": \"Go-http-client/2.0\", \n \"X-Amzn-Trace-Id\": \"Root=1-60468d3b-36d312777a03ec3e1c564e3b\", \n \"X-Hub-Signature\": \"3fc5d4b8ff4efb404be24faf543667d29902d6a1306bd0c1ef2084497300cee9\"\n }, \n \"json\": {\n \"success\": true\n }, \n \"origin\": \"191.35.122.74\", \n \"url\": \"https://httpbin.org/post\"\n}\n", + "response_status_code":200, + "execution_duration":547, + "success":true, + "error":"", + "created_at":"2021-03-08T20:46:51.680846Z" + } + ], + "limit":50, + "offset":0 +} +``` + +### Get delivery attempt + +```bash +curl --location --request GET 'http://localhost:8000/v1/delivery-attempts/d72719d6-5a79-4df7-a2c2-2029ab0e1848' +``` + +```javascript +{ + "id":"d72719d6-5a79-4df7-a2c2-2029ab0e1848", + "webhook_id":"a6e9a525-ac5a-488c-b118-bd7327ce6d8d", + "delivery_id":"bc76122c-e56b-45c7-8dc3-b80a861191d5", + "raw_request":"POST /post HTTP/1.1\r\nHost: httpbin.org\r\nContent-Type: application/json\r\nX-Hub-Signature: 3fc5d4b8ff4efb404be24faf543667d29902d6a1306bd0c1ef2084497300cee9\r\n\r\n{\"success\": true}", + "raw_response":"HTTP/2.0 200 OK\r\nContent-Length: 538\r\nAccess-Control-Allow-Credentials: true\r\nAccess-Control-Allow-Origin: *\r\nContent-Type: application/json\r\nDate: Mon, 08 Mar 2021 20:46:51 GMT\r\nServer: gunicorn/19.9.0\r\n\r\n{\n \"args\": {}, \n \"data\": \"{\\\"success\\\": true}\", \n \"files\": {}, \n \"form\": {}, \n \"headers\": {\n \"Accept-Encoding\": \"gzip\", \n \"Content-Length\": \"17\", \n \"Content-Type\": \"application/json\", \n \"Host\": \"httpbin.org\", \n \"User-Agent\": \"Go-http-client/2.0\", \n \"X-Amzn-Trace-Id\": \"Root=1-60468d3b-36d312777a03ec3e1c564e3b\", \n \"X-Hub-Signature\": \"3fc5d4b8ff4efb404be24faf543667d29902d6a1306bd0c1ef2084497300cee9\"\n }, \n \"json\": {\n \"success\": true\n }, \n \"origin\": \"191.35.122.74\", \n \"url\": \"https://httpbin.org/post\"\n}\n", + "response_status_code":200, + "execution_duration":547, + "success":true, + "error":"", + "created_at":"2021-03-08T20:46:51.680846Z" +} +``` + +## How to build docker image + +``` +docker build -f Dockerfile -t postmand . +``` diff --git a/cmd/postmand/main.go b/cmd/postmand/main.go index 910a2bc..c66345c 100644 --- a/cmd/postmand/main.go +++ b/cmd/postmand/main.go @@ -23,7 +23,7 @@ func healthcheckServer(db *sqlx.DB, logger *zap.Logger) { pingHandler := handler.NewPing(pingService, logger) mux := http.NewRouter(logger) mux.Get("/healthz", pingHandler.Healthz) - server := http.NewServer(mux, env.GetInt("POSTMAND_HEALTH_CHECK_HTTP_PORT", 8000), logger) + server := http.NewServer(mux, env.GetInt("POSTMAND_HEALTH_CHECK_HTTP_PORT", 8001), logger) server.Run() } @@ -66,7 +66,7 @@ func main() { db, env.GetString("POSTMAND_DATABASE_MIGRATION_DIR", "file:///db/migrations"), ) - migrationService := service.NewMigration(migrationRepository) + migrationService := service.NewMigration(migrationRepository, logger) return migrationService.Run(c.Context) }, }, diff --git a/http/handler/delivery.go b/http/handler/delivery.go index ac61f55..8114935 100644 --- a/http/handler/delivery.go +++ b/http/handler/delivery.go @@ -23,7 +23,7 @@ type Delivery struct { // List deliveries. func (d Delivery) List(w http.ResponseWriter, r *http.Request) { - listOptions, err := makeListOptions(r, []string{}) + listOptions, err := makeListOptions(r, []string{"webhook_id", "status"}) if err != nil { er := errorResponses["internal_server_error"] makeErrorResponse(w, &er, d.logger) diff --git a/service/migration.go b/service/migration.go index 87cec2c..0d482fd 100644 --- a/service/migration.go +++ b/service/migration.go @@ -4,19 +4,29 @@ import ( "context" "github.com/allisson/postmand" + "go.uber.org/zap" ) // Migration implements postmand.MigrationService interface. type Migration struct { migrationRepo postmand.MigrationRepository + logger *zap.Logger } // Run database migrations. func (m Migration) Run(ctx context.Context) error { - return m.migrationRepo.Run(ctx) + m.logger.Info("migration-started") + if err := m.migrationRepo.Run(ctx); err != nil { + m.logger.Error("migration-error", zap.Error(err)) + } + m.logger.Info("migration-completed") + return nil } // NewMigration will create an implementation of postmand.MigrationService. -func NewMigration(migrationRepo postmand.MigrationRepository) *Migration { - return &Migration{migrationRepo: migrationRepo} +func NewMigration(migrationRepo postmand.MigrationRepository, logger *zap.Logger) *Migration { + return &Migration{ + migrationRepo: migrationRepo, + logger: logger, + } } diff --git a/service/migration_test.go b/service/migration_test.go index a046182..f3ce2b0 100644 --- a/service/migration_test.go +++ b/service/migration_test.go @@ -7,11 +7,13 @@ import ( "github.com/allisson/postmand/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "go.uber.org/zap" ) func TestMigration(t *testing.T) { + logger, _ := zap.NewDevelopment() migrationRepository := &mocks.MigrationRepository{} - migrationService := NewMigration(migrationRepository) + migrationService := NewMigration(migrationRepository, logger) migrationRepository.On("Run", mock.Anything).Return(nil) ctx := context.Background() err := migrationService.Run(ctx) diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..e69de29