From 6dcbd3e7d8d108636510a3f22752ddbd5031f31f Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Mon, 27 Jun 2022 17:18:55 +0300 Subject: [PATCH] feat(telemetry): basic telemetry client and local setup * Disabled by default, to enable set `WERF_TELEMETRY=1` env var. * werf uses opentelemetry client to send metrics over OTLP. * Send werf runs counter with {OS, arch, version, command, exitCode} labels. * Run `make local-telemtry` to setup local opentelemetry collector with prometheus (using docker compose). Signed-off-by: Timofey Kirillov --- Makefile | 5 ++ cmd/werf/common/telemetry.go | 25 +++++++ cmd/werf/main.go | 12 +++- go.mod | 18 ++++- go.sum | 39 ++++++++++ pkg/telemetry/controller.go | 72 +++++++++++++++++++ pkg/telemetry/metrics.go | 39 ++++++++++ pkg/telemetry/telemetry.go | 69 ++++++++++++++++++ scripts/local-telemetry/docker-compose.yaml | 20 ++++++ .../otel-collector-config.yaml | 22 ++++++ scripts/local-telemetry/prometheus.yaml | 6 ++ 11 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 cmd/werf/common/telemetry.go create mode 100644 pkg/telemetry/controller.go create mode 100644 pkg/telemetry/metrics.go create mode 100644 pkg/telemetry/telemetry.go create mode 100644 scripts/local-telemetry/docker-compose.yaml create mode 100644 scripts/local-telemetry/otel-collector-config.yaml create mode 100644 scripts/local-telemetry/prometheus.yaml diff --git a/Makefile b/Makefile index 6be0b9df7a..abe926ce18 100644 --- a/Makefile +++ b/Makefile @@ -62,3 +62,8 @@ build-images: clean: rm -f $$GOPATH/bin/werf rm -f $$GOPATH/buildah-test + +.PHOME: local-telemetry +local-telemetry: + cd scripts/local-telemetry ; docker-compose down + cd scripts/local-telemetry ; docker-compose up -d diff --git a/cmd/werf/common/telemetry.go b/cmd/werf/common/telemetry.go new file mode 100644 index 0000000000..a270276f6d --- /dev/null +++ b/cmd/werf/common/telemetry.go @@ -0,0 +1,25 @@ +package common + +import ( + "context" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/telemetry" +) + +func InitTelemetry(ctx context.Context) { + // TODO: append error to the ~/.werf/telemetry_errors.log + err := telemetry.Init(ctx) + logboek.Context(ctx).Debug().LogF("Telemetry: init error: %s\n", err) +} + +func ShutdownTelemetry(ctx context.Context, exitCode int) { + telemetry.ChangeMetrics(func(metrics *telemetry.Metrics) { + metrics.ExitCode = exitCode + }) + + // TODO: append error to the ~/.werf/telemetry_errors.log + err := telemetry.Shutdown(ctx) + + logboek.Context(ctx).Debug().LogF("Telemetry: shutdown error: %s\n", err) +} diff --git a/cmd/werf/main.go b/cmd/werf/main.go index f0c72a87d9..b55ec4ef69 100644 --- a/cmd/werf/main.go +++ b/cmd/werf/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "log" "os" @@ -52,8 +53,13 @@ import ( ) func main() { + ctx := context.Background() + + common.InitTelemetry(ctx) + shouldTerminate, err := common.ContainerBackendProcessStartupHook() if err != nil { + common.ShutdownTelemetry(ctx, 1) common.TerminateWithError(err.Error(), 1) } if shouldTerminate { @@ -65,6 +71,7 @@ func main() { logrus.StandardLogger().SetOutput(logboek.OutStream()) if err := process_exterminator.Init(); err != nil { + common.ShutdownTelemetry(ctx, 1) common.TerminateWithError(fmt.Sprintf("process exterminator initialization failed: %s", err), 1) } @@ -72,9 +79,12 @@ func main() { if err := rootCmd.Execute(); err != nil { if helm_v3.IsPluginError(err) { + common.ShutdownTelemetry(ctx, helm_v3.PluginErrorCode(err)) common.TerminateWithError(err.Error(), helm_v3.PluginErrorCode(err)) + } else { + common.ShutdownTelemetry(ctx, 1) + common.TerminateWithError(err.Error(), 1) } - common.TerminateWithError(err.Error(), 1) } } diff --git a/go.mod b/go.mod index 9fe82df9a0..bbf5f5b92b 100644 --- a/go.mod +++ b/go.mod @@ -111,6 +111,7 @@ require ( github.com/blang/semver v3.5.1+incompatible // indirect github.com/bugsnag/bugsnag-go v1.5.3 // indirect github.com/bugsnag/panicwrap v1.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect @@ -144,7 +145,8 @@ require ( github.com/go-errors/errors v1.0.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect - github.com/go-logr/logr v1.2.2 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.19.5 // indirect github.com/go-openapi/errors v0.19.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -169,6 +171,7 @@ require ( github.com/gorilla/mux v1.8.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -267,6 +270,15 @@ require ( go.mongodb.org/mongo-driver v1.5.1 // indirect go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect go.opencensus.io v0.23.0 // indirect + go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0 // indirect + go.opentelemetry.io/otel/metric v0.30.0 // indirect + go.opentelemetry.io/otel/sdk v1.7.0 // indirect + go.opentelemetry.io/otel/sdk/metric v0.30.0 // indirect + go.opentelemetry.io/otel/trace v1.7.0 // indirect + go.opentelemetry.io/proto/otlp v0.16.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect @@ -276,8 +288,8 @@ require ( golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8 // indirect - google.golang.org/grpc v1.44.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/grpc v1.46.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect diff --git a/go.sum b/go.sum index ef646b9561..ce70e74a8f 100644 --- a/go.sum +++ b/go.sum @@ -310,6 +310,7 @@ github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYX github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -354,6 +355,8 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -690,6 +693,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= @@ -779,7 +783,10 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v0.3.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= @@ -1022,6 +1029,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.0.0-20191010200024-a3d713f9b7f8/go.mod h1:KyKXa9ciM8+lgMXwOVsXi7UxGrsf9mM61Mzs+xKUrKE= @@ -1132,7 +1140,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= @@ -2143,21 +2154,45 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.2 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM= +go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b h1:wZT15nCrRJsT1PJpZ3rYPTsPH2LnkDq8s6CNpgIsyiw= +go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b/go.mod h1:lnHkMUGUQfY3wBpkzmPQH6RuCyURze1hKrrF7xTxJ60= +go.opentelemetry.io/otel/exporters/jaeger v1.7.0 h1:wXgjiRldljksZkZrldGVe6XrG9u3kYDyQmkZwmm5dI0= +go.opentelemetry.io/otel/exporters/jaeger v1.7.0/go.mod h1:PwQAOqBgqbLQRKlj466DuD2qyMjbtcPpfPfj+AqbSBs= +go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 h1:7Yxsak1q4XrJ5y7XBnNwqWx9amMZvoidCctv62XOQ6Y= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0/go.mod h1:M1hVZHNxcbkAlcvrOMlpQ4YOO3Awf+4N2dxkZL3xm04= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 h1:Os0ds8fJp2AUa9DNraFWIycgUzevz47i6UvnSh+8LQ0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0/go.mod h1:8Lz1GGcrx1kPGE3zqDrK7ZcPzABEfIQqBjq7roQa5ZA= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0 h1:MrUowGDjf4jKGMgjDAIP5Czh6YGdCHc46gfTwlF6eQI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0/go.mod h1:WulNodDa6sY6ZADi664BgKD6SvXLLQXVZEQ81q5ps9U= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v0.30.0 h1:Hs8eQZ8aQgs0U49diZoaS6Uaxw3+bBE3lcMUKBFIk3c= +go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.3.0 h1:3278edCoH89MEJ0Ky8WQXVmDQv3FX4ZJ3Pp+9fJreAI= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= +go.opentelemetry.io/otel/sdk v1.7.0 h1:4OmStpcKVOfvDOgCt7UriAPtKolwIhxpnSNI/yK+1B0= +go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/sdk/metric v0.30.0 h1:XTqQ4y3erR2Oj8xSAOL5ovO5011ch2ELg51z4fVkpME= +go.opentelemetry.io/otel/sdk/metric v0.30.0/go.mod h1:8AKFRi5HyvTR0RRty3paN1aMC9HMT+NzcEhw/BLkLX8= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= +go.opentelemetry.io/otel/trace v1.7.0 h1:O37Iogk1lEkMRXewVtZ1BBTVn5JEp8GrJvP92bJqC6o= +go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= +go.opentelemetry.io/proto/otlp v0.16.0 h1:WHzDWdXUvbc5bG2ObdrGfaNpQz7ft7QN9HHmJlbiB1E= +go.opentelemetry.io/proto/otlp v0.16.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -2851,6 +2886,8 @@ google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5 google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2866,6 +2903,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/pkg/telemetry/controller.go b/pkg/telemetry/controller.go new file mode 100644 index 0000000000..830dea15a8 --- /dev/null +++ b/pkg/telemetry/controller.go @@ -0,0 +1,72 @@ +package telemetry + +import ( + "context" + "fmt" + neturl "net/url" + "time" + + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + controller "go.opentelemetry.io/otel/sdk/metric/controller/basic" + processor "go.opentelemetry.io/otel/sdk/metric/processor/basic" + "go.opentelemetry.io/otel/sdk/metric/selector/simple" +) + +var ( + ctrl *controller.Controller + exporter *otlpmetric.Exporter +) + +func SetupController(ctx context.Context, url string) error { + { + e, err := NewExporter(url) + if err != nil { + return fmt.Errorf("unable to create telemetry exporter: %w", err) + } + exporter = e + } + ctrl = NewController(exporter) + + if err := exporter.Start(ctx); err != nil { + return fmt.Errorf("error starting telemetry exporter: %w", err) + } + + return nil +} + +func NewExporter(url string) (*otlpmetric.Exporter, error) { + urlObj, err := neturl.Parse(url) + if err != nil { + return nil, fmt.Errorf("bad url: %w", err) + } + + var opts []otlpmetrichttp.Option + + if urlObj.Scheme == "http" { + opts = append(opts, otlpmetrichttp.WithInsecure()) + } + + opts = append(opts, + otlpmetrichttp.WithEndpoint(urlObj.Host), + otlpmetrichttp.WithURLPath(urlObj.Path), + otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false}), + otlpmetrichttp.WithTimeout(5*time.Second), + ) + + client := otlpmetrichttp.NewClient(opts...) + + return otlpmetric.NewUnstarted(client), nil +} + +func NewController(exporter *otlpmetric.Exporter) *controller.Controller { + checkpointerFactory := processor.NewFactory(simple.NewWithHistogramDistribution(), exporter) + + return controller.New( + checkpointerFactory, + controller.WithExporter(exporter), + controller.WithCollectTimeout(5*time.Second), + controller.WithCollectPeriod(0), + controller.WithPushTimeout(5*time.Second), + ) +} diff --git a/pkg/telemetry/metrics.go b/pkg/telemetry/metrics.go new file mode 100644 index 0000000000..014ff2e56a --- /dev/null +++ b/pkg/telemetry/metrics.go @@ -0,0 +1,39 @@ +package telemetry + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/instrument" + + "github.com/werf/logboek" +) + +type Metrics struct { + Version string + Command string + OS string + Arch string + ExitCode int +} + +func (m *Metrics) WriteOpenTelemetry(ctx context.Context, meter metric.Meter) error { + labels := []attribute.KeyValue{ + attribute.String("version", m.Version), + attribute.String("command", m.Command), + attribute.String("os", m.OS), + attribute.String("arch", m.Arch), + attribute.Int("exit_code", m.ExitCode), + } + + runCounter, err := meter.SyncInt64().Counter("runs", instrument.WithDescription("werf runs counter")) + if err != nil { + return fmt.Errorf("unable to record runs counter: %w", err) + } + runCounter.Add(ctx, 1, labels...) + logboek.Context(ctx).Debug().LogF("Telemetry: incremented runs counter: %#v\n", runCounter) + + return nil +} diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go new file mode 100644 index 0000000000..498dbd3c45 --- /dev/null +++ b/pkg/telemetry/telemetry.go @@ -0,0 +1,69 @@ +package telemetry + +import ( + "context" + "fmt" + "runtime" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/util" + "github.com/werf/werf/pkg/werf" +) + +const ( + URL = "http://localhost:4318/v1/metrics" +) + +var metrics *Metrics + +func ChangeMetrics(changeFunc func(metrics *Metrics)) { + if !IsEnabled() { + return + } + changeFunc(metrics) +} + +func Init(ctx context.Context) error { + if !IsEnabled() { + return nil + } + + metrics = &Metrics{ + Version: werf.Version, + Command: "TODO", + OS: runtime.GOOS, + Arch: runtime.GOARCH, + } + + if err := SetupController(ctx, URL); err != nil { + return fmt.Errorf("unable to setup telemetry controller for url %q: %w", URL, err) + } + + return nil +} + +func Shutdown(ctx context.Context) error { + if !IsEnabled() { + return nil + } + + if err := metrics.WriteOpenTelemetry(ctx, ctrl.Meter("werf")); err != nil { + return fmt.Errorf("unable to write open telemetry data: %w", err) + } + + if err := ctrl.Collect(ctx); err != nil { + return fmt.Errorf("otel controller collection failed: %w", err) + } + + if err := exporter.Export(ctx, ctrl.Resource(), ctrl); err != nil { + return fmt.Errorf("unable to export telemetry: %w", err) + } + + logboek.Context(ctx).Debug().LogF("Telemetry collection done\n") + + return nil +} + +func IsEnabled() bool { + return util.GetBoolEnvironmentDefaultFalse("WERF_TELEMETRY") +} diff --git a/scripts/local-telemetry/docker-compose.yaml b/scripts/local-telemetry/docker-compose.yaml new file mode 100644 index 0000000000..1d3871d13a --- /dev/null +++ b/scripts/local-telemetry/docker-compose.yaml @@ -0,0 +1,20 @@ +version: "2" +services: + + otel-collector: + image: otel/opentelemetry-collector-contrib-dev:latest + command: [ "--config=/etc/otel-collector-config.yaml" ] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "8888:8888" # Prometheus metrics exposed by the collector server itself + - "8889:8889" # Collected metrics for prometheus + - "4318:4318" # OTLP HTTP receiver + + prometheus: + container_name: prometheus + image: prom/prometheus:latest + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" diff --git a/scripts/local-telemetry/otel-collector-config.yaml b/scripts/local-telemetry/otel-collector-config.yaml new file mode 100644 index 0000000000..8de083f402 --- /dev/null +++ b/scripts/local-telemetry/otel-collector-config.yaml @@ -0,0 +1,22 @@ +receivers: + otlp: + protocols: + http: + +exporters: + prometheus: + endpoint: "0.0.0.0:8889" + +processors: + batch: + +service: + telemetry: + logs: + level: "debug" + + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] diff --git a/scripts/local-telemetry/prometheus.yaml b/scripts/local-telemetry/prometheus.yaml new file mode 100644 index 0000000000..b8e00e2746 --- /dev/null +++ b/scripts/local-telemetry/prometheus.yaml @@ -0,0 +1,6 @@ +scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 5s + static_configs: + - targets: ['otel-collector:8889'] + - targets: ['otel-collector:8888']