From 2e404a963be263498244a473418c025f5646ef10 Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Wed, 29 Jun 2022 18:40:07 +0300 Subject: [PATCH] feat(telemetry): experiments with opentelemetry, traces and clickhouse storage Signed-off-by: Timofey Kirillov --- cmd/werf/main.go | 3 + go.mod | 12 ++-- go.sum | 12 ++-- pkg/telemetry/event.go | 35 +++++++++++ .../{controller.go => metric_exporter.go} | 20 +++---- pkg/telemetry/telemetry.go | 33 +++++++---- pkg/telemetry/trace_exporter.go | 59 +++++++++++++++++++ scripts/local-telemetry/docker-compose.yaml | 33 +++++++---- .../otel-collector-config.yaml | 24 +++++--- 9 files changed, 176 insertions(+), 55 deletions(-) create mode 100644 pkg/telemetry/event.go rename pkg/telemetry/{controller.go => metric_exporter.go} (73%) create mode 100644 pkg/telemetry/trace_exporter.go diff --git a/cmd/werf/main.go b/cmd/werf/main.go index b55ec4ef69..92f9603530 100644 --- a/cmd/werf/main.go +++ b/cmd/werf/main.go @@ -50,6 +50,7 @@ import ( "github.com/werf/werf/cmd/werf/synchronization" "github.com/werf/werf/cmd/werf/version" "github.com/werf/werf/pkg/process_exterminator" + "github.com/werf/werf/pkg/telemetry" ) func main() { @@ -57,6 +58,8 @@ func main() { common.InitTelemetry(ctx) + telemetry.MessageEvent(ctx, "command started") + shouldTerminate, err := common.ContainerBackendProcessStartupHook() if err != nil { common.ShutdownTelemetry(ctx, 1) diff --git a/go.mod b/go.mod index bbf5f5b92b..1719e7fb03 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,13 @@ require ( github.com/werf/kubedog v0.6.5-0.20220608151530-2555924b0ce9 github.com/werf/lockgate v0.0.0-20200729113342-ec2c142f71ea github.com/werf/logboek v0.5.4 + go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0 + go.opentelemetry.io/otel/metric v0.30.0 + go.opentelemetry.io/otel/sdk/metric v0.30.0 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 golang.org/x/net v0.0.0-20220225172249-27dd8689420f gopkg.in/errgo.v2 v2.1.0 @@ -270,13 +277,8 @@ require ( go.mongodb.org/mongo-driver v1.5.1 // indirect go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0 // indirect - go.opentelemetry.io/otel/metric v0.30.0 // indirect go.opentelemetry.io/otel/sdk v1.7.0 // indirect - go.opentelemetry.io/otel/sdk/metric v0.30.0 // indirect go.opentelemetry.io/otel/trace v1.7.0 // indirect go.opentelemetry.io/proto/otlp v0.16.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect diff --git a/go.sum b/go.sum index ce70e74a8f..e3217482b6 100644 --- a/go.sum +++ b/go.sum @@ -310,6 +310,7 @@ github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYX github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -781,7 +782,6 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -940,6 +940,7 @@ github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -1140,7 +1141,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= @@ -2154,12 +2154,9 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.2 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b h1:wZT15nCrRJsT1PJpZ3rYPTsPH2LnkDq8s6CNpgIsyiw= go.opentelemetry.io/otel v1.7.1-0.20220624212736-ef6c0da0de3b/go.mod h1:lnHkMUGUQfY3wBpkzmPQH6RuCyURze1hKrrF7xTxJ60= -go.opentelemetry.io/otel/exporters/jaeger v1.7.0 h1:wXgjiRldljksZkZrldGVe6XrG9u3kYDyQmkZwmm5dI0= -go.opentelemetry.io/otel/exporters/jaeger v1.7.0/go.mod h1:PwQAOqBgqbLQRKlj466DuD2qyMjbtcPpfPfj+AqbSBs= go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= @@ -2169,15 +2166,16 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 h1:Os0ds8fJp2AUa9DNra go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0/go.mod h1:8Lz1GGcrx1kPGE3zqDrK7ZcPzABEfIQqBjq7roQa5ZA= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0 h1:MrUowGDjf4jKGMgjDAIP5Czh6YGdCHc46gfTwlF6eQI= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.30.0/go.mod h1:WulNodDa6sY6ZADi664BgKD6SvXLLQXVZEQ81q5ps9U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0 h1:giGm8w67Ja7amYNfYMdme7xSp2pIxThWopw8+QP51Yk= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0 h1:Ydage/P0fRrSPpZeCVxzjqGcI6iVmG2xb43+IR8cjqM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/metric v0.30.0 h1:Hs8eQZ8aQgs0U49diZoaS6Uaxw3+bBE3lcMUKBFIk3c= go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.3.0 h1:3278edCoH89MEJ0Ky8WQXVmDQv3FX4ZJ3Pp+9fJreAI= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= go.opentelemetry.io/otel/sdk v1.7.0 h1:4OmStpcKVOfvDOgCt7UriAPtKolwIhxpnSNI/yK+1B0= go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= @@ -2884,7 +2882,6 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -2901,7 +2898,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/pkg/telemetry/event.go b/pkg/telemetry/event.go new file mode 100644 index 0000000000..6a4dc58886 --- /dev/null +++ b/pkg/telemetry/event.go @@ -0,0 +1,35 @@ +package telemetry + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "go.opentelemetry.io/otel/attribute" +) + +func MessageEvent(ctx context.Context, msg string) error { + if !IsEnabled() { + return nil + } + + trc := tracerProvider.Tracer("telemetry.werf.io") + + _, span := trc.Start(ctx, "message") + + span.SetAttributes(attribute.Key("ts").Int64(time.Now().UnixMilli())) + span.SetAttributes(attribute.Key("executionID").String(executionID)) + span.SetAttributes(attribute.Key("projectID").String(projectID)) + span.SetAttributes(attribute.Key("command").String(command)) + span.SetAttributes(attribute.Key("eventType").String("message")) + + data, err := json.Marshal(map[string]interface{}{"message": msg}) + if err != nil { + return fmt.Errorf("unable to marshal message data: %w", err) + } + span.SetAttributes(attribute.Key("data").String(string(data))) + span.End() + + return nil +} diff --git a/pkg/telemetry/controller.go b/pkg/telemetry/metric_exporter.go similarity index 73% rename from pkg/telemetry/controller.go rename to pkg/telemetry/metric_exporter.go index 830dea15a8..850f65b38b 100644 --- a/pkg/telemetry/controller.go +++ b/pkg/telemetry/metric_exporter.go @@ -14,28 +14,28 @@ import ( ) var ( - ctrl *controller.Controller - exporter *otlpmetric.Exporter + ctrl *controller.Controller + metricExporter *otlpmetric.Exporter ) -func SetupController(ctx context.Context, url string) error { +func SetupMetricExporter(ctx context.Context, url string) error { { - e, err := NewExporter(url) + e, err := NewMetricExporter(url) if err != nil { - return fmt.Errorf("unable to create telemetry exporter: %w", err) + return fmt.Errorf("unable to create telemetry metric exporter: %w", err) } - exporter = e + metricExporter = e } - ctrl = NewController(exporter) + ctrl = NewController(metricExporter) - if err := exporter.Start(ctx); err != nil { - return fmt.Errorf("error starting telemetry exporter: %w", err) + if err := metricExporter.Start(ctx); err != nil { + return fmt.Errorf("error starting telemetry metric exporter: %w", err) } return nil } -func NewExporter(url string) (*otlpmetric.Exporter, error) { +func NewMetricExporter(url string) (*otlpmetric.Exporter, error) { urlObj, err := neturl.Parse(url) if err != nil { return nil, fmt.Errorf("bad url: %w", err) diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index 498dbd3c45..9809c9ce8f 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -3,18 +3,27 @@ package telemetry import ( "context" "fmt" + "os" "runtime" + "github.com/google/uuid" + "github.com/werf/logboek" "github.com/werf/werf/pkg/util" "github.com/werf/werf/pkg/werf" ) const ( - URL = "http://localhost:4318/v1/metrics" + MetricsURL = "http://localhost:4318/v1/metrics" + TracesURL = "http://localhost:4318/v1/traces" ) -var metrics *Metrics +var ( + executionID string + projectID string + command string + metrics *Metrics +) func ChangeMetrics(changeFunc func(metrics *Metrics)) { if !IsEnabled() { @@ -28,6 +37,10 @@ func Init(ctx context.Context) error { return nil } + executionID = uuid.New().String() + projectID = "26a38bba-51ed-4f93-8104-e6f51a5454ef" + command = fmt.Sprintf("%v", os.Args) + metrics = &Metrics{ Version: werf.Version, Command: "TODO", @@ -35,8 +48,8 @@ func Init(ctx context.Context) error { Arch: runtime.GOARCH, } - if err := SetupController(ctx, URL); err != nil { - return fmt.Errorf("unable to setup telemetry controller for url %q: %w", URL, err) + if err := SetupTraceExporter(ctx, TracesURL); err != nil { + return fmt.Errorf("unable to setup telemetry traces exporter for url %q: %w", TracesURL, err) } return nil @@ -47,16 +60,12 @@ func Shutdown(ctx context.Context) error { return nil } - if err := metrics.WriteOpenTelemetry(ctx, ctrl.Meter("werf")); err != nil { - return fmt.Errorf("unable to write open telemetry data: %w", err) - } - - if err := ctrl.Collect(ctx); err != nil { - return fmt.Errorf("otel controller collection failed: %w", err) + if err := traceExporter.Shutdown(ctx); err != nil { + return fmt.Errorf("unable to shutdown trace exporter: %w", err) } - if err := exporter.Export(ctx, ctrl.Resource(), ctrl); err != nil { - return fmt.Errorf("unable to export telemetry: %w", err) + if err := tracerProvider.Shutdown(ctx); err != nil { + return fmt.Errorf("unable to shutdown trace provider: %w", err) } logboek.Context(ctx).Debug().LogF("Telemetry collection done\n") diff --git a/pkg/telemetry/trace_exporter.go b/pkg/telemetry/trace_exporter.go new file mode 100644 index 0000000000..ece174dfbe --- /dev/null +++ b/pkg/telemetry/trace_exporter.go @@ -0,0 +1,59 @@ +package telemetry + +import ( + "context" + "fmt" + neturl "net/url" + "time" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/trace" +) + +var ( + tracerProvider *trace.TracerProvider + traceExporter *otlptrace.Exporter +) + +func SetupTraceExporter(ctx context.Context, url string) error { + { + e, err := NewTraceExporter(url) + if err != nil { + return fmt.Errorf("unable to create telemetry trace exporter: %w", err) + } + traceExporter = e + } + + tracerProvider = trace.NewTracerProvider(trace.WithSyncer(traceExporter)) + + if err := traceExporter.Start(ctx); err != nil { + return fmt.Errorf("error starting telemetry trace exporter: %w", err) + } + + return nil +} + +func NewTraceExporter(url string) (*otlptrace.Exporter, error) { + urlObj, err := neturl.Parse(url) + if err != nil { + return nil, fmt.Errorf("bad url: %w", err) + } + + var opts []otlptracehttp.Option + + if urlObj.Scheme == "http" { + opts = append(opts, otlptracehttp.WithInsecure()) + } + + opts = append(opts, + otlptracehttp.WithEndpoint(urlObj.Host), + otlptracehttp.WithURLPath(urlObj.Path), + otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}), + otlptracehttp.WithTimeout(5*time.Second), + ) + + client := otlptracehttp.NewClient(opts...) + + return otlptrace.NewUnstarted(client), nil +} diff --git a/scripts/local-telemetry/docker-compose.yaml b/scripts/local-telemetry/docker-compose.yaml index 1d3871d13a..9b1a90b9cc 100644 --- a/scripts/local-telemetry/docker-compose.yaml +++ b/scripts/local-telemetry/docker-compose.yaml @@ -1,20 +1,29 @@ -version: "2" +version: "3" + +networks: + otel-clickhouse: + services: otel-collector: - image: otel/opentelemetry-collector-contrib-dev:latest - command: [ "--config=/etc/otel-collector-config.yaml" ] + image: ghcr.io/werf/telemetrywerfio-collector:dev + # image: otel/opentelemetry-collector-contrib-dev:latest + command: + - "--config=/etc/otel-collector-config.yaml" + - "--set=service.telemetry.logs.level=DEBUG" volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml ports: - - "8888:8888" # Prometheus metrics exposed by the collector server itself - - "8889:8889" # Collected metrics for prometheus - - "4318:4318" # OTLP HTTP receiver + - "4318:4318" # OTLP HTTP receiver + depends_on: + - clickhouse + networks: + - otel-clickhouse - prometheus: - container_name: prometheus - image: prom/prometheus:latest - volumes: - - ./prometheus.yaml:/etc/prometheus/prometheus.yml + clickhouse: + image: clickhouse/clickhouse-server:latest ports: - - "9090:9090" + - "9000:9000" + - "8123:8123" + networks: + - otel-clickhouse diff --git a/scripts/local-telemetry/otel-collector-config.yaml b/scripts/local-telemetry/otel-collector-config.yaml index 8de083f402..e260aed804 100644 --- a/scripts/local-telemetry/otel-collector-config.yaml +++ b/scripts/local-telemetry/otel-collector-config.yaml @@ -3,20 +3,28 @@ receivers: protocols: http: -exporters: - prometheus: - endpoint: "0.0.0.0:8889" - processors: batch: +exporters: + telemetrywerfio: + dsn: tcp://clickhouse:9000?database=default + timeout: 5s + sending_queue: + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + service: telemetry: logs: level: "debug" pipelines: - metrics: - receivers: [otlp] - processors: [batch] - exporters: [prometheus] + traces: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ telemetrywerfio ]