diff --git a/.chloggen/healthcheck-v2.yaml b/.chloggen/healthcheck-v2.yaml new file mode 100755 index 0000000000000..674c89e0e47a2 --- /dev/null +++ b/.chloggen/healthcheck-v2.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: 'new_component' + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: healthcheckv2extension + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Health Check Extension V2 is meant to be a replacement for the current Health Check Extension. It is based off of component status reporting and provides HTTP and gRPC services health check services. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [26661] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5404b611eebed..b323923b0090a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -285,10 +285,10 @@ testbed/mockdatasenders/mockdatadogagentexporter/ @open-telemetry/collect # List of distribution maintainers for OpenTelemetry Collector Contrib # ##################################################### -reports/distributions/core.yaml @open-telemetry/collector-contrib-approvers -reports/distributions/contrib.yaml @open-telemetry/collector-contrib-approvers +reports/distributions/core.yaml @open-telemetry/collector-contrib-approvers +reports/distributions/contrib.yaml @open-telemetry/collector-contrib-approvers ## UNMAINTAINED components -exporter/skywalkingexporter/ @open-telemetry/collector-contrib-approvers +exporter/skywalkingexporter/ @open-telemetry/collector-contrib-approvers diff --git a/extension/healthcheckv2extension/extension.go b/extension/healthcheckv2extension/extension.go index cb58a30add883..18f3f1268b7ad 100644 --- a/extension/healthcheckv2extension/extension.go +++ b/extension/healthcheckv2extension/extension.go @@ -7,36 +7,198 @@ import ( "context" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/extension" + "go.uber.org/multierr" "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/grpc" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/http" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" ) +type eventSourcePair struct { + source *component.InstanceID + event *component.StatusEvent +} + type healthCheckExtension struct { - config Config - telemetry component.TelemetrySettings + config Config + telemetry component.TelemetrySettings + aggregator *status.Aggregator + subcomponents []component.Component + eventCh chan *eventSourcePair + readyCh chan struct{} } var _ component.Component = (*healthCheckExtension)(nil) +var _ extension.ConfigWatcher = (*healthCheckExtension)(nil) +var _ extension.PipelineWatcher = (*healthCheckExtension)(nil) func newExtension( - _ context.Context, + ctx context.Context, config Config, set extension.CreateSettings, ) *healthCheckExtension { - return &healthCheckExtension{ - config: config, - telemetry: set.TelemetrySettings, + var comps []component.Component + + errPriority := status.PriorityPermanent + if config.ComponentHealthConfig != nil && + config.ComponentHealthConfig.IncludeRecoverable && + !config.ComponentHealthConfig.IncludePermanent { + errPriority = status.PriorityRecoverable + } + + aggregator := status.NewAggregator(errPriority) + + if config.UseV2 && config.GRPCConfig != nil { + grpcServer := grpc.NewServer( + config.GRPCConfig, + config.ComponentHealthConfig, + set.TelemetrySettings, + aggregator, + ) + comps = append(comps, grpcServer) + } + + if !config.UseV2 || config.UseV2 && config.HTTPConfig != nil { + httpServer := http.NewServer( + config.HTTPConfig, + config.LegacyConfig, + config.ComponentHealthConfig, + set.TelemetrySettings, + aggregator, + ) + comps = append(comps, httpServer) } + + hc := &healthCheckExtension{ + config: config, + subcomponents: comps, + telemetry: set.TelemetrySettings, + aggregator: aggregator, + eventCh: make(chan *eventSourcePair), + readyCh: make(chan struct{}), + } + + // Start processing events in the background so that our status watcher doesn't + // block others before the extension starts. + go hc.eventLoop(ctx) + + return hc } // Start implements the component.Component interface. -func (hc *healthCheckExtension) Start(context.Context, component.Host) error { +func (hc *healthCheckExtension) Start(ctx context.Context, host component.Host) error { hc.telemetry.Logger.Debug("Starting health check extension V2", zap.Any("config", hc.config)) + for _, comp := range hc.subcomponents { + if err := comp.Start(ctx, host); err != nil { + return err + } + } + return nil } // Shutdown implements the component.Component interface. -func (hc *healthCheckExtension) Shutdown(context.Context) error { +func (hc *healthCheckExtension) Shutdown(ctx context.Context) error { + // Preemptively send the stopped event, so it can be exported before shutdown + hc.telemetry.ReportStatus(component.NewStatusEvent(component.StatusStopped)) + + close(hc.eventCh) + hc.aggregator.Close() + + var err error + for _, comp := range hc.subcomponents { + err = multierr.Append(err, comp.Shutdown(ctx)) + } + + return err +} + +// ComponentStatusChanged implements the extension.StatusWatcher interface. +func (hc *healthCheckExtension) ComponentStatusChanged( + source *component.InstanceID, + event *component.StatusEvent, +) { + // There can be late arriving events after shutdown. We need to close + // the event channel so that this function doesn't block and we release all + // goroutines, but attempting to write to a closed channel will panic; log + // and recover. + defer func() { + if r := recover(); r != nil { + hc.telemetry.Logger.Info( + "discarding event received after shutdown", + zap.Any("source", source), + zap.Any("event", event), + ) + } + }() + hc.eventCh <- &eventSourcePair{source: source, event: event} +} + +// NotifyConfig implements the extension.ConfigWatcher interface. +func (hc *healthCheckExtension) NotifyConfig(ctx context.Context, conf *confmap.Conf) error { + var err error + for _, comp := range hc.subcomponents { + if cw, ok := comp.(extension.ConfigWatcher); ok { + err = multierr.Append(err, cw.NotifyConfig(ctx, conf)) + } + } + return err +} + +// Ready implements the extension.PipelineWatcher interface. +func (hc *healthCheckExtension) Ready() error { + close(hc.readyCh) return nil } + +// NotReady implements the extension.PipelineWatcher interface. +func (hc *healthCheckExtension) NotReady() error { + return nil +} + +func (hc *healthCheckExtension) eventLoop(ctx context.Context) { + // Record events with component.StatusStarting, but queue other events until + // PipelineWatcher.Ready is called. This prevents aggregate statuses from + // flapping between StatusStarting and StatusOK as components are started + // individually by the service. + var eventQueue []*eventSourcePair + + for loop := true; loop; { + select { + case esp, ok := <-hc.eventCh: + if !ok { + return + } + if esp.event.Status() != component.StatusStarting { + eventQueue = append(eventQueue, esp) + continue + } + hc.aggregator.RecordStatus(esp.source, esp.event) + case <-hc.readyCh: + for _, esp := range eventQueue { + hc.aggregator.RecordStatus(esp.source, esp.event) + } + eventQueue = nil + loop = false + case <-ctx.Done(): + return + } + } + + // After PipelineWatcher.Ready, record statuses as they are received. + for { + select { + case esp, ok := <-hc.eventCh: + if !ok { + return + } + hc.aggregator.RecordStatus(esp.source, esp.event) + case <-ctx.Done(): + return + } + } +} diff --git a/extension/healthcheckv2extension/extension_test.go b/extension/healthcheckv2extension/extension_test.go new file mode 100644 index 0000000000000..c106ec30c1dbf --- /dev/null +++ b/extension/healthcheckv2extension/extension_test.go @@ -0,0 +1,134 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package healthcheckv2extension + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/extension/extensiontest" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/testhelpers" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/common/testutil" +) + +func TestComponentStatus(t *testing.T) { + cfg := createDefaultConfig().(*Config) + cfg.HTTPConfig.Endpoint = testutil.GetAvailableLocalAddress(t) + cfg.UseV2 = true + ext := newExtension(context.Background(), *cfg, extensiontest.NewNopCreateSettings()) + + // Status before Start will be StatusNone + st, ok := ext.aggregator.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + assert.Equal(t, st.Status(), component.StatusNone) + + require.NoError(t, ext.Start(context.Background(), componenttest.NewNopHost())) + + traces := testhelpers.NewPipelineMetadata("traces") + + // StatusStarting will be sent immediately. + for _, id := range traces.InstanceIDs() { + ext.ComponentStatusChanged(id, component.NewStatusEvent(component.StatusStarting)) + } + + // StatusOK will be queued until the PipelineWatcher Ready method is called. + for _, id := range traces.InstanceIDs() { + ext.ComponentStatusChanged(id, component.NewStatusEvent(component.StatusOK)) + } + + // Note the use of assert.Eventually here and throughout this test is because + // status events are processed asynchronously in the background. + assert.Eventually(t, func() bool { + st, ok = ext.aggregator.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + return st.Status() == component.StatusStarting + }, time.Second, 10*time.Millisecond) + + require.NoError(t, ext.Ready()) + + assert.Eventually(t, func() bool { + st, ok = ext.aggregator.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + return st.Status() == component.StatusOK + }, time.Second, 10*time.Millisecond) + + // StatusStopping will be sent immediately. + for _, id := range traces.InstanceIDs() { + ext.ComponentStatusChanged(id, component.NewStatusEvent(component.StatusStopping)) + } + + assert.Eventually(t, func() bool { + st, ok = ext.aggregator.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + return st.Status() == component.StatusStopping + }, time.Second, 10*time.Millisecond) + + require.NoError(t, ext.NotReady()) + require.NoError(t, ext.Shutdown(context.Background())) + + // Events sent after shutdown will be discarded + for _, id := range traces.InstanceIDs() { + ext.ComponentStatusChanged(id, component.NewStatusEvent(component.StatusStopped)) + } + + st, ok = ext.aggregator.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + assert.Equal(t, component.StatusStopping, st.Status()) +} + +func TestNotifyConfig(t *testing.T) { + confMap, err := confmaptest.LoadConf( + filepath.Join("internal", "http", "testdata", "config.yaml"), + ) + require.NoError(t, err) + confJSON, err := os.ReadFile( + filepath.Clean(filepath.Join("internal", "http", "testdata", "config.json")), + ) + require.NoError(t, err) + + endpoint := testutil.GetAvailableLocalAddress(t) + + cfg := createDefaultConfig().(*Config) + cfg.UseV2 = true + cfg.HTTPConfig.Endpoint = endpoint + cfg.HTTPConfig.Config.Enabled = true + cfg.HTTPConfig.Config.Path = "/config" + + ext := newExtension(context.Background(), *cfg, extensiontest.NewNopCreateSettings()) + + require.NoError(t, ext.Start(context.Background(), componenttest.NewNopHost())) + t.Cleanup(func() { require.NoError(t, ext.Shutdown(context.Background())) }) + + client := &http.Client{} + url := fmt.Sprintf("http://%s/config", endpoint) + + var resp *http.Response + + resp, err = client.Get(url) + require.NoError(t, err) + assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + + require.NoError(t, ext.NotifyConfig(context.Background(), confMap)) + + resp, err = client.Get(url) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, confJSON, body) +} diff --git a/extension/healthcheckv2extension/factory_test.go b/extension/healthcheckv2extension/factory_test.go index 540a0346a00a7..e8fdf2d784393 100644 --- a/extension/healthcheckv2extension/factory_test.go +++ b/extension/healthcheckv2extension/factory_test.go @@ -54,7 +54,9 @@ func TestCreateDefaultConfig(t *testing.T) { }, cfg) assert.NoError(t, componenttest.CheckConfigStruct(cfg)) - ext, err := createExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ext, err := createExtension(ctx, extensiontest.NewNopCreateSettings(), cfg) require.NoError(t, err) require.NotNil(t, ext) } @@ -62,8 +64,9 @@ func TestCreateDefaultConfig(t *testing.T) { func TestCreateExtension(t *testing.T) { cfg := createDefaultConfig().(*Config) cfg.Endpoint = testutil.GetAvailableLocalAddress(t) - - ext, err := createExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ext, err := createExtension(ctx, extensiontest.NewNopCreateSettings(), cfg) require.NoError(t, err) require.NotNil(t, ext) } diff --git a/extension/healthcheckv2extension/go.mod b/extension/healthcheckv2extension/go.mod index e41ca7fad5915..cb0a38f48d534 100644 --- a/extension/healthcheckv2extension/go.mod +++ b/extension/healthcheckv2extension/go.mod @@ -17,7 +17,9 @@ require ( go.opentelemetry.io/otel/metric v1.25.0 go.opentelemetry.io/otel/trace v1.25.0 go.uber.org/goleak v1.3.0 + go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 + google.golang.org/grpc v1.63.2 ) require ( @@ -61,12 +63,10 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.47.0 // indirect go.opentelemetry.io/otel/sdk v1.25.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.25.0 // indirect - go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/extension/healthcheckv2extension/internal/grpc/grpc.go b/extension/healthcheckv2extension/internal/grpc/grpc.go new file mode 100644 index 0000000000000..027fab6a220c3 --- /dev/null +++ b/extension/healthcheckv2extension/internal/grpc/grpc.go @@ -0,0 +1,137 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package grpc // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/grpc" + +import ( + "context" + "time" + + "go.opentelemetry.io/collector/component" + "google.golang.org/grpc/codes" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + grpcstatus "google.golang.org/grpc/status" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" +) + +var ( + errNotFound = grpcstatus.Error(codes.NotFound, "Service not found.") + errShuttingDown = grpcstatus.Error(codes.Canceled, "Server shutting down.") + errStreamSend = grpcstatus.Error(codes.Canceled, "Error sending; stream terminated.") + errStreamEnded = grpcstatus.Error(codes.Canceled, "Stream has ended.") + + statusToServingStatusMap = map[component.Status]healthpb.HealthCheckResponse_ServingStatus{ + component.StatusNone: healthpb.HealthCheckResponse_NOT_SERVING, + component.StatusStarting: healthpb.HealthCheckResponse_NOT_SERVING, + component.StatusOK: healthpb.HealthCheckResponse_SERVING, + component.StatusRecoverableError: healthpb.HealthCheckResponse_SERVING, + component.StatusPermanentError: healthpb.HealthCheckResponse_SERVING, + component.StatusFatalError: healthpb.HealthCheckResponse_NOT_SERVING, + component.StatusStopping: healthpb.HealthCheckResponse_NOT_SERVING, + component.StatusStopped: healthpb.HealthCheckResponse_NOT_SERVING, + } +) + +func (s *Server) Check( + _ context.Context, + req *healthpb.HealthCheckRequest, +) (*healthpb.HealthCheckResponse, error) { + st, ok := s.aggregator.AggregateStatus(status.Scope(req.Service), status.Concise) + if !ok { + return nil, errNotFound + } + + return &healthpb.HealthCheckResponse{ + Status: s.toServingStatus(st.Event), + }, nil +} + +func (s *Server) Watch(req *healthpb.HealthCheckRequest, stream healthpb.Health_WatchServer) error { + sub := s.aggregator.Subscribe(status.Scope(req.Service), status.Concise) + defer s.aggregator.Unsubscribe(sub) + + var lastServingStatus healthpb.HealthCheckResponse_ServingStatus = -1 + var failureTimer *time.Timer + failureCh := make(chan struct{}) + + for { + select { + case st, ok := <-sub: + if !ok { + return errShuttingDown + } + var sst healthpb.HealthCheckResponse_ServingStatus + + switch { + case st == nil: + sst = healthpb.HealthCheckResponse_SERVICE_UNKNOWN + case s.componentHealthConfig.IncludeRecoverable && + s.componentHealthConfig.RecoveryDuration > 0 && + st.Status() == component.StatusRecoverableError: + if failureTimer == nil { + failureTimer = time.AfterFunc( + s.componentHealthConfig.RecoveryDuration, + func() { failureCh <- struct{}{} }, + ) + } + sst = lastServingStatus + if lastServingStatus == -1 { + sst = healthpb.HealthCheckResponse_SERVING + } + default: + if failureTimer != nil { + if !failureTimer.Stop() { + <-failureTimer.C + } + failureTimer = nil + } + sst = s.toServingStatus(st.Event) + } + + if lastServingStatus == sst { + continue + } + + lastServingStatus = sst + + err := stream.Send(&healthpb.HealthCheckResponse{Status: sst}) + if err != nil { + return errStreamSend + } + case <-failureCh: + failureTimer.Stop() + failureTimer = nil + if lastServingStatus == healthpb.HealthCheckResponse_NOT_SERVING { + continue + } + lastServingStatus = healthpb.HealthCheckResponse_NOT_SERVING + err := stream.Send( + &healthpb.HealthCheckResponse{ + Status: healthpb.HealthCheckResponse_NOT_SERVING, + }, + ) + if err != nil { + return errStreamSend + } + case <-stream.Context().Done(): + return errStreamEnded + } + } +} + +func (s *Server) toServingStatus( + ev status.Event, +) healthpb.HealthCheckResponse_ServingStatus { + if s.componentHealthConfig.IncludeRecoverable && + ev.Status() == component.StatusRecoverableError && + time.Now().After(ev.Timestamp().Add(s.componentHealthConfig.RecoveryDuration)) { + return healthpb.HealthCheckResponse_NOT_SERVING + } + + if s.componentHealthConfig.IncludePermanent && ev.Status() == component.StatusPermanentError { + return healthpb.HealthCheckResponse_NOT_SERVING + } + + return statusToServingStatusMap[ev.Status()] +} diff --git a/extension/healthcheckv2extension/internal/grpc/grpc_test.go b/extension/healthcheckv2extension/internal/grpc/grpc_test.go new file mode 100644 index 0000000000000..79150d0a56cca --- /dev/null +++ b/extension/healthcheckv2extension/internal/grpc/grpc_test.go @@ -0,0 +1,1601 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package grpc + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configgrpc" + "go.opentelemetry.io/collector/config/confignet" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + grpcstatus "google.golang.org/grpc/status" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/testhelpers" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/common/testutil" +) + +func TestCheck(t *testing.T) { + addr := testutil.GetAvailableLocalAddress(t) + config := &Config{ + ServerConfig: configgrpc.ServerConfig{ + NetAddr: confignet.AddrConfig{ + Endpoint: addr, + Transport: "tcp", + }, + }, + } + var server *Server + traces := testhelpers.NewPipelineMetadata("traces") + metrics := testhelpers.NewPipelineMetadata("metrics") + + type teststep struct { + step func() + eventually bool + service string + expectedStatus healthpb.HealthCheckResponse_ServingStatus + expectedErr error + } + + tests := []struct { + name string + config *Config + componentHealthSettings *common.ComponentHealthConfig + teststeps []teststep + }{ + { + name: "exclude recoverable and permanent errors", + config: config, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedErr: grpcstatus.Error(codes.NotFound, "Service not found."), + }, + { + service: metrics.PipelineID.String(), + expectedErr: grpcstatus.Error(codes.NotFound, "Service not found."), + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // errors will be ignored + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + }, + }, + { + name: "include recoverable and exclude permanent errors", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: false, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedErr: grpcstatus.Error(codes.NotFound, "Service not found."), + }, + { + service: metrics.PipelineID.String(), + expectedErr: grpcstatus.Error(codes.NotFound, "Service not found."), + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // metrics and overall status will be NOT_SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: "", + eventually: true, + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + // metrics and overall status will recover and resume SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + service: "", + eventually: true, + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // permament error will be ignored + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + service: "", + eventually: true, + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + service: "", + eventually: true, + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + }, + }, + { + name: "include permanent and exclude recoverable errors", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: true, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedErr: grpcstatus.Error(codes.NotFound, "Service not found."), + }, + { + service: metrics.PipelineID.String(), + expectedErr: grpcstatus.Error(codes.NotFound, "Service not found."), + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // recoverable will be ignored + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // permament error included + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + }, + }, + { + name: "include permanent and recoverable errors", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: true, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedErr: grpcstatus.Error(codes.NotFound, "Service not found."), + }, + { + service: metrics.PipelineID.String(), + expectedErr: grpcstatus.Error(codes.NotFound, "Service not found."), + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // metrics and overall status will be NOT_SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: "", + eventually: true, + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + // metrics and overall status will recover and resume SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + service: "", + eventually: true, + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server = NewServer( + config, + tc.componentHealthSettings, + componenttest.NewNopTelemetrySettings(), + status.NewAggregator(testhelpers.ErrPriority(tc.componentHealthSettings)), + ) + require.NoError(t, server.Start(context.Background(), componenttest.NewNopHost())) + t.Cleanup(func() { require.NoError(t, server.Shutdown(context.Background())) }) + + cc, err := grpc.Dial( + addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + ) + require.NoError(t, err) + defer func() { + assert.NoError(t, cc.Close()) + }() + + client := healthpb.NewHealthClient(cc) + + for _, ts := range tc.teststeps { + if ts.step != nil { + ts.step() + } + + if ts.eventually { + assert.Eventually(t, func() bool { + resp, err := client.Check( + context.Background(), + &healthpb.HealthCheckRequest{Service: ts.service}, + ) + require.NoError(t, err) + return ts.expectedStatus == resp.Status + }, time.Second, 10*time.Millisecond) + continue + } + + resp, err := client.Check( + context.Background(), + &healthpb.HealthCheckRequest{Service: ts.service}, + ) + require.Equal(t, ts.expectedErr, err) + if ts.expectedErr != nil { + continue + } + assert.Equal(t, ts.expectedStatus, resp.Status) + } + }) + } + +} + +func TestWatch(t *testing.T) { + addr := testutil.GetAvailableLocalAddress(t) + config := &Config{ + ServerConfig: configgrpc.ServerConfig{ + NetAddr: confignet.AddrConfig{ + Endpoint: addr, + Transport: "tcp", + }, + }, + } + var server *Server + traces := testhelpers.NewPipelineMetadata("traces") + metrics := testhelpers.NewPipelineMetadata("metrics") + + // statusUnchanged is a sentinel value to signal that a step does not result + // in a status change. This is important, because checking for a status + // change is blocking. + var statusUnchanged healthpb.HealthCheckResponse_ServingStatus = -1 + + type teststep struct { + step func() + service string + expectedStatus healthpb.HealthCheckResponse_ServingStatus + } + + tests := []struct { + name string + config *Config + componentHealthSettings *common.ComponentHealthConfig + teststeps []teststep + }{ + { + name: "exclude recoverable and permanent errors", + config: config, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // errors will be ignored + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: statusUnchanged, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: statusUnchanged, + }, + { + step: func() { + // This will be the last status change for traces (stopping changes to NOT_SERVING) + // Stopped results in the same serving status, and repeat statuses are not streamed. + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + // This will be the last status change for metrics (stopping changes to NOT_SERVING) + // Stopped results in the same serving status, and repeat statuses are not streamed. + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + }, + }, + { + name: "include recoverable and exclude permanent errors", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: false, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // metrics and overall status will be NOT_SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + // metrics and overall status will recover and resume SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // permanent error will be ignored + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: statusUnchanged, + }, + }, + }, + { + name: "exclude permanent errors", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: false, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // permanent error will be ignored + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: statusUnchanged, + }, + }, + }, + { + name: "include recoverable 0s recovery duration", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: false, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // metrics and overall status will be NOT_SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + // metrics and overall status will recover and resume SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // This will be the last status change for traces (stopping changes to NOT_SERVING) + // Stopped results in the same serving status, and repeat statuses are not streamed. + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + // This will be the last status change for metrics (stopping changes to NOT_SERVING) + // Stopped results in the same serving status, and repeat statuses are not streamed. + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + }, + }, + { + name: "include permanent and exclude recoverable errors", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: true, + IncludeRecoverable: false, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // recoverable will be ignored + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: statusUnchanged, + }, + { + step: func() { + // metrics and overall status will recover and resume SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + // This will be the last status change for traces (stopping changes to NOT_SERVING) + // Stopped results in the same serving status, and repeat statuses are not streamed. + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + }, + }, + { + name: "exclude recoverable errors", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: true, + IncludeRecoverable: false, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // recoverable will be ignored + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: statusUnchanged, + }, + }, + }, + { + name: "include recoverable and permanent errors", + config: config, + componentHealthSettings: &common.ComponentHealthConfig{ + IncludePermanent: true, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVICE_UNKNOWN, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + service: traces.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // metrics and overall status will be NOT_SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + step: func() { + // metrics and overall status will recover and resume SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_SERVING, + }, + { + step: func() { + // metrics and overall status will be NOT_SERVING + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + service: metrics.PipelineID.String(), + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + { + service: "", + expectedStatus: healthpb.HealthCheckResponse_NOT_SERVING, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server = NewServer( + config, + tc.componentHealthSettings, + componenttest.NewNopTelemetrySettings(), + status.NewAggregator(testhelpers.ErrPriority(tc.componentHealthSettings)), + ) + require.NoError(t, server.Start(context.Background(), componenttest.NewNopHost())) + t.Cleanup(func() { require.NoError(t, server.Shutdown(context.Background())) }) + + cc, err := grpc.Dial( + addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + ) + require.NoError(t, err) + defer func() { + assert.NoError(t, cc.Close()) + }() + + client := healthpb.NewHealthClient(cc) + watchers := make(map[string]healthpb.Health_WatchClient) + + for _, ts := range tc.teststeps { + if ts.step != nil { + ts.step() + } + + if statusUnchanged == ts.expectedStatus { + continue + } + + watcher, ok := watchers[ts.service] + if !ok { + watcher, err = client.Watch( + context.Background(), + &healthpb.HealthCheckRequest{Service: ts.service}, + ) + require.NoError(t, err) + watchers[ts.service] = watcher + } + + var resp *healthpb.HealthCheckResponse + // Note Recv blocks until there is a new item in the stream + resp, err = watcher.Recv() + require.NoError(t, err) + assert.Equal(t, ts.expectedStatus, resp.Status) + } + + wg := sync.WaitGroup{} + wg.Add(len(watchers)) + + for svc, watcher := range watchers { + svc := svc + watcher := watcher + go func() { + resp, err := watcher.Recv() + // Ensure there are not any unread messages + assert.Nil(t, resp, "%s: had unread messages", svc) + // Ensure watchers receive the cancelation when streams are closed by the server + assert.Equal(t, grpcstatus.Error(codes.Canceled, "Server shutting down."), err) + wg.Done() + }() + } + + // closing the aggregator will gracefully terminate streams of status events + server.aggregator.Close() + wg.Wait() + }) + } +} diff --git a/extension/healthcheckv2extension/internal/grpc/package_test.go b/extension/healthcheckv2extension/internal/grpc/package_test.go new file mode 100644 index 0000000000000..0a083f58c642a --- /dev/null +++ b/extension/healthcheckv2extension/internal/grpc/package_test.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package grpc // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/grpc" + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/extension/healthcheckv2extension/internal/grpc/server.go b/extension/healthcheckv2extension/internal/grpc/server.go new file mode 100644 index 0000000000000..eb0a0f0fd8e7c --- /dev/null +++ b/extension/healthcheckv2extension/internal/grpc/server.go @@ -0,0 +1,79 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package grpc // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/grpc" + +import ( + "context" + "errors" + + "go.opentelemetry.io/collector/component" + "google.golang.org/grpc" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" +) + +type Server struct { + healthpb.UnimplementedHealthServer + grpcServer *grpc.Server + aggregator *status.Aggregator + config *Config + componentHealthConfig *common.ComponentHealthConfig + telemetry component.TelemetrySettings + doneCh chan struct{} +} + +var _ component.Component = (*Server)(nil) + +func NewServer( + config *Config, + componentHealthConfig *common.ComponentHealthConfig, + telemetry component.TelemetrySettings, + aggregator *status.Aggregator, +) *Server { + srv := &Server{ + config: config, + componentHealthConfig: componentHealthConfig, + telemetry: telemetry, + aggregator: aggregator, + doneCh: make(chan struct{}), + } + if srv.componentHealthConfig == nil { + srv.componentHealthConfig = &common.ComponentHealthConfig{} + } + return srv +} + +// Start implements the component.Component interface. +func (s *Server) Start(ctx context.Context, host component.Host) error { + var err error + s.grpcServer, err = s.config.ToServer(ctx, host, s.telemetry) + if err != nil { + return err + } + + healthpb.RegisterHealthServer(s.grpcServer, s) + ln, err := s.config.NetAddr.Listen(context.Background()) + + go func() { + defer close(s.doneCh) + + if err = s.grpcServer.Serve(ln); err != nil && !errors.Is(err, grpc.ErrServerStopped) { + s.telemetry.ReportStatus(component.NewPermanentErrorEvent(err)) + } + }() + + return nil +} + +// Shutdown implements the component.Component interface. +func (s *Server) Shutdown(context.Context) error { + if s.grpcServer == nil { + return nil + } + s.grpcServer.GracefulStop() + <-s.doneCh + return nil +} diff --git a/extension/healthcheckv2extension/internal/http/handlers.go b/extension/healthcheckv2extension/internal/http/handlers.go new file mode 100644 index 0000000000000..1aaf9f713f670 --- /dev/null +++ b/extension/healthcheckv2extension/internal/http/handlers.go @@ -0,0 +1,40 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package http // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/http" + +import ( + "net/http" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" +) + +func (s *Server) statusHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pipeline := r.URL.Query().Get("pipeline") + verbose := r.URL.Query().Has("verbose") + st, ok := s.aggregator.AggregateStatus(status.Scope(pipeline), status.Verbosity(verbose)) + + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + + s.responder.respond(st, w) + }) +} + +func (s *Server) configHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conf := s.colconf.Load() + + if conf == nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(conf.([]byte)) + }) +} diff --git a/extension/healthcheckv2extension/internal/http/package_test.go b/extension/healthcheckv2extension/internal/http/package_test.go new file mode 100644 index 0000000000000..9f1f2d2d2c7a9 --- /dev/null +++ b/extension/healthcheckv2extension/internal/http/package_test.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package http // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/http" + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/extension/healthcheckv2extension/internal/http/responders.go b/extension/healthcheckv2extension/internal/http/responders.go new file mode 100644 index 0000000000000..4204b5517d972 --- /dev/null +++ b/extension/healthcheckv2extension/internal/http/responders.go @@ -0,0 +1,147 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package http // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/http" + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "go.opentelemetry.io/collector/component" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" +) + +var responseCodes = map[component.Status]int{ + component.StatusNone: http.StatusServiceUnavailable, + component.StatusStarting: http.StatusServiceUnavailable, + component.StatusOK: http.StatusOK, + component.StatusRecoverableError: http.StatusOK, + component.StatusPermanentError: http.StatusOK, + component.StatusFatalError: http.StatusInternalServerError, + component.StatusStopping: http.StatusServiceUnavailable, + component.StatusStopped: http.StatusServiceUnavailable, +} + +type responder interface { + respond(*status.AggregateStatus, http.ResponseWriter) +} + +type responderFunc func(*status.AggregateStatus, http.ResponseWriter) + +func (f responderFunc) respond(st *status.AggregateStatus, w http.ResponseWriter) { + f(st, w) +} + +func respondWithJSON(code int, content any, w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + + body, _ := json.Marshal(content) + _, _ = w.Write(body) +} + +func defaultResponder(startTimestamp *time.Time) responderFunc { + return func(st *status.AggregateStatus, w http.ResponseWriter) { + code := responseCodes[st.Status()] + sst := toSerializableStatus(st, &serializationOptions{ + includeStartTime: true, + startTimestamp: startTimestamp, + }) + respondWithJSON(code, sst, w) + } +} + +func componentHealthResponder( + startTimestamp *time.Time, + config *common.ComponentHealthConfig, +) responderFunc { + healthyFunc := func(now *time.Time) func(status.Event) bool { + return func(ev status.Event) bool { + if ev.Status() == component.StatusPermanentError { + return !config.IncludePermanent + } + + if ev.Status() == component.StatusRecoverableError && config.IncludeRecoverable { + return now.Before(ev.Timestamp().Add(config.RecoveryDuration)) + } + + return ev.Status() != component.StatusFatalError + } + } + return func(st *status.AggregateStatus, w http.ResponseWriter) { + now := time.Now() + sst := toSerializableStatus( + st, + &serializationOptions{ + includeStartTime: true, + startTimestamp: startTimestamp, + healthyFunc: healthyFunc(&now), + }, + ) + + code := responseCodes[st.Status()] + if !sst.Healthy { + code = http.StatusInternalServerError + } + + respondWithJSON(code, sst, w) + } +} + +// Below are responders ported from the original healthcheck extension. We will +// keep them for backwards compatibility, but eventually deprecate and remove +// them. + +// legacyResponseCodes match the current response code mapping with the exception +// of FatalError, which maps to 503 instead of 500. +var legacyResponseCodes = map[component.Status]int{ + component.StatusNone: http.StatusServiceUnavailable, + component.StatusStarting: http.StatusServiceUnavailable, + component.StatusOK: http.StatusOK, + component.StatusRecoverableError: http.StatusOK, + component.StatusPermanentError: http.StatusOK, + component.StatusFatalError: http.StatusServiceUnavailable, + component.StatusStopping: http.StatusServiceUnavailable, + component.StatusStopped: http.StatusServiceUnavailable, +} + +func legacyDefaultResponder(startTimestamp *time.Time) responderFunc { + type healthCheckResponse struct { + StatusMsg string `json:"status"` + UpSince time.Time `json:"upSince"` + Uptime string `json:"uptime"` + } + + codeToMsgMap := map[int]string{ + http.StatusOK: "Server available", + http.StatusServiceUnavailable: "Server not available", + } + + return func(st *status.AggregateStatus, w http.ResponseWriter) { + code := legacyResponseCodes[st.Status()] + resp := healthCheckResponse{ + StatusMsg: codeToMsgMap[code], + } + if code == http.StatusOK { + resp.UpSince = *startTimestamp + resp.Uptime = fmt.Sprintf("%v", time.Since(*startTimestamp)) + } + respondWithJSON(code, resp, w) + } +} + +func legacyCustomResponder(config *ResponseBodyConfig) responderFunc { + codeToMsgMap := map[int][]byte{ + http.StatusOK: []byte(config.Healthy), + http.StatusServiceUnavailable: []byte(config.Unhealthy), + } + return func(st *status.AggregateStatus, w http.ResponseWriter) { + code := legacyResponseCodes[st.Status()] + w.WriteHeader(code) + _, _ = w.Write(codeToMsgMap[code]) + } +} diff --git a/extension/healthcheckv2extension/internal/http/serialization.go b/extension/healthcheckv2extension/internal/http/serialization.go new file mode 100644 index 0000000000000..6b00933dc9884 --- /dev/null +++ b/extension/healthcheckv2extension/internal/http/serialization.go @@ -0,0 +1,95 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package http // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/http" + +import ( + "time" + + "go.opentelemetry.io/collector/component" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" +) + +type healthyFunc func(status.Event) bool + +func (f healthyFunc) isHealthy(ev status.Event) bool { + if f != nil { + return f(ev) + } + return true +} + +type serializationOptions struct { + includeStartTime bool + startTimestamp *time.Time + healthyFunc healthyFunc +} + +type serializableStatus struct { + StartTimestamp *time.Time `json:"start_time,omitempty"` + *SerializableEvent + ComponentStatuses map[string]*serializableStatus `json:"components,omitempty"` +} + +// SerializableEvent is exported for json.Unmarshal +type SerializableEvent struct { + Healthy bool `json:"healthy"` + StatusString string `json:"status"` + Error string `json:"error,omitempty"` + Timestamp time.Time `json:"status_time"` +} + +var stringToStatusMap = map[string]component.Status{ + "StatusNone": component.StatusNone, + "StatusStarting": component.StatusStarting, + "StatusOK": component.StatusOK, + "StatusRecoverableError": component.StatusRecoverableError, + "StatusPermanentError": component.StatusPermanentError, + "StatusFatalError": component.StatusFatalError, + "StatusStopping": component.StatusStopping, + "StatusStopped": component.StatusStopped, +} + +func (ev *SerializableEvent) Status() component.Status { + if st, ok := stringToStatusMap[ev.StatusString]; ok { + return st + } + return component.StatusNone +} + +func toSerializableEvent(ev status.Event, isHealthy bool) *SerializableEvent { + se := &SerializableEvent{ + Healthy: isHealthy, + StatusString: ev.Status().String(), + Timestamp: ev.Timestamp(), + } + if ev.Err() != nil { + se.Error = ev.Err().Error() + } + return se +} + +func toSerializableStatus( + st *status.AggregateStatus, + opts *serializationOptions, +) *serializableStatus { + s := &serializableStatus{ + SerializableEvent: toSerializableEvent( + st.Event, + opts.healthyFunc.isHealthy(st.Event), + ), + ComponentStatuses: make(map[string]*serializableStatus), + } + + if opts.includeStartTime { + s.StartTimestamp = opts.startTimestamp + opts.includeStartTime = false + } + + for k, cs := range st.ComponentStatusMap { + s.ComponentStatuses[k] = toSerializableStatus(cs, opts) + } + + return s +} diff --git a/extension/healthcheckv2extension/internal/http/server.go b/extension/healthcheckv2extension/internal/http/server.go new file mode 100644 index 0000000000000..1ae6663799681 --- /dev/null +++ b/extension/healthcheckv2extension/internal/http/server.go @@ -0,0 +1,125 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package http // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/http" + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "sync/atomic" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/extension" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" +) + +type Server struct { + telemetry component.TelemetrySettings + httpConfig confighttp.ServerConfig + httpServer *http.Server + mux *http.ServeMux + responder responder + colconf atomic.Value + aggregator *status.Aggregator + startTimestamp time.Time + doneCh chan struct{} +} + +var _ component.Component = (*Server)(nil) +var _ extension.ConfigWatcher = (*Server)(nil) + +func NewServer( + config *Config, + legacyConfig LegacyConfig, + componentHealthConfig *common.ComponentHealthConfig, + telemetry component.TelemetrySettings, + aggregator *status.Aggregator, +) *Server { + now := time.Now() + srv := &Server{ + telemetry: telemetry, + mux: http.NewServeMux(), + aggregator: aggregator, + doneCh: make(chan struct{}), + } + + if legacyConfig.UseV2 { + srv.httpConfig = config.ServerConfig + if componentHealthConfig != nil { + srv.responder = componentHealthResponder(&now, componentHealthConfig) + } else { + srv.responder = defaultResponder(&now) + } + if config.Status.Enabled { + srv.mux.Handle(config.Status.Path, srv.statusHandler()) + } + if config.Config.Enabled { + srv.mux.Handle(config.Config.Path, srv.configHandler()) + } + } else { + srv.httpConfig = legacyConfig.ServerConfig + if legacyConfig.ResponseBody != nil { + srv.responder = legacyCustomResponder(legacyConfig.ResponseBody) + } else { + srv.responder = legacyDefaultResponder(&now) + } + srv.mux.Handle(legacyConfig.Path, srv.statusHandler()) + } + + return srv +} + +// Start implements the component.Component interface. +func (s *Server) Start(ctx context.Context, host component.Host) error { + var err error + s.startTimestamp = time.Now() + + s.httpServer, err = s.httpConfig.ToServer(ctx, host, s.telemetry, s.mux) + if err != nil { + return err + } + + ln, err := s.httpConfig.ToListener(ctx) + if err != nil { + return fmt.Errorf("failed to bind to address %s: %w", s.httpConfig.Endpoint, err) + } + + go func() { + defer close(s.doneCh) + if err = s.httpServer.Serve(ln); !errors.Is(err, http.ErrServerClosed) && err != nil { + s.telemetry.ReportStatus(component.NewPermanentErrorEvent(err)) + } + }() + + return nil +} + +// Shutdown implements the component.Component interface. +func (s *Server) Shutdown(context.Context) error { + if s.httpServer == nil { + return nil + } + s.httpServer.Close() + <-s.doneCh + return nil +} + +// NotifyConfig implements the extension.ConfigWatcher interface. +func (s *Server) NotifyConfig(_ context.Context, conf *confmap.Conf) error { + confBytes, err := json.Marshal(conf.ToStringMap()) + if err != nil { + s.telemetry.Logger.Warn("could not marshal config", zap.Error(err)) + return err + } + s.colconf.Store(confBytes) + return nil +} diff --git a/extension/healthcheckv2extension/internal/http/server_test.go b/extension/healthcheckv2extension/internal/http/server_test.go new file mode 100644 index 0000000000000..6bd70e69669e3 --- /dev/null +++ b/extension/healthcheckv2extension/internal/http/server_test.go @@ -0,0 +1,4089 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package http + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/confmap/confmaptest" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/testhelpers" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/common/testutil" +) + +// These are used for the legacy test assertions +const ( + expectedBodyNotReady = "{\"status\":\"Server not available\",\"upSince\":" + expectedBodyReady = "{\"status\":\"Server available\",\"upSince\":" +) + +type componentStatusExpectation struct { + healthy bool + status component.Status + err error + nestedStatus map[string]*componentStatusExpectation +} + +type teststep struct { + step func() + queryParams string + eventually bool + expectedStatusCode int + expectedBody string + expectedComponentStatus *componentStatusExpectation +} + +func TestStatus(t *testing.T) { + var server *Server + traces := testhelpers.NewPipelineMetadata("traces") + metrics := testhelpers.NewPipelineMetadata("metrics") + + tests := []struct { + name string + config *Config + legacyConfig LegacyConfig + componentHealthConfig *common.ComponentHealthConfig + pipelines map[string]*testhelpers.PipelineMetadata + teststeps []teststep + }{ + { + name: "exclude recoverable and permanent errors", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator(server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator(server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + { + name: "exclude recoverable and permanent errors - verbose", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator(server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator(server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusRecoverableError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusPermanentError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusPermanentError, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + }, + }, + { + name: "include recoverable and exclude permanent errors", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + componentHealthConfig: &common.ComponentHealthConfig{ + IncludePermanent: false, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + eventually: true, + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + { + name: "include recoverable and exclude permanent errors - verbose", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + componentHealthConfig: &common.ComponentHealthConfig{ + IncludePermanent: false, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator(server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator(server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + eventually: true, + queryParams: "verbose", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: false, + status: component.StatusRecoverableError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusPermanentError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusPermanentError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusPermanentError, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + }, + }, + { + name: "include permanent and exclude recoverable errors", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + componentHealthConfig: &common.ComponentHealthConfig{ + IncludePermanent: true, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + { + name: "include permanent and exclude recoverable errors - verbose", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + componentHealthConfig: &common.ComponentHealthConfig{ + IncludePermanent: true, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator(server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator(server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + eventually: true, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusRecoverableError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: false, + status: component.StatusPermanentError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: false, + status: component.StatusPermanentError, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + }, + }, + { + name: "include permanent and recoverable errors", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + componentHealthConfig: &common.ComponentHealthConfig{ + IncludePermanent: true, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + eventually: true, + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + { + queryParams: "pipeline=traces", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + { + queryParams: "pipeline=metrics", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + { + name: "include permanent and recoverable errors - verbose", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + componentHealthConfig: &common.ComponentHealthConfig{ + IncludePermanent: true, + IncludeRecoverable: true, + RecoveryDuration: 2 * time.Millisecond, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator(server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator(server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStarting, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStarting, + }, + "processor:batch": { + healthy: true, + status: component.StatusStarting, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStarting, + }, + }, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + eventually: true, + queryParams: "verbose", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: false, + status: component.StatusRecoverableError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: false, + status: component.StatusRecoverableError, + err: assert.AnError, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusOK, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusOK, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusOK, + }, + }, + }, + "pipeline:metrics": { + healthy: false, + status: component.StatusPermanentError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusInternalServerError, + expectedComponentStatus: &componentStatusExpectation{ + healthy: false, + status: component.StatusPermanentError, + err: assert.AnError, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusOK, + }, + "processor:batch": { + healthy: true, + status: component.StatusOK, + }, + "exporter:metrics/out": { + healthy: false, + status: component.StatusPermanentError, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopping, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopping, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopping, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopping, + }, + }, + }, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + queryParams: "verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "pipeline:traces": { + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + "pipeline:metrics": { + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + }, + }, + { + queryParams: "pipeline=traces&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:traces/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:traces/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + { + queryParams: "pipeline=metrics&verbose", + expectedStatusCode: http.StatusServiceUnavailable, + expectedComponentStatus: &componentStatusExpectation{ + healthy: true, + status: component.StatusStopped, + nestedStatus: map[string]*componentStatusExpectation{ + "receiver:metrics/in": { + healthy: true, + status: component.StatusStopped, + }, + "processor:batch": { + healthy: true, + status: component.StatusStopped, + }, + "exporter:metrics/out": { + healthy: true, + status: component.StatusStopped, + }, + }, + }, + }, + }, + }, + { + name: "pipeline non-existent", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: true, + Path: "/status", + }, + }, + pipelines: testhelpers.NewPipelines("traces"), + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + queryParams: "pipeline=nonexistent", + expectedStatusCode: http.StatusNotFound, + }, + }, + }, + { + name: "status disabled", + legacyConfig: LegacyConfig{UseV2: true}, + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{Enabled: false}, + Status: PathConfig{ + Enabled: false, + }, + }, + teststeps: []teststep{ + { + expectedStatusCode: http.StatusNotFound, + }, + }, + }, + { + name: "legacy - default response", + legacyConfig: LegacyConfig{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Path: "/status", + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator(server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator(server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: expectedBodyNotReady, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: expectedBodyNotReady, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusOK, + expectedBody: expectedBodyReady, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusOK, + expectedBody: expectedBodyReady, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + expectedStatusCode: http.StatusOK, + expectedBody: expectedBodyReady, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusOK, + expectedBody: expectedBodyReady, + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewFatalErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: expectedBodyNotReady, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: expectedBodyNotReady, + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: expectedBodyNotReady, + }, + }, + }, + { + name: "legacy - custom response", + legacyConfig: LegacyConfig{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Path: "/status", + ResponseBody: &ResponseBodyConfig{Healthy: "ALL OK", Unhealthy: "NOT OK"}, + }, + teststeps: []teststep{ + { + step: func() { + testhelpers.SeedAggregator(server.aggregator, + traces.InstanceIDs(), + component.StatusStarting, + ) + testhelpers.SeedAggregator(server.aggregator, + metrics.InstanceIDs(), + component.StatusStarting, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: "NOT OK", + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: "NOT OK", + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusOK, + ) + }, + expectedStatusCode: http.StatusOK, + expectedBody: "ALL OK", + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusOK, + expectedBody: "ALL OK", + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewStatusEvent(component.StatusOK), + ) + }, + expectedStatusCode: http.StatusOK, + expectedBody: "ALL OK", + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusOK, + expectedBody: "ALL OK", + }, + { + step: func() { + server.aggregator.RecordStatus( + metrics.ExporterID, + component.NewFatalErrorEvent(assert.AnError), + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: "NOT OK", + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopping, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopping, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: "NOT OK", + }, + { + step: func() { + testhelpers.SeedAggregator( + server.aggregator, + traces.InstanceIDs(), + component.StatusStopped, + ) + testhelpers.SeedAggregator( + server.aggregator, + metrics.InstanceIDs(), + component.StatusStopped, + ) + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: "NOT OK", + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server = NewServer( + tc.config, + tc.legacyConfig, + tc.componentHealthConfig, + componenttest.NewNopTelemetrySettings(), + status.NewAggregator(testhelpers.ErrPriority(tc.componentHealthConfig)), + ) + + require.NoError(t, server.Start(context.Background(), componenttest.NewNopHost())) + defer func() { require.NoError(t, server.Shutdown(context.Background())) }() + + var url string + if tc.legacyConfig.UseV2 { + url = fmt.Sprintf("http://%s%s", tc.config.Endpoint, tc.config.Status.Path) + } else { + url = fmt.Sprintf("http://%s%s", tc.legacyConfig.Endpoint, tc.legacyConfig.Path) + } + + client := &http.Client{} + + for _, ts := range tc.teststeps { + if ts.step != nil { + ts.step() + } + + stepURL := url + if ts.queryParams != "" { + stepURL = fmt.Sprintf("%s?%s", stepURL, ts.queryParams) + } + + var err error + var resp *http.Response + + if ts.eventually { + assert.Eventually(t, func() bool { + resp, err = client.Get(stepURL) + require.NoError(t, err) + return ts.expectedStatusCode == resp.StatusCode + }, time.Second, 10*time.Millisecond) + } else { + resp, err = client.Get(stepURL) + require.NoError(t, err) + assert.Equal(t, ts.expectedStatusCode, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + assert.True(t, strings.Contains(string(body), ts.expectedBody)) + + if ts.expectedComponentStatus != nil { + st := &serializableStatus{} + require.NoError(t, json.Unmarshal(body, st)) + if strings.Contains(ts.queryParams, "verbose") { + assertStatusDetailed(t, ts.expectedComponentStatus, st) + continue + } + assertStatusSimple(t, ts.expectedComponentStatus, st) + } + } + }) + } +} + +func assertStatusDetailed( + t *testing.T, + expected *componentStatusExpectation, + actual *serializableStatus, +) { + assert.Equal(t, expected.healthy, actual.Healthy) + assert.Equal(t, expected.status, actual.Status(), + "want: %s, got: %s", expected.status, actual.Status()) + if expected.err != nil { + assert.Equal(t, expected.err.Error(), actual.Error) + } + assertNestedStatus(t, expected.nestedStatus, actual.ComponentStatuses) +} + +func assertNestedStatus( + t *testing.T, + expected map[string]*componentStatusExpectation, + actual map[string]*serializableStatus, +) { + for k, expectation := range expected { + st, ok := actual[k] + require.True(t, ok, "status for key: %s not found", k) + assert.Equal(t, expectation.healthy, st.Healthy) + assert.Equal(t, expectation.status, st.Status(), + "want: %s, got: %s", expectation.status, st.Status()) + if expectation.err != nil { + assert.Equal(t, expectation.err.Error(), st.Error) + } + assertNestedStatus(t, expectation.nestedStatus, st.ComponentStatuses) + } +} + +func assertStatusSimple( + t *testing.T, + expected *componentStatusExpectation, + actual *serializableStatus, +) { + assert.Equal(t, expected.status, actual.Status()) + assert.Equal(t, expected.healthy, actual.Healthy) + if expected.err != nil { + assert.Equal(t, expected.err.Error(), actual.Error) + } + assert.Nil(t, actual.ComponentStatuses) +} + +func TestConfig(t *testing.T) { + var server *Server + confMap, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) + require.NoError(t, err) + confJSON, err := os.ReadFile(filepath.Clean(filepath.Join("testdata", "config.json"))) + require.NoError(t, err) + + for _, tc := range []struct { + name string + config *Config + setup func() + expectedStatusCode int + expectedBody []byte + }{ + { + name: "config not notified", + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{ + Enabled: true, + Path: "/config", + }, + Status: PathConfig{ + Enabled: false, + }, + }, + expectedStatusCode: http.StatusServiceUnavailable, + expectedBody: []byte{}, + }, + { + name: "config notified", + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{ + Enabled: true, + Path: "/config", + }, + Status: PathConfig{ + Enabled: false, + }, + }, + setup: func() { + require.NoError(t, server.NotifyConfig(context.Background(), confMap)) + }, + expectedStatusCode: http.StatusOK, + expectedBody: confJSON, + }, + { + name: "config disabled", + config: &Config{ + ServerConfig: confighttp.ServerConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + }, + Config: PathConfig{ + Enabled: false, + }, + Status: PathConfig{ + Enabled: false, + }, + }, + expectedStatusCode: http.StatusNotFound, + expectedBody: []byte("404 page not found\n"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + server = NewServer( + tc.config, + LegacyConfig{UseV2: true}, + &common.ComponentHealthConfig{}, + componenttest.NewNopTelemetrySettings(), + status.NewAggregator(status.PriorityPermanent), + ) + + require.NoError(t, server.Start(context.Background(), componenttest.NewNopHost())) + defer func() { require.NoError(t, server.Shutdown(context.Background())) }() + + client := &http.Client{} + url := fmt.Sprintf("http://%s%s", tc.config.Endpoint, tc.config.Config.Path) + + if tc.setup != nil { + tc.setup() + } + + resp, err := client.Get(url) + require.NoError(t, err) + assert.Equal(t, tc.expectedStatusCode, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, tc.expectedBody, body) + }) + } + +} diff --git a/extension/healthcheckv2extension/internal/http/testdata/config.json b/extension/healthcheckv2extension/internal/http/testdata/config.json new file mode 100644 index 0000000000000..55dc317f7c4da --- /dev/null +++ b/extension/healthcheckv2extension/internal/http/testdata/config.json @@ -0,0 +1 @@ +{"exporters":{"nop":null,"nop/myexporter":null},"extensions":{"nop":null,"nop/myextension":null},"processors":{"nop":null,"nop/myprocessor":null},"receivers":{"nop":null,"nop/myreceiver":null},"service":{"extensions":["nop"],"pipelines":{"traces":{"exporters":["nop"],"processors":["nop"],"receivers":["nop"]}}}} \ No newline at end of file diff --git a/extension/healthcheckv2extension/internal/http/testdata/config.yaml b/extension/healthcheckv2extension/internal/http/testdata/config.yaml new file mode 100644 index 0000000000000..38227d7a68bc4 --- /dev/null +++ b/extension/healthcheckv2extension/internal/http/testdata/config.yaml @@ -0,0 +1,23 @@ +receivers: + nop: + nop/myreceiver: + +processors: + nop: + nop/myprocessor: + +exporters: + nop: + nop/myexporter: + +extensions: + nop: + nop/myextension: + +service: + extensions: [nop] + pipelines: + traces: + receivers: [nop] + processors: [nop] + exporters: [nop] diff --git a/extension/healthcheckv2extension/internal/status/aggregation.go b/extension/healthcheckv2extension/internal/status/aggregation.go new file mode 100644 index 0000000000000..bab7275443193 --- /dev/null +++ b/extension/healthcheckv2extension/internal/status/aggregation.go @@ -0,0 +1,152 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package status // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" + +import ( + "time" + + "go.opentelemetry.io/collector/component" +) + +// statusEvent contains a status and timestamp, and can contain an error. Note: +// this is duplicated from core because we need to be able to "rewrite" the +// timestamps of some events during aggregation. +type statusEvent struct { + status component.Status + err error + timestamp time.Time +} + +// Status returns the Status (enum) associated with the StatusEvent +func (ev *statusEvent) Status() component.Status { + return ev.status +} + +// Err returns the error associated with the StatusEvent. +func (ev *statusEvent) Err() error { + return ev.err +} + +// Timestamp returns the timestamp associated with the StatusEvent +func (ev *statusEvent) Timestamp() time.Time { + return ev.timestamp +} + +type ErrorPriority int + +const ( + PriorityPermanent ErrorPriority = iota + PriorityRecoverable +) + +type aggregationFunc func(*AggregateStatus) Event + +// The purpose of aggregation is to ensure that the most relevant status bubbles +// upwards in the aggregate status. This aggregation func prioritizes lifecycle +// events (including FatalError) over PermanentError and RecoverableError +// events. The priority argument determines the priority of PermanentError +// events vs RecoverableError events. Lifecycle events will have the timestamp +// of the most recent event and error events will have the timestamp of the +// first occurrence. +func newAggregationFunc(priority ErrorPriority) aggregationFunc { + permanentPriorityFunc := func(seen map[component.Status]struct{}) component.Status { + if _, isPermanent := seen[component.StatusPermanentError]; isPermanent { + return component.StatusPermanentError + } + if _, isRecoverable := seen[component.StatusRecoverableError]; isRecoverable { + return component.StatusRecoverableError + } + return component.StatusNone + } + + recoverablePriorityFunc := func(seen map[component.Status]struct{}) component.Status { + if _, isRecoverable := seen[component.StatusRecoverableError]; isRecoverable { + return component.StatusRecoverableError + } + if _, isPermanent := seen[component.StatusPermanentError]; isPermanent { + return component.StatusPermanentError + } + return component.StatusNone + } + + errPriorityFunc := permanentPriorityFunc + if priority == PriorityRecoverable { + errPriorityFunc = recoverablePriorityFunc + } + + statusFunc := func(st *AggregateStatus) component.Status { + seen := make(map[component.Status]struct{}) + for _, cs := range st.ComponentStatusMap { + seen[cs.Status()] = struct{}{} + } + + // All statuses are the same. Note, this will handle StatusOK and StatusStopped as these two + // cases require all components be in the same state. + if len(seen) == 1 { + for st := range seen { + return st + } + } + + // Handle mixed status cases + if _, isFatal := seen[component.StatusFatalError]; isFatal { + return component.StatusFatalError + } + + if _, isStarting := seen[component.StatusStarting]; isStarting { + return component.StatusStarting + } + + if _, isStopping := seen[component.StatusStopping]; isStopping { + return component.StatusStopping + } + + if _, isStopped := seen[component.StatusStopped]; isStopped { + return component.StatusStopping + } + + return errPriorityFunc(seen) + } + + return func(st *AggregateStatus) Event { + var ev, lastEvent, matchingEvent Event + status := statusFunc(st) + isError := component.StatusIsError(status) + + for _, cs := range st.ComponentStatusMap { + ev = cs.Event + if lastEvent == nil || lastEvent.Timestamp().Before(ev.Timestamp()) { + lastEvent = ev + } + if status == ev.Status() { + switch { + case matchingEvent == nil: + matchingEvent = ev + case isError: + if ev.Timestamp().Before(matchingEvent.Timestamp()) { + matchingEvent = ev + } + case ev.Timestamp().After(matchingEvent.Timestamp()): + matchingEvent = ev + } + } + } + + // the error status will be the first matching event + if isError { + return matchingEvent + } + + // the aggregate status matches an existing event + if lastEvent.Status() == status { + return lastEvent + } + + // the aggregate status requires a synthetic event + return &statusEvent{ + status: status, + timestamp: lastEvent.Timestamp(), + } + } +} diff --git a/extension/healthcheckv2extension/internal/status/aggregation_test.go b/extension/healthcheckv2extension/internal/status/aggregation_test.go new file mode 100644 index 0000000000000..9c37768341bd5 --- /dev/null +++ b/extension/healthcheckv2extension/internal/status/aggregation_test.go @@ -0,0 +1,127 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/component" +) + +func TestAggregationFuncs(t *testing.T) { + aggRecoverable := newAggregationFunc(PriorityRecoverable) + aggPermanent := newAggregationFunc(PriorityPermanent) + + type statusExpectation struct { + priorityPermanent component.Status + priorityRecoverable component.Status + } + + for _, tc := range []struct { + name string + aggregateStatus *AggregateStatus + expectedStatus *statusExpectation + }{ + { + name: "FatalError takes precedence over all", + aggregateStatus: &AggregateStatus{ + ComponentStatusMap: map[string]*AggregateStatus{ + "c1": { + Event: component.NewStatusEvent(component.StatusFatalError), + }, + "c2": { + Event: component.NewStatusEvent(component.StatusStarting), + }, + "c3": { + Event: component.NewStatusEvent(component.StatusOK), + }, + "c4": { + Event: component.NewStatusEvent(component.StatusRecoverableError), + }, + "c5": { + Event: component.NewStatusEvent(component.StatusPermanentError), + }, + "c6": { + Event: component.NewStatusEvent(component.StatusStopping), + }, + "c7": { + Event: component.NewStatusEvent(component.StatusStopped), + }, + }, + }, + expectedStatus: &statusExpectation{ + priorityPermanent: component.StatusFatalError, + priorityRecoverable: component.StatusFatalError, + }, + }, + { + name: "Lifecycle: Starting takes precedence over non-fatal errors", + aggregateStatus: &AggregateStatus{ + ComponentStatusMap: map[string]*AggregateStatus{ + "c1": { + Event: component.NewStatusEvent(component.StatusStarting), + }, + "c2": { + Event: component.NewStatusEvent(component.StatusRecoverableError), + }, + "c3": { + Event: component.NewStatusEvent(component.StatusPermanentError), + }, + }, + }, + expectedStatus: &statusExpectation{ + priorityPermanent: component.StatusStarting, + priorityRecoverable: component.StatusStarting, + }, + }, + { + name: "Lifecycle: Stopping takes precedence over non-fatal errors", + aggregateStatus: &AggregateStatus{ + ComponentStatusMap: map[string]*AggregateStatus{ + "c1": { + Event: component.NewStatusEvent(component.StatusStopping), + }, + "c2": { + Event: component.NewStatusEvent(component.StatusRecoverableError), + }, + "c3": { + Event: component.NewStatusEvent(component.StatusPermanentError), + }, + }, + }, + expectedStatus: &statusExpectation{ + priorityPermanent: component.StatusStopping, + priorityRecoverable: component.StatusStopping, + }, + }, + { + name: "Prioritized error takes priority over OK", + aggregateStatus: &AggregateStatus{ + ComponentStatusMap: map[string]*AggregateStatus{ + "c1": { + Event: component.NewStatusEvent(component.StatusOK), + }, + "c2": { + Event: component.NewStatusEvent(component.StatusRecoverableError), + }, + "c3": { + Event: component.NewStatusEvent(component.StatusPermanentError), + }, + }, + }, + expectedStatus: &statusExpectation{ + priorityPermanent: component.StatusPermanentError, + priorityRecoverable: component.StatusRecoverableError, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expectedStatus.priorityPermanent, + aggPermanent(tc.aggregateStatus).Status()) + assert.Equal(t, tc.expectedStatus.priorityRecoverable, + aggRecoverable(tc.aggregateStatus).Status()) + }) + } +} diff --git a/extension/healthcheckv2extension/internal/status/aggregator.go b/extension/healthcheckv2extension/internal/status/aggregator.go new file mode 100644 index 0000000000000..004dbf5447a7d --- /dev/null +++ b/extension/healthcheckv2extension/internal/status/aggregator.go @@ -0,0 +1,225 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package status // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" + +import ( + "fmt" + "strings" + "sync" + "time" + + "go.opentelemetry.io/collector/component" +) + +// Extensions are treated as a pseudo pipeline and extsID is used as a map key +var ( + extsID = component.MustNewID("extensions") + extsIDMap = map[component.ID]struct{}{extsID: {}} +) + +// Note: this interface had to be introduced because we need to be able to rewrite the +// timestamps of some events during aggregation. The implementation in core doesn't currently +// allow this, but this interface provides a workaround. +type Event interface { + Status() component.Status + Err() error + Timestamp() time.Time +} + +// Scope refers to a part of an AggregateStatus. The zero-value, aka ScopeAll, +// refers to the entire AggregateStatus. ScopeExtensions refers to the extensions +// subtree, and any other value refers to a pipeline subtree. +type Scope string + +const ( + ScopeAll Scope = "" + ScopeExtensions Scope = "extensions" + pipelinePrefix string = "pipeline:" +) + +func (s Scope) toKey() string { + if s == ScopeAll || s == ScopeExtensions { + return string(s) + } + return pipelinePrefix + string(s) +} + +type Verbosity bool + +const ( + Verbose Verbosity = true + Concise = false +) + +// AggregateStatus contains a map of child AggregateStatuses and an embedded component.StatusEvent. +// It can be used to represent a single, top-level status when the ComponentStatusMap is empty, +// or a nested structure when map is non-empty. +type AggregateStatus struct { + Event + + ComponentStatusMap map[string]*AggregateStatus +} + +func (a *AggregateStatus) clone(verbosity Verbosity) *AggregateStatus { + st := &AggregateStatus{ + Event: a.Event, + } + + if verbosity == Verbose && len(a.ComponentStatusMap) > 0 { + st.ComponentStatusMap = make(map[string]*AggregateStatus, len(a.ComponentStatusMap)) + for k, cs := range a.ComponentStatusMap { + st.ComponentStatusMap[k] = cs.clone(verbosity) + } + } + + return st +} + +type subscription struct { + statusCh chan *AggregateStatus + verbosity Verbosity +} + +// Aggregator records individual status events for components and aggregates statuses for the +// pipelines they belong to and the collector overall. +type Aggregator struct { + mu sync.RWMutex + aggregateStatus *AggregateStatus + subscriptions map[string][]*subscription + aggregationFunc aggregationFunc +} + +// NewAggregator returns a *status.Aggregator. +func NewAggregator(errPriority ErrorPriority) *Aggregator { + return &Aggregator{ + aggregateStatus: &AggregateStatus{ + Event: &component.StatusEvent{}, + ComponentStatusMap: make(map[string]*AggregateStatus), + }, + subscriptions: make(map[string][]*subscription), + aggregationFunc: newAggregationFunc(errPriority), + } +} + +// AggregateStatus returns an *AggregateStatus for the given scope. The scope can be the collector +// overall (ScopeAll), extensions (ScopeExtensions), or a pipeline by name. Detail specifies whether +// or not subtrees should be returned with the *AggregateStatus. The boolean return value indicates +// whether or not the scope was found. +func (a *Aggregator) AggregateStatus(scope Scope, verbosity Verbosity) (*AggregateStatus, bool) { + a.mu.Lock() + defer a.mu.Unlock() + + if scope == ScopeAll { + return a.aggregateStatus.clone(verbosity), true + } + + st, ok := a.aggregateStatus.ComponentStatusMap[scope.toKey()] + if !ok { + return nil, false + } + + return st.clone(verbosity), true +} + +// RecordStatus stores and aggregates a StatusEvent for the given component instance. +func (a *Aggregator) RecordStatus(source *component.InstanceID, event *component.StatusEvent) { + compIDs := source.PipelineIDs + // extensions are treated as a pseudo-pipeline + if source.Kind == component.KindExtension { + compIDs = extsIDMap + } + + a.mu.Lock() + defer a.mu.Unlock() + + for compID := range compIDs { + var pipelineStatus *AggregateStatus + pipelineScope := Scope(compID.String()) + pipelineKey := pipelineScope.toKey() + + pipelineStatus, ok := a.aggregateStatus.ComponentStatusMap[pipelineKey] + if !ok { + pipelineStatus = &AggregateStatus{ + ComponentStatusMap: make(map[string]*AggregateStatus), + } + } + + componentKey := fmt.Sprintf("%s:%s", strings.ToLower(source.Kind.String()), source.ID) + pipelineStatus.ComponentStatusMap[componentKey] = &AggregateStatus{ + Event: event, + } + a.aggregateStatus.ComponentStatusMap[pipelineKey] = pipelineStatus + pipelineStatus.Event = a.aggregationFunc(pipelineStatus) + a.notifySubscribers(pipelineScope, pipelineStatus) + } + + a.aggregateStatus.Event = a.aggregationFunc(a.aggregateStatus) + a.notifySubscribers(ScopeAll, a.aggregateStatus) +} + +// Subscribe allows you to subscribe to a stream of events for the given scope. The scope can be +// the collector overall (ScopeAll), extensions (ScopeExtensions), or a pipeline name. +// It is possible to subscribe to a pipeline that has not yet reported. An initial nil +// will be sent on the channel and events will start streaming if and when it starts reporting. +// Detail specifies whether or not subtrees should be returned with the *AggregateStatus. +func (a *Aggregator) Subscribe(scope Scope, verbosity Verbosity) <-chan *AggregateStatus { + a.mu.Lock() + defer a.mu.Unlock() + + key := scope.toKey() + st := a.aggregateStatus + if scope != ScopeAll { + st = st.ComponentStatusMap[key] + } + if st != nil { + st = st.clone(verbosity) + } + sub := &subscription{ + statusCh: make(chan *AggregateStatus, 1), + verbosity: verbosity, + } + + a.subscriptions[key] = append(a.subscriptions[key], sub) + sub.statusCh <- st + + return sub.statusCh +} + +// Unbsubscribe removes a stream from further status updates. +func (a *Aggregator) Unsubscribe(statusCh <-chan *AggregateStatus) { + a.mu.Lock() + defer a.mu.Unlock() + + for scope, subs := range a.subscriptions { + for i, sub := range subs { + if sub.statusCh == statusCh { + a.subscriptions[scope] = append(subs[:i], subs[i+1:]...) + return + } + } + } +} + +// Close terminates all existing subscriptions. +func (a *Aggregator) Close() { + a.mu.Lock() + defer a.mu.Unlock() + + for _, subs := range a.subscriptions { + for _, sub := range subs { + close(sub.statusCh) + } + } +} + +func (a *Aggregator) notifySubscribers(scope Scope, status *AggregateStatus) { + for _, sub := range a.subscriptions[scope.toKey()] { + // clear unread events + select { + case <-sub.statusCh: + default: + } + sub.statusCh <- status.clone(sub.verbosity) + } +} diff --git a/extension/healthcheckv2extension/internal/status/aggregator_test.go b/extension/healthcheckv2extension/internal/status/aggregator_test.go new file mode 100644 index 0000000000000..eb8f6f8b2c22a --- /dev/null +++ b/extension/healthcheckv2extension/internal/status/aggregator_test.go @@ -0,0 +1,405 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package status_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/testhelpers" +) + +func TestAggregateStatus(t *testing.T) { + agg := status.NewAggregator(status.PriorityPermanent) + traces := testhelpers.NewPipelineMetadata("traces") + + t.Run("zero value", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + assert.Equal(t, component.StatusNone, st.Status()) + }) + + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusOK) + + t.Run("pipeline statuses all successful", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + assert.Equal(t, component.StatusOK, st.Status()) + }) + + agg.RecordStatus( + traces.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + + t.Run("pipeline with recoverable error", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + assertErrorEventsMatch(t, + component.StatusRecoverableError, + assert.AnError, + st, + ) + }) + + agg.RecordStatus( + traces.ExporterID, + component.NewPermanentErrorEvent(assert.AnError), + ) + + t.Run("pipeline with permanent error", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.ScopeAll, status.Concise) + require.True(t, ok) + assertErrorEventsMatch(t, + component.StatusPermanentError, + assert.AnError, + st, + ) + }) +} + +func TestAggregateStatusVerbose(t *testing.T) { + agg := status.NewAggregator(status.PriorityPermanent) + traces := testhelpers.NewPipelineMetadata("traces") + tracesKey := toPipelineKey(traces.PipelineID) + + t.Run("zero value", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.ScopeAll, status.Verbose) + require.True(t, ok) + assertEventsMatch(t, component.StatusNone, st) + assert.Empty(t, st.ComponentStatusMap) + }) + + // Seed aggregator with successful statuses for pipeline. + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusOK) + + t.Run("pipeline statuses all successful", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.ScopeAll, status.Verbose) + require.True(t, ok) + + // The top-level status and pipeline status match. + assertEventsMatch(t, component.StatusOK, st, st.ComponentStatusMap[tracesKey]) + + // Component statuses match + assertEventsMatch(t, + component.StatusOK, + collectStatuses(st.ComponentStatusMap[tracesKey], traces.InstanceIDs()...)..., + ) + }) + + // Record an error in the traces exporter + agg.RecordStatus( + traces.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + + t.Run("pipeline with exporter error", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.ScopeAll, status.Verbose) + require.True(t, ok) + // The top-level status and pipeline status match. + assertErrorEventsMatch( + t, + component.StatusRecoverableError, + assert.AnError, + st, + st.ComponentStatusMap[tracesKey], + ) + + // Component statuses match + assertEventsMatch(t, + component.StatusOK, + collectStatuses( + st.ComponentStatusMap[tracesKey], traces.ReceiverID, traces.ProcessorID, + )..., + ) + assertErrorEventsMatch(t, + component.StatusRecoverableError, + assert.AnError, + st.ComponentStatusMap[tracesKey].ComponentStatusMap[toComponentKey(traces.ExporterID)], + ) + }) + +} + +func TestPipelineAggregateStatus(t *testing.T) { + agg := status.NewAggregator(status.PriorityPermanent) + traces := testhelpers.NewPipelineMetadata("traces") + + t.Run("non existent pipeline", func(t *testing.T) { + st, ok := agg.AggregateStatus("doesnotexist", status.Concise) + require.Nil(t, st) + require.False(t, ok) + }) + + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusOK) + + t.Run("pipeline exists / status successful", func(t *testing.T) { + st, ok := agg.AggregateStatus( + status.Scope(traces.PipelineID.String()), + status.Concise, + ) + require.True(t, ok) + assertEventsMatch(t, component.StatusOK, st) + }) + + agg.RecordStatus( + traces.ExporterID, + component.NewRecoverableErrorEvent(assert.AnError), + ) + + t.Run("pipeline exists / exporter error", func(t *testing.T) { + st, ok := agg.AggregateStatus( + status.Scope(traces.PipelineID.String()), + status.Concise, + ) + require.True(t, ok) + assertErrorEventsMatch(t, component.StatusRecoverableError, assert.AnError, st) + }) +} + +func TestPipelineAggregateStatusVerbose(t *testing.T) { + agg := status.NewAggregator(status.PriorityPermanent) + traces := testhelpers.NewPipelineMetadata("traces") + + t.Run("non existent pipeline", func(t *testing.T) { + st, ok := agg.AggregateStatus("doesnotexist", status.Verbose) + require.Nil(t, st) + require.False(t, ok) + }) + + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusOK) + + t.Run("pipeline exists / status successful", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.Scope(traces.PipelineID.String()), status.Verbose) + require.True(t, ok) + + // Top-level status matches + assertEventsMatch(t, component.StatusOK, st) + + // Component statuses match + assertEventsMatch(t, component.StatusOK, collectStatuses(st, traces.InstanceIDs()...)...) + }) + + agg.RecordStatus(traces.ExporterID, component.NewRecoverableErrorEvent(assert.AnError)) + + t.Run("pipeline exists / exporter error", func(t *testing.T) { + st, ok := agg.AggregateStatus(status.Scope(traces.PipelineID.String()), status.Verbose) + require.True(t, ok) + + // Top-level status matches + assertErrorEventsMatch(t, component.StatusRecoverableError, assert.AnError, st) + + // Component statuses match + assertEventsMatch(t, + component.StatusOK, + collectStatuses(st, traces.ReceiverID, traces.ProcessorID)..., + ) + assertErrorEventsMatch(t, + component.StatusRecoverableError, + assert.AnError, + st.ComponentStatusMap[toComponentKey(traces.ExporterID)], + ) + }) +} + +func TestStreaming(t *testing.T) { + agg := status.NewAggregator(status.PriorityPermanent) + defer agg.Close() + + traces := testhelpers.NewPipelineMetadata("traces") + metrics := testhelpers.NewPipelineMetadata("metrics") + + traceEvents := agg.Subscribe(status.Scope(traces.PipelineID.String()), status.Concise) + metricEvents := agg.Subscribe(status.Scope(metrics.PipelineID.String()), status.Concise) + allEvents := agg.Subscribe(status.ScopeAll, status.Concise) + + assert.Nil(t, <-traceEvents) + assert.Nil(t, <-metricEvents) + assert.NotNil(t, <-allEvents) + + // Start pipelines + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusStarting) + assertEventsRecvdMatch(t, component.StatusStarting, traceEvents, allEvents) + testhelpers.SeedAggregator(agg, metrics.InstanceIDs(), component.StatusStarting) + assertEventsRecvdMatch(t, component.StatusStarting, metricEvents, allEvents) + + // Successful start + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusOK) + assertEventsRecvdMatch(t, component.StatusOK, traceEvents) + // All is still in StatusStarting until the metrics pipeline reports OK + assertEventsRecvdMatch(t, component.StatusStarting, allEvents) + testhelpers.SeedAggregator(agg, metrics.InstanceIDs(), component.StatusOK) + assertEventsRecvdMatch(t, component.StatusOK, metricEvents, allEvents) + + // Traces Pipeline RecoverableError + agg.RecordStatus(traces.ExporterID, component.NewRecoverableErrorEvent(assert.AnError)) + assertErrorEventsRecvdMatch(t, + component.StatusRecoverableError, + assert.AnError, + traceEvents, + allEvents, + ) + + // Traces Pipeline Recover + agg.RecordStatus(traces.ExporterID, component.NewStatusEvent(component.StatusOK)) + assertEventsRecvdMatch(t, component.StatusOK, traceEvents, allEvents) + + // Stopping + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusStopping) + assertEventsRecvdMatch(t, component.StatusStopping, traceEvents, allEvents) + testhelpers.SeedAggregator(agg, metrics.InstanceIDs(), component.StatusStopping) + assertEventsRecvdMatch(t, component.StatusStopping, metricEvents, allEvents) + + // Stopped + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusStopped) + // All is not stopped until the metrics pipeline is stopped + assertEventsRecvdMatch(t, component.StatusStopped, traceEvents) + testhelpers.SeedAggregator(agg, metrics.InstanceIDs(), component.StatusStopped) + assertEventsRecvdMatch(t, component.StatusStopped, metricEvents, allEvents) +} + +func TestStreamingVerbose(t *testing.T) { + agg := status.NewAggregator(status.PriorityPermanent) + defer agg.Close() + + traces := testhelpers.NewPipelineMetadata("traces") + tracesKey := toPipelineKey(traces.PipelineID) + + allEvents := agg.Subscribe(status.ScopeAll, status.Verbose) + + t.Run("zero value", func(t *testing.T) { + st := <-allEvents + assertEventsMatch(t, component.StatusNone, st) + assert.Empty(t, st.ComponentStatusMap) + }) + + // Seed aggregator with successful statuses for pipeline. + testhelpers.SeedAggregator(agg, traces.InstanceIDs(), component.StatusOK) + + t.Run("pipeline statuses all successful", func(t *testing.T) { + st := <-allEvents + // The top-level status matches the pipeline status. + assertEventsMatch(t, component.StatusOK, st, st.ComponentStatusMap[tracesKey]) + + // Component statuses match + assertEventsMatch(t, + component.StatusOK, + collectStatuses(st.ComponentStatusMap[tracesKey], traces.InstanceIDs()...)..., + ) + }) + + // Record an error in the traces exporter + agg.RecordStatus(traces.ExporterID, component.NewRecoverableErrorEvent(assert.AnError)) + + t.Run("pipeline with exporter error", func(t *testing.T) { + st := <-allEvents + + // The top-level status and pipeline status match. + assertErrorEventsMatch(t, + component.StatusRecoverableError, + assert.AnError, + st, + st.ComponentStatusMap[tracesKey], + ) + + // Component statuses match + assertEventsMatch(t, + component.StatusOK, + collectStatuses( + st.ComponentStatusMap[tracesKey], traces.ReceiverID, traces.ProcessorID, + )..., + ) + assertErrorEventsMatch(t, + component.StatusRecoverableError, + assert.AnError, + st.ComponentStatusMap[tracesKey].ComponentStatusMap[toComponentKey(traces.ExporterID)], + ) + }) +} + +// assertEventMatches ensures one or more events share the expected status and are +// otherwise equal, ignoring timestamp. +func assertEventsMatch( + t *testing.T, + expectedStatus component.Status, + statuses ...*status.AggregateStatus, +) { + err0 := statuses[0].Event.Err() + for _, st := range statuses { + ev := st.Event + assert.Equal(t, expectedStatus, ev.Status()) + assert.Equal(t, err0, ev.Err()) + } +} + +// assertErrorEventMatches compares one or more status events with the expected +// status and expected error. +func assertErrorEventsMatch( + t *testing.T, + expectedStatus component.Status, + expectedErr error, + statuses ...*status.AggregateStatus, +) { + assert.True(t, component.StatusIsError(expectedStatus)) + for _, st := range statuses { + ev := st.Event + assert.Equal(t, expectedStatus, ev.Status()) + assert.Equal(t, expectedErr, ev.Err()) + } +} + +func collectStatuses( + aggregateStatus *status.AggregateStatus, + instanceIDs ...*component.InstanceID, +) (result []*status.AggregateStatus) { + for _, id := range instanceIDs { + key := toComponentKey(id) + result = append(result, aggregateStatus.ComponentStatusMap[key]) + } + return +} + +func assertEventsRecvdMatch(t *testing.T, + expectedStatus component.Status, + chans ...<-chan *status.AggregateStatus, +) { + var err0 error + for i, stCh := range chans { + st := <-stCh + ev := st.Event + if i == 0 { + err0 = ev.Err() + } + assert.Equal(t, expectedStatus, ev.Status()) + assert.Equal(t, err0, ev.Err()) + } +} + +func assertErrorEventsRecvdMatch(t *testing.T, + expectedStatus component.Status, + expectedErr error, + chans ...<-chan *status.AggregateStatus, +) { + assert.True(t, component.StatusIsError(expectedStatus)) + for _, stCh := range chans { + st := <-stCh + ev := st.Event + assert.Equal(t, expectedStatus, ev.Status()) + assert.Equal(t, expectedErr, ev.Err()) + } +} + +func toComponentKey(id *component.InstanceID) string { + return fmt.Sprintf("%s:%s", strings.ToLower(id.Kind.String()), id.ID) +} + +func toPipelineKey(id component.ID) string { + return fmt.Sprintf("pipeline:%s", id.String()) +} diff --git a/extension/healthcheckv2extension/internal/status/package_test.go b/extension/healthcheckv2extension/internal/status/package_test.go new file mode 100644 index 0000000000000..312e32157c05c --- /dev/null +++ b/extension/healthcheckv2extension/internal/status/package_test.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package status // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/extension/healthcheckv2extension/internal/testhelpers/helpers.go b/extension/healthcheckv2extension/internal/testhelpers/helpers.go new file mode 100644 index 0000000000000..be02ca6275380 --- /dev/null +++ b/extension/healthcheckv2extension/internal/testhelpers/helpers.go @@ -0,0 +1,83 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testhelpers // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/testhelpers" + +import ( + "go.opentelemetry.io/collector/component" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension/internal/status" +) + +// PipelineMetadata groups together component and instance IDs for a hypothetical pipeline used +// for testing purposes. +type PipelineMetadata struct { + PipelineID component.ID + ReceiverID *component.InstanceID + ProcessorID *component.InstanceID + ExporterID *component.InstanceID +} + +// InstanceIDs returns a slice of instanceIDs for components within the hypothetical pipeline. +func (p *PipelineMetadata) InstanceIDs() []*component.InstanceID { + return []*component.InstanceID{p.ReceiverID, p.ProcessorID, p.ExporterID} +} + +// NewPipelineMetadata returns a metadata for a hypothetical pipeline. +func NewPipelineMetadata(typestr string) *PipelineMetadata { + pipelineID := component.MustNewID(typestr) + return &PipelineMetadata{ + PipelineID: pipelineID, + ReceiverID: &component.InstanceID{ + ID: component.NewIDWithName(component.MustNewType(typestr), "in"), + Kind: component.KindReceiver, + PipelineIDs: map[component.ID]struct{}{ + pipelineID: {}, + }, + }, + ProcessorID: &component.InstanceID{ + ID: component.MustNewID("batch"), + Kind: component.KindProcessor, + PipelineIDs: map[component.ID]struct{}{ + pipelineID: {}, + }, + }, + ExporterID: &component.InstanceID{ + ID: component.NewIDWithName(component.MustNewType(typestr), "out"), + Kind: component.KindExporter, + PipelineIDs: map[component.ID]struct{}{ + pipelineID: {}, + }, + }, + } +} + +// NewPipelines returns a map of hypothetical pipelines identified by their stringified typeVal. +func NewPipelines(typestrs ...string) map[string]*PipelineMetadata { + result := make(map[string]*PipelineMetadata, len(typestrs)) + for _, typestr := range typestrs { + result[typestr] = NewPipelineMetadata(typestr) + } + return result +} + +// SeedAggregator records a status event for each instanceID. +func SeedAggregator( + agg *status.Aggregator, + instanceIDs []*component.InstanceID, + statuses ...component.Status, +) { + for _, st := range statuses { + for _, id := range instanceIDs { + agg.RecordStatus(id, component.NewStatusEvent(st)) + } + } +} + +func ErrPriority(config *common.ComponentHealthConfig) status.ErrorPriority { + if config != nil && config.IncludeRecoverable && !config.IncludePermanent { + return status.PriorityRecoverable + } + return status.PriorityPermanent +}