Merge pull request #1026 from xmidt-org/denopink/feat/verbose-connect…

…ion-convey-logging

feat: add verbose logging for device convey during connection and uniform timestamp to all device wrp messsages (expect for message sent to devices `writePump`)

.github/workflows/ci.yml

@@ -20,10 +20,11 @@ on:
 
 jobs:
   ci:
-    uses: xmidt-org/shared-go/.github/workflows/ci.yml@d0737af7255c581ce3cc09b69b96ba16a5913b4e # v4.0.0
+    uses: xmidt-org/shared-go/.github/workflows/ci.yml@5bc4b83f25ff4c944cd6253ba189e50d1997ab3c # v4.1.0
     with:
       copyright-skip: true
       lint-skip:      true
       style-skip:     true
+      tests-skip:     true
       release-type:   library
     secrets: inherit

device/manager.go

@@ -13,6 +13,7 @@ import (
 	"github.com/xmidt-org/webpa-common/v2/convey"
 	"github.com/xmidt-org/webpa-common/v2/convey/conveymetric"
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 
 	"github.com/gorilla/websocket"
 	"github.com/xmidt-org/webpa-common/v2/convey/conveyhttp"
@@ -25,6 +26,9 @@ const MaxDevicesHeader = "X-Xmidt-Max-Devices"
 // DefaultWRPContentType is the content type used on inbound WRP messages which don't provide one.
 const DefaultWRPContentType = "application/octet-stream"
 
+// WRPTimestampMetadataKey is the uniform timestamp given to all device wrp messsages (expect for message sent to devices `writePump`)
+const WRPTimestampMetadataKey = "/xdmit-timestamp"
+
 // emptyBuffer is solely used as an address of a global empty buffer.
 // This sentinel value will reset pointers of the writePump's encoder
 // such that the gc can clean things up.
@@ -269,6 +273,8 @@ func (m *manager) Connect(response http.ResponseWriter, request *http.Request, r
 	go m.readPump(d, InstrumentReader(c, d.statistics), closeOnce)
 	go m.writePump(d, InstrumentWriter(c, d.statistics), pinger, closeOnce)
 
+	d.logger.Debug("Connection metadata", zap.String("deviceID", string(d.ID())), zap.String("conveyCompliance", convey.GetCompliance(cvyErr).String()), zap.Strings("conveyHeaderKeys", maps.Keys(cvy)), zap.Any("conveyHeader", cvy))
+
 	return d, nil
 }
 
@@ -345,6 +351,7 @@ func (m *manager) wrpSourceIsValid(message *wrp.Message, d *device) bool {
 
 // nolint: typecheck
 func addDeviceMetadataContext(message *wrp.Message, deviceMetadata *Metadata) {
+	message.Metadata[WRPTimestampMetadataKey] = time.Now().Format(time.RFC3339Nano)
 	message.PartnerIDs = []string{deviceMetadata.PartnerIDClaim()}
 
 	// nolint: typecheck

go.mod

@@ -1,6 +1,6 @@
 module github.com/xmidt-org/webpa-common/v2
 
-go 1.13
+go 1.21
 
 require (
 	github.com/aws/aws-sdk-go v1.47.5
@@ -13,18 +13,15 @@ require (
 	github.com/gorilla/schema v1.2.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/hashicorp/consul/api v1.25.1
-	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
 	github.com/justinas/alice v1.2.0
 	github.com/miekg/dns v1.1.56
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/segmentio/ksuid v1.0.4
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/cast v1.5.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.17.0
 	github.com/stretchr/testify v1.8.4
-	github.com/ugorji/go v1.2.7 // indirect
 	github.com/ugorji/go/codec v1.2.11
 	github.com/xmidt-org/argus v0.9.10
 	github.com/xmidt-org/candlelight v0.0.16
@@ -34,4 +31,99 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0
 	go.uber.org/fx v1.20.1
 	go.uber.org/zap v1.26.0
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
+)
+
+require (
+	emperror.dev/emperror v0.33.0 // indirect
+	emperror.dev/errors v0.8.1 // indirect
+	github.com/GaryBoone/GoStats v0.0.0-20130122001700-1993eafbef57 // indirect
+	github.com/VividCortex/gohistogram v1.0.0 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/fatih/color v1.14.1 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/goccy/go-json v0.10.0 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/glog v1.1.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/uuid v1.3.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
+	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jtacoma/uritemplates v1.0.0 // indirect
+	github.com/lestrrat-go/blackmagic v1.0.1 // indirect
+	github.com/lestrrat-go/httpcc v1.0.1 // indirect
+	github.com/lestrrat-go/httprc v1.0.4 // indirect
+	github.com/lestrrat-go/iter v1.0.2 // indirect
+	github.com/lestrrat-go/jwx/v2 v2.0.8 // indirect
+	github.com/lestrrat-go/option v1.0.1 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/openzipkin/zipkin-go v0.4.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/sagikazarmark/locafero v0.3.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/xmidt-org/arrange v0.4.0 // indirect
+	github.com/xmidt-org/bascule v0.11.4 // indirect
+	github.com/xmidt-org/chronon v0.1.1 // indirect
+	github.com/xmidt-org/clortho v0.0.4 // indirect
+	github.com/xmidt-org/httpaux v0.4.0 // indirect
+	github.com/xmidt-org/touchstone v0.1.2 // indirect
+	go.opentelemetry.io/otel v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/zipkin v1.14.0 // indirect
+	go.opentelemetry.io/otel/metric v0.37.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.14.0 // indirect
+	go.opentelemetry.io/otel/trace v1.14.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	go.uber.org/dig v1.17.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/tools v0.13.0 // indirect
+	google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
+	google.golang.org/grpc v1.58.2 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )