Skip to content

Commit

Permalink
all: add more fields to gateway-mt and linksharing events
Browse files Browse the repository at this point in the history 
copy over some additional fields from logging to eventkit fields
to help in debugging. This is especially needed as we are no longer
logging as much as we used to on 4xx status events. This also adds
all query values, and request and response headers.

Updates #387

Change-Id: I0faaa9236e55d7706ce0c7781d3e31bf15505fe8
  • Loading branch information
@halkyon
halkyon committed Feb 15, 2024
1 parent c8e393f commit 16c9add
Show file tree
Hide file tree
Showing 5 changed files with 199 additions and 48 deletions.
97 changes: 61 additions & 36 deletions pkg/httplog/httplog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package httplog

import (
"encoding/json"
"net/http"
"net/url"
"strings"
Expand All @@ -14,6 +15,26 @@ import (
xhttp "storj.io/minio/cmd/http"
)

// Known headers and query string values that should be redacted and not logged.
// References:
// https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
// https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
var (
confidentialQueries = map[string]struct{}{
"prefix": {},
xhttp.AmzAccessKeyID: {},
xhttp.AmzSignatureV2: {},
xhttp.AmzSignature: {},
xhttp.AmzCredential: {},
}

confidentialHeaders = map[string]struct{}{
xhttp.Authorization: {},
"Cookie": {},
xhttp.AmzCopySource: {},
}
)

// StatusLevel takes an HTTP status and returns an appropriate log level.
func StatusLevel(status int) zapcore.Level {
switch {
Expand All @@ -28,56 +49,60 @@ func StatusLevel(status int) zapcore.Level {

// RequestQueryLogObject encodes a URL query string into a zap logging object.
type RequestQueryLogObject struct {
Query url.Values
LogAll bool
Query url.Values
InsecureDisableConfidentialSanitization bool
}

// MarshalLogObject implements the zapcore.ObjectMarshaler interface.
func (o *RequestQueryLogObject) MarshalLogObject(enc zapcore.ObjectEncoder) error {
func (o RequestQueryLogObject) MarshalLogObject(enc zapcore.ObjectEncoder) error {
for k, v := range o.Query {
if o.LogAll {
enc.AddString(k, strings.Join(v, ","))
continue
}

var val string
// obfuscate any credentials or confidential information in the query value.
// https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
// https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
switch k {
case "prefix", xhttp.AmzAccessKeyID, xhttp.AmzSignatureV2, xhttp.AmzSignature, xhttp.AmzCredential:
val = "[...]"
default:
val = strings.Join(v, ",")
}
enc.AddString(k, val)
enc.AddString(k, hideConfidentialQuery(k, v, o.InsecureDisableConfidentialSanitization))
}
return nil
}

// MarshalJSON implements json.Marshal.
func (o RequestQueryLogObject) MarshalJSON() ([]byte, error) {
data := make(map[string]string)
for k, v := range o.Query {
data[k] = hideConfidentialQuery(k, v, o.InsecureDisableConfidentialSanitization)
}
return json.Marshal(data)
}

// HeadersLogObject encodes an http.Header into a zap logging object.
type HeadersLogObject struct {
Headers http.Header
LogAll bool
Headers http.Header
InsecureDisableConfidentialSanitization bool
}

// MarshalLogObject implements the zapcore.ObjectMarshaler interface.
func (o *HeadersLogObject) MarshalLogObject(enc zapcore.ObjectEncoder) error {
func (o HeadersLogObject) MarshalLogObject(enc zapcore.ObjectEncoder) error {
for k, v := range o.Headers {
if o.LogAll {
enc.AddString(k, strings.Join(v, ","))
continue
}

var val string
// obfuscate any credentials and sensitive information in headers.
switch k {
case xhttp.Authorization, "Cookie", xhttp.AmzCopySource:
val = "[...]"
default:
val = strings.Join(v, ",")
}
enc.AddString(k, val)
enc.AddString(k, hideConfidentialHeader(k, v, o.InsecureDisableConfidentialSanitization))
}
return nil
}

// MarshalJSON implements json.Marshal.
func (o HeadersLogObject) MarshalJSON() ([]byte, error) {
data := make(map[string]string)
for k, v := range o.Headers {
data[k] = hideConfidentialHeader(k, v, o.InsecureDisableConfidentialSanitization)
}
return json.Marshal(data)
}

func hideConfidentialQuery(k string, vals []string, disable bool) string {
if _, ok := confidentialQueries[k]; ok && !disable {
return "[...]"
}
return strings.Join(vals, ",")
}

func hideConfidentialHeader(k string, vals []string, disable bool) string {
if _, ok := confidentialHeaders[k]; ok && !disable {
return "[...]"
}
return strings.Join(vals, ",")
}
86 changes: 86 additions & 0 deletions pkg/httplog/httplog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,19 @@
package httplog

import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"testing"

"github.com/stretchr/testify/require"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest/observer"

"storj.io/common/testcontext"
xhttp "storj.io/minio/cmd/http"
)

func TestStatusLevel(t *testing.T) {
Expand Down Expand Up @@ -62,3 +68,83 @@ func TestStatusLevel(t *testing.T) {
})
}
}

func TestConfidentialLogFields(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()

tests := []struct {
header string
query string
}{
{query: xhttp.AmzAccessKeyID, header: ""},
{query: xhttp.AmzSignatureV2, header: ""},
{query: xhttp.AmzSignature, header: ""},
{query: xhttp.AmzCredential, header: ""},
{query: "prefix", header: ""},
{header: xhttp.Authorization, query: ""},
{header: "Cookie", query: ""},
{header: xhttp.AmzCopySource, query: ""},
}
for i, test := range tests {
observedZapCore, observedLogs := observer.New(zap.DebugLevel)
observedLogger := zap.New(observedZapCore)

observedLogger.Debug("hello",
zap.Object("headers", &HeadersLogObject{Headers: http.Header{test.header: []string{"value"}}}),
zap.Object("query", &RequestQueryLogObject{Query: url.Values{test.query: []string{"value"}}}))

if test.header != "" {
require.Len(t, observedLogs.All(), 1, i)
fields, ok := observedLogs.All()[0].ContextMap()["headers"].(map[string]interface{})
require.True(t, ok, i)
require.Equal(t, "[...]", fields[test.header], i)
}

if test.query != "" {
require.Len(t, observedLogs.All(), 1, i)
fields, ok := observedLogs.All()[0].ContextMap()["query"].(map[string]interface{})
require.True(t, ok, i)
require.Equal(t, "[...]", fields[test.query], i)
}
}
}

func TestConfidentalJSONFields(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()

tests := []struct {
header string
query string
}{
{query: xhttp.AmzAccessKeyID, header: ""},
{query: xhttp.AmzSignatureV2, header: ""},
{query: xhttp.AmzSignature, header: ""},
{query: xhttp.AmzCredential, header: ""},
{query: "prefix", header: ""},
{header: xhttp.Authorization, query: ""},
{header: "Cookie", query: ""},
{header: xhttp.AmzCopySource, query: ""},
}
for i, test := range tests {
var b []byte
var err error
var key string
switch {
case test.header != "":
key = test.header
b, err = json.Marshal(&HeadersLogObject{Headers: http.Header{key: []string{"value"}}})
case test.query != "":
key = test.query
b, err = json.Marshal(&RequestQueryLogObject{Query: url.Values{key: []string{"value"}}})
default:
t.Error("misconfigured test")
}

require.NoError(t, err)
result := make(map[string]string)
require.NoError(t, json.Unmarshal(b, &result), i)
require.Equal(t, "[...]", result[key], i)
}
}
18 changes: 17 additions & 1 deletion pkg/linksharing/sharing/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package sharing

import (
"encoding/hex"
"encoding/json"
"net/http"
"time"

Expand All @@ -13,6 +14,7 @@ import (

"storj.io/common/useragent"
"storj.io/edge/pkg/auth/authdb"
"storj.io/edge/pkg/httplog"
"storj.io/edge/pkg/trustedip"
"storj.io/eventkit"
privateAccess "storj.io/uplink/private/access"
Expand Down Expand Up @@ -63,6 +65,17 @@ func EventHandler(h http.Handler) http.Handler {
rw.WriteHeader(http.StatusOK)
}

var queryJSON, requestHeadersJSON, responseHeadersJSON string
if b, err := json.Marshal(&httplog.RequestQueryLogObject{Query: r.URL.Query()}); err == nil {
queryJSON = string(b)
}
if b, err := json.Marshal(&httplog.HeadersLogObject{Headers: r.Header}); err == nil {
requestHeadersJSON = string(b)
}
if b, err := json.Marshal(&httplog.HeadersLogObject{Headers: rw.Header()}); err == nil {
responseHeadersJSON = string(b)
}

ek.Event("present",
eventkit.String("protocol", r.Proto),
eventkit.String("host", r.Host),
Expand All @@ -77,6 +90,9 @@ func EventHandler(h http.Handler) http.Handler {
eventkit.String("encryption-key-hash", encKeyHash),
eventkit.String("macaroon-head", macHead),
eventkit.String("satellite-address", satelliteAddress),
eventkit.String("remote-ip", trustedip.GetClientIP(trustedip.NewListTrustAll(), r)))
eventkit.String("remote-ip", trustedip.GetClientIP(trustedip.NewListTrustAll(), r)),
eventkit.String("query", queryJSON),
eventkit.String("request-headers", requestHeadersJSON),
eventkit.String("response-headers", responseHeadersJSON))
}))
}
31 changes: 28 additions & 3 deletions pkg/server/middleware/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package middleware

import (
"encoding/hex"
"encoding/json"
"net/http"
"time"

Expand All @@ -14,6 +15,8 @@ import (
"storj.io/common/grant"
"storj.io/common/useragent"
"storj.io/edge/pkg/auth/authdb"
"storj.io/edge/pkg/httplog"
"storj.io/edge/pkg/server/gwlog"
"storj.io/edge/pkg/trustedip"
"storj.io/eventkit"
)
Expand All @@ -40,8 +43,7 @@ func CollectEvent(h http.Handler) http.Handler {
credentials := GetAccess(r.Context())
if credentials != nil {
if credentials.AccessGrant != "" {
access, err := grant.ParseAccess(credentials.AccessGrant)
if err == nil {
if access, err := grant.ParseAccess(credentials.AccessGrant); err == nil {
macHead = hex.EncodeToString(access.APIKey.Head())
satelliteAddress = access.SatelliteAddress
}
Expand All @@ -54,12 +56,29 @@ func CollectEvent(h http.Handler) http.Handler {
}
}

gl, ok := gwlog.FromContext(r.Context())
if !ok {
gl = gwlog.New()
r = r.WithContext(gl.WithContext(r.Context()))
}

h.ServeHTTP(w, r)

if !rw.WroteHeader() {
rw.WriteHeader(http.StatusOK)
}

var queryJSON, requestHeadersJSON, responseHeadersJSON string
if b, err := json.Marshal(&httplog.RequestQueryLogObject{Query: r.URL.Query()}); err == nil {
queryJSON = string(b)
}
if b, err := json.Marshal(&httplog.HeadersLogObject{Headers: r.Header}); err == nil {
requestHeadersJSON = string(b)
}
if b, err := json.Marshal(&httplog.HeadersLogObject{Headers: rw.Header()}); err == nil {
responseHeadersJSON = string(b)
}

ek.Event("gmt",
eventkit.String("protocol", r.Proto),
eventkit.String("method", r.Method),
Expand All @@ -71,6 +90,12 @@ func CollectEvent(h http.Handler) http.Handler {
eventkit.String("encryption-key-hash", encKeyHash),
eventkit.String("macaroon-head", macHead),
eventkit.String("satellite-address", satelliteAddress),
eventkit.String("remote-ip", trustedip.GetClientIP(trustedip.NewListTrustAll(), r)))
eventkit.String("remote-ip", trustedip.GetClientIP(trustedip.NewListTrustAll(), r)),
eventkit.String("error", gl.TagValue("error")),
eventkit.String("request-id", gl.RequestID),
eventkit.String("api-operation", gl.API),
eventkit.String("query", queryJSON),
eventkit.String("request-headers", requestHeadersJSON),
eventkit.String("response-headers", responseHeadersJSON))
}))
}
15 changes: 7 additions & 8 deletions pkg/server/middleware/log.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,7 @@ func logGatewayResponse(log *zap.Logger, r *http.Request, rw whmon.ResponseWrite
}
}
if credentials != nil && credentials.AccessGrant != "" {
access, err := grant.ParseAccess(credentials.AccessGrant)
if err == nil {
if access, err := grant.ParseAccess(credentials.AccessGrant); err == nil {
macaroonHead = hex.EncodeToString(access.APIKey.Head())
satelliteAddress = access.SatelliteAddress
}
Expand All @@ -143,16 +142,16 @@ func logGatewayResponse(log *zap.Logger, r *http.Request, rw whmon.ResponseWrite
zap.String("macaroon-head", macaroonHead),
zap.String("satellite-address", satelliteAddress),
zap.Object("query", &httplog.RequestQueryLogObject{
Query: r.URL.Query(),
LogAll: insecureLogAll,
Query: r.URL.Query(),
InsecureDisableConfidentialSanitization: insecureLogAll,
}),
zap.Object("request-headers", &httplog.HeadersLogObject{
Headers: r.Header,
LogAll: insecureLogAll,
Headers: r.Header,
InsecureDisableConfidentialSanitization: insecureLogAll,
}),
zap.Object("response-headers", &httplog.HeadersLogObject{
Headers: rw.Header(),
LogAll: true, // we don't need to hide any known response header values.
Headers: rw.Header(),
InsecureDisableConfidentialSanitization: true, // we don't need to hide any known response header values.
}),
}

Expand Down

0 comments on commit 16c9add

Please sign in to comment.