Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fatal error: concurrent map iteration and map write #160

Open
rcmorano opened this issue Apr 22, 2022 · 0 comments
Open

fatal error: concurrent map iteration and map write #160

rcmorano opened this issue Apr 22, 2022 · 0 comments

Comments

@rcmorano
Copy link

rcmorano commented Apr 22, 2022 •
edited

I have setup a high volume pipeline with eel in the middle and found that the more workers I configure, the more frequently it crashes like this [0] when it's under high load. The attached error was caused using a worker pool of 30.

[0]

fatal error: concurrent map iteration and map write

goroutine 426404 [running]:
runtime.throw({0x974589?, 0x7f012504ca68?})
	/usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc000508ea0 sp=0xc000508e70 pc=0x437531
runtime.mapiternext(0x0?)
	/usr/local/go/src/runtime/map.go:871 +0x4eb fp=0xc000508f10 sp=0xc000508ea0 pc=0x41196b
runtime.mapiterinit(0x4673ae?, 0x43a0c7?, 0xc000508f48?)
	/usr/local/go/src/runtime/map.go:861 +0x228 fp=0xc000508f30 sp=0xc000508f10 pc=0x411428
github.com/Comcast/eel/util.(*DefaultLogWriter).log(0xc0000aaf30, {0x95c6fd, 0x4}, {0xc0000ba570, 0x24}, 0x40f2c7?, {0xc00002efc0, 0x4, 0x1?})
	/go/src/github.com/Comcast/eel/util/defaultlogger.go:94 +0x12f fp=0xc000509088 sp=0xc000508f30 pc=0x73f72f
github.com/Comcast/eel/util.(*DefaultLogger).Info(0xc000509488?, {0xc00002efc0?, 0xc0005094a8?, 0x1?})
	/go/src/github.com/Comcast/eel/util/defaultlogger.go:123 +0x4e fp=0xc0005090e0 sp=0xc000509088 pc=0x73fdee
github.com/Comcast/eel/jtl.HandleEvent({0xa472e0, 0xc000132c30}, {0xa45550?, 0xc0005421c0}, 0xc00053e500)
	/go/src/github.com/Comcast/eel/jtl/inproc.go:178 +0x36a8 fp=0xc0005099e0 sp=0xc0005090e0 pc=0x84b008
github.com/Comcast/eel/jtl.EventHandler({0xa45550, 0xc0005421c0}, 0x0?)
	/go/src/github.com/Comcast/eel/jtl/inproc.go:32 +0x4e fp=0xc000509a20 sp=0xc0005099e0 pc=0x84792e
net/http.HandlerFunc.ServeHTTP(0x72?, {0xa45550?, 0xc0005421c0?}, 0xc00050fbd0?)
	/usr/local/go/src/net/http/server.go:2084 +0x2f fp=0xc000509a48 sp=0xc000509a20 pc=0x6be78f
net/http.(*ServeMux).ServeHTTP(0x0?, {0xa45550, 0xc0005421c0}, 0xc00053e500)
	/usr/local/go/src/net/http/server.go:2462 +0x149 fp=0xc000509a98 sp=0xc000509a48 pc=0x6c05e9
net/http.serverHandler.ServeHTTP({0xa43f20?}, {0xa45550, 0xc0005421c0}, 0xc00053e500)
	/usr/local/go/src/net/http/server.go:2916 +0x43b fp=0xc000509b58 sp=0xc000509a98 pc=0x6c223b
net/http.(*conn).serve(0xc00023c140, {0xa45928, 0xc00030ce10})
	/usr/local/go/src/net/http/server.go:1966 +0x5d7 fp=0xc000509fb8 sp=0xc000509b58 pc=0x6bd237
net/http.(*Server).Serve.func3()
	/usr/local/go/src/net/http/server.go:3071 +0x2e fp=0xc000509fe0 sp=0xc000509fb8 pc=0x6c2b8e
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc000509fe8 sp=0xc000509fe0 pc=0x469581
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3071 +0x4db

goroutine 1 [chan receive, 6 minutes]:
main.main()
	/go/src/github.com/Comcast/eel/main.go:210 +0x3e5

goroutine 21 [chan receive, 6 minutes]:
github.com/Comcast/eel/jtl.(*TraceLogger).processTraceLogLoop.func1()
	/go/src/github.com/Comcast/eel/jtl/tracelogger.go:78 +0x18f
created by github.com/Comcast/eel/jtl.(*TraceLogger).processTraceLogLoop
	/go/src/github.com/Comcast/eel/jtl/tracelogger.go:74 +0x8a

goroutine 36 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 37 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 38 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 39 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 40 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 41 [semacquire]:
sync.runtime_Semacquire(0x10?)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0004df6b0?)
	/usr/local/go/src/sync/waitgroup.go:136 +0x52
github.com/Comcast/eel/jtl.handleEvent({0xa472e0, 0xc000590460}, 0xa40530?, 0xc0003bd3e0, {0x0?, 0x1?}, 0x0?, 0x0?)
	/go/src/github.com/Comcast/eel/jtl/outproc.go:169 +0x2c5
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:86 +0x85
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 42 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 43 [runnable]:
github.com/Comcast/eel/jtl.(*JExprItem).parse(0xc000132cd0, {0xc0000c1db0?, 0x8?})
	/go/src/github.com/Comcast/eel/jtl/parser.go:110 +0x96
github.com/Comcast/eel/jtl.NewJExpr({0xc0000c1db0, 0xa})
	/go/src/github.com/Comcast/eel/jtl/parser.go:70 +0xe8
github.com/Comcast/eel/jtl.(*JDoc).ParseExpression(0xc0000c1db0?, {0xa472e0, 0xc000132c30}, {0x8cdd80?, 0xc0004fac80?})
	/go/src/github.com/Comcast/eel/jtl/jpath.go:885 +0x65
github.com/Comcast/eel/jtl.(*JDoc).GetStringValueForExpression(0xa472e0?, {0xa472e0, 0xc000132c30}, {0xc0000c1db0?, 0xc000302800?})
	/go/src/github.com/Comcast/eel/jtl/jpath.go:967 +0x55
github.com/Comcast/eel/jtl.(*HandlerFactory).GetHandlersForEvent(0xc0003022e0, {0xa472e0, 0xc000132c30}, 0x8cdd80?)
	/go/src/github.com/Comcast/eel/jtl/handlers.go:399 +0xa5
github.com/Comcast/eel/jtl.handleEvent({0xa472e0, 0xc000132c30}, 0xa40530?, 0xc00031acf0, {0x0?, 0x1?}, 0x0?, 0x0?)
	/go/src/github.com/Comcast/eel/jtl/outproc.go:31 +0xda
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:86 +0x85
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 44 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 45 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 46 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 47 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 48 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 49 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 50 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 51 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 52 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 53 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 54 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 55 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 56 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 57 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 58 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 59 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 60 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 61 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 62 [semacquire]:
sync.runtime_Semacquire(0x10?)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc000500ca0?)
	/usr/local/go/src/sync/waitgroup.go:136 +0x52
github.com/Comcast/eel/jtl.handleEvent({0xa472e0, 0xc000537900}, 0xa40530?, 0xc000341020, {0x0?, 0x1?}, 0x0?, 0x0?)
	/go/src/github.com/Comcast/eel/jtl/outproc.go:169 +0x2c5
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:86 +0x85
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 63 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 64 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 65 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 66 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 67 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 68 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56

goroutine 69 [select]:
github.com/Comcast/eel/jtl.(*WorkDispatcher).Start.func1()
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:125 +0x12b
created by github.com/Comcast/eel/jtl.(*WorkDispatcher).Start
	/go/src/github.com/Comcast/eel/jtl/workerpool.go:122 +0x2ae

goroutine 70 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f00fe1d8dc8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000314a00?, 0xc00003b900?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Accept(0xc000314a00)
	/usr/local/go/src/internal/poll/fd_unix.go:614 +0x22c
net.(*netFD).accept(0xc000314a00)
	/usr/local/go/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc00031a8e8)
	/usr/local/go/src/net/tcpsock_posix.go:139 +0x28
net.(*TCPListener).Accept(0xc00031a8e8)
	/usr/local/go/src/net/tcpsock.go:288 +0x3d
net/http.(*Server).Serve(0xc0003a0000, {0xa45340, 0xc00031a8e8})
	/usr/local/go/src/net/http/server.go:3039 +0x385
net/http.(*Server).ListenAndServe(0xc0003a0000)
	/usr/local/go/src/net/http/server.go:2968 +0x7d
net/http.ListenAndServe(...)
	/usr/local/go/src/net/http/server.go:3222
github.com/Comcast/eel/jtl.(*WebhookPlugin).startWebhookServices(0xc000302fe0, {0xa472e0?, 0xc00009ebe0})
	/go/src/github.com/Comcast/eel/jtl/webhookplugin.go:75 +0x530
created by github.com/Comcast/eel/jtl.(*WebhookPlugin).StartWebhookConsumer
	/go/src/github.com/Comcast/eel/jtl/webhookplugin.go:57 +0x10e

goroutine 512878 [select]:
net/http.(*persistConn).roundTrip(0xc0003cfd40, 0xc000300e00)
	/usr/local/go/src/net/http/transport.go:2620 +0x974
net/http.(*Transport).roundTrip(0xc00035e000, 0xc0000d9000)
	/usr/local/go/src/net/http/transport.go:594 +0x7c9
net/http.(*Transport).RoundTrip(0xc0000d9000?, 0xa42220?)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x19
net/http.send(0xc0000d8f00, {0xa42220, 0xc00035e000}, {0x94c840?, 0x4cad01?, 0xd07a40?})
	/usr/local/go/src/net/http/client.go:252 +0x5d8
net/http.(*Client).send(0xc00030c540, 0xc0000d8f00, {0xc000055000?, 0xc0003cfc20?, 0xd07a40?})
	/usr/local/go/src/net/http/client.go:176 +0x9b
net/http.(*Client).do(0xc00030c540, 0xc0000d8f00)
	/usr/local/go/src/net/http/client.go:725 +0x8f5
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:593
github.com/Comcast/eel/util.HitEndpoint({0xa472e0, 0xc000537db0}, {0xc00033a090, 0x2f}, {0xc0003d4000, 0x2c0}, {0xc0003247a0, 0x4}, 0xc0004fde30, 0x0)
	/go/src/github.com/Comcast/eel/util/http.go:155 +0xb45
github.com/Comcast/eel/util.(*DefaultRetrier).Retry(0x60, {0xa472e0, 0xc000537db0}, {0xc00033a090, 0x2f}, {0xc0003d4000, 0x2c0}, {0xc0003247a0, 0x4}, 0xc0004fde30, ...)
	/go/src/github.com/Comcast/eel/util/defaultretrier.go:39 +0x1a7
github.com/Comcast/eel/util.(*DefaultRetrier).RetryEndpoint(0xc00038a7b8?, {0xa472e0?, 0xc000537db0?}, {0xc00033a090?, 0x415665?}, {0xc0003d4000?, 0x4490f3?}, {0xc0003247a0?, 0xc00038a4f8?}, 0xc0004fde30, ...)
	/go/src/github.com/Comcast/eel/util/defaultretrier.go:28 +0x65
github.com/Comcast/eel/jtl.(*HttpPublisher).Publish(0xc00039ab40)
	/go/src/github.com/Comcast/eel/jtl/httppublisher.go:60 +0xc3
github.com/Comcast/eel/jtl.handleEvent.func1.1({0xa472e0?, 0xc000537e00}, {0xa47368, 0xc00039ab40})
	/go/src/github.com/Comcast/eel/jtl/outproc.go:149 +0x102
created by github.com/Comcast/eel/jtl.handleEvent.func1
	/go/src/github.com/Comcast/eel/jtl/outproc.go:146 +0x25e5

goroutine 512948 [select]:
net/http.(*persistConn).writeLoop(0xc0003cfd40)
	/usr/local/go/src/net/http/transport.go:2392 +0xf5
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1751 +0x1791

goroutine 512941 [select]:
net/http.(*Transport).getConn(0xc00035e000, 0xc0005fec00, {{}, 0x0, {0xc00033a090, 0x4}, {0xc000333548, 0x12}, 0x0})
	/usr/local/go/src/net/http/transport.go:1375 +0x5c6
net/http.(*Transport).roundTrip(0xc00035e000, 0xc000419700)
	/usr/local/go/src/net/http/transport.go:581 +0x76f
net/http.(*Transport).RoundTrip(0xc000419700?, 0xa42220?)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x19
net/http.send(0xc000419600, {0xa42220, 0xc00035e000}, {0x94c840?, 0x4cad01?, 0xd07a40?})
	/usr/local/go/src/net/http/client.go:252 +0x5d8
net/http.(*Client).send(0xc00030c540, 0xc000419600, {0xc000055400?, 0xc0005410e0?, 0xd07a40?})
	/usr/local/go/src/net/http/client.go:176 +0x9b
net/http.(*Client).do(0xc00030c540, 0xc000419600)
	/usr/local/go/src/net/http/client.go:725 +0x8f5
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:593
github.com/Comcast/eel/util.HitEndpoint({0xa472e0, 0xc000590820}, {0xc00033a090, 0x2f}, {0xc00024f900, 0x4a6}, {0xc0003247a0, 0x4}, 0xc0004833e0, 0x0)
	/go/src/github.com/Comcast/eel/util/http.go:155 +0xb45
github.com/Comcast/eel/util.(*DefaultRetrier).Retry(0x60, {0xa472e0, 0xc000590820}, {0xc00033a090, 0x2f}, {0xc00024f900, 0x4a6}, {0xc0003247a0, 0x4}, 0xc0004833e0, ...)
	/go/src/github.com/Comcast/eel/util/defaultretrier.go:39 +0x1a7
github.com/Comcast/eel/util.(*DefaultRetrier).RetryEndpoint(0xc00011a7b8?, {0xa472e0?, 0xc000590820?}, {0xc00033a090?, 0x415665?}, {0xc00024f900?, 0x4490f3?}, {0xc0003247a0?, 0xc00011a4f8?}, 0xc0004833e0, ...)
	/go/src/github.com/Comcast/eel/util/defaultretrier.go:28 +0x65
github.com/Comcast/eel/jtl.(*HttpPublisher).Publish(0xc00023cb40)
	/go/src/github.com/Comcast/eel/jtl/httppublisher.go:60 +0xc3
github.com/Comcast/eel/jtl.handleEvent.func1.1({0xa472e0?, 0xc000590870}, {0xa47368, 0xc00023cb40})
	/go/src/github.com/Comcast/eel/jtl/outproc.go:149 +0x102
created by github.com/Comcast/eel/jtl.handleEvent.func1
	/go/src/github.com/Comcast/eel/jtl/outproc.go:146 +0x25e5

goroutine 512860 [IO wait]:
internal/poll.runtime_pollWait(0x7f00fe1d8648, 0x72)
	/usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000600480?, 0xc000482c11?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc000600480, {0xc000482c11, 0x1, 0x1})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000600480, {0xc000482c11?, 0xc000286118?, 0xc000116768?})
	/usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000562000, {0xc000482c11?, 0xc0006a6fa0?, 0xc0001de4e0?})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc000482c00)
	/usr/local/go/src/net/http/server.go:672 +0x3f
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:668 +0xca

goroutine 512947 [IO wait]:
internal/poll.runtime_pollWait(0x7f00fe1d8af8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000314480?, 0xc0003cb000?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc000314480, {0xc0003cb000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000314480, {0xc0003cb000?, 0x1?, 0xc00033a0a5?})
	/usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000326230, {0xc0003cb000?, 0x0?, 0xc00038a510?})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0003cfd40, {0xc0003cb000?, 0x4070dd?, 0x60?})
	/usr/local/go/src/net/http/transport.go:1929 +0x4e
bufio.(*Reader).fill(0xc00075f6e0)
	/usr/local/go/src/bufio/bufio.go:106 +0x103
bufio.(*Reader).Peek(0xc00075f6e0, 0x1)
	/usr/local/go/src/bufio/bufio.go:144 +0x5d
net/http.(*persistConn).readLoop(0xc0003cfd40)
	/usr/local/go/src/net/http/transport.go:2093 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1750 +0x173e

goroutine 512942 [IO wait]:
internal/poll.runtime_pollWait(0x7f00fe13dca0, 0x77)
	/usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000601180?, 0x95c1ae?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitWrite(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).WaitWrite(...)
	/usr/local/go/src/internal/poll/fd_unix.go:741
net.(*netFD).connect(0xc000601180, {0xa458b8?, 0xc0000c0020}, {0xc00049d370?, 0x40ef65?}, {0xa42660?, 0xc000029140?})
	/usr/local/go/src/net/fd_unix.go:141 +0x716
net.(*netFD).dial(0xc000601180, {0xa458b8, 0xc0000c0020}, {0xa462e0?, 0x0?}, {0xa462e0?, 0xc000483c50}, 0x0?)
	/usr/local/go/src/net/sock_posix.go:149 +0x394
net.socket({0xa458b8, 0xc0000c0020}, {0x95c1ae, 0x3}, 0x2, 0x1, 0x0?, 0x60?, {0xa462e0, 0x0}, ...)
	/usr/local/go/src/net/sock_posix.go:70 +0x2b2
net.internetSocket({0xa458b8, 0xc0000c0020}, {0x95c1ae, 0x3}, {0xa462e0, 0x0}, {0xa462e0, 0xc000483c50}, 0xc0004df760?, 0x0, ...)
	/usr/local/go/src/net/ipsock_posix.go:142 +0xf8
net.(*sysDialer).doDialTCP(0xc000601100, {0xa458b8, 0xc0000c0020}, 0x0, 0xd?)
	/usr/local/go/src/net/tcpsock_posix.go:65 +0xa5
net.(*sysDialer).dialTCP(0x1?, {0xa458b8?, 0xc0000c0020?}, 0x4cad96?, 0xc0005c55f0?)
	/usr/local/go/src/net/tcpsock_posix.go:61 +0x59
net.(*sysDialer).dialSingle(0xc000601100, {0xa458b8, 0xc0000c0020}, {0xa43ea8?, 0xc000483c50})
	/usr/local/go/src/net/dial.go:583 +0x28b
net.(*sysDialer).dialSerial(0xc000601100, {0xa458b8, 0xc0000c0020}, {0xc000511d50?, 0x1, 0x95c5e1?})
	/usr/local/go/src/net/dial.go:551 +0x312
net.(*Dialer).DialContext(0xc0003221e0, {0xa458b8, 0xc0000c0020}, {0x95c1ae, 0x3}, {0xc000333548, 0x12})
	/usr/local/go/src/net/dial.go:428 +0x736
net.(*Dialer).Dial(0x7f00fe0cf060?, {0x95c1ae?, 0x12504c5b8?}, {0xc000333548?, 0x118?})
	/usr/local/go/src/net/dial.go:351 +0x45
net/http.(*Transport).dial(0x9137076f19a24fbe?, {0xa458f0?, 0xc000799e00?}, {0x95c1ae?, 0xc000511c60?}, {0xc000333548?, 0xc000419600?})
	/usr/local/go/src/net/http/transport.go:1172 +0x5a
net/http.(*Transport).dialConn(0xc00035e000, {0xa458f0, 0xc000799e00}, {{}, 0x0, {0xc00033a090, 0x4}, {0xc000333548, 0x12}, 0x0})
	/usr/local/go/src/net/http/transport.go:1607 +0x83f
net/http.(*Transport).dialConnFor(0xa47368?, 0xc0000d44d0)
	/usr/local/go/src/net/http/transport.go:1449 +0xb0
created by net/http.(*Transport).queueForDial
	/usr/local/go/src/net/http/transport.go:1418 +0x3d2

goroutine 512861 [chan send]:
github.com/Comcast/eel/jtl.(*lexer).emit(...)
	/go/src/github.com/Comcast/eel/jtl/lexer.go:158
github.com/Comcast/eel/jtl.lexText(0xc00002f000)
	/go/src/github.com/Comcast/eel/jtl/lexer.go:201 +0x30f
github.com/Comcast/eel/jtl.(*lexer).run(0xc00002f000)
	/go/src/github.com/Comcast/eel/jtl/lexer.go:141 +0x2a
created by github.com/Comcast/eel/jtl.lex
	/go/src/github.com/Comcast/eel/jtl/lexer.go:134 +0xf8
{"app.id":"eel","debug_action":"event_after_transformation","destination":"unknown","env.name":"default","gears.app.id":"tenant1","gears.partner.id":"","handler":"oura-to-dgraph","instance.id":"oura-2-dgraph-etl-5b7bfc9448-gjpvl","log.id":"dc9c18bf-d5a2-40e5-8f59-0c9091c851c5","log.level":"debug","log.timestamp":"2022-04-22T15:41:07.491325766Z","payload":{"set":{"context":{"block_hash":"be342e54bc765e74ed04fd51850d9fd86abf8799807657db8ecfd32841d15aa9","block_number":5058097,"certificate_idx":null,"input_idx":null,"output_address":null,"output_idx":null,"slot":16009473,"timestamp":1607575764,"tx_hash":"ff9c00eefe24cf99d19dc5e1b405f8d3b71c4d18afb95ca2ed150daf4c936a9f","tx_idx":1},"fingerprint":"16009473.tx.175673335737938583721856672132598484816","timestamp":1607575764000,"transaction":{"fee":186276,"hash":"ff9c00eefe24cf99d19dc5e1b405f8d3b71c4d18afb95ca2ed150daf4c936a9f","input_count":1,"inputs":[{"index":0,"tx_id":"a08daab5ef88fbaaf4b869ceb2e94ba17ccda4ebc5f874bc0eb67217bf7db136"}],"metadata":null,"mint":null,"mint_count":0,"network_id":null,"output_count":1,"outputs":[{"address":"addr1qy8rdzdyv0c7j2yhnraxqxjt0t0l6nwxa0xnja4v0vjkq2vxal9tylsn3za5s72ulkv9zf6nwyp2hel82lht689cxucqa8a6ey","amount":12726539187,"assets":[]}],"total_output":12726539187,"ttl":16012994,"validity_interval_start":null},"variant":"Transaction"}},"topic":"","tx.traceId":"dc9c18bf-d5a2-40e5-8f59-0c9091c851c5"}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@rcmorano