Skip to content

Instantly share code, notes, and snippets.

@derekperkins

derekperkins/block

Created April 1, 2022 20:23
Show Gist options
  • Save derekperkins/a2a9bc0d83a0589e21c8820c93e949e8 to your computer and use it in GitHub Desktop.
Save derekperkins/a2a9bc0d83a0589e21c8820c93e949e8 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
block
--- contention:
cycles/second=2250006157
55949963957215 1174 @ 0x448208 0x10f5411 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x10f5410 vitess.io/vitess/go/timer.(*Timer).run+0xb0 vitess.io/vitess/go/timer/timer.go:98
53095068945274 9478 @ 0x448208 0x9a2e3b 0x9a3525 0x9b87e6 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x9a2e3a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/[email protected]/internal/transport/controlbuf.go:406
# 0x9a3524 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/[email protected]/internal/transport/controlbuf.go:533
# 0x9b87e5 google.golang.org/grpc/internal/transport.NewServerTransport.func2+0xc5 google.golang.org/[email protected]/internal/transport/http2_server.go:304
53094056692142 4719 @ 0x448208 0x1266c25 0x128e46a 0x1363f74 0x1357db3 0xb2a049 0xb0423a 0xb040df 0x9f8824 0x9f9cc5 0x9f3ab8 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x1266c24 vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x1e4 vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169
# 0x128e469 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x29 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1521
# 0x1363f73 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x173 vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:320
# 0x1357db2 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0xd2 vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:1106
# 0xb2a048 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/[email protected]/server_metrics.go:121
# 0xb04239 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/[email protected]/chain.go:49
# 0xb040de github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/[email protected]/chain.go:58
# 0x9f8823 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/[email protected]/server.go:1548
# 0x9f9cc4 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/[email protected]/server.go:1621
# 0x9f3ab7 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/[email protected]/server.go:940
25956646950774 2307 @ 0x406098 0xafe4b6 0x469301
# 0x406097 runtime.chanrecv1+0x17 runtime/chan.go:439
# 0xafe4b5 vitess.io/vitess/go/stats.(*Rates).track+0x35 vitess.io/vitess/go/stats/rates.go:99
4126614620026 99419 @ 0x448208 0x9c372a 0x9c3467 0x9c42d2 0x4a8d7a 0x9c4245 0x9c421b 0x9eee87 0x9ef6e6 0x9efd8e 0xa013f0 0xa007a5 0x9ff636 0xa004df 0xb28637 0x13535ac 0x136c3d9 0x11fb5e9 0x1211f51 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x9c3729 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/[email protected]/internal/transport/transport.go:189
# 0x9c3466 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/[email protected]/internal/transport/transport.go:169
# 0x9c42d1 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/[email protected]/internal/transport/transport.go:483
# 0x4a8d79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c4244 io.ReadFull+0xa4 io/io.go:347
# 0x9c421a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/[email protected]/internal/transport/transport.go:467
# 0x9eee86 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/[email protected]/rpc_util.go:559
# 0x9ef6e5 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/[email protected]/rpc_util.go:690
# 0x9efd8d google.golang.org/grpc.recv+0x6d google.golang.org/[email protected]/rpc_util.go:758
# 0xa013ef google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/[email protected]/stream.go:957
# 0xa007a4 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/[email protected]/stream.go:808
# 0x9ff635 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/[email protected]/stream.go:662
# 0xa004de google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/[email protected]/stream.go:807
# 0xb28636 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/[email protected]/client_metrics.go:160
# 0x13535ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:461
# 0x136c3d8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:648
# 0x11fb5e8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x1211f50 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
4123174590719 65052 @ 0x448208 0x9a2e3b 0x9a3525 0x9ac3a5 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x9a2e3a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/[email protected]/internal/transport/controlbuf.go:406
# 0x9a3524 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/[email protected]/internal/transport/controlbuf.go:533
# 0x9ac3a4 google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x64 google.golang.org/[email protected]/internal/transport/http2_client.go:396
4074636121346 88519 @ 0x448208 0x11fc968 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x11fc967 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer.func1+0x87 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:124
4073292701444 88535 @ 0x47158c 0x11fc4be 0x1212d9a 0x1211e32 0x469301
# 0x47158b sync.(*Cond).Wait+0x8b sync/cond.go:56
# 0x11fc4bd vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x13d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98
# 0x1212d99 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x219 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:346
# 0x1211e31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
4068133247001 185 @ 0x448208 0x9b61c9 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x9b61c8 google.golang.org/grpc/internal/transport.(*http2Client).keepalive+0x168 google.golang.org/[email protected]/internal/transport/http2_client.go:1525
2576243253445 130 @ 0x4060d8 0x10f50cb 0x469301
# 0x4060d7 runtime.chanrecv2+0x17 runtime/chan.go:444
# 0x10f50ca vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x4a vitess.io/vitess/go/timer/suspendable_ticker.go:74
1377446186171 2449 @ 0x4060d8 0x11c6905 0x469301
# 0x4060d7 runtime.chanrecv2+0x17 runtime/chan.go:444
# 0x11c6904 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xa4 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39
1375922793173 1221 @ 0x448208 0xf761f9 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0xf761f8 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x1f8 go.etcd.io/etcd/client/[email protected]/lease.go:579
1375484944184 611 @ 0x448208 0xf75e85 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0xf75e84 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x84 go.etcd.io/etcd/client/[email protected]/lease.go:539
1361246753886 121 @ 0x4060d8 0x13df10a 0x469301
# 0x4060d7 runtime.chanrecv2+0x17 runtime/chan.go:444
# 0x13df109 k8s.io/klog/v2.(*loggingT).flushDaemon+0x69 k8s.io/klog/[email protected]/klog.go:1169
1349996677453 60 @ 0x448208 0x12daf39 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x12daf38 go.opencensus.io/stats/view.(*worker).start+0xb8 [email protected]/stats/view/worker.go:276
1349996579346 60 @ 0x448208 0x11bab49 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0x11bab48 github.com/patrickmn/go-cache.(*janitor).Run+0x88 github.com/patrickmn/[email protected]+incompatible/cache.go:1079
1282499483555 19 @ 0x4060d8 0x7a6f2a 0x469301
# 0x4060d7 runtime.chanrecv2+0x17 runtime/chan.go:444
# 0x7a6f29 github.com/golang/glog.(*loggingT).flushDaemon+0x69 github.com/golang/[email protected]/glog.go:882
1232343135 5 @ 0x47184d 0x47182a 0x7b53ec 0x7b534f 0x7b528a 0x7b4eda 0x7b4545 0xaff52b 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x47184c sync.(*Mutex).Lock+0xec sync/mutex.go:81
# 0x471829 sync.(*Map).Load+0xc9 sync/map.go:107
# 0x7b53eb encoding/json.typeEncoder+0x4b encoding/json/encode.go:382
# 0x7b534e encoding/json.valueEncoder+0x2e encoding/json/encode.go:378
# 0x7b5289 encoding/json.(*encodeState).reflectValue+0x49 encoding/json/encode.go:360
# 0x7b4ed9 encoding/json.(*encodeState).marshal+0xf9 encoding/json/encode.go:332
# 0x7b4544 encoding/json.Marshal+0x44 encoding/json/encode.go:161
# 0xaff52a vitess.io/vitess/go/stats.(*RatesFunc).String+0x2a vitess.io/vitess/go/stats/rates.go:208
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
945593978 4 @ 0xafed4c 0xafed32 0xaff359 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0xafed4b sync.(*Mutex).Lock+0x6b sync/mutex.go:81
# 0xafed31 vitess.io/vitess/go/stats.(*Rates).Get+0x51 vitess.io/vitess/go/stats/rates.go:145
# 0xaff358 vitess.io/vitess/go/stats.(*Rates).String+0x18 vitess.io/vitess/go/stats/rates.go:179
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
909966825 2 @ 0x474bd1 0x7b5635 0x7b52b8 0x7b4eda 0x7b4545 0xb002f5 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x474bd0 sync.(*WaitGroup).Wait+0x70 sync/waitgroup.go:130
# 0x7b5634 encoding/json.typeEncoder.func1+0x54 encoding/json/encode.go:396
# 0x7b52b7 encoding/json.(*encodeState).reflectValue+0x77 encoding/json/encode.go:360
# 0x7b4ed9 encoding/json.(*encodeState).marshal+0xf9 encoding/json/encode.go:332
# 0x7b4544 encoding/json.Marshal+0x44 encoding/json/encode.go:161
# 0xb002f4 vitess.io/vitess/go/stats.(*Timings).String+0xb4 vitess.io/vitess/go/stats/timings.go:126
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
804904402 4 @ 0x474bd1 0x7b5635 0x7b52b8 0x7b4eda 0x7b4545 0xaf8445 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x474bd0 sync.(*WaitGroup).Wait+0x70 sync/waitgroup.go:130
# 0x7b5634 encoding/json.typeEncoder.func1+0x54 encoding/json/encode.go:396
# 0x7b52b7 encoding/json.(*encodeState).reflectValue+0x77 encoding/json/encode.go:360
# 0x7b4ed9 encoding/json.(*encodeState).marshal+0xf9 encoding/json/encode.go:332
# 0x7b4544 encoding/json.Marshal+0x44 encoding/json/encode.go:161
# 0xaf8444 expvar.Func.String+0x24 expvar/expvar.go:262
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
50389313 103 @ 0x448208 0xfcdf57 0x104ba5d 0x10fa553 0x10fb639 0x10f989c 0x10f8ce5 0x10f8b5b 0x10f5483 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0xfcdf56 vitess.io/vitess/go/mysql.Connect+0x4d6 vitess.io/vitess/go/mysql/client.go:152
# 0x104ba5c vitess.io/vitess/go/vt/dbconfigs.(*Connector).Connect+0x3c vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:194
# 0x10fa552 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x32 vitess.io/vitess/go/vt/dbconnpool/connection.go:38
# 0x10fb638 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x38 vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:115
# 0x10f989b vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x3b vitess.io/vitess/go/pools/resource_pool.go:340
# 0x10f8ce4 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0xc4 vitess.io/vitess/go/pools/resource_pool.go:235
# 0x10f8b5a vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x7a vitess.io/vitess/go/pools/resource_pool.go:237
# 0x10f5482 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
17593803 103 @ 0x448208 0xfcde31 0x104ba5d 0x10fa553 0x10fb639 0x10f989c 0x10f8ce5 0x10f8b5b 0x10f5483 0x469301
# 0x448207 runtime.selectgo+0x407 runtime/select.go:492
# 0xfcde30 vitess.io/vitess/go/mysql.Connect+0x3b0 vitess.io/vitess/go/mysql/client.go:124
# 0x104ba5c vitess.io/vitess/go/vt/dbconfigs.(*Connector).Connect+0x3c vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:194
# 0x10fa552 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x32 vitess.io/vitess/go/vt/dbconnpool/connection.go:38
# 0x10fb638 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x38 vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:115
# 0x10f989b vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x3b vitess.io/vitess/go/pools/resource_pool.go:340
# 0x10f8ce4 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0xc4 vitess.io/vitess/go/pools/resource_pool.go:235
# 0x10f8b5a vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x7a vitess.io/vitess/go/pools/resource_pool.go:237
# 0x10f5482 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
3096316 157 @ 0x47158c 0x6a99d5 0x6ae4c7 0x6af71f 0x469301
# 0x47158b sync.(*Cond).Wait+0x8b sync/cond.go:56
# 0x6a99d4 net/http.(*connReader).abortPendingRead+0xb4 net/http/server.go:720
# 0x6ae4c6 net/http.(*response).finishRequest+0x86 net/http/server.go:1611
# 0x6af71e net/http.(*conn).serve+0xb3e net/http/server.go:1935
3059281 6 @ 0xaf974c 0xaf972f 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0xaf974b sync.(*Mutex).Lock+0x6b sync/mutex.go:81
# 0xaf972e vitess.io/vitess/go/stats.(*counters).String+0x4e vitess.io/vitess/go/stats/counters.go:36
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
2189227 53 @ 0x11fc0e5 0x11fc0be 0x1211fce 0x136c430 0x11fb5e9 0x1211f51 0x469301
# 0x11fc0e4 sync.(*Mutex).Lock+0x84 sync/mutex.go:81
# 0x11fc0bd vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Send+0x5d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:69
# 0x1211fcd vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1.1+0x2d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:149
# 0x136c42f vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0xef vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:657
# 0x11fb5e8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x1211f50 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
1958537 46 @ 0x7a5dad 0x7a5d6e 0x7a5cb0 0x7a79be 0x125c349 0x469301
# 0x7a5dac sync.(*Mutex).Lock+0x8c sync/mutex.go:81
# 0x7a5d6d github.com/golang/glog.(*loggingT).output+0x4d github.com/golang/[email protected]/glog.go:672
# 0x7a5caf github.com/golang/glog.(*loggingT).printf+0x10f github.com/golang/[email protected]/glog.go:655
# 0x7a79bd github.com/golang/glog.Errorf+0x3d github.com/golang/[email protected]/glog.go:1121
# 0x125c348 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge.func1+0x268 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:818
1677034 17 @ 0x127b52c 0x127b4e8 0x1289ab4 0x1289988 0x125c259 0x469301
# 0x127b52b sync.(*Mutex).Lock+0x8b sync/mutex.go:81
# 0x127b4e7 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).StartRequest+0x47 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:348
# 0x1289ab3 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML+0x73 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1031
# 0x1289987 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages+0x67 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1024
# 0x125c258 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge.func1+0x178 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:815
1270666 5 @ 0x47305b 0x47159e 0x11fc4be 0x1212d9a 0x1211e32 0x469301
# 0x47305a sync.(*Mutex).Lock+0x3a sync/mutex.go:81
# 0x47159d sync.(*Cond).Wait+0x9d sync/cond.go:57
# 0x11fc4bd vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x13d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98
# 0x1212d99 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x219 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:346
# 0x1211e31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
1159942 6 @ 0x9c31e5 0x9c31be 0x9c38b1 0x9c3809 0x9c3467 0x9c42d2 0x4a8d7a 0x9c4245 0x9c421b 0x9eee87 0x9ef6e6 0x9efd8e 0xa013f0 0xa007a5 0x9ff636 0xa004df 0xb28637 0x13535ac 0x136c3d9 0x11fb5e9 0x1211f51 0x469301
# 0x9c31e4 sync.(*Mutex).Lock+0x44 sync/mutex.go:81
# 0x9c31bd google.golang.org/grpc/internal/transport.(*recvBuffer).load+0x1d google.golang.org/[email protected]/internal/transport/transport.go:120
# 0x9c38b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).readAdditional+0x50 google.golang.org/[email protected]/internal/transport/transport.go:213
# 0x9c3808 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0x188 google.golang.org/[email protected]/internal/transport/transport.go:208
# 0x9c3466 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/[email protected]/internal/transport/transport.go:169
# 0x9c42d1 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/[email protected]/internal/transport/transport.go:483
# 0x4a8d79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c4244 io.ReadFull+0xa4 io/io.go:347
# 0x9c421a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/[email protected]/internal/transport/transport.go:467
# 0x9eee86 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/[email protected]/rpc_util.go:559
# 0x9ef6e5 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/[email protected]/rpc_util.go:690
# 0x9efd8d google.golang.org/grpc.recv+0x6d google.golang.org/[email protected]/rpc_util.go:758
# 0xa013ef google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/[email protected]/stream.go:957
# 0xa007a4 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/[email protected]/stream.go:808
# 0x9ff635 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/[email protected]/stream.go:662
# 0xa004de google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/[email protected]/stream.go:807
# 0xb28636 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/[email protected]/client_metrics.go:160
# 0x13535ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:461
# 0x136c3d8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:648
# 0x11fb5e8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x1211f50 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
766755 2 @ 0x1203a4d 0x1203a30 0xafada3 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x1203a4c sync.(*Mutex).Lock+0x6c sync/mutex.go:81
# 0x1203a2f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).register.func2+0x4f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:79
# 0xafada2 vitess.io/vitess/go/stats.CountersFuncWithMultiLabels.String+0x62 vitess.io/vitess/go/stats/counters.go:274
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
684405 3 @ 0x120092d 0x1200910 0xafada3 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x120092c sync.(*Mutex).Lock+0x6c sync/mutex.go:81
# 0x120090f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).register.func18+0x4f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:308
# 0xafada2 vitess.io/vitess/go/stats.CountersFuncWithMultiLabels.String+0x62 vitess.io/vitess/go/stats/counters.go:274
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
493785 1 @ 0x12028cd 0x12028b0 0xafc85c 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x12028cc sync.(*Mutex).Lock+0x6c sync/mutex.go:81
# 0x12028af vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).register.func8+0x4f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:150
# 0xafc85b vitess.io/vitess/go/stats.StringMapFunc.String+0x1b vitess.io/vitess/go/stats/export.go:240
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
456818 2 @ 0x12041c5 0x12041a7 0xaf9466 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x12041c4 sync.(*Mutex).Lock+0x64 sync/mutex.go:81
# 0x12041a6 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).maxReplicationLagSeconds+0x46 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:367
# 0xaf9465 vitess.io/vitess/go/stats.CounterFunc.String+0x25 vitess.io/vitess/go/stats/counter.go:117
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
332663 1 @ 0x120116b 0x120114b 0xaf9466 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x120116a sync.(*Mutex).Lock+0x6a sync/mutex.go:81
# 0x120114a vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).register.func15+0x4a vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:266
# 0xaf9465 vitess.io/vitess/go/stats.CounterFunc.String+0x25 vitess.io/vitess/go/stats/counter.go:117
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
282825 1 @ 0x1200c68 0x1200c48 0xaf9466 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x1200c67 sync.(*Mutex).Lock+0x67 sync/mutex.go:81
# 0x1200c47 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).register.func17+0x47 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:294
# 0xaf9465 vitess.io/vitess/go/stats.CounterFunc.String+0x25 vitess.io/vitess/go/stats/counter.go:117
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
212287 1 @ 0x1201a4d 0x1201a30 0xafada3 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x1201a4c sync.(*Mutex).Lock+0x6c sync/mutex.go:81
# 0x1201a2f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).register.func12+0x4f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:214
# 0xafada2 vitess.io/vitess/go/stats.CountersFuncWithMultiLabels.String+0x62 vitess.io/vitess/go/stats/counters.go:274
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
69210 1 @ 0x12017eb 0x12017cb 0xaf9466 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x12017ea sync.(*Mutex).Lock+0x6a sync/mutex.go:81
# 0x12017ca vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).register.func13+0x4a vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:232
# 0xaf9465 vitess.io/vitess/go/stats.CounterFunc.String+0x25 vitess.io/vitess/go/stats/counter.go:117
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
47520 1 @ 0x1200e2d 0x1200e10 0xafada3 0x505123 0x505cd3 0x508588 0x5027f5 0xaf8cd3 0xaf8935 0xaf8b30 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x1200e2c sync.(*Mutex).Lock+0x6c sync/mutex.go:81
# 0x1200e0f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vrStats).register.func16+0x4f vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/stats.go:281
# 0xafada2 vitess.io/vitess/go/stats.CountersFuncWithMultiLabels.String+0x62 vitess.io/vitess/go/stats/counters.go:274
# 0x505122 fmt.(*pp).handleMethods+0x382 fmt/print.go:626
# 0x505cd2 fmt.(*pp).printArg+0x692 fmt/print.go:709
# 0x508587 fmt.(*pp).doPrintf+0x287 fmt/print.go:1026
# 0x5027f4 fmt.Fprintf+0x74 fmt/print.go:204
# 0xaf8cd2 expvar.expvarHandler.func1+0x132 expvar/expvar.go:341
# 0xaf8934 expvar.Do+0x154 expvar/expvar.go:328
# 0xaf8b2f expvar.expvarHandler+0x14f expvar/expvar.go:336
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
37373 1 @ 0x127b52c 0x127b4e8 0x128c3df 0x1287985 0x128551c 0x128ebf2 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x127b52b sync.(*Mutex).Lock+0x8b sync/mutex.go:81
# 0x127b4e7 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).StartRequest+0x47 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:348
# 0x128c3de vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x45e vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1298
# 0x1287984 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).Execute+0x384 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:723
# 0x128551b vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).IsHealthy+0x9b vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:394
# 0x128ebf1 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).registerDebugHealthHandler.func1+0xf1 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1603
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
29273 1 @ 0x9c2faf 0x9c2f88 0x9b22d4 0x9b22b4 0x9b5ded 0x469301
# 0x9c2fae sync.(*Mutex).Lock+0x6e sync/mutex.go:81
# 0x9c2f87 google.golang.org/grpc/internal/transport.(*recvBuffer).put+0x47 google.golang.org/[email protected]/internal/transport/transport.go:99
# 0x9b22d3 google.golang.org/grpc/internal/transport.(*Stream).write+0x373 google.golang.org/[email protected]/internal/transport/transport.go:457
# 0x9b22b3 google.golang.org/grpc/internal/transport.(*http2Client).handleData+0x353 google.golang.org/[email protected]/internal/transport/http2_client.go:1053
# 0x9b5dec google.golang.org/grpc/internal/transport.(*http2Client).reader+0x8ac google.golang.org/[email protected]/internal/transport/http2_client.go:1486
27338 1 @ 0x11fc3e5 0x11fc3c4 0x1212d9a 0x1211e32 0x469301
# 0x11fc3e4 sync.(*Mutex).Lock+0x64 sync/mutex.go:81
# 0x11fc3c3 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x43 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:89
# 0x1212d99 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x219 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:346
# 0x1211e31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
21915 1 @ 0x9a2932 0x9a2908 0x9b1cb5 0x9b1c78 0x9acc27 0x9c431f 0x4a8d7a 0x9c4245 0x9c421b 0x9eef0b 0x9ef6e6 0x9efd8e 0xa013f0 0xa007a5 0x9ff636 0xa004df 0xb28637 0x13535ac 0x136c3d9 0x11fb5e9 0x1211f51 0x469301
# 0x9a2931 sync.(*Mutex).Lock+0x51 sync/mutex.go:81
# 0x9a2907 google.golang.org/grpc/internal/transport.(*controlBuffer).executeAndPut+0x27 google.golang.org/[email protected]/internal/transport/controlbuf.go:329
# 0x9b1cb4 google.golang.org/grpc/internal/transport.(*controlBuffer).put+0x74 google.golang.org/[email protected]/internal/transport/controlbuf.go:323
# 0x9b1c77 google.golang.org/grpc/internal/transport.(*http2Client).updateWindow+0x37 google.golang.org/[email protected]/internal/transport/http2_client.go:969
# 0x9acc26 google.golang.org/grpc/internal/transport.(*http2Client).newStream.func3+0x26 google.golang.org/[email protected]/internal/transport/http2_client.go:442
# 0x9c431e google.golang.org/grpc/internal/transport.(*transportReader).Read+0x7e google.golang.org/[email protected]/internal/transport/transport.go:488
# 0x4a8d79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c4244 io.ReadFull+0xa4 io/io.go:347
# 0x9c421a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/[email protected]/internal/transport/transport.go:467
# 0x9eef0a google.golang.org/grpc.(*parser).recvMsg+0xca google.golang.org/[email protected]/rpc_util.go:578
# 0x9ef6e5 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/[email protected]/rpc_util.go:690
# 0x9efd8d google.golang.org/grpc.recv+0x6d google.golang.org/[email protected]/rpc_util.go:758
# 0xa013ef google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/[email protected]/stream.go:957
# 0xa007a4 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/[email protected]/stream.go:808
# 0x9ff635 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/[email protected]/stream.go:662
# 0xa004de google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/[email protected]/stream.go:807
# 0xb28636 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/[email protected]/client_metrics.go:160
# 0x13535ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:461
# 0x136c3d8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:648
# 0x11fb5e8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x1211f50 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
4680 1 @ 0x6a9505 0x6a94de 0x6a9858 0x469301
# 0x6a9504 sync.(*Mutex).Lock+0x44 sync/mutex.go:81
# 0x6a94dd net/http.(*connReader).lock+0x1d net/http/server.go:649
# 0x6a9857 net/http.(*connReader).backgroundRead+0x57 net/http/server.go:673
Raw
goroutines?debug=1
goroutine profile: total 529
202 @ 0x4387d6 0x44966c 0x449646 0x465145 0x4731e5 0x12564b4 0x1256492 0x1270c7f 0x1289393 0x128c491 0x1289205 0x136386b 0x1356db0 0xb2a049 0xb0423a 0xb040df 0x9f8824 0x9f9cc5 0x9f3ab8 0x469301
# 0x465144 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x4731e4 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x12564b3 sync.(*Mutex).Lock+0x93 sync/mutex.go:81
# 0x1256491 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe+0x71 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:127
# 0x1270c7e vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x21e vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:362
# 0x1289392 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0x112 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:985
# 0x128c490 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x510 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1308
# 0x1289204 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0xc4 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:969
# 0x136386a vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1aa vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:291
# 0x1356daf vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0xcf vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:971
# 0xb2a048 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/[email protected]/server_metrics.go:121
# 0xb04239 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/[email protected]/chain.go:49
# 0xb040de github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/[email protected]/chain.go:58
# 0x9f8823 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/[email protected]/server.go:1548
# 0x9f9cc4 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/[email protected]/server.go:1621
# 0x9f3ab7 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/[email protected]/server.go:940
47 @ 0x4387d6 0x448572 0x10f5411 0x469301
# 0x10f5410 vitess.io/vitess/go/timer.(*Timer).run+0xb0 vitess.io/vitess/go/timer/timer.go:98
39 @ 0x4387d6 0x430e53 0x463949 0x4ed8d2 0x4eec3a 0x4eec28 0x5b3ca9 0x5c4fe5 0x571374 0x4a8d7a 0x983cce 0x983c88 0x984415 0x9bab72 0x9f36e2 0x9f3006 0x469301
# 0x463948 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:234
# 0x4ed8d1 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4eec39 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89
# 0x4eec27 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167
# 0x5b3ca8 net.(*netFD).Read+0x28 net/fd_posix.go:56
# 0x5c4fe4 net.(*conn).Read+0x44 net/net.go:183
# 0x571373 bufio.(*Reader).Read+0x1b3 bufio/bufio.go:227
# 0x4a8d79 io.ReadAtLeast+0x99 io/io.go:328
# 0x983ccd io.ReadFull+0x6d io/io.go:347
# 0x983c87 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/[email protected]/http2/frame.go:237
# 0x984414 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/[email protected]/http2/frame.go:492
# 0x9bab71 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams+0xb1 google.golang.org/[email protected]/internal/transport/http2_server.go:541
# 0x9f36e1 google.golang.org/grpc.(*Server).serveStreams+0x141 google.golang.org/[email protected]/server.go:924
# 0x9f3005 google.golang.org/grpc.(*Server).handleRawConn.func1+0x45 google.golang.org/[email protected]/server.go:874
39 @ 0x4387d6 0x448572 0x1266c25 0x128e46a 0x1363f74 0x1357db3 0xb2a049 0xb0423a 0xb040df 0x9f8824 0x9f9cc5 0x9f3ab8 0x469301
# 0x1266c24 vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x1e4 vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169
# 0x128e469 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x29 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1521
# 0x1363f73 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x173 vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:320
# 0x1357db2 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0xd2 vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:1106
# 0xb2a048 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/[email protected]/server_metrics.go:121
# 0xb04239 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/[email protected]/chain.go:49
# 0xb040de github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/[email protected]/chain.go:58
# 0x9f8823 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/[email protected]/server.go:1548
# 0x9f9cc4 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/[email protected]/server.go:1621
# 0x9f3ab7 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/[email protected]/server.go:940
39 @ 0x4387d6 0x448572 0x9a2e3b 0x9a3525 0x9b87e6 0x469301
# 0x9a2e3a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/[email protected]/internal/transport/controlbuf.go:406
# 0x9a3524 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/[email protected]/internal/transport/controlbuf.go:533
# 0x9b87e5 google.golang.org/grpc/internal/transport.NewServerTransport.func2+0xc5 google.golang.org/[email protected]/internal/transport/http2_server.go:304
39 @ 0x4387d6 0x448572 0x9be925 0x469301
# 0x9be924 google.golang.org/grpc/internal/transport.(*http2Server).keepalive+0x244 google.golang.org/[email protected]/internal/transport/http2_server.go:1061
19 @ 0x4387d6 0x40666c 0x406098 0xafe4b6 0x469301
# 0xafe4b5 vitess.io/vitess/go/stats.(*Rates).track+0x35 vitess.io/vitess/go/stats/rates.go:99
17 @ 0x4387d6 0x46531d 0x4652fd 0x47158c 0x1259b48 0x469301
# 0x4652fc sync.runtime_notifyListWait+0x11c runtime/sema.go:513
# 0x47158b sync.(*Cond).Wait+0x8b sync/cond.go:56
# 0x1259b47 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend+0x427 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:525
6 @ 0x4387d6 0x448572 0x9fe298 0x469301
# 0x9fe297 google.golang.org/grpc.newClientStreamWithParams.func4+0x97 google.golang.org/[email protected]/stream.go:363
5 @ 0x4387d6 0x40666c 0x406098 0x9e834f 0x469301
# 0x9e834e google.golang.org/grpc.(*addrConn).resetTransport+0x48e google.golang.org/[email protected]/clientconn.go:1219
5 @ 0x4387d6 0x430e53 0x463949 0x4ed8d2 0x4eec3a 0x4eec28 0x5b3ca9 0x5c4fe5 0x571374 0x4a8d7a 0x983cce 0x983c88 0x984415 0x9b5954 0x469301
# 0x463948 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:234
# 0x4ed8d1 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4eec39 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89
# 0x4eec27 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167
# 0x5b3ca8 net.(*netFD).Read+0x28 net/fd_posix.go:56
# 0x5c4fe4 net.(*conn).Read+0x44 net/net.go:183
# 0x571373 bufio.(*Reader).Read+0x1b3 bufio/bufio.go:227
# 0x4a8d79 io.ReadAtLeast+0x99 io/io.go:328
# 0x983ccd io.ReadFull+0x6d io/io.go:347
# 0x983c87 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/[email protected]/http2/frame.go:237
# 0x984414 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/[email protected]/http2/frame.go:492
# 0x9b5953 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x413 google.golang.org/[email protected]/internal/transport/http2_client.go:1451
5 @ 0x4387d6 0x448572 0x9a2e3b 0x9a3525 0x9ac3a5 0x469301
# 0x9a2e3a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/[email protected]/internal/transport/controlbuf.go:406
# 0x9a3524 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/[email protected]/internal/transport/controlbuf.go:533
# 0x9ac3a4 google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x64 google.golang.org/[email protected]/internal/transport/http2_client.go:396
5 @ 0x4387d6 0x448572 0x9df665 0x469301
# 0x9df664 google.golang.org/grpc.(*ccBalancerWrapper).watcher+0xa4 google.golang.org/[email protected]/balancer_conn_wrappers.go:71
3 @ 0x4387d6 0x40666c 0x406098 0x11fbf49 0x469301
# 0x11fbf48 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.newRelayLog.func1+0x48 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:58
3 @ 0x4387d6 0x40666c 0x4060d8 0x10f50cb 0x469301
# 0x10f50ca vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x4a vitess.io/vitess/go/timer/suspendable_ticker.go:74
3 @ 0x4387d6 0x448572 0x11fc968 0x469301
# 0x11fc967 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer.func1+0x87 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:124
3 @ 0x4387d6 0x448572 0x12119a5 0x121156b 0x121553a 0x1214cc8 0x11f08da 0x11ef9a6 0x469301
# 0x12119a4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply+0x3e4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:158
# 0x121156a vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).play+0x32a vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:125
# 0x1215539 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vreplicator).replicate+0x7d9 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go:249
# 0x1214cc7 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vreplicator).Replicate+0x27 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go:165
# 0x11f08d9 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*controller).runBlp+0xab9 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/controller.go:271
# 0x11ef9a5 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*controller).run+0xa5 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/controller.go:152
3 @ 0x4387d6 0x448572 0x9b61c9 0x469301
# 0x9b61c8 google.golang.org/grpc/internal/transport.(*http2Client).keepalive+0x168 google.golang.org/[email protected]/internal/transport/http2_client.go:1525
3 @ 0x4387d6 0x448572 0x9c372a 0x9c3467 0x9c42d2 0x4a8d7a 0x9c4245 0x9c421b 0x9eee87 0x9ef6e6 0x9efd8e 0xa013f0 0xa007a5 0x9ff636 0xa004df 0xb28637 0x13535ac 0x136c3d9 0x11fb5e9 0x1211f51 0x469301
# 0x9c3729 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/[email protected]/internal/transport/transport.go:189
# 0x9c3466 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/[email protected]/internal/transport/transport.go:169
# 0x9c42d1 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/[email protected]/internal/transport/transport.go:483
# 0x4a8d79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c4244 io.ReadFull+0xa4 io/io.go:347
# 0x9c421a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/[email protected]/internal/transport/transport.go:467
# 0x9eee86 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/[email protected]/rpc_util.go:559
# 0x9ef6e5 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/[email protected]/rpc_util.go:690
# 0x9efd8d google.golang.org/grpc.recv+0x6d google.golang.org/[email protected]/rpc_util.go:758
# 0xa013ef google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/[email protected]/stream.go:957
# 0xa007a4 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/[email protected]/stream.go:808
# 0x9ff635 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/[email protected]/stream.go:662
# 0xa004de google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/[email protected]/stream.go:807
# 0xb28636 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/[email protected]/client_metrics.go:160
# 0x13535ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:461
# 0x136c3d8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:648
# 0x11fb5e8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x1211f50 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
3 @ 0x4387d6 0x46531d 0x4652fd 0x47158c 0x11fc4be 0x1212d9a 0x1211e32 0x469301
# 0x4652fc sync.runtime_notifyListWait+0x11c runtime/sema.go:513
# 0x47158b sync.(*Cond).Wait+0x8b sync/cond.go:56
# 0x11fc4bd vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x13d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98
# 0x1212d99 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x219 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:346
# 0x1211e31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
2 @ 0x4387d6 0x448572 0x494465 0x469301
# 0x494464 context.propagateCancel.func1+0xa4 context/context.go:279
2 @ 0x4387d6 0x448572 0x9c372a 0x9c3467 0x9c42d2 0x4a8d7a 0x9c4245 0x9c421b 0x9eee87 0x9ef6e6 0x9efd8e 0xa013f0 0xa007a5 0x9ff636 0xa004df 0xee040c 0xf84cca 0x469301
# 0x9c3729 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/[email protected]/internal/transport/transport.go:189
# 0x9c3466 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/[email protected]/internal/transport/transport.go:169
# 0x9c42d1 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/[email protected]/internal/transport/transport.go:483
# 0x4a8d79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c4244 io.ReadFull+0xa4 io/io.go:347
# 0x9c421a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/[email protected]/internal/transport/transport.go:467
# 0x9eee86 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/[email protected]/rpc_util.go:559
# 0x9ef6e5 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/[email protected]/rpc_util.go:690
# 0x9efd8d google.golang.org/grpc.recv+0x6d google.golang.org/[email protected]/rpc_util.go:758
# 0xa013ef google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/[email protected]/stream.go:957
# 0xa007a4 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/[email protected]/stream.go:808
# 0x9ff635 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/[email protected]/stream.go:662
# 0xa004de google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/[email protected]/stream.go:807
# 0xee040b go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv+0x4b go.etcd.io/etcd/api/[email protected]/etcdserverpb/rpc.pb.go:6714
# 0xf84cc9 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient+0x69 go.etcd.io/etcd/client/[email protected]/watch.go:757
2 @ 0x4387d6 0x448572 0xf82894 0x469301
# 0xf82893 go.etcd.io/etcd/client/v3.(*watchGrpcStream).run+0x373 go.etcd.io/etcd/client/[email protected]/watch.go:537
2 @ 0x4387d6 0x448572 0xf850a5 0x469301
# 0xf850a4 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream+0x264 go.etcd.io/etcd/client/[email protected]/watch.go:803
2 @ 0x4387d6 0x448572 0xf971cf 0x469301
# 0xf971ce vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1+0x1ae vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78
1 @ 0x40b614 0x42caca 0x42cabe 0x42cabd 0x42c9c5 0x4066d4 0x4060d8 0x11c6905 0x469301
# 0x11c6904 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xa4 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39
1 @ 0x40bb54 0x4657f8 0xb395d9 0x469301
# 0x4657f7 os/signal.signal_recv+0x97 runtime/sigqueue.go:169
# 0xb395d8 os/signal.loop+0x18 os/signal/signal_unix.go:24
1 @ 0x4387d6 0x40666c 0x406098 0xb72834 0x1e7ec06 0x1e7ebfe 0x438407 0x469301
# 0xb72833 vitess.io/vitess/go/vt/servenv.Run+0x213 vitess.io/vitess/go/vt/servenv/run.go:57
# 0x1e7ec05 vitess.io/vitess/go/vt/servenv.RunDefault+0x825 vitess.io/vitess/go/vt/servenv/servenv.go:229
# 0x1e7ebfd main.main+0x81d vitess.io/vitess/go/cmd/vttablet/vttablet.go:120
# 0x438406 runtime.main+0x226 runtime/proc.go:255
1 @ 0x4387d6 0x40666c 0x4060d8 0x104a678 0x469301
# 0x104a677 vitess.io/vitess/go/vt/dbconfigs.init.0.func1+0x37 vitess.io/vitess/go/vt/dbconfigs/credentials.go:283
1 @ 0x4387d6 0x40666c 0x4060d8 0x105fc0d 0x469301
# 0x105fc0c vitess.io/vitess/go/vt/srvtopo.NewSrvVSchemaWatcher.func1+0x10c vitess.io/vitess/go/vt/srvtopo/watch_srvvschema.go:49
1 @ 0x4387d6 0x40666c 0x4060d8 0x128527c 0x469301
# 0x128527b vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL.func1+0x7b vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:344
1 @ 0x4387d6 0x40666c 0x4060d8 0x13df10a 0x469301
# 0x13df109 k8s.io/klog/v2.(*loggingT).flushDaemon+0x69 k8s.io/klog/[email protected]/klog.go:1169
1 @ 0x4387d6 0x40666c 0x4060d8 0x7a6f2a 0x469301
# 0x7a6f29 github.com/golang/glog.(*loggingT).flushDaemon+0x69 github.com/golang/[email protected]/glog.go:882
1 @ 0x4387d6 0x40666c 0x4060d8 0xdbda3e 0x469301
# 0xdbda3d vitess.io/vitess/go/vt/topo.(*Server).WatchShard.func1+0xbd vitess.io/vitess/go/vt/topo/shard.go:683
1 @ 0x4387d6 0x40666c 0x4060d8 0xdc41be 0x469301
# 0xdc41bd vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema.func1+0xbd vitess.io/vitess/go/vt/topo/srv_vschema.go:74
1 @ 0x4387d6 0x430e53 0x463949 0x4ed8d2 0x4eec3a 0x4eec28 0x5b3ca9 0x5c4fe5 0x6a9ced 0x570dc3 0x57198f 0x571be7 0x63c4b9 0x6a51f9 0x6a51fa 0x6ab0a5 0x6af445 0x469301
# 0x463948 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:234
# 0x4ed8d1 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4eec39 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89
# 0x4eec27 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167
# 0x5b3ca8 net.(*netFD).Read+0x28 net/fd_posix.go:56
# 0x5c4fe4 net.(*conn).Read+0x44 net/net.go:183
# 0x6a9cec net/http.(*connReader).Read+0x16c net/http/server.go:780
# 0x570dc2 bufio.(*Reader).fill+0x102 bufio/bufio.go:101
# 0x57198e bufio.(*Reader).ReadSlice+0x2e bufio/bufio.go:360
# 0x571be6 bufio.(*Reader).ReadLine+0x26 bufio/bufio.go:389
# 0x63c4b8 net/textproto.(*Reader).readLineSlice+0x98 net/textproto/reader.go:57
# 0x6a51f8 net/textproto.(*Reader).ReadLine+0x78 net/textproto/reader.go:38
# 0x6a51f9 net/http.readRequest+0x79 net/http/request.go:1029
# 0x6ab0a4 net/http.(*conn).readRequest+0x224 net/http/server.go:966
# 0x6af444 net/http.(*conn).serve+0x864 net/http/server.go:1856
1 @ 0x4387d6 0x430e53 0x463949 0x4ed8d2 0x4f0c2c 0x4f0c19 0x5b5455 0x5ce068 0x5cd23d 0x6b41b4 0x6b27b4 0x469301
# 0x463948 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:234
# 0x4ed8d1 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4f0c2b internal/poll.(*pollDesc).waitRead+0x22b internal/poll/fd_poll_runtime.go:89
# 0x4f0c18 internal/poll.(*FD).Accept+0x218 internal/poll/fd_unix.go:402
# 0x5b5454 net.(*netFD).accept+0x34 net/fd_unix.go:173
# 0x5ce067 net.(*TCPListener).accept+0x27 net/tcpsock_posix.go:140
# 0x5cd23c net.(*TCPListener).Accept+0x3c net/tcpsock.go:262
# 0x6b41b3 net/http.(*Server).Serve+0x393 net/http/server.go:3002
# 0x6b27b3 net/http.Serve+0x73 net/http/server.go:2506
1 @ 0x4387d6 0x430e53 0x463949 0x4ed8d2 0x4f0c2c 0x4f0c19 0x5b5455 0x5ce068 0x5cd23d 0x9f22a2 0xb6eb0c 0x469301
# 0x463948 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:234
# 0x4ed8d1 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4f0c2b internal/poll.(*pollDesc).waitRead+0x22b internal/poll/fd_poll_runtime.go:89
# 0x4f0c18 internal/poll.(*FD).Accept+0x218 internal/poll/fd_unix.go:402
# 0x5b5454 net.(*netFD).accept+0x34 net/fd_unix.go:173
# 0x5ce067 net.(*TCPListener).accept+0x27 net/tcpsock_posix.go:140
# 0x5cd23c net.(*TCPListener).Accept+0x3c net/tcpsock.go:262
# 0x9f22a1 google.golang.org/grpc.(*Server).Serve+0x361 google.golang.org/[email protected]/server.go:786
# 0xb6eb0b vitess.io/vitess/go/vt/servenv.serveGRPC.func1+0x2b vitess.io/vitess/go/vt/servenv/grpc_server.go:255
1 @ 0x4387d6 0x448572 0x11bab49 0x469301
# 0x11bab48 github.com/patrickmn/go-cache.(*janitor).Run+0x88 github.com/patrickmn/[email protected]+incompatible/cache.go:1079
1 @ 0x4387d6 0x448572 0x1251508 0x469301
# 0x1251507 vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).Operate+0x1e7 vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:242
1 @ 0x4387d6 0x448572 0x12daf39 0x469301
# 0x12daf38 go.opencensus.io/stats/view.(*worker).start+0xb8 [email protected]/stats/view/worker.go:276
1 @ 0x4387d6 0x448572 0x13aaa12 0x469301
# 0x13aaa11 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).shardSyncLoop+0x1b1 vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:67
1 @ 0x4387d6 0x448572 0x9c372a 0x9c3467 0x9c42d2 0x4a8d7a 0x9c4245 0x9c421b 0x9eee87 0x9ef6e6 0x9efd8e 0xa013f0 0xa007a5 0x9ff636 0xa004df 0xee0aec 0xf75078 0x469301
# 0x9c3729 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/[email protected]/internal/transport/transport.go:189
# 0x9c3466 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/[email protected]/internal/transport/transport.go:169
# 0x9c42d1 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/[email protected]/internal/transport/transport.go:483
# 0x4a8d79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c4244 io.ReadFull+0xa4 io/io.go:347
# 0x9c421a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/[email protected]/internal/transport/transport.go:467
# 0x9eee86 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/[email protected]/rpc_util.go:559
# 0x9ef6e5 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/[email protected]/rpc_util.go:690
# 0x9efd8d google.golang.org/grpc.recv+0x6d google.golang.org/[email protected]/rpc_util.go:758
# 0xa013ef google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/[email protected]/stream.go:957
# 0xa007a4 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/[email protected]/stream.go:808
# 0x9ff635 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/[email protected]/stream.go:662
# 0xa004de google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/[email protected]/stream.go:807
# 0xee0aeb go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv+0x4b go.etcd.io/etcd/api/[email protected]/etcdserverpb/rpc.pb.go:6853
# 0xf75077 go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop+0x297 go.etcd.io/etcd/client/[email protected]/lease.go:447
1 @ 0x4387d6 0x448572 0xae7a5f 0x469301
# 0xae7a5e vitess.io/vitess/go/cache/ristretto.(*Cache).processItems+0x13e vitess.io/vitess/go/cache/ristretto/cache.go:458
1 @ 0x4387d6 0x448572 0xae8c0c 0x469301
# 0xae8c0b vitess.io/vitess/go/cache/ristretto.(*defaultPolicy).processItems+0x8b vitess.io/vitess/go/cache/ristretto/policy.go:107
1 @ 0x4387d6 0x448572 0xf75e85 0x469301
# 0xf75e84 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x84 go.etcd.io/etcd/client/[email protected]/lease.go:539
1 @ 0x4387d6 0x448572 0xf761f9 0x469301
# 0xf761f8 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x1f8 go.etcd.io/etcd/client/[email protected]/lease.go:579
1 @ 0x4387d6 0x44966c 0x449646 0x465025 0x474bd1 0x9f36ec 0x9f3006 0x469301
# 0x465024 sync.runtime_Semacquire+0x24 runtime/sema.go:56
# 0x474bd0 sync.(*WaitGroup).Wait+0x70 sync/waitgroup.go:130
# 0x9f36eb google.golang.org/grpc.(*Server).serveStreams+0x14b google.golang.org/[email protected]/server.go:950
# 0x9f3005 google.golang.org/grpc.(*Server).handleRawConn.func1+0x45 google.golang.org/[email protected]/server.go:874
1 @ 0x4387d6 0x44966c 0x449646 0x465145 0x4731e5 0x1255f2d 0x1255f04 0x127c597 0x127bf65 0x127a7b8 0x127a345 0x12852e5 0x13a40f8 0x13a3e3c 0x13be6ac 0x137f398 0xb29e07 0xb03f1a 0xb03dbf 0x137f258 0x9f5fcf 0x9f9d0a 0x9f3ab8 0x469301
# 0x465144 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x4731e4 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x1255f2c sync.(*Mutex).Lock+0x8c sync/mutex.go:81
# 0x1255f03 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Close+0x63 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:91
# 0x127c596 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).unserveCommon+0x1d6 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:511
# 0x127bf64 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).unservePrimary+0x24 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:429
# 0x127a7b7 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).execTransition+0x97 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:254
# 0x127a344 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).SetServingType+0x284 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:216
# 0x12852e4 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).SetServingType+0x44 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:367
# 0x13a40f7 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).demotePrimary+0x217 vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:439
# 0x13a3e3b vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).DemotePrimary+0x5b vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:390
# 0x13be6ab vitess.io/vitess/go/vt/vttablet/grpctmserver.(*server).DemotePrimary+0x1cb vitess.io/vitess/go/vt/vttablet/grpctmserver/server.go:444
# 0x137f397 vitess.io/vitess/go/vt/proto/tabletmanagerservice._TabletManager_DemotePrimary_Handler.func1+0x77 vitess.io/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice_grpc.pb.go:1577
# 0xb29e06 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1+0x86 github.com/grpc-ecosystem/[email protected]/server_metrics.go:107
# 0xb03f19 github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1+0x39 github.com/grpc-ecosystem/[email protected]/chain.go:25
# 0xb03dbe github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1+0xbe github.com/grpc-ecosystem/[email protected]/chain.go:34
# 0x137f257 vitess.io/vitess/go/vt/proto/tabletmanagerservice._TabletManager_DemotePrimary_Handler+0x137 vitess.io/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice_grpc.pb.go:1579
# 0x9f5fce google.golang.org/grpc.(*Server).processUnaryRPC+0xc6e google.golang.org/[email protected]/server.go:1292
# 0x9f9d09 google.golang.org/grpc.(*Server).handleStream+0xa29 google.golang.org/[email protected]/server.go:1617
# 0x9f3ab7 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/[email protected]/server.go:940
1 @ 0x4387d6 0x44966c 0x449646 0x465145 0x4731e5 0x12567f3 0x12567d6 0x1165d15 0x11641b7 0x11635b3 0x1162c6d 0x1162c6e 0x10f5483 0x469301
# 0x465144 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x4731e4 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x12567f2 sync.(*Mutex).Lock+0x92 sync/mutex.go:81
# 0x12567d5 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).schemaChanged+0x75 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:140
# 0x1165d14 vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).broadcast+0x2b4 vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:546
# 0x11641b6 vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).reload+0xa36 vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:405
# 0x11635b2 vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).ReloadAt+0x1f2 vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:286
# 0x1162c6c vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Reload+0x2c vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:268
# 0x1162c6d vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Open.func2+0x2d vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:207
# 0x10f5482 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
1 @ 0x4387d6 0x44966c 0x449646 0x465145 0x4731e5 0x1258c25 0x1258bf7 0x1256549 0x1270c7f 0x1289393 0x128c491 0x1289205 0x136386b 0x1356db0 0xb2a049 0xb0423a 0xb040df 0x9f8824 0x9f9cc5 0x9f3ab8 0x469301
# 0x465144 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x4731e4 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x1258c24 sync.(*Mutex).Lock+0xa4 sync/mutex.go:81
# 0x1258bf6 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe+0x76 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:388
# 0x1256548 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe+0x128 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:136
# 0x1270c7e vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x21e vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:362
# 0x1289392 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0x112 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:985
# 0x128c490 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x510 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1308
# 0x1289204 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0xc4 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:969
# 0x136386a vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1aa vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:291
# 0x1356daf vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0xcf vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:971
# 0xb2a048 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/[email protected]/server_metrics.go:121
# 0xb04239 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/[email protected]/chain.go:49
# 0xb040de github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/[email protected]/chain.go:58
# 0x9f8823 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/[email protected]/server.go:1548
# 0x9f9cc4 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/[email protected]/server.go:1621
# 0x9f3ab7 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/[email protected]/server.go:940
1 @ 0x4387d6 0x44966c 0x449646 0x465145 0x4731e5 0x125ab5e 0x125ab2f 0x1259293 0x1259045 0x469301
# 0x465144 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x4731e4 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x125ab5d sync.(*Mutex).Lock+0x7d sync/mutex.go:81
# 0x125ab2e vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).stopVStream+0x4e vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:631
# 0x1259292 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).unsubscribe+0x1d2 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:439
# 0x1259044 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1+0x44 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:416
1 @ 0x4387d6 0x44966c 0x449646 0x465145 0x4731e5 0x125b0a8 0x125b074 0x11e4f9c 0x11e62d5 0x11e63df 0x11e68c9 0x11e69d3 0x11d969d 0x125afb3 0x125ad30 0x469301
# 0x465144 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x4731e4 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x125b0a7 sync.(*Mutex).Lock+0xa7 sync/mutex.go:81
# 0x125b073 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream.func1+0x73 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:668
# 0x11e4f9b vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.newUVStreamer.func1+0xdb vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:105
# 0x11e62d4 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).sendEventsForCurrentPos+0x114 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:309
# 0x11e63de vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).setStreamStartPosition+0x9e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:322
# 0x11e68c8 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).init+0x28 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:350
# 0x11e69d2 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x72 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:368
# 0x11d969c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x1bc vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227
# 0x125afb2 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x132 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:667
# 0x125ad2f vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x6f vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:643
1 @ 0x4387d6 0x44966c 0x449646 0x465145 0x4731e5 0x125bb91 0x125bb7b 0x10f5483 0x469301
# 0x465144 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x4731e4 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x125bb90 sync.(*Mutex).Lock+0x390 sync/mutex.go:81
# 0x125bb7a vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x37a vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:777
# 0x10f5482 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
1 @ 0x463465 0xb3e8f5 0xb3e70d 0xb3b88b 0xb6363a 0xb641ee 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x463464 runtime/pprof.runtime_goroutineProfileWithLabels+0x24 runtime/mprof.go:746
# 0xb3e8f4 runtime/pprof.writeRuntimeProfile+0xb4 runtime/pprof/pprof.go:724
# 0xb3e70c runtime/pprof.writeGoroutine+0x4c runtime/pprof/pprof.go:684
# 0xb3b88a runtime/pprof.(*Profile).WriteTo+0x14a runtime/pprof/pprof.go:331
# 0xb63639 net/http/pprof.handler.ServeHTTP+0x499 net/http/pprof/pprof.go:253
# 0xb641ed net/http/pprof.Index+0x12d net/http/pprof/pprof.go:371
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
1 @ 0x469301
Raw
logs
I0401 19:11:09.972142 1817963 message_manager.go:646] Context canceled, exiting vstream
I0401 19:11:11.858632 1817963 uvstreamer.go:367] Stream() called
I0401 19:11:11.858852 1817963 message_manager.go:630] messageManager - stopVStream called. Acquiring streamMu lock
I0401 19:11:11.858866 1817963 message_manager.go:632] messageManager - acquired streamMu lock
I0401 19:11:11.858870 1817963 message_manager.go:634] messageManager - calling stream cancel
I0401 19:11:11.858874 1817963 cache.go:116] messager cache - Clearing cache. Acquiring my lock
I0401 19:11:11.858878 1817963 cache.go:118] messager cache - acquired lock
I0401 19:11:11.858881 1817963 cache.go:123] messager cache - cache cleared
I0401 19:11:11.858962 1817963 message_manager.go:646] Context canceled, exiting vstream
I0401 19:11:12.506185 1817963 snapshot_conn.go:79] Locking table searches__extractor__msgs for copying
I0401 19:11:12.507237 1817963 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs
I0401 19:11:13.157549 1817963 uvstreamer.go:367] Stream() called
I0401 19:11:13.157614 1817963 message_manager.go:630] messageManager - stopVStream called. Acquiring streamMu lock
I0401 19:11:13.157624 1817963 message_manager.go:632] messageManager - acquired streamMu lock
I0401 19:11:13.157628 1817963 message_manager.go:634] messageManager - calling stream cancel
I0401 19:11:13.157633 1817963 cache.go:116] messager cache - Clearing cache. Acquiring my lock
I0401 19:11:13.157636 1817963 cache.go:118] messager cache - acquired lock
I0401 19:11:13.157640 1817963 cache.go:123] messager cache - cache cleared
I0401 19:11:13.157658 1817963 message_manager.go:646] Context canceled, exiting vstream
E0401 19:11:13.630643 1817963 message_manager.go:396] Terminating connection due to error sending field info: EOF
I0401 19:11:14.932349 1817963 uvstreamer.go:367] Stream() called
I0401 19:11:14.932526 1817963 message_manager.go:630] messageManager - stopVStream called. Acquiring streamMu lock
I0401 19:11:14.932536 1817963 message_manager.go:632] messageManager - acquired streamMu lock
I0401 19:11:14.932540 1817963 message_manager.go:634] messageManager - calling stream cancel
I0401 19:11:14.932545 1817963 cache.go:116] messager cache - Clearing cache. Acquiring my lock
I0401 19:11:14.932548 1817963 cache.go:118] messager cache - acquired lock
I0401 19:11:14.932552 1817963 cache.go:123] messager cache - cache cleared
I0401 19:11:14.932604 1817963 message_manager.go:646] Context canceled, exiting vstream
E0401 19:11:15.524285 1817963 message_manager.go:396] Terminating connection due to error sending field info: EOF
I0401 19:11:17.286647 1817963 uvstreamer.go:367] Stream() called
I0401 19:11:17.286718 1817963 message_manager.go:630] messageManager - stopVStream called. Acquiring streamMu lock
I0401 19:11:17.286737 1817963 message_manager.go:632] messageManager - acquired streamMu lock
I0401 19:11:17.286740 1817963 message_manager.go:634] messageManager - calling stream cancel
I0401 19:11:17.286748 1817963 cache.go:116] messager cache - Clearing cache. Acquiring my lock
I0401 19:11:17.286752 1817963 cache.go:118] messager cache - acquired lock
I0401 19:11:17.286756 1817963 cache.go:123] messager cache - cache cleared
I0401 19:11:17.286797 1817963 message_manager.go:646] Context canceled, exiting vstream
E0401 19:11:20.328147 1817963 message_manager.go:396] Terminating connection due to error sending field info: EOF
E0401 19:11:27.359574 1817963 message_manager.go:396] Terminating connection due to error sending field info: EOF
I0401 19:11:31.516406 1817963 uvstreamer.go:367] Stream() called
I0401 19:11:31.517264 1817963 uvstreamer.go:302] sendEventsForCurrentPos
I0401 19:11:31.517404 1817963 snapshot_conn.go:79] Locking table rankings__metrics_retriever__msgs for copying
I0401 19:11:31.517473 1817963 message_manager.go:630] messageManager - stopVStream called. Acquiring streamMu lock
I0401 19:11:31.518521 1817963 snapshot_conn.go:72] Tables unlocked: rankings__metrics_retriever__msgs
I0401 19:11:31.936290 1817963 uvstreamer.go:367] Stream() called
I0401 19:11:31.936406 1817963 message_manager.go:630] messageManager - stopVStream called. Acquiring streamMu lock
I0401 19:11:31.936430 1817963 message_manager.go:632] messageManager - acquired streamMu lock
I0401 19:11:31.936444 1817963 message_manager.go:634] messageManager - calling stream cancel
I0401 19:11:31.936451 1817963 cache.go:116] messager cache - Clearing cache. Acquiring my lock
I0401 19:11:31.936454 1817963 cache.go:118] messager cache - acquired lock
I0401 19:11:31.936458 1817963 cache.go:123] messager cache - cache cleared
I0401 19:11:31.936513 1817963 message_manager.go:646] Context canceled, exiting vstream
I0401 19:11:33.039948 1817963 uvstreamer.go:367] Stream() called
I0401 19:11:33.040396 1817963 message_manager.go:630] messageManager - stopVStream called. Acquiring streamMu lock
I0401 19:11:33.040425 1817963 message_manager.go:632] messageManager - acquired streamMu lock
I0401 19:11:33.040429 1817963 message_manager.go:634] messageManager - calling stream cancel
I0401 19:11:33.040435 1817963 cache.go:116] messager cache - Clearing cache. Acquiring my lock
I0401 19:11:33.040439 1817963 cache.go:118] messager cache - acquired lock
I0401 19:11:33.040442 1817963 cache.go:123] messager cache - cache cleared
I0401 19:11:33.040536 1817963 message_manager.go:646] Context canceled, exiting vstream
E0401 19:11:34.641239 1817963 message_manager.go:396] Terminating connection due to error sending field info: EOF
I0401 19:11:42.634411 1817963 snapshot_conn.go:79] Locking table searches__extractor__msgs for copying
I0401 19:11:42.635260 1817963 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs
I0401 19:12:12.758003 1817963 snapshot_conn.go:79] Locking table searches__extractor__msgs for copying
I0401 19:12:12.759004 1817963 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs
I0401 19:12:41.411490 1817963 message_manager.go:630] messageManager - stopVStream called. Acquiring streamMu lock
I0401 19:12:41.411514 1817963 message_manager.go:632] messageManager - acquired streamMu lock
I0401 19:12:41.411519 1817963 message_manager.go:634] messageManager - calling stream cancel
I0401 19:12:41.411537 1817963 cache.go:116] messager cache - Clearing cache. Acquiring my lock
I0401 19:12:41.411541 1817963 cache.go:118] messager cache - acquired lock
I0401 19:12:41.411546 1817963 cache.go:123] messager cache - cache cleared
I0401 19:12:41.411556 1817963 vstreamer.go:924] stream (at source tablet) ended @ 0602b256-3011-11ea-be15-82083c4d9668:1-15175234,2ac235ed-7176-11e9-b0fa-dabd372f7bb6:1-53608823,3772b195-6974-11ea-920b-92a325b23988:1-186,3daef343-3011-11ea-85cd-3e0b88929dd7:1-38372999,5ab1ba81-716e-11e9-850d-eeee6a5aaa6d:1-321393,7a5f4de6-6740-11e9-bf86-0a580a302102:1-251259,86146468-696e-11ea-b1ff-6eac01039183:1-70323,96217fbf-51c8-11e9-be7b-1ecc41b1d20c:1-1438808,9a1dcf78-6740-11e9-9ada-0a580a302802:1-13,9ea18f36-69ab-11ea-a7c9-5ec22c061b6d:1-19602636,a0c71122-597d-11e9-a573-0a580a300c05:1-5262758,c82ca0af-698b-11ea-a343-9697394464d9:1-52701,ebbb0482-69ee-11ea-b35a-eae11a254278:1-404134786,f7e7a719-e829-11ea-835e-da09945a0734:1-324053616,fd53af9e-4cbf-11e9-a52a-1a935972d8f4:1-7966040
I0401 19:12:41.411576 1817963 binlog_connection.go:279] closing binlog socket to unblock reads
E0401 19:12:41.411506 1817963 message_manager.go:598] Error sending messages: &{[] 0 0 [[INT64(804868096) INT64(105256323) INT64(1648829561572192779) INT64(1648829561572192779) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804868251) INT64(105290580) INT64(1648829561694614639) INT64(1648829561694614639) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804869803) INT64(105320549) INT64(1648829561118483134) INT64(1648829561118483134) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804869809) INT64(105343764) INT64(1648829561800977019) INT64(1648829561800977019) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804605663) INT64(107494197) INT64(1648826669008054590) INT64(1648826669008054590) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804605720) INT64(19284268) INT64(1648826668940457832) INT64(1648826668940457832) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804609087) INT64(107460383) INT64(1648826671479661870) INT64(1648826671479661870) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804609547) INT64(107467801) INT64(1648826669677486403) INT64(1648826669677486403) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804609557) INT64(107507813) INT64(1648826669177619640) INT64(1648826669177619640) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")] [INT64(804609563) INT64(107491190) INT64(1648826671395038518) INT64(1648826671395038518) JSON("{\"isDebug\": \"false\", \"tryCount\": \"0\"}") VARBINARY("")]] 0 }: EOF
I0401 19:12:41.411652 1817963 binlog_connection.go:153] connection closed during binlog stream (possibly intentional): read unix @->/vtdataroot/tabletdata/mysql.sock: use of closed network connection
io.ReadFull(header size) failed (errno 2013) (sqlstate HY000)
I0401 19:12:41.411673 1817963 binlog_connection.go:287] waiting for binlog dump thread to end
I0401 19:12:41.411679 1817963 binlog_connection.go:293] closing binlog MySQL client with serverID 363446208. Will recycle ID.
I0401 19:12:41.411687 1817963 message_manager.go:646] Context canceled, exiting vstream
I0401 19:47:50.002926 1817963 tablegc.go:387] TableGC: check tables
I0401 19:56:29.704657 1817963 rpc_replication.go:388] DemotePrimary
I0401 19:56:29.706324 1817963 rpc_replication.go:438] DemotePrimary disabling query service
I0401 19:56:30.102201 1817963 state_manager.go:214] Starting transition to PRIMARY Not Serving, timestamp: 2022-03-28 23:58:14.437817374 +0000 UTC
I0401 19:56:30.102247 1817963 state_manager.go:496] Started execution of unserveCommon
I0401 19:56:30.102250 1817963 state_manager.go:498] Finished execution of handleShutdownGracePeriod
I0401 19:56:30.102252 1817963 state_manager.go:501] Started online ddl executor close
I0401 19:56:30.102256 1817963 executor.go:299] onlineDDL Executor - Acquiring lock - initMutex
I0401 19:56:30.365826 1817963 executor.go:301] onlineDDL Executor - Acquired lock - initMutex
I0401 19:56:30.365835 1817963 executor.go:307] onlineDDL Executor - Stopping timer ticks
I0401 19:56:30.548862 1817963 executor.go:309] onlineDDL Executor - Closing the conpool
I0401 19:56:30.548869 1817963 pool.go:144] connpool - started execution of Close
I0401 19:56:30.548872 1817963 pool.go:146] connpool - found the pool
I0401 19:56:30.548875 1817963 pool.go:153] connpool - calling close on the pool
I0401 19:56:30.548945 1817963 pool.go:155] connpool - acquiring lock
I0401 19:56:30.548949 1817963 pool.go:157] connpool - acquired lock
I0401 19:56:30.548954 1817963 pool.go:160] connpool - closing dbaPool
I0401 19:56:30.548969 1817963 pool.go:162] connpool - finished execution of Close
I0401 19:56:30.549023 1817963 executor.go:312] onlineDDL Executor - finished Close execution
I0401 19:56:30.549028 1817963 state_manager.go:503] Finished online ddl executor close
I0401 19:56:30.549031 1817963 state_manager.go:504] Started table garbage collector close
I0401 19:56:30.549036 1817963 tablegc.go:205] TableGC - started execution of Close. Acquiring initMutex lock
I0401 19:56:30.549039 1817963 tablegc.go:207] TableGC - acquired lock
I0401 19:56:30.549044 1817963 tablegc.go:215] TableGC: closing
I0401 19:56:30.549050 1817963 tablegc.go:219] TableGC - closing pool
I0401 19:56:30.549054 1817963 pool.go:144] connpool - started execution of Close
I0401 19:56:30.549058 1817963 pool.go:146] connpool - found the pool
I0401 19:56:30.549061 1817963 pool.go:153] connpool - calling close on the pool
I0401 19:56:30.549112 1817963 pool.go:155] connpool - acquiring lock
I0401 19:56:30.549128 1817963 pool.go:157] connpool - acquired lock
I0401 19:56:30.549131 1817963 pool.go:160] connpool - closing dbaPool
I0401 19:56:30.549135 1817963 pool.go:162] connpool - finished execution of Close
I0401 19:56:30.549138 1817963 tablegc.go:222] TableGC - finished execution of Close
I0401 19:56:30.549141 1817963 state_manager.go:506] Finished table garbage collector close
I0401 19:56:30.549144 1817963 state_manager.go:507] Started lag throttler close
I0401 19:56:30.549149 1817963 throttler.go:266] Throttler - started execution of Close. Acquiring initMutex lock
I0401 19:56:30.549152 1817963 throttler.go:268] Throttler - acquired initMutex lock
I0401 19:56:30.549155 1817963 throttler.go:278] Throttler - finished suspending tickers
I0401 19:56:30.549158 1817963 throttler.go:281] Throttler - closing pool
I0401 19:56:30.549160 1817963 pool.go:144] connpool - started execution of Close
I0401 19:56:30.549164 1817963 pool.go:146] connpool - found the pool
I0401 19:56:30.549170 1817963 pool.go:153] connpool - calling close on the pool
I0401 19:56:30.549176 1817963 pool.go:155] connpool - acquiring lock
I0401 19:56:30.549178 1817963 pool.go:157] connpool - acquired lock
I0401 19:56:30.549181 1817963 pool.go:160] connpool - closing dbaPool
I0401 19:56:30.549185 1817963 pool.go:162] connpool - finished execution of Close
I0401 19:56:30.549188 1817963 throttler.go:284] Throttler - finished execution of Close
I0401 19:56:30.549191 1817963 state_manager.go:509] Finished lag throttler close
I0401 19:56:30.549194 1817963 state_manager.go:510] Started messager close
I0401 19:56:30.549198 1817963 engine.go:90] messager Engine - started execution of Close. Acquiring mu lock
E0401 19:56:50.231865 1817963 message_manager.go:818] Unable to delete messages: Code: CLUSTER_EVENT
operation not allowed in state SHUTTING_DOWN
E0401 19:56:50.627034 1817963 message_manager.go:818] Unable to delete messages: Code: CLUSTER_EVENT
operation not allowed in state SHUTTING_DOWN
E0401 19:56:50.231889 1817963 message_manager.go:818] Unable to delete messages: Code: CLUSTER_EVENT
operation not allowed in state SHUTTING_DOWN
...
message manager continues forever, no other logs
Raw
mutex
--- mutex:
cycles/second=2250006157
sampling period=1
808066 3 @ 0x473349 0x11fc2a2 0x1211fce 0x136c430 0x11fb5e9 0x1211f51 0x469301
# 0x473348 sync.(*Mutex).Unlock+0x28 sync/mutex.go:190
# 0x11fc2a1 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Send+0x241 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:84
# 0x1211fcd vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1.1+0x2d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:149
# 0x136c42f vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0xef vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:657
# 0x11fb5e8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x1211f50 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
696398 20 @ 0x7a6245 0x7a621b 0x7a5cb0 0x7a79be 0x125c349 0x469301
# 0x7a6244 sync.(*Mutex).Unlock+0x524 sync/mutex.go:190
# 0x7a621a github.com/golang/glog.(*loggingT).output+0x4fa github.com/golang/[email protected]/glog.go:735
# 0x7a5caf github.com/golang/glog.(*loggingT).printf+0x10f github.com/golang/[email protected]/glog.go:655
# 0x7a79bd github.com/golang/glog.Errorf+0x3d github.com/golang/[email protected]/glog.go:1121
# 0x125c348 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge.func1+0x268 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:818
402099 27 @ 0x473349 0x47157e 0x11fc4be 0x1212d9a 0x1211e32 0x469301
# 0x473348 sync.(*Mutex).Unlock+0x28 sync/mutex.go:190
# 0x47157d sync.(*Cond).Wait+0x7d sync/cond.go:55
# 0x11fc4bd vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x13d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98
# 0x1212d99 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x219 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:346
# 0x1211e31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
322514 17 @ 0x473349 0x11fc5c2 0x1212d9a 0x1211e32 0x469301
# 0x473348 sync.(*Mutex).Unlock+0x28 sync/mutex.go:190
# 0x11fc5c1 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x241 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:108
# 0x1212d99 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x219 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:346
# 0x1211e31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
312389 3 @ 0x9c3068 0x9c3045 0x9b22d4 0x9b22b4 0x9b5ded 0x469301
# 0x9c3067 sync.(*Mutex).Unlock+0x127 sync/mutex.go:190
# 0x9c3044 google.golang.org/grpc/internal/transport.(*recvBuffer).put+0x104 google.golang.org/[email protected]/internal/transport/transport.go:110
# 0x9b22d3 google.golang.org/grpc/internal/transport.(*Stream).write+0x373 google.golang.org/[email protected]/internal/transport/transport.go:457
# 0x9b22b3 google.golang.org/grpc/internal/transport.(*http2Client).handleData+0x353 google.golang.org/[email protected]/internal/transport/http2_client.go:1053
# 0x9b5dec google.golang.org/grpc/internal/transport.(*http2Client).reader+0x8ac google.golang.org/[email protected]/internal/transport/http2_client.go:1486
254317 9 @ 0x473349 0x127b61b 0x1289ab4 0x1289988 0x125c259 0x469301
# 0x473348 sync.(*Mutex).Unlock+0x28 sync/mutex.go:190
# 0x127b61a vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).StartRequest+0x17a vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:359
# 0x1289ab3 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML+0x73 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1031
# 0x1289987 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages+0x67 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1024
# 0x125c258 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge.func1+0x178 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:815
6683 1 @ 0x473349 0x127e787 0x12816ac 0x4c3b25 0x4c30a5 0x949cfd 0x949a1b 0x4c3b25 0x4c30a5 0x949cfd 0x9453df 0x943f8c 0x942a0a 0x942585 0x9422dd 0x940ead 0x94111d 0x9407d8 0x95d7aa 0x95d78e 0xb74a45 0x6b060f 0x6b1f09 0x6b3c3b 0x6af6e8 0x469301
# 0x473348 sync.(*Mutex).Unlock+0x28 sync/mutex.go:190
# 0x127e786 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).AppendDetails+0x886 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:751
# 0x12816ab vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).AddStatusPart.func1+0xcb vitess.io/vitess/go/vt/vttablet/tabletserver/status.go:245
# 0x4c3b24 reflect.Value.call+0x844 reflect/value.go:556
# 0x4c30a4 reflect.Value.Call+0xc4 reflect/value.go:339
# 0x949cfc text/template.safeCall+0x9c text/template/funcs.go:368
# 0x949a1a text/template.call+0x3da text/template/funcs.go:353
# 0x4c3b24 reflect.Value.call+0x844 reflect/value.go:556
# 0x4c30a4 reflect.Value.Call+0xc4 reflect/value.go:339
# 0x949cfc text/template.safeCall+0x9c text/template/funcs.go:368
# 0x9453de text/template.(*state).evalCall+0x81e text/template/exec.go:724
# 0x943f8b text/template.(*state).evalFunction+0x1eb text/template/exec.go:579
# 0x942a09 text/template.(*state).evalCommand+0xe9 text/template/exec.go:466
# 0x942584 text/template.(*state).evalPipeline+0x144 text/template/exec.go:435
# 0x9422dc text/template.(*state).walkTemplate+0x17c text/template/exec.go:411
# 0x940eac text/template.(*state).walk+0x10c text/template/exec.go:268
# 0x94111c text/template.(*state).walk+0x37c text/template/exec.go:263
# 0x9407d7 text/template.(*Template).execute+0x2d7 text/template/exec.go:217
# 0x95d7a9 text/template.(*Template).Execute+0x69 text/template/exec.go:200
# 0x95d78d html/template.(*Template).ExecuteTemplate+0x4d html/template/template.go:139
# 0xb74a44 vitess.io/vitess/go/vt/servenv.(*statusPage).statusHandler+0x244 vitess.io/vitess/go/vt/servenv/status.go:223
# 0x6b060e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2047
# 0x6b1f08 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2425
# 0x6b3c3a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2879
# 0x6af6e7 net/http.(*conn).serve+0xb07 net/http/server.go:1930
Raw
show processlist
mysql> show processlist;
+--------+-------------+-----------------+----------+------------------+------+-----------------------------------------------------------------+------------------+---------+-----------+---------------+
| Id | User | Host | db | Command | Time | State | Info | Time_ms | Rows_sent | Rows_examined |
+--------+-------------+-----------------+----------+------------------+------+-----------------------------------------------------------------+------------------+---------+-----------+---------------+
| 22 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838930 | 0 | 0 |
| 33 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840617 | 0 | 0 |
| 34 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839160 | 0 | 0 |
| 36 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840876 | 0 | 0 |
| 37 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838047 | 0 | 0 |
| 38 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839550 | 0 | 0 |
| 39 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840601 | 0 | 0 |
| 40 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839301 | 0 | 0 |
| 41 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840819 | 0 | 0 |
| 42 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840845 | 0 | 0 |
| 43 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838843 | 0 | 0 |
| 44 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839157 | 0 | 0 |
| 45 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838050 | 0 | 0 |
| 46 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838935 | 0 | 0 |
| 47 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840607 | 0 | 0 |
| 48 | vt_app | localhost | keywords | Sleep | 824 | | NULL | 824685 | 1 | 1 |
| 49 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840818 | 0 | 0 |
| 50 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840621 | 0 | 0 |
| 51 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840823 | 0 | 0 |
| 53 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840598 | 0 | 0 |
| 55 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840451 | 0 | 0 |
| 56 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840146 | 0 | 0 |
| 57 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838782 | 0 | 0 |
| 58 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840748 | 0 | 0 |
| 59 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838839 | 0 | 0 |
| 60 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838501 | 0 | 0 |
| 61 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838106 | 0 | 0 |
| 62 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839320 | 0 | 0 |
| 63 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840492 | 0 | 0 |
| 64 | vt_app | localhost | keywords | Sleep | 834 | | NULL | 834685 | 1 | 1 |
| 65 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839305 | 0 | 0 |
| 66 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840872 | 0 | 0 |
| 67 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838507 | 0 | 0 |
| 68 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839315 | 0 | 0 |
| 69 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840455 | 0 | 0 |
| 70 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839734 | 0 | 0 |
| 72 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839553 | 0 | 0 |
| 73 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840976 | 0 | 0 |
| 74 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840611 | 0 | 0 |
| 75 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840141 | 0 | 0 |
| 76 | vt_app | localhost | keywords | Sleep | 831 | | NULL | 831412 | 0 | 0 |
| 77 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839473 | 0 | 0 |
| 78 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840503 | 0 | 0 |
| 79 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838778 | 0 | 0 |
| 80 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839476 | 0 | 0 |
| 81 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840628 | 0 | 0 |
| 82 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839731 | 0 | 0 |
| 83 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840745 | 0 | 0 |
| 84 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838110 | 0 | 0 |
| 85 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840624 | 0 | 0 |
| 330070 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840562 | 0 | 0 |
| 330071 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841501 | 0 | 0 |
| 330073 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832532 | 0 | 0 |
| 330075 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841556 | 0 | 0 |
| 330077 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840401 | 0 | 0 |
| 330078 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839256 | 0 | 0 |
| 330083 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839713 | 0 | 0 |
| 330084 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832532 | 0 | 0 |
| 330085 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840404 | 0 | 0 |
| 330095 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841278 | 0 | 0 |
| 330110 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841258 | 0 | 0 |
| 330111 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838800 | 0 | 0 |
| 330112 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838735 | 0 | 0 |
| 330113 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832416 | 0 | 0 |
| 330114 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839291 | 0 | 0 |
| 330115 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839114 | 0 | 0 |
| 330116 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840588 | 0 | 0 |
| 330118 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838457 | 0 | 0 |
| 330119 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839534 | 0 | 0 |
| 330120 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840764 | 0 | 0 |
| 330122 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840437 | 0 | 0 |
| 330123 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840428 | 0 | 0 |
| 330124 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832562 | 0 | 0 |
| 330154 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838889 | 0 | 0 |
| 330158 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840727 | 0 | 0 |
| 330159 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840770 | 0 | 0 |
| 330161 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841299 | 0 | 0 |
| 330162 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839515 | 0 | 0 |
| 330163 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840575 | 0 | 0 |
| 330164 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838064 | 0 | 0 |
| 330165 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832532 | 0 | 0 |
| 330166 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840848 | 0 | 0 |
| 330168 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838821 | 0 | 0 |
| 330169 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838754 | 0 | 0 |
| 330170 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832533 | 0 | 0 |
| 330171 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839267 | 0 | 0 |
| 330174 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832393 | 0 | 0 |
| 330175 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841316 | 0 | 0 |
| 330177 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841499 | 0 | 0 |
| 330178 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840560 | 0 | 0 |
| 330179 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838062 | 0 | 0 |
| 330180 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840583 | 0 | 0 |
| 330181 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838006 | 0 | 0 |
| 330182 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839281 | 0 | 0 |
| 330183 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839458 | 0 | 0 |
| 330184 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840560 | 0 | 0 |
| 330185 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840802 | 0 | 0 |
| 330186 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840124 | 0 | 0 |
| 330187 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838031 | 0 | 0 |
| 330189 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840803 | 0 | 0 |
| 330209 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840472 | 0 | 0 |
| 330211 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840434 | 0 | 0 |
| 330216 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832393 | 0 | 0 |
| 330217 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840598 | 0 | 0 |
| 330218 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832532 | 0 | 0 |
| 330219 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840822 | 0 | 0 |
| 330220 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840937 | 0 | 0 |
| 330221 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841296 | 0 | 0 |
| 330223 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839695 | 0 | 0 |
| 330225 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840702 | 0 | 0 |
| 330228 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840586 | 0 | 0 |
| 330229 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840098 | 0 | 0 |
| 330230 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841524 | 0 | 0 |
| 330231 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832562 | 0 | 0 |
| 330260 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832435 | 0 | 0 |
| 330262 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840096 | 0 | 0 |
| 330263 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839439 | 0 | 0 |
| 330264 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839264 | 0 | 0 |
| 330267 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832416 | 0 | 0 |
| 330269 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840779 | 0 | 0 |
| 330270 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840608 | 0 | 0 |
| 330272 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840820 | 0 | 0 |
| 330273 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839693 | 0 | 0 |
| 550935 | vt_filtered | localhost | keywords | Sleep | 0 | | NULL | 639 | 0 | 1 |
| 588030 | vt_filtered | localhost | keywords | Sleep | 0 | | NULL | 637 | 0 | 1 |
| 588031 | vt_filtered | localhost | keywords | Sleep | 0 | | NULL | 637 | 0 | 1 |
| 600931 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838909 | 0 | 0 |
| 600932 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840958 | 0 | 0 |
| 600933 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838485 | 0 | 0 |
| 600935 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832416 | 0 | 0 |
| 600936 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832532 | 0 | 0 |
| 600937 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838802 | 0 | 0 |
| 600938 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839514 | 0 | 0 |
| 600939 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840700 | 0 | 0 |
| 600940 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 838891 | 0 | 0 |
| 600941 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840582 | 0 | 0 |
| 600942 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839116 | 0 | 0 |
| 600944 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841257 | 0 | 0 |
| 600945 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840562 | 0 | 0 |
| 600954 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838083 | 0 | 0 |
| 600963 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832532 | 0 | 0 |
| 600964 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840763 | 0 | 0 |
| 600965 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832562 | 0 | 0 |
| 600966 | vt_app | localhost | keywords | Sleep | 832 | | NULL | 832532 | 0 | 0 |
| 600972 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838008 | 0 | 0 |
| 600973 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839258 | 0 | 0 |
| 600974 | vt_app | localhost | keywords | Sleep | 840 | | NULL | 840577 | 0 | 0 |
| 600975 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838734 | 0 | 0 |
| 600976 | vt_app | localhost | keywords | Sleep | 838 | | NULL | 838459 | 0 | 0 |
| 600977 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839437 | 0 | 0 |
| 600978 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 841558 | 0 | 0 |
| 607117 | vt_app | localhost | keywords | Sleep | 839 | | NULL | 839138 | 0 | 0 |
| 611857 | vt_app | localhost | keywords | Sleep | 841 | | NULL | 840935 | 0 | 0 |
| 614928 | vt_dba | localhost | keywords | Sleep | 2295 | | NULL | 2295419 | 46 | 666 |
| 616134 | vt_dba | localhost | keywords | Sleep | 317 | | NULL | 317483 | 0 | 0 |
| 616135 | vt_dba | localhost | keywords | Sleep | 317 | | NULL | 317482 | 0 | 0 |
| 616175 | vt_dba | localhost | keywords | Sleep | 177 | | NULL | 177374 | 0 | 0 |
| 616194 | vt_repl | 10.48.1.7:36394 | NULL | Binlog Dump GTID | 98 | Source has sent all binlog to replica; waiting for more updates | NULL | 97924 | 0 | 0 |
| 616210 | vt_dba | localhost | NULL | Sleep | 38 | | NULL | 38082 | 0 | 0 |
| 616211 | vt_dba | localhost | NULL | Sleep | 38 | | NULL | 38081 | 0 | 0 |
| 616212 | vt_dba | localhost | NULL | Sleep | 38 | | NULL | 38081 | 0 | 0 |
| 616214 | vt_dba | localhost | NULL | Sleep | 26 | | NULL | 26080 | 0 | 0 |
| 616215 | vt_dba | localhost | NULL | Sleep | 26 | | NULL | 26079 | 0 | 0 |
| 616217 | root | localhost | NULL | Query | 0 | init | show processlist | 0 | 0 | 0 |
| 616219 | vt_dba | localhost | NULL | Sleep | 14 | | NULL | 14078 | 0 | 0 |
| 616220 | vt_dba | localhost | NULL | Sleep | 14 | | NULL | 14077 | 0 | 0 |
| 616221 | vt_dba | localhost | NULL | Sleep | 14 | | NULL | 14077 | 0 | 0 |
| 616222 | vt_dba | localhost | NULL | Sleep | 14 | | NULL | 14077 | 0 | 0 |
| 616223 | vt_dba | localhost | NULL | Sleep | 14 | | NULL | 14076 | 0 | 0 |
+--------+-------------+-----------------+----------+------------------+------+-----------------------------------------------------------------+------------------+---------+-----------+---------------+
169 rows in set (0.00 sec)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment