Created
July 26, 2023 11:30
-
-
Save ptecza/f47688cd357083ae350586bdf2e2e280 to your computer and use it in GitHub Desktop.
Debugging Docker daemon
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
ubuntu@my-swarm-node-2:~$ curl -s --unix-socket /var/run/docker.sock http://./debug/pprof/goroutine\?debug\=2 > docker.sock-debug.txt | |
ubuntu@my-swarm-node-2:~$ head -200 docker.sock-debug.txt | |
goroutine 2276198 [running]: | |
runtime/pprof.writeGoroutineStacks({0x560b9f8fa920, 0xc00c0001c0}) | |
/usr/local/go/src/runtime/pprof/pprof.go:703 +0x70 | |
runtime/pprof.writeGoroutine({0x560b9f8fa920?, 0xc00c0001c0?}, 0xc007a62000?) | |
/usr/local/go/src/runtime/pprof/pprof.go:692 +0x2b | |
runtime/pprof.(*Profile).WriteTo(0x560b9f7162c0?, {0x560b9f8fa920?, 0xc00c0001c0?}, 0xc?) | |
/usr/local/go/src/runtime/pprof/pprof.go:329 +0x14b | |
net/http/pprof.handler.ServeHTTP({0xc00072b9c1, 0x9}, {0x560b9f916940, 0xc00c0001c0}, 0x560b9f417940?) | |
/usr/local/go/src/net/http/pprof/pprof.go:259 +0x4a5 | |
github.com/docker/docker/api/server/router/debug.handlePprof({0x560b9f7162c0?, 0xc00b7f0db0?}, {0x560b9f916940, 0xc00c0001c0}, 0xc00e6535a0?, 0x560b9d2b6527?) | |
/go/src/github.com/docker/docker/api/server/router/debug/debug_routes.go:10 +0x59 | |
github.com/docker/docker/api/server/middleware.ExperimentalMiddleware.WrapHandler.func1({0x560b9f918180, 0xc00b7f0ed0}, {0x560b9f916940?, 0xc00c0001c0?}, 0x560b9f3ffa40?, 0xc00b6208e0?) | |
/go/src/github.com/docker/docker/api/server/middleware/experimental.go:26 +0x15b | |
github.com/docker/docker/api/server/middleware.VersionMiddleware.WrapHandler.func1({0x560b9f918180, 0xc00b7f0ea0}, {0x560b9f916940, 0xc00c0001c0}, 0xc00e6537e8?, 0xc00e6537a0?) | |
/go/src/github.com/docker/docker/api/server/middleware/version.go:62 +0x4d7 | |
github.com/docker/docker/pkg/authorization.(*Middleware).WrapHandler.func1({0x560b9f918180, 0xc00b7f0ea0}, {0x560b9f916940?, 0xc00c0001c0?}, 0xc00d810f00, 0xc00b620890?) | |
/go/src/github.com/docker/docker/pkg/authorization/middleware.go:59 +0x649 | |
github.com/docker/docker/api/server.(*Server).makeHTTPHandler.func1({0x560b9f916940, 0xc00c0001c0}, 0xc00d810700) | |
/go/src/github.com/docker/docker/api/server/server.go:53 +0x1ce | |
net/http.HandlerFunc.ServeHTTP(0xc00d810000?, {0x560b9f916940?, 0xc00c0001c0?}, 0xc00e6539e8?) | |
/usr/local/go/src/net/http/server.go:2122 +0x2f | |
github.com/docker/docker/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc008a980c0, {0x560b9f916940, 0xc00c0001c0}, 0xc00ad5d200) | |
/go/src/github.com/docker/docker/vendor/github.com/gorilla/mux/mux.go:210 +0x1cf | |
net/http.serverHandler.ServeHTTP({0xc00b7f0ab0?}, {0x560b9f916940, 0xc00c0001c0}, 0xc00ad5d200) | |
/usr/local/go/src/net/http/server.go:2936 +0x316 | |
net/http.(*conn).serve(0xc00b83fa70, {0x560b9f918180, 0xc007de6d20}) | |
/usr/local/go/src/net/http/server.go:1995 +0x612 | |
created by net/http.(*Server).Serve | |
/usr/local/go/src/net/http/server.go:3089 +0x5ed | |
goroutine 1 [semacquire, 634 minutes, locked to thread]: | |
sync.runtime_Semacquire(0xc003f695e0?) | |
/usr/local/go/src/runtime/sema.go:62 +0x27 | |
sync.(*WaitGroup).Wait(0xc000830d00?) | |
/usr/local/go/src/sync/waitgroup.go:116 +0x4b | |
main.(*DaemonCli).start(0xc000830d00, 0xc000f30540) | |
/go/src/github.com/docker/docker/cmd/dockerd/daemon.go:304 +0x174b | |
main.runDaemon(...) | |
/go/src/github.com/docker/docker/cmd/dockerd/docker_unix.go:14 | |
main.newDaemonCommand.func1(0xc000f36000?, {0xc000255920?, 0x3?, 0x3?}) | |
/go/src/github.com/docker/docker/cmd/dockerd/docker.go:39 +0x94 | |
github.com/docker/docker/vendor/github.com/spf13/cobra.(*Command).execute(0xc000f36000, {0xc00012c010, 0x3, 0x3}) | |
/go/src/github.com/docker/docker/vendor/github.com/spf13/cobra/command.go:916 +0x862 | |
github.com/docker/docker/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc000f36000) | |
/go/src/github.com/docker/docker/vendor/github.com/spf13/cobra/command.go:1044 +0x3bd | |
github.com/docker/docker/vendor/github.com/spf13/cobra.(*Command).Execute(...) | |
/go/src/github.com/docker/docker/vendor/github.com/spf13/cobra/command.go:968 | |
main.main() | |
/go/src/github.com/docker/docker/cmd/dockerd/docker.go:109 +0x18c | |
goroutine 210 [sync.Cond.Wait, 634 minutes]: | |
sync.runtime_notifyListWait(0xc00159a450, 0x0) | |
/usr/local/go/src/runtime/sema.go:527 +0x14c | |
sync.(*Cond).Wait(0x0?) | |
/usr/local/go/src/sync/cond.go:70 +0x8c | |
github.com/docker/docker/daemon/logger.(*messageRing).Dequeue(0xc00159a400) | |
/go/src/github.com/docker/docker/daemon/logger/ring.go:189 +0x4b | |
github.com/docker/docker/daemon/logger.(*RingLogger).run(0xc000136500) | |
/go/src/github.com/docker/docker/daemon/logger/ring.go:124 +0x85 | |
created by github.com/docker/docker/daemon/logger.newRingLogger | |
/go/src/github.com/docker/docker/daemon/logger/ring.go:45 +0x1f6 | |
goroutine 3810 [IO wait]: | |
internal/poll.runtime_pollWait(0x7f358ab0f918, 0x72) | |
/usr/local/go/src/runtime/netpoll.go:306 +0x89 | |
internal/poll.(*pollDesc).wait(0xc008d87bc0?, 0xc0041dc000?, 0x1) | |
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
internal/poll.(*pollDesc).waitRead(...) | |
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
internal/poll.(*FD).Read(0xc008d87bc0, {0xc0041dc000, 0x8000, 0x8000}) | |
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 | |
os.(*File).read(...) | |
/usr/local/go/src/os/file_posix.go:31 | |
os.(*File).Read(0xc0070de598, {0xc0041dc000?, 0x0?, 0x0?}) | |
/usr/local/go/src/os/file.go:118 +0x5e | |
github.com/docker/docker/vendor/github.com/containerd/fifo.(*fifo).Read(0xc0086238c0, {0xc0041dc000, 0x8000, 0x8000}) | |
/go/src/github.com/docker/docker/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe | |
io.copyBuffer({0x560b9f8f4660, 0xc008facc00}, {0x7f358044c0d8, 0xc0086238c0}, {0xc0041dc000, 0x8000, 0x8000}) | |
/usr/local/go/src/io/io.go:427 +0x1b2 | |
io.CopyBuffer({0x560b9f8f4660?, 0xc008facc00?}, {0x7f358044c0d8?, 0xc0086238c0?}, {0xc0041dc000?, 0x560b9d2addbd?, 0xc00154cf50?}) | |
/usr/local/go/src/io/io.go:400 +0x3c | |
github.com/docker/docker/pkg/pools.Copy({0x560b9f8f4660, 0xc008facc00}, {0x7f358044c0d8, 0xc0086238c0}) | |
/go/src/github.com/docker/docker/pkg/pools/pools.go:81 +0x7d | |
github.com/docker/docker/container/stream.(*Config).CopyToPipe.func1.1() | |
/go/src/github.com/docker/docker/container/stream/streams.go:123 +0x68 | |
created by github.com/docker/docker/container/stream.(*Config).CopyToPipe.func1 | |
/go/src/github.com/docker/docker/container/stream/streams.go:122 +0xd8 | |
goroutine 67 [select]: | |
github.com/docker/docker/vendor/go.opencensus.io/stats/view.(*worker).start(0xc000a88c00) | |
/go/src/github.com/docker/docker/vendor/go.opencensus.io/stats/view/worker.go:276 +0xad | |
created by github.com/docker/docker/vendor/go.opencensus.io/stats/view.init.0 | |
/go/src/github.com/docker/docker/vendor/go.opencensus.io/stats/view/worker.go:34 +0x96 | |
goroutine 87 [select, 634 minutes]: | |
io.(*pipe).read(0xc00012ad20, {0xc001003eec, 0xc114, 0x560ba0db1400?}) | |
/usr/local/go/src/io/pipe.go:57 +0xb1 | |
io.(*PipeReader).Read(0xc001003e99?, {0xc001003eec?, 0x0?, 0xc000279e18?}) | |
/usr/local/go/src/io/pipe.go:136 +0x25 | |
bufio.(*Scanner).Scan(0xc000279f28) | |
/usr/local/go/src/bufio/scan.go:214 +0x876 | |
github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).writerScanner(0xc00070f080?, 0xc000014958, 0xc000de5050) | |
/go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:86 +0x11f | |
created by github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).WriterLevel | |
/go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:57 +0x3d1 | |
goroutine 88 [select, 634 minutes]: | |
io.(*pipe).read(0xc00012ad80, {0xc001010618, 0xf9e8, 0xc00009ee00?}) | |
/usr/local/go/src/io/pipe.go:57 +0xb1 | |
io.(*PipeReader).Read(0xc00101059f?, {0xc001010618?, 0x6?, 0x100000000000008?}) | |
/usr/local/go/src/io/pipe.go:136 +0x25 | |
bufio.(*Scanner).Scan(0xc00009ef28) | |
/usr/local/go/src/bufio/scan.go:214 +0x876 | |
github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).writerScanner(0x4?, 0xc000014968, 0xc000de5060) | |
/go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:86 +0x11f | |
created by github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).WriterLevel | |
/go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:57 +0x3d1 | |
goroutine 89 [select, 634 minutes]: | |
io.(*pipe).read(0xc00012ade0, {0xc0004b2000, 0x10000, 0x560b9d2cbf65?}) | |
/usr/local/go/src/io/pipe.go:57 +0xb1 | |
io.(*PipeReader).Read(0x7f35b36e8428?, {0xc0004b2000?, 0x560b9d2b647d?, 0x0?}) | |
/usr/local/go/src/io/pipe.go:136 +0x25 | |
bufio.(*Scanner).Scan(0xc000275f28) | |
/usr/local/go/src/bufio/scan.go:214 +0x876 | |
github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).writerScanner(0x0?, 0xc000014978, 0xc000de5070) | |
/go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:86 +0x11f | |
created by github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).WriterLevel | |
/go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:57 +0x3d1 | |
goroutine 100 [syscall, 634 minutes]: | |
os/signal.signal_recv() | |
/usr/local/go/src/runtime/sigqueue.go:152 +0x2f | |
os/signal.loop() | |
/usr/local/go/src/os/signal/signal_unix.go:23 +0x19 | |
created by os/signal.Notify.func1.1 | |
/usr/local/go/src/os/signal/signal.go:151 +0x2a | |
goroutine 91 [chan receive, 634 minutes]: | |
github.com/docker/docker/cmd/dockerd/trap.Trap.func1() | |
/go/src/github.com/docker/docker/cmd/dockerd/trap/trap.go:31 +0x5c | |
created by github.com/docker/docker/cmd/dockerd/trap.Trap | |
/go/src/github.com/docker/docker/cmd/dockerd/trap/trap.go:29 +0xd9 | |
goroutine 92 [chan receive, 634 minutes]: | |
main.(*DaemonCli).start.func2() | |
/go/src/github.com/docker/docker/cmd/dockerd/daemon.go:198 +0x4b | |
created by main.(*DaemonCli).start | |
/go/src/github.com/docker/docker/cmd/dockerd/daemon.go:192 +0x91b | |
goroutine 74 [IO wait, 34 minutes]: | |
internal/poll.runtime_pollWait(0x7f358ab0fbe8, 0x72) | |
/usr/local/go/src/runtime/netpoll.go:306 +0x89 | |
internal/poll.(*pollDesc).wait(0xc000570000?, 0xc000266000?, 0x0) | |
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
internal/poll.(*pollDesc).waitRead(...) | |
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
internal/poll.(*FD).Read(0xc000570000, {0xc000266000, 0x8000, 0x8000}) | |
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 | |
net.(*netFD).Read(0xc000570000, {0xc000266000?, 0x60100000000?, 0x8?}) | |
/usr/local/go/src/net/fd_posix.go:55 +0x29 | |
net.(*conn).Read(0xc0004ce040, {0xc000266000?, 0x560b9d3194d0?, 0x560b9d2e6367?}) | |
/usr/local/go/src/net/net.go:183 +0x45 | |
bufio.(*Reader).Read(0xc00070efc0, {0xc000df6200, 0x9, 0x7f35b36dff18?}) | |
/usr/local/go/src/bufio/bufio.go:237 +0x1bb | |
io.ReadAtLeast({0x560b9f8f34e0, 0xc00070efc0}, {0xc000df6200, 0x9, 0x9}, 0x9) | |
/usr/local/go/src/io/io.go:332 +0x9a | |
io.ReadFull(...) | |
/usr/local/go/src/io/io.go:351 | |
github.com/docker/docker/vendor/golang.org/x/net/http2.readFrameHeader({0xc000df6200?, 0x9?, 0xc0000a1d5f?}, {0x560b9f8f34e0?, 0xc00070efc0?}) | |
/go/src/github.com/docker/docker/vendor/golang.org/x/net/http2/frame.go:237 +0x6e | |
github.com/docker/docker/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc000df61c0) | |
/go/src/github.com/docker/docker/vendor/golang.org/x/net/http2/frame.go:498 +0x95 | |
github.com/docker/docker/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000ad4000) | |
/go/src/github.com/docker/docker/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1516 +0x45e | |
created by github.com/docker/docker/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client | |
/go/src/github.com/docker/docker/vendor/google.golang.org/grpc/internal/transport/http2_client.go:369 +0x1b4f | |
goroutine 101 [chan receive, 634 minutes]: | |
github.com/docker/docker/daemon.(*Daemon).setupDumpStackTrap.func1() | |
/go/src/github.com/docker/docker/daemon/debugtrap_unix.go:19 +0x58 | |
created by github.com/docker/docker/daemon.(*Daemon).setupDumpStackTrap | |
/go/src/github.com/docker/docker/daemon/debugtrap_unix.go:18 +0xd5 | |
goroutine 115 [IO wait, 634 minutes]: | |
internal/poll.runtime_pollWait(0x7f358ab0fdc8, 0x72) | |
/usr/local/go/src/runtime/netpoll.go:306 +0x89 | |
internal/poll.(*pollDesc).wait(0xc000570c80?, 0xc000276d10?, 0x0) | |
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
internal/poll.(*pollDesc).waitRead(...) | |
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
internal/poll.(*FD).Accept(0xc000570c80) | |
/usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd | |
net.(*netFD).accept(0xc000570c80) | |
/usr/local/go/src/net/fd_unix.go:172 +0x35 | |
net.(*UnixListener).accept(0x560b9d3f8ae6?) | |
/usr/local/go/src/net/unixsock_posix.go:172 +0x1c | |
net.(*UnixListener).Accept(0xc00057d3e0) | |
/usr/local/go/src/net/unixsock.go:260 +0x3d | |
net/http.(*Server).Serve(0xc0004ca000, {0x560b9f916760, 0xc00057d3e0}) | |
ubuntu@my-swarm-node-2:~$ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment