Skip to content

bizarre panic on daemon #768

Closed
Closed

Description

@whyrusleeping

Im not really sure what happened, but heres a stack dump:

whyrusleeping@alarm ~> ipfs daemon
Initializing daemon...
API server listening on /ip4/127.0.0.1/tcp/5001
runtime: g19800: leftover defer sp=0x30312e31 pc=0x2f30312e
    defer 0x13d2f040 sp=0x30312e31 pc=0x2f30312e
    defer 0x137a3733 sp=0xd4611010 pc=0x70692f13
fatal error: unexpected signal during runtime execution
[signal 0xb code=0x1 addr=0x33312e39 pc=0x5c598]

runtime stack:
runtime.throw(0x721d90, 0x2a)
    /home/whyrusleeping/go/src/runtime/panic.go:508 +0x8c
runtime.sigpanic()
    /home/whyrusleeping/go/src/runtime/sigpanic_unix.go:12 +0x58
runtime.gentraceback(0x43774, 0x11c18aa8, 0x0, 0x11e6df40, 0x0, 0x0, 0x7fffffff, 0x7adb1c, 0x0, 0x0, ...)
    /home/whyrusleeping/go/src/runtime/traceback.go:412 +0x5ac
runtime.scanstack(0x11e6df40)
    /home/whyrusleeping/go/src/runtime/mgc.go:1019 +0x234
runtime.gcphasework(0x11e6df40)
    /home/whyrusleeping/go/src/runtime/mgc.go:1161 +0xb8
runtime.stopg(0x11e6df40, 0x4)
    /home/whyrusleeping/go/src/runtime/proc1.go:447 +0xb4
runtime.markroot(0x10c10000, 0x1a8)
    /home/whyrusleeping/go/src/runtime/mgc.go:734 +0x178
runtime.parfordo(0x10c10000)
    /home/whyrusleeping/go/src/runtime/parfor.go:69 +0x170
runtime.gcscan_m()
    /home/whyrusleeping/go/src/runtime/mgc.go:1910 +0x154
runtime.systemstack(0x10c12500)
    /home/whyrusleeping/go/src/runtime/asm_arm.s:246 +0x80
runtime.mstart()
    /home/whyrusleeping/go/src/runtime/proc1.go:714

goroutine 3 [garbage collection scan]:
runtime.systemstack_switch()
    /home/whyrusleeping/go/src/runtime/asm_arm.s:194 +0x4 fp=0x10c18f84 sp=0x10c18f80
runtime.gcwork(0x0)
    /home/whyrusleeping/go/src/runtime/malloc.go:539 +0x174 fp=0x10c18fdc sp=0x10c18f84
runtime.backgroundgc()
    /home/whyrusleeping/go/src/runtime/mgc0.go:87 +0x5c fp=0x10c18fec sp=0x10c18fdc
runtime.goexit()
    /home/whyrusleeping/go/src/runtime/asm_arm.s:1364 +0x4 fp=0x10c18fec sp=0x10c18fec
created by runtime.gogc
    /home/whyrusleeping/go/src/runtime/malloc.go:499 +0xe4

goroutine 1 [select, 36 minutes]:
github.com/jbenet/go-ipfs/core/corehttp.listenAndServe(0x10c46990, 0xb6d8cfd0, 0x11135bd0, 0x111332a0, 0x0, 0x0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/core/corehttp/corehttp.go:59 +0x580
github.com/jbenet/go-ipfs/core/corehttp.ListenAndServe(0x10c46990, 0x11133260, 0x17, 0x11641d18, 0x3, 0x3, 0x0, 0x0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/core/corehttp/corehttp.go:38 +0x1bc
main.daemonFunc(0xb6d8d698, 0x10c5a0c0, 0xb6d8db80, 0x10cec5c0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/cmd/ipfs/daemon.go:135 +0x121c
github.com/jbenet/go-ipfs/commands.(*Command).Call(0x9be480, 0xb6d8d698, 0x10c5a0c0, 0x0, 0x0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/commands/command.go:98 +0x424
main.callCommand(0xb6d8d670, 0x10cfa580, 0xb6d8d698, 0x10c5a0c0, 0x9be480, 0x9b9548, 0x0, 0x0, 0x0, 0x0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/cmd/ipfs/main.go:325 +0x820
main.(*cmdInvocation).Run(0x10cfa5a0, 0xb6d8d670, 0x10cfa580, 0x0, 0x0, 0x0, 0x0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/cmd/ipfs/main.go:164 +0x1dc
main.main()
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/cmd/ipfs/main.go:135 +0x578

goroutine 5 [syscall, 36 minutes]:
os/signal.loop()
    /home/whyrusleeping/go/src/os/signal/signal_unix.go:21 +0x1c
created by os/signal.init·1
    /home/whyrusleeping/go/src/os/signal/signal_unix.go:27 +0x38

goroutine 10 [chan receive, 36 minutes]:
main.func·014()
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/cmd/ipfs/main.go:479 +0x50
created by main.(*cmdInvocation).setupInterruptHandler
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/cmd/ipfs/main.go:503 +0x130

goroutine 11 [select]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0x10c54370)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:205 +0x1d4
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:236 +0x234

goroutine 12 [select, 30 minutes]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0x10c3c0e0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go:120 +0x3bc
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.openDB
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go:122 +0x824

goroutine 13 [select]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0x10c3c0e0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go:73 +0x124
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.openDB
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go:122 +0x840

goroutine 14 [select, 30 minutes]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0x10c3c0e0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go:812 +0x6a8
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.openDB
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go:122 +0x88c

goroutine 15 [select, 35 minutes]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0x10c3c0e0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go:759 +0x248
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.openDB
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go:122 +0x8a8

goroutine 16 [select, 35 minutes]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.(*DB).jWriter(0x10c3c0e0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go:37 +0x170
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb.openDB
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go:122 +0x8c4

goroutine 33 [chan receive, 36 minutes]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup.(*contextGroup).closeOnContextDone(0x10ce65c0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup/ctxgroup.go:218 +0x74
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup.newContextGroup
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup/ctxgroup.go:146 +0x1ac

goroutine 34 [select]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream.(*Swarm).connGarbageCollect(0x10c92dc0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/swarm.go:356 +0x1bc
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream.NewSwarm
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream/swarm.go:61 +0x2c0

goroutine 35 [chan receive, 36 minutes]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup.(*contextGroup).closeOnContextDone(0x116900c0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup/ctxgroup.go:218 +0x74
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup.newContextGroup
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup/ctxgroup.go:146 +0x1ac

goroutine 192 [select]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/periodic.func·001(0xb6d8c838, 0x11691ac0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/periodic/periodic.go:51 +0x20c
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess.func·003()
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/goprocess.go:182 +0x8c
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess.Go
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/goprocess.go:185 +0x3f0

goroutine 37 [chan receive, 36 minutes]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup.(*contextGroup).closeOnContextDone(0x116905c0)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup/ctxgroup.go:218 +0x74
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup.newContextGroup
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup/ctxgroup.go:146 +0x1ac

goroutine 38 [chan receive, 36 minutes]:
github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup.(*contextGroup).closeOnContextDone(0x11690740)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup/ctxgroup.go:218 +0x74
created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup.newContextGroup
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup/ctxgroup.go:146 +0x1ac

goroutine 39 [select]:
github.com/jbenet/go-ipfs/routing/dht.(*ProviderManager).run(0x10c95470)
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/routing/dht/providers.go:55 +0xbb4
created by github.com/jbenet/go-ipfs/routing/dht.NewProviderManager
    /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/routing/dht/providers.go:44 +0x260

There were more traces but they didnt seem all that relevant

Activity

  1. whyrusleeping commented on Feb 13, 2015

    @whyrusleeping
    MemberAuthor

    So, after @jbenet showed me a nice stack overflow question, this appears to be a map operation happening in a thread-unsafe manner.

  2. whyrusleeping commented on Feb 13, 2015

    @whyrusleeping
    MemberAuthor

    We should run the race detector some more, it might catch the issue

  3. jbenet commented on Feb 13, 2015

    @jbenet
    Member

    It may be golang tip

    —
    Sent from Mailbox

    On Fri, Feb 13, 2015 at 11:14 AM, Jeromy Johnson notifications@github.com
    wrote:

    We should run the race detector some more, it might catch the issue

    Reply to this email directly or view it on GitHub:
    #768 (comment)

  4. whyrusleeping commented on Feb 17, 2015

    @whyrusleeping
    MemberAuthor

    Got another weird one:

    whyrusleeping@alarm ~> ipfs daemon
    Initializing daemon...
    API server listening on /ip4/127.0.0.1/tcp/5001
    Gateway server listening on /ip4/127.0.0.1/tcp/8080
    fatal error: unexpected signal during runtime execution
    [signal 0xb code=0x2 addr=0x2c25c41a pc=0x51738]
    
    goroutine 825 [running]:
    runtime.throw(0x72b5e0, 0x2a)
        /home/whyrusleeping/go/src/runtime/panic.go:508 +0x8c fp=0x10f01e7c sp=0x10f01e70
    runtime.sigpanic()
        /home/whyrusleeping/go/src/runtime/sigpanic_unix.go:12 +0x58 fp=0x10f01ea4 sp=0x10f01e7c
    runtime.selectgoImpl(0x10f01f94, 0x0, 0xc)
        /home/whyrusleeping/go/src/runtime/select.go:565 +0x1094 fp=0x10f01f64 sp=0x10f01ea8
    runtime.selectgo(0x10f01f94)
        /home/whyrusleeping/go/src/runtime/select.go:183 +0x10 fp=0x10f01f74 sp=0x10f01f64
    github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/yamux.(*Session).keepalive(0x115bc540)
        /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/yamux/session.go:246 +0xe8 fp=0x10f01fe4 sp=0x10f01f74
    runtime.goexit()
        /home/whyrusleeping/go/src/runtime/asm_arm.s:1364 +0x4 fp=0x10f01fe4 sp=0x10f01fe4
    created by github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/yamux.newSession
        /home/whyrusleeping/gopkg/src/github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/yamux/session.go:96 +0x44c
    
    
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    kind/bugA bug in existing code (including security flaws)

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions