Crash: "fatal error: concurrent map iteration and map write"

[Firehed]

This happened when submitting (or processing?) a large number of new jobs with a unique identifier on them.

 fatal error: concurrent map iteration and map write 
 goroutine 48 [running]: 
 runtime.throw(0x91b410, 0x26) 
 	/usr/local/go/src/runtime/panic.go:596 +0x95 fp=0xc420023de8 sp=0xc420023dc8 
 runtime.mapiternext(0xc420023f68) 
 	/usr/local/go/src/runtime/hashmap.go:737 +0x7ee fp=0xc420023e98 sp=0xc420023de8 
 github.com/appscode/g2/pkg/server.(*Server).WatchJobTimeout(0xc42014b220) 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:841 +0x13e fp=0xc420023fd8 sp=0xc420023e98 
 runtime.goexit() 
 	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc420023fe0 sp=0xc420023fd8 
 created by github.com/appscode/g2/pkg/server.(*Server).Start 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:153 +0x215 
 goroutine 1 [IO wait]: 
 net.runtime_pollWait(0x7fbe42e35660, 0x72, 0x0) 
 	/usr/local/go/src/runtime/netpoll.go:164 +0x59 
 net.(*pollDesc).wait(0xc420048068, 0x72, 0x0, 0xc42000a360) 
 	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x38 
 net.(*pollDesc).waitRead(0xc420048068, 0xffffffffffffffff, 0x0) 
 	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x34 
 net.(*netFD).accept(0xc420048000, 0x0, 0xb30180, 0xc42000a360) 
 	/usr/local/go/src/net/fd_unix.go:430 +0x1e5 
 net.(*TCPListener).accept(0xc4200ae020, 0xc42005fbd0, 0x7fa4b0, 0x455ab0) 
 	/usr/local/go/src/net/tcpsock_posix.go:136 +0x2e 
 net.(*TCPListener).Accept(0xc4200ae020, 0x926b20, 0xc42000a280, 0xc42014b220, 0xb386c0) 
 	/usr/local/go/src/net/tcpsock.go:228 +0x49 
 github.com/appscode/g2/pkg/server.(*Server).Start(0xc42014b220) 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:164 +0x24e 
 main.NewCmdRun.func1(0xc4201706c0, 0xb85770, 0x0, 0x0) 
 	/go/src/github.com/appscode/g2/cmd/gearmand/run.go:20 +0x8f 
 github.com/appscode/g2/vendor/github.com/spf13/cobra.(*Command).execute(0xc4201706c0, 0xb85770, 0x0, 0x0, 0xc4201706c0, 0xb85770) 
 	/go/src/github.com/appscode/g2/vendor/github.com/spf13/cobra/command.go:651 +0x23a 
 github.com/appscode/g2/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc420170240, 0xc420137f70, 0xc42012d000, 0xc4201294a0) 
 	/go/src/github.com/appscode/g2/vendor/github.com/spf13/cobra/command.go:726 +0x339 
 github.com/appscode/g2/vendor/github.com/spf13/cobra.(*Command).Execute(0xc420170240, 0xc420137f68, 0x1) 
 	/go/src/github.com/appscode/g2/vendor/github.com/spf13/cobra/command.go:685 +0x2b 
 main.main() 
 	/go/src/github.com/appscode/g2/cmd/gearmand/main.go:32 +0x177 
 goroutine 5 [chan receive, 2 minutes]: 
 github.com/appscode/g2/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0xb692c0) 
 	/go/src/github.com/appscode/g2/vendor/github.com/golang/glog/glog.go:879 +0x7a 
 created by github.com/appscode/g2/vendor/github.com/golang/glog.init.1 
 	/go/src/github.com/appscode/g2/vendor/github.com/golang/glog/glog.go:410 +0x21d 
 goroutine 35 [runnable]: 
 github.com/appscode/g2/pkg/server.genJid(0x8565c0, 0xc42013e4e0) 
 	/go/src/github.com/appscode/g2/pkg/server/util.go:78 +0x7d 
 github.com/appscode/g2/pkg/server.init.1.func1() 
 	/go/src/github.com/appscode/g2/pkg/server/util.go:71 +0x22 
 created by github.com/appscode/g2/pkg/server.init.1 
 	/go/src/github.com/appscode/g2/pkg/server/util.go:73 +0x1b5 
 goroutine 36 [select]: 
 github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc4201560e0) 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x23f 
 created by github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x19c 
 goroutine 41 [select, 2 minutes]: 
 github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc420192000) 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5d9 
 created by github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb.openDB 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x583 
 goroutine 42 [select, 2 minutes]: 
 github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc420192000) 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:101 +0x249 
 created by github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb.openDB 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5a5 
 goroutine 43 [select, 2 minutes]: 
 github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc420192000) 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x638 
 created by github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb.openDB 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x722 
 goroutine 44 [select, 2 minutes]: 
 github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc420192000) 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x242 
 created by github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb.openDB 
 	/go/src/github.com/appscode/g2/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x744 
 goroutine 45 [runnable]: 
 github.com/appscode/g2/pkg/server.(*Server).EvtLoop(0xc42014b220) 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:796 +0x139 
 created by github.com/appscode/g2/pkg/server.(*Server).Start 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:137 +0x1be 
 goroutine 46 [IO wait, 2 minutes]: 
 net.runtime_pollWait(0x7fbe42e355a0, 0x72, 0x0) 
 	/usr/local/go/src/runtime/netpoll.go:164 +0x59 
 net.(*pollDesc).wait(0xc42010c768, 0x72, 0x0, 0xc420190380) 
 	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x38 
 net.(*pollDesc).waitRead(0xc42010c768, 0xffffffffffffffff, 0x0) 
 	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x34 
 net.(*netFD).accept(0xc42010c700, 0x0, 0xb30180, 0xc420190380) 
 	/usr/local/go/src/net/fd_unix.go:430 +0x1e5 
 net.(*TCPListener).accept(0xc420138030, 0xc420128e40, 0xc42003ce08, 0x4e2bed) 
 	/usr/local/go/src/net/tcpsock_posix.go:136 +0x2e 
 net.(*TCPListener).AcceptTCP(0xc420138030, 0x6e9f83, 0xc42003ce20, 0xc42003ce18) 
 	/usr/local/go/src/net/tcpsock.go:215 +0x49 
 net/http.tcpKeepAliveListener.Accept(0xc420138030, 0xc420128e10, 0x879600, 0xb5a940, 0x8be000) 
 	/usr/local/go/src/net/http/server.go:3044 +0x2f 
 net/http.(*Server).Serve(0xc420146160, 0xb35b00, 0xc420138030, 0x0, 0x0) 
 	/usr/local/go/src/net/http/server.go:2643 +0x228 
 net/http.(*Server).ListenAndServe(0xc420146160, 0xc420146160, 0x2) 
 	/usr/local/go/src/net/http/server.go:2585 +0xb0 
 net/http.ListenAndServe(0x906c6d, 0x5, 0x0, 0x0, 0x0, 0x0) 
 	/usr/local/go/src/net/http/server.go:2787 +0x7f 
 github.com/appscode/g2/pkg/server.(*Server).Start.func1(0xc42014b220) 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:148 +0x1ee 
 created by github.com/appscode/g2/pkg/server.(*Server).Start 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:149 +0x33c 
 goroutine 47 [chan receive, 2 minutes]: 
 github.com/appscode/g2/pkg/server.(*Server).WatcherLoop(0xc42014b220) 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:809 +0x7c 
 created by github.com/appscode/g2/pkg/server.(*Server).Start 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:152 +0x1f0 
 goroutine 49 [select, 2 minutes]: 
 github.com/appscode/g2/vendor/gopkg.in/robfig/cron%2ev2.(*Cron).run(0xc42012e5a0) 
 	/go/src/github.com/appscode/g2/vendor/gopkg.in/robfig/cron.v2/cron.go:182 +0x96c 
 created by github.com/appscode/g2/vendor/gopkg.in/robfig/cron%2ev2.(*Cron).Start 
 	/go/src/github.com/appscode/g2/vendor/gopkg.in/robfig/cron.v2/cron.go:157 +0x43 
 goroutine 8 [runnable]: 
 github.com/appscode/g2/pkg/server.(*session).handleBinaryConnection(0xc42000a280, 0xc42014b220, 0xb386c0, 0xc42000c028, 0xc42014e1e0, 0x2, 0xc42014e120) 
 	/go/src/github.com/appscode/g2/pkg/server/session.go:131 +0xa77 
 github.com/appscode/g2/pkg/server.(*session).handleConnection(0xc42000a280, 0xc42014b220, 0xb386c0, 0xc42000c028) 
 	/go/src/github.com/appscode/g2/pkg/server/session.go:61 +0x3bd 
 created by github.com/appscode/g2/pkg/server.(*Server).Start 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:170 +0x2e0 
 goroutine 65 [IO wait, 2 minutes]: 
 net.runtime_pollWait(0x7fbe42e354e0, 0x72, 0xa) 
 	/usr/local/go/src/runtime/netpoll.go:164 +0x59 
 net.(*pollDesc).wait(0xc420110148, 0x72, 0xb31780, 0xb2d450) 
 	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x38 
 net.(*pollDesc).waitRead(0xc420110148, 0xc4205b0000, 0x40000) 
 	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x34 
 net.(*netFD).Read(0xc4201100e0, 0xc4205b0000, 0x40000, 0x40000, 0x0, 0xb31780, 0xb2d450) 
 	/usr/local/go/src/net/fd_unix.go:250 +0x1b7 
 net.(*conn).Read(0xc42000c008, 0xc4205b0000, 0x40000, 0x40000, 0x0, 0x0, 0x0) 
 	/usr/local/go/src/net/net.go:181 +0x70 
 bufio.(*Reader).fill(0xc42014e0c0) 
 	/usr/local/go/src/bufio/bufio.go:97 +0x117 
 bufio.(*Reader).ReadSlice(0xc42014e0c0, 0xc42003db0a, 0xc42003d7f8, 0xc420024400, 0x7fbe42ec9000, 0x0, 0x0) 
 	/usr/local/go/src/bufio/bufio.go:338 +0xbb 
 bufio.(*Reader).ReadBytes(0xc42014e0c0, 0xc42014e00a, 0xc42003db48, 0xc42012c5e0, 0x2, 0x8, 0x2) 
 	/usr/local/go/src/bufio/bufio.go:416 +0x66 
 github.com/appscode/g2/pkg/server.(*session).handleAdminConnection(0xc4200ee740, 0xc42014b220, 0xb386c0, 0xc42000c008, 0xc42014e0c0, 0x1, 0xc42014e000) 
 	/go/src/github.com/appscode/g2/pkg/server/session.go:183 +0x48 
 github.com/appscode/g2/pkg/server.(*session).handleConnection(0xc4200ee740, 0xc42014b220, 0xb386c0, 0xc42000c008) 
 	/go/src/github.com/appscode/g2/pkg/server/session.go:63 +0x429 
 created by github.com/appscode/g2/pkg/server.(*Server).Start 
 	/go/src/github.com/appscode/g2/pkg/server/server.go:170 +0x2e0 
 goroutine 66 [chan receive, 2 minutes]: 
 github.com/appscode/g2/pkg/server.queueingWriter(0xc42014e000, 0xc42014e060) 
 	/go/src/github.com/appscode/g2/pkg/server/client.go:29 +0x11f 
 created by github.com/appscode/g2/pkg/server.(*session).handleConnection 
 	/go/src/github.com/appscode/g2/pkg/server/session.go:48 +0x20a 
 goroutine 67 [chan receive, 2 minutes]: 
 github.com/appscode/g2/pkg/server.writer(0xb386c0, 0xc42000c008, 0xc42014e060) 
 	/go/src/github.com/appscode/g2/pkg/server/util.go:301 +0x11a 
 created by github.com/appscode/g2/pkg/server.(*session).handleConnection 
 	/go/src/github.com/appscode/g2/pkg/server/session.go:49 +0x246 
 goroutine 9 [chan receive]: 
 github.com/appscode/g2/pkg/server.queueingWriter(0xc42014e120, 0xc42014e180) 
 	/go/src/github.com/appscode/g2/pkg/server/client.go:29 +0x11f 
 created by github.com/appscode/g2/pkg/server.(*session).handleConnection 
 	/go/src/github.com/appscode/g2/pkg/server/session.go:48 +0x20a 
 goroutine 10 [chan receive]: 
 github.com/appscode/g2/pkg/server.writer(0xb386c0, 0xc42000c028, 0xc42014e180) 
 	/go/src/github.com/appscode/g2/pkg/server/util.go:301 +0x11a 
 created by github.com/appscode/g2/pkg/server.(*session).handleConnection 
 	/go/src/github.com/appscode/g2/pkg/server/session.go:49 +0x246