Skip to content

Anchore pod running in Kubernetes keeps crashing #33

Open
Open
Anchore pod running in Kubernetes keeps crashing#33
@Entropy70

Description

@Entropy70
Entropy70
opened on Nov 19, 2021

When I check the logs of the anchore service pod I get a fatal error, pasted below.

fatal error: concurrent map read and map write

goroutine 606433 [running]:
runtime.throw(0x8e5c35, 0x21)
        /usr/local/go/src/runtime/panic.go:774 +0x72 fp=0xc00067b178 sp=0xc00067b148 pc=0x42d4c2
runtime.mapaccess2(0x853280, 0xc00008ecc0, 0xc000565090, 0xc000565090, 0xc0002eb4bc)
        /usr/local/go/src/runtime/map.go:470 +0x278 fp=0xc00067b1c0 sp=0xc00067b178 pc=0x40d968
reflect.mapaccess(0x853280, 0xc00008ecc0, 0xc000565090, 0x8df8d9)
        /usr/local/go/src/runtime/map.go:1319 +0x3f fp=0xc00067b1f8 sp=0xc00067b1c0 pc=0x40faff
reflect.Value.MapIndex(0x853280, 0xc000851308, 0x95, 0x83f000, 0xc000565090, 0x98, 0xc0006e82a0, 0x879f60, 0xcfd660)
        /usr/local/go/src/reflect/value.go:1176 +0x16d fp=0xc00067b270 sp=0xc00067b1f8 pc=0x48952d
encoding/json.mapEncoder.encode(0x900da8, 0xc0006e82a0, 0x853280, 0xc000851308, 0x95, 0x100)
        /usr/local/go/src/encoding/json/encode.go:706 +0x302 fp=0xc00067b3b8 sp=0xc00067b270 pc=0x4e2032
encoding/json.mapEncoder.encode-fm(0xc0006e82a0, 0x853280, 0xc000851308, 0x95, 0x100)
        /usr/local/go/src/encoding/json/encode.go:682 +0x64 fp=0xc00067b3f8 sp=0xc00067b3b8 pc=0x4eca14
encoding/json.structEncoder.encode(0xc0000c4fc0, 0x3, 0x4, 0xc000d56390, 0xc0006e82a0, 0x8898c0, 0xc0008512c0, 0x99, 0x880100)
        /usr/local/go/src/encoding/json/encode.go:664 +0x306 fp=0xc00067b498 sp=0xc00067b3f8 pc=0x4e1ae6
encoding/json.structEncoder.encode-fm(0xc0006e82a0, 0x8898c0, 0xc0008512c0, 0x99, 0x100)
        /usr/local/go/src/encoding/json/encode.go:635 +0x7f fp=0xc00067b4f0 sp=0xc00067b498 pc=0x4ec98f
encoding/json.(*encodeState).reflectValue(0xc0006e82a0, 0x8898c0, 0xc0008512c0, 0x99, 0x100)
        /usr/local/go/src/encoding/json/encode.go:337 +0x82 fp=0xc00067b528 sp=0xc00067b4f0 pc=0x4df322
encoding/json.(*encodeState).marshal(0xc0006e82a0, 0x8898c0, 0xc0008512c0, 0x860100, 0x0, 0x0)
        /usr/local/go/src/encoding/json/encode.go:309 +0x10b fp=0xc00067b5a8 sp=0xc00067b528 pc=0x4dedcb
encoding/json.(*Encoder).Encode(0xc00067b6a8, 0x8898c0, 0xc0008512c0, 0xa5f0c0, 0xc00026a2a0)
        /usr/local/go/src/encoding/json/stream.go:202 +0x8a fp=0xc00067b638 sp=0xc00067b5a8 pc=0x4eb57a
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.SendJsonResponse(0xc00067b858, 0x8898c0, 0xc0008512c0, 0x8efe84, 0x3a)
        /home/circleci/project/pkg/http/api/v1/handler.go:264 +0x212 fp=0xc00067b708 sp=0xc00067b638 pc=0x7fc472
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).GetMetadata(0xc0001ca000, 0xa65440, 0xc00026a2a0, 0xc00021da00)
        /home/circleci/project/pkg/http/api/v1/handler.go:258 +0x30e fp=0xc00067b850 sp=0xc00067b708 pc=0x7fc23e
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).GetMetadata-fm(0xa65440, 0xc00026a2a0, 0xc00021da00)
        /home/circleci/project/pkg/http/api/v1/handler.go:243 +0x48 fp=0xc00067b880 sp=0xc00067b850 pc=0x7fe288
net/http.HandlerFunc.ServeHTTP(0xc00008d6c0, 0xa65440, 0xc00026a2a0, 0xc00021da00)
        /usr/local/go/src/net/http/server.go:2007 +0x44 fp=0xc00067b8a8 sp=0xc00067b880 pc=0x671da4
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).LoggerMiddleware.func1(0xa65440, 0xc00026a2a0, 0xc00021da00)
        /home/circleci/project/pkg/http/api/v1/handler.go:85 +0x37b fp=0xc00067ba48 sp=0xc00067b8a8 pc=0x7fcf8b
net/http.HandlerFunc.ServeHTTP(0xc0008cb720, 0xa65440, 0xc00026a2a0, 0xc00021da00)
        /usr/local/go/src/net/http/server.go:2007 +0x44 fp=0xc00067ba70 sp=0xc00067ba48 pc=0x671da4
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).AuthenticationMiddleware.func1(0xa65440, 0xc00026a2a0, 0xc00021da00)
        /home/circleci/project/pkg/http/api/v1/handler.go:68 +0x6b fp=0xc00067bac0 sp=0xc00067ba70 pc=0x7fcb6b
net/http.HandlerFunc.ServeHTTP(0xc0008cb740, 0xa65440, 0xc00026a2a0, 0xc00021da00)
        /usr/local/go/src/net/http/server.go:2007 +0x44 fp=0xc00067bae8 sp=0xc00067bac0 pc=0x671da4
github.com/gorilla/mux.(*Router).ServeHTTP(0xc0001cc000, 0xa65440, 0xc00026a2a0, 0xc0002d2400)
        /go/pkg/mod/github.com/gorilla/mux@v1.7.3/mux.go:212 +0xe2 fp=0xc00067bc18 sp=0xc00067bae8 pc=0x7f6032
net/http.serverHandler.ServeHTTP(0xc0001d4000, 0xa65440, 0xc00026a2a0, 0xc0002d2400)
        /usr/local/go/src/net/http/server.go:2802 +0xa4 fp=0xc00067bc48 sp=0xc00067bc18 pc=0x674464
net/http.(*conn).serve(0xc000a3c500, 0xa66200, 0xc000d5e900)
        /usr/local/go/src/net/http/server.go:1890 +0x875 fp=0xc00067bfc8 sp=0xc00067bc48 pc=0x670dd5
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc00067bfd0 sp=0xc00067bfc8 pc=0x459b81
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f5ab46393d8, 0x72, 0x0)
        /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0000fc098, 0x72, 0x0, 0x0, 0x8d89dd)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0000fc080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc0000fc080, 0xc0005d5848, 0x6753f4, 0xc0001d40a0)
        /usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0000a3200, 0x6195fcaf, 0xc0005d5848, 0x4ade96)
        /usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0000a3200, 0xc0005d5898, 0x18, 0xc000000180, 0x674924)
        /usr/local/go/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0001d4000, 0xa651c0, 0xc0000a3200, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:2896 +0x280
net/http.(*Server).ListenAndServe(0xc0001d4000, 0xc0001d4000, 0xc000113c48)
        /usr/local/go/src/net/http/server.go:2825 +0xb7
net/http.ListenAndServe(...)
        /usr/local/go/src/net/http/server.go:3081
main.main()
        /home/circleci/project/cmd/anchore-adapter/main.go:56 +0x9ed

goroutine 628903 [select]:
net/http.(*persistConn).roundTrip(0xc0002c7320, 0xc000a2bd10, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:2431 +0x770
net/http.(*Transport).roundTrip(0xc00076c000, 0xc00021d800, 0x0, 0xc000abd328, 0x40000000040c378)
        /usr/local/go/src/net/http/transport.go:535 +0x9b6
net/http.(*Transport).RoundTrip(0xc00076c000, 0xc00021d800, 0xc00076c000, 0x0, 0x0)
        /usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc00021d800, 0xa5eec0, 0xc00076c000, 0x0, 0x0, 0x0, 0xc00000e020, 0x3, 0x1, 0x0)
        /usr/local/go/src/net/http/client.go:250 +0x443
net/http.(*Client).send(0xc0001c5e90, 0xc00021d800, 0x0, 0x0, 0x0, 0xc00000e020, 0x0, 0x1, 0x0)
        /usr/local/go/src/net/http/client.go:174 +0xfa
net/http.(*Client).do(0xc0001c5e90, 0xc00021d800, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/client.go:641 +0x3ce
net/http.(*Client).Do(...)
        /usr/local/go/src/net/http/client.go:509
github.com/parnurzeal/gorequest.(*SuperAgent).getResponseBytes(0xc0004651e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /go/pkg/mod/github.com/parnurzeal/gorequest@v0.2.15/gorequest.go:1065 +0x406
github.com/parnurzeal/gorequest.(*SuperAgent).EndBytes(0xc0004651e0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x7f5ab69fe400, 0xc000068380, 0x7f5ab69fe460, 0x0, ...)
        /go/pkg/mod/github.com/parnurzeal/gorequest@v0.2.15/gorequest.go:948 +0x40
github.com/anchore/harbor-scanner-adapter/pkg/adapter/anchore/client.sendRequest(0xc0004651e0, 0x0, 0x0, 0x0, 0x1, 0xc000aac3c0, 0x0, 0x1)
        /home/circleci/project/pkg/adapter/anchore/client/client.go:527 +0x25c
github.com/anchore/harbor-scanner-adapter/pkg/adapter/anchore/client.QueryVulnerabilityRecords(0xc0001c6060, 0xc0002c6fc0, 0x12, 0x12, 0xc000318140, 0x4, 0x4, 0x0, 0x0, 0x0, ...)
        /home/circleci/project/pkg/adapter/anchore/client/client.go:251 +0x79a
github.com/anchore/harbor-scanner-adapter/pkg/adapter/anchore/client.GetVulnerabilityDescriptions(0xc0001c6060, 0xc000abeb10, 0xd096c0, 0xc0005f8a80)
        /home/circleci/project/pkg/adapter/anchore/client/client.go:160 +0x37c
github.com/anchore/harbor-scanner-adapter/pkg/adapter/anchore.(*HarborScannerAdapter).GetHarborVulnerabilityReport(0xc0000b2060, 0xc0004a2e71, 0x74, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /home/circleci/project/pkg/adapter/anchore/adapter.go:271 +0x8c9
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).GetHarborVulnerabilityReport(...)
        /home/circleci/project/pkg/http/api/v1/handler.go:226
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).GetScanReport(0xc0001ca000, 0xa65440, 0xc0001d5c00, 0xc00038e800)
        /home/circleci/project/pkg/http/api/v1/handler.go:189 +0x236
net/http.HandlerFunc.ServeHTTP(0xc00008d810, 0xa65440, 0xc0001d5c00, 0xc00038e800)
        /usr/local/go/src/net/http/server.go:2007 +0x44
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).LoggerMiddleware.func1(0xa65440, 0xc0001d5c00, 0xc00038e800)
        /home/circleci/project/pkg/http/api/v1/handler.go:85 +0x37b
net/http.HandlerFunc.ServeHTTP(0xc000415dc0, 0xa65440, 0xc0001d5c00, 0xc00038e800)
        /usr/local/go/src/net/http/server.go:2007 +0x44
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).AuthenticationMiddleware.func1(0xa65440, 0xc0001d5c00, 0xc00038e800)
        /home/circleci/project/pkg/http/api/v1/handler.go:68 +0x6b
net/http.HandlerFunc.ServeHTTP(0xc000415e00, 0xa65440, 0xc0001d5c00, 0xc00038e800)
        /usr/local/go/src/net/http/server.go:2007 +0x44
github.com/gorilla/mux.(*Router).ServeHTTP(0xc0001cc000, 0xa65440, 0xc0001d5c00, 0xc00038e300)
        /go/pkg/mod/github.com/gorilla/mux@v1.7.3/mux.go:212 +0xe2
net/http.serverHandler.ServeHTTP(0xc0001d4000, 0xa65440, 0xc0001d5c00, 0xc00038e300)
        /usr/local/go/src/net/http/server.go:2802 +0xa4
net/http.(*conn).serve(0xc000a3cf00, 0xa66200, 0xc000b05ec0)
        /usr/local/go/src/net/http/server.go:1890 +0x875
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 602157 [runnable]:
fmt.(*pp).free(0xc00048c000)
        /usr/local/go/src/fmt/print.go:146 +0x102
fmt.Sprintf(0x8d7446, 0x2, 0xc000676dd8, 0x1, 0x1, 0x0, 0x4d2c88)
        /usr/local/go/src/fmt/print.go:221 +0xb4
github.com/sirupsen/logrus.(*TextFormatter).appendValue(0xc0000b6050, 0xc000ad66c0, 0x83f000, 0xc00008dd80)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/text_formatter.go:293 +0xe3
github.com/sirupsen/logrus.(*TextFormatter).appendKeyValue(0xc0000b6050, 0xc000ad66c0, 0x8d784c, 0x3, 0x83f000, 0xc00008dd80)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/text_formatter.go:281 +0x8f
github.com/sirupsen/logrus.(*TextFormatter).Format(0xc0000b6050, 0xc00027ec40, 0xc00027ec40, 0x0, 0x0, 0x8e, 0x4666d2)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/text_formatter.go:198 +0xa32
github.com/sirupsen/logrus.(*Entry).write(0xc00027ec40)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/entry.go:255 +0xa1
github.com/sirupsen/logrus.Entry.log(0xc0000c0000, 0xc000a3f440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/entry.go:231 +0x19d
github.com/sirupsen/logrus.(*Entry).Log(0xc0003e8000, 0xc000000004, 0xc000677698, 0x1, 0x1)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/entry.go:268 +0xeb
github.com/sirupsen/logrus.(*Logger).Log(0xc0000c0000, 0x4, 0xc000677698, 0x1, 0x1)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/logger.go:192 +0x7d
github.com/sirupsen/logrus.(*Logger).Info(...)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/logger.go:206
github.com/sirupsen/logrus.Info(...)
        /go/pkg/mod/github.com/sirupsen/logrus@v1.4.2/exported.go:109
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.SendJsonResponse(0xc000677858, 0x8898c0, 0xc00079c5a0, 0x8efe84, 0x3a)
        /home/circleci/project/pkg/http/api/v1/handler.go:262 +0x83
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).GetMetadata(0xc0001ca000, 0xa65440, 0xc0001d41c0, 0xc00038ca00)
        /home/circleci/project/pkg/http/api/v1/handler.go:258 +0x30e
net/http.HandlerFunc.ServeHTTP(0xc00008d6c0, 0xa65440, 0xc0001d41c0, 0xc00038ca00)
        /usr/local/go/src/net/http/server.go:2007 +0x44
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).LoggerMiddleware.func1(0xa65440, 0xc0001d41c0, 0xc00038ca00)
        /home/circleci/project/pkg/http/api/v1/handler.go:85 +0x37b
net/http.HandlerFunc.ServeHTTP(0xc000981340, 0xa65440, 0xc0001d41c0, 0xc00038ca00)
        /usr/local/go/src/net/http/server.go:2007 +0x44
github.com/anchore/harbor-scanner-adapter/pkg/http/api/v1.(*APIHandler).AuthenticationMiddleware.func1(0xa65440, 0xc0001d41c0, 0xc00038ca00)
        /home/circleci/project/pkg/http/api/v1/handler.go:68 +0x6b
net/http.HandlerFunc.ServeHTTP(0xc000981360, 0xa65440, 0xc0001d41c0, 0xc00038ca00)
        /usr/local/go/src/net/http/server.go:2007 +0x44
github.com/gorilla/mux.(*Router).ServeHTTP(0xc0001cc000, 0xa65440, 0xc0001d41c0, 0xc00038c800)
        /go/pkg/mod/github.com/gorilla/mux@v1.7.3/mux.go:212 +0xe2
net/http.serverHandler.ServeHTTP(0xc0001d4000, 0xa65440, 0xc0001d41c0, 0xc00038c800)
        /usr/local/go/src/net/http/server.go:2802 +0xa4
net/http.(*conn).serve(0xc00022caa0, 0xa66200, 0xc0003c5980)
        /usr/local/go/src/net/http/server.go:1890 +0x875
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:2928 +0x384

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions