wasm operator always restart

[xiaowujee]

operator version quay.io/solo-io/wasme:0.0.32
istio version 1.8.2
kubernetes version 1.18.3

filterDeployment:

apiVersion: wasme.io/v1
kind: FilterDeployment
metadata:
  name: bookinfo-custom-demo1
  namespace: bookinfo
spec:
  deployment:
    istio:
      kind: Deployment
      labels:
        app: details
        version: v1      
  filter:
    image: webassemblyhub.io/xiaowu/wasme-test:v0.20

operator error log

time="2021-03-11T02:07:49Z" level=info msg="image is already cached" cache="{wasme-cache wasme}" image="webassemblyhub.io/xiaowu/wasme-test:v0.20"

time="2021-03-11T02:07:49Z" level=info msg="updated workload sidecar annotations" filter="id:\"bookinfo-custom-demo1.bookinfo\" image:\"webassemblyhub.io/xiaowu/wasme-test:v0.20\" rootID:\"root_id\" " workload=details-v1

time="2021-03-11T02:07:49Z" level=info msg="created Istio EnvoyFilter resource" envoy_filter_resource=details-v1-bookinfo-custom-demo1.bookinfo.bookinfo filter="id:\"bookinfo-custom-demo1.bookinfo\" image:\"webassemblyhub.io/xiaowu/wasme-test:v0.20\" rootID:\"root_id\" " workload=details-v1

{"level":"debug","ts":1615428469.5524452,"msg":"applied filter to workload","result":"state:Succeeded "}

panic: reflect: slice index out of range [recovered]

	panic: reflect: slice index out of range [recovered]

	panic: reflect: slice index out of range

goroutine 200 [running]:

k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)

	/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.0.0-20191028221656-72ed19daf4bb/pkg/util/runtime/runtime.go:55 +0x105

panic(0x1d78d20, 0x2590e10)

	/opt/hostedtoolcache/go/1.14.12/x64/src/runtime/panic.go:969 +0x166

encoding/json.(*encodeState).marshal.func1(0xc001839220)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:326 +0x9a

panic(0x1d78d20, 0x2590e10)

	/opt/hostedtoolcache/go/1.14.12/x64/src/runtime/panic.go:969 +0x166

reflect.Value.Index(0x1d203e0, 0xc001040920, 0x197, 0x1, 0x2010000, 0xc00092c5a0, 0x199)

	/opt/hostedtoolcache/go/1.14.12/x64/src/reflect/value.go:946 +0x1e7

encoding/json.arrayEncoder.encode(0xc000b95620, 0xc000533200, 0x1d203e0, 0xc001040920, 0x197, 0x1d70000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:886 +0x8d

encoding/json.sliceEncoder.encode(0xc000b92c80, 0xc000533200, 0x1d203e0, 0xc001040920, 0x197, 0x0)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:860 +0x8f

encoding/json.structEncoder.encode(0xc000292900, 0x10, 0x10, 0xc000b95650, 0xc000533200, 0x2138280, 0xc001040840, 0x199, 0x0)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:759 +0x2bb

encoding/json.structEncoder.encode(0xc000a6c900, 0x5, 0x8, 0xc000b95830, 0xc000533200, 0x1fa0f80, 0xc001040820, 0x199, 0x1e10000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:759 +0x2bb

encoding/json.ptrEncoder.encode(0xc000b95860, 0xc000533200, 0x215e2c0, 0xc001040820, 0x16, 0x2150000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:915 +0x116

encoding/json.(*encodeState).reflectValue(0xc000533200, 0x215e2c0, 0xc001040820, 0x16, 0xc001040000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:358 +0x82

encoding/json.interfaceEncoder(0xc000533200, 0x20e4860, 0xc00189f580, 0x94, 0x0)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:714 +0xac

encoding/json.structEncoder.encode(0xc0008a06c0, 0x4, 0x4, 0xc000383710, 0xc000533200, 0x1fea120, 0xc00189f580, 0x99, 0x1fe0000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:759 +0x2bb

encoding/json.(*encodeState).reflectValue(0xc000533200, 0x1fea120, 0xc00189f580, 0x99, 0xc001830000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:358 +0x82

encoding/json.(*encodeState).marshal(0xc000533200, 0x1fea120, 0xc00189f580, 0x580000, 0x0, 0x0)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:330 +0xf0

encoding/json.(*Encoder).Encode(0xc0011e6320, 0x1fea120, 0xc00189f580, 0x5aff8b, 0xc00112c120)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/stream.go:206 +0x8a

go.uber.org/zap/zapcore.(*jsonEncoder).encodeReflected(0xc00112c120, 0x1fea120, 0xc00189f580, 0x9dc1b6348c84dd2f, 0xffffffffffffffd2, 0x3500, 0x3438323435313631, 0x3430333337353936)

	/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/json_encoder.go:157 +0x63

go.uber.org/zap/zapcore.(*jsonEncoder).AddReflected(0xc00112c120, 0x219b31e, 0x5, 0x1fea120, 0xc00189f580, 0xc0014dc000, 0x16)

	/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/json_encoder.go:165 +0x4c

go.uber.org/zap/zapcore.Field.AddTo(0x219b31e, 0x5, 0x17, 0x0, 0x0, 0x0, 0x1fea120, 0xc00189f580, 0x2654f80, 0xc00112c120)

	/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/field.go:164 +0xb74

go.uber.org/zap/zapcore.addFields(0x2654f80, 0xc00112c120, 0xc000532300, 0x2, 0x2)

	/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/field.go:204 +0xcf

go.uber.org/zap/zapcore.(*jsonEncoder).EncodeEntry(0xc000a403c0, 0xff, 0xc00a7cbd622bef27, 0x2bdbb3326e, 0x39a10e0, 0x0, 0x0, 0x21a7856, 0xd, 0x0, ...)

	/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/json_encoder.go:383 +0x1e4

sigs.k8s.io/controller-runtime/pkg/log/zap.(*KubeAwareEncoder).EncodeEntry(0xc00000c1a0, 0xff, 0xc00a7cbd622bef27, 0x2bdbb3326e, 0x39a10e0, 0x0, 0x0, 0x21a7856, 0xd, 0x0, ...)

	/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.8/pkg/log/zap/kube_helpers.go:128 +0x174

go.uber.org/zap/zapcore.(*ioCore).Write(0xc000a403f0, 0xff, 0xc00a7cbd622bef27, 0x2bdbb3326e, 0x39a10e0, 0x0, 0x0, 0x21a7856, 0xd, 0x0, ...)

	/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/core.go:86 +0xa9

go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000981970, 0xc000532300, 0x2, 0x2)

	/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/entry.go:216 +0x117

github.com/go-logr/zapr.(*infoLogger).Info(0xc000025e10, 0x21a7856, 0xd, 0xc00189f5c0, 0x4, 0x4)

	/home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.1.1/zapr.go:70 +0xdd

github.com/solo-io/skv2/pkg/events.(*cache).handleEvent(0xc00094fc20, 0x25e41c0, 0xc00189f580, 0x26457a0, 0xc00094ff00)

	/home/runner/go/pkg/mod/github.com/solo-io/skv2@v0.8.1/pkg/events/cache.go:84 +0x16d

github.com/solo-io/skv2/pkg/events.(*cache).Update(0xc00094fc20, 0x7f1b913832b0, 0xc001040820, 0x25dcf00, 0xc001040820, 0x7f1b913832b0, 0xc000190340, 0x25dcf00, 0xc000190340, 0x26457a0, ...)

	/home/runner/go/pkg/mod/github.com/solo-io/skv2@v0.8.1/pkg/events/cache.go:102 +0xab

sigs.k8s.io/controller-runtime/pkg/source/internal.EventHandler.OnUpdate(0x7f1b911839b8, 0xc00094fc20, 0x26457a0, 0xc00094ff00, 0x0, 0x0, 0x0, 0x215e2c0, 0xc001040820, 0x215e2c0, ...)

	/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.8/pkg/source/internal/eventsource.go:124 +0x444

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc000421738, 0x2, 0x1)

	/home/runner/go/pkg/mod/k8s.io/client-go@v0.0.0-20191016111102-bec269661e48/tools/cache/shared_informer.go:656 +0x1e8

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc001839e18, 0x2000000a87748, 0x0)

	/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.0.0-20191028221656-72ed19daf4bb/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/home/runner/go/pkg/mod/k8s.io/client-go@v0.0.0-20191016111102-bec269661e48/tools/cache/shared_informer.go:652 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0009e1760)

	/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.0.0-20191028221656-72ed19daf4bb/pkg/util/wait/wait.go:152 +0x5f

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001839f60, 0xdf8475800, 0x0, 0x1, 0xc0004f48a0)

	/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.0.0-20191028221656-72ed19daf4bb/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.0.0-20191028221656-72ed19daf4bb/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000b91880)

	/home/runner/go/pkg/mod/k8s.io/client-go@v0.0.0-20191016111102-bec269661e48/tools/cache/shared_informer.go:650 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0003d9020, 0xc000b929a0)

	/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.0.0-20191028221656-72ed19daf4bb/pkg/util/wait/wait.go:71 +0x51

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.0.0-20191028221656-72ed19daf4bb/pkg/util/wait/wait.go:69 +0x62

[Sodman]

Hi @xiaowujee, I haven't been able to reproduce this on my end. Does this happen consistently for you? Can you check the logs of the cache pod and see if there's anything suspicious in there? Also I would try restarting the cache pod and see if that makes any difference.