Skip to content

sidecar.MD

Latest commit

 

History

History
266 lines (238 loc) · 8.75 KB

sidecar.MD

File metadata and controls

266 lines (238 loc) · 8.75 KB
Raw

Deploy toolbox

kubectl create ns sidecar
kubectl label ns sidecar istio-injection=enabled
kubectl apply -f nginx.yaml -n sidecar
kubectl apply -f toolbox.yaml -n sidecar

排查命令

istioctl pc listener -n sidecar toolbox-666487ff7f-qc2gb --port 15006
istioctl pc route -n sidecar toolbox-666487ff7f-qc2gb --name="inbound|80||" -ojson
istioctl pc cluster -n sidecar toolbox-666487ff7f-qc2gb
istioctl pc endpoint -n sidecar toolbox-666487ff7f-qc2gb

验证

# 查看webhook
[root@master1 sidecar]# k get mutatingwebhookconfigurations
NAME                         WEBHOOKS   AGE
istio-revision-tag-default   4          90m
istio-sidecar-injector       4          92m
[root@master1 sidecar]# 
[root@master1 sidecar]# k get mutatingwebhookconfigurations  istio-sidecar-injector  -oyaml


# 检查服务访问是否正常：从toolbox 访问nginx
kubectl exec -ti  -n sidecar toolbox-666487ff7f-qc2gb   bash 
[root@toolbox-666487ff7f-qc2gb /]# curl nginx


# 从客户端toolbox梳理流量出去到nginx
[root@master1 sidecar]# docker ps  |grep tool
cc659edfb835        5bd54fe7908d                         "/usr/local/bin/pilo…"   6 minutes ago       Up 6 minutes                                                          k8s_istio-proxy_toolbox-666487ff7f-qc2gb_sidecar_260c9f68-2971-4cb1-b2ab-18ae373bf9a0_0
8e7b639357c5        centos                               "tail -f /dev/null"      6 minutes ago       Up 6 minutes                                                          k8s_toolbox_toolbox-666487ff7f-qc2gb_sidecar_260c9f68-2971-4cb1-b2ab-18ae373bf9a0_0
4b624eeaa894        k8s.gcr.io/pause:3.2                 "/pause"                 7 minutes ago       Up 7 minutes                                                          k8s_POD_toolbox-666487ff7f-qc2gb_sidecar_260c9f68-2971-4cb1-b2ab-18ae373bf9a0_0

[root@master1 sidecar]# docker inspect 8e7b639357c5 |grep -i pid
            "Pid": 15697,
            "PidMode": "",
            "PidsLimit": null,
[root@master1 sidecar]# nsenter   -t     15697  -n iptables-save 
# Generated by iptables-save v1.4.21 on Tue Feb 15 17:32:16 2022
*raw
:PREROUTING ACCEPT [2199:411868]
:OUTPUT ACCEPT [1771:218842]
COMMIT
# Completed on Tue Feb 15 17:32:16 2022
# Generated by iptables-save v1.4.21 on Tue Feb 15 17:32:16 2022
*mangle
:PREROUTING ACCEPT [2199:411868]
:INPUT ACCEPT [2199:411868]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [1771:218842]
:POSTROUTING ACCEPT [1771:218842]
COMMIT
# Completed on Tue Feb 15 17:32:16 2022
# Generated by iptables-save v1.4.21 on Tue Feb 15 17:32:16 2022
*filter
:INPUT ACCEPT [2199:411868]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [1771:218842]
COMMIT
# Completed on Tue Feb 15 17:32:16 2022
# Generated by iptables-save v1.4.21 on Tue Feb 15 17:32:16 2022
*nat
:PREROUTING ACCEPT [206:12360]
:INPUT ACCEPT [206:12360]
:OUTPUT ACCEPT [42:3545]
:POSTROUTING ACCEPT [43:3605]
:ISTIO_INBOUND - [0:0]
:ISTIO_IN_REDIRECT - [0:0]
:ISTIO_OUTPUT - [0:0]
:ISTIO_REDIRECT - [0:0]
-A PREROUTING -p tcp -j ISTIO_INBOUND
# -A OUTPUT -p tcp -j ISTIO_OUTPUT
-A ISTIO_INBOUND -p tcp -m tcp --dport 15008 -j RETURN
-A ISTIO_INBOUND -p tcp -m tcp --dport 22 -j RETURN
-A ISTIO_INBOUND -p tcp -m tcp --dport 15090 -j RETURN
-A ISTIO_INBOUND -p tcp -m tcp --dport 15021 -j RETURN
-A ISTIO_INBOUND -p tcp -m tcp --dport 15020 -j RETURN
-A ISTIO_INBOUND -p tcp -j ISTIO_IN_REDIRECT
-A ISTIO_IN_REDIRECT -p tcp -j REDIRECT --to-ports 15006
-A ISTIO_OUTPUT -s 127.0.0.6/32 -o lo -j RETURN
-A ISTIO_OUTPUT ! -d 127.0.0.1/32 -o lo -m owner --uid-owner 1337 -j ISTIO_IN_REDIRECT
-A ISTIO_OUTPUT -o lo -m owner ! --uid-owner 1337 -j RETURN
-A ISTIO_OUTPUT -m owner --uid-owner 1337 -j RETURN
-A ISTIO_OUTPUT ! -d 127.0.0.1/32 -o lo -m owner --gid-owner 1337 -j ISTIO_IN_REDIRECT
-A ISTIO_OUTPUT -o lo -m owner ! --gid-owner 1337 -j RETURN
-A ISTIO_OUTPUT -m owner --gid-owner 1337 -j RETURN
-A ISTIO_OUTPUT -d 127.0.0.1/32 -j RETURN
-A ISTIO_OUTPUT -j ISTIO_REDIRECT
# -A ISTIO_REDIRECT -p tcp -j REDIRECT --to-ports 15001
COMMIT
# Completed on Tue Feb 15 17:32:16 2022

Check clusters

kubectl get po -n sidecar
istioctl pc cluster -n sidecar toolbox-68f79dd5f8-hkgkl

Check tcp dump to make sure the traffic forward happen in client side(244941 is toolbox process id)

nsenter -t 244941 -n tcpdump -i eth0 -vvv port 80

Or check toolbox sidecar logs and we can see the traffic is passthrough

k logs -f toolbox-68f79dd5f8-hkgkl -n sidecar -c istio-proxy

Default access log format

[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%"
%RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION%
%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%"
"%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n

Data path

Client -> Server

Client side

Traffic is hijacked to envoy 15001 by iptables

nsenter -t 244941 -n iptables-save
---
-A OUTPUT -p tcp -j ISTIO_OUTPUT
-A ISTIO_OUTPUT -j ISTIO_REDIRECT
-A ISTIO_REDIRECT -p tcp -j REDIRECT --to-ports 15001
---

Check envoy config

istioctl pc listener -n sidecar toolbox-68f79dd5f8-dcz5g --port 15001 -ojson
---
    "name": "virtualOutbound",
    "address": {
        "socketAddress": {
            "address": "0.0.0.0",
            "portValue": 15001
        }
    },
    "useOriginalDst": true
---
then it is handed over to listener 80
---

    "name": "0.0.0.0_80",
    "address": {
        "socketAddress": {
            "address": "0.0.0.0",
            "portValue": 80
        }
    },
    "routeConfigName": "80"
---
then it is handle by route 80
istioctl pc route -n sidecar toolbox-68f79dd5f8-dcz5g --name=80
---
{
                "name": "nginx.sidecar.svc.cluster.local:80",
                "domains": [
                    "nginx.sidecar.svc.cluster.local",
                    "nginx.sidecar.svc.cluster.local:80",
                    "nginx",
                    "nginx:80",
                    "nginx.sidecar.svc",
                    "nginx.sidecar.svc:80",
                    "nginx.sidecar",
                    "nginx.sidecar:80",
                    "10.109.253.161",
                    "10.109.253.161:80"
                ],
                "routes": [
                    {
                        "name": "default",
                        "match": {
                            "prefix": "/"
                        },
                        "route": {
                            "cluster": "outbound|80||nginx.sidecar.svc.cluster.local",
---
then it is handed over to cluster outbound|80||nginx.sidecar.svc.cluster.local
and the endpoint is the actual pod ip
istioctl pc endpoint -n sidecar toolbox-68f79dd5f8-dcz5g
---
192.168.166.189:80               HEALTHY     OK                outbound|80||nginx.sidecar.svc.cluster.local
---
then the request is send to kernel, going to iptables again
-A OUTPUT -p tcp -j ISTIO_OUTPUT
-A ISTIO_OUTPUT -m owner --uid-owner 1337 -j RETURN
---
iptables return and the traffic is send out!

Server side is almost same but easier

iptables first, hijacked to 15006:

istioctl pc listener -n sidecar nginx-deployment-6799fc88d8-qq68n --port 15006
{
        "name": "virtualInbound",
        "address": {
            "socketAddress": {
                "address": "0.0.0.0",
                "portValue": 15006
            }
        },
---
    {
        "name": "0.0.0.0_80",
        "address": {
            "socketAddress": {
                "address": "0.0.0.0",
                "portValue": 80
            }
        },
        "filters": [
            "inbound_0.0.0.0_80"
        }
         "domains": [
                                            "*"
                                        ],
                                        "routes": [
                                            {
                                                "name": "default",
                                                "match": {
                                                    "prefix": "/"
                                                },
                                                "route": {
                                                    "cluster": "inbound|80||",
---
istioctl pc route -n sidecar nginx-deployment-6799fc88d8-qq68n --name="inbound|80||" -ojson
---
"name": "inbound|80||",
"decorator": {
    "operation": "nginx.sidecar.svc.cluster.local:80/*"
}
---
istioctl pc cluster -n sidecar nginx-deployment-6799fc88d8-qq68n
---
nginx.sidecar.svc.cluster.local                                                             80        -          outbound      EDS
---
istioctl pc endpoint -n sidecar nginx-deployment-6799fc88d8-qq68n
---
192.168.166.189:80               HEALTHY     OK                outbound|80||nginx.sidecar.svc.cluster.local
---
then it is send to kernel, handled by OUTPUT chain
-A ISTIO_OUTPUT -m owner --uid-owner 1337 -j RETURN