Skip to content

Commit 2d2524f

Browse files
greeshma1196greeshma1196
authored
zone aware routing: latency by zone + rps by zone (#4111)
This PR introduces two changes: 1. When I operate skipper-ingress controller across multiple availability zones, I want backend request latency labeled by the destination endpoint's zone, So that I can plot per-zone latency and RPS by zone RPS by zone: <img width="1490" height="845" alt="image" src="https://github.com/user-attachments/assets/1b31464f-67ed-471f-8992-16ee658e4835" /> Latency by zone: <img width="1486" height="853" alt="image" src="https://github.com/user-attachments/assets/935f047a-a052-40a0-aed4-7f534d860174" /> 2. When I run skipper-ingress that consume routes from RouteSRV, I want each backend endpoint's availability zone to survive the eskip wire format, So that per-zone backend latency can be plotted when RouteSRV is used, the same as in the in-process kubernetes dataclient mode - ref: #4103 Result: In the test cluster, there were 2 instances of skipper-ingress one set to Zone-A, the other to Zone-B route-dump of skipper-ingress in zone A ``` kube_rg__skipper_loadtest_target__loadtest_target__all__0_0: Host(/^(skipper-loadtest[.]pg9-test[.]zalan[.]do[.]?(:[0-9]+)?)$/) && PathSubtree("/") -> disableAccessLog(2, 3, 404, 429) -> fifo(2000, 20, "1s") -> disableAccessLog() -> oauthTokeninfoValidate("{optOutAnnotations: [iam.zalando.org/public], optOutHosts: [\"^.*[.]ingress[.]cluster[.]local\"], unauthorizedResponse: \"Authentication required, see https://cloud.docs.zalando.net/howtos/authenticate-requests/#default-authentication\\n\"}") -> stateBagToTag("auth-user", "client.uid") -> <powerOfRandomNChoices, "http://[2a05:d024:90:1103:6d8f::4]:9090?zone=eu-central-1a", "http://[2a05:d024:90:1103:931f::b]:9090?zone=eu-central-1a", "http://[2a05:d024:90:1103:931f::c]:9090?zone=eu-central-1a">; ``` route-dump of skipper-ingress in zone B ``` kube_rg__skipper_loadtest_target__loadtest_target__all__0_0: Host(/^(skipper-loadtest[.]pg9-test[.]zalan[.]do[.]?(:[0-9]+)?)$/) && PathSubtree("/") -> disableAccessLog(2, 3, 404, 429) -> fifo(2000, 20, "1s") -> disableAccessLog() -> oauthTokeninfoValidate("{optOutAnnotations: [iam.zalando.org/public], optOutHosts: [\"^.*[.]ingress[.]cluster[.]local\"], unauthorizedResponse: \"Authentication required, see https://cloud.docs.zalando.net/howtos/authenticate-requests/#default-authentication\\n\"}") -> stateBagToTag("auth-user", "client.uid") -> <powerOfRandomNChoices, "http://[2a05:d024:90:1104:99f6::20]:9090?zone=eu-central-1b", "http://[2a05:d024:90:1104:99f6::24]:9090?zone=eu-central-1b", "http://[2a05:d024:90:1104:99f6::c]:9090?zone=eu-central-1b">; ``` --------- Signed-off-by: greeshma1196 <greeshma.mathew@gmail.com>
1 parent 2948b38 commit 2d2524f

37 files changed

Lines changed: 490 additions & 84 deletions

‎CONTEXT.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,13 @@ Central route server precomputes per-zone route sets; each proxy fetches `/route
261261
**DataClient Mode**:
262262
Each proxy instance filters endpoints to its own zone at the dataclient layer during endpoint fetch.
263263

264+
**Backend zone**:
265+
The availability zone of the selected destination endpoint, sourced from the
266+
Kubernetes EndpointSlice `zone` field (e.g. `eu-central-1c`). It is the zone the
267+
traffic is sent *to*. Recorded as the `zone` metric label whenever present, else
268+
empty.
269+
_Avoid_: AZ, endpoint zone, destination zone
270+
264271
## RouteSRV
265272

266273
**RouteSRV**:

‎config/config.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ type Config struct {
101101
ServeMethodMetric bool `yaml:"serve-method-metric"`
102102
ServeStatusCodeMetric bool `yaml:"serve-status-code-metric"`
103103
BackendHostMetrics bool `yaml:"backend-host-metrics"`
104+
BackendZoneMetrics bool `yaml:"backend-zone-metrics"`
104105
ProxyRequestMetrics bool `yaml:"proxy-request-metrics"`
105106
ProxyResponseMetrics bool `yaml:"proxy-response-metrics"`
106107
AllFiltersMetrics bool `yaml:"all-filters-metrics"`
@@ -484,6 +485,7 @@ func NewConfig() *Config {
484485
flag.BoolVar(&cfg.ServeMethodMetric, "serve-method-metric", true, "enables the HTTP method as a domain of the total serve time metric. It affects both route and host split metrics")
485486
flag.BoolVar(&cfg.ServeStatusCodeMetric, "serve-status-code-metric", true, "enables the HTTP response status code as a domain of the total serve time metric. It affects both route and host split metrics")
486487
flag.BoolVar(&cfg.BackendHostMetrics, "backend-host-metrics", false, "enables reporting total serve time metrics for each backend")
488+
flag.BoolVar(&cfg.BackendZoneMetrics, "backend-zone-metrics", false, "enables reporting backend serve time metrics per availability zone of the destination endpoint")
487489
flag.BoolVar(&cfg.ProxyRequestMetrics, "proxy-request-metrics", false, "enables reporting latency / time spent in handling the request part of the proxy operation i.e., the duration from entry till before the backend round trip")
488490
flag.BoolVar(&cfg.ProxyResponseMetrics, "proxy-response-metrics", false, "enables reporting latency / time spent in handling the response part of the proxy operation i.e., the duration from after the backend round trip till the response is served")
489491
flag.BoolVar(&cfg.AllFiltersMetrics, "all-filters-metrics", false, "enables reporting combined filter metrics for each route")
@@ -990,6 +992,7 @@ func (c *Config) ToOptions() skipper.Options {
990992
EnableProxyRequestMetrics: c.ProxyRequestMetrics,
991993
EnableProxyResponseMetrics: c.ProxyResponseMetrics,
992994
EnableBackendHostMetrics: c.BackendHostMetrics,
995+
EnableBackendZoneMetrics: c.BackendZoneMetrics,
993996
EnableAllFiltersMetrics: c.AllFiltersMetrics,
994997
EnableCombinedResponseMetrics: c.CombinedResponseMetrics,
995998
EnableRouteResponseMetrics: c.RouteResponseMetrics,

‎docs/operation/operation.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,31 @@ If enabled these metrics are also exposed in /metrics, and the example json stru
461461
}
462462
```
463463

464+
### Backend zone metrics
465+
466+
Skipper can label backend request latency by the availability zone of the
467+
destination endpoint. This is disabled by default and enabled with:
468+
469+
-backend-zone-metrics
470+
enables reporting backend serve time metrics per availability zone of the destination endpoint
471+
472+
When enabled, the existing `skipper_backend_duration_seconds` histogram gains a
473+
`zone` label. The zone is the availability zone of the selected load-balanced
474+
endpoint, sourced from the Kubernetes EndpointSlice `zone` field. It is
475+
available in both the in-process kubernetes dataclient mode and RouteSRV mode.
476+
Non-load-balanced backends (shunt, loopback, network) have no zone and produce
477+
no zone observation. The `zone` observation is recorded independently of the `route` and
478+
`host` labels, so per-zone series have empty `route` and `host`.
479+
480+
Same-zone versus cross-zone traffic is a query-time concern: compare the `zone`
481+
label against the availability zone of the skipper instance. Example, plotting
482+
the p99 backend latency per zone:
483+
484+
```promql
485+
histogram_quantile(0.99,
486+
sum by (zone, le) (rate(skipper_backend_duration_seconds_bucket{zone!=""}[5m])))
487+
```
488+
464489
### Connection metrics
465490

466491
This option will enable known loadbalancer connections metrics, like

‎eskip/eskip.go‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
)
1717

1818
const duplicateHeaderPredicateErrorFmt = "duplicate header predicate: %s"
19+
const zoneQueryParam = "zone"
1920

2021
var (
2122
errDuplicatePathTreePredicate = errors.New("duplicate path tree predicate")
@@ -361,9 +362,7 @@ func NewLBEndpoints(eps []string) []*LBEndpoint {
361362
result := make([]*LBEndpoint, len(eps))
362363

363364
for i := range eps {
364-
result[i] = &LBEndpoint{
365-
Address: eps[i],
366-
}
365+
result[i] = newLBEndpoint(eps[i])
367366
}
368367

369368
return result
@@ -391,6 +390,36 @@ func (ep LBEndpoint) String() string {
391390
return ep.Address
392391
}
393392

393+
// StringWithZone returns the endpoint address with the availability zone encoded
394+
// as a "zone" query parameter.
395+
func (ep LBEndpoint) StringWithZone() string {
396+
if ep.Zone == "" {
397+
return ep.Address
398+
}
399+
return ep.Address + "?" + zoneQueryParam + "=" + url.QueryEscape(ep.Zone)
400+
}
401+
402+
// newLBEndpoint builds an LBEndpoint from a endpoint string, extracting the
403+
// availability zone from the "zone" query parameter if present. Only the "zone"
404+
// key is stripped; the remaining address is kept clean. Address must not carry
405+
// the zone param so that it can be used as it was originally defined
406+
func newLBEndpoint(s string) *LBEndpoint {
407+
u, err := url.Parse(s)
408+
if err != nil {
409+
return &LBEndpoint{Address: s}
410+
}
411+
412+
q := u.Query()
413+
zone := q.Get(zoneQueryParam)
414+
if zone == "" {
415+
return &LBEndpoint{Address: s}
416+
}
417+
418+
q.Del(zoneQueryParam)
419+
u.RawQuery = q.Encode()
420+
return &LBEndpoint{Address: u.String(), Zone: zone}
421+
}
422+
394423
type RoutePredicate func(*Route) bool
395424

396425
// RouteInfo contains a route id, plus the loaded and parsed route or

‎eskip/eskip_test.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -896,3 +896,59 @@ func BenchmarkRouteStringNoRepeatedPredicates(b *testing.B) {
896896
}
897897
stringSink = s
898898
}
899+
900+
func TestLBEndpointStringWithZone(t *testing.T) {
901+
for _, ti := range []struct {
902+
name string
903+
ep LBEndpoint
904+
want string
905+
}{{
906+
name: "test lb endpoint without zone",
907+
ep: LBEndpoint{Address: "http://10.0.0.1:8080"},
908+
want: "http://10.0.0.1:8080",
909+
}, {
910+
name: "test lb endpoint with zone",
911+
ep: LBEndpoint{Address: "http://10.0.0.1:8080", Zone: "eu-central-1a"},
912+
want: "http://10.0.0.1:8080?zone=eu-central-1a",
913+
}} {
914+
t.Run(ti.name, func(t *testing.T) {
915+
if got := ti.ep.StringWithZone(); got != ti.want {
916+
t.Errorf("StringWithZone() = %q, want %q", got, ti.want)
917+
}
918+
})
919+
}
920+
}
921+
922+
func TestNewLBEndpoint(t *testing.T) {
923+
for _, ti := range []struct {
924+
name string
925+
in string
926+
wantAddr string
927+
wantZone string
928+
}{{
929+
name: "test lb endpoint with zone",
930+
in: "http://10.0.0.1:8080?zone=eu-central-1a",
931+
wantAddr: "http://10.0.0.1:8080",
932+
wantZone: "eu-central-1a",
933+
}, {
934+
name: "test lb endpoint without zone",
935+
in: "http://10.0.0.1:8080",
936+
wantAddr: "http://10.0.0.1:8080",
937+
wantZone: "",
938+
}, {
939+
name: "test non-URL string",
940+
in: "not a url",
941+
wantAddr: "not a url",
942+
wantZone: "",
943+
}} {
944+
t.Run(ti.name, func(t *testing.T) {
945+
ep := newLBEndpoint(ti.in)
946+
if ep.Address != ti.wantAddr {
947+
t.Errorf("Address = %q, want %q", ep.Address, ti.wantAddr)
948+
}
949+
if ep.Zone != ti.wantZone {
950+
t.Errorf("Zone = %q, want %q", ep.Zone, ti.wantZone)
951+
}
952+
})
953+
}
954+
}

‎eskip/json.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,15 @@ func newJSONRoute(r *Route) *jsonRoute {
3333
}
3434

3535
if cr.BackendType != NetworkBackend || cr.Backend != "" {
36+
eps := make([]string, len(cr.LBEndpoints))
37+
for i, ep := range cr.LBEndpoints {
38+
eps[i] = ep.StringWithZone()
39+
}
3640
jr.Backend = &jsonBackend{
3741
Type: cr.BackendType.String(),
3842
Address: cr.Backend,
3943
Algorithm: cr.LBAlgorithm,
40-
Endpoints: LBEndpointString(cr.LBEndpoints),
44+
Endpoints: eps,
4145
}
4246
}
4347

‎eskip/json_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,22 @@ func TestMarshalUnmarshalJSON(t *testing.T) {
9191
[]*Route{{Id: "beef", BackendType: LBBackend, LBAlgorithm: "yolo", LBEndpoints: []*LBEndpoint{{Address: "localhost"}}}},
9292
`[{"id":"beef","backend":{"type":"lb","algorithm":"yolo","endpoints":["localhost"]}}]`,
9393
},
94+
{
95+
"lb backend with zone",
96+
[]*Route{{Id: "beef", BackendType: LBBackend, LBAlgorithm: "roundRobin", LBEndpoints: []*LBEndpoint{
97+
{Address: "http://10.0.0.1:8080", Zone: "eu-central-1a"},
98+
{Address: "http://10.0.0.2:8080", Zone: "eu-central-1b"},
99+
}}},
100+
`[{"id":"beef","backend":{"type":"lb","algorithm":"roundRobin","endpoints":["http://10.0.0.1:8080?zone=eu-central-1a","http://10.0.0.2:8080?zone=eu-central-1b"]}}]`,
101+
},
102+
{
103+
"lb backend without zone",
104+
[]*Route{{Id: "beef", BackendType: LBBackend, LBAlgorithm: "roundRobin", LBEndpoints: []*LBEndpoint{
105+
{Address: "http://10.0.0.1:8080"},
106+
{Address: "http://10.0.0.2:8080"},
107+
}}},
108+
`[{"id":"beef","backend":{"type":"lb","algorithm":"roundRobin","endpoints":["http://10.0.0.1:8080","http://10.0.0.2:8080"]}}]`,
109+
},
94110
{
95111
"shunt backend",
96112
[]*Route{{Id: "shunty", BackendType: ShuntBackend}},

‎eskip/roundtrip_test.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,63 @@ func TestRoundtripRegexp(t *testing.T) {
122122
})
123123
}
124124
}
125+
126+
func TestRoundtripLBZone(t *testing.T) {
127+
t.Run("endpoints have zone", func(t *testing.T) {
128+
in := &Route{
129+
Id: "r1",
130+
BackendType: LBBackend,
131+
LBAlgorithm: "roundRobin",
132+
LBEndpoints: []*LBEndpoint{
133+
{Address: "http://10.0.0.1:8080", Zone: "eu-central-1a"},
134+
{Address: "http://10.0.0.2:8080", Zone: "eu-central-1b"},
135+
},
136+
}
137+
138+
serialized := in.String()
139+
t.Logf("serialized: %s", serialized)
140+
141+
outs, err := Parse(serialized)
142+
require.NoError(t, err)
143+
require.Len(t, outs, 1)
144+
145+
out := outs[0]
146+
require.Len(t, out.LBEndpoints, 2)
147+
148+
assert.Equal(t, "http://10.0.0.1:8080", out.LBEndpoints[0].Address)
149+
assert.Equal(t, "eu-central-1a", out.LBEndpoints[0].Zone)
150+
assert.Equal(t, "http://10.0.0.2:8080", out.LBEndpoints[1].Address)
151+
assert.Equal(t, "eu-central-1b", out.LBEndpoints[1].Zone)
152+
153+
assert.Equal(t, serialized, out.String(), "round-trip must be stable")
154+
})
155+
156+
t.Run("endpoints do not have zone", func(t *testing.T) {
157+
in := &Route{
158+
Id: "r2",
159+
BackendType: LBBackend,
160+
LBAlgorithm: "roundRobin",
161+
LBEndpoints: []*LBEndpoint{
162+
{Address: "http://10.0.0.1:8080"},
163+
{Address: "http://10.0.0.2:8080"},
164+
},
165+
}
166+
167+
serialized := in.String()
168+
t.Logf("serialized: %s", serialized)
169+
170+
outs, err := Parse(serialized)
171+
require.NoError(t, err)
172+
require.Len(t, outs, 1)
173+
174+
out := outs[0]
175+
require.Len(t, out.LBEndpoints, 2)
176+
177+
assert.Equal(t, "http://10.0.0.1:8080", out.LBEndpoints[0].Address)
178+
assert.Equal(t, "", out.LBEndpoints[0].Zone)
179+
assert.Equal(t, "http://10.0.0.2:8080", out.LBEndpoints[1].Address)
180+
assert.Equal(t, "", out.LBEndpoints[1].Zone)
181+
182+
assert.Equal(t, serialized, out.String(), "round-trip must be stable")
183+
})
184+
}

‎eskip/string.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ func lbBackendString(r *Route) string {
167167
b.WriteString(", ")
168168
}
169169
b.WriteByte('"')
170-
b.WriteString(ep.String())
170+
b.WriteString(ep.StringWithZone())
171171
b.WriteByte('"')
172172
}
173173
b.WriteByte('>')

‎eskip/string_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,12 @@ func TestRouteString(t *testing.T) {
123123
{Address: "http://127.0.0.1:9998"},
124124
}},
125125
`Method("GET") -> <random, "http://127.0.0.1:9997", "http://127.0.0.1:9998">`,
126+
}, {
127+
&Route{Method: "GET", LBAlgorithm: "random", BackendType: LBBackend, LBEndpoints: []*LBEndpoint{
128+
{Address: "http://127.0.0.1:9997", Zone: "eu-central-1a"},
129+
{Address: "http://127.0.0.1:9998", Zone: "eu-central-1b"},
130+
}},
131+
`Method("GET") -> <random, "http://127.0.0.1:9997?zone=eu-central-1a", "http://127.0.0.1:9998?zone=eu-central-1b">`,
126132
}, {
127133
// test slash escaping
128134
&Route{Path: `/`, PathRegexps: []string{`/`}, Filters: []*Filter{{"afilter", []interface{}{`/`}}}, BackendType: ShuntBackend},

0 commit comments

Comments
 (0)