Skip to content

Commit 41a90d0

Browse files
MarcinFalkowskiMarcinFalkowski
authored
PROM-5262 | x-service-tag-preference (#450)
1 parent b71802a commit 41a90d0

40 files changed

+1106
-459
lines changed

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@
33
Lists all changes with user impact.
44
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
55

6+
## [0.22.12]
7+
### Changed
8+
- add service tag preference routing
9+
- optimize auto service tag routing (don't send config to envoys if disabled, reduce number of metadata ser per route)
10+
- fix and refactor e2e tests
11+
612
## [0.22.11]
713
### Changed
814
- Implemented handling of initialResourcesVersions in DeltaRequest for ADS

‎docs/configuration.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,12 @@ Property
148148
**envoy-control.envoy.snapshot.routing.service-tags.enabled** | If set to true, service tags routing will be enabled | false
149149
**envoy-control.envoy.snapshot.routing.service-tags.metadata-key** | What key to use in endpoint metadata to store its service tags | tag
150150
**envoy-control.envoy.snapshot.routing.service-tags.header** | What header to use in service tag rules | x-service-tag
151-
**envoy-control.envoy.snapshot.routing.service-tags.preference-header** | What header to use for service tag preference list. Used for sending info to upstream if 'auto service tags' is in force. In the future also read from downstream request. | x-service-tag-preference
151+
**envoy-control.envoy.snapshot.routing.service-tags.preference-routing.header** | What header to use for service tag preference list. Used for sending info to upstream if 'auto service tags' is enabled and for routing if preference routing is enabled. | x-service-tag-preference
152+
**envoy-control.envoy.snapshot.routing.service-tags.preference-routing.enable-for-all** | enable preference routing for all services (services can still be excluded by `disable-for-services` property) | false
153+
**envoy-control.envoy.snapshot.routing.service-tags.preference-routing.enable-for-services** | enable preference routing for selected services | []
154+
**envoy-control.envoy.snapshot.routing.service-tags.preference-routing.disable-for-services** | disable preference routing for selected services | []
155+
**envoy-control.envoy.snapshot.routing.service-tags.preference-routing.default-preference-env** | environment variable which default service tag preference for an envoy will be taken from | DEFAULT_SERVICE_TAG_PREFERENCE
156+
**envoy-control.envoy.snapshot.routing.service-tags.preference-routing.default-preference-fallback** | default service tag preference for an envoy if env set in `default-preference-env` is absent | global
152157
**envoy-control.envoy.snapshot.routing.service-tags.routing-excluded-tags** | List of tags predicates that cannot be used for routing. This supports an exact matching (just "string" - EXACT matching) prefixes (PREFIX matching) and regexes (REGEX matching) | empty list
153158
**envoy-control.envoy.snapshot.routing.service-tags.allowed-tags-combinations** | List of rules, which tags can be conbined together and requested together. Details below | empty list
154159
**(...).allowed-tags-combinations[].service-name** | The rule will apply only for this service | ""

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroup.kt‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.logger
99
import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties
1010
import io.envoyproxy.envoy.config.core.v3.Node as NodeV3
1111

12-
@Suppress("MagicNumber")
13-
val MIN_ENVOY_VERSION_SUPPORTING_UPSTREAM_METADATA = envoyVersion(1, 24)
1412
@Suppress("MagicNumber")
1513
val MIN_ENVOY_VERSION_SUPPORTING_JWT_FAILURE_STATUS = envoyVersion(1, 26)
1614

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,14 +275,39 @@ class ServiceTagsProperties {
275275
var enabled = false
276276
var metadataKey = "tag"
277277
var header = "x-service-tag"
278-
var preferenceHeader = "x-service-tag-preference"
278+
var preferenceRouting = ServiceTagPreferenceProperties()
279279
var routingExcludedTags: MutableList<StringMatcher> = mutableListOf()
280+
// TODO[PROM-6067]: remove service tag combinations feature
280281
var allowedTagsCombinations: MutableList<ServiceTagsCombinationsProperties> = mutableListOf()
281282
var autoServiceTagEnabled = false
282283
var rejectRequestsWithDuplicatedAutoServiceTag = true
283284
var addUpstreamServiceTagsHeader: Boolean = false
284285

286+
// TODO[PROM-6055]: Ultimately, autoServiceTag feature should be removed, when preference routing
287+
// will handle all cases
285288
fun isAutoServiceTagEffectivelyEnabled() = enabled && autoServiceTagEnabled
289+
fun shouldRejectRequestsWithDuplicatedAutoServiceTag() =
290+
isAutoServiceTagEffectivelyEnabled() && rejectRequestsWithDuplicatedAutoServiceTag
291+
}
292+
293+
class ServiceTagPreferenceProperties {
294+
var enableForAll = false
295+
// TODO(PROM-6088): remove this option: ultimately all services should use it
296+
var enableForServices: List<String> = emptyList()
297+
var disableForServices: List<String> = emptyList()
298+
var header = "x-service-tag-preference"
299+
var defaultPreferenceEnv = "DEFAULT_SERVICE_TAG_PREFERENCE"
300+
var defaultPreferenceFallback = "global"
301+
302+
fun isEnabledFor(service: String): Boolean {
303+
val enabled = enableForAll || enableForServices.contains(service)
304+
if (!enabled) {
305+
return false
306+
}
307+
val disabled = disableForServices.contains(service)
308+
return !disabled
309+
}
310+
fun isEnabledForSome() = enableForAll || enableForServices.isNotEmpty()
286311
}
287312

288313
class StringMatcher {

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -497,6 +497,7 @@ class EnvoyClustersFactory(
497497
private fun Cluster.Builder.configureLbSubsets(): Cluster.Builder {
498498
val canaryEnabled = properties.loadBalancing.canary.enabled
499499
val tagsEnabled = properties.routing.serviceTags.enabled
500+
val tagPreferenceEnabled = properties.routing.serviceTags.preferenceRouting.isEnabledForSome()
500501

501502
if (!canaryEnabled && !tagsEnabled) {
502503
return this
@@ -533,6 +534,9 @@ class EnvoyClustersFactory(
533534
if (tagsEnabled && canaryEnabled) {
534535
addTagsAndCanarySelector()
535536
}
537+
if (tagPreferenceEnabled) {
538+
setMetadataFallbackPolicy(Cluster.LbSubsetConfig.LbSubsetMetadataFallbackPolicy.FALLBACK_LIST)
539+
}
536540
}
537541
)
538542
}

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/EnvoyDefaultFilters.kt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3
77
import pl.allegro.tech.servicemesh.envoycontrol.groups.Group
88
import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot
99
import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties
10+
import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.HttpFilterFactory
1011

1112
class EnvoyDefaultFilters(
1213
private val snapshotProperties: SnapshotProperties,
@@ -34,7 +35,7 @@ class EnvoyDefaultFilters(
3435
private val defaultHeaderToMetadataConfig = headerToMetadataConfig(defaultServiceTagHeaderToMetadataFilterRules)
3536
private val headerToMetadataHttpFilter = headerToMetadataHttpFilter(defaultHeaderToMetadataConfig)
3637
private val defaultHeaderToMetadataFilter = { _: Group, _: GlobalSnapshot -> headerToMetadataHttpFilter }
37-
private val defaultServiceTagFilter = { _: Group, _: GlobalSnapshot -> serviceTagFilterFactory.luaEgressFilter() }
38+
private val defaultServiceTagFilters = serviceTagFilterFactory.egressFilters()
3839
private val envoyRouterHttpFilter = envoyRouterHttpFilter()
3940

4041
/**
@@ -73,9 +74,9 @@ class EnvoyDefaultFilters(
7374
compressionFilterFactory.brotliCompressionFilter(group)
7475
}
7576

76-
val defaultEgressFilters = listOf(
77+
val defaultEgressFilters: List<HttpFilterFactory> = listOf(
7778
defaultHeaderToMetadataFilter,
78-
defaultServiceTagFilter,
79+
*defaultServiceTagFilters,
7980
defaultGzipCompressionFilter,
8081
defaultBrotliCompressionFilter,
8182
defaultEnvoyRouterHttpFilter,

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/LuaFilterFactory.kt‎

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import com.google.protobuf.Any
44
import com.google.protobuf.ListValue
55
import com.google.protobuf.Struct
66
import com.google.protobuf.Value
7+
import io.envoyproxy.envoy.config.core.v3.DataSource
78
import io.envoyproxy.envoy.config.core.v3.Metadata
89
import io.envoyproxy.envoy.extensions.filters.http.lua.v3.Lua
910
import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter
@@ -15,14 +16,8 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filt
1516

1617
class LuaFilterFactory(private val snapshotProperties: SnapshotProperties) {
1718

18-
private val ingressRbacLoggingScript: String = this::class.java.classLoader
19-
.getResource("lua/ingress_rbac_logging.lua")!!.readText()
20-
2119
private val ingressRbacLoggingFilter: HttpFilter? = if (snapshotProperties.incomingPermissions.enabled) {
22-
HttpFilter.newBuilder()
23-
.setName("envoy.lua")
24-
.setTypedConfig(Any.pack(Lua.newBuilder().setInlineCode(ingressRbacLoggingScript).build()))
25-
.build()
20+
createLuaFilter(luaFile = "lua/ingress_rbac_logging.lua", filterName = "envoy.lua")
2621
} else {
2722
null
2823
}
@@ -32,28 +27,15 @@ class LuaFilterFactory(private val snapshotProperties: SnapshotProperties) {
3227
fun ingressRbacLoggingFilter(group: Group): HttpFilter? =
3328
ingressRbacLoggingFilter.takeIf { group.proxySettings.incoming.permissionsEnabled }
3429

35-
private val ingressClientNameHeaderScript: String = this::class.java.classLoader
36-
.getResource("lua/ingress_client_name_header.lua")!!.readText()
37-
3830
private val ingressClientNameHeaderFilter: HttpFilter =
39-
HttpFilter.newBuilder()
40-
.setName("ingress.client.lua")
41-
.setTypedConfig(Any.pack(Lua.newBuilder().setInlineCode(ingressClientNameHeaderScript).build()))
42-
.build()
43-
44-
private val ingressCurrentZoneHeaderScript: String = this::class.java.classLoader
45-
.getResource("lua/ingress_current_zone_header.lua")!!.readText()
31+
createLuaFilter(luaFile = "lua/ingress_client_name_header.lua", filterName = "ingress.client.lua")
4632

4733
private val ingressCurrentZoneHeaderFilter: HttpFilter =
48-
HttpFilter.newBuilder()
49-
.setName("ingress.zone.lua")
50-
.setTypedConfig(Any.pack(Lua.newBuilder().setInlineCode(ingressCurrentZoneHeaderScript).build()))
51-
.build()
34+
createLuaFilter(luaFile = "lua/ingress_current_zone_header.lua", filterName = "ingress.zone.lua")
5235

53-
private val sanUriWildcardRegexForLua = SanUriMatcherFactory(
54-
snapshotProperties.incomingPermissions.tlsAuthentication
55-
)
56-
.sanUriWildcardRegexForLua
36+
private val sanUriWildcardRegexForLua =
37+
SanUriMatcherFactory(snapshotProperties.incomingPermissions.tlsAuthentication)
38+
.sanUriWildcardRegexForLua
5739

5840
fun ingressScriptsMetadata(
5941
group: Group,
@@ -97,6 +79,38 @@ class LuaFilterFactory(private val snapshotProperties: SnapshotProperties) {
9779
ingressClientNameHeaderFilter.takeIf { trustedClientIdentityHeader.isNotEmpty() }
9880

9981
fun ingressCurrentZoneHeaderFilter(): HttpFilter = ingressCurrentZoneHeaderFilter
82+
83+
companion object {
84+
private val placeholderFormat = "\"%([0-9a-z_]+)%\"".toRegex(RegexOption.IGNORE_CASE)
85+
private val replacementFormat = "[a-z0-9_-]+".toRegex(RegexOption.IGNORE_CASE)
86+
87+
fun createLuaFilter(
88+
luaFile: String,
89+
filterName: String,
90+
variables: Map<String, String> = emptyMap()
91+
): HttpFilter {
92+
val scriptTemplate = this::class.java.classLoader.getResource(luaFile)!!.readText()
93+
94+
val script = scriptTemplate.replace(placeholderFormat) { match ->
95+
val key = match.groupValues[1]
96+
val replacement = variables[key]
97+
?: throw IllegalArgumentException("Missing replacement for placeholder: $key")
98+
require(replacement.matches(replacementFormat)) { "invalid replacement format: '$replacement'" }
99+
"\"${replacement}\""
100+
}
101+
102+
return HttpFilter.newBuilder()
103+
.setName(filterName)
104+
.setTypedConfig(
105+
Any.pack(
106+
Lua.newBuilder().setDefaultSourceCode(
107+
DataSource.newBuilder()
108+
.setInlineString(script)
109+
).build()
110+
)
111+
).build()
112+
}
113+
}
100114
}
101115

102116
sealed class LuaMetadataProperty<T>(open val value: T) {
@@ -157,7 +171,8 @@ sealed class LuaMetadataProperty<T>(open val value: T) {
157171

158172
override fun toValue(): Value {
159173
return Value.newBuilder()
160-
.setStructValue(Struct.newBuilder()
174+
.setStructValue(
175+
Struct.newBuilder()
161176
.apply {
162177
value.forEach {
163178
putFields(

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/ServiceTagFilterFactory.kt‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,18 @@
11
package pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters
22

3-
import com.google.protobuf.Any
43
import io.envoyproxy.envoy.extensions.filters.http.header_to_metadata.v3.Config
5-
import io.envoyproxy.envoy.extensions.filters.http.lua.v3.Lua
64
import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter
5+
import pl.allegro.tech.servicemesh.envoycontrol.groups.Group
76
import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ServiceTagsProperties
7+
import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.HttpFilterFactory
8+
import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters.LuaFilterFactory.Companion.createLuaFilter
89

910
class ServiceTagFilterFactory(private val properties: ServiceTagsProperties) {
1011

1112
companion object {
1213
const val AUTO_SERVICE_TAG_PREFERENCE_METADATA = "auto_service_tag_preference"
13-
const val SERVICE_TAG_METADATA_KEY_METADATA = "service_tag_metadata_key"
14-
const val REJECT_REQUEST_SERVICE_TAG_DUPLICATE = "reject_request_service_tag_duplicate"
1514
}
1615

17-
private val luaEgressScript: String = this::class.java.classLoader
18-
.getResource("lua/egress_service_tags.lua")!!.readText()
19-
2016
fun headerToMetadataFilterRules(): List<Config.Rule> {
2117
return listOf(
2218
Config.Rule.newBuilder()
@@ -33,13 +29,40 @@ class ServiceTagFilterFactory(private val properties: ServiceTagsProperties) {
3329
)
3430
}
3531

36-
fun luaEgressFilter(): HttpFilter = HttpFilter.newBuilder()
37-
.setName("envoy.lua.servicetags")
38-
.setTypedConfig(
39-
Any.pack(
40-
Lua.newBuilder()
41-
.setInlineCode(luaEgressScript)
42-
.build()
32+
fun egressFilters(): Array<HttpFilterFactory> = arrayOf(
33+
{ group: Group, _ -> luaEgressServiceTagPreferenceFilter(group) },
34+
{ _, _ -> luaEgressAutoServiceTagsFilter }
35+
)
36+
37+
private fun luaEgressServiceTagPreferenceFilter(group: Group): HttpFilter? =
38+
if (properties.preferenceRouting.isEnabledFor(group.serviceName)) {
39+
luaEgressServiceTagPreferenceFilter
40+
} else {
41+
null
42+
}
43+
44+
private val luaEgressAutoServiceTagsFilter: HttpFilter? =
45+
if (properties.shouldRejectRequestsWithDuplicatedAutoServiceTag()) {
46+
createLuaFilter(
47+
luaFile = "lua/egress_auto_service_tags.lua",
48+
filterName = "envoy.lua.auto_service_tags",
49+
variables = mapOf(
50+
"SERVICE_TAG_METADATA_KEY" to properties.metadataKey,
51+
)
4352
)
44-
).build()
53+
} else {
54+
null
55+
}
56+
57+
private val luaEgressServiceTagPreferenceFilter: HttpFilter = createLuaFilter(
58+
luaFile = "lua/egress_service_tag_preference.lua",
59+
filterName = "envoy.lua.service_tag_preference",
60+
variables = mapOf(
61+
"SERVICE_TAG_METADATA_KEY" to properties.metadataKey,
62+
"SERVICE_TAG_HEADER" to properties.header,
63+
"SERVICE_TAG_PREFERENCE_HEADER" to properties.preferenceRouting.header,
64+
"DEFAULT_SERVICE_TAG_PREFERENCE_ENV" to properties.preferenceRouting.defaultPreferenceEnv,
65+
"DEFAULT_SERVICE_TAG_PREFERENCE_FALLBACK" to properties.preferenceRouting.defaultPreferenceFallback,
66+
)
67+
)
4568
}

0 commit comments

Comments
 (0)