Skip to content

Commit 5962f10

Browse files
andy-fongEItanyadavidjumani
authored
[1.18] AI backport of (#10494) and (#10640) (#10649)
Co-authored-by: Eitan Yarmush <[email protected]> Co-authored-by: David Jumani <[email protected]>
1 parent 5d4dac8 commit 5962f10

File tree

13 files changed

+590
-421
lines changed

13 files changed

+590
-421
lines changed

‎changelog/v1.18.10/ai-semantic-caching-distance.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
changelog:
2+
- type: FIX
3+
issueLink: https://github.com/solo-io/solo-projects/issues/7440
4+
resolvesIssue: false
5+
description: >
6+
Fixes an issue where the ai semantic caching distance is not being set correctly in the cache. Also
7+
move the distance threshold to the cache configuration, rather than per datastore.
8+
9+

‎changelog/v1.18.10/ai-upstream-custom-hostname.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
changelog:
2+
- type: FIX
3+
issueLink: https://github.com/solo-io/solo-projects/issues/7831
4+
resolvesIssue: false
5+
description: >
6+
Added optional `hostname` to the AI Upstream CustomHost setting to override SNI and host header value

‎docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/enterprise/options/ai/ai.proto.sk.md

+6-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/helm/gloo/crds/gateway.solo.io_v1_RouteOption.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,8 @@ spec:
275275
type: boolean
276276
type: object
277277
type: object
278+
distanceThreshold:
279+
type: number
278280
embedding:
279281
properties:
280282
azureOpenai:

‎install/helm/gloo/crds/gateway.solo.io_v1_RouteTable.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,8 @@ spec:
385385
type: boolean
386386
type: object
387387
type: object
388+
distanceThreshold:
389+
type: number
388390
embedding:
389391
properties:
390392
azureOpenai:

‎install/helm/gloo/crds/gateway.solo.io_v1_VirtualService.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -3492,6 +3492,8 @@ spec:
34923492
type: boolean
34933493
type: object
34943494
type: object
3495+
distanceThreshold:
3496+
type: number
34953497
embedding:
34963498
properties:
34973499
azureOpenai:

‎install/helm/gloo/crds/gloo.solo.io_v1_Upstream.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ spec:
4747
properties:
4848
host:
4949
type: string
50+
hostname:
51+
nullable: true
52+
type: string
5053
port:
5154
maximum: 4294967295
5255
minimum: 0
@@ -121,6 +124,9 @@ spec:
121124
properties:
122125
host:
123126
type: string
127+
hostname:
128+
nullable: true
129+
type: string
124130
port:
125131
maximum: 4294967295
126132
minimum: 0
@@ -157,6 +163,9 @@ spec:
157163
properties:
158164
host:
159165
type: string
166+
hostname:
167+
nullable: true
168+
type: string
160169
port:
161170
maximum: 4294967295
162171
minimum: 0
@@ -231,6 +240,9 @@ spec:
231240
properties:
232241
host:
233242
type: string
243+
hostname:
244+
nullable: true
245+
type: string
234246
port:
235247
maximum: 4294967295
236248
minimum: 0
@@ -259,6 +271,9 @@ spec:
259271
properties:
260272
host:
261273
type: string
274+
hostname:
275+
nullable: true
276+
type: string
262277
port:
263278
maximum: 4294967295
264279
minimum: 0
@@ -322,6 +337,9 @@ spec:
322337
properties:
323338
host:
324339
type: string
340+
hostname:
341+
nullable: true
342+
type: string
325343
port:
326344
maximum: 4294967295
327345
minimum: 0

‎projects/gloo/api/v1/enterprise/options/ai/ai.proto

+14-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ option go_package = "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise
44

55
import "github.com/solo-io/solo-kit/api/v1/ref.proto";
66
import "google/protobuf/struct.proto";
7+
import "google/protobuf/wrappers.proto";
78
import "extproto/ext.proto";
89
option (extproto.equal_all) = true;
910
option (extproto.hash_all) = true;
@@ -60,10 +61,13 @@ message UpstreamSpec {
6061
// Send requests to a custom host and port, such as to proxy the request,
6162
// or to use a different backend that is API-compliant with the upstream version.
6263
message CustomHost {
63-
// Custom host to send the traffic requests to.
64+
// Custom host or IP address to send the traffic requests to.
6465
string host = 1;
6566
// Custom port to send the traffic requests to.
6667
uint32 port = 2;
68+
// Optional: hostname used to set the SNI (if is secure connection) and the host request header.
69+
// If hostname is not set, host will be used instead
70+
google.protobuf.StringValue hostname = 3;
6771
}
6872

6973
// Settings for the [OpenAI](https://platform.openai.com/docs/overview) LLM provider.
@@ -431,6 +435,7 @@ message SemanticCache {
431435
//
432436
// +kubebuilder:validation:Minimum=0
433437
// +kubebuilder:validation:Maximum=1
438+
// Deprecated: Prefer setting the distance threshold in the RouteOptions.SemanticCache resource.
434439
float score_threshold = 2;
435440
}
436441

@@ -474,6 +479,14 @@ message SemanticCache {
474479
uint32 ttl = 3;
475480
// The caching mode to use for the request and response lifecycle. Supported values include `READ_WRITE` or `READ_ONLY`.
476481
Mode mode = 4;
482+
483+
// Distance score threshold value between 0.0 and 1.0 that determines how similar
484+
// two queries must be in order to return a cached result.
485+
// The lower the number, the more similar the queries must be for a cache hit.
486+
//
487+
// +kubebuilder:validation:Minimum=0
488+
// +kubebuilder:validation:Maximum=1
489+
float distance_threshold = 5;
477490
}
478491

479492
// [Retrieval augmented generation (RAG)](https://research.ibm.com/blog/retrieval-augmented-generation-RAG)

‎projects/gloo/pkg/api/v1/enterprise/options/ai/ai.pb.clone.go

+10
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎projects/gloo/pkg/api/v1/enterprise/options/ai/ai.pb.equal.go

+14
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)