Skip to content

Commit 3cda82d

Browse files
connortbotchitalian
connortbot
and
chitalian
authored
Cache -> RMT CH [backend] (#3757)
* Track cache hits in rmt, update cache metric api * web API for new cache logging * types * rev stuff * revert admin changes?? * revert admin changes?? * linting --------- Co-authored-by: Justin <justintorre75@gmail.com>
1 parent f6c550e commit 3cda82d

File tree

43 files changed

+1544
-1207
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1544
-1207
lines changed

‎bifrost/lib/clients/jawnTypes/private.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1043,6 +1043,7 @@ Json: JsonObject;
10431043
scores_column?: components["schemas"]["Partial_TextOperators_"];
10441044
request_body?: components["schemas"]["Partial_VectorOperators_"];
10451045
response_body?: components["schemas"]["Partial_VectorOperators_"];
1046+
cache_enabled?: components["schemas"]["Partial_BooleanOperators_"];
10461047
};
10471048
/** @description Make all properties in T optional */
10481049
Partial_SessionsRequestResponseRMTToOperators_: {
@@ -1338,6 +1339,8 @@ Json: JsonObject;
13381339
assets: string[];
13391340
target_url: string;
13401341
model: string;
1342+
cache_reference_id: string | null;
1343+
cache_enabled: boolean;
13411344
};
13421345
"ResultSuccess_HeliconeRequest-Array_": {
13431346
data: components["schemas"]["HeliconeRequest"][];
@@ -1764,6 +1767,13 @@ Json: JsonObject;
17641767
id: string;
17651768
};
17661769
request: {
1770+
cacheReferenceId?: string;
1771+
cacheControl?: string;
1772+
/** Format: double */
1773+
cacheBucketMaxSize?: number;
1774+
/** Format: double */
1775+
cacheSeed?: number;
1776+
cacheEnabled?: boolean;
17671777
experimentRowIndex?: string;
17681778
experimentColumnId?: string;
17691779
heliconeTemplate?: components["schemas"]["TemplateWithInputs"];

‎bifrost/lib/clients/jawnTypes/public.ts‎

Lines changed: 102 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -217,18 +217,6 @@ export interface paths {
217217
"/v2/experiment/{experimentId}/{requestId}/{scoreKey}": {
218218
get: operations["GetExperimentScore"];
219219
};
220-
"/v1/trace/custom/v1/log": {
221-
post: operations["LogCustomTraceLegacy"];
222-
};
223-
"/v1/trace/custom/log": {
224-
post: operations["LogCustomTrace"];
225-
};
226-
"/v1/trace/log": {
227-
post: operations["LogTrace"];
228-
};
229-
"/v1/trace/log-python": {
230-
post: operations["LogPythonTrace"];
231-
};
232220
"/v1/stripe/subscription/cost-for-prompts": {
233221
get: operations["GetCostForPrompts"];
234222
};
@@ -278,6 +266,18 @@ export interface paths {
278266
"/v1/stripe/webhook": {
279267
post: operations["HandleStripeWebhook"];
280268
};
269+
"/v1/trace/custom/v1/log": {
270+
post: operations["LogCustomTraceLegacy"];
271+
};
272+
"/v1/trace/custom/log": {
273+
post: operations["LogCustomTrace"];
274+
};
275+
"/v1/trace/log": {
276+
post: operations["LogTrace"];
277+
};
278+
"/v1/trace/log-python": {
279+
post: operations["LogPythonTrace"];
280+
};
281281
"/v1/session/has-session": {
282282
get: operations["HasSession"];
283283
};
@@ -717,6 +717,7 @@ export interface components {
717717
scores_column?: components["schemas"]["Partial_TextOperators_"];
718718
request_body?: components["schemas"]["Partial_VectorOperators_"];
719719
response_body?: components["schemas"]["Partial_VectorOperators_"];
720+
cache_enabled?: components["schemas"]["Partial_BooleanOperators_"];
720721
};
721722
/** @description From T, pick a set of properties whose keys are in the union K */
722723
"Pick_FilterLeaf.user_metrics-or-request_response_rmt_": {
@@ -1322,6 +1323,8 @@ export interface components {
13221323
assets: string[];
13231324
target_url: string;
13241325
model: string;
1326+
cache_reference_id: string | null;
1327+
cache_enabled: boolean;
13251328
};
13261329
"ResultSuccess_HeliconeRequest-Array_": {
13271330
data: components["schemas"]["HeliconeRequest"][];
@@ -1687,6 +1690,41 @@ Json: JsonObject;
16871690
error: null;
16881691
};
16891692
"Result_ScoreV2-or-null.string_": components["schemas"]["ResultSuccess_ScoreV2-or-null_"] | components["schemas"]["ResultError_string_"];
1693+
UpgradeToProRequest: {
1694+
addons?: {
1695+
evals?: boolean;
1696+
experiments?: boolean;
1697+
prompts?: boolean;
1698+
alerts?: boolean;
1699+
};
1700+
/** Format: double */
1701+
seats?: number;
1702+
/** @enum {string} */
1703+
ui_mode?: "embedded" | "hosted";
1704+
};
1705+
UpgradeToTeamBundleRequest: {
1706+
/** @enum {string} */
1707+
ui_mode?: "embedded" | "hosted";
1708+
};
1709+
LLMUsage: {
1710+
model: string;
1711+
provider: string;
1712+
/** Format: double */
1713+
prompt_tokens: number;
1714+
/** Format: double */
1715+
completion_tokens: number;
1716+
/** Format: double */
1717+
total_count: number;
1718+
/** Format: double */
1719+
amount: number;
1720+
description: string;
1721+
totalCost: {
1722+
/** Format: double */
1723+
prompt_token: number;
1724+
/** Format: double */
1725+
completion_token: number;
1726+
};
1727+
};
16901728
OTELTrace: {
16911729
resourceSpans: {
16921730
scopeSpans: {
@@ -1743,41 +1781,6 @@ Json: JsonObject;
17431781
};
17441782
}[];
17451783
};
1746-
UpgradeToProRequest: {
1747-
addons?: {
1748-
evals?: boolean;
1749-
experiments?: boolean;
1750-
prompts?: boolean;
1751-
alerts?: boolean;
1752-
};
1753-
/** Format: double */
1754-
seats?: number;
1755-
/** @enum {string} */
1756-
ui_mode?: "embedded" | "hosted";
1757-
};
1758-
UpgradeToTeamBundleRequest: {
1759-
/** @enum {string} */
1760-
ui_mode?: "embedded" | "hosted";
1761-
};
1762-
LLMUsage: {
1763-
model: string;
1764-
provider: string;
1765-
/** Format: double */
1766-
prompt_tokens: number;
1767-
/** Format: double */
1768-
completion_tokens: number;
1769-
/** Format: double */
1770-
total_count: number;
1771-
/** Format: double */
1772-
amount: number;
1773-
description: string;
1774-
totalCost: {
1775-
/** Format: double */
1776-
prompt_token: number;
1777-
/** Format: double */
1778-
completion_token: number;
1779-
};
1780-
};
17811784
SessionResult: {
17821785
created_at: string;
17831786
latest_request_created_at: string;
@@ -4014,58 +4017,6 @@ export interface operations {
40144017
};
40154018
};
40164019
};
4017-
LogCustomTraceLegacy: {
4018-
requestBody: {
4019-
content: {
4020-
"application/json": unknown;
4021-
};
4022-
};
4023-
responses: {
4024-
/** @description No content */
4025-
204: {
4026-
content: never;
4027-
};
4028-
};
4029-
};
4030-
LogCustomTrace: {
4031-
requestBody: {
4032-
content: {
4033-
"application/json": unknown;
4034-
};
4035-
};
4036-
responses: {
4037-
/** @description No content */
4038-
204: {
4039-
content: never;
4040-
};
4041-
};
4042-
};
4043-
LogTrace: {
4044-
requestBody: {
4045-
content: {
4046-
"application/json": components["schemas"]["OTELTrace"];
4047-
};
4048-
};
4049-
responses: {
4050-
/** @description No content */
4051-
204: {
4052-
content: never;
4053-
};
4054-
};
4055-
};
4056-
LogPythonTrace: {
4057-
requestBody: {
4058-
content: {
4059-
"application/json": unknown;
4060-
};
4061-
};
4062-
responses: {
4063-
/** @description No content */
4064-
204: {
4065-
content: never;
4066-
};
4067-
};
4068-
};
40694020
GetCostForPrompts: {
40704021
responses: {
40714022
/** @description Ok */
@@ -4315,6 +4266,58 @@ export interface operations {
43154266
};
43164267
};
43174268
};
4269+
LogCustomTraceLegacy: {
4270+
requestBody: {
4271+
content: {
4272+
"application/json": unknown;
4273+
};
4274+
};
4275+
responses: {
4276+
/** @description No content */
4277+
204: {
4278+
content: never;
4279+
};
4280+
};
4281+
};
4282+
LogCustomTrace: {
4283+
requestBody: {
4284+
content: {
4285+
"application/json": unknown;
4286+
};
4287+
};
4288+
responses: {
4289+
/** @description No content */
4290+
204: {
4291+
content: never;
4292+
};
4293+
};
4294+
};
4295+
LogTrace: {
4296+
requestBody: {
4297+
content: {
4298+
"application/json": components["schemas"]["OTELTrace"];
4299+
};
4300+
};
4301+
responses: {
4302+
/** @description No content */
4303+
204: {
4304+
content: never;
4305+
};
4306+
};
4307+
};
4308+
LogPythonTrace: {
4309+
requestBody: {
4310+
content: {
4311+
"application/json": unknown;
4312+
};
4313+
};
4314+
responses: {
4315+
/** @description No content */
4316+
204: {
4317+
content: never;
4318+
};
4319+
};
4320+
};
43184321
HasSession: {
43194322
responses: {
43204323
/** @description Ok */

‎bifrost/packages/llm-mapper/types.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { ProviderName } from "../cost/providers/mappings";
22

3+
export const DEFAULT_UUID = "00000000-0000-0000-0000-000000000000";
4+
35
export type MapperType =
46
| "openai-chat"
57
| "openai-response"
@@ -208,6 +210,8 @@ type HeliconeMetadata = {
208210
requestId: string;
209211
path: string;
210212
countryCode: string | null;
213+
cacheEnabled: boolean;
214+
cacheReferenceId: string | null;
211215
createdAt: string;
212216
totalTokens: number | null;
213217
promptTokens: number | null;
@@ -336,4 +340,6 @@ export interface HeliconeRequest {
336340
assets: Array<string>;
337341
target_url: string;
338342
model: string;
343+
cache_reference_id: string | null;
344+
cache_enabled: boolean;
339345
}

‎bifrost/packages/llm-mapper/utils/getMappedContent.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ const metaDataFromHeliconeRequest = (
6565
return {
6666
requestId: heliconeRequest.request_id,
6767
countryCode: heliconeRequest.country_code,
68+
cacheEnabled: heliconeRequest.cache_enabled ?? false,
69+
cacheReferenceId: heliconeRequest.cache_reference_id ?? null,
6870
cost: modelCost({
6971
provider: heliconeRequest.provider,
7072
model: model,

‎clickhouse/migrations/schema_47_cache_to_request_response_rmt.sql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
ALTER TABLE request_response_rmt
2+
ADD COLUMN IF NOT EXISTS cache_reference_id UUID
3+
AFTER completion_audio_tokens,
4+
ADD COLUMN IF NOT EXISTS cache_enabled Bool DEFAULT false
5+
AFTER country_code;

0 commit comments

Comments
 (0)