Skip to content

Commit 332dd28

Browse files
0xGingi0xGingi
committed
fix tts/sst/custom provider price tracking & hide reasoning verbosity for models that don't support it
1 parent 65d843a commit 332dd28

8 files changed

Lines changed: 900 additions & 672 deletions

File tree

‎src/lib/db/queries/model-performance.ts‎

Lines changed: 273 additions & 216 deletions
Large diffs are not rendered by default.

‎src/lib/utils/model-capabilities.ts‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import type { NanoGPTModel } from '$lib/backend/models/nano-gpt';
22

33
export function supportsImages(model: NanoGPTModel): boolean {
4-
if (model.architecture?.output_modalities && !model.architecture.output_modalities.includes('image')) {
4+
if (
5+
model.architecture?.output_modalities &&
6+
!model.architecture.output_modalities.includes('image')
7+
) {
58
return false;
69
}
710
return true;
@@ -16,14 +19,21 @@ export function supportsVision(model: NanoGPTModel): boolean {
1619
}
1720

1821
export function isImageOnlyModel(model: NanoGPTModel): boolean {
19-
return (model.architecture?.output_modalities?.includes('image') &&
20-
model.architecture?.output_modalities?.length === 1) ?? false;
22+
return (
23+
(model.architecture?.output_modalities?.includes('image') &&
24+
model.architecture?.output_modalities?.length === 1) ??
25+
false
26+
);
2127
}
2228

2329
export function supportsReasoning(model: NanoGPTModel): boolean {
2430
return model.capabilities?.reasoning ?? false;
2531
}
2632

33+
export function supportsVerbosity(model: NanoGPTModel): boolean {
34+
return model.additionalParams?.verbosity !== undefined;
35+
}
36+
2737
export function supportsDocuments(model: NanoGPTModel): boolean {
2838
// For now, assume all models that support images also support documents
2939
// This can be refined later based on specific model capabilities

‎src/routes/account/analytics/+page.server.ts‎

Lines changed: 101 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -2,114 +2,120 @@ import { redirect } from '@sveltejs/kit';
22
import type { PageServerLoad } from './$types';
33
import { auth } from '$lib/auth';
44
import {
5-
getModelPerformanceStatsByUser,
6-
calculateAllModelPerformanceStats,
5+
getModelPerformanceStatsByUser,
6+
calculateAllModelPerformanceStats,
77
} from '$lib/db/queries/model-performance';
88

99
export const load: PageServerLoad = async ({ request }) => {
10-
const session = await auth.api.getSession({ headers: request.headers });
10+
const session = await auth.api.getSession({ headers: request.headers });
1111

12-
if (!session?.user?.id) {
13-
throw redirect(302, '/');
14-
}
12+
if (!session?.user?.id) {
13+
throw redirect(302, '/');
14+
}
1515

16-
const userId = session.user.id;
16+
const userId = session.user.id;
1717

18-
try {
19-
console.log(`[analytics] Loading analytics for user ${userId}`);
18+
try {
19+
console.log(`[analytics] Loading analytics for user ${userId}`);
2020

21-
// Always recalculate stats to ensure they're up-to-date
22-
const stats = await calculateAllModelPerformanceStats(userId);
23-
console.log(`[analytics] Calculated stats for ${stats.length} models`);
21+
// Always recalculate stats to ensure they're up-to-date
22+
const stats = await calculateAllModelPerformanceStats(userId);
23+
console.log(`[analytics] Calculated stats for ${stats.length} models`);
2424

25-
// Calculate additional insights
26-
const totalMessages = stats.reduce((sum, s) => sum + s.totalMessages, 0);
27-
const totalCost = stats.reduce((sum, s) => sum + s.totalCost, 0);
28-
const avgRating =
29-
stats.filter((s) => s.avgRating !== null).length > 0
30-
? stats.reduce((sum, s) => sum + (s.avgRating ?? 0), 0) /
31-
stats.filter((s) => s.avgRating !== null).length
32-
: null;
25+
// Calculate additional insights
26+
const totalMessages = stats.reduce((sum, s) => sum + s.totalMessages, 0);
27+
const totalCost = stats.reduce((sum, s) => {
28+
const cost = s.totalCost;
29+
if (cost === null || cost === undefined || isNaN(cost)) {
30+
return sum;
31+
}
32+
return sum + cost;
33+
}, 0);
34+
const avgRating =
35+
stats.filter((s) => s.avgRating !== null).length > 0
36+
? stats.reduce((sum, s) => sum + (s.avgRating ?? 0), 0) /
37+
stats.filter((s) => s.avgRating !== null).length
38+
: null;
3339

34-
// Find most used model
35-
const mostUsedModel = stats.length > 0
36-
? stats.reduce(
37-
(prev, current) => (current.totalMessages > prev.totalMessages ? current : prev),
38-
stats[0]!
39-
)
40-
: null;
40+
// Find most used model
41+
const mostUsedModel =
42+
stats.length > 0
43+
? stats.reduce(
44+
(prev, current) => (current.totalMessages > prev.totalMessages ? current : prev),
45+
stats[0]!
46+
)
47+
: null;
4148

42-
// Find best rated model (with at least 5 messages)
43-
const qualifiedModels = stats.filter((s) => s.totalMessages >= 5 && s.avgRating !== null);
44-
const bestRatedModel =
45-
qualifiedModels.length > 0
46-
? qualifiedModels.reduce((prev, current) =>
47-
(current.avgRating ?? 0) > (prev.avgRating ?? 0) ? current : prev
48-
)
49-
: null;
49+
// Find best rated model (with at least 5 messages)
50+
const qualifiedModels = stats.filter((s) => s.totalMessages >= 5 && s.avgRating !== null);
51+
const bestRatedModel =
52+
qualifiedModels.length > 0
53+
? qualifiedModels.reduce((prev, current) =>
54+
(current.avgRating ?? 0) > (prev.avgRating ?? 0) ? current : prev
55+
)
56+
: null;
5057

51-
// Find most cost-effective model (lowest cost per message with at least 5 messages)
52-
const modelsWithCost = stats.filter((s) => s.totalMessages >= 5 && s.totalCost > 0);
53-
const mostCostEffective =
54-
modelsWithCost.length > 0
55-
? modelsWithCost.reduce((prev, current) => {
56-
const prevCostPerMsg = prev.totalCost / prev.totalMessages;
57-
const currentCostPerMsg = current.totalCost / current.totalMessages;
58-
return currentCostPerMsg < prevCostPerMsg ? current : prev;
59-
})
60-
: null;
58+
// Find most cost-effective model (lowest cost per message with at least 5 messages)
59+
const modelsWithCost = stats.filter((s) => s.totalMessages >= 5 && s.totalCost > 0);
60+
const mostCostEffective =
61+
modelsWithCost.length > 0
62+
? modelsWithCost.reduce((prev, current) => {
63+
const prevCostPerMsg = prev.totalCost / prev.totalMessages;
64+
const currentCostPerMsg = current.totalCost / current.totalMessages;
65+
return currentCostPerMsg < prevCostPerMsg ? current : prev;
66+
})
67+
: null;
6168

62-
// Find fastest model by tokens/sec (requires avgTokens and avgResponseTime with at least 5 messages)
63-
const modelsWithSpeed = stats.filter(
64-
(s) =>
65-
s.totalMessages >= 5 &&
66-
s.avgTokens !== null &&
67-
s.avgTokens !== undefined &&
68-
s.avgResponseTime !== null &&
69-
s.avgResponseTime !== undefined &&
70-
s.avgResponseTime > 0
71-
);
69+
// Find fastest model by tokens/sec (requires avgTokens and avgResponseTime with at least 5 messages)
70+
const modelsWithSpeed = stats.filter(
71+
(s) =>
72+
s.totalMessages >= 5 &&
73+
s.avgTokens !== null &&
74+
s.avgTokens !== undefined &&
75+
s.avgResponseTime !== null &&
76+
s.avgResponseTime !== undefined &&
77+
s.avgResponseTime > 0
78+
);
7279

73-
const fastestModel =
74-
modelsWithSpeed.length > 0
75-
? modelsWithSpeed.reduce((prev, current) => {
76-
const prevSpeed =
77-
(prev.avgTokens ?? 0) /
78-
(((prev.avgResponseTime ?? 0) / 1000) || 1); // tokens per second
79-
const currentSpeed =
80-
(current.avgTokens ?? 0) /
81-
(((current.avgResponseTime ?? 0) / 1000) || 1);
82-
return currentSpeed > prevSpeed ? current : prev;
83-
})
84-
: null;
80+
const fastestModel =
81+
modelsWithSpeed.length > 0
82+
? modelsWithSpeed.reduce((prev, current) => {
83+
const prevSpeed = (prev.avgTokens ?? 0) / ((prev.avgResponseTime ?? 0) / 1000 || 1); // tokens per second
84+
const currentSpeed =
85+
(current.avgTokens ?? 0) / ((current.avgResponseTime ?? 0) / 1000 || 1);
86+
return currentSpeed > prevSpeed ? current : prev;
87+
})
88+
: null;
8589

86-
console.log(`[analytics] Generated insights: ${totalMessages} messages, $${totalCost.toFixed(2)} cost`);
90+
console.log(
91+
`[analytics] Generated insights: ${totalMessages} messages, $${totalCost.toFixed(2)} cost`
92+
);
8793

88-
return {
89-
stats,
90-
insights: {
91-
totalMessages,
92-
totalCost,
93-
avgRating,
94-
mostUsedModel,
95-
bestRatedModel,
96-
mostCostEffective,
97-
fastestModel,
98-
},
99-
};
100-
} catch (err) {
101-
console.error(`[analytics] Error loading analytics for user ${userId}:`, err);
102-
// Return empty data on error rather than crashing
103-
return {
104-
stats: [],
105-
insights: {
106-
totalMessages: 0,
107-
totalCost: 0,
108-
avgRating: null,
109-
mostUsedModel: null,
110-
bestRatedModel: null,
111-
mostCostEffective: null,
112-
},
113-
};
114-
}
94+
return {
95+
stats,
96+
insights: {
97+
totalMessages,
98+
totalCost,
99+
avgRating,
100+
mostUsedModel,
101+
bestRatedModel,
102+
mostCostEffective,
103+
fastestModel,
104+
},
105+
};
106+
} catch (err) {
107+
console.error(`[analytics] Error loading analytics for user ${userId}:`, err);
108+
// Return empty data on error rather than crashing
109+
return {
110+
stats: [],
111+
insights: {
112+
totalMessages: 0,
113+
totalCost: 0,
114+
avgRating: null,
115+
mostUsedModel: null,
116+
bestRatedModel: null,
117+
mostCostEffective: null,
118+
},
119+
};
120+
}
115121
};

‎src/routes/account/analytics/+page.svelte‎

Lines changed: 45 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -20,42 +20,42 @@
2020
let isRefreshing = $state(false);
2121
2222
const sortedStats = $derived.by(() => {
23-
const stats = [...data.stats];
23+
const stats = [...data.stats].filter((s) => s.totalMessages > 0);
2424
stats.sort((a, b) => {
2525
let aVal: number;
2626
let bVal: number;
2727
28-
switch (sortColumn) {
29-
case 'model':
30-
return sortDirection === 'asc'
31-
? a.modelId.localeCompare(b.modelId)
32-
: b.modelId.localeCompare(a.modelId);
33-
case 'rating':
34-
aVal = a.avgRating ?? 0;
35-
bVal = b.avgRating ?? 0;
36-
break;
37-
case 'uses':
38-
aVal = a.totalMessages;
39-
bVal = b.totalMessages;
40-
break;
41-
case 'cost':
42-
aVal = a.totalMessages > 0 ? a.totalCost / a.totalMessages : 0;
43-
bVal = b.totalMessages > 0 ? b.totalCost / b.totalMessages : 0;
44-
break;
45-
case 'thumbsRatio':
46-
const aTotal = a.thumbsUpCount + a.thumbsDownCount;
47-
const bTotal = b.thumbsUpCount + b.thumbsDownCount;
48-
aVal = aTotal > 0 ? a.thumbsUpCount / aTotal : 0;
49-
bVal = bTotal > 0 ? b.thumbsUpCount / bTotal : 0;
50-
break;
51-
case 'speed':
52-
aVal = getTokensPerSecond(a);
53-
bVal = getTokensPerSecond(b);
54-
break;
55-
default:
56-
aVal = 0;
57-
bVal = 0;
58-
}
28+
switch (sortColumn) {
29+
case 'model':
30+
return sortDirection === 'asc'
31+
? a.modelId.localeCompare(b.modelId)
32+
: b.modelId.localeCompare(a.modelId);
33+
case 'rating':
34+
aVal = a.avgRating ?? 0;
35+
bVal = b.avgRating ?? 0;
36+
break;
37+
case 'uses':
38+
aVal = a.totalMessages;
39+
bVal = b.totalMessages;
40+
break;
41+
case 'cost':
42+
aVal = a.totalMessages > 0 && !isNaN(a.totalCost) ? a.totalCost / a.totalMessages : 0;
43+
bVal = b.totalMessages > 0 && !isNaN(b.totalCost) ? b.totalCost / b.totalMessages : 0;
44+
break;
45+
case 'thumbsRatio':
46+
const aTotal = a.thumbsUpCount + a.thumbsDownCount;
47+
const bTotal = b.thumbsUpCount + b.thumbsDownCount;
48+
aVal = aTotal > 0 ? a.thumbsUpCount / aTotal : 0;
49+
bVal = bTotal > 0 ? b.thumbsUpCount / bTotal : 0;
50+
break;
51+
case 'speed':
52+
aVal = getTokensPerSecond(a);
53+
bVal = getTokensPerSecond(b);
54+
break;
55+
default:
56+
aVal = 0;
57+
bVal = 0;
58+
}
5959
6060
return sortDirection === 'asc' ? aVal - bVal : bVal - aVal;
6161
});
@@ -84,9 +84,15 @@
8484
}
8585
8686
function formatCost(cost: number): string {
87+
if (isNaN(cost)) return 'N/A';
8788
return `$${cost.toFixed(6)}`;
8889
}
8990
91+
function formatCostSafe(cost: number | null | undefined): string {
92+
if (cost === null || cost === undefined || isNaN(cost)) return 'N/A';
93+
return `$${cost.toFixed(2)}`;
94+
}
95+
9096
function getThumbsRatio(upCount: number, downCount: number): string {
9197
const total = upCount + downCount;
9298
if (total === 0) return 'N/A';
@@ -164,7 +170,7 @@
164170
<div class="flex items-center justify-between">
165171
<div>
166172
<p class="text-muted-foreground text-sm">Total Cost</p>
167-
<p class="text-2xl font-bold">${data.insights.totalCost.toFixed(2)}</p>
173+
<p class="text-2xl font-bold">{formatCostSafe(data.insights.totalCost)}</p>
168174
</div>
169175
<DollarSignIcon class="text-muted-foreground size-8" />
170176
</div>
@@ -209,7 +215,7 @@
209215
</div>
210216
{/if}
211217

212-
{#if data.insights.mostCostEffective}
218+
{#if data.insights.mostCostEffective && !isNaN(data.insights.mostCostEffective.totalCost)}
213219
<div class="bg-card rounded-lg border p-6">
214220
<h3 class="mb-2 text-lg font-semibold">Most Cost-Effective</h3>
215221
<p class="text-muted-foreground text-sm">
@@ -348,7 +354,11 @@
348354
</td>
349355
<td class="py-3 pr-4 text-center">{stat.totalMessages}</td>
350356
<td class="py-3 pr-4 text-center">
351-
{formatCost(stat.totalCost / stat.totalMessages)}
357+
{#if !isNaN(stat.totalCost) && stat.totalMessages > 0}
358+
{formatCost(stat.totalCost / stat.totalMessages)}
359+
{:else}
360+
<span class="text-muted-foreground text-sm">N/A</span>
361+
{/if}
352362
</td>
353363
<td class="py-3 pr-4">
354364
<div class="flex items-center justify-center gap-2">

0 commit comments

Comments
 (0)