Skip to content

Commit 3f5f611

Browse files
caswelltomclaude
andcommitted
Add token usage & cost analytics (v1.1.0)
- New DB columns: prompt_tokens, completion_tokens, model_name on msgs table - OpenAI: stream_options include_usage captures token counts from final SSE chunk - Claude: captures input tokens from message_start, output from message_delta - conversation_manager: add_message() stores granular token + model data - New token_cost_manager: rate card (OpenAI/Claude/DeepSeek), cost estimation, formatting - New token_analytics.php: admin-only page with cost by model, per-student breakdown, rate card - Analytics dashboard: admin link to Token Usage & Cost page - Welcome screen improvements: highlights voice/quiz features (v1.0.20) - Fix SOLA_NEXT tags appearing raw in Practice Speaking transcript (v1.0.21) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 2362ba5 commit 3f5f611

16 files changed

+701
-9
lines changed

amd/build/ui.min.js.map

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

amd/build/voice.min.js.map

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

analytics.php

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@
8080
'url_7' => (new moodle_url('/local/ai_course_assistant/analytics.php', ['courseid' => $courseid, 'range' => 7]))->out(false),
8181
'url_30' => (new moodle_url('/local/ai_course_assistant/analytics.php', ['courseid' => $courseid, 'range' => 30]))->out(false),
8282
'url_all' => (new moodle_url('/local/ai_course_assistant/analytics.php', ['courseid' => $courseid, 'range' => 0]))->out(false),
83+
'token_analytics_url' => has_capability('moodle/site:config', context_system::instance())
84+
? (new moodle_url('/local/ai_course_assistant/token_analytics.php'))->out(false)
85+
: null,
8386
];
8487

8588
echo $OUTPUT->header();

classes/conversation_manager.php

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,10 @@ public static function add_message(
7777
string $role,
7878
string $message,
7979
int $tokensused = 0,
80-
string $provider = ''
80+
string $provider = '',
81+
?int $prompttokens = null,
82+
?int $completiontokens = null,
83+
?string $modelname = null
8184
): int {
8285
global $DB;
8386

@@ -87,7 +90,13 @@ public static function add_message(
8790
$record->courseid = $courseid;
8891
$record->role = $role;
8992
$record->message = $message;
90-
$record->tokens_used = $tokensused;
93+
// Keep tokens_used as the sum for backward compatibility with existing analytics queries.
94+
$record->tokens_used = ($prompttokens !== null && $completiontokens !== null)
95+
? ($prompttokens + $completiontokens)
96+
: $tokensused;
97+
$record->prompt_tokens = $prompttokens;
98+
$record->completion_tokens = $completiontokens;
99+
$record->model_name = ($role === 'assistant' && $modelname !== null) ? $modelname : null;
91100
$record->provider = ($role === 'assistant' && $provider !== '') ? $provider : null;
92101
$record->timecreated = time();
93102

classes/provider/base_provider.php

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,18 @@ protected function check_http_error(int $httpcode, string $response): void {
188188
throw new \moodle_exception('chat:error', 'local_ai_course_assistant', '', null, "HTTP {$httpcode}: {$response}");
189189
}
190190

191+
/**
192+
* Get token usage from the last streaming call.
193+
*
194+
* Default implementation returns null. Providers that support usage reporting
195+
* (OpenAI-compatible with stream_options, Claude) override this.
196+
*
197+
* @return array|null
198+
*/
199+
public function get_last_token_usage(): ?array {
200+
return null;
201+
}
202+
191203
/**
192204
* Factory method to create a provider from plugin config, with optional per-course overrides.
193205
*

classes/provider/claude_provider.php

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,18 @@ class claude_provider extends base_provider {
3030
/** @var string Anthropic API version */
3131
private const API_VERSION = '2023-06-01';
3232

33+
/** @var array|null Token usage from the last streaming call */
34+
private ?array $last_token_usage = null;
35+
36+
/**
37+
* Get token usage from the last streaming call.
38+
*
39+
* @return array|null ['prompt_tokens', 'completion_tokens', 'model'] or null.
40+
*/
41+
public function get_last_token_usage(): ?array {
42+
return $this->last_token_usage;
43+
}
44+
3345
protected function get_default_model(): string {
3446
return 'claude-sonnet-4-5-20250929';
3547
}
@@ -99,6 +111,7 @@ public function chat_completion_stream(string $systemprompt, array $messages, ca
99111
$body = $this->build_body($systemprompt, $messages, true, $options);
100112

101113
$buffer = '';
114+
$this->last_token_usage = null;
102115

103116
$this->http_post_stream($url, $this->get_headers(), $body, function ($data) use ($callback, &$buffer) {
104117
$buffer .= $data;
@@ -123,8 +136,26 @@ public function chat_completion_stream(string $systemprompt, array $messages, ca
123136
continue;
124137
}
125138

126-
// Anthropic streams content_block_delta events.
127-
if (($event['type'] ?? '') === 'content_block_delta') {
139+
$eventtype = $event['type'] ?? '';
140+
141+
// message_start carries input token count and the model name.
142+
if ($eventtype === 'message_start') {
143+
$this->last_token_usage = [
144+
'prompt_tokens' => (int) ($event['message']['usage']['input_tokens'] ?? 0),
145+
'completion_tokens' => 0,
146+
'model' => $event['message']['model'] ?? $this->model,
147+
];
148+
}
149+
150+
// message_delta carries output (completion) token count.
151+
if ($eventtype === 'message_delta' && isset($event['usage']['output_tokens'])) {
152+
if ($this->last_token_usage !== null) {
153+
$this->last_token_usage['completion_tokens'] = (int) $event['usage']['output_tokens'];
154+
}
155+
}
156+
157+
// content_block_delta carries the actual text chunks.
158+
if ($eventtype === 'content_block_delta') {
128159
$text = $event['delta']['text'] ?? '';
129160
if ($text !== '') {
130161
$callback($text);

classes/provider/openai_compatible_provider.php

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,18 @@
2828
*/
2929
abstract class openai_compatible_provider extends base_provider {
3030

31+
/** @var array|null Token usage from the last streaming call */
32+
protected ?array $last_token_usage = null;
33+
34+
/**
35+
* Get token usage from the last streaming call.
36+
*
37+
* @return array|null ['prompt_tokens', 'completion_tokens', 'model'] or null.
38+
*/
39+
public function get_last_token_usage(): ?array {
40+
return $this->last_token_usage;
41+
}
42+
3143
/**
3244
* Get the chat completions endpoint path.
3345
*
@@ -87,6 +99,8 @@ protected function build_body(string $systemprompt, array $messages, bool $strea
8799

88100
if ($stream) {
89101
$body['stream'] = true;
102+
// Request usage data in the final streaming chunk.
103+
$body['stream_options'] = ['include_usage' => true];
90104
}
91105

92106
return json_encode($body);
@@ -110,6 +124,7 @@ public function chat_completion_stream(string $systemprompt, array $messages, ca
110124
$body = $this->build_body($systemprompt, $messages, true, $options);
111125

112126
$buffer = '';
127+
$this->last_token_usage = null;
113128

114129
$this->http_post_stream($url, $this->get_headers(), $body, function ($data) use ($callback, &$buffer) {
115130
$buffer .= $data;
@@ -133,6 +148,16 @@ public function chat_completion_stream(string $systemprompt, array $messages, ca
133148
continue;
134149
}
135150

151+
// Capture usage from the final usage-only chunk (stream_options: include_usage: true).
152+
// This chunk has empty choices[] and a populated usage object.
153+
if (!empty($event['usage'])) {
154+
$this->last_token_usage = [
155+
'prompt_tokens' => (int) ($event['usage']['prompt_tokens'] ?? 0),
156+
'completion_tokens' => (int) ($event['usage']['completion_tokens'] ?? 0),
157+
'model' => $event['model'] ?? $this->model,
158+
];
159+
}
160+
136161
$content = $event['choices'][0]['delta']['content'] ?? '';
137162
if ($content !== '') {
138163
$callback($content);

classes/provider/provider_interface.php

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,15 @@ public function chat_completion(string $systemprompt, array $messages, array $op
4646
* @throws \moodle_exception On API errors.
4747
*/
4848
public function chat_completion_stream(string $systemprompt, array $messages, callable $callback, array $options = []): void;
49+
50+
/**
51+
* Get token usage from the last streaming call.
52+
*
53+
* Must be called immediately after chat_completion_stream() completes.
54+
* Returns null if the provider does not report usage data.
55+
*
56+
* @return array|null Array with keys 'prompt_tokens' (int), 'completion_tokens' (int),
57+
* 'model' (string), or null if not available.
58+
*/
59+
public function get_last_token_usage(): ?array;
4960
}

classes/token_cost_manager.php

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
<?php
2+
// This file is part of Moodle - http://moodle.org/
3+
//
4+
// Moodle is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Moodle is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16+
17+
namespace local_ai_course_assistant;
18+
19+
/**
20+
* Token cost manager — rate cards and cost estimation.
21+
*
22+
* Rates are USD per 1,000,000 tokens (industry standard as of early 2026).
23+
* Model strings are matched by prefix (longest match wins) so new dated
24+
* model variants (e.g. gpt-4o-2024-11-20) are covered automatically.
25+
*
26+
* Update the rate card when providers change pricing.
27+
*
28+
* @package local_ai_course_assistant
29+
* @copyright 2025 AI Course Assistant
30+
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
31+
*/
32+
class token_cost_manager {
33+
34+
/**
35+
* Rate card: USD per 1,000,000 tokens.
36+
* Keys are model prefix strings matched with str_starts_with.
37+
* Values: ['input' => float, 'output' => float].
38+
*
39+
* Prices sourced from provider pricing pages (March 2026).
40+
*/
41+
private static array $rate_cards = [
42+
// ── OpenAI ────────────────────────────────────────────────────────────
43+
'gpt-4o-mini' => ['input' => 0.15, 'output' => 0.60],
44+
'gpt-4o' => ['input' => 2.50, 'output' => 10.00],
45+
'gpt-4-turbo' => ['input' => 10.00, 'output' => 30.00],
46+
'gpt-4' => ['input' => 30.00, 'output' => 60.00],
47+
'gpt-3.5-turbo' => ['input' => 0.50, 'output' => 1.50],
48+
'o1-mini' => ['input' => 3.00, 'output' => 12.00],
49+
'o1-preview' => ['input' => 15.00, 'output' => 60.00],
50+
'o1' => ['input' => 15.00, 'output' => 60.00],
51+
'o3-mini' => ['input' => 1.10, 'output' => 4.40],
52+
'o3' => ['input' => 10.00, 'output' => 40.00],
53+
54+
// ── Anthropic Claude ──────────────────────────────────────────────────
55+
'claude-haiku' => ['input' => 0.80, 'output' => 4.00],
56+
'claude-sonnet' => ['input' => 3.00, 'output' => 15.00],
57+
'claude-opus' => ['input' => 15.00, 'output' => 75.00],
58+
59+
// ── DeepSeek ──────────────────────────────────────────────────────────
60+
'deepseek-chat' => ['input' => 0.14, 'output' => 0.28],
61+
'deepseek-reasoner' => ['input' => 0.55, 'output' => 2.19],
62+
];
63+
64+
/**
65+
* Estimate the cost of a single API call in USD.
66+
*
67+
* Returns null if the model is not in the rate card (e.g. Ollama local models).
68+
*
69+
* @param string $modelname Exact model string from the API response.
70+
* @param int $prompttokens
71+
* @param int $completiontokens
72+
* @return float|null Cost in USD, or null if model is not known.
73+
*/
74+
public static function estimate_cost(string $modelname, int $prompttokens, int $completiontokens): ?float {
75+
$rates = self::get_rates($modelname);
76+
if ($rates === null) {
77+
return null;
78+
}
79+
$inputcost = ($prompttokens / 1_000_000) * $rates['input'];
80+
$outputcost = ($completiontokens / 1_000_000) * $rates['output'];
81+
return $inputcost + $outputcost;
82+
}
83+
84+
/**
85+
* Look up rate card by model name (prefix match, longest prefix wins).
86+
*
87+
* @param string $modelname
88+
* @return array|null ['input' => float, 'output' => float] or null.
89+
*/
90+
public static function get_rates(string $modelname): ?array {
91+
$modelname = strtolower(trim($modelname));
92+
$best = null;
93+
$bestlen = 0;
94+
foreach (self::$rate_cards as $prefix => $rates) {
95+
if (str_starts_with($modelname, $prefix) && strlen($prefix) > $bestlen) {
96+
$best = $rates;
97+
$bestlen = strlen($prefix);
98+
}
99+
}
100+
return $best;
101+
}
102+
103+
/**
104+
* Format a dollar amount for display.
105+
*
106+
* Uses more decimal places for sub-cent amounts so the value is meaningful.
107+
*
108+
* @param float|null $cost
109+
* @return string e.g. "$0.000142", "$0.0183", "$1.24", or "—" if null.
110+
*/
111+
public static function format_cost(?float $cost): string {
112+
if ($cost === null) {
113+
return '';
114+
}
115+
if ($cost < 0.0001) {
116+
return '$' . number_format($cost, 6);
117+
}
118+
if ($cost < 0.01) {
119+
return '$' . number_format($cost, 4);
120+
}
121+
if ($cost < 1.0) {
122+
return '$' . number_format($cost, 3);
123+
}
124+
return '$' . number_format($cost, 2);
125+
}
126+
127+
/**
128+
* Return all rate card entries for display in the admin UI.
129+
*
130+
* @return array [['model', 'input_per_1m', 'output_per_1m'], ...]
131+
*/
132+
public static function get_all_rates(): array {
133+
$result = [];
134+
foreach (self::$rate_cards as $prefix => $rates) {
135+
$result[] = [
136+
'model' => $prefix . '',
137+
'input_per_1m' => '$' . number_format($rates['input'], 2),
138+
'output_per_1m' => '$' . number_format($rates['output'], 2),
139+
];
140+
}
141+
return $result;
142+
}
143+
}

db/install.xml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333
<FIELD NAME="role" TYPE="char" LENGTH="20" NOTNULL="true" SEQUENCE="false" COMMENT="user or assistant"/>
3434
<FIELD NAME="message" TYPE="text" NOTNULL="true" SEQUENCE="false"/>
3535
<FIELD NAME="tokens_used" TYPE="int" LENGTH="10" NOTNULL="false" DEFAULT="0" SEQUENCE="false"/>
36+
<FIELD NAME="prompt_tokens" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="Input (prompt) tokens sent to the AI provider"/>
37+
<FIELD NAME="completion_tokens" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="Output (completion) tokens generated by the AI provider"/>
38+
<FIELD NAME="model_name" TYPE="char" LENGTH="100" NOTNULL="false" SEQUENCE="false" COMMENT="Exact model identifier used for this response (e.g. gpt-4o-mini)"/>
3639
<FIELD NAME="provider" TYPE="char" LENGTH="50" NOTNULL="false" SEQUENCE="false" COMMENT="AI provider used for this response (assistant messages only)"/>
3740
<FIELD NAME="timecreated" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
3841
</FIELDS>

0 commit comments

Comments
 (0)