From b82dc0cbe0fd97dd21697c2112c4b1feb9fd014a Mon Sep 17 00:00:00 2001 From: Sebastian Mendel Date: Wed, 19 Aug 2026 16:14:43 +0200 Subject: [PATCH] feat(analytics): break usage and cost down by calling extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The caller identity from ADR-177 reached telemetry only, where no cost is recorded — so it could not answer the question it exists for. It now travels the cost path as well (ADR-178): source_extension is a column on tx_nrllm_service_usage and part of its daily aggregation key, fed from the same request metadata TelemetryMiddleware reads, so the cost row and the telemetry row cannot disagree about who called. The Analytics module gains a By-extension chart and a per-extension table (cost, requests, tokens). Calls that name no caller are listed as Unattributed rather than hidden. trackUsage() grows one optional trailing parameter; api-surface.txt is regenerated additively. Signed-off-by: Sebastian Mendel --- CHANGELOG.md | 9 +++ .../Backend/AnalyticsController.php | 7 ++ .../Provider/Middleware/UsageMiddleware.php | 14 ++++ Classes/Service/UsageAnalyticsService.php | 14 +++- .../UsageAnalyticsServiceInterface.php | 9 +++ Classes/Service/UsageTrackerService.php | 10 +++ .../Service/UsageTrackerServiceInterface.php | 5 ++ Documentation/Administration/Analytics.rst | 32 +++++++- Resources/Private/Language/de.locallang.xlf | 24 ++++++ Resources/Private/Language/locallang.xlf | 18 +++++ .../Templates/Backend/Analytics/Index.html | 32 ++++++++ .../Public/JavaScript/Backend/Analytics.js | 1 + .../Service/UsageAnalyticsServiceTest.php | 19 +++++ .../Service/UsageTrackerServiceTest.php | 57 ++++++++++++++ Tests/Unit/Api/api-surface.txt | 2 +- Tests/Unit/Fixture/RecordingUsageTracker.php | 3 + .../Middleware/UsageMiddlewareTest.php | 74 +++++++++++++++++++ ext_tables.sql | 5 ++ 18 files changed, 331 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29e90f7a4..b5a4f507b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Added +- **Analytics answers "which extension spent what"** (ADR-178). The usage + table carries the caller's extension key next to the money, so the + Analytics module gains a *By extension* chart and a per-extension table + with cost, requests and tokens. Calls that name no caller — wizard + tasks, scheduler runs, anything unannotated — are listed as + *Unattributed*. `UsageTrackerServiceInterface::trackUsage()` grows one + optional trailing `$sourceExtension` parameter (additive); rows written + before this change stay unattributed. + - **Agent-harness verification.** `docs/ARCHITECTURE.md` (component map + phpat dependency-rule summary), `docs/exec-plans/` scaffold, `Build/Scripts/verify-harness.sh`, and a `harness-verify.yml` workflow — a thin caller of the shared `script-check` reusable — that fails CI on an AGENTS.md line-budget or dead-reference regression. - A caller can choose the correlation id its call is traced under, through diff --git a/Classes/Controller/Backend/AnalyticsController.php b/Classes/Controller/Backend/AnalyticsController.php index 87e7e5905..761e9f3e2 100644 --- a/Classes/Controller/Backend/AnalyticsController.php +++ b/Classes/Controller/Backend/AnalyticsController.php @@ -70,6 +70,7 @@ public function indexAction(): ResponseInterface $byProvider = $this->analytics->getBreakdownByProvider($period->from, $period->to); $byModel = $this->analytics->getBreakdownByModel($period->from, $period->to); $byService = $this->analytics->getBreakdownByService($period->from, $period->to); + $bySource = $this->analytics->getBreakdownBySourceExtension($period->from, $period->to); $moduleTemplate->assignMultiple([ 'preset' => $period->preset, @@ -82,6 +83,11 @@ public function indexAction(): ResponseInterface 'byProvider' => $byProvider, 'byModel' => $byModel, 'byService' => $byService, + // Who called: the extension key a consumer named via + // withCallerSource() (ADR-178). The compatibility layer tags every + // bridged third-party call, so this is the AI inventory of the + // installation, priced. + 'bySource' => $bySource, 'perUser' => $this->analytics->getPerUserUsage($period->from, $period->to), // Reads tx_nrllm_telemetry, not the usage table: the runs a sibling // configuration answered for after the requested one failed. @@ -99,6 +105,7 @@ public function indexAction(): ResponseInterface 'byProvider' => $byProvider, 'byModel' => $byModel, 'byService' => $byService, + 'bySource' => $bySource, ], JSON_THROW_ON_ERROR | JSON_HEX_TAG | JSON_HEX_AMP | JSON_HEX_APOS | JSON_HEX_QUOT), ]); diff --git a/Classes/Provider/Middleware/UsageMiddleware.php b/Classes/Provider/Middleware/UsageMiddleware.php index c12950102..b7cf2a7db 100644 --- a/Classes/Provider/Middleware/UsageMiddleware.php +++ b/Classes/Provider/Middleware/UsageMiddleware.php @@ -251,9 +251,22 @@ private function track( taskUid: $taskUid, beUserUid: $beUserUid, countsAsRequest: $countsAsRequest, + sourceExtension: $this->sourceExtension($context), ); } + /** + * The caller identity as {@see TelemetryMiddleware} reads it — same + * metadata key, so the cost row and the telemetry row cannot disagree + * about who called (ADR-178). + */ + private function sourceExtension(ProviderCallContext $context): string + { + $value = $context->metadata[TelemetryMiddleware::METADATA_SOURCE_EXTENSION] ?? null; + + return is_string($value) ? $value : ''; + } + /** * One token figure as the PROVIDER reported it, or null where it reported * nothing (ADR-174). @@ -301,6 +314,7 @@ private function trackSpecialized(ProviderCallContext $context, mixed $result): taskUid: $record->taskUid, beUserUid: $record->beUserUid, countsAsRequest: $record->countsAsRequest, + sourceExtension: $this->sourceExtension($context), ); } diff --git a/Classes/Service/UsageAnalyticsService.php b/Classes/Service/UsageAnalyticsService.php index 1ccf41384..f1370ba8f 100644 --- a/Classes/Service/UsageAnalyticsService.php +++ b/Classes/Service/UsageAnalyticsService.php @@ -40,6 +40,7 @@ 'service_provider', 'model_id', 'service_type', + 'source_extension', 'model_uid', 'configuration_uid', 'task_uid', @@ -117,6 +118,15 @@ public function getBreakdownByService(DateTimeInterface $from, DateTimeInterface return $this->breakdown('service_type', $from, $to); } + public function getBreakdownBySourceExtension(DateTimeInterface $from, DateTimeInterface $to): array + { + // Unattributed rows are the normal case for anything that does not + // annotate (a wizard task, a scheduler run), so they get a label of + // their own instead of the generic "unknown" the other breakdowns use + // for a missing provider or model (ADR-178). + return $this->breakdown('source_extension', $from, $to, 'unattributed'); + } + public function getTotalsGroupedBy(string $column, DateTimeInterface $from, DateTimeInterface $to): array { $this->assertGroupableColumn($column); @@ -317,7 +327,7 @@ private function assertGroupableColumn(string $column): void /** * @return list */ - private function breakdown(string $column, DateTimeInterface $from, DateTimeInterface $to): array + private function breakdown(string $column, DateTimeInterface $from, DateTimeInterface $to, string $emptyLabel = 'unknown'): array { $this->assertGroupableColumn($column); $qb = $this->connectionPool->getQueryBuilderForTable(self::TABLE); @@ -333,7 +343,7 @@ private function breakdown(string $column, DateTimeInterface $from, DateTimeInte foreach ($rows as $row) { $label = is_string($row[$column] ?? null) ? $row[$column] : ''; if ($label === '') { - $label = 'unknown'; + $label = $emptyLabel; } $out[] = [ diff --git a/Classes/Service/UsageAnalyticsServiceInterface.php b/Classes/Service/UsageAnalyticsServiceInterface.php index 7e1f3198a..f99e1d1b7 100644 --- a/Classes/Service/UsageAnalyticsServiceInterface.php +++ b/Classes/Service/UsageAnalyticsServiceInterface.php @@ -44,6 +44,15 @@ public function getBreakdownByModel(DateTimeInterface $from, DateTimeInterface $ */ public function getBreakdownByService(DateTimeInterface $from, DateTimeInterface $to): array; + /** + * Cost, requests and tokens per calling extension — the `source_extension` + * a consumer named via `AbstractOptions::withCallerSource()` (ADR-178). + * Calls that named nobody are grouped under `unattributed`. + * + * @return list + */ + public function getBreakdownBySourceExtension(DateTimeInterface $from, DateTimeInterface $to): array; + /** * Sum cost/requests/tokens grouped by an internal column, keyed by that * column's value. $column MUST be a hardcoded internal column name diff --git a/Classes/Service/UsageTrackerService.php b/Classes/Service/UsageTrackerService.php index 2da5ffe85..c0d5d696e 100644 --- a/Classes/Service/UsageTrackerService.php +++ b/Classes/Service/UsageTrackerService.php @@ -70,8 +70,12 @@ public function trackUsage( int $taskUid = 0, ?int $beUserUid = null, bool $countsAsRequest = true, + string $sourceExtension = '', ): void { $beUser = $beUserUid ?? $this->getCurrentBackendUserId(); + // Mirrors the telemetry column width; a longer claim is truncated + // rather than rejected (ADR-178: attribution is a label, not a key). + $sourceExtension = substr($sourceExtension, 0, 64); $today = strtotime('today'); $now = time(); // Sub-calls of a larger operation (e.g. a translation's language-detection @@ -90,6 +94,10 @@ public function trackUsage( // indexed (config_lookup) and grouped on by the analytics module, so two // configurations on the same model must not merge into one row (which // would keep only the first configuration_uid and misattribute usage). + // source_extension joins the key for the same reason (ADR-178): two + // extensions calling one model on one day must stay two rows, or the + // per-extension cost breakdown attributes everything to whoever wrote + // the row first. $existingUid = $queryBuilder ->select('uid') ->from(self::TABLE) @@ -101,6 +109,7 @@ public function trackUsage( $queryBuilder->expr()->eq('model_uid', $modelUid), $queryBuilder->expr()->eq('model_id', $queryBuilder->createNamedParameter($modelId)), $queryBuilder->expr()->eq('task_uid', $taskUid), + $queryBuilder->expr()->eq('source_extension', $queryBuilder->createNamedParameter($sourceExtension)), $queryBuilder->expr()->eq('request_date', $today), ) ->executeQuery() @@ -143,6 +152,7 @@ public function trackUsage( 'model_uid' => $modelUid, 'model_id' => $modelId, 'task_uid' => $taskUid, + 'source_extension' => $sourceExtension, 'be_user' => $beUser, 'request_count' => $requestIncrement, 'tokens_used' => $metrics['tokens'] ?? 0, diff --git a/Classes/Service/UsageTrackerServiceInterface.php b/Classes/Service/UsageTrackerServiceInterface.php index d356390e2..d67738217 100644 --- a/Classes/Service/UsageTrackerServiceInterface.php +++ b/Classes/Service/UsageTrackerServiceInterface.php @@ -47,6 +47,10 @@ interface UsageTrackerServiceInterface * request row (e.g. the language-detection step of a * translation), so the metrics (tokens/cost) are still * aggregated but the request is counted only once. + * @param string $sourceExtension Extension key the caller named via + * `AbstractOptions::withCallerSource()`; '' when the call + * is unattributed. Part of the daily aggregation key, so + * per-extension cost stays separable (ADR-178). */ public function trackUsage( string $serviceType, @@ -58,6 +62,7 @@ public function trackUsage( int $taskUid = 0, ?int $beUserUid = null, bool $countsAsRequest = true, + string $sourceExtension = '', ): void; /** diff --git a/Documentation/Administration/Analytics.rst b/Documentation/Administration/Analytics.rst index 3881e87e1..bf8302299 100644 --- a/Documentation/Administration/Analytics.rst +++ b/Documentation/Administration/Analytics.rst @@ -86,7 +86,7 @@ continuous rather than skipping gaps. Breakdown charts ================ -Three bar charts split the window's usage along different axes: +Four bar charts split the window's usage along different axes: - **By provider** — cost and requests per ``service_provider`` (OpenAI, Anthropic, Ollama, …). @@ -95,6 +95,36 @@ Three bar charts split the window's usage along different axes: usage table, so it only reflects usage recorded after that change. - **By service** — cost and requests per service type (chat, vision, translation, speech, image). +- **By extension** — cost and requests per calling extension, see + :ref:`administration-analytics-per-extension`. + +.. _administration-analytics-per-extension: + +Per-extension table +=================== + +A table lists cost, requests and tokens per **calling extension**, ordered +by cost. It answers which piece of software spent what: a consumer names +itself with ``AbstractOptions::withCallerSource()`` (:ref:`ADR-177 +`), and that name is stored on the usage row alongside the money +(:ref:`ADR-178 `). + +Where the entries come from: + +- The compatibility layer `nr-llm-compat + `__ tags every call it + reroutes, so an intercepted third-party extension (``ai_filemetadata``, + ``texter``, ``ns_t3ai``, …) shows up under its own extension key. +- Any other consumer that annotates its calls appears the same way. +- Everything else — wizard tasks, scheduler runs, playground calls and any + consumer that does not annotate — is listed as **Unattributed**. That is + the normal state, not an error. + +.. note:: + The name is what the caller claims. Attribution is an inventory of the + installation, not an access control: a caller can name itself anything, + and nothing verifies it. Rows written before the column existed are + unattributed — no migration invents an origin for them. .. _administration-analytics-per-user: diff --git a/Resources/Private/Language/de.locallang.xlf b/Resources/Private/Language/de.locallang.xlf index 38365a035..edd8ae4cf 100644 --- a/Resources/Private/Language/de.locallang.xlf +++ b/Resources/Private/Language/de.locallang.xlf @@ -1212,6 +1212,30 @@ By service Nach Dienst + + By extension + Nach Extension + + + Per calling extension + Pro aufrufender Extension + + + What each extension spent through this installation. An extension appears here once it names itself on its calls; usage from wizards, scheduler runs and other unnamed callers is listed as unattributed. + Was jede Extension über diese Installation ausgegeben hat. Eine Extension erscheint hier, sobald sie sich bei ihren Aufrufen benennt; Verbrauch aus Assistenten, Scheduler-Läufen und anderen unbenannten Aufrufern steht unter "Nicht zugeordnet". + + + Unattributed + Nicht zugeordnet + + + No usage recorded in this period. + In diesem Zeitraum wurde kein Verbrauch erfasst. + + + Extension + Extension + Per backend user Pro Backend-Benutzer diff --git a/Resources/Private/Language/locallang.xlf b/Resources/Private/Language/locallang.xlf index f44216697..f7b7d5b65 100644 --- a/Resources/Private/Language/locallang.xlf +++ b/Resources/Private/Language/locallang.xlf @@ -916,6 +916,24 @@ By service + + By extension + + + Per calling extension + + + What each extension spent through this installation. An extension appears here once it names itself on its calls; usage from wizards, scheduler runs and other unnamed callers is listed as unattributed. + + + Unattributed + + + No usage recorded in this period. + + + Extension + Per backend user diff --git a/Resources/Private/Templates/Backend/Analytics/Index.html b/Resources/Private/Templates/Backend/Analytics/Index.html index 0e79a4d88..8126546aa 100644 --- a/Resources/Private/Templates/Backend/Analytics/Index.html +++ b/Resources/Private/Templates/Backend/Analytics/Index.html @@ -71,8 +71,40 @@

+

+ Per-extension table (ADR-178): which extension spent what +

+

+ + +
+ + + + + + + + + + + + + + +
+ + + {s.label} + + ~${s.cost -> f:format.number(decimals: 2)}{s.requests -> f:format.number(decimals: 0)}{s.tokens -> f:format.number(decimals: 0)}
+
+ +

+ + Per-user table

diff --git a/Resources/Public/JavaScript/Backend/Analytics.js b/Resources/Public/JavaScript/Backend/Analytics.js index 973501c5a..e6fa96639 100644 --- a/Resources/Public/JavaScript/Backend/Analytics.js +++ b/Resources/Public/JavaScript/Backend/Analytics.js @@ -57,6 +57,7 @@ class Analytics { this.renderBreakdown('nrllm-provider-chart', this.data.byProvider || []); this.renderBreakdown('nrllm-model-chart', this.data.byModel || []); this.renderBreakdown('nrllm-service-chart', this.data.byService || []); + this.renderBreakdown('nrllm-source-chart', this.data.bySource || []); } /** diff --git a/Tests/Functional/Service/UsageAnalyticsServiceTest.php b/Tests/Functional/Service/UsageAnalyticsServiceTest.php index 6761ff903..612f826f4 100644 --- a/Tests/Functional/Service/UsageAnalyticsServiceTest.php +++ b/Tests/Functional/Service/UsageAnalyticsServiceTest.php @@ -144,6 +144,25 @@ public function breakdownByModelOrdersByCostDesc(): void self::assertSame('gpt-4o', $byModel[1]['label']); } + #[Test] + public function breakdownBySourceExtensionSeparatesCallersAndLabelsUnattributedUsage(): void + { + // ADR-178: the dashboard axis that answers "which extension spent what". + $this->insertRow(self::LIT_2026_06_01, ['source_extension' => 'ai_filemetadata', 'estimated_cost' => 0.10]); + $this->insertRow(self::LIT_2026_06_01, ['source_extension' => 'texter', 'estimated_cost' => 0.50, 'model_uid' => 2]); + $this->insertRow(self::LIT_2026_06_01, ['estimated_cost' => 0.20, 'model_uid' => 3]); + + $bySource = $this->service->getBreakdownBySourceExtension( + new DateTimeImmutable(self::LIT_2026_06_01), + new DateTimeImmutable(self::LIT_2026_06_01), + ); + + self::assertSame('texter', $bySource[0]['label']); + self::assertSame('unattributed', $bySource[1]['label']); + self::assertEqualsWithDelta(0.20, $bySource[1]['cost'], 0.0001); + self::assertSame('ai_filemetadata', $bySource[2]['label']); + } + #[Test] public function getTotalsGroupedByKeysByColumnValue(): void { diff --git a/Tests/Functional/Service/UsageTrackerServiceTest.php b/Tests/Functional/Service/UsageTrackerServiceTest.php index 07009302d..763da2e83 100644 --- a/Tests/Functional/Service/UsageTrackerServiceTest.php +++ b/Tests/Functional/Service/UsageTrackerServiceTest.php @@ -183,6 +183,63 @@ public function trackUsageKeepsSeparateRecordsForDifferentModelIds(): void self::assertSame(2, $count); } + #[Test] + public function trackUsageKeepsSeparateRecordsForDifferentSourceExtensions(): void + { + // ADR-178: two extensions calling the same model on the same day must + // stay two rows, or the per-extension cost breakdown attributes + // everything to whoever wrote the row first. + $this->service->trackUsage('completion', 'openai', ['tokens' => 10, 'cost' => 0.01], modelId: 'gpt-4o', sourceExtension: 'ai_filemetadata'); + $this->service->trackUsage('completion', 'openai', ['tokens' => 20, 'cost' => 0.02], modelId: 'gpt-4o', sourceExtension: 'texter'); + + $connection = $this->connectionPool->getConnectionForTable(self::TABLE); + self::assertSame(2, $connection->count('*', self::TABLE, ['model_id' => 'gpt-4o'])); + + $row = $connection->select(['estimated_cost'], self::TABLE, ['source_extension' => 'texter'])->fetchAssociative(); + self::assertIsArray($row); + self::assertIsNumeric($row['estimated_cost']); + self::assertEqualsWithDelta(0.02, (float)$row['estimated_cost'], 0.0001); + + // The same extension again aggregates into ITS row only. + $this->service->trackUsage('completion', 'openai', ['tokens' => 5, 'cost' => 0.01], modelId: 'gpt-4o', sourceExtension: 'texter'); + self::assertSame(2, $connection->count('*', self::TABLE, ['model_id' => 'gpt-4o'])); + + $row = $connection->select(['estimated_cost'], self::TABLE, ['source_extension' => 'texter'])->fetchAssociative(); + self::assertIsArray($row); + self::assertIsNumeric($row['estimated_cost']); + self::assertEqualsWithDelta(0.03, (float)$row['estimated_cost'], 0.0001); + } + + #[Test] + public function trackUsageKeepsUnattributedCallsSeparateFromAttributedOnes(): void + { + // An unannotated caller (wizard task, scheduler run) writes '' and must + // not be merged into an extension's row. + $this->service->trackUsage('completion', 'openai', ['tokens' => 10, 'cost' => 0.01], modelId: 'gpt-4o'); + $this->service->trackUsage('completion', 'openai', ['tokens' => 10, 'cost' => 0.04], modelId: 'gpt-4o', sourceExtension: 'ns_t3ai'); + + $connection = $this->connectionPool->getConnectionForTable(self::TABLE); + self::assertSame(2, $connection->count('*', self::TABLE, ['model_id' => 'gpt-4o'])); + + $row = $connection->select(['estimated_cost'], self::TABLE, ['source_extension' => ''])->fetchAssociative(); + self::assertIsArray($row); + self::assertIsNumeric($row['estimated_cost']); + self::assertEqualsWithDelta(0.01, (float)$row['estimated_cost'], 0.0001); + } + + #[Test] + public function trackUsageTruncatesAnOverlongSourceExtension(): void + { + // The column is varchar(64); a longer claim is a label, not a key, so it + // is cut rather than rejected (ADR-178). + $this->service->trackUsage('completion', 'openai', ['tokens' => 1, 'cost' => 0.01], sourceExtension: str_repeat('a', 100)); + + $connection = $this->connectionPool->getConnectionForTable(self::TABLE); + $row = $connection->select(['source_extension'], self::TABLE, ['service_type' => 'completion'])->fetchAssociative(); + self::assertIsArray($row); + self::assertSame(str_repeat('a', 64), $row['source_extension']); + } + #[Test] public function trackUsageKeepsSeparateRecordsForDifferentConfigurations(): void { diff --git a/Tests/Unit/Api/api-surface.txt b/Tests/Unit/Api/api-surface.txt index 097ad98b8..fa4917325 100644 --- a/Tests/Unit/Api/api-surface.txt +++ b/Tests/Unit/Api/api-surface.txt @@ -1668,7 +1668,7 @@ Netresearch\NrLlm\Service\UsageTrackerServiceInterface (interface) method getTodayUsage(string $serviceType, string $provider): ?array method getUsageReport(string $serviceType, DateTimeInterface $from, DateTimeInterface $to): array method getUserUsage(int $beUserUid, DateTimeInterface $from, DateTimeInterface $to): array - method trackUsage(string $serviceType, string $provider, array $metrics = …, ?int $configurationUid = …, int $modelUid = …, string $modelId = …, int $taskUid = …, ?int $beUserUid = …, bool $countsAsRequest = …): void + method trackUsage(string $serviceType, string $provider, array $metrics = …, ?int $configurationUid = …, int $modelUid = …, string $modelId = …, int $taskUid = …, ?int $beUserUid = …, bool $countsAsRequest = …, string $sourceExtension = …): void Netresearch\NrLlm\Service\UseCase\UseCasePackProviderInterface (interface) const TAG_NAME = "nr_llm.use_case_pack" diff --git a/Tests/Unit/Fixture/RecordingUsageTracker.php b/Tests/Unit/Fixture/RecordingUsageTracker.php index ccff9526f..97d77f629 100644 --- a/Tests/Unit/Fixture/RecordingUsageTracker.php +++ b/Tests/Unit/Fixture/RecordingUsageTracker.php @@ -32,6 +32,7 @@ final class RecordingUsageTracker implements UsageTrackerServiceInterface * taskUid: int, * beUserUid: ?int, * countsAsRequest: bool, + * sourceExtension: string, * }> */ public array $calls = []; @@ -46,6 +47,7 @@ public function trackUsage( int $taskUid = 0, ?int $beUserUid = null, bool $countsAsRequest = true, + string $sourceExtension = '', ): void { $this->calls[] = [ 'serviceType' => $serviceType, @@ -57,6 +59,7 @@ public function trackUsage( 'taskUid' => $taskUid, 'beUserUid' => $beUserUid, 'countsAsRequest' => $countsAsRequest, + 'sourceExtension' => $sourceExtension, ]; } diff --git a/Tests/Unit/Provider/Middleware/UsageMiddlewareTest.php b/Tests/Unit/Provider/Middleware/UsageMiddlewareTest.php index 4765a67c6..6ac312ef1 100644 --- a/Tests/Unit/Provider/Middleware/UsageMiddlewareTest.php +++ b/Tests/Unit/Provider/Middleware/UsageMiddlewareTest.php @@ -21,6 +21,7 @@ use Netresearch\NrLlm\Provider\Middleware\MiddlewarePipeline; use Netresearch\NrLlm\Provider\Middleware\ProviderCallContext; use Netresearch\NrLlm\Provider\Middleware\ProviderOperation; +use Netresearch\NrLlm\Provider\Middleware\TelemetryMiddleware; use Netresearch\NrLlm\Provider\Middleware\Usage\ProviderUsageRecord; use Netresearch\NrLlm\Provider\Middleware\Usage\UsageMetricsExtractorInterface; use Netresearch\NrLlm\Provider\Middleware\UsageMiddleware; @@ -164,6 +165,79 @@ public function skipRequestCountMetadataRecordsMetricsButNotTheRequest(): void ); } + #[Test] + public function callerSourceMetadataReachesTheCostRow(): void + { + // ADR-178: the cost row is attributed from the SAME metadata key the + // telemetry row uses, so the two cannot disagree about who called. + $this->tracker->expects(self::once()) + ->method('trackUsage') + ->with( + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + 'ai_filemetadata', + ); + + $response = new CompletionResponse( + content: 'hi', + model: 'gpt-4o-mini', + usage: new UsageStatistics(100, 50, 150, 0.0012), + finishReason: 'stop', + provider: 'openai', + ); + + $this->pipeline()->run( + context: ProviderCallContext::forConfiguration( + ProviderOperation::Chat, + $this->configuration(uid: 7), + [TelemetryMiddleware::METADATA_SOURCE_EXTENSION => 'ai_filemetadata'], + ), + terminal: static fn(): CompletionResponse => $response, + ); + } + + #[Test] + public function anUnannotatedCallIsRecordedAsUnattributed(): void + { + $this->tracker->expects(self::once()) + ->method('trackUsage') + ->with( + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + self::anything(), + '', + ); + + $response = new CompletionResponse( + content: 'hi', + model: 'gpt-4o-mini', + usage: new UsageStatistics(100, 50, 150, 0.0012), + finishReason: 'stop', + provider: 'openai', + ); + + $this->pipeline()->run( + context: ProviderCallContext::forConfiguration( + ProviderOperation::Chat, + $this->configuration(uid: 7), + ), + terminal: static fn(): CompletionResponse => $response, + ); + } + #[Test] public function requestIsCountedByDefaultWithoutSkipMetadata(): void { diff --git a/ext_tables.sql b/ext_tables.sql index 99998ce23..17679a73d 100644 --- a/ext_tables.sql +++ b/ext_tables.sql @@ -367,6 +367,11 @@ CREATE TABLE tx_nrllm_service_usage ( -- Task dimension (per-task usage tracking) task_uid int(11) unsigned DEFAULT '0' NOT NULL, + -- Caller dimension (ADR-178): the extension key a consumer named via + -- withCallerSource(). Part of the daily aggregation key, so two + -- extensions on the same model produce two rows. '' = unattributed. + source_extension varchar(64) DEFAULT '' NOT NULL, + -- User context be_user int(11) unsigned DEFAULT '0' NOT NULL,