Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]

### Added
- **Analytics answers "which extension spent what"** (ADR-178). The usage
table carries the caller's extension key next to the money, so the
Analytics module gains a *By extension* chart and a per-extension table
with cost, requests and tokens. Calls that name no caller — wizard
tasks, scheduler runs, anything unannotated — are listed as
*Unattributed*. `UsageTrackerServiceInterface::trackUsage()` grows one
optional trailing `$sourceExtension` parameter (additive); rows written
before this change stay unattributed.

- **Agent-harness verification.** `docs/ARCHITECTURE.md` (component map + phpat dependency-rule summary), `docs/exec-plans/` scaffold, `Build/Scripts/verify-harness.sh`, and a `harness-verify.yml` workflow — a thin caller of the shared `script-check` reusable — that fails CI on an AGENTS.md line-budget or dead-reference regression.

- A caller can choose the correlation id its call is traced under, through
Expand Down
7 changes: 7 additions & 0 deletions Classes/Controller/Backend/AnalyticsController.php
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public function indexAction(): ResponseInterface
$byProvider = $this->analytics->getBreakdownByProvider($period->from, $period->to);
$byModel = $this->analytics->getBreakdownByModel($period->from, $period->to);
$byService = $this->analytics->getBreakdownByService($period->from, $period->to);
$bySource = $this->analytics->getBreakdownBySourceExtension($period->from, $period->to);

$moduleTemplate->assignMultiple([
'preset' => $period->preset,
Expand All @@ -82,6 +83,11 @@ public function indexAction(): ResponseInterface
'byProvider' => $byProvider,
'byModel' => $byModel,
'byService' => $byService,
// Who called: the extension key a consumer named via
// withCallerSource() (ADR-178). The compatibility layer tags every
// bridged third-party call, so this is the AI inventory of the
// installation, priced.
'bySource' => $bySource,
'perUser' => $this->analytics->getPerUserUsage($period->from, $period->to),
// Reads tx_nrllm_telemetry, not the usage table: the runs a sibling
// configuration answered for after the requested one failed.
Expand All @@ -99,6 +105,7 @@ public function indexAction(): ResponseInterface
'byProvider' => $byProvider,
'byModel' => $byModel,
'byService' => $byService,
'bySource' => $bySource,
], JSON_THROW_ON_ERROR | JSON_HEX_TAG | JSON_HEX_AMP | JSON_HEX_APOS | JSON_HEX_QUOT),
]);

Expand Down
14 changes: 14 additions & 0 deletions Classes/Provider/Middleware/UsageMiddleware.php
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,22 @@ private function track(
taskUid: $taskUid,
beUserUid: $beUserUid,
countsAsRequest: $countsAsRequest,
sourceExtension: $this->sourceExtension($context),
);
}

/**
* The caller identity as {@see TelemetryMiddleware} reads it — same
* metadata key, so the cost row and the telemetry row cannot disagree
* about who called (ADR-178).
*/
private function sourceExtension(ProviderCallContext $context): string
{
$value = $context->metadata[TelemetryMiddleware::METADATA_SOURCE_EXTENSION] ?? null;

return is_string($value) ? $value : '';
}

/**
* One token figure as the PROVIDER reported it, or null where it reported
* nothing (ADR-174).
Expand Down Expand Up @@ -301,6 +314,7 @@ private function trackSpecialized(ProviderCallContext $context, mixed $result):
taskUid: $record->taskUid,
beUserUid: $record->beUserUid,
countsAsRequest: $record->countsAsRequest,
sourceExtension: $this->sourceExtension($context),
);
}

Expand Down
14 changes: 12 additions & 2 deletions Classes/Service/UsageAnalyticsService.php
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
'service_provider',
'model_id',
'service_type',
'source_extension',
'model_uid',
'configuration_uid',
'task_uid',
Expand Down Expand Up @@ -117,6 +118,15 @@ public function getBreakdownByService(DateTimeInterface $from, DateTimeInterface
return $this->breakdown('service_type', $from, $to);
}

public function getBreakdownBySourceExtension(DateTimeInterface $from, DateTimeInterface $to): array
{
// Unattributed rows are the normal case for anything that does not
// annotate (a wizard task, a scheduler run), so they get a label of
// their own instead of the generic "unknown" the other breakdowns use
// for a missing provider or model (ADR-178).
return $this->breakdown('source_extension', $from, $to, 'unattributed');
}

public function getTotalsGroupedBy(string $column, DateTimeInterface $from, DateTimeInterface $to): array
{
$this->assertGroupableColumn($column);
Expand Down Expand Up @@ -317,7 +327,7 @@ private function assertGroupableColumn(string $column): void
/**
* @return list<array{label: string, cost: float, requests: int, tokens: int}>
*/
private function breakdown(string $column, DateTimeInterface $from, DateTimeInterface $to): array
private function breakdown(string $column, DateTimeInterface $from, DateTimeInterface $to, string $emptyLabel = 'unknown'): array
{
$this->assertGroupableColumn($column);
$qb = $this->connectionPool->getQueryBuilderForTable(self::TABLE);
Expand All @@ -333,7 +343,7 @@ private function breakdown(string $column, DateTimeInterface $from, DateTimeInte
foreach ($rows as $row) {
$label = is_string($row[$column] ?? null) ? $row[$column] : '';
if ($label === '') {
$label = 'unknown';
$label = $emptyLabel;
}

$out[] = [
Expand Down
9 changes: 9 additions & 0 deletions Classes/Service/UsageAnalyticsServiceInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ public function getBreakdownByModel(DateTimeInterface $from, DateTimeInterface $
*/
public function getBreakdownByService(DateTimeInterface $from, DateTimeInterface $to): array;

/**
* Cost, requests and tokens per calling extension — the `source_extension`
* a consumer named via `AbstractOptions::withCallerSource()` (ADR-178).
* Calls that named nobody are grouped under `unattributed`.
*
* @return list<array{label: string, cost: float, requests: int, tokens: int}>
*/
public function getBreakdownBySourceExtension(DateTimeInterface $from, DateTimeInterface $to): array;

/**
* Sum cost/requests/tokens grouped by an internal column, keyed by that
* column's value. $column MUST be a hardcoded internal column name
Expand Down
10 changes: 10 additions & 0 deletions Classes/Service/UsageTrackerService.php
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,12 @@ public function trackUsage(
int $taskUid = 0,
?int $beUserUid = null,
bool $countsAsRequest = true,
string $sourceExtension = '',
): void {
$beUser = $beUserUid ?? $this->getCurrentBackendUserId();
// Mirrors the telemetry column width; a longer claim is truncated
// rather than rejected (ADR-178: attribution is a label, not a key).
$sourceExtension = substr($sourceExtension, 0, 64);
$today = strtotime('today');
$now = time();
// Sub-calls of a larger operation (e.g. a translation's language-detection
Expand All @@ -90,6 +94,10 @@ public function trackUsage(
// indexed (config_lookup) and grouped on by the analytics module, so two
// configurations on the same model must not merge into one row (which
// would keep only the first configuration_uid and misattribute usage).
// source_extension joins the key for the same reason (ADR-178): two
// extensions calling one model on one day must stay two rows, or the
// per-extension cost breakdown attributes everything to whoever wrote
// the row first.
$existingUid = $queryBuilder
->select('uid')
->from(self::TABLE)
Expand All @@ -101,6 +109,7 @@ public function trackUsage(
$queryBuilder->expr()->eq('model_uid', $modelUid),
$queryBuilder->expr()->eq('model_id', $queryBuilder->createNamedParameter($modelId)),
$queryBuilder->expr()->eq('task_uid', $taskUid),
$queryBuilder->expr()->eq('source_extension', $queryBuilder->createNamedParameter($sourceExtension)),
$queryBuilder->expr()->eq('request_date', $today),
)
->executeQuery()
Expand Down Expand Up @@ -143,6 +152,7 @@ public function trackUsage(
'model_uid' => $modelUid,
'model_id' => $modelId,
'task_uid' => $taskUid,
'source_extension' => $sourceExtension,
'be_user' => $beUser,
'request_count' => $requestIncrement,
'tokens_used' => $metrics['tokens'] ?? 0,
Expand Down
5 changes: 5 additions & 0 deletions Classes/Service/UsageTrackerServiceInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ interface UsageTrackerServiceInterface
* request row (e.g. the language-detection step of a
* translation), so the metrics (tokens/cost) are still
* aggregated but the request is counted only once.
* @param string $sourceExtension Extension key the caller named via
* `AbstractOptions::withCallerSource()`; '' when the call
* is unattributed. Part of the daily aggregation key, so
* per-extension cost stays separable (ADR-178).
*/
public function trackUsage(
string $serviceType,
Expand All @@ -58,6 +62,7 @@ public function trackUsage(
int $taskUid = 0,
?int $beUserUid = null,
bool $countsAsRequest = true,
string $sourceExtension = '',
): void;

/**
Expand Down
32 changes: 31 additions & 1 deletion Documentation/Administration/Analytics.rst
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ continuous rather than skipping gaps.
Breakdown charts
================

Three bar charts split the window's usage along different axes:
Four bar charts split the window's usage along different axes:

- **By provider** — cost and requests per ``service_provider``
(OpenAI, Anthropic, Ollama, …).
Expand All @@ -95,6 +95,36 @@ Three bar charts split the window's usage along different axes:
usage table, so it only reflects usage recorded after that change.
- **By service** — cost and requests per service type (chat, vision,
translation, speech, image).
- **By extension** — cost and requests per calling extension, see
:ref:`administration-analytics-per-extension`.

.. _administration-analytics-per-extension:

Per-extension table
===================

A table lists cost, requests and tokens per **calling extension**, ordered
by cost. It answers which piece of software spent what: a consumer names
itself with ``AbstractOptions::withCallerSource()`` (:ref:`ADR-177
<adr-177>`), and that name is stored on the usage row alongside the money
(:ref:`ADR-178 <adr-178>`).

Where the entries come from:

- The compatibility layer `nr-llm-compat
<https://github.com/netresearch/t3x-nr-llm-compat>`__ tags every call it
reroutes, so an intercepted third-party extension (``ai_filemetadata``,
``texter``, ``ns_t3ai``, …) shows up under its own extension key.
- Any other consumer that annotates its calls appears the same way.
- Everything else — wizard tasks, scheduler runs, playground calls and any
consumer that does not annotate — is listed as **Unattributed**. That is
the normal state, not an error.

.. note::
The name is what the caller claims. Attribution is an inventory of the
installation, not an access control: a caller can name itself anything,
and nothing verifies it. Rows written before the column existed are
unattributed — no migration invents an origin for them.

.. _administration-analytics-per-user:

Expand Down
24 changes: 24 additions & 0 deletions Resources/Private/Language/de.locallang.xlf
Original file line number Diff line number Diff line change
Expand Up @@ -1212,6 +1212,30 @@
<source>By service</source>
<target>Nach Dienst</target>
</trans-unit>
<trans-unit id="analytics.breakdown.source">
<source>By extension</source>
<target>Nach Extension</target>
</trans-unit>
<trans-unit id="analytics.source.title">
<source>Per calling extension</source>
<target>Pro aufrufender Extension</target>
</trans-unit>
<trans-unit id="analytics.source.description">
<source>What each extension spent through this installation. An extension appears here once it names itself on its calls; usage from wizards, scheduler runs and other unnamed callers is listed as unattributed.</source>
<target>Was jede Extension über diese Installation ausgegeben hat. Eine Extension erscheint hier, sobald sie sich bei ihren Aufrufen benennt; Verbrauch aus Assistenten, Scheduler-Läufen und anderen unbenannten Aufrufern steht unter "Nicht zugeordnet".</target>
</trans-unit>
<trans-unit id="analytics.source.unattributed">
<source>Unattributed</source>
<target>Nicht zugeordnet</target>
</trans-unit>
<trans-unit id="analytics.source.empty">
<source>No usage recorded in this period.</source>
<target>In diesem Zeitraum wurde kein Verbrauch erfasst.</target>
</trans-unit>
<trans-unit id="analytics.table.extension">
<source>Extension</source>
<target>Extension</target>
</trans-unit>
<trans-unit id="analytics.peruser.title">
<source>Per backend user</source>
<target>Pro Backend-Benutzer</target>
Expand Down
18 changes: 18 additions & 0 deletions Resources/Private/Language/locallang.xlf
Original file line number Diff line number Diff line change
Expand Up @@ -916,6 +916,24 @@
<trans-unit id="analytics.breakdown.service">
<source>By service</source>
</trans-unit>
<trans-unit id="analytics.breakdown.source">
<source>By extension</source>
</trans-unit>
<trans-unit id="analytics.source.title">
<source>Per calling extension</source>
</trans-unit>
<trans-unit id="analytics.source.description">
<source>What each extension spent through this installation. An extension appears here once it names itself on its calls; usage from wizards, scheduler runs and other unnamed callers is listed as unattributed.</source>
</trans-unit>
<trans-unit id="analytics.source.unattributed">
<source>Unattributed</source>
</trans-unit>
<trans-unit id="analytics.source.empty">
<source>No usage recorded in this period.</source>
</trans-unit>
<trans-unit id="analytics.table.extension">
<source>Extension</source>
</trans-unit>
<trans-unit id="analytics.peruser.title">
<source>Per backend user</source>
</trans-unit>
Expand Down
32 changes: 32 additions & 0 deletions Resources/Private/Templates/Backend/Analytics/Index.html
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,40 @@ <h1><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:an
<div class="nrllm-chart-box"><h3><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.breakdown.provider" /></h3><div class="nrllm-canvas-wrap"><canvas id="nrllm-provider-chart"></canvas></div></div>
<div class="nrllm-chart-box"><h3><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.breakdown.model" /></h3><div class="nrllm-canvas-wrap"><canvas id="nrllm-model-chart"></canvas></div></div>
<div class="nrllm-chart-box"><h3><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.breakdown.service" /></h3><div class="nrllm-canvas-wrap"><canvas id="nrllm-service-chart"></canvas></div></div>
<div class="nrllm-chart-box"><h3><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.breakdown.source" /></h3><div class="nrllm-canvas-wrap"><canvas id="nrllm-source-chart"></canvas></div></div>
</div>

<f:comment>Per-extension table (ADR-178): which extension spent what</f:comment>
<h3><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.source.title" /></h3>
<p class="text-body-secondary"><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.source.description" /></p>
<f:if condition="{bySource}">
<f:then>
<div class="table-fit">
<table class="table table-striped">
<thead>
<tr><th scope="col"><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.table.extension" /></th><th scope="col"><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.table.cost" /></th><th scope="col"><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.table.requests" /></th><th scope="col"><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.table.tokens" /></th></tr>
</thead>
<tbody>
<f:for each="{bySource}" as="s">
<tr>
<td>
<f:if condition="{s.label} == 'unattributed'">
<f:then><span class="text-body-secondary"><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.source.unattributed" /></span></f:then>
<f:else><code>{s.label}</code></f:else>
</f:if>
</td>
<td>~${s.cost -> f:format.number(decimals: 2)}</td>
<td>{s.requests -> f:format.number(decimals: 0)}</td>
<td>{s.tokens -> f:format.number(decimals: 0)}</td>
</tr>
</f:for>
</tbody>
</table>
</div>
</f:then>
<f:else><p class="text-body-secondary"><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.source.empty" /></p></f:else>
</f:if>

<f:comment>Per-user table</f:comment>
<h3><f:translate key="LLL:EXT:nr_llm/Resources/Private/Language/locallang.xlf:analytics.peruser.title" /></h3>
<f:if condition="{perUser}">
Expand Down
1 change: 1 addition & 0 deletions Resources/Public/JavaScript/Backend/Analytics.js
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ class Analytics {
this.renderBreakdown('nrllm-provider-chart', this.data.byProvider || []);
this.renderBreakdown('nrllm-model-chart', this.data.byModel || []);
this.renderBreakdown('nrllm-service-chart', this.data.byService || []);
this.renderBreakdown('nrllm-source-chart', this.data.bySource || []);
}

/**
Expand Down
19 changes: 19 additions & 0 deletions Tests/Functional/Service/UsageAnalyticsServiceTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,25 @@ public function breakdownByModelOrdersByCostDesc(): void
self::assertSame('gpt-4o', $byModel[1]['label']);
}

#[Test]
public function breakdownBySourceExtensionSeparatesCallersAndLabelsUnattributedUsage(): void
{
// ADR-178: the dashboard axis that answers "which extension spent what".
$this->insertRow(self::LIT_2026_06_01, ['source_extension' => 'ai_filemetadata', 'estimated_cost' => 0.10]);
$this->insertRow(self::LIT_2026_06_01, ['source_extension' => 'texter', 'estimated_cost' => 0.50, 'model_uid' => 2]);
$this->insertRow(self::LIT_2026_06_01, ['estimated_cost' => 0.20, 'model_uid' => 3]);

$bySource = $this->service->getBreakdownBySourceExtension(
new DateTimeImmutable(self::LIT_2026_06_01),
new DateTimeImmutable(self::LIT_2026_06_01),
);

self::assertSame('texter', $bySource[0]['label']);
self::assertSame('unattributed', $bySource[1]['label']);
self::assertEqualsWithDelta(0.20, $bySource[1]['cost'], 0.0001);
self::assertSame('ai_filemetadata', $bySource[2]['label']);
}

#[Test]
public function getTotalsGroupedByKeysByColumnValue(): void
{
Expand Down
Loading
Loading