MO-523: Align analytics event envelope

.github/workflows/sonarcloud.yml

@@ -64,6 +64,6 @@ jobs:
         run: |
           ./.sonar/scanner/dotnet-sonarscanner begin /k:"DEFRA_waste-obligations" /o:"defra" /d:sonar.token="$SONAR_TOKEN" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.vscoveragexml.reportsPaths=coverage*.xml /d:sonar.coverage.exclusions=compose/journey-tests/**
           dotnet build --no-incremental
-          dotnet test --test-modules tests/Api.Tests/bin/Debug/net10.0/Api.Tests.dll --no-build --coverage --coverage-output-format xml --coverage-output "coverage.Api.Tests.xml"
-          dotnet test --test-modules tests/Api.IntegrationTests/bin/Debug/net10.0/Api.IntegrationTests.dll --no-build --coverage --coverage-output-format xml --coverage-output "coverage.Api.IntegrationTests.xml"
+          dotnet test --test-modules tests/Api.Tests/bin/Debug/net10.0/Api.Tests.dll --no-build -- --coverage --coverage-output-format xml --coverage-output "$GITHUB_WORKSPACE/coverage.Api.Tests.xml"
+          dotnet test --test-modules tests/Api.IntegrationTests/bin/Debug/net10.0/Api.IntegrationTests.dll --no-build -- --coverage --coverage-output-format xml --coverage-output "$GITHUB_WORKSPACE/coverage.Api.IntegrationTests.xml"
           ./.sonar/scanner/dotnet-sonarscanner end /d:sonar.token="$SONAR_TOKEN"

docs/analytics-compliance-declaration-events.md

@@ -4,7 +4,7 @@ The analytics topic receives generic analytics event envelopes. Each envelope id
 
 Compliance declaration create, update, and delete operations are captured internally in the same transaction as the declaration change. The analytics processor reads undispatched changes, serialises them as analytics events, and publishes them to the analytics SNS topic configured by `AnalyticsAuditEventProcessor:TopicArn`.
 
-The nested compliance declaration payload is serialised using the embedded [compliance declaration schema](../src/Api/Schemas/ComplianceDeclaration/compliance-declaration.v1.3.schema.json). Its version history is recorded in the [compliance declaration schema changelog](../src/Api/Schemas/ComplianceDeclaration/CHANGELOG.md). For compliance declarations, the analytics message `schemaVersion` is currently `compliance_declaration.v1.3`.
+The nested compliance declaration payload is serialised using the embedded [compliance declaration schema](../src/Api/Schemas/ComplianceDeclaration/compliance-declaration.v1.3.schema.json). Its version history is recorded in the [compliance declaration schema changelog](../src/Api/Schemas/ComplianceDeclaration/CHANGELOG.md). For compliance declarations, the analytics message `schemaVersion` is currently `compliance_declaration_v1.3`.
 
 ## Message transport
 
@@ -22,17 +22,18 @@ All analytics events use this generic envelope:
   "sequence": 123,
   "entity": "entity_name",
   "entityId": "entity_name_entity-id",
-  "operation": "insert",
+  "operation": "create",
   "eventType": "domain.event",
   "deletedReason": null,
   "piiKeyRef": null,
-  "occurredAt": "2026-01-02T03:04:05+00:00",
-  "recordedAt": "2026-01-02T03:04:06+00:00",
-  "actor": "service:waste-obligations",
+  "occurredAt": "2026-01-02T03:04:05.000Z",
+  "recordedAt": "2026-01-02T03:04:06.000Z",
+  "actor": "user:e72be574-8b5b-4836-af47-dd7e0c0d1d87",
+  "correlationId": "cdp-request-id",
   "version": 1,
   "before": null,
   "after": {},
-  "schemaVersion": "entity_name.v1.0"
+  "schemaVersion": "entity_name_v1.0"
 }
 ```
 
@@ -45,38 +46,45 @@ All analytics events use this generic envelope:
 | `operation` | Event operation. See the operation values below. |
 | `eventType` | Business event name. |
 | `deletedReason` | Reason the entity was deleted. This is only set when `operation` is `delete`; otherwise it is `null`. |
-| `piiKeyRef` | Currently always `null`. |
-| `occurredAt` | Time the entity change occurred, in ISO 8601 format with offset. |
-| `recordedAt` | Time the analytics event was recorded, in ISO 8601 format with offset. |
-| `actor` | Service actor that wrote the event. |
+| `piiKeyRef` | Always `null`; PII classification and protection are outside the scope of this change. |
+| `occurredAt` | Time the entity change occurred, as a UTC ISO 8601 timestamp with millisecond precision. |
+| `recordedAt` | Time the analytics event was recorded, as a UTC ISO 8601 timestamp with millisecond precision. |
+| `actor` | `user:<UUID>` for user submissions and status amendments; `service:waste-obligations` for system-driven deletions. |
+| `correlationId` | The incoming `x-cdp-request-id` when present; omitted when no request ID is available. |
 | `version` | Entity version after the operation. |
-| `before` | Previous entity state. This is `null` for inserts. |
+| `before` | Previous entity state. This is `null` for creates. |
 | `after` | New entity state. This is `null` for deletes. |
 | `schemaVersion` | Entity-qualified schema version for the `before` and `after` payloads. |
 
 ## Operation values
 
 | Operation | Meaning | `before` | `after` | `deletedReason` |
 | --- | --- | --- | --- | --- |
-| `insert` | Entity was created. | `null` | Created entity state. | `null` |
+| `create` | Entity was created. | `null` | Created entity state. | `null` |
 | `update` | Entity was changed. | Previous entity state. | Updated entity state. | `null` |
 | `delete` | Entity was deleted. | Previous entity state. | `null` | Delete reason. |
 
+## Controlled vocabulary
+
+The publisher validates the current compliance declaration entity, envelope operation, event type, actor prefix, deletion reason, and schema version before serialising a message. The supported operations are `create`, `update`, and `delete`; actor prefixes are `service`, `user`, `system`, and `integration`; and the supported declaration event types are `submission.created`, `submission.amended`, and `submission.removed`. Create and update events use the responsible audit user as `user:<UUID>`; the system-driven delete flow retains `service:waste-obligations`.
+
+The only current deletion reason is `elevated_system_allowed_removal`. This replaces the legacy free-text value `elevated system allowed removal`, which remains normalised for undispatched historical outbox events. The governed list maintained with the shared event schema must include `elevated_system_allowed_removal` before additional deletion paths are introduced.
+
 ## Compliance declaration events
 
-Compliance declaration events use `entity` set to `compliance_declaration`. The `entityId` value is prefixed with the entity type, for example `compliance_declaration_65f1f6570bb08052a8a27b01`.
+Compliance declaration events use `entity` set to `compliance_declaration`. The current service identity is a Mongo ObjectId, so `entityId` remains prefixed as `compliance_declaration_65f1f6570bb08052a8a27b01`. The proposed `cdec_<ULID>` format is not implemented because it is incompatible with the existing immutable identifier model; it requires a corrected specification before it can be adopted.
 
 The current compliance declaration event types are:
 
 | Event type | Operation | Description |
 | --- | --- | --- |
-| `submission.created` | `insert` | Compliance declaration was created. |
+| `submission.created` | `create` | Compliance declaration was created. |
 | `submission.amended` | `update` | Compliance declaration was updated. |
 | `submission.removed` | `delete` | Compliance declaration was deleted. |
 
 ## Created event
 
-When a compliance declaration is created, the analytics topic receives an `insert` event with `eventType` set to `submission.created`.
+When a compliance declaration is created, the analytics topic receives a `create` event with `eventType` set to `submission.created`.
 
 The `before` value is `null`. The `after` value is the created compliance declaration, serialised according to the linked compliance declaration schema.
 
@@ -86,13 +94,14 @@ The `before` value is `null`. The `after` value is the created compliance declar
   "sequence": 123,
   "entity": "compliance_declaration",
   "entityId": "compliance_declaration_65f1f6570bb08052a8a27b01",
-  "operation": "insert",
+  "operation": "create",
   "eventType": "submission.created",
   "deletedReason": null,
   "piiKeyRef": null,
-  "occurredAt": "2026-01-02T03:04:05+00:00",
-  "recordedAt": "2026-01-02T03:04:06+00:00",
-  "actor": "service:waste-obligations",
+  "occurredAt": "2026-01-02T03:04:05.000Z",
+  "recordedAt": "2026-01-02T03:04:06.000Z",
+  "actor": "user:e72be574-8b5b-4836-af47-dd7e0c0d1d87",
+  "correlationId": "cdp-request-id",
   "version": 1,
   "before": null,
   "after": {
@@ -152,7 +161,7 @@ The `before` value is `null`. The `after` value is the created compliance declar
     "isRegulation43Compliant": true,
     "obligationCoveragePercentage": 40
   },
-  "schemaVersion": "compliance_declaration.v1.3"
+  "schemaVersion": "compliance_declaration_v1.3"
 }
 ```
 
@@ -172,9 +181,10 @@ The `before` value is the declaration state before the update. The `after` value
   "eventType": "submission.amended",
   "deletedReason": null,
   "piiKeyRef": null,
-  "occurredAt": "2026-01-02T03:05:05+00:00",
-  "recordedAt": "2026-01-02T03:05:06+00:00",
-  "actor": "service:waste-obligations",
+  "occurredAt": "2026-01-02T03:05:05.000Z",
+  "recordedAt": "2026-01-02T03:05:06.000Z",
+  "actor": "user:e72be574-8b5b-4836-af47-dd7e0c0d1d87",
+  "correlationId": "cdp-request-id",
   "version": 2,
   "before": {
     "id": "65f1f6570bb08052a8a27b01",
@@ -300,6 +310,6 @@ The `before` value is the declaration state before the update. The `after` value
     "isRegulation43Compliant": true,
     "obligationCoveragePercentage": 40
   },
-  "schemaVersion": "compliance_declaration.v1.3"
+  "schemaVersion": "compliance_declaration_v1.3"
 }
 ```

docs/compliance-declaration-end-to-end-event-flow.md

@@ -11,7 +11,7 @@ The service uses two audit concepts:
 
 | Action | Endpoint | Declaration change | Audit event operation | Audit event type | Analytics event |
 | --- | --- | --- | --- | --- | --- |
-| Create | `POST /organisations/{organisationId}/compliance-declarations` | Creates a submitted declaration, assigns version `1`, and adds a `Submitted` entry to the embedded `audit` array. | `insert` | `submission.created` | `insert` with `before: null` and `after` set to the created declaration. |
+| Create | `POST /organisations/{organisationId}/compliance-declarations` | Creates a submitted declaration, assigns version `1`, and adds a `Submitted` entry to the embedded `audit` array. | `insert` | `submission.created` | `create` with `before: null` and `after` set to the created declaration. |
 | Update status | `PATCH /organisations/{organisationId}/compliance-declarations/{complianceDeclarationId}` | Applies a valid status transition, increments the version, updates `updated`, and appends an embedded audit entry. | `update` | `submission.amended` | `update` with the previous declaration in `before` and the updated declaration in `after`. |
 | Delete | `DELETE /compliance-declarations/{id}` | Deletes the declaration using the current version as an optimistic concurrency check. | `delete` | `submission.removed` | `delete` with the previous declaration in `before`, `after: null`, and `deletedReason` set. |
 
@@ -114,7 +114,7 @@ flowchart TD
 | `before` | The previous declaration BSON document, or `null` for create. |
 | `after` | The new declaration BSON document, or `null` for delete. |
 | `schemaVersion` | The declaration schema version, currently `v1.3`. |
-| `traceId` | The propagated trace header value, used for service logging and not included in the analytics envelope. |
+| `traceId` | The propagated `x-cdp-request-id`, used for service logging and mapped to the analytics envelope's `correlationId` when available. |
 | `dispatches` | A per-process outcome map, initially empty. |
 
 The `AuditEvent` collection is indexed by sequence, entity/entity id/version, and the analytics dispatch fields so the dispatcher can read the oldest undispatched or retryable events efficiently.
@@ -184,10 +184,13 @@ sequenceDiagram
 
 The processor reads audit events where `dispatches.analytics` does not exist, or where it is `Failed` and `nextAttemptAt` is due. Each event is mapped to the analytics envelope:
 
-- `eventId`, `sequence`, `entity`, `operation`, `eventType`, timestamps, actor, version, `before`, and `after` are copied from the audit event.
+- `eventId`, `sequence`, `entity`, `eventType`, timestamps, actor, version, `before`, and `after` are copied from the audit event. Submission and status-amendment events record their responsible user as `user:<UUID>`; the system-driven deletion flow records `service:waste-obligations`.
 - `entityId` is changed from the raw ObjectId string to `compliance_declaration_{objectId}`.
-- `schemaVersion` is changed from `v1.3` to `compliance_declaration.v1.3`.
-- `piiKeyRef` is currently set to `null`.
+- `insert` is mapped to the envelope operation `create`; `update` and `delete` are unchanged.
+- `schemaVersion` is changed from `v1.3` to `compliance_declaration_v1.3`.
+- `traceId` is mapped to `correlationId` when the originating request supplied `x-cdp-request-id`.
+- `piiKeyRef` is set to `null` under the instructed PII scope exclusion.
+- The emitted entity, operation, event type, actor prefix, deletion reason, and schema version are validated against the current governed vocabulary before publishing.
 
 The serializer loads the embedded compliance declaration JSON schema and uses it to write the `before` and `after` BSON documents with the expected field names and JSON value formats.
 

src/Api/Schemas/EmbeddedEntityJsonSchemaProvider.cs

@@ -50,14 +50,24 @@ private string FindResourceName(string schemaFileName)
 
     private static string GetSchemaFileName(string entity, string schemaVersionValue)
     {
-        var schemaVersionPrefix = $"{entity}.";
-        var schemaVersion = schemaVersionValue.StartsWith(schemaVersionPrefix, StringComparison.Ordinal)
-            ? schemaVersionValue[schemaVersionPrefix.Length..]
-            : schemaVersionValue;
+        var schemaVersion = GetSchemaVersion(entity, schemaVersionValue);
 
         return $"{ToKebabCase(entity)}.{schemaVersion}{SchemaFileSuffix}";
     }
 
+    private static string GetSchemaVersion(string entity, string schemaVersionValue)
+    {
+        var dotPrefix = $"{entity}.";
+        if (schemaVersionValue.StartsWith(dotPrefix, StringComparison.Ordinal))
+            return schemaVersionValue[dotPrefix.Length..];
+
+        var underscorePrefix = $"{entity}_";
+
+        return schemaVersionValue.StartsWith(underscorePrefix, StringComparison.Ordinal)
+            ? schemaVersionValue[underscorePrefix.Length..]
+            : schemaVersionValue;
+    }
+
     private static string ToKebabCase(string value)
     {
         var builder = new StringBuilder(value.Length);

src/Api/Services/ComplianceDeclarationService.cs

@@ -4,6 +4,7 @@
 using Defra.WasteObligations.Api.Utils.Logging;
 using Defra.WasteObligations.Api.Utils.Metrics;
 using Defra.WasteObligations.AuditEvents;
+using Defra.WasteObligations.AuditEvents.Analytics;
 using MongoDB.Bson;
 using MongoDB.Driver;
 
@@ -19,7 +20,7 @@ public class ComplianceDeclarationService(
     IUnsubmittedEligibilityVisibilityService unsubmittedEligibilityVisibilityService
 ) : IComplianceDeclarationService
 {
-    private const string Actor = "service:waste-obligations";
+    private const string SystemActor = "service:waste-obligations";
     private const string ComplianceDeclarationEntity = "compliance_declaration";
 
     public async Task<ComplianceDeclaration> Create(
@@ -49,7 +50,7 @@ await unsubmittedEligibilityVisibilityService.Refresh(
                 await auditEventService.RecordEvent(
                     transactionSession,
                     new AuditEventRequest(
-                        Actor,
+                        GetActor(complianceDeclaration),
                         ComplianceDeclarationEntity,
                         AuditEventOperation.Insert,
                         "submission.created",
@@ -151,11 +152,11 @@ await unsubmittedEligibilityVisibilityService.Refresh(
                 await auditEventService.RecordEvent(
                     transactionSession,
                     new AuditEventRequest(
-                        Actor,
+                        SystemActor,
                         ComplianceDeclarationEntity,
                         AuditEventOperation.Delete,
                         "submission.removed",
-                        "elevated system allowed removal",
+                        AnalyticsEventVocabulary.ElevatedSystemAllowedRemoval,
                         current.Id.ToString(),
                         current.Version + 1,
                         current.ToBsonDocument(),
@@ -279,7 +280,7 @@ await unsubmittedEligibilityVisibilityService.Refresh(
                 await auditEventService.RecordEvent(
                     transactionSession,
                     new AuditEventRequest(
-                        Actor,
+                        GetActor(updated),
                         ComplianceDeclarationEntity,
                         AuditEventOperation.Update,
                         "submission.amended",
@@ -412,4 +413,7 @@ private static SortDefinition<ComplianceDeclaration> SortByReversedDirection(
         SortDefinition<ComplianceDeclaration> ascending,
         SortDefinition<ComplianceDeclaration> descending
     ) => direction is ComplianceDeclarationSortDirection.Ascending ? descending : ascending;
+
+    private static string GetActor(ComplianceDeclaration complianceDeclaration) =>
+        complianceDeclaration.Audit.LastOrDefault()?.User is { } user ? $"user:{user.Id}" : SystemActor;
 }

src/AuditEvents/Analytics/AnalyticsEvent.cs

@@ -29,17 +29,23 @@ public record AnalyticsEvent
     [JsonPropertyName("piiKeyRef")]
     public string? PiiKeyRef { get; init; }
 
-    [Description("ISO 8601 extended format with offset")]
+    [Description("UTC ISO 8601 format with millisecond precision")]
+    [JsonConverter(typeof(UtcMillisecondDateTimeOffsetJsonConverter))]
     [JsonPropertyName("occurredAt")]
     public DateTimeOffset OccurredAt { get; init; }
 
-    [Description("ISO 8601 extended format with offset")]
+    [Description("UTC ISO 8601 format with millisecond precision")]
+    [JsonConverter(typeof(UtcMillisecondDateTimeOffsetJsonConverter))]
     [JsonPropertyName("recordedAt")]
     public DateTimeOffset RecordedAt { get; init; }
 
     [JsonPropertyName("actor")]
     public required string Actor { get; init; }
 
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    [JsonPropertyName("correlationId")]
+    public string? CorrelationId { get; init; }
+
     [JsonPropertyName("version")]
     public int Version { get; init; }
 

src/AuditEvents/Analytics/AnalyticsEventMappers.cs

@@ -4,23 +4,30 @@ namespace Defra.WasteObligations.AuditEvents.Analytics;
 
 public static class AnalyticsEventMappers
 {
-    public static AnalyticsEvent ToAnalyticsEvent(this AuditEvent auditEvent) =>
-        new()
+    public static AnalyticsEvent ToAnalyticsEvent(this AuditEvent auditEvent)
+    {
+        var analyticsEvent = new AnalyticsEvent
         {
             EventId = auditEvent.EventId,
             Sequence = auditEvent.Sequence,
             Entity = auditEvent.Entity,
             EntityId = $"{auditEvent.Entity}_{auditEvent.EntityId}",
-            Operation = auditEvent.Operation,
+            Operation = AnalyticsEventVocabulary.ToOperation(auditEvent.Operation),
             EventType = auditEvent.EventType,
-            DeletedReason = auditEvent.DeletedReason,
+            DeletedReason = AnalyticsEventVocabulary.NormalizeDeletedReason(auditEvent.DeletedReason),
             PiiKeyRef = null,
             OccurredAt = auditEvent.OccurredAt,
             RecordedAt = auditEvent.RecordedAt,
             Actor = auditEvent.Actor,
+            CorrelationId = auditEvent.TraceId,
             Version = auditEvent.Version,
             Before = auditEvent.Before,
             After = auditEvent.After,
-            SchemaVersion = $"{auditEvent.Entity}.{auditEvent.SchemaVersion}",
+            SchemaVersion = $"{auditEvent.Entity}_{auditEvent.SchemaVersion}",
         };
+
+        AnalyticsEventVocabulary.Validate(analyticsEvent);
+
+        return analyticsEvent;
+    }
 }