Skip to content

Commit 1cddf76

Browse files
TheOneAdonisTheOneAdonis
committed
fix(manager-server): keep response headers out of fail body
1 parent ffd42f8 commit 1cddf76

5 files changed

Lines changed: 186 additions & 21 deletions

File tree

‎apps/manager-server/internal/repository/usagemonitoring/repository_test.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package usagemonitoring_test
33
import (
44
"context"
55
"database/sql"
6+
"encoding/json"
67
"errors"
78
"fmt"
89
"path/filepath"
@@ -560,6 +561,58 @@ func TestUsageMonitoringSearchIndexTracksProjectionInsertUpdateAndDelete(t *test
560561
}
561562
}
562563

564+
func TestSuccessfulResponseHeadersDoNotEnterFailureSearchStorage(t *testing.T) {
565+
sqlDB, db := newMonitoringRepositoryStore(t)
566+
ctx := context.Background()
567+
marker := strings.Repeat("unindexed-success-header-marker-", 128)
568+
payload, err := json.Marshal(map[string]any{
569+
"timestamp": "2026-04-25T00:00:00Z",
570+
"failed": false,
571+
"provider": "openai",
572+
"model": "gpt-5.4",
573+
"endpoint": "POST /v1/chat/completions",
574+
"tokens": map[string]any{"input_tokens": 1, "total_tokens": 1},
575+
"response_headers": map[string]any{
576+
"Content-Type": []any{"application/json"},
577+
"X-CPAMP-Unindexed-Diagnostic": []any{marker},
578+
},
579+
})
580+
if err != nil {
581+
t.Fatalf("marshal successful event: %v", err)
582+
}
583+
event, err := usage.NormalizeRaw(payload)
584+
if err != nil {
585+
t.Fatalf("normalize successful event: %v", err)
586+
}
587+
if _, err := db.InsertEvents(ctx, []usage.Event{event}); err != nil {
588+
t.Fatalf("insert successful event: %v", err)
589+
}
590+
591+
var eventID int64
592+
var failBody, failSummary, metadataJSON, rawJSON string
593+
if err := sqlDB.QueryRowContext(ctx, `select id, coalesce(fail_body, ''), coalesce(fail_summary, ''),
594+
coalesce(response_metadata_json, ''), coalesce(raw_json, '')
595+
from usage_events where event_hash = ?`, event.EventHash).Scan(&eventID, &failBody, &failSummary, &metadataJSON, &rawJSON); err != nil {
596+
t.Fatalf("read persisted successful event: %v", err)
597+
}
598+
if failBody != "" || failSummary != "" {
599+
t.Fatalf("persisted failure fields = body:%q summary:%q", failBody, failSummary)
600+
}
601+
if !strings.Contains(metadataJSON, "application/json") || !strings.Contains(rawJSON, marker) {
602+
t.Fatalf("persisted metadata/raw json missing: metadata=%q rawHasMarker=%v", metadataJSON, strings.Contains(rawJSON, marker))
603+
}
604+
605+
catchUpMonitoringRepository(t, ctx, db)
606+
var searchText string
607+
if err := sqlDB.QueryRowContext(ctx, `select search_text from usage_monitoring_event_projection_v1 where event_id = ?`, eventID).Scan(&searchText); err != nil {
608+
t.Fatalf("read successful event projection: %v", err)
609+
}
610+
if strings.Contains(searchText, marker) {
611+
t.Fatalf("projection search text contains response header marker")
612+
}
613+
assertSearchIndexCount(t, ctx, sqlDB, marker, 0)
614+
}
615+
563616
func TestMigrationBackfillsSearchIndexForExistingProjection(t *testing.T) {
564617
sqlDB, db := newMonitoringRepositoryStore(t)
565618
ctx := context.Background()

‎apps/manager-server/internal/usage/event.go‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -810,27 +810,9 @@ func readFailFields(record map[string]any) (int64, string) {
810810
if body == "" {
811811
body = readString(record, "fail_body", "failBody")
812812
}
813-
if headers, ok := compactJSON(first(record, "response_headers", "responseHeaders", "headers")); ok && headers != "{}" && headers != "[]" {
814-
if body == "" {
815-
body = headers
816-
} else {
817-
body = body + "\n" + headers
818-
}
819-
}
820813
return statusCode, body
821814
}
822815

823-
func compactJSON(value any) (string, bool) {
824-
if value == nil {
825-
return "", false
826-
}
827-
data, err := json.Marshal(value)
828-
if err != nil {
829-
return "", false
830-
}
831-
return string(data), true
832-
}
833-
834816
func readOptionalInt(record map[string]any, keys ...string) *int64 {
835817
value := readInt(record, keys...)
836818
if value == 0 && first(record, keys...) == nil {

‎apps/manager-server/internal/usage/import_test.go‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -583,12 +583,17 @@ func TestNormalizeRawReadsCPA7118UsageFields(t *testing.T) {
583583
}
584584
if !event.Failed || event.FailStatusCode != 429 ||
585585
!strings.Contains(event.FailBody, "rate limit exceeded") ||
586-
!strings.Contains(event.FailBody, "Retry-After") {
586+
strings.Contains(event.FailBody, "Retry-After") {
587587
t.Fatalf("event failure = %#v", event)
588588
}
589-
if !strings.Contains(event.FailSummary, "rate limit exceeded") || !strings.Contains(event.FailSummary, "Retry-After") {
589+
if !strings.Contains(event.FailSummary, "rate limit exceeded") || strings.Contains(event.FailSummary, "Retry-After") {
590590
t.Fatalf("fail summary = %q", event.FailSummary)
591591
}
592+
if event.ResponseMetadata == nil || event.ResponseMetadata.Errors == nil ||
593+
event.ResponseMetadata.Errors.RetryAfterSeconds == nil ||
594+
*event.ResponseMetadata.Errors.RetryAfterSeconds != 30 {
595+
t.Fatalf("response metadata = %#v", event.ResponseMetadata)
596+
}
592597
if event.LatencyMS == nil || *event.LatencyMS != 1500 {
593598
t.Fatalf("latency = %#v", event.LatencyMS)
594599
}
@@ -611,12 +616,51 @@ func TestNormalizeRawReadsCPA7118UsageFields(t *testing.T) {
611616
detail.Tokens.CacheCreationTokens != 1 || detail.FailStatusCode != 429 ||
612617
detail.Tokens.CachedTokens != 0 || detail.Tokens.CacheTokens != 0 ||
613618
!strings.Contains(detail.FailSummary, "rate limit exceeded") ||
614-
!strings.Contains(detail.FailSummary, "Retry-After") || detail.TTFTMS == nil ||
619+
strings.Contains(detail.FailSummary, "Retry-After") || detail.ResponseMetadata == nil ||
620+
detail.ResponseMetadata.Errors == nil || detail.TTFTMS == nil ||
615621
*detail.TTFTMS != 450 {
616622
t.Fatalf("detail = %#v", detail)
617623
}
618624
}
619625

626+
func TestNormalizeRawKeepsSuccessfulResponseHeadersOutOfFailureFields(t *testing.T) {
627+
marker := strings.Repeat("large-success-header-marker-", 256)
628+
payload, err := json.Marshal(map[string]any{
629+
"timestamp": "2026-04-25T00:00:00Z",
630+
"source": "user@example.com",
631+
"tokens": map[string]any{
632+
"input_tokens": 1,
633+
"total_tokens": 1,
634+
},
635+
"failed": false,
636+
"provider": "openai",
637+
"model": "gpt-5.4",
638+
"endpoint": "POST /v1/chat/completions",
639+
"response_headers": map[string]any{
640+
"Content-Type": []any{"application/json"},
641+
"X-CPAMP-Unindexed-Diagnostic": []any{marker},
642+
},
643+
})
644+
if err != nil {
645+
t.Fatalf("marshal payload: %v", err)
646+
}
647+
648+
event, err := NormalizeRaw(payload)
649+
if err != nil {
650+
t.Fatalf("normalize successful response: %v", err)
651+
}
652+
if event.Failed || event.FailBody != "" || event.FailSummary != "" {
653+
t.Fatalf("successful failure fields = failed:%v body:%q summary:%q", event.Failed, event.FailBody, event.FailSummary)
654+
}
655+
if event.ResponseMetadata == nil || event.ResponseMetadata.Response == nil ||
656+
event.ResponseMetadata.Response.ContentType != "application/json" || event.ResponseMetadataJSON == "" {
657+
t.Fatalf("response metadata = %#v json=%q", event.ResponseMetadata, event.ResponseMetadataJSON)
658+
}
659+
if !strings.Contains(event.RawJSON, marker) {
660+
t.Fatalf("raw json did not preserve response headers")
661+
}
662+
}
663+
620664
func TestNormalizeRawReadsAnthropicCacheUsageFields(t *testing.T) {
621665
payload := `{
622666
"timestamp": "2026-04-25T00:00:00Z",

‎apps/manager-server/internal/worker/account_action_candidate_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,50 @@ func TestAccountActionCandidateFromEventUsesHeaderErrorCode(t *testing.T) {
139139
}
140140
}
141141

142+
func TestAccountActionCandidateUsesNormalizedHeaderMetadataWithoutFailBodyHeaders(t *testing.T) {
143+
payload := `{
144+
"timestamp": "2026-04-25T00:00:00Z",
145+
"failed": true,
146+
"fail": {"status_code": 401, "body": "upstream rejected request"},
147+
"provider": "codex",
148+
"model": "gpt-5.4",
149+
"endpoint": "POST /v1/chat/completions",
150+
"auth_file_snapshot": "codex-auth.json",
151+
"auth_index": "auth-1",
152+
"account_snapshot": "user@example.com",
153+
"response_headers": {
154+
"X-OpenAI-IDE-Error-Code": ["token_invalidated"]
155+
}
156+
}`
157+
event, err := usage.NormalizeRaw([]byte(payload))
158+
if err != nil {
159+
t.Fatalf("normalize header-only account action event: %v", err)
160+
}
161+
if event.FailBody != "upstream rejected request" || strings.Contains(event.FailBody, "token_invalidated") {
162+
t.Fatalf("fail body = %q", event.FailBody)
163+
}
164+
if event.HeaderErrorKind != "auth" || event.HeaderErrorCode != "token_invalidated" {
165+
t.Fatalf("header error = kind:%q code:%q metadata:%#v", event.HeaderErrorKind, event.HeaderErrorCode, event.ResponseMetadata)
166+
}
167+
168+
// Ensure classification is supplied by structured metadata, not raw JSON.
169+
event.RawJSON = ""
170+
candidate, ok := accountActionCandidateFromEvent(event, time.Now())
171+
if !ok {
172+
t.Fatal("candidate not detected from structured response metadata")
173+
}
174+
if candidate.ActionType != model.AccountActionTypeReauth || candidate.ReasonCode != credentialpolicy.ReasonTokenRevoked {
175+
t.Fatalf("candidate = %#v", candidate)
176+
}
177+
var evidence map[string]any
178+
if err := json.Unmarshal([]byte(candidate.EvidenceJSON), &evidence); err != nil {
179+
t.Fatalf("decode evidence: %v", err)
180+
}
181+
if evidence["errorCode"] != "token_invalidated" {
182+
t.Fatalf("evidence = %#v", evidence)
183+
}
184+
}
185+
142186
func TestAccountActionCandidateFromEventClassifiesXAIAuthenticationFailures(t *testing.T) {
143187
shouldNotRetry := false
144188
tests := []struct {

‎apps/manager-server/internal/worker/rate_limit_auto_disable_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -941,6 +941,48 @@ func TestQuotaAutoDisableCandidateUsesResponseHeaderReset(t *testing.T) {
941941
}
942942
}
943943

944+
func TestQuotaAutoDisableCandidateUsesNormalizedHeaderMetadataWithoutFailBodyHeaders(t *testing.T) {
945+
now := time.Unix(1_700_000_000, 0)
946+
payload := fmt.Sprintf(`{
947+
"timestamp": %q,
948+
"failed": true,
949+
"fail": {"status_code": 429, "body": "rate limit exceeded"},
950+
"provider": "codex",
951+
"model": "gpt-5.4",
952+
"endpoint": "POST /v1/chat/completions",
953+
"auth_file_snapshot": "codex-auth.json",
954+
"auth_index": "auth-1",
955+
"account_snapshot": "user@example.com",
956+
"response_headers": {
957+
"x-codex-rate-limit-reached-type": ["primary"],
958+
"x-codex-primary-used-percent": ["100"],
959+
"x-codex-primary-reset-after-seconds": ["300"],
960+
"x-codex-primary-window-minutes": ["300"]
961+
}
962+
}`, now.UTC().Format(time.RFC3339Nano))
963+
event, err := usage.NormalizeRaw([]byte(payload))
964+
if err != nil {
965+
t.Fatalf("normalize header-only quota event: %v", err)
966+
}
967+
if event.FailBody != "rate limit exceeded" || strings.Contains(event.FailBody, "x-codex") {
968+
t.Fatalf("fail body = %q", event.FailBody)
969+
}
970+
if event.ResponseMetadata == nil || event.ResponseMetadata.Quota == nil {
971+
t.Fatalf("response metadata = %#v", event.ResponseMetadata)
972+
}
973+
974+
// Isolate the structured path: historical raw/failure fallbacks are covered
975+
// separately and must not be required for newly normalized events.
976+
event.RawJSON = ""
977+
candidate, ok := quotaAutoDisableCandidateFromEvent(event, "http://cpa", "key", now)
978+
if !ok {
979+
t.Fatal("candidate not detected from structured response metadata")
980+
}
981+
if got := candidate.ResetAt.Unix(); got != now.Add(5*time.Minute).Unix() {
982+
t.Fatalf("reset unix = %d", got)
983+
}
984+
}
985+
944986
func TestQuotaAutoDisableCandidateUsesReachedWindowResetWithoutReachedType(t *testing.T) {
945987
now := time.Unix(1_700_000_000, 0)
946988
event := usage.Event{

0 commit comments

Comments
 (0)