fix: always log metadata for free tier exceeded, skip only S3 bodies (#5545)

* fix: PTB should read bodies from S3 even when freeLimitExceeded

PTB requests always store bodies to S3 for billing purposes, so Jawn
should not treat them as "not_stored_exceeded_free". Only non-PTB
requests with freeLimitExceeded should skip S3 body reading.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* feat: always log metadata to ClickHouse, skip only S3 bodies for exceeded free tier

- Remove early exit that skipped Kafka logging entirely
- Metadata (request count, latency, tokens, cost) always logged to ClickHouse
- Only S3 body storage is skipped for non-PTB exceeded free tier
- This allows tracking request volume for exceeded orgs

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: PTB with free tier exceeded skips S3 only if usage extraction succeeded

- PTB + got usage: skip S3 bodies (we have what we need for billing)
- PTB + failed to get usage: store S3 bodies (Jawn needs to extract)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* chore: fix outdated comment about early exit

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Justin Torre <justin@Justins-MacBook-Pro.local>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: 3 people

valhalla/jawn/src/lib/handlers/LoggingHandler.ts

@@ -172,10 +172,12 @@ export class LoggingHandler extends AbstractLogHandler {
 
       context.sizeBytes = size ?? 0;
       // Determine storage location:
-      // 1. If free tier limit exceeded, bodies were not stored
+      // 1. If free tier limit exceeded AND not PTB, bodies were not stored
       // 2. If size is small enough, use clickhouse
       // 3. Otherwise use s3
-      if (context.message.heliconeMeta.freeLimitExceeded) {
+      // Note: PTB always stores bodies to S3 for billing purposes
+      const isPTB = context.message.heliconeMeta.isPassthroughBilling;
+      if (context.message.heliconeMeta.freeLimitExceeded && !isPTB) {
         context.storageLocation = "not_stored_exceeded_free";
       } else {
         context.storageLocation =

worker/src/lib/dbLogger/DBLoggable.ts

@@ -797,14 +797,9 @@ export class DBLoggable {
       org.data.tier === "free" && org.data.freeLimitExceeded === currentMonth;
     const isPassthroughBilling = this.request.escrowInfo ? true : false;
 
-    // Skip logging entirely if free limit exceeded and not PTB
-    // This saves processing - we don't need to extract usage or send to Kafka
-    if (freeLimitExceeded && !isPassthroughBilling) {
-      console.log(
-        `[FreeTierLimit] Skipping logging entirely for org ${authParams.organizationId} - free tier limit exceeded and not PTB`
-      );
-      return ok(null);
-    }
+    // Note: We always log metadata to Kafka/ClickHouse even when free limit exceeded
+    // This allows tracking request volume for exceeded orgs
+    // We just skip S3 body storage for non-PTB exceeded requests
 
     const { body: rawResponseBody, endTime: responseEndTime } =
       await this.response.getResponseBody();
@@ -840,13 +835,19 @@ export class DBLoggable {
       failedToGetUsage = true;
     }
 
-    // Skip S3 storage only if BOTH request and response are omitted
-    // If only one is omitted, still store to S3 - Jawn will respect the omit flags
-    // IMPORTANT: For PTB, if we failed to get usage, always store to S3 so Jawn can extract it for billing
-    const skipS3Storage =
+    // Skip S3 storage if:
+    // 1. Free tier limit exceeded AND (not PTB OR we got usage successfully)
+    //    - Non-PTB: always skip bodies
+    //    - PTB with usage: skip bodies (we have what we need for billing)
+    //    - PTB without usage: store bodies (Jawn needs to extract for billing)
+    // 2. Both omit headers are set (but not if PTB failed to get usage)
+    const skipS3ForFreeTier =
+      freeLimitExceeded && (!isPassthroughBilling || !failedToGetUsage);
+    const skipS3ForOmitHeaders =
       !(isPassthroughBilling && failedToGetUsage) &&
       requestHeaders?.omitHeaders?.omitRequest === true &&
       requestHeaders?.omitHeaders?.omitResponse === true;
+    const skipS3Storage = skipS3ForFreeTier || skipS3ForOmitHeaders;
 
     if (S3_ENABLED === "true" && !skipS3Storage) {
       try {
@@ -952,8 +953,7 @@ export class DBLoggable {
           this.request.attempt?.endpoint.providerModelId ?? undefined,
         stripeCustomerId: requestHeaders.stripeCustomerId ?? undefined,
         aiGatewayBodyMapping: aiGatewayBodyMapping ?? undefined,
-        // Note: If we reach here with freeLimitExceeded=true, it means PTB is enabled
-        // (non-PTB exceeded requests exit early above)
+        // Pass freeLimitExceeded to Jawn so it knows bodies may not be in S3
         freeLimitExceeded: freeLimitExceeded ? true : undefined,
       },
       log: {