add more truncation fields on save

Author: franciscao633

src/lib/constants.ts

@@ -262,6 +262,9 @@ export const DATETIME_REGEX =
 export const URL_LENGTH = 500;
 export const PAGE_TITLE_LENGTH = 500;
 export const EVENT_NAME_LENGTH = 50;
+export const TAG_LENGTH = 50;
+export const HOSTNAME_LENGTH = 100;
+export const FIELD_VALUE_LENGTH = 255;
 
 export const UTM_PARAMS = ['utm_campaign', 'utm_content', 'utm_medium', 'utm_source', 'utm_term'];
 

src/queries/sql/events/saveEvent.ts

@@ -1,5 +1,12 @@
 import clickhouse from '@/lib/clickhouse';
-import { EVENT_NAME_LENGTH, PAGE_TITLE_LENGTH, URL_LENGTH } from '@/lib/constants';
+import {
+  EVENT_NAME_LENGTH,
+  FIELD_VALUE_LENGTH,
+  HOSTNAME_LENGTH,
+  PAGE_TITLE_LENGTH,
+  TAG_LENGTH,
+  URL_LENGTH,
+} from '@/lib/constants';
 import { uuid } from '@/lib/crypto';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import kafka from '@/lib/kafka';
@@ -69,6 +76,10 @@ export async function saveEvent(args: SaveEventArgs) {
   });
 }
 
+function truncate(value: string | null | undefined, maxLength: number) {
+  return value ? value.substring(0, maxLength) : value;
+}
+
 async function relationalQuery({
   websiteId,
   sessionId,
@@ -110,27 +121,27 @@ async function relationalQuery({
       websiteId,
       sessionId,
       visitId,
-      urlPath: urlPath?.substring(0, URL_LENGTH),
-      urlQuery: urlQuery?.substring(0, URL_LENGTH),
-      utmSource,
-      utmMedium,
-      utmCampaign,
-      utmContent,
-      utmTerm,
-      referrerPath: referrerPath?.substring(0, URL_LENGTH),
-      referrerQuery: referrerQuery?.substring(0, URL_LENGTH),
-      referrerDomain: referrerDomain?.substring(0, URL_LENGTH),
-      pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH),
-      gclid,
-      fbclid,
-      msclkid,
-      ttclid,
-      lifatid,
-      twclid,
+      urlPath: truncate(urlPath, URL_LENGTH),
+      urlQuery: truncate(urlQuery, URL_LENGTH),
+      utmSource: truncate(utmSource, FIELD_VALUE_LENGTH),
+      utmMedium: truncate(utmMedium, FIELD_VALUE_LENGTH),
+      utmCampaign: truncate(utmCampaign, FIELD_VALUE_LENGTH),
+      utmContent: truncate(utmContent, FIELD_VALUE_LENGTH),
+      utmTerm: truncate(utmTerm, FIELD_VALUE_LENGTH),
+      referrerPath: truncate(referrerPath, URL_LENGTH),
+      referrerQuery: truncate(referrerQuery, URL_LENGTH),
+      referrerDomain: truncate(referrerDomain, URL_LENGTH),
+      pageTitle: truncate(pageTitle, PAGE_TITLE_LENGTH),
+      gclid: truncate(gclid, FIELD_VALUE_LENGTH),
+      fbclid: truncate(fbclid, FIELD_VALUE_LENGTH),
+      msclkid: truncate(msclkid, FIELD_VALUE_LENGTH),
+      ttclid: truncate(ttclid, FIELD_VALUE_LENGTH),
+      lifatid: truncate(lifatid, FIELD_VALUE_LENGTH),
+      twclid: truncate(twclid, FIELD_VALUE_LENGTH),
       eventType,
-      eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
-      tag,
-      hostname,
+      eventName: truncate(eventName, EVENT_NAME_LENGTH) ?? null,
+      tag: truncate(tag, TAG_LENGTH),
+      hostname: truncate(hostname, HOSTNAME_LENGTH),
       lcp,
       inp,
       cls,
@@ -145,8 +156,8 @@ async function relationalQuery({
       websiteId,
       sessionId,
       eventId: websiteEventId,
-      urlPath: urlPath?.substring(0, URL_LENGTH),
-      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
+      urlPath: truncate(urlPath, URL_LENGTH),
+      eventName: truncate(eventName, EVENT_NAME_LENGTH),
       eventData,
       createdAt,
     });
@@ -158,7 +169,7 @@ async function relationalQuery({
         websiteId,
         sessionId,
         eventId: websiteEventId,
-        eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
+        eventName: truncate(eventName, EVENT_NAME_LENGTH),
         currency,
         revenue,
         createdAt,
@@ -221,34 +232,34 @@ async function clickhouseQuery({
     country: country,
     region: country && region ? (region.includes('-') ? region : `${country}-${region}`) : null,
     city: city,
-    url_path: urlPath?.substring(0, URL_LENGTH),
-    url_query: urlQuery?.substring(0, URL_LENGTH),
-    utm_source: utmSource,
-    utm_medium: utmMedium,
-    utm_campaign: utmCampaign,
-    utm_content: utmContent,
-    utm_term: utmTerm,
-    referrer_path: referrerPath?.substring(0, URL_LENGTH),
-    referrer_query: referrerQuery?.substring(0, URL_LENGTH),
-    referrer_domain: referrerDomain?.substring(0, URL_LENGTH),
-    page_title: pageTitle?.substring(0, PAGE_TITLE_LENGTH),
-    gclid: gclid,
-    fbclid: fbclid,
-    msclkid: msclkid,
-    ttclid: ttclid,
-    li_fat_id: lifatid,
-    twclid: twclid,
+    url_path: truncate(urlPath, URL_LENGTH),
+    url_query: truncate(urlQuery, URL_LENGTH),
+    utm_source: truncate(utmSource, FIELD_VALUE_LENGTH),
+    utm_medium: truncate(utmMedium, FIELD_VALUE_LENGTH),
+    utm_campaign: truncate(utmCampaign, FIELD_VALUE_LENGTH),
+    utm_content: truncate(utmContent, FIELD_VALUE_LENGTH),
+    utm_term: truncate(utmTerm, FIELD_VALUE_LENGTH),
+    referrer_path: truncate(referrerPath, URL_LENGTH),
+    referrer_query: truncate(referrerQuery, URL_LENGTH),
+    referrer_domain: truncate(referrerDomain, URL_LENGTH),
+    page_title: truncate(pageTitle, PAGE_TITLE_LENGTH),
+    gclid: truncate(gclid, FIELD_VALUE_LENGTH),
+    fbclid: truncate(fbclid, FIELD_VALUE_LENGTH),
+    msclkid: truncate(msclkid, FIELD_VALUE_LENGTH),
+    ttclid: truncate(ttclid, FIELD_VALUE_LENGTH),
+    li_fat_id: truncate(lifatid, FIELD_VALUE_LENGTH),
+    twclid: truncate(twclid, FIELD_VALUE_LENGTH),
     event_type: eventType,
-    event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
-    tag: tag,
+    event_name: truncate(eventName, EVENT_NAME_LENGTH) ?? null,
+    tag: truncate(tag, TAG_LENGTH),
     distinct_id: distinctId,
     created_at: getUTCString(createdAt),
     browser: browser,
     os: os,
     device: device,
     screen: screen,
     language: language,
-    hostname: hostname,
+    hostname: truncate(hostname, HOSTNAME_LENGTH),
     lcp: lcp,
     inp: inp,
     cls: cls,
@@ -267,8 +278,8 @@ async function clickhouseQuery({
       websiteId,
       sessionId,
       eventId,
-      urlPath: urlPath?.substring(0, URL_LENGTH),
-      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
+      urlPath: truncate(urlPath, URL_LENGTH),
+      eventName: truncate(eventName, EVENT_NAME_LENGTH),
       eventData,
       createdAt,
     });