Skip to content

fix: truncate tracking data to prevent Prisma length mismatch errors … #4269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
KAUSHALCODER123 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: truncate tracking data to prevent Prisma length mismatch errors … #4269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/lib/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,18 @@ export const DATETIME_REGEX =
export const URL_LENGTH = 500;
export const PAGE_TITLE_LENGTH = 500;
export const EVENT_NAME_LENGTH = 50;
export const UTM_LENGTH = 255;
export const CLICK_ID_LENGTH = 255;
export const TAG_LENGTH = 50;
export const HOSTNAME_LENGTH = 100;
export const DATA_LENGTH = 500;
export const CURRENCY_LENGTH = 10;
export const SESSION_FIELD_LENGTH = 20;
export const SCREEN_LENGTH = 11;
export const LANGUAGE_LENGTH = 35;
export const COUNTRY_LENGTH = 2;
export const CITY_LENGTH = 50;
export const DISTINCT_ID_LENGTH = 50;

export const UTM_PARAMS = ['utm_campaign', 'utm_content', 'utm_medium', 'utm_source', 'utm_term'];

Expand Down
65 changes: 37 additions & 28 deletions src/queries/sql/events/saveEvent.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_NAME_LENGTH, PAGE_TITLE_LENGTH, URL_LENGTH } from '@/lib/constants';
import {
CLICK_ID_LENGTH,
CURRENCY_LENGTH,
EVENT_NAME_LENGTH,
HOSTNAME_LENGTH,
PAGE_TITLE_LENGTH,
TAG_LENGTH,
URL_LENGTH,
UTM_LENGTH,
} from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import kafka from '@/lib/kafka';
Expand Down Expand Up @@ -112,25 +121,25 @@ async function relationalQuery({
visitId,
urlPath: urlPath?.substring(0, URL_LENGTH),
urlQuery: urlQuery?.substring(0, URL_LENGTH),
utmSource,
utmMedium,
utmCampaign,
utmContent,
utmTerm,
utmSource: utmSource?.substring(0, UTM_LENGTH),
utmMedium: utmMedium?.substring(0, UTM_LENGTH),
utmCampaign: utmCampaign?.substring(0, UTM_LENGTH),
utmContent: utmContent?.substring(0, UTM_LENGTH),
utmTerm: utmTerm?.substring(0, UTM_LENGTH),
referrerPath: referrerPath?.substring(0, URL_LENGTH),
referrerQuery: referrerQuery?.substring(0, URL_LENGTH),
referrerDomain: referrerDomain?.substring(0, URL_LENGTH),
pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH),
gclid,
fbclid,
msclkid,
ttclid,
lifatid,
twclid,
gclid: gclid?.substring(0, CLICK_ID_LENGTH),
fbclid: fbclid?.substring(0, CLICK_ID_LENGTH),
msclkid: msclkid?.substring(0, CLICK_ID_LENGTH),
ttclid: ttclid?.substring(0, CLICK_ID_LENGTH),
lifatid: lifatid?.substring(0, CLICK_ID_LENGTH),
twclid: twclid?.substring(0, CLICK_ID_LENGTH),
eventType,
eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
tag,
hostname,
tag: tag?.substring(0, TAG_LENGTH),
hostname: hostname?.substring(0, HOSTNAME_LENGTH),
lcp,
inp,
cls,
Expand Down Expand Up @@ -159,7 +168,7 @@ async function relationalQuery({
sessionId,
eventId: websiteEventId,
eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
currency,
currency: currency?.substring(0, CURRENCY_LENGTH),
revenue,
createdAt,
});
Expand Down Expand Up @@ -223,32 +232,32 @@ async function clickhouseQuery({
city: city,
url_path: urlPath?.substring(0, URL_LENGTH),
url_query: urlQuery?.substring(0, URL_LENGTH),
utm_source: utmSource,
utm_medium: utmMedium,
utm_campaign: utmCampaign,
utm_content: utmContent,
utm_term: utmTerm,
utm_source: utmSource?.substring(0, UTM_LENGTH),
utm_medium: utmMedium?.substring(0, UTM_LENGTH),
utm_campaign: utmCampaign?.substring(0, UTM_LENGTH),
utm_content: utmContent?.substring(0, UTM_LENGTH),
utm_term: utmTerm?.substring(0, UTM_LENGTH),
referrer_path: referrerPath?.substring(0, URL_LENGTH),
referrer_query: referrerQuery?.substring(0, URL_LENGTH),
referrer_domain: referrerDomain?.substring(0, URL_LENGTH),
page_title: pageTitle?.substring(0, PAGE_TITLE_LENGTH),
gclid: gclid,
fbclid: fbclid,
msclkid: msclkid,
ttclid: ttclid,
li_fat_id: lifatid,
twclid: twclid,
gclid: gclid?.substring(0, CLICK_ID_LENGTH),
fbclid: fbclid?.substring(0, CLICK_ID_LENGTH),
msclkid: msclkid?.substring(0, CLICK_ID_LENGTH),
ttclid: ttclid?.substring(0, CLICK_ID_LENGTH),
li_fat_id: lifatid?.substring(0, CLICK_ID_LENGTH),
twclid: twclid?.substring(0, CLICK_ID_LENGTH),
event_type: eventType,
event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
tag: tag,
tag: tag?.substring(0, TAG_LENGTH),
distinct_id: distinctId,
created_at: getUTCString(createdAt),
browser: browser,
os: os,
device: device,
screen: screen,
language: language,
hostname: hostname,
hostname: hostname?.substring(0, HOSTNAME_LENGTH),
lcp: lcp,
inp: inp,
cls: cls,
Expand Down
10 changes: 5 additions & 5 deletions src/queries/sql/events/saveEventData.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import clickhouse from '@/lib/clickhouse';
import { DATA_TYPE } from '@/lib/constants';
import { DATA_LENGTH, DATA_TYPE } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { flattenJSON, getStringValue } from '@/lib/data';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
Expand Down Expand Up @@ -34,8 +34,8 @@ async function relationalQuery(data: SaveEventDataArgs) {
id: uuid(),
websiteEventId: eventId,
websiteId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
dataKey: a.key?.substring(0, DATA_LENGTH),
stringValue: getStringValue(a.value, a.dataType)?.substring(0, DATA_LENGTH),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
Expand All @@ -62,9 +62,9 @@ async function clickhouseQuery(data: SaveEventDataArgs) {
event_id: eventId,
url_path: urlPath,
event_name: eventName,
data_key: key,
data_key: key?.substring(0, DATA_LENGTH),
data_type: dataType,
string_value: getStringValue(value, dataType),
string_value: getStringValue(value, dataType)?.substring(0, DATA_LENGTH),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
created_at: getUTCString(createdAt),
Expand Down
21 changes: 20 additions & 1 deletion src/queries/sql/sessions/createSession.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
import type { Prisma } from '@/generated/prisma/client';
import {
CITY_LENGTH,
COUNTRY_LENGTH,
DISTINCT_ID_LENGTH,
LANGUAGE_LENGTH,
SCREEN_LENGTH,
SESSION_FIELD_LENGTH,
} from '@/lib/constants';
import prisma from '@/lib/prisma';

const FUNCTION_NAME = 'createSession';
Expand Down Expand Up @@ -38,7 +46,18 @@ export async function createSession(data: Prisma.SessionCreateInput) {
)
on conflict (session_id) do nothing
`,
data,
{
...data,
browser: data.browser?.substring(0, SESSION_FIELD_LENGTH),
os: data.os?.substring(0, SESSION_FIELD_LENGTH),
device: data.device?.substring(0, SESSION_FIELD_LENGTH),
screen: data.screen?.substring(0, SCREEN_LENGTH),
language: data.language?.substring(0, LANGUAGE_LENGTH),
country: data.country?.substring(0, COUNTRY_LENGTH),
region: data.region?.substring(0, SESSION_FIELD_LENGTH),
city: data.city?.substring(0, CITY_LENGTH),
distinctId: data.distinctId?.substring(0, DISTINCT_ID_LENGTH),
},
FUNCTION_NAME,
);
}
10 changes: 5 additions & 5 deletions src/queries/sql/sessions/saveSessionData.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import clickhouse from '@/lib/clickhouse';
import { DATA_TYPE } from '@/lib/constants';
import { DATA_LENGTH, DATA_TYPE } from '@/lib/constants';
Copy link
Copy Markdown
Contributor

@greptile-apps greptile-apps Bot May 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Import DISTINCT_ID_LENGTH and apply it to distinctId so the Prisma write doesn't exceed the @db.VarChar(50) limit on SessionData.distinctId.

Suggested change
import { DATA_LENGTH, DATA_TYPE } from '@/lib/constants';
import { DATA_LENGTH, DISTINCT_ID_LENGTH, DATA_TYPE } from '@/lib/constants';

import { uuid } from '@/lib/crypto';
import { flattenJSON, getStringValue } from '@/lib/data';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
Expand Down Expand Up @@ -37,8 +37,8 @@ export async function relationalQuery({
id: uuid(),
websiteId,
sessionId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
dataKey: a.key?.substring(0, DATA_LENGTH),
stringValue: getStringValue(a.value, a.dataType)?.substring(0, DATA_LENGTH),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
Expand Down Expand Up @@ -86,9 +86,9 @@ async function clickhouseQuery({
return {
website_id: websiteId,
session_id: sessionId,
data_key: key,
data_key: key?.substring(0, DATA_LENGTH),
data_type: dataType,
string_value: getStringValue(value, dataType),
string_value: getStringValue(value, dataType)?.substring(0, DATA_LENGTH),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
distinct_id: distinctId,
Expand Down