diff --git a/public/intl/messages/en-US.json b/public/intl/messages/en-US.json index d4a150ead6..3c2695865f 100644 --- a/public/intl/messages/en-US.json +++ b/public/intl/messages/en-US.json @@ -4,6 +4,7 @@ "account": "Account", "action": "Action", "actions": "Actions", + "active": "Active", "activity": "Activity", "add": "Add", "add-board": "Add board", @@ -51,6 +52,7 @@ "cohorts": "Cohorts", "compare": "Compare", "compare-dates": "Compare dates", + "change": "Change", "confirm": "Confirm", "confirm-password": "Confirm password", "contains": "Contains", @@ -153,6 +155,7 @@ "insights-description": "Dive deeper into your data by using segments and filters.", "inp": "INP", "invalid-url": "Invalid URL", + "inactive": "Inactive", "is": "Is", "is-false": "Is false", "is-not": "Is not", @@ -309,7 +312,9 @@ "sms": "SMS", "source": "Source", "sources": "Sources", + "sort-by": "Sort by", "start-step": "Start Step", + "status": "Status", "steps": "Steps", "sum": "Sum", "support": "Support", @@ -342,6 +347,7 @@ "toggle-charts": "Toggle charts", "total": "Total", "total-records": "Total records", + "results-per-page": "Results per page", "tracking-code": "Tracking code", "traffic": "Traffic", "transactions": "Transactions", @@ -384,6 +390,8 @@ "visit-duration": "Visit duration", "visitors": "Visitors", "visits": "Visits", + "most-visitors-today": "Most visitors today", + "most-views-today": "Most views today", "website": "Website", "website-id": "Website ID", "websites": "Websites", diff --git a/src/app/(main)/websites/WebsitesDataTable.tsx b/src/app/(main)/websites/WebsitesDataTable.tsx index 5527848716..e27b7a1de0 100644 --- a/src/app/(main)/websites/WebsitesDataTable.tsx +++ b/src/app/(main)/websites/WebsitesDataTable.tsx @@ -1,44 +1,164 @@ -import { Icon, Row } from '@umami/react-zen'; +import { Icon, ListItem, Row, Select, Text } from '@umami/react-zen'; import Link from '@/components/common/Link'; import { DataGrid } from '@/components/common/DataGrid'; -import { useLoginQuery, useNavigation, useUserWebsitesQuery } from '@/components/hooks'; +import { + useLoginQuery, + useMessages, + useNavigation, + useTimezone, + useUserWebsitesQuery, +} from '@/components/hooks'; import { Favicon } from '@/index'; +import { DEFAULT_PAGE_SIZE } from '@/lib/constants'; import { WebsitesTable } from './WebsitesTable'; +const PAGE_SIZE_OPTIONS = [20, 50, 100]; +const SORT_OPTIONS = [ + { + id: 'visitors', + orderBy: 'visitors', + sortDescending: true, + }, + { + id: 'pageviews', + orderBy: 'pageviews', + sortDescending: true, + }, + { + id: 'name', + orderBy: 'name', + sortDescending: false, + }, + { + id: 'domain', + orderBy: 'domain', + sortDescending: false, + }, +] as const; +const DEFAULT_SORT_ID = 'visitors'; +const DEFAULT_SORT = SORT_OPTIONS.find(option => option.id === DEFAULT_SORT_ID) || SORT_OPTIONS[0]; + +function getSortById(id?: string) { + return SORT_OPTIONS.find(option => option.id === id) || DEFAULT_SORT; +} + +function getSelectedSort(orderBy?: string, sortDescending?: boolean | string) { + const isDescending = sortDescending === true || sortDescending === 'true'; + + return ( + SORT_OPTIONS.find( + option => option.orderBy === orderBy && option.sortDescending === isDescending, + ) || DEFAULT_SORT + ); +} + export function WebsitesDataTable({ userId, teamId, allowEdit = true, allowView = true, showActions = true, + showStats = false, }: { userId?: string; teamId?: string; allowEdit?: boolean; allowView?: boolean; showActions?: boolean; + showStats?: boolean; }) { const { user } = useLoginQuery(); - const queryResult = useUserWebsitesQuery({ userId: userId || user?.id, teamId }); - const { renderUrl } = useNavigation(); + const { t, labels } = useMessages(); + const { renderUrl, router, query, updateParams } = useNavigation(); + const { timezone, canonicalizeTimezone } = useTimezone(); + const pageSize = Number(query.pageSize) || DEFAULT_PAGE_SIZE; + const selectedSort = getSelectedSort(query.orderBy, query.sortDescending); + const queryResult = useUserWebsitesQuery( + { userId: userId || user?.id, teamId }, + { + pageSize, + orderBy: selectedSort.orderBy, + sortDescending: selectedSort.sortDescending, + timezone: canonicalizeTimezone(timezone), + includeMetrics: showStats || undefined, + }, + ); const renderLink = (row: any) => ( - {row.name} + + {row.name} + + + ); + + const handlePageSizeChange = (value: string) => { + router.push(updateParams({ page: 1, pageSize: value })); + }; + + const handleSortChange = (value: string) => { + const sort = getSortById(value); + const isDefault = sort.id === DEFAULT_SORT_ID; + + router.push( + updateParams({ + page: 1, + orderBy: isDefault ? undefined : sort.orderBy, + sortDescending: isDefault ? undefined : sort.sortDescending ? 'true' : 'false', + }), + ); + }; + + const renderActions = () => ( + + + + {t(labels.sortBy)} + + + + + + {t(labels.resultsPerPage)} + + + ); return ( - + {({ data }) => ( )} diff --git a/src/app/(main)/websites/WebsitesPage.tsx b/src/app/(main)/websites/WebsitesPage.tsx index 4decd8f196..c58de9c1dd 100644 --- a/src/app/(main)/websites/WebsitesPage.tsx +++ b/src/app/(main)/websites/WebsitesPage.tsx @@ -27,7 +27,7 @@ export function WebsitesPage() { {showActions && } - + diff --git a/src/app/(main)/websites/WebsitesTable.tsx b/src/app/(main)/websites/WebsitesTable.tsx index e1ed4a7d18..84d824f6e7 100644 --- a/src/app/(main)/websites/WebsitesTable.tsx +++ b/src/app/(main)/websites/WebsitesTable.tsx @@ -1,35 +1,205 @@ -import { DataColumn, DataTable, type DataTableProps, Icon } from '@umami/react-zen'; +import { + Column, + DataColumn, + DataTable, + type DataTableProps, + Icon, + Text, + useTheme, +} from '@umami/react-zen'; import type { ReactNode } from 'react'; +import { useMemo } from 'react'; import { DateDistance } from '@/components/common/DateDistance'; import { LinkButton } from '@/components/common/LinkButton'; import { SortableLabel } from '@/components/common/SortableLabel'; import { useMessages, useNavigation } from '@/components/hooks'; import { SquarePen } from '@/components/icons'; +import { ChangeLabel } from '@/components/metrics/ChangeLabel'; +import { getThemeColors } from '@/lib/colors'; +import { formatLongNumber } from '@/lib/format'; export interface WebsitesTableProps extends DataTableProps { showActions?: boolean; allowEdit?: boolean; allowView?: boolean; + showStats?: boolean; renderLink?: (row: any) => ReactNode; } -export function WebsitesTable({ showActions, renderLink, ...props }: WebsitesTableProps) { +function WebsiteMetric({ + label, + value, + formatValue = formatLongNumber, +}: { + label: string; + value?: number; + formatValue?: (value: number) => string; +}) { + return ( + + {formatValue(value || 0)} + + {label} + + + ); +} + +function WebsiteActivitySparkline({ values }: { values?: number[] }) { + const { theme } = useTheme(); + const { colors } = useMemo(() => getThemeColors(theme), [theme]); + const series = Array.from({ length: 7 }, (_, index) => Number(values?.[index]) || 0); + const width = 84; + const height = 32; + const barWidth = 8; + const gap = 4; + const maxValue = Math.max(...series, 1); + + return ( + + ); +} + +function WebsiteStatus({ + isActive, + activeVisitors, + activeLabel, + inactiveLabel, + onlineLabel, +}: { + isActive?: boolean; + activeVisitors?: number; + activeLabel: string; + inactiveLabel: string; + onlineLabel: string; +}) { + return ( + + + {isActive ? activeLabel : inactiveLabel} + + + {activeVisitors ? `${formatLongNumber(activeVisitors)} ${onlineLabel}` : inactiveLabel} + + + ); +} + +export function WebsitesTable({ + showActions, + showStats, + renderLink, + ...props +}: WebsitesTableProps) { const { t, labels } = useMessages(); const { renderUrl } = useNavigation(); return ( - }> - {renderLink} - - } /> } - width="200px" + id={showStats ? 'website' : 'name'} + label={showStats ? t(labels.website) : } > - {(row: any) => } + {(row: any) => { + if (!showStats) { + return renderLink ? renderLink(row) : row.name; + } + + return ( + + {renderLink ? renderLink(row) : {row.name}} + + {row.domain} + + + ); + }} + {!showStats && } />} + {!showStats && ( + } + width="200px" + > + {(row: any) => } + + )} + {showStats && ( + } align="end" width="160px"> + {(row: any) => } + + )} + {showStats && ( + } align="end" width="160px"> + {(row: any) => } + + )} + {showStats && ( + + {(row: any) => ( + `${Math.round(value)}%`} + /> + )} + + )} + {showStats && ( + + {(row: any) => ( + + {`${Math.round(Math.abs(row.metrics?.change || 0))}%`} + + )} + + )} + {showStats && ( + + {(row: any) => } + + )} + {showStats && ( + + {(row: any) => ( + + )} + + )} {showActions && ( {(row: any) => { diff --git a/src/app/api/me/websites/route.ts b/src/app/api/me/websites/route.ts index 809c6e502c..aa540c0789 100644 --- a/src/app/api/me/websites/route.ts +++ b/src/app/api/me/websites/route.ts @@ -1,14 +1,20 @@ import { z } from 'zod'; import { getQueryFilters, parseRequest } from '@/lib/request'; import { json } from '@/lib/response'; -import { pagingParams, sortingParams } from '@/lib/schema'; +import { pagingParams, searchParams, timezoneParam, websiteSortingParams } from '@/lib/schema'; import { getAllUserWebsitesIncludingTeamAccess, getUserWebsites } from '@/queries/prisma'; export async function GET(request: Request) { const schema = z.object({ ...pagingParams, - ...sortingParams, + ...searchParams, + ...websiteSortingParams, + timezone: timezoneParam.optional(), includeTeams: z.string().optional(), + includeMetrics: z + .enum(['true', 'false']) + .transform(value => value === 'true') + .optional(), }); const { auth, query, error } = await parseRequest(request, schema); @@ -17,7 +23,10 @@ export async function GET(request: Request) { return error(); } - const filters = await getQueryFilters(query); + const filters = { + ...(await getQueryFilters(query)), + includeMetrics: query.includeMetrics, + }; if (query.includeTeams) { return json(await getAllUserWebsitesIncludingTeamAccess(auth.user.id, filters)); diff --git a/src/app/api/teams/[teamId]/websites/route.ts b/src/app/api/teams/[teamId]/websites/route.ts index 5d010799bb..97235c89a3 100644 --- a/src/app/api/teams/[teamId]/websites/route.ts +++ b/src/app/api/teams/[teamId]/websites/route.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; import { getQueryFilters, parseRequest } from '@/lib/request'; import { json, unauthorized } from '@/lib/response'; -import { pagingParams, searchParams, sortingParams } from '@/lib/schema'; +import { pagingParams, searchParams, timezoneParam, websiteSortingParams } from '@/lib/schema'; import { canViewTeam } from '@/permissions'; import { getTeamWebsites } from '@/queries/prisma'; @@ -9,7 +9,12 @@ export async function GET(request: Request, { params }: { params: Promise<{ team const schema = z.object({ ...pagingParams, ...searchParams, - ...sortingParams, + ...websiteSortingParams, + timezone: timezoneParam.optional(), + includeMetrics: z + .enum(['true', 'false']) + .transform(value => value === 'true') + .optional(), }); const { teamId } = await params; const { auth, query, error } = await parseRequest(request, schema); @@ -22,7 +27,10 @@ export async function GET(request: Request, { params }: { params: Promise<{ team return unauthorized(); } - const filters = await getQueryFilters(query); + const filters = { + ...(await getQueryFilters(query)), + includeMetrics: query.includeMetrics, + }; const websites = await getTeamWebsites(teamId, filters); diff --git a/src/app/api/users/[userId]/websites/route.ts b/src/app/api/users/[userId]/websites/route.ts index c706dae9e1..ce25d91ecb 100644 --- a/src/app/api/users/[userId]/websites/route.ts +++ b/src/app/api/users/[userId]/websites/route.ts @@ -1,15 +1,20 @@ import { z } from 'zod'; import { getQueryFilters, parseRequest } from '@/lib/request'; import { json, unauthorized } from '@/lib/response'; -import { pagingParams, searchParams, sortingParams } from '@/lib/schema'; +import { pagingParams, searchParams, timezoneParam, websiteSortingParams } from '@/lib/schema'; import { getAllUserWebsitesIncludingTeamAccess, getUserWebsites } from '@/queries/prisma/website'; export async function GET(request: Request, { params }: { params: Promise<{ userId: string }> }) { const schema = z.object({ ...pagingParams, ...searchParams, - ...sortingParams, + ...websiteSortingParams, + timezone: timezoneParam.optional(), includeTeams: z.string().optional(), + includeMetrics: z + .enum(['true', 'false']) + .transform(value => value === 'true') + .optional(), }); const { auth, query, error } = await parseRequest(request, schema); @@ -24,7 +29,10 @@ export async function GET(request: Request, { params }: { params: Promise<{ user return unauthorized(); } - const filters = await getQueryFilters(query); + const filters = { + ...(await getQueryFilters(query)), + includeMetrics: query.includeMetrics, + }; if (query.includeTeams) { return json(await getAllUserWebsitesIncludingTeamAccess(userId, filters)); diff --git a/src/app/api/websites/route.ts b/src/app/api/websites/route.ts index fd4553fb8c..0d913c076a 100644 --- a/src/app/api/websites/route.ts +++ b/src/app/api/websites/route.ts @@ -4,7 +4,7 @@ import { uuid } from '@/lib/crypto'; import { fetchAccount } from '@/lib/load'; import { getQueryFilters, parseRequest } from '@/lib/request'; import { json, unauthorized } from '@/lib/response'; -import { pagingParams, searchParams, sortingParams } from '@/lib/schema'; +import { pagingParams, searchParams, timezoneParam, websiteSortingParams } from '@/lib/schema'; import { canCreateTeamWebsite, canCreateWebsite } from '@/permissions'; import { createShare, createWebsite, getWebsiteCount } from '@/queries/prisma'; import { getAllUserWebsitesIncludingTeamAccess, getUserWebsites } from '@/queries/prisma/website'; @@ -15,8 +15,13 @@ export async function GET(request: Request) { const schema = z.object({ ...pagingParams, ...searchParams, - ...sortingParams, + ...websiteSortingParams, + timezone: timezoneParam.optional(), includeTeams: z.string().optional(), + includeMetrics: z + .enum(['true', 'false']) + .transform(value => value === 'true') + .optional(), }); const { auth, query, error } = await parseRequest(request, schema); @@ -27,7 +32,10 @@ export async function GET(request: Request) { const userId = auth.user.id; - const filters = await getQueryFilters(query); + const filters = { + ...(await getQueryFilters(query)), + includeMetrics: query.includeMetrics, + }; if (query.includeTeams) { return json(await getAllUserWebsitesIncludingTeamAccess(userId, filters)); diff --git a/src/components/messages.ts b/src/components/messages.ts index caa56ef548..3db42adca9 100644 --- a/src/components/messages.ts +++ b/src/components/messages.ts @@ -53,6 +53,7 @@ export const labels: Record = { shareUrl: 'label.share-url', action: 'label.action', actions: 'label.actions', + active: 'label.active', domain: 'label.domain', websiteId: 'label.website-id', resetWebsite: 'label.reset-website', @@ -256,6 +257,10 @@ export const labels: Record = { month: 'label.month', date: 'label.date', pageOf: 'label.page-of', + sortBy: 'label.sort-by', + resultsPerPage: 'label.results-per-page', + mostVisitorsToday: 'label.most-visitors-today', + mostViewsToday: 'label.most-views-today', period: 'label.period', cumulative: 'label.cumulative', create: 'label.create', @@ -292,7 +297,9 @@ export const labels: Record = { heatmapDescription: 'label.heatmap-description', compareDates: 'label.compare-dates', compare: 'label.compare', + change: 'label.change', current: 'label.current', + inactive: 'label.inactive', previous: 'label.previous', previousPeriod: 'label.previous-period', previousYear: 'label.previous-year', @@ -383,6 +390,8 @@ export const labels: Record = { replay: 'label.replay', replayId: 'label.replay-id', replayEnabled: 'label.replay-enabled', + replayCode: 'label.replay-code', + status: 'label.status', recorderCode: 'label.recorder-code', sampleRate: 'label.sample-rate', maskLevel: 'label.mask-level', diff --git a/src/lib/schema.ts b/src/lib/schema.ts index 25a5f47264..f62dac2805 100644 --- a/src/lib/schema.ts +++ b/src/lib/schema.ts @@ -94,6 +94,22 @@ export const sortingParams = { }), }; +export const websiteOrderByParam = z.enum(['name', 'domain', 'createdAt', 'visitors', 'pageviews']); + +export const websiteSortingParams = { + orderBy: websiteOrderByParam.optional(), + sortDescending: z + .enum(['true', 'false']) + .optional() + .transform(value => { + if (value === undefined) { + return undefined; + } + + return value === 'true'; + }), +}; + export const userRoleParam = z.enum(['admin', 'user', 'view-only']); export const teamRoleParam = z.enum(['team-member', 'team-view-only', 'team-manager']); diff --git a/src/queries/prisma/website.ts b/src/queries/prisma/website.ts index 254e27e158..f4c5944122 100644 --- a/src/queries/prisma/website.ts +++ b/src/queries/prisma/website.ts @@ -1,9 +1,425 @@ +import { endOfDay, startOfDay, subDays } from 'date-fns'; +import { toZonedTime, fromZonedTime } from 'date-fns-tz'; import type { Prisma, Website } from '@/generated/prisma/client'; -import { ROLES } from '@/lib/constants'; +import clickhouse from '@/lib/clickhouse'; +import { DEFAULT_PAGE_SIZE, EVENT_TYPE, ROLES } from '@/lib/constants'; +import { normalizeTimezone } from '@/lib/date'; import prisma from '@/lib/prisma'; import redis from '@/lib/redis'; import { sanitizeSortFilters } from '@/lib/sort'; import type { QueryFilters } from '@/lib/types'; +import { + getWebsiteListActiveVisitors, + getWebsiteListActivity, + getWebsiteListStats, + type WebsiteListStats, +} from '@/queries/sql'; + +const ACTIVITY_ORDER_FIELDS = ['pageviews', 'visitors'] as const; +const WEBSITE_ACTIVITY_DAYS = 7; + +interface WebsiteListResult { + data: any[]; + count: any; + page: number; + pageSize: number; + orderBy?: string; + search?: string; + sortDescending?: boolean; +} + +type WebsiteQueryFilters = QueryFilters & { + includeMetrics?: boolean; +}; + +interface WebsiteListMetricsSummary { + pageviews: number; + visitors: number; + visits: number; + bounces: number; +} + +interface WebsiteListComparisonSummary { + visitors: number; +} + +interface WebsiteListDecorationOptions { + currentStats?: WebsiteListStats[]; +} + +function isValidDate(value?: Date) { + return value instanceof Date && !Number.isNaN(value.getTime()); +} + +function isActivityOrderBy(orderBy?: string): orderBy is (typeof ACTIVITY_ORDER_FIELDS)[number] { + return ACTIVITY_ORDER_FIELDS.includes(orderBy as (typeof ACTIVITY_ORDER_FIELDS)[number]); +} + +function getActivityTimezone(filters: QueryFilters = {}) { + const timezone = + filters.timezone && filters.timezone.toLowerCase() !== 'utc' + ? normalizeTimezone(filters.timezone) + : 'UTC'; + + return timezone; +} + +function getTodayDateRange(filters: QueryFilters = {}) { + const timezone = getActivityTimezone(filters); + const zonedNow = toZonedTime(new Date(), timezone); + + return { + startDate: fromZonedTime(startOfDay(zonedNow), timezone), + endDate: fromZonedTime(endOfDay(zonedNow), timezone), + }; +} + +function getActivityDateRange(filters: QueryFilters = {}) { + if (isValidDate(filters.startDate) && isValidDate(filters.endDate)) { + return { + startDate: filters.startDate, + endDate: filters.endDate, + }; + } + + return getTodayDateRange(filters); +} + +function getRecentActivityDateRange(filters: QueryFilters = {}, days = WEBSITE_ACTIVITY_DAYS) { + const timezone = getActivityTimezone(filters); + const zonedNow = toZonedTime(new Date(), timezone); + + return { + startDate: fromZonedTime(startOfDay(subDays(zonedNow, days - 1)), timezone), + endDate: fromZonedTime(endOfDay(zonedNow), timezone), + timezone, + }; +} + +function getPreviousTodayDateRange(filters: QueryFilters = {}) { + const timezone = getActivityTimezone(filters); + const zonedNow = toZonedTime(new Date(), timezone); + const previousDay = subDays(zonedNow, 1); + + return { + startDate: fromZonedTime(startOfDay(previousDay), timezone), + endDate: fromZonedTime(endOfDay(previousDay), timezone), + }; +} + +function getActivityOrderQuery( + orderBy: (typeof ACTIVITY_ORDER_FIELDS)[number], + websiteFilterWhereClause: string, +) { + if (orderBy === 'visitors') { + return ` + select + session.website_id as website_id, + count(*) as value + from session + where session.created_at between {{startDate}} and {{endDate}} + and session.website_id in ( + select filtered_website.website_id + from website filtered_website + where ${websiteFilterWhereClause} + ) + group by session.website_id + `; + } + + if (orderBy === 'pageviews' && clickhouse.enabled) { + return null; + } + + if (orderBy === 'pageviews') { + return ` + select + website_event.website_id as website_id, + count(*) as value + from website_event + where website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type NOT IN (${EVENT_TYPE.customEvent}, ${EVENT_TYPE.performance}) + and website_event.website_id in ( + select filtered_website.website_id + from website filtered_website + where ${websiteFilterWhereClause} + ) + group by website_event.website_id + `; + } + + return null; +} + +function getWebsiteActivityFilter( + where: Prisma.WebsiteWhereInput = {}, + search?: string, + alias = 'website', +): { whereClause: string; queryParams: Record } | null { + const queryParams: Record = {}; + const conditions = [`${alias}.deleted_at is null`]; + + if (search) { + conditions.push(`(${alias}.name ilike {{search}} or ${alias}.domain ilike {{search}})`); + queryParams.search = `%${search}%`; + } + + if (where?.userId) { + conditions.push(`${alias}.user_id = {{userId::uuid}}`); + queryParams.userId = where.userId; + } else if (where?.teamId) { + conditions.push(`${alias}.team_id = {{teamId::uuid}}`); + queryParams.teamId = where.teamId; + } else if (Array.isArray(where?.OR)) { + const userFilter = where.OR.find((item: Prisma.WebsiteWhereInput) => item?.userId); + const teamFilter = where.OR.find( + (item: Prisma.WebsiteWhereInput) => item?.team?.members?.some?.userId, + ); + const teamMemberFilter = teamFilter?.team?.members?.some; + + if (!userFilter?.userId || !teamMemberFilter?.userId || !teamMemberFilter?.role) { + return null; + } + + conditions.push(`( + ${alias}.user_id = {{userId::uuid}} + or exists ( + select 1 + from team + inner join team_user on team_user.team_id = team.team_id + where team.team_id = ${alias}.team_id + and team.deleted_at is null + and team_user.user_id = {{teamUserId::uuid}} + and team_user.role = {{teamUserRole}} + ) + )`); + queryParams.userId = userFilter.userId; + queryParams.teamUserId = teamMemberFilter.userId; + queryParams.teamUserRole = teamMemberFilter.role; + } else { + const unsupportedKeys = Object.keys(where || {}).filter( + key => !['AND', 'deletedAt'].includes(key), + ); + + if (unsupportedKeys.length > 0) { + return null; + } + } + + return { + whereClause: conditions.join('\n and '), + queryParams, + }; +} + +async function fetchActivitySortedWebsitePage( + criteria: Prisma.WebsiteFindManyArgs, + filters: QueryFilters, + orderBy: (typeof ACTIVITY_ORDER_FIELDS)[number], +) { + const activityFilter = getWebsiteActivityFilter(criteria.where, filters.search); + const scopedActivityFilter = getWebsiteActivityFilter( + criteria.where, + filters.search, + 'filtered_website', + ); + const activityOrderQuery = scopedActivityFilter + ? getActivityOrderQuery(orderBy, scopedActivityFilter.whereClause) + : null; + + if (!activityOrderQuery || !activityFilter || !scopedActivityFilter) { + return null; + } + + const { rawQuery } = prisma; + const { page = 1, pageSize, sortDescending = true } = filters; + const size = +pageSize || DEFAULT_PAGE_SIZE; + const offset = size * (+page - 1); + const direction = sortDescending ? 'desc' : 'asc'; + const { startDate, endDate } = getActivityDateRange(filters); + const queryParams = { + ...activityFilter.queryParams, + startDate, + endDate, + }; + + const countResult = (await rawQuery( + ` + select count(*) as count + from website + where ${activityFilter.whereClause} + `, + activityFilter.queryParams, + )) as { count: number }[]; + const [{ count = 0 } = {}] = countResult; + const total = Number(count) || 0; + + if (total === 0) { + return { + ids: [], + count: 0, + page: +page, + pageSize: size, + orderBy, + search: filters.search, + sortDescending, + }; + } + + const rows = (await rawQuery( + ` + select website.website_id as id + from website + left join ( + ${activityOrderQuery} + ) activity on activity.website_id = website.website_id + where ${activityFilter.whereClause} + order by coalesce(activity.value, 0) ${direction}, website.name asc, website.website_id asc + limit ${size} offset ${offset} + `, + queryParams, + )) as { id: string }[]; + + return { + ids: rows.map(row => row.id), + count: total, + page: +page, + pageSize: size, + orderBy, + search: filters.search, + sortDescending, + }; +} + +async function fetchActivitySortedWebsiteDetails( + criteria: Prisma.WebsiteFindManyArgs, + ids: string[], +) { + if (ids.length === 0) { + return []; + } + + const websites = await prisma.client.website.findMany({ + ...criteria, + where: { + id: { + in: ids, + }, + }, + }); + + const websiteById = new Map( + websites.map(website => [website.id, website] as const), + ); + + return ids.map(id => websiteById.get(id)).filter(Boolean); +} + +async function getWebsitesByActivityFallback( + criteria: Prisma.WebsiteFindManyArgs, + filters: WebsiteQueryFilters, + orderBy: (typeof ACTIVITY_ORDER_FIELDS)[number], +) { + const { page = 1, pageSize, sortDescending = true, search } = filters; + const size = +pageSize || DEFAULT_PAGE_SIZE; + const websiteRefs = await prisma.client.website.findMany({ + where: criteria.where, + select: { + id: true, + name: true, + }, + }); + const count = websiteRefs.length; + + if (count === 0) { + return decorateWebsiteList( + { + data: [], + count, + page: +page, + pageSize: size, + orderBy, + search, + sortDescending, + }, + filters, + ); + } + + const { startDate, endDate } = getActivityDateRange(filters); + const stats = await getWebsiteListStats( + websiteRefs.map(website => website.id), + { + startDate, + endDate, + }, + ); + const statsByWebsiteId = new Map( + stats.map( + stat => + [ + stat.websiteId, + { + pageviews: Number(stat.pageviews) || 0, + visitors: Number(stat.visitors) || 0, + visits: Number(stat.visits) || 0, + bounces: Number(stat.bounces) || 0, + }, + ] as const, + ), + ); + const direction = sortDescending ? -1 : 1; + const ids = [...websiteRefs] + .sort((a, b) => { + const aValue = statsByWebsiteId.get(a.id)?.[orderBy] || 0; + const bValue = statsByWebsiteId.get(b.id)?.[orderBy] || 0; + + if (aValue !== bValue) { + return (aValue - bValue) * direction; + } + + return a.name.localeCompare(b.name); + }) + .slice(size * (+page - 1), size * (+page - 1) + size) + .map(website => website.id); + const data = await fetchActivitySortedWebsiteDetails(criteria, ids); + + return decorateWebsiteList( + { + data, + count, + page: +page, + pageSize: size, + orderBy, + search, + sortDescending, + }, + filters, + { currentStats: stats }, + ); +} + +async function getWebsitesByActivity( + criteria: Prisma.WebsiteFindManyArgs, + filters: WebsiteQueryFilters, + orderBy: (typeof ACTIVITY_ORDER_FIELDS)[number], +) { + const pageData = await fetchActivitySortedWebsitePage(criteria, filters, orderBy); + + if (pageData) { + const data = await fetchActivitySortedWebsiteDetails(criteria, pageData.ids); + + return decorateWebsiteList( + { + ...pageData, + data, + }, + filters, + ); + } + + return getWebsitesByActivityFallback(criteria, filters, orderBy); +} const WEBSITE_SORT_FIELDS = ['name', 'domain', 'createdAt'] as const; @@ -25,7 +441,11 @@ export async function getWebsite(websiteId: string) { return attachShareIdToWebsite(website); } -export async function getWebsites(criteria: Prisma.WebsiteFindManyArgs, filters: QueryFilters) { +export async function getWebsites( + criteria: Prisma.WebsiteFindManyArgs, + filters: WebsiteQueryFilters, +) { + const { orderBy } = filters; const sortFilters = sanitizeSortFilters(filters, WEBSITE_SORT_FIELDS); const { search } = sortFilters; const { getSearchParameters, pagedQuery } = prisma; @@ -41,12 +461,19 @@ export async function getWebsites(criteria: Prisma.WebsiteFindManyArgs, filters: deletedAt: null, }; + if (isActivityOrderBy(orderBy)) { + return getWebsitesByActivity({ ...criteria, where }, filters, orderBy); + } + const websites = await pagedQuery('website', { ...criteria, where }, sortFilters); - return attachShareIdToWebsites(websites); + return decorateWebsiteList(websites, filters); } -export async function getAllUserWebsitesIncludingTeamAccess(userId: string, filters?: QueryFilters) { +export async function getAllUserWebsitesIncludingTeamAccess( + userId: string, + filters?: WebsiteQueryFilters, +) { return getWebsites( { where: { @@ -70,7 +497,7 @@ export async function getAllUserWebsitesIncludingTeamAccess(userId: string, filt ); } -export async function getUserWebsites(userId: string, filters?: QueryFilters) { +export async function getUserWebsites(userId: string, filters?: WebsiteQueryFilters) { return getWebsites( { where: { @@ -89,7 +516,7 @@ export async function getUserWebsites(userId: string, filters?: QueryFilters) { ); } -export async function getTeamWebsites(teamId: string, filters?: QueryFilters) { +export async function getTeamWebsites(teamId: string, filters?: WebsiteQueryFilters) { return getWebsites( { where: { @@ -287,8 +714,9 @@ export async function attachShareIdToWebsites(websites: { count: any; page: number; pageSize: number; - orderBy: string; - search: string; + orderBy?: string; + search?: string; + sortDescending?: boolean; }) { const websiteIds = websites.data.map(website => website.id); @@ -309,7 +737,9 @@ export async function attachShareIdToWebsites(websites: { }, }); - const shareByWebsiteId = new Map(shares.map(share => [share.entityId, share.slug])); + const shareByWebsiteId = new Map( + shares.map(share => [share.entityId, share.slug] as const), + ); return { ...websites, @@ -319,3 +749,105 @@ export async function attachShareIdToWebsites(websites: { })), }; } + +async function attachMetricsToWebsites( + websites: WebsiteListResult, + filters: WebsiteQueryFilters = {}, + options: WebsiteListDecorationOptions = {}, +) { + const websiteIds = websites.data.map(website => website.id); + + if (websiteIds.length === 0) { + return websites; + } + + const { startDate, endDate } = getTodayDateRange(filters); + const previousDateRange = getPreviousTodayDateRange(filters); + const recentActivityRange = getRecentActivityDateRange(filters); + const [stats, previousStats, activity, activeVisitors] = await Promise.all([ + options.currentStats ?? + getWebsiteListStats(websiteIds, { + startDate, + endDate, + }), + getWebsiteListStats(websiteIds, previousDateRange), + getWebsiteListActivity(websiteIds, recentActivityRange), + getWebsiteListActiveVisitors(websiteIds), + ]); + + const statsByWebsiteId = new Map( + stats.map( + stat => + [ + stat.websiteId, + { + pageviews: Number(stat.pageviews) || 0, + visitors: Number(stat.visitors) || 0, + visits: Number(stat.visits) || 0, + bounces: Number(stat.bounces) || 0, + }, + ] as const, + ), + ); + const previousStatsByWebsiteId = new Map( + previousStats.map( + stat => + [ + stat.websiteId, + { + visitors: Number(stat.visitors) || 0, + }, + ] as const, + ), + ); + const activityByWebsiteId = new Map( + activity.map(item => [item.websiteId, item.activity.map(value => Number(value) || 0)] as const), + ); + const activeVisitorsByWebsiteId = new Map( + activeVisitors.map(item => [item.websiteId, Number(item.visitors) || 0] as const), + ); + + return { + ...websites, + data: websites.data.map(website => ({ + ...website, + metrics: { + visitors: statsByWebsiteId.get(website.id)?.visitors || 0, + pageviews: statsByWebsiteId.get(website.id)?.pageviews || 0, + bounceRate: (() => { + const visits = statsByWebsiteId.get(website.id)?.visits || 0; + const bounces = statsByWebsiteId.get(website.id)?.bounces || 0; + + return visits > 0 ? (Math.min(visits, bounces) / visits) * 100 : 0; + })(), + change: (() => { + const currentVisitors = statsByWebsiteId.get(website.id)?.visitors || 0; + const previousVisitors = previousStatsByWebsiteId.get(website.id)?.visitors || 0; + + if (previousVisitors === 0) { + return currentVisitors > 0 ? 100 : 0; + } + + return ((currentVisitors - previousVisitors) / previousVisitors) * 100; + })(), + activeVisitors: activeVisitorsByWebsiteId.get(website.id) || 0, + isActive: (activeVisitorsByWebsiteId.get(website.id) || 0) > 0, + activity: activityByWebsiteId.get(website.id) || Array(WEBSITE_ACTIVITY_DAYS).fill(0), + }, + })), + }; +} + +async function decorateWebsiteList( + websites: WebsiteListResult, + filters: WebsiteQueryFilters = {}, + options: WebsiteListDecorationOptions = {}, +) { + if (!filters.includeMetrics) { + return attachShareIdToWebsites(websites); + } + + const websitesWithShares = await attachShareIdToWebsites(websites); + + return attachMetricsToWebsites(websitesWithShares, filters, options); +} diff --git a/src/queries/sql/getWebsiteListActiveVisitors.ts b/src/queries/sql/getWebsiteListActiveVisitors.ts new file mode 100644 index 0000000000..070264f3dc --- /dev/null +++ b/src/queries/sql/getWebsiteListActiveVisitors.ts @@ -0,0 +1,62 @@ +import { subMinutes } from 'date-fns'; +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; + +const FUNCTION_NAME = 'getWebsiteListActiveVisitors'; + +export interface WebsiteListActiveVisitors { + websiteId: string; + visitors: number; +} + +export function getWebsiteListActiveVisitors(...args: [websiteIds: string[]]) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteIds: string[]) { + const { rawQuery } = prisma; + const startDate = subMinutes(new Date(), 5); + + return rawQuery( + ` + select + website_id as "websiteId", + cast(count(distinct session_id) as bigint) as "visitors" + from website_event + where website_id = ANY({{websiteIds}}::uuid[]) + and created_at >= {{startDate}} + group by website_id + `, + { + websiteIds, + startDate, + }, + FUNCTION_NAME, + ) as Promise; +} + +async function clickhouseQuery(websiteIds: string[]) { + const { rawQuery } = clickhouse; + const startDate = subMinutes(new Date(), 5); + + return rawQuery( + ` + select + website_id as websiteId, + count(distinct session_id) as visitors + from website_event + where website_id in {websiteIds:Array(UUID)} + and created_at >= {startDate:DateTime64} + group by website_id + `, + { + websiteIds, + startDate, + }, + FUNCTION_NAME, + ) as Promise; +} diff --git a/src/queries/sql/getWebsiteListActivity.ts b/src/queries/sql/getWebsiteListActivity.ts new file mode 100644 index 0000000000..2e2e897c81 --- /dev/null +++ b/src/queries/sql/getWebsiteListActivity.ts @@ -0,0 +1,138 @@ +import { addDays } from 'date-fns'; +import { formatInTimeZone } from 'date-fns-tz'; +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE } from '@/lib/constants'; +import { normalizeTimezone } from '@/lib/date'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; + +const FUNCTION_NAME = 'getWebsiteListActivity'; + +export interface WebsiteListActivityFilters { + startDate: Date; + endDate: Date; + timezone?: string; +} + +interface WebsiteListActivityBucket { + websiteId: string; + bucket: string; + value: number; +} + +export interface WebsiteListActivity { + websiteId: string; + activity: number[]; +} + +function getRequiredDateRange(filters: WebsiteListActivityFilters) { + const { startDate, endDate } = filters; + const hasValidStartDate = startDate instanceof Date && !Number.isNaN(startDate.getTime()); + const hasValidEndDate = endDate instanceof Date && !Number.isNaN(endDate.getTime()); + + if (!hasValidStartDate || !hasValidEndDate) { + throw new Error('startDate and endDate are required for getWebsiteListActivity'); + } + + return { + startDate, + endDate, + timezone: filters.timezone ? normalizeTimezone(filters.timezone) : 'UTC', + }; +} + +function formatResults( + data: WebsiteListActivityBucket[], + filters: WebsiteListActivityFilters, +): WebsiteListActivity[] { + const { startDate, timezone } = getRequiredDateRange(filters); + const buckets = Array.from({ length: 7 }, (_, index) => + formatInTimeZone(addDays(startDate, index), timezone, 'yyyy-MM-dd'), + ); + const activityByWebsiteId = new Map>(); + + data.forEach(({ websiteId, bucket, value }) => { + const websiteBuckets = activityByWebsiteId.get(websiteId) || new Map(); + + websiteBuckets.set(bucket, Number(value) || 0); + activityByWebsiteId.set(websiteId, websiteBuckets); + }); + + return Array.from(activityByWebsiteId.entries()).map(([websiteId, websiteBuckets]) => ({ + websiteId, + activity: buckets.map(bucket => websiteBuckets.get(bucket) || 0), + })); +} + +export function getWebsiteListActivity( + ...args: [websiteIds: string[], filters: WebsiteListActivityFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteIds: string[], + filters: WebsiteListActivityFilters, +): Promise { + const { rawQuery } = prisma; + const { startDate, endDate, timezone } = getRequiredDateRange(filters); + + const result = (await rawQuery( + ` + select + website_event.website_id as "websiteId", + to_char(date_trunc('day', timezone({{timezone}}, website_event.created_at)), 'YYYY-MM-DD') as "bucket", + cast(count(*) as bigint) as "value" + from website_event + where website_event.website_id = ANY({{websiteIds}}::uuid[]) + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type NOT IN (${EVENT_TYPE.customEvent}, ${EVENT_TYPE.performance}) + group by 1, 2 + order by 2 + `, + { + websiteIds, + startDate, + endDate, + timezone, + }, + FUNCTION_NAME, + )) as WebsiteListActivityBucket[]; + + return formatResults(result, { startDate, endDate, timezone }); +} + +async function clickhouseQuery( + websiteIds: string[], + filters: WebsiteListActivityFilters, +): Promise { + const { rawQuery } = clickhouse; + const { startDate, endDate, timezone } = getRequiredDateRange(filters); + + const result = (await rawQuery( + ` + select + website_id as websiteId, + formatDateTime(toTimezone(created_at, {timezone:String}), '%Y-%m-%d') as bucket, + sum(views) as value + from website_event_stats_hourly + where website_id in {websiteIds:Array(UUID)} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type NOT IN (${EVENT_TYPE.customEvent}, ${EVENT_TYPE.performance}) + group by website_id, bucket + order by bucket + `, + { + websiteIds, + startDate, + endDate, + timezone, + }, + FUNCTION_NAME, + )) as WebsiteListActivityBucket[]; + + return formatResults(result, { startDate, endDate, timezone }); +} diff --git a/src/queries/sql/getWebsiteListStats.ts b/src/queries/sql/getWebsiteListStats.ts new file mode 100644 index 0000000000..1050d0c202 --- /dev/null +++ b/src/queries/sql/getWebsiteListStats.ts @@ -0,0 +1,117 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import type { QueryFilters } from '@/lib/types'; + +const FUNCTION_NAME = 'getWebsiteListStats'; + +export interface WebsiteListStatsFilters extends QueryFilters { + startDate: Date; + endDate: Date; +} + +export interface WebsiteListStats { + websiteId: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; +} + +function getRequiredDateRange(filters: WebsiteListStatsFilters) { + const { startDate, endDate } = filters; + const hasValidStartDate = startDate instanceof Date && !Number.isNaN(startDate.getTime()); + const hasValidEndDate = endDate instanceof Date && !Number.isNaN(endDate.getTime()); + + if (!hasValidStartDate || !hasValidEndDate) { + throw new Error('startDate and endDate are required for getWebsiteListStats'); + } + + return { startDate, endDate }; +} + +export function getWebsiteListStats( + ...args: [websiteIds: string[], filters: WebsiteListStatsFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +function relationalQuery( + websiteIds: string[], + filters: WebsiteListStatsFilters, +): Promise { + const { rawQuery } = prisma; + const { startDate, endDate } = getRequiredDateRange(filters); + + return rawQuery( + ` + select + t.website_id as "websiteId", + cast(coalesce(sum(t.c), 0) as bigint) as "pageviews", + cast(count(distinct t.session_id) as bigint) as "visitors", + cast(count(distinct t.visit_id) as bigint) as "visits", + cast(coalesce(sum(case when t.c = 1 then 1 else 0 end), 0) as bigint) as "bounces" + from ( + select + website_event.website_id, + website_event.session_id, + website_event.visit_id, + count(*) as "c" + from website_event + where website_event.website_id = ANY({{websiteIds}}::uuid[]) + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type NOT IN (${EVENT_TYPE.customEvent}, ${EVENT_TYPE.performance}) + group by 1, 2, 3 + ) as t + group by t.website_id + `, + { + websiteIds, + startDate, + endDate, + }, + FUNCTION_NAME, + ); +} + +function clickhouseQuery( + websiteIds: string[], + filters: WebsiteListStatsFilters, +): Promise { + const { rawQuery } = clickhouse; + const { startDate, endDate } = getRequiredDateRange(filters); + + return rawQuery( + ` + select + website_id as websiteId, + sum(c) as pageviews, + uniq(session_id) as visitors, + uniq(visit_id) as visits, + sumIf(1, c = 1) as bounces + from ( + select + website_id, + session_id, + visit_id, + sum(views) as c + from website_event_stats_hourly + where website_id in {websiteIds:Array(UUID)} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type NOT IN (${EVENT_TYPE.customEvent}, ${EVENT_TYPE.performance}) + group by website_id, session_id, visit_id + ) as t + group by website_id + `, + { + websiteIds, + startDate, + endDate, + }, + FUNCTION_NAME, + ); +} diff --git a/src/queries/sql/index.ts b/src/queries/sql/index.ts index a5bbb41939..58282a5b29 100644 --- a/src/queries/sql/index.ts +++ b/src/queries/sql/index.ts @@ -24,6 +24,9 @@ export * from './getRealtimeActivity'; export * from './getRealtimeData'; export * from './getValues'; export * from './getWebsiteDateRange'; +export * from './getWebsiteListActivity'; +export * from './getWebsiteListActiveVisitors'; +export * from './getWebsiteListStats'; export * from './getWebsiteStats'; export * from './getWeeklyTraffic'; export * from './heatmap/extractHeatmapEvents';