diff --git a/apps/web/app/(ee)/api/admin/analytics/route.ts b/apps/web/app/(ee)/api/admin/analytics/route.ts index dc11691ced7..57a98beaa8f 100644 --- a/apps/web/app/(ee)/api/admin/analytics/route.ts +++ b/apps/web/app/(ee)/api/admin/analytics/route.ts @@ -1,11 +1,11 @@ import { getAnalytics } from "@/lib/analytics/get-analytics"; import { withAdmin } from "@/lib/auth"; -import { analyticsQuerySchema } from "@/lib/zod/schemas/analytics"; +import { parseAnalyticsQuery } from "@/lib/zod/schemas/analytics"; import { NextResponse } from "next/server"; // GET /api/admin/analytics – get analytics for admin export const GET = withAdmin(async ({ searchParams }) => { - const parsedParams = analyticsQuerySchema.parse(searchParams); + const parsedParams = parseAnalyticsQuery(searchParams); const response = await getAnalytics(parsedParams); diff --git a/apps/web/app/(ee)/api/admin/events/route.ts b/apps/web/app/(ee)/api/admin/events/route.ts index f6696fc3df9..cc791e4f46f 100644 --- a/apps/web/app/(ee)/api/admin/events/route.ts +++ b/apps/web/app/(ee)/api/admin/events/route.ts @@ -1,11 +1,11 @@ import { getEvents } from "@/lib/analytics/get-events"; import { withAdmin } from "@/lib/auth"; -import { eventsQuerySchema } from "@/lib/zod/schemas/analytics"; +import { parseEventsQuery } from "@/lib/zod/schemas/analytics"; import { NextResponse } from "next/server"; // GET /api/admin/events – get events for admin export const GET = withAdmin(async ({ searchParams }) => { - const parsedParams = eventsQuerySchema.parse(searchParams); + const parsedParams = parseEventsQuery(searchParams); const response = await getEvents(parsedParams); diff --git a/apps/web/app/(ee)/api/cron/export/events/workspace/route.ts b/apps/web/app/(ee)/api/cron/export/events/workspace/route.ts index 821c00f8e25..ac00dd4e1da 100644 --- a/apps/web/app/(ee)/api/cron/export/events/workspace/route.ts +++ b/apps/web/app/(ee)/api/cron/export/events/workspace/route.ts @@ -26,7 +26,6 @@ const payloadSchema = eventsQuerySchema.extend({ userId: z.string(), linkId: z.string().optional(), folderIds: z.array(z.string()).optional(), - folderId: z.string().optional(), dataAvailableFrom: z.string().optional(), }); @@ -78,7 +77,7 @@ export async function POST(req: Request) { ); } - const { linkId, folderIds, folderId, dataAvailableFrom, ...eventFilters } = + const { linkId, folderIds, dataAvailableFrom, ...eventFilters } = filters; // Fetch events in batches and build CSV @@ -89,7 +88,6 @@ export async function POST(req: Request) { ...(linkId && { linkId }), workspaceId, folderIds, - folderId: folderId || "", dataAvailableFrom: dataAvailableFrom ? new Date(dataAvailableFrom) : workspace.createdAt, diff --git a/apps/web/app/(ee)/api/cron/usage/utils.ts b/apps/web/app/(ee)/api/cron/usage/utils.ts index 78a01a6033d..a3eb0f1898a 100644 --- a/apps/web/app/(ee)/api/cron/usage/utils.ts +++ b/apps/web/app/(ee)/api/cron/usage/utils.ts @@ -122,7 +122,7 @@ export const updateUsage = async () => { event: "clicks", groupBy: "top_links", interval: "30d", - root: false, + root: { values: ["false"], operator: "IS", sqlOperator: "IN" }, }); const topFive = topLinks.slice(0, 5); diff --git a/apps/web/app/(ee)/api/events/export/route.ts b/apps/web/app/(ee)/api/events/export/route.ts index 2296f1a66df..2fe17be4492 100644 --- a/apps/web/app/(ee)/api/events/export/route.ts +++ b/apps/web/app/(ee)/api/events/export/route.ts @@ -2,6 +2,7 @@ import { eventsExportColumnAccessors, eventsExportColumnNames, } from "@/lib/analytics/events-export-helpers"; +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getAnalytics } from "@/lib/analytics/get-analytics"; import { getEvents } from "@/lib/analytics/get-events"; import { getFolderIdsToFilter } from "@/lib/analytics/get-folder-ids-to-filter"; @@ -34,8 +35,20 @@ export const GET = withWorkspace( }) .parse(searchParams); - const { event, domain, interval, start, end, columns, key, folderId } = - parsedParams; + const { + event, + domain: domainFilter, + interval, + start, + end, + columns, + key, + folderId: folderIdFilter, + } = parsedParams; + + // Extract string values for specific link/folder lookup + const domain = getFirstFilterValue(domainFilter); + const folderId = getFirstFilterValue(folderIdFilter); if (domain) { await getDomainOrThrow({ workspace, domain }); @@ -100,7 +113,6 @@ export const GET = withWorkspace( userId: session.user.id, ...(link && { linkId: link.id }), folderIds: folderIds ? folderIds : undefined, - folderId: folderId || "", dataAvailableFrom: workspace.createdAt.toISOString(), }, }); @@ -114,7 +126,6 @@ export const GET = withWorkspace( workspaceId: workspace.id, limit: MAX_EVENTS_TO_EXPORT, folderIds, - folderId: folderId || "", }); const data = response.map((row) => diff --git a/apps/web/app/(ee)/api/events/route.ts b/apps/web/app/(ee)/api/events/route.ts index 84390b27c7e..452a59d85b7 100644 --- a/apps/web/app/(ee)/api/events/route.ts +++ b/apps/web/app/(ee)/api/events/route.ts @@ -1,3 +1,4 @@ +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getEvents } from "@/lib/analytics/get-events"; import { getFolderIdsToFilter } from "@/lib/analytics/get-folder-ids-to-filter"; import { getDomainOrThrow } from "@/lib/api/domains/get-domain-or-throw"; @@ -6,7 +7,7 @@ import { throwIfClicksUsageExceeded } from "@/lib/api/links/usage-checks"; import { assertValidDateRangeForPlan } from "@/lib/api/utils/assert-valid-date-range-for-plan"; import { withWorkspace } from "@/lib/auth"; import { verifyFolderAccess } from "@/lib/folder/permissions"; -import { eventsQuerySchema } from "@/lib/zod/schemas/analytics"; +import { parseEventsQuery } from "@/lib/zod/schemas/analytics"; import { Link } from "@dub/prisma/client"; import { NextResponse } from "next/server"; @@ -15,7 +16,7 @@ export const GET = withWorkspace( async ({ searchParams, workspace, session }) => { throwIfClicksUsageExceeded(workspace); - const parsedParams = eventsQuerySchema.parse(searchParams); + const parsedParams = parseEventsQuery(searchParams); let { event, @@ -24,11 +25,15 @@ export const GET = withWorkspace( end, linkId, externalId, - domain, + domain: domainFilter, key, - folderId, + folderId: folderIdFilter, } = parsedParams; + // Extract string values for specific link/folder lookup + const domain = getFirstFilterValue(domainFilter); + const folderId = getFirstFilterValue(folderIdFilter); + let link: Link | null = null; if (domain) { @@ -78,7 +83,7 @@ export const GET = withWorkspace( ...(link && { linkId: link.id }), workspaceId: workspace.id, folderIds, - folderId: folderId || "", + folderId: folderIdFilter || undefined, }); console.timeEnd("getEvents"); diff --git a/apps/web/app/(ee)/api/partner-profile/programs/[programId]/analytics/export/route.ts b/apps/web/app/(ee)/api/partner-profile/programs/[programId]/analytics/export/route.ts index da549eb6168..bf3c0cd0683 100644 --- a/apps/web/app/(ee)/api/partner-profile/programs/[programId]/analytics/export/route.ts +++ b/apps/web/app/(ee)/api/partner-profile/programs/[programId]/analytics/export/route.ts @@ -1,4 +1,5 @@ import { VALID_ANALYTICS_ENDPOINTS } from "@/lib/analytics/constants"; +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getAnalytics } from "@/lib/analytics/get-analytics"; import { convertToCSV } from "@/lib/analytics/utils"; import { DubApiError } from "@/lib/api/errors"; @@ -37,7 +38,10 @@ export const GET = withPartnerProfile( const parsedParams = partnerProfileAnalyticsQuerySchema.parse(searchParams); - let { linkId, domain, key, ...rest } = parsedParams; + let { linkId, domain: domainFilter, key, ...rest } = parsedParams; + + // Extract string value for link lookup + const domain = getFirstFilterValue(domainFilter); if (linkId) { if (!links.some((link) => link.id === linkId)) { diff --git a/apps/web/app/(ee)/api/partner-profile/programs/[programId]/analytics/route.ts b/apps/web/app/(ee)/api/partner-profile/programs/[programId]/analytics/route.ts index fec8ed9d2ec..d71db79463f 100644 --- a/apps/web/app/(ee)/api/partner-profile/programs/[programId]/analytics/route.ts +++ b/apps/web/app/(ee)/api/partner-profile/programs/[programId]/analytics/route.ts @@ -1,3 +1,4 @@ +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getAnalytics } from "@/lib/analytics/get-analytics"; import { DubApiError } from "@/lib/api/errors"; import { getProgramEnrollmentOrThrow } from "@/lib/api/programs/get-program-enrollment-or-throw"; @@ -23,9 +24,11 @@ export const GET = withPartnerProfile( }, }); - let { linkId, domain, key, ...rest } = + let { linkId, domain: domainFilter, key, ...rest } = partnerProfileAnalyticsQuerySchema.parse(searchParams); + const domain = getFirstFilterValue(domainFilter); + if (linkId) { if (!links.some((link) => link.id === linkId)) { throw new DubApiError({ diff --git a/apps/web/app/(ee)/api/partner-profile/programs/[programId]/events/export/route.ts b/apps/web/app/(ee)/api/partner-profile/programs/[programId]/events/export/route.ts index dd8ac6cfab5..1d74bf226b9 100644 --- a/apps/web/app/(ee)/api/partner-profile/programs/[programId]/events/export/route.ts +++ b/apps/web/app/(ee)/api/partner-profile/programs/[programId]/events/export/route.ts @@ -2,6 +2,7 @@ import { eventsExportColumnAccessors, eventsExportColumnNames, } from "@/lib/analytics/events-export-helpers"; +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getAnalytics } from "@/lib/analytics/get-analytics"; import { getEvents } from "@/lib/analytics/get-events"; import { convertToCSV } from "@/lib/analytics/utils"; @@ -59,7 +60,9 @@ export const GET = withPartnerProfile( .parse(searchParams); const { event, columns: columnsParam } = parsedParams; - let { linkId, domain, key, ...rest } = parsedParams; + let { linkId, domain: domainFilter, key, ...rest } = parsedParams; + + const domain = getFirstFilterValue(domainFilter); // Default columns based on event type if not provided const defaultColumns: Record = { diff --git a/apps/web/app/(ee)/api/partner-profile/programs/[programId]/events/route.ts b/apps/web/app/(ee)/api/partner-profile/programs/[programId]/events/route.ts index 7a032dd508b..629b36d8269 100644 --- a/apps/web/app/(ee)/api/partner-profile/programs/[programId]/events/route.ts +++ b/apps/web/app/(ee)/api/partner-profile/programs/[programId]/events/route.ts @@ -1,3 +1,4 @@ +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getEvents } from "@/lib/analytics/get-events"; import { DubApiError } from "@/lib/api/errors"; import { getProgramEnrollmentOrThrow } from "@/lib/api/programs/get-program-enrollment-or-throw"; @@ -38,9 +39,11 @@ export const GET = withPartnerProfile( }); } - let { linkId, domain, key, ...rest } = + let { linkId, domain: domainFilter, key, ...rest } = partnerProfileEventsQuerySchema.parse(searchParams); + const domain = getFirstFilterValue(domainFilter); + if (linkId) { if (!links.some((link) => link.id === linkId)) { throw new DubApiError({ diff --git a/apps/web/app/api/analytics/dashboard/route.ts b/apps/web/app/api/analytics/dashboard/route.ts index a5984041b30..68a39902e7a 100644 --- a/apps/web/app/api/analytics/dashboard/route.ts +++ b/apps/web/app/api/analytics/dashboard/route.ts @@ -1,10 +1,11 @@ +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getAnalytics } from "@/lib/analytics/get-analytics"; import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors"; import { assertValidDateRangeForPlan } from "@/lib/api/utils/assert-valid-date-range-for-plan"; import { exceededLimitError } from "@/lib/exceeded-limit-error"; import { PlanProps } from "@/lib/types"; import { redis } from "@/lib/upstash"; -import { analyticsQuerySchema } from "@/lib/zod/schemas/analytics"; +import { parseAnalyticsQuery } from "@/lib/zod/schemas/analytics"; import { prisma } from "@dub/prisma"; import { DUB_DEMO_LINKS, DUB_WORKSPACE_ID, getSearchParams } from "@dub/utils"; import { waitUntil } from "@vercel/functions"; @@ -16,9 +17,13 @@ export const dynamic = "force-dynamic"; export const GET = async (req: Request) => { try { const searchParams = getSearchParams(req.url); - const parsedParams = analyticsQuerySchema.parse(searchParams); + const parsedParams = parseAnalyticsQuery(searchParams); - const { domain, key, folderId, interval, start, end } = parsedParams; + const { domain: domainFilter, key, folderId: folderIdFilter, interval, start, end } = parsedParams; + + // Extract string values for specific link/folder lookup + const domain = getFirstFilterValue(domainFilter); + const folderId = getFirstFilterValue(folderIdFilter); if ((!domain || !key) && !folderId) { throw new DubApiError({ diff --git a/apps/web/app/api/analytics/export/route.ts b/apps/web/app/api/analytics/export/route.ts index 3caf41659c5..c0780fb50f8 100644 --- a/apps/web/app/api/analytics/export/route.ts +++ b/apps/web/app/api/analytics/export/route.ts @@ -1,4 +1,5 @@ import { VALID_ANALYTICS_ENDPOINTS } from "@/lib/analytics/constants"; +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getAnalytics } from "@/lib/analytics/get-analytics"; import { getFolderIdsToFilter } from "@/lib/analytics/get-folder-ids-to-filter"; import { convertToCSV } from "@/lib/analytics/utils"; @@ -8,7 +9,7 @@ import { throwIfClicksUsageExceeded } from "@/lib/api/links/usage-checks"; import { assertValidDateRangeForPlan } from "@/lib/api/utils/assert-valid-date-range-for-plan"; import { withWorkspace } from "@/lib/auth"; import { verifyFolderAccess } from "@/lib/folder/permissions"; -import { analyticsQuerySchema } from "@/lib/zod/schemas/analytics"; +import { parseAnalyticsQuery } from "@/lib/zod/schemas/analytics"; import { Link } from "@dub/prisma/client"; import JSZip from "jszip"; @@ -17,11 +18,15 @@ export const GET = withWorkspace( async ({ searchParams, workspace, session }) => { throwIfClicksUsageExceeded(workspace); - const parsedParams = analyticsQuerySchema.parse(searchParams); + const parsedParams = parseAnalyticsQuery(searchParams); - const { interval, start, end, linkId, externalId, domain, key, folderId } = + const { interval, start, end, linkId, externalId, domain: domainFilter, key, folderId: folderIdFilter } = parsedParams; + // Extract string values for specific link/folder lookup + const domain = getFirstFilterValue(domainFilter); + const folderId = getFirstFilterValue(folderIdFilter); + let link: Link | null = null; if (domain) { @@ -80,7 +85,7 @@ export const GET = withWorkspace( ...(link && { linkId: link.id }), groupBy: endpoint, folderIds, - folderId: folderId || "", + folderId: folderIdFilter || undefined, }); if (!response || response.length === 0) return; diff --git a/apps/web/app/api/analytics/route.ts b/apps/web/app/api/analytics/route.ts index 77a5fb6cb44..36d2d929570 100644 --- a/apps/web/app/api/analytics/route.ts +++ b/apps/web/app/api/analytics/route.ts @@ -1,4 +1,5 @@ import { VALID_ANALYTICS_ENDPOINTS } from "@/lib/analytics/constants"; +import { getFirstFilterValue } from "@/lib/analytics/filter-helpers"; import { getAnalytics } from "@/lib/analytics/get-analytics"; import { getFolderIdsToFilter } from "@/lib/analytics/get-folder-ids-to-filter"; import { getDomainOrThrow } from "@/lib/api/domains/get-domain-or-throw"; @@ -13,7 +14,7 @@ import { withWorkspace } from "@/lib/auth"; import { verifyFolderAccess } from "@/lib/folder/permissions"; import { analyticsPathParamsSchema, - analyticsQuerySchema, + parseAnalyticsQuery, } from "@/lib/zod/schemas/analytics"; import { Link } from "@dub/prisma/client"; import { NextResponse } from "next/server"; @@ -32,7 +33,7 @@ export const GET = withWorkspace( oldEvent = undefined; } - const parsedParams = analyticsQuerySchema.parse(searchParams); + const parsedParams = parseAnalyticsQuery(searchParams); let { event, @@ -42,12 +43,17 @@ export const GET = withWorkspace( end, linkId, externalId, - domain, + domain: domainFilter, key, - folderId, + folderId: folderIdFilter, programId, } = parsedParams; + // Extract string values for specific link/folder lookup + // When domain+key is provided, it's for getting a specific link (not filtering) + const domain = getFirstFilterValue(domainFilter); + const folderId = getFirstFilterValue(folderIdFilter); + let link: Link | null = null; event = oldEvent || event; diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/links/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/links/page-client.tsx index 5b356b25085..11f233a9256 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/links/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/links/page-client.tsx @@ -93,6 +93,7 @@ function WorkspaceLinks() { activeFilters, onSelect, onRemove, + onRemoveFilter, onRemoveAll, setSearch, setSelectedFilter, @@ -247,6 +248,7 @@ function WorkspaceLinks() { activeFilters={activeFilters} onSelect={onSelect} onRemove={onRemove} + onRemoveFilter={onRemoveFilter} onRemoveAll={onRemoveAll} /> diff --git a/apps/web/lib/analytics/filter-helpers.ts b/apps/web/lib/analytics/filter-helpers.ts new file mode 100644 index 00000000000..ea98bd9e4f5 --- /dev/null +++ b/apps/web/lib/analytics/filter-helpers.ts @@ -0,0 +1,175 @@ +import { ParsedFilter, type SQLOperator } from "@dub/utils"; + +/** + * Advanced filter structure for Tinybird's filters JSON parameter. + * Used for event-level dimensional filters. + */ +export interface AdvancedFilter { + field: string; + operator: SQLOperator; + values: string[]; +} + +/** + * Extract the first string value from a ParsedFilter. + * Useful for API routes that need a single value (e.g., domain, folderId) + * for lookups, even when the filter supports multiple values. + */ +export function getFirstFilterValue( + filter: ParsedFilter | string | undefined, +): string | undefined { + if (!filter) return undefined; + if (typeof filter === "string") return filter; + return filter.values?.[0]; +} + +/** + * Prepare trigger and region filters for Tinybird pipes. + * Handles backward compatibility for qr parameter and region splitting. + */ +export function prepareFiltersForPipe(params: { + qr?: boolean; + trigger?: ParsedFilter; + region?: string | ParsedFilter; + country?: ParsedFilter; +}) { + // Handle qr backward compatibility + let triggerForPipe = params.trigger; + if (params.qr && !params.trigger) { + triggerForPipe = { + operator: "IS" as const, + sqlOperator: "IN" as const, + values: ["qr"], + }; + } + + // Handle region split (format: "US-CA") + let countryForPipe = params.country; + let regionForPipe = params.region; + if (params.region && typeof params.region === "string") { + const split = params.region.split("-"); + countryForPipe = { + operator: "IS" as const, + sqlOperator: "IN" as const, + values: [split[0]], + }; + regionForPipe = split[1]; + } + + return { triggerForPipe, countryForPipe, regionForPipe }; +} + +/** + * Normalize a filter value that may be a plain string (e.g. from partner-profile + * routes) or an already-parsed ParsedFilter into a consistent ParsedFilter. + * + * Useful when callers pass a raw ID string but extractWorkspaceLinkFilters + * expects a ParsedFilter with sqlOperator. + */ +export function ensureParsedFilter( + value: string | ParsedFilter | undefined, +): ParsedFilter | undefined { + if (!value) return undefined; + if (typeof value === "string") { + return { + operator: "IS" as const, + sqlOperator: "IN" as const, + values: [value], + }; + } + return value; +} + +/** + * Extract workspace link filters (domain, tagIds, folderId, partnerId, root) into + * separate values and operators for Tinybird. + * + * These filters are applied on the workspace_links node in Tinybird, + * so they need to be passed as separate parameters (not in the filters JSON). + */ +export function extractWorkspaceLinkFilters(params: { + domain?: ParsedFilter; + tagIds?: ParsedFilter; + folderId?: ParsedFilter; + partnerId?: ParsedFilter; + groupId?: ParsedFilter; + tenantId?: ParsedFilter; + root?: ParsedFilter; +}) { + const extractFilter = (filter?: ParsedFilter) => ({ + values: filter?.values, + operator: (filter?.sqlOperator === "NOT IN" ? "NOT IN" : "IN") as + | "IN" + | "NOT IN", + }); + + const domain = extractFilter(params.domain); + const tagIds = extractFilter(params.tagIds); + const folderId = extractFilter(params.folderId); + const partnerId = extractFilter(params.partnerId); + const groupId = extractFilter(params.groupId); + const tenantId = extractFilter(params.tenantId); + const root = extractFilter(params.root); + + return { + domain: domain.values, + domainOperator: domain.operator, + tagIds: tagIds.values, + tagIdsOperator: tagIds.operator, + folderId: folderId.values, + folderIdOperator: folderId.operator, + partnerId: partnerId.values, + partnerIdOperator: partnerId.operator, + groupId: groupId.values, + groupIdOperator: groupId.operator, + tenantId: tenantId.values, + tenantIdOperator: tenantId.operator, + root: root.values, + rootOperator: root.operator, + }; +} + +/** + * Build advanced filters array for Tinybird's filters JSON parameter. + * Extracts event-level dimensional filters from params and formats them + * for the filters JSON that gets passed to Tinybird pipes. + */ +const SUPPORTED_FIELDS = [ + "country", + "city", + "continent", + "device", + "browser", + "os", + "referer", + "refererUrl", + "url", + "trigger", + "utm_source", + "utm_medium", + "utm_campaign", + "utm_term", + "utm_content", + "saleType", +] as const; + +type SupportedField = (typeof SUPPORTED_FIELDS)[number]; + +export function buildAdvancedFilters( + params: Partial>, +): AdvancedFilter[] { + const filters: AdvancedFilter[] = []; + + for (const field of SUPPORTED_FIELDS) { + const parsed = params[field]; + if (!parsed) continue; + + filters.push({ + field, + operator: parsed.sqlOperator, + values: parsed.values, + }); + } + + return filters; +} diff --git a/apps/web/lib/analytics/get-analytics.ts b/apps/web/lib/analytics/get-analytics.ts index dffdb1295c0..ca1758a3c22 100644 --- a/apps/web/lib/analytics/get-analytics.ts +++ b/apps/web/lib/analytics/get-analytics.ts @@ -12,6 +12,12 @@ import { DIMENSIONAL_ANALYTICS_FILTERS, SINGULAR_ANALYTICS_ENDPOINTS, } from "./constants"; +import { + buildAdvancedFilters, + ensureParsedFilter, + extractWorkspaceLinkFilters, + prepareFiltersForPipe, +} from "./filter-helpers"; import { queryParser } from "./query-parser"; import { AnalyticsFilters } from "./types"; import { formatUTCDateTimeClickhouse } from "./utils/format-utc-datetime-clickhouse"; @@ -23,7 +29,6 @@ export const getAnalytics = async (params: AnalyticsFilters) => { event, groupBy, workspaceId, - linkId, linkIds, interval, start, @@ -38,16 +43,14 @@ export const getAnalytics = async (params: AnalyticsFilters) => { query, } = params; - const tagIds = combineTagIds(params); - // get all-time clicks count if: - // 1. linkId or linkIds is defined + // 1. linkIds is defined // 2. type is count // 3. interval is all // 4. no custom start or end date is provided // 5. no other dimensional filters are applied if ( - (linkId || linkIds) && + linkIds && groupBy === "count" && interval === "all" && !start && @@ -56,24 +59,6 @@ export const getAnalytics = async (params: AnalyticsFilters) => { (filter) => !params[filter as keyof AnalyticsFilters], ) ) { - const columns = - event === "composite" - ? `clicks, leads, sales, saleAmount` - : event === "sales" - ? `sales, saleAmount` - : `${event}`; - - // Handle single linkId - if (linkId) { - const response = await conn.execute( - `SELECT ${columns} FROM Link WHERE id = ?`, - [linkId], - ); - - return analyticsResponse["count"].parse(response.rows[0]); - } - - // Handle multiple linkIds with aggregation if (linkIds && linkIds.length > 0) { const linkIdsToFilter = linkIds.map(() => "?").join(","); const aggregateColumns = @@ -92,9 +77,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => { } } - if (groupBy === "trigger") { - groupBy = "triggers"; - } + if (groupBy === "trigger") groupBy = "triggers"; const { startDate, endDate, granularity } = getStartEndDates({ interval, @@ -104,27 +87,25 @@ export const getAnalytics = async (params: AnalyticsFilters) => { timezone, }); - if (qr) { - trigger = "qr"; - } - - if (region) { - const split = region.split("-"); - country = split[0]; - region = split[1]; - } + const { triggerForPipe, countryForPipe, regionForPipe } = + prepareFiltersForPipe({ + qr, + trigger, + region, + country, + }); // Create a Tinybird pipe const pipe = tb.buildPipe({ - pipe: ["count", "timeseries"].includes(groupBy) + pipe: ["count", "timeseries"].includes(groupBy!) ? `v3_${groupBy}` : [ - "top_folders", - "top_link_tags", - "top_domains", - "top_partners", - "top_groups", - ].includes(groupBy) + "top_folders", + "top_link_tags", + "top_domains", + "top_partners", + "top_groups", + ].includes(groupBy!) ? "v3_group_by_link_metadata" : "v3_group_by", parameters: analyticsFilterTB, @@ -140,29 +121,78 @@ export const getAnalytics = async (params: AnalyticsFilters) => { }), }); - const filters = queryParser(query); + const metadataFilters = queryParser(query) || []; - const response = await pipe({ + const advancedFilters = buildAdvancedFilters({ ...params, + country: countryForPipe, + trigger: triggerForPipe, + }); + + const allFilters = [...metadataFilters, ...advancedFilters]; + + // Normalize partnerId (may be a plain string from partner-profile routes) + const partnerIdFilter = ensureParsedFilter(params.partnerId); + + const { + domain: domainParam, + domainOperator, + tagIds: tagIdsParam, + tagIdsOperator, + folderId: folderIdParam, + folderIdOperator, + partnerId: partnerIdParam, + partnerIdOperator, + groupId: groupIdParam, + groupIdOperator, + tenantId: tenantIdParam, + tenantIdOperator, + root: rootParam, + rootOperator, + } = extractWorkspaceLinkFilters({ + ...params, + partnerId: partnerIdFilter, + }); + + const tinybirdParams: any = { + workspaceId, + linkId: params.linkId, + linkIds: params.linkIds, + folderIds: params.folderIds, + customerId: params.customerId, + programId: params.programId, + partnerId: partnerIdParam, + partnerIdOperator, + tenantId: tenantIdParam, + tenantIdOperator, + groupId: groupIdParam, + groupIdOperator, + domain: domainParam, + domainOperator, + tagIds: tagIdsParam, + tagIdsOperator, + folderId: folderIdParam, + folderIdOperator, + root: rootParam, + rootOperator, groupBy, eventType: event, - workspaceId, - tagIds, - trigger, start: formatUTCDateTimeClickhouse(startDate), end: formatUTCDateTimeClickhouse(endDate), granularity, timezone, - country, - region, - filters: filters ? JSON.stringify(filters) : undefined, - }); + region: typeof regionForPipe === "string" ? regionForPipe : undefined, + + filters: allFilters.length > 0 ? JSON.stringify(allFilters) : undefined, + }; + + const response = await pipe(tinybirdParams); if (groupBy === "count") { const { groupByField, ...rest } = response.data[0]; // Return the count value for deprecated count endpoints if (isDeprecatedClicksEndpoint) { - return rest[event]; + return rest[event!]; // Return the object for regular count endpoints } else { return rest; @@ -326,12 +356,12 @@ export const getAnalytics = async (params: AnalyticsFilters) => { } // Return array for other endpoints - const schema = analyticsResponse[groupBy]; + const schema = analyticsResponse[groupBy!]; return response.data.map((item) => schema.parse({ ...item, - [SINGULAR_ANALYTICS_ENDPOINTS[groupBy]]: item.groupByField, + [SINGULAR_ANALYTICS_ENDPOINTS[groupBy!]]: item.groupByField, }), ); }; diff --git a/apps/web/lib/analytics/get-events.ts b/apps/web/lib/analytics/get-events.ts index 48021fa25bd..859299d4260 100644 --- a/apps/web/lib/analytics/get-events.ts +++ b/apps/web/lib/analytics/get-events.ts @@ -21,6 +21,12 @@ import { saleEventResponseSchema, saleEventSchemaTBEndpoint, } from "../zod/schemas/sales"; +import { + buildAdvancedFilters, + ensureParsedFilter, + extractWorkspaceLinkFilters, + prepareFiltersForPipe, +} from "./filter-helpers"; import { queryParser } from "./query-parser"; import { EventsFilters } from "./types"; import { formatUTCDateTimeClickhouse } from "./utils/format-utc-datetime-clickhouse"; @@ -53,15 +59,13 @@ export const getEvents = async (params: EventsFilters) => { timezone, }); - if (qr) { - trigger = "qr"; - } - - if (region) { - const split = region.split("-"); - country = split[0]; - region = split[1]; - } + const { triggerForPipe, countryForPipe, regionForPipe } = + prepareFiltersForPipe({ + qr, + trigger, + region, + country, + }); // support legacy order param if (order && order !== "desc") { @@ -79,21 +83,84 @@ export const getEvents = async (params: EventsFilters) => { }[eventType] ?? clickEventSchemaTBEndpoint, }); - const filters = queryParser(query); + const metadataFilters = queryParser(query) || []; + + // Build advanced filters for event-level dimensions + const advancedFilters = buildAdvancedFilters({ + country: countryForPipe, + trigger: triggerForPipe, + city: params.city, + continent: params.continent, + device: params.device, + browser: params.browser, + os: params.os, + referer: params.referer, + refererUrl: params.refererUrl, + url: params.url, + utm_source: params.utm_source, + utm_medium: params.utm_medium, + utm_campaign: params.utm_campaign, + utm_term: params.utm_term, + utm_content: params.utm_content, + saleType: params.saleType, + }); + + const allFilters = [...metadataFilters, ...advancedFilters]; + + const partnerIdFilter = ensureParsedFilter(params.partnerId); - const response = await pipe({ + const { + domain: domainParam, + domainOperator, + tagIds: tagIdsParam, + tagIdsOperator, + folderId: folderIdParam, + folderIdOperator, + partnerId: partnerIdParam, + partnerIdOperator, + groupId: groupIdParam, + groupIdOperator, + tenantId: tenantIdParam, + tenantIdOperator, + root: rootParam, + rootOperator, + } = extractWorkspaceLinkFilters({ ...params, + partnerId: partnerIdFilter, + }); + + const tinybirdParams: any = { eventType, workspaceId, - trigger, - country, - region, + linkId: params.linkId, + linkIds: params.linkIds, + folderIds: params.folderIds, + customerId: params.customerId, + programId: params.programId, + partnerId: partnerIdParam, + partnerIdOperator, + tenantId: tenantIdParam, + tenantIdOperator, + groupId: groupIdParam, + groupIdOperator, + ...(typeof triggerForPipe !== 'object' && triggerForPipe ? { trigger: triggerForPipe } : {}), + ...(typeof countryForPipe !== 'object' && countryForPipe ? { country: countryForPipe } : {}), + ...(typeof regionForPipe === 'string' ? { region: regionForPipe } : {}), + // Workspace links filters with operators + ...(domainParam ? { domain: domainParam, domainOperator } : {}), + ...(tagIdsParam ? { tagIds: tagIdsParam, tagIdsOperator } : {}), + ...(folderIdParam ? { folderId: folderIdParam, folderIdOperator } : {}), + ...(rootParam ? { root: rootParam, rootOperator } : {}), order: sortOrder, offset: (params.page - 1) * params.limit, + limit: params.limit, + sortBy: params.sortBy, start: formatUTCDateTimeClickhouse(startDate), end: formatUTCDateTimeClickhouse(endDate), - filters: filters ? JSON.stringify(filters) : undefined, - }); + filters: allFilters.length > 0 ? JSON.stringify(allFilters) : undefined, + }; + + const response = await pipe(tinybirdParams); const [linksMap, customersMap] = await Promise.all([ getLinksMap(response.data.map((d) => d.link_id)), @@ -118,6 +185,11 @@ export const getEvents = async (params: EventsFilters) => { link = decodeLinkIfCaseSensitive(link); + const transformedLink = transformLink(link, { skipDecodeKey: true }); + if (transformedLink.testVariants && !Array.isArray(transformedLink.testVariants)) { + transformedLink.testVariants = null; + } + const eventData = { ...evt, // use link domain & key from mysql instead of tinybird @@ -133,7 +205,7 @@ export const getEvents = async (params: EventsFilters) => { refererUrl: evt.referer_url_processed ?? "", }), // transformLink -> add shortLink, qrCode, workspaceId, etc. - link: transformLink(link, { skipDecodeKey: true }), + link: transformedLink, ...(evt.event === "lead" || evt.event === "sale" ? { eventId: evt.event_id, diff --git a/apps/web/lib/analytics/query-parser.ts b/apps/web/lib/analytics/query-parser.ts index 8518732a31e..7403c9fe152 100644 --- a/apps/web/lib/analytics/query-parser.ts +++ b/apps/web/lib/analytics/query-parser.ts @@ -91,15 +91,27 @@ function parseCondition(condition: string): InternalFilter | null { .replace(/\[['"]/g, ".") // Replace [' or [" with . .replace(/['"]\]/g, ""); // Remove trailing '] or "] + // Security: Validate metadata key contains only safe characters + // Only allow alphanumeric and underscore (no dots — nested keys are not supported) + if (!/^[a-zA-Z0-9_]+$/.test(extractedKey)) return null; + operand = `metadata.${extractedKey}`; } else { operand = fieldOrMetadata; } + // Security: Sanitize value to prevent SQL injection + // Remove potentially dangerous characters from the value + const sanitizedValue = value.trim() + .replace(/^['"`]|['"`]$/g, "") + .replace(/[;\\]|--|\*\/|\/\*/g, ""); + + if (!sanitizedValue) return null; + return { operand, operator: mapOperator(operator), - value: value.trim().replace(/^['"`]|['"`]$/g, ""), + value: sanitizedValue, }; } diff --git a/apps/web/lib/analytics/types.ts b/apps/web/lib/analytics/types.ts index 008e097026d..8c4f5d9ab46 100644 --- a/apps/web/lib/analytics/types.ts +++ b/apps/web/lib/analytics/types.ts @@ -1,3 +1,4 @@ +import { ParsedFilter } from "@dub/utils"; import * as z from "zod/v4"; import { analyticsQuerySchema, @@ -37,30 +38,32 @@ export type AnalyticsSaleUnit = (typeof ANALYTICS_SALE_UNIT)[number]; export type DeviceTabs = "devices" | "browsers" | "os" | "triggers"; -export type AnalyticsFilters = Override< - z.infer, - { +export type AnalyticsFilters = Partial, 'start' | 'end' | 'partnerId'>> & { + workspaceId?: string; + dataAvailableFrom?: Date; + isDeprecatedClicksEndpoint?: boolean; + linkIds?: string[]; + folderIds?: string[]; // TODO: remove this once it's been added to the public API + start?: Date | null; + end?: Date | null; + // Accept plain string (from partner-profile routes) or ParsedFilter (from API schema) + partnerId?: string | ParsedFilter; +}; + +// Structural fields from eventsQuerySchema that should remain required +type EventsStructuralFields = Pick, 'event' | 'page' | 'limit' | 'sortBy'>; + +export type EventsFilters = Partial, 'start' | 'end' | 'partnerId' | keyof EventsStructuralFields>> & + EventsStructuralFields & { workspaceId?: string; dataAvailableFrom?: Date; - isDeprecatedClicksEndpoint?: boolean; - linkIds?: string[]; - folderIds?: string[]; // TODO: remove this once it's been added to the public API + customerId?: string; + folderIds?: string[]; start?: Date | null; end?: Date | null; - - // TODO: Fix the schema so that we can avoid the override here - device?: string | undefined; - browser?: string | undefined; - os?: string | undefined; - } ->; - -export type EventsFilters = z.infer & { - workspaceId?: string; - dataAvailableFrom?: Date; - customerId?: string; - folderIds?: string[]; -}; + // Accept plain string (from partner-profile routes) or ParsedFilter (from API schema) + partnerId?: string | ParsedFilter; + }; const partnerAnalyticsSchema = analyticsQuerySchema .pick({ diff --git a/apps/web/lib/stripe/index.ts b/apps/web/lib/stripe/index.ts index 9e01e9a52ac..8b71428b16a 100644 --- a/apps/web/lib/stripe/index.ts +++ b/apps/web/lib/stripe/index.ts @@ -26,4 +26,4 @@ export const stripeAppClient = ({ mode }: { mode?: StripeMode }) => { version: "0.1.0", }, }); -}; +}; \ No newline at end of file diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts index c528951c6ed..0fa6bd13487 100644 --- a/apps/web/lib/zod/schemas/analytics.ts +++ b/apps/web/lib/zod/schemas/analytics.ts @@ -12,11 +12,11 @@ import { DUB_FOUNDING_DATE, capitalize, formatDate, + parseFilterValue, } from "@dub/utils"; import * as z from "zod/v4"; import { booleanQuerySchema } from "./misc"; import { parseDateSchema } from "./utils"; -import { UTMTemplateSchema } from "./utm"; const analyticsEvents = z .enum([...EVENT_TYPES, "composite"], { @@ -54,14 +54,19 @@ export const analyticsPathParamsSchema = z.object({ }); // Query schema for GET /analytics and GET /events endpoints -export const analyticsQuerySchema = z - .object({ +export const analyticsQuerySchema = z.object({ event: analyticsEvents, groupBy: analyticsGroupBy, domain: z .string() .optional() - .describe("The domain to filter analytics for."), + .transform(parseFilterValue) + .describe( + "The domain to filter analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `dub.co`, `dub.co,google.com`, `-spam.com`." + ) + .meta({ example: "dub.co" }), key: z .string() .optional() @@ -73,15 +78,15 @@ export const analyticsQuerySchema = z .optional() .describe( "The unique ID of the short link on Dub to retrieve analytics for.", + ) + .meta({ deprecated: false }), // Keep for backward compatibility + linkIds: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe( + "The link IDs to retrieve analytics for. Supports comma-separated values or array format.", ), - // TODO: Add this to the public API when we can properly verify linkIds ownership in /api/analytics - // linkIds: z - // .union([z.string(), z.array(z.string())]) - // .transform((v) => (Array.isArray(v) ? v : v.split(","))) - // .optional() - // .describe( - // "A list of link IDs to retrieve analytics for. Takes precidence over ", - // ), externalId: z .string() .optional() @@ -91,9 +96,13 @@ export const analyticsQuerySchema = z tenantId: z .string() .optional() + .transform(parseFilterValue) .describe( - "The ID of the tenant that created the link inside your system.", - ), + "The ID of the tenant that created the link inside your system. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `tenant_123`, `tenant_123,tenant_456`, `-tenant_789`." + ) + .meta({ example: "tenant_123" }), programId: z .string() .optional() @@ -101,7 +110,13 @@ export const analyticsQuerySchema = z partnerId: z .string() .optional() - .describe("The ID of the partner to retrieve analytics for."), + .transform(parseFilterValue) + .describe( + "The ID of the partner to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `pn_123`, `pn_123,pn_456`, `-pn_789`." + ) + .meta({ example: "pn_123" }), customerId: z .string() .optional() @@ -135,86 +150,208 @@ export const analyticsQuerySchema = z country: z .string() .optional() + .transform(parseFilterValue) .describe( - "The country to retrieve analytics for. Must be passed as a 2-letter ISO 3166-1 country code. See https://d.to/geo for more information.", - ), + "The country to retrieve analytics for. Must be passed as a 2-letter ISO 3166-1 country code. " + + "Supports advanced filtering: " + + "• Single value: `US` (IS US) " + + "• Multiple values: `US,BR,FR` (IS ONE OF) " + + "• Exclude single: `-US` (IS NOT US) " + + "• Exclude multiple: `-US,BR` (IS NOT ONE OF). " + + "See https://d.to/geo for country codes.", + ) + .meta({ + example: "US", + }), city: z .string() .optional() - .describe("The city to retrieve analytics for.") + .transform(parseFilterValue) + .describe( + "The city to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `New York`, `New York,London`, `-New York`." + ) .meta({ example: "New York" }), region: z .string() .optional() .describe("The ISO 3166-2 region code to retrieve analytics for."), continent: z - .enum(CONTINENT_CODES) + .string() .optional() - .describe("The continent to retrieve analytics for."), + .transform(parseFilterValue) + .describe( + "The continent to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Valid values: AF, AN, AS, EU, NA, OC, SA. " + + "Examples: `NA`, `NA,EU`, `-AS`." + ) + .meta({ example: "NA" }), device: z .string() .optional() - .transform((v) => capitalize(v) as string | undefined) - .describe("The device to retrieve analytics for.") + .transform((v) => { + if (!v) return undefined; + // Capitalize each value + const parsed = parseFilterValue(v); + if (!parsed) return undefined; + return { + ...parsed, + values: parsed.values.map((val) => capitalize(val)).filter(Boolean) as string[], + }; + }) + .describe( + "The device to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `Desktop`, `Mobile,Tablet`, `-Mobile`." + ) .meta({ example: "Desktop" }), browser: z .string() .optional() - .transform((v) => capitalize(v) as string | undefined) - .describe("The browser to retrieve analytics for.") + .transform((v) => { + if (!v) return undefined; + const parsed = parseFilterValue(v); + if (!parsed) return undefined; + return { + ...parsed, + values: parsed.values.map((val) => capitalize(val)).filter(Boolean) as string[], + }; + }) + .describe( + "The browser to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `Chrome`, `Chrome,Firefox,Safari`, `-IE`." + ) .meta({ example: "Chrome" }), os: z .string() .optional() .transform((v) => { - if (v === "iOS") return "iOS"; - return capitalize(v) as string | undefined; + if (!v) return undefined; + const parsed = parseFilterValue(v); + if (!parsed) return undefined; + return { + ...parsed, + values: parsed.values.map((val) => (val === "iOS" ? "iOS" : capitalize(val))).filter(Boolean) as string[], + }; }) - .describe("The OS to retrieve analytics for.") + .describe( + "The OS to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `Windows`, `Mac,Windows,Linux`, `-Windows`." + ) .meta({ example: "Windows" }), trigger: z - .enum(TRIGGER_TYPES) + .string() .optional() + .transform(parseFilterValue) .describe( - "The trigger to retrieve analytics for. If undefined, returns all trigger types.", - ), + "The trigger to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Valid values: qr, link. " + + "Examples: `qr`, `qr,link`, `-qr`. " + + "If undefined, returns all trigger types." + ) + .meta({ example: "qr" }), referer: z .string() .optional() - .describe("The referer hostname to retrieve analytics for.") + .transform(parseFilterValue) + .describe( + "The referer hostname to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `google.com`, `google.com,twitter.com`, `-facebook.com`." + ) .meta({ example: "google.com" }), refererUrl: z .string() .optional() - .describe("The full referer URL to retrieve analytics for.") + .transform(parseFilterValue) + .describe( + "The full referer URL to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`)." + ) .meta({ example: "https://dub.co/blog" }), - url: z.string().optional().describe("The URL to retrieve analytics for."), + url: z + .string() + .optional() + .transform(parseFilterValue) + .describe( + "The destination URL to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `https://example.com`, `https://example.com,https://other.com`, `-https://spam.com`." + ) + .meta({ example: "https://example.com" }), tagIds: z - .union([z.string(), z.array(z.string())]) - .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .string() .optional() - .describe("The tag IDs to retrieve analytics for."), + .transform(parseFilterValue) + .describe( + "The tag IDs to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `tag_123`, `tag_123,tag_456`, `-tag_789`." + ) + .meta({ example: "tag_123" }), folderId: z .string() .optional() + .transform(parseFilterValue) .describe( - "The folder ID to retrieve analytics for. If not provided, return analytics for unsorted links.", - ), + "The folder ID to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `folder_123`, `folder_123,folder_456`, `-folder_789`. " + + "If not provided, return analytics for all links." + ) + .meta({ example: "folder_123" }), groupId: z .string() .optional() - .describe("The group ID to retrieve analytics for."), - root: booleanQuerySchema + .transform(parseFilterValue) + .describe( + "The group ID to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `grp_123`, `grp_123,grp_456`, `-grp_789`." + ) + .meta({ example: "grp_123" }), + root: z + .string() .optional() + .transform((v) => { + if (!v) return undefined; + // Normalize boolean values to "true" or "false" strings for consistency + const parsed = parseFilterValue(v); + if (!parsed) return undefined; + return { + ...parsed, + values: parsed.values.map((val) => { + // Normalize various truthy/falsy values to "true"/"false" + if (val === "true" || val === "1" || val === "yes") return "true"; + if (val === "false" || val === "0" || val === "no") return "false"; + return val; + }), + }; + }) .describe( - "Filter for root domains. If true, filter for domains only. If false, filter for links only. If undefined, return both.", - ), + "Filter for root domains. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `true` (root domains only), `false` (regular links only), `true,false` (both). " + + "If undefined, return both." + ) + .meta({ example: "true" }), saleType: z - .enum(["new", "recurring"]) + .string() .optional() + .transform(parseFilterValue) .describe( - "Filter sales by type: 'new' for first-time purchases, 'recurring' for repeat purchases. If undefined, returns both.", - ), + "Filter sales by type. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Valid values: `new` (first-time purchases), `recurring` (repeat purchases). " + + "Examples: `new`, `new,recurring`, `-recurring`. " + + "If undefined, returns both." + ) + .meta({ example: "new" }), query: z .string() .max(10000) @@ -239,66 +376,158 @@ export const analyticsQuerySchema = z "Deprecated: Use the `trigger` field instead. Filter for QR code scans. If true, filter for QR codes only. If false, filter for links only. If undefined, return both.", ) .meta({ deprecated: true }), - }) - .extend(UTMTemplateSchema.omit({ id: true, name: true }).shape); - -// Analytics filter params for Tinybird endpoints -export const analyticsFilterTB = z - .object({ - eventType: analyticsEvents, - workspaceId: z.string().optional(), - customerId: z.string().optional(), - root: z.boolean().optional(), - saleType: z.string().optional(), - trigger: z.enum(TRIGGER_TYPES).optional(), - start: z.string(), - end: z.string(), - granularity: z.enum(["minute", "hour", "day", "month"]).optional(), - timezone: z.string().optional(), - // TODO: remove this once it's been added to the public API - linkIds: z - .union([z.string(), z.array(z.string())]) - .transform((v) => (Array.isArray(v) ? v : v.split(","))) + utm_source: z + .string() .optional() - .describe("The link IDs to retrieve analytics for."), - folderIds: z - .union([z.string(), z.array(z.string())]) - .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .transform(parseFilterValue) + .describe( + "The UTM source to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `google`, `google,twitter`, `-spam`." + ) + .meta({ example: "google" }), + utm_medium: z + .string() .optional() - .describe("The folder IDs to retrieve analytics for."), - filters: z + .transform(parseFilterValue) + .describe( + "The UTM medium to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `cpc`, `cpc,social`, `-email`." + ) + .meta({ example: "cpc" }), + utm_campaign: z .string() .optional() - .describe("The filters to apply to the analytics."), - }) - .extend( - analyticsQuerySchema.pick({ - groupBy: true, - browser: true, - city: true, - country: true, - continent: true, - region: true, - device: true, - domain: true, - linkId: true, - os: true, - referer: true, - refererUrl: true, - tagIds: true, - url: true, - utm_source: true, - utm_medium: true, - utm_campaign: true, - utm_term: true, - utm_content: true, - programId: true, - partnerId: true, - tenantId: true, - folderId: true, - groupId: true, - }).shape, - ); + .transform(parseFilterValue) + .describe( + "The UTM campaign to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`). " + + "Examples: `summer_sale`, `summer_sale,winter_sale`, `-old_campaign`." + ) + .meta({ example: "summer_sale" }), + utm_term: z + .string() + .optional() + .transform(parseFilterValue) + .describe( + "The UTM term to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`)." + ) + .meta({ example: "keyword" }), + utm_content: z + .string() + .optional() + .transform(parseFilterValue) + .describe( + "The UTM content to retrieve analytics for. " + + "Supports advanced filtering: single value, multiple values (comma-separated), or exclusion (prefix with `-`)." + ) + .meta({ example: "banner" }), +}); + +/** + * Parse analytics query parameters with backward compatibility + * Converts deprecated singular fields (linkId, tagId) to their plural equivalents + */ +export function parseAnalyticsQuery(searchParams: unknown) { + const data = analyticsQuerySchema.parse(searchParams); + + // Backward compatibility: convert linkId to linkIds + if (data.linkId && !data.linkIds) { + data.linkIds = [data.linkId]; + } + + // Backward compatibility: convert tagId to tagIds + if (data.tagId && !data.tagIds) { + // Convert single tagId to the multi-value format + data.tagIds = { operator: "IS" as const, sqlOperator: "IN" as const, values: [data.tagId] }; + } + + return data; +} + +// Analytics filter params for Tinybird endpoints +export const analyticsFilterTB = z.object({ + eventType: analyticsEvents, + workspaceId: z.string().optional(), + customerId: z.string().optional(), + start: z.string(), + end: z.string(), + granularity: z.enum(["minute", "hour", "day", "month"]).optional(), + timezone: z.string().optional(), + groupBy: analyticsGroupBy, + // Link-specific filters (not using advanced filtering) + linkId: z.string().optional(), + linkIds: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe("The link IDs to retrieve analytics for."), + folderIds: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe("The folder IDs to retrieve analytics for."), + folderId: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe("The folder ID(s) to retrieve analytics for (with operator support)."), + domain: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe("The domain(s) to retrieve analytics for."), + domainOperator: z.enum(["IN", "NOT IN"]).optional(), + tagIds: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe("The tag IDs to retrieve analytics for."), + tagIdsOperator: z.enum(["IN", "NOT IN"]).optional(), + folderIdOperator: z.enum(["IN", "NOT IN"]).optional(), + root: z + .union([z.string(), z.boolean(), z.array(z.union([z.string(), z.boolean()]))]) + .transform((v) => { + const normalize = (val: string | boolean): boolean => { + if (typeof val === 'boolean') return val; + return val === 'true' || val === '1' || val === 'yes'; + }; + if (Array.isArray(v)) return v.map(normalize); + return [normalize(v)]; + }) + .optional() + .describe("Filter for root domain links."), + rootOperator: z.enum(["IN", "NOT IN"]).optional(), + // Program/Partner/Group filters (not using advanced filtering for programId/tenantId/groupId) + programId: z.string().optional(), + partnerId: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe("The partner ID(s) to retrieve analytics for (with operator support)."), + partnerIdOperator: z.enum(["IN", "NOT IN"]).optional(), + tenantId: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe("The tenant ID(s) to retrieve analytics for (with operator support)."), + tenantIdOperator: z.enum(["IN", "NOT IN"]).optional(), + groupId: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .optional() + .describe("The group ID(s) to retrieve analytics for (with operator support)."), + groupIdOperator: z.enum(["IN", "NOT IN"]).optional(), + // Region is a special case - it's the subdivision part of a region code + region: z.string().optional(), + // All dimensional filters now go through the JSON filters parameter + filters: z + .string() + .optional() + .describe("JSON array of advanced filters with operators (IN, NOT IN)."), +}); export const eventsFilterTB = analyticsFilterTB .omit({ granularity: true, timezone: true }) @@ -341,3 +570,24 @@ export const eventsQuerySchema = analyticsQuerySchema .describe("DEPRECATED. Use `sortOrder` instead.") .meta({ deprecated: true }), }); + +/** + * Parse events query parameters with backward compatibility + * Converts deprecated singular fields (linkId, tagId) to their plural equivalents + */ +export function parseEventsQuery(searchParams: unknown) { + const data = eventsQuerySchema.parse(searchParams); + + // Backward compatibility: convert linkId to linkIds + if (data.linkId && !data.linkIds) { + data.linkIds = [data.linkId]; + } + + // Backward compatibility: convert tagId to tagIds + if (data.tagId && !data.tagIds) { + // Convert single tagId to the multi-value format + data.tagIds = { operator: "IS" as const, sqlOperator: "IN" as const, values: [data.tagId] }; + } + + return data; +} diff --git a/apps/web/scripts/download-top-links.ts b/apps/web/scripts/download-top-links.ts index 31ffb6d71be..3cd1ab44bd3 100644 --- a/apps/web/scripts/download-top-links.ts +++ b/apps/web/scripts/download-top-links.ts @@ -11,7 +11,7 @@ async function main() { groupBy: "top_links", workspaceId: "xxx", interval: "30d", - root: false, + root: { values: ["false"], operator: "IS", sqlOperator: "IN" }, }).then(async (data) => { return await Promise.all( data.map( diff --git a/apps/web/scripts/dub-wrapped.ts b/apps/web/scripts/dub-wrapped.ts index 395a4600784..2cca0f32088 100644 --- a/apps/web/scripts/dub-wrapped.ts +++ b/apps/web/scripts/dub-wrapped.ts @@ -61,7 +61,7 @@ async function main() { event: "clicks", groupBy: "top_links", interval: "ytd", - root: false, + root: { values: ["false"], operator: "IS", sqlOperator: "IN" }, }), getAnalytics({ workspaceId: projectId!, diff --git a/apps/web/scripts/get-top-links-for-workspace.ts b/apps/web/scripts/get-top-links-for-workspace.ts index fa52e738a34..8e3fe35eb8d 100644 --- a/apps/web/scripts/get-top-links-for-workspace.ts +++ b/apps/web/scripts/get-top-links-for-workspace.ts @@ -39,7 +39,7 @@ async function main() { groupBy: "top_links", workspaceId: workspace.id, interval: "30d", - root: false, + root: { values: ["false"], operator: "IS", sqlOperator: "IN" }, }).then(async (data) => { const topFive = data.slice(0, 5); return await Promise.all( diff --git a/apps/web/tests/analytics/advanced-filters.test.ts b/apps/web/tests/analytics/advanced-filters.test.ts new file mode 100644 index 00000000000..42c71bcd182 --- /dev/null +++ b/apps/web/tests/analytics/advanced-filters.test.ts @@ -0,0 +1,588 @@ +import { parseFilterValue, buildFilterValue } from "@dub/utils"; +import { + buildAdvancedFilters, + ensureParsedFilter, + extractWorkspaceLinkFilters, +} from "@/lib/analytics/filter-helpers"; +import { describe, expect, test } from "vitest"; + +describe("Advanced Filters - Unit Tests", () => { + describe("parseFilterValue", () => { + describe("Single Values", () => { + test("single positive value", () => { + const result = parseFilterValue("US"); + expect(result).toEqual({ + operator: "IS", + sqlOperator: "IN", + values: ["US"], + }); + }); + + test("single negative value", () => { + const result = parseFilterValue("-US"); + expect(result).toEqual({ + operator: "IS_NOT", + sqlOperator: "NOT IN", + values: ["US"], + }); + }); + }); + + describe("Multiple Values", () => { + test("multiple positive values", () => { + const result = parseFilterValue("US,BR,FR"); + expect(result).toEqual({ + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["US", "BR", "FR"], + }); + }); + + test("multiple negative values", () => { + const result = parseFilterValue("-US,BR,FR"); + expect(result).toEqual({ + operator: "IS_NOT_ONE_OF", + sqlOperator: "NOT IN", + values: ["US", "BR", "FR"], + }); + }); + + test("two values", () => { + const result = parseFilterValue("mobile,desktop"); + expect(result).toEqual({ + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["mobile", "desktop"], + }); + }); + }); + + describe("Edge Cases", () => { + test("empty string returns undefined", () => { + expect(parseFilterValue("")).toBeUndefined(); + }); + + test("undefined returns undefined", () => { + expect(parseFilterValue(undefined)).toBeUndefined(); + }); + + test("filters empty values from comma-separated", () => { + const result = parseFilterValue("US,,BR"); + expect(result?.values).toEqual(["US", "BR"]); + }); + + test("trailing comma", () => { + const result = parseFilterValue("US,BR,"); + expect(result?.values).toEqual(["US", "BR"]); + }); + + test("only commas returns undefined", () => { + expect(parseFilterValue(",,,")).toBeUndefined(); + }); + + test("minus sign only returns undefined", () => { + expect(parseFilterValue("-")).toBeUndefined(); + }); + + test("minus with empty values returns undefined", () => { + expect(parseFilterValue("-,,,")).toBeUndefined(); + }); + }); + + describe("Array Input", () => { + test("array with single value", () => { + const result = parseFilterValue(["US"]); + expect(result).toEqual({ + operator: "IS", + sqlOperator: "IN", + values: ["US"], + }); + }); + + test("array with multiple values", () => { + const result = parseFilterValue(["US", "BR", "FR"]); + expect(result).toEqual({ + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["US", "BR", "FR"], + }); + }); + }); + }); + + describe("buildFilterValue", () => { + test("rebuild single positive value", () => { + const result = buildFilterValue({ + operator: "IS", + sqlOperator: "IN", + values: ["US"], + }); + expect(result).toBe("US"); + }); + + test("rebuild multiple positive values", () => { + const result = buildFilterValue({ + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["US", "BR", "FR"], + }); + expect(result).toBe("US,BR,FR"); + }); + + test("rebuild single negative value", () => { + const result = buildFilterValue({ + operator: "IS_NOT", + sqlOperator: "NOT IN", + values: ["US"], + }); + expect(result).toBe("-US"); + }); + + test("rebuild multiple negative values", () => { + const result = buildFilterValue({ + operator: "IS_NOT_ONE_OF", + sqlOperator: "NOT IN", + values: ["US", "BR"], + }); + expect(result).toBe("-US,BR"); + }); + + describe("Round-trip", () => { + test("single positive", () => { + const original = "US"; + const parsed = parseFilterValue(original)!; + const rebuilt = buildFilterValue(parsed); + expect(rebuilt).toBe(original); + }); + + test("multiple positive", () => { + const original = "US,BR,FR"; + const parsed = parseFilterValue(original)!; + const rebuilt = buildFilterValue(parsed); + expect(rebuilt).toBe(original); + }); + + test("single negative", () => { + const original = "-US"; + const parsed = parseFilterValue(original)!; + const rebuilt = buildFilterValue(parsed); + expect(rebuilt).toBe(original); + }); + + test("multiple negative", () => { + const original = "-US,BR,FR"; + const parsed = parseFilterValue(original)!; + const rebuilt = buildFilterValue(parsed); + expect(rebuilt).toBe(original); + }); + }); + }); + + describe("buildAdvancedFilters", () => { + test("single field with IN operator (single value)", () => { + const result = buildAdvancedFilters({ + country: { + operator: "IS", + sqlOperator: "IN", + values: ["US"], + }, + }); + expect(result).toEqual([ + { + field: "country", + operator: "IN", + values: ["US"], + }, + ]); + }); + + test("single field with IN operator", () => { + const result = buildAdvancedFilters({ + country: { + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["US", "BR", "FR"], + }, + }); + expect(result).toEqual([ + { + field: "country", + operator: "IN", + values: ["US", "BR", "FR"], + }, + ]); + }); + + test("single field with NOT IN operator", () => { + const result = buildAdvancedFilters({ + device: { + operator: "IS_NOT_ONE_OF", + sqlOperator: "NOT IN", + values: ["Mobile", "Tablet"], + }, + }); + expect(result).toEqual([ + { + field: "device", + operator: "NOT IN", + values: ["Mobile", "Tablet"], + }, + ]); + }); + + test("multiple fields combined", () => { + const result = buildAdvancedFilters({ + country: { + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["US", "BR"], + }, + device: { + operator: "IS", + sqlOperator: "IN", + values: ["Desktop"], + }, + }); + expect(result).toHaveLength(2); + expect(result).toContainEqual({ + field: "country", + operator: "IN", + values: ["US", "BR"], + }); + expect(result).toContainEqual({ + field: "device", + operator: "IN", + values: ["Desktop"], + }); + }); + + test("empty params returns empty array", () => { + const result = buildAdvancedFilters({}); + expect(result).toEqual([]); + }); + + test("skips undefined fields", () => { + const result = buildAdvancedFilters({ + country: { + operator: "IS", + sqlOperator: "IN", + values: ["US"], + }, + city: undefined, + device: undefined, + }); + expect(result).toHaveLength(1); + expect(result[0].field).toBe("country"); + }); + + test("handles all supported fields", () => { + const result = buildAdvancedFilters({ + country: { operator: "IS", sqlOperator: "IN", values: ["US"] }, + city: { operator: "IS", sqlOperator: "IN", values: ["NYC"] }, + device: { operator: "IS", sqlOperator: "IN", values: ["Mobile"] }, + browser: { operator: "IS", sqlOperator: "IN", values: ["Chrome"] }, + os: { operator: "IS", sqlOperator: "IN", values: ["Mac"] }, + }); + expect(result).toHaveLength(5); + expect(result.map((f) => f.field)).toEqual([ + "country", + "city", + "device", + "browser", + "os", + ]); + }); + + test("maintains insertion order", () => { + const result = buildAdvancedFilters({ + device: { operator: "IS", sqlOperator: "IN", values: ["Mobile"] }, + country: { operator: "IS", sqlOperator: "IN", values: ["US"] }, + browser: { operator: "IS", sqlOperator: "IN", values: ["Chrome"] }, + }); + // Should maintain order from SUPPORTED_FIELDS, not insertion order + expect(result[0].field).toBe("country"); + expect(result[1].field).toBe("device"); + expect(result[2].field).toBe("browser"); + }); + }); + + describe("ensureParsedFilter", () => { + test("returns undefined for undefined input", () => { + expect(ensureParsedFilter(undefined)).toBeUndefined(); + }); + + test("returns undefined for empty string", () => { + expect(ensureParsedFilter("")).toBeUndefined(); + }); + + test("converts plain string to IS ParsedFilter", () => { + const result = ensureParsedFilter("pn_abc123"); + expect(result).toEqual({ + operator: "IS", + sqlOperator: "IN", + values: ["pn_abc123"], + }); + }); + + test("passes through ParsedFilter unchanged (IS)", () => { + const input = { + operator: "IS" as const, + sqlOperator: "IN" as const, + values: ["pn_abc123"], + }; + expect(ensureParsedFilter(input)).toEqual(input); + }); + + test("passes through ParsedFilter unchanged (IS_NOT)", () => { + const input = { + operator: "IS_NOT" as const, + sqlOperator: "NOT IN" as const, + values: ["pn_abc123"], + }; + expect(ensureParsedFilter(input)).toEqual(input); + }); + + test("passes through ParsedFilter with multiple values", () => { + const input = { + operator: "IS_ONE_OF" as const, + sqlOperator: "IN" as const, + values: ["pn_abc123", "pn_def456"], + }; + expect(ensureParsedFilter(input)).toEqual(input); + }); + + test("passes through negated ParsedFilter with multiple values", () => { + const input = { + operator: "IS_NOT_ONE_OF" as const, + sqlOperator: "NOT IN" as const, + values: ["pn_abc123", "pn_def456"], + }; + expect(ensureParsedFilter(input)).toEqual(input); + }); + }); + + describe("extractWorkspaceLinkFilters - partnerId", () => { + test("extracts single partnerId with IN operator", () => { + const result = extractWorkspaceLinkFilters({ + partnerId: { + operator: "IS", + sqlOperator: "IN", + values: ["pn_abc123"], + }, + }); + expect(result.partnerId).toEqual(["pn_abc123"]); + expect(result.partnerIdOperator).toBe("IN"); + }); + + test("extracts multiple partnerIds with IN operator", () => { + const result = extractWorkspaceLinkFilters({ + partnerId: { + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["pn_abc123", "pn_def456", "pn_ghi789"], + }, + }); + expect(result.partnerId).toEqual(["pn_abc123", "pn_def456", "pn_ghi789"]); + expect(result.partnerIdOperator).toBe("IN"); + }); + + test("extracts single partnerId with NOT IN operator", () => { + const result = extractWorkspaceLinkFilters({ + partnerId: { + operator: "IS_NOT", + sqlOperator: "NOT IN", + values: ["pn_abc123"], + }, + }); + expect(result.partnerId).toEqual(["pn_abc123"]); + expect(result.partnerIdOperator).toBe("NOT IN"); + }); + + test("extracts multiple partnerIds with NOT IN operator", () => { + const result = extractWorkspaceLinkFilters({ + partnerId: { + operator: "IS_NOT_ONE_OF", + sqlOperator: "NOT IN", + values: ["pn_abc123", "pn_def456"], + }, + }); + expect(result.partnerId).toEqual(["pn_abc123", "pn_def456"]); + expect(result.partnerIdOperator).toBe("NOT IN"); + }); + + test("returns undefined partnerId when not provided", () => { + const result = extractWorkspaceLinkFilters({}); + expect(result.partnerId).toBeUndefined(); + expect(result.partnerIdOperator).toBe("IN"); // default + }); + + test("works alongside other workspace link filters", () => { + const result = extractWorkspaceLinkFilters({ + domain: { + operator: "IS", + sqlOperator: "IN", + values: ["dub.sh"], + }, + partnerId: { + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["pn_abc123", "pn_def456"], + }, + folderId: { + operator: "IS_NOT", + sqlOperator: "NOT IN", + values: ["fold_xyz"], + }, + }); + expect(result.domain).toEqual(["dub.sh"]); + expect(result.domainOperator).toBe("IN"); + expect(result.partnerId).toEqual(["pn_abc123", "pn_def456"]); + expect(result.partnerIdOperator).toBe("IN"); + expect(result.folderId).toEqual(["fold_xyz"]); + expect(result.folderIdOperator).toBe("NOT IN"); + }); + }); + + describe("extractWorkspaceLinkFilters - groupId", () => { + test("extracts single groupId with IN operator", () => { + const result = extractWorkspaceLinkFilters({ + groupId: { + operator: "IS", + sqlOperator: "IN", + values: ["grp_abc123"], + }, + }); + expect(result.groupId).toEqual(["grp_abc123"]); + expect(result.groupIdOperator).toBe("IN"); + }); + + test("extracts multiple groupIds with IN operator", () => { + const result = extractWorkspaceLinkFilters({ + groupId: { + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["grp_abc123", "grp_def456", "grp_ghi789"], + }, + }); + expect(result.groupId).toEqual(["grp_abc123", "grp_def456", "grp_ghi789"]); + expect(result.groupIdOperator).toBe("IN"); + }); + + test("extracts single groupId with NOT IN operator", () => { + const result = extractWorkspaceLinkFilters({ + groupId: { + operator: "IS_NOT", + sqlOperator: "NOT IN", + values: ["grp_abc123"], + }, + }); + expect(result.groupId).toEqual(["grp_abc123"]); + expect(result.groupIdOperator).toBe("NOT IN"); + }); + + test("extracts multiple groupIds with NOT IN operator", () => { + const result = extractWorkspaceLinkFilters({ + groupId: { + operator: "IS_NOT_ONE_OF", + sqlOperator: "NOT IN", + values: ["grp_abc123", "grp_def456"], + }, + }); + expect(result.groupId).toEqual(["grp_abc123", "grp_def456"]); + expect(result.groupIdOperator).toBe("NOT IN"); + }); + + test("returns undefined groupId when not provided", () => { + const result = extractWorkspaceLinkFilters({}); + expect(result.groupId).toBeUndefined(); + expect(result.groupIdOperator).toBe("IN"); // default + }); + + test("works alongside partnerId and other filters", () => { + const result = extractWorkspaceLinkFilters({ + partnerId: { + operator: "IS", + sqlOperator: "IN", + values: ["pn_abc123"], + }, + groupId: { + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["grp_abc123", "grp_def456"], + }, + domain: { + operator: "IS_NOT", + sqlOperator: "NOT IN", + values: ["spam.com"], + }, + }); + expect(result.partnerId).toEqual(["pn_abc123"]); + expect(result.partnerIdOperator).toBe("IN"); + expect(result.groupId).toEqual(["grp_abc123", "grp_def456"]); + expect(result.groupIdOperator).toBe("IN"); + expect(result.domain).toEqual(["spam.com"]); + expect(result.domainOperator).toBe("NOT IN"); + }); + }); + + describe("extractWorkspaceLinkFilters - tenantId", () => { + test("extracts single tenantId with IN operator", () => { + const result = extractWorkspaceLinkFilters({ + tenantId: { + operator: "IS", + sqlOperator: "IN", + values: ["tenant_abc123"], + }, + }); + expect(result.tenantId).toEqual(["tenant_abc123"]); + expect(result.tenantIdOperator).toBe("IN"); + }); + + test("extracts multiple tenantIds with IN operator", () => { + const result = extractWorkspaceLinkFilters({ + tenantId: { + operator: "IS_ONE_OF", + sqlOperator: "IN", + values: ["tenant_abc", "tenant_def", "tenant_ghi"], + }, + }); + expect(result.tenantId).toEqual(["tenant_abc", "tenant_def", "tenant_ghi"]); + expect(result.tenantIdOperator).toBe("IN"); + }); + + test("extracts tenantId with NOT IN operator", () => { + const result = extractWorkspaceLinkFilters({ + tenantId: { + operator: "IS_NOT_ONE_OF", + sqlOperator: "NOT IN", + values: ["tenant_abc", "tenant_def"], + }, + }); + expect(result.tenantId).toEqual(["tenant_abc", "tenant_def"]); + expect(result.tenantIdOperator).toBe("NOT IN"); + }); + + test("returns undefined tenantId when not provided", () => { + const result = extractWorkspaceLinkFilters({}); + expect(result.tenantId).toBeUndefined(); + expect(result.tenantIdOperator).toBe("IN"); + }); + + test("works alongside all other workspace link filters", () => { + const result = extractWorkspaceLinkFilters({ + domain: { operator: "IS", sqlOperator: "IN", values: ["dub.sh"] }, + partnerId: { operator: "IS", sqlOperator: "IN", values: ["pn_abc"] }, + groupId: { operator: "IS", sqlOperator: "IN", values: ["grp_abc"] }, + tenantId: { operator: "IS_ONE_OF", sqlOperator: "IN", values: ["t1", "t2"] }, + folderId: { operator: "IS_NOT", sqlOperator: "NOT IN", values: ["fold_x"] }, + }); + expect(result.domain).toEqual(["dub.sh"]); + expect(result.partnerId).toEqual(["pn_abc"]); + expect(result.groupId).toEqual(["grp_abc"]); + expect(result.tenantId).toEqual(["t1", "t2"]); + expect(result.tenantIdOperator).toBe("IN"); + expect(result.folderId).toEqual(["fold_x"]); + expect(result.folderIdOperator).toBe("NOT IN"); + }); + }); +}); diff --git a/apps/web/tests/analytics/index.test.ts b/apps/web/tests/analytics/index.test.ts index 13f17a0e804..360733200ee 100644 --- a/apps/web/tests/analytics/index.test.ts +++ b/apps/web/tests/analytics/index.test.ts @@ -4,6 +4,7 @@ import { describe, expect, test } from "vitest"; import * as z from "zod/v4"; import { env } from "../utils/env"; import { IntegrationHarness } from "../utils/integration"; +import { E2E_PARTNER, E2E_PARTNERS, E2E_PARTNER_GROUP } from "../utils/resource"; describe.runIf(env.CI).sequential("GET /analytics", async () => { const h = new IntegrationHarness(); @@ -58,4 +59,473 @@ describe.runIf(env.CI).sequential("GET /analytics", async () => { data.every((event) => event.metadata?.productId === "premiumProductId"), ).toBe(true); }); + + describe("Advanced Filters", () => { + test("single country filter", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "count", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "US", + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + }); + + test("multiple countries filter (IS ONE OF)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "countries", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "US,CA,GB", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + + // All returned countries should be in the filter + data.forEach((item: any) => { + expect(["US", "CA", "GB"]).toContain(item.country); + }); + }); + + test("exclude country (IS NOT)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "countries", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "-US", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + + // No country should be US + data.forEach((item: any) => { + expect(item.country).not.toBe("US"); + }); + }); + + test("exclude multiple countries (IS NOT ONE OF)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "countries", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "-US,GB", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + + // No country should be US or GB + data.forEach((item: any) => { + expect(["US", "GB"]).not.toContain(item.country); + }); + }); + + test("multiple devices filter", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "devices", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + device: "mobile,desktop", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + + // All returned devices should be in the filter + data.forEach((item: any) => { + expect(["Mobile", "Desktop"]).toContain(item.device); + }); + }); + + test("country AND device filters combined", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "count", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "US", + device: "desktop", + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + // Should return clicks that are both from US AND desktop + }); + + test("timeseries with country filter", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "timeseries", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "US", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + // Timeseries should only include US data + }); + + test("backward compatibility - old format still works", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "count", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "US", + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + // Old single-value format should still work + }); + }); + + describe("partnerId Filters", () => { + test("single partnerId filter (count)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "count", + workspaceId, + interval: "all", + partnerId: E2E_PARTNER.id, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + expect(data).toHaveProperty("leads"); + expect(data).toHaveProperty("sales"); + }); + + test("multiple partnerIds filter (IS ONE OF)", async () => { + const partnerIds = E2E_PARTNERS.map((p) => p.id).join(","); + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "count", + workspaceId, + interval: "all", + partnerId: partnerIds, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + expect(data).toHaveProperty("leads"); + expect(data).toHaveProperty("sales"); + }); + + test("exclude partnerId (IS NOT)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "count", + workspaceId, + interval: "all", + partnerId: `-${E2E_PARTNER.id}`, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + expect(data).toHaveProperty("leads"); + expect(data).toHaveProperty("sales"); + }); + + test("exclude multiple partnerIds (IS NOT ONE OF)", async () => { + const partnerIds = E2E_PARTNERS.map((p) => p.id).join(","); + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "count", + workspaceId, + interval: "all", + partnerId: `-${partnerIds}`, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + expect(data).toHaveProperty("leads"); + expect(data).toHaveProperty("sales"); + }); + + test("partnerId with timeseries groupBy", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "timeseries", + workspaceId, + interval: "30d", + partnerId: E2E_PARTNER.id, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("partnerId with top_links groupBy", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "top_links", + workspaceId, + interval: "all", + partnerId: E2E_PARTNER.id, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("partnerId combined with country filter", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "count", + workspaceId, + interval: "all", + partnerId: E2E_PARTNER.id, + country: "US", + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + }); + + test("backward compatibility - single partnerId still works", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "count", + workspaceId, + interval: "all", + partnerId: E2E_PARTNER.id, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + }); + }); + + describe("groupId Filters", () => { + test("single groupId filter (count)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "count", + workspaceId, + interval: "all", + groupId: E2E_PARTNER_GROUP.id, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + expect(data).toHaveProperty("leads"); + expect(data).toHaveProperty("sales"); + }); + + test("exclude groupId (IS NOT)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "count", + workspaceId, + interval: "all", + groupId: `-${E2E_PARTNER_GROUP.id}`, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + expect(data).toHaveProperty("leads"); + expect(data).toHaveProperty("sales"); + }); + + test("groupId with timeseries groupBy", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "timeseries", + workspaceId, + interval: "30d", + groupId: E2E_PARTNER_GROUP.id, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("groupId combined with partnerId filter", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "count", + workspaceId, + interval: "all", + groupId: E2E_PARTNER_GROUP.id, + partnerId: E2E_PARTNER.id, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + }); + + test("backward compatibility - single groupId still works", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "count", + workspaceId, + interval: "all", + groupId: E2E_PARTNER_GROUP.id, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + }); + }); + + describe("tenantId Filters", () => { + test("single tenantId filter (count)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "count", + workspaceId, + interval: "all", + tenantId: E2E_PARTNER.tenantId, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + expect(data).toHaveProperty("leads"); + expect(data).toHaveProperty("sales"); + }); + + test("exclude tenantId (IS NOT)", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "composite", + groupBy: "count", + workspaceId, + interval: "all", + tenantId: `-${E2E_PARTNER.tenantId}`, + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + expect(data).toHaveProperty("leads"); + expect(data).toHaveProperty("sales"); + }); + + test("tenantId with timeseries groupBy", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "timeseries", + workspaceId, + interval: "30d", + tenantId: E2E_PARTNER.tenantId, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("tenantId combined with country filter", async () => { + const { status, data } = await http.get({ + path: `/analytics`, + query: { + event: "clicks", + groupBy: "count", + workspaceId, + interval: "all", + tenantId: E2E_PARTNER.tenantId, + country: "US", + }, + }); + + expect(status).toEqual(200); + expect(data).toHaveProperty("clicks"); + }); + }); }); diff --git a/apps/web/tests/analytics/normalize-filter.test.ts b/apps/web/tests/analytics/normalize-filter.test.ts new file mode 100644 index 00000000000..408b5ffeee2 --- /dev/null +++ b/apps/web/tests/analytics/normalize-filter.test.ts @@ -0,0 +1,200 @@ +import { normalizeActiveFilter } from "@dub/ui"; +import { describe, expect, test } from "vitest"; + +describe("normalizeActiveFilter", () => { + describe("New format (already normalized)", () => { + test("returns unchanged if already has operator and values array", () => { + const input = { + key: "country", + operator: "IS_ONE_OF" as const, + values: ["US", "BR"], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual(input); + }); + + test("handles IS_NOT operator", () => { + const input = { + key: "device", + operator: "IS_NOT" as const, + values: ["Mobile"], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual(input); + }); + + test("handles IS_NOT_ONE_OF operator", () => { + const input = { + key: "country", + operator: "IS_NOT_ONE_OF" as const, + values: ["US", "BR", "FR"], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual(input); + }); + }); + + describe("Legacy singular format { key, value }", () => { + test("converts single value to IS operator with values array", () => { + const input = { + key: "country", + value: "US", + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "country", + operator: "IS", + values: ["US"], + }); + }); + + test("handles string values", () => { + const input = { + key: "domain", + value: "dub.sh", + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "domain", + operator: "IS", + values: ["dub.sh"], + }); + }); + + test("handles non-string values", () => { + const input = { + key: "userId", + value: 123, + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "userId", + operator: "IS", + values: [123], + }); + }); + }); + + describe("Legacy plural format { key, values }", () => { + test("single value becomes IS operator", () => { + const input = { + key: "tagIds", + values: ["tag_123"], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "tagIds", + operator: "IS", + values: ["tag_123"], + }); + }); + + test("multiple values becomes IS_ONE_OF operator", () => { + const input = { + key: "tagIds", + values: ["tag_123", "tag_456", "tag_789"], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "tagIds", + operator: "IS_ONE_OF", + values: ["tag_123", "tag_456", "tag_789"], + }); + }); + + test("two values becomes IS_ONE_OF operator", () => { + const input = { + key: "country", + values: ["US", "BR"], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "country", + operator: "IS_ONE_OF", + values: ["US", "BR"], + }); + }); + + test("empty values array becomes IS with empty array", () => { + const input = { + key: "tagIds", + values: [], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "tagIds", + operator: "IS", + values: [], + }); + }); + }); + + describe("Edge cases", () => { + test("handles filter without operator or values", () => { + const input = { + key: "country", + } as any; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "country", + operator: "IS", + values: [], + }); + }); + + test("preserves other properties not in the spec", () => { + const input = { + key: "country", + value: "US", + // @ts-expect-error - testing extra properties + extraProp: "should be ignored", + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "country", + operator: "IS", + values: ["US"], + }); + // Extra properties are not preserved + expect(result).not.toHaveProperty("extraProp"); + }); + }); + + describe("Real-world scenarios", () => { + test("Links page tags filter (multiple: true, hideOperator: true)", () => { + const input = { + key: "tagIds", + values: ["tag_abc", "tag_def"], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "tagIds", + operator: "IS_ONE_OF", + values: ["tag_abc", "tag_def"], + }); + }); + + test("Analytics page with advanced filters", () => { + const input = { + key: "country", + operator: "IS_NOT_ONE_OF" as const, + values: ["US", "GB"], + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual(input); + }); + + test("Single domain filter (legacy format)", () => { + const input = { + key: "domain", + value: "dub.sh", + }; + const result = normalizeActiveFilter(input); + expect(result).toEqual({ + key: "domain", + operator: "IS", + values: ["dub.sh"], + }); + }); + }); +}); diff --git a/apps/web/tests/events/index.test.ts b/apps/web/tests/events/index.test.ts new file mode 100644 index 00000000000..802da8417c7 --- /dev/null +++ b/apps/web/tests/events/index.test.ts @@ -0,0 +1,379 @@ +import { clickEventResponseSchema } from "@/lib/zod/schemas/clicks"; +import { leadEventResponseSchema } from "@/lib/zod/schemas/leads"; +import { saleEventResponseSchema } from "@/lib/zod/schemas/sales"; +import { describe, expect, test } from "vitest"; +import * as z from "zod/v4"; +import { env } from "../utils/env"; +import { IntegrationHarness } from "../utils/integration"; +import { E2E_PARTNER, E2E_PARTNERS, E2E_PARTNER_GROUP } from "../utils/resource"; + +describe.runIf(env.CI).sequential("GET /events", async () => { + const h = new IntegrationHarness(); + const { workspace, http } = await h.init(); + const workspaceId = workspace.id; + + test("get clicks events", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + }, + }); + + const responseSchema = z.array(clickEventResponseSchema.strict()); + const parsed = responseSchema.safeParse(data); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + expect(parsed.success).toBeTruthy(); + }); + + test("get leads events", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "leads", + workspaceId, + interval: "30d", + }, + }); + + const responseSchema = z.array(leadEventResponseSchema.strict()); + const parsed = responseSchema.safeParse(data); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + expect(parsed.success).toBeTruthy(); + }); + + test("get sales events", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "sales", + workspaceId, + interval: "30d", + }, + }); + + const responseSchema = z.array(saleEventResponseSchema.strict()); + const parsed = responseSchema.safeParse(data); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + expect(parsed.success).toBeTruthy(); + }); + + describe("Advanced Filters", () => { + test("filter events by country", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "US", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by multiple countries (IS ONE OF)", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "US,CA,GB", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("exclude country (IS NOT)", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "-US", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by device", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + device: "desktop", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by utm_source", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + utm_source: "e2e", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by metadata query", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "sales", + workspaceId, + interval: "30d", + query: "metadata['productId']:premiumProductId", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("combine multiple filters", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + country: "US", + device: "desktop", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by single partnerId", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + partnerId: E2E_PARTNER.id, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by multiple partnerIds (IS ONE OF)", async () => { + const partnerIds = E2E_PARTNERS.map((p) => p.id).join(","); + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + partnerId: partnerIds, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("exclude events by partnerId (IS NOT)", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + partnerId: `-${E2E_PARTNER.id}`, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by partnerId combined with country", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + partnerId: E2E_PARTNER.id, + country: "US", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by single groupId", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + groupId: E2E_PARTNER_GROUP.id, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("exclude events by groupId (IS NOT)", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + groupId: `-${E2E_PARTNER_GROUP.id}`, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by groupId combined with partnerId", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + groupId: E2E_PARTNER_GROUP.id, + partnerId: E2E_PARTNER.id, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("filter events by single tenantId", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + tenantId: E2E_PARTNER.tenantId, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + + test("exclude events by tenantId (IS NOT)", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "all", + tenantId: `-${E2E_PARTNER.tenantId}`, + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + }); + }); + + describe("Pagination and Sorting", () => { + test("sort events by timestamp descending (default)", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + + if (data.length > 1) { + for (let i = 0; i < data.length - 1; i++) { + const current = new Date(data[i].timestamp).getTime(); + const next = new Date(data[i + 1].timestamp).getTime(); + expect(current).toBeGreaterThanOrEqual(next); + } + } + }); + + test("sort events by timestamp ascending", async () => { + const { status, data } = await http.get({ + path: `/events`, + query: { + event: "clicks", + workspaceId, + interval: "30d", + domain: "dub.sh", + key: "checkly-check", + sortOrder: "asc", + }, + }); + + expect(status).toEqual(200); + expect(Array.isArray(data)).toBe(true); + + if (data.length > 1) { + for (let i = 0; i < data.length - 1; i++) { + const current = new Date(data[i].timestamp).getTime(); + const next = new Date(data[i + 1].timestamp).getTime(); + expect(current).toBeLessThanOrEqual(next); + } + } + }); + }); +}); diff --git a/apps/web/ui/analytics/toggle.tsx b/apps/web/ui/analytics/toggle.tsx index dbbefa0599d..fcf3dbbcac3 100644 --- a/apps/web/ui/analytics/toggle.tsx +++ b/apps/web/ui/analytics/toggle.tsx @@ -67,8 +67,10 @@ export function AnalyticsToggle({ activeFilters, onSelect, onRemove, + onRemoveFilter, onRemoveAll, onOpenFilter, + onToggleOperator, streaming, activeFiltersWithStreaming, } = useAnalyticsFilters({ partnerPage, dashboardProps }); @@ -81,6 +83,7 @@ export function AnalyticsToggle({ onSelect={onSelect} onRemove={onRemove} onOpenFilter={onOpenFilter} + isAdvancedFilter={true} askAI /> ); @@ -288,7 +291,10 @@ export function AnalyticsToggle({ activeFilters={activeFiltersWithStreaming} onSelect={onSelect} onRemove={onRemove} + onRemoveFilter={onRemoveFilter} onRemoveAll={onRemoveAll} + onToggleOperator={onToggleOperator} + isAdvancedFilter={true} />
searchParamsObj.tagIds?.split(",")?.filter(Boolean) ?? [], - [searchParamsObj.tagIds], - ); - const selectedCustomerId = searchParamsObj.customerId; const { data: selectedCustomerWorkspace } = useCustomer({ @@ -119,68 +118,66 @@ export function useAnalyticsFilters({ const [requestedFilters, setRequestedFilters] = useState([]); + const parseFilterParam = useCallback((value: string): ParsedFilter | undefined => { + return parseFilterValue(value); + }, []); + const activeFilters = useMemo(() => { - const { domain, key, root, folderId, ...params } = searchParamsObj; + const { domain, key, tagIds, root, folderId, ...params } = searchParamsObj; // Handle special cases first - const filters = [ - // Handle domain/key special case - ...(domain && !key ? [{ key: "domain", value: domain }] : []), + const filters: Array<{ key: string; operator: FilterOperator; values: any[] }> = [ + // Handle domain/key special case for links ...(domain && key ? [ { key: "link", - value: linkConstructor({ domain, key, pretty: true }), + operator: "IS" as FilterOperator, + values: [linkConstructor({ domain, key, pretty: true })], }, ] : []), - // Handle tagIds special case - ...(selectedTagIds.length > 0 - ? [{ key: "tagIds", value: selectedTagIds }] - : []), - // Handle root special case - convert string to boolean - ...(root ? [{ key: "root", value: root === "true" }] : []), - // Handle folderId special case - ...(folderId ? [{ key: "folderId", value: folderId }] : []), // Handle customerId special case ...(selectedCustomer ? [ { key: "customerId", - value: + operator: "IS" as FilterOperator, + values: [ selectedCustomer.email || selectedCustomer["name"] || selectedCustomer["externalId"], + ], }, ] : []), ]; - // Handle all other filters dynamically + // Handle all filters dynamically (including domain, tagIds, folderId, root) VALID_ANALYTICS_FILTERS.forEach((filter) => { // Skip special cases we handled above - if ( - ["domain", "key", "tagId", "tagIds", "root", "customerId"].includes( - filter, - ) - ) - return; - // also skip date range filters and qr + if (["key", "tagId", "customerId"].includes(filter)) return; + // Also skip date range filters and qr if (["interval", "start", "end", "qr"].includes(filter)) return; + // Skip domain if we're showing a specific link (domain + key) + if (filter === "domain" && domain && key) return; - const value = params[filter]; + const value = params[filter] || (filter === "domain" ? domain : filter === "tagIds" ? tagIds : filter === "root" ? root : filter === "folderId" ? folderId : undefined); if (value) { - filters.push({ key: filter, value }); + const parsed = parseFilterParam(value); + if (parsed) { + filters.push({ key: filter, operator: parsed.operator, values: parsed.values }); + } } }); return filters; }, [ searchParamsObj, - selectedTagIds, partnerPage, selectedCustomerId, selectedCustomer, + parseFilterParam, ]); const isRequested = useCallback( @@ -414,6 +411,7 @@ export function useAnalyticsFilters({ key: "ai", icon: Magic, label: "Ask AI", + singleSelect: true, separatorAfter: true, options: aiFilterSuggestions?.map(({ icon, value }) => ({ @@ -529,12 +527,12 @@ export function useAnalyticsFilters({ label: "Link type", options: [ { - value: true, + value: "true", icon: Globe2, label: "Root domain link", }, { - value: false, + value: "false", icon: Hyperlink, label: "Regular short link", }, @@ -546,14 +544,21 @@ export function useAnalyticsFilters({ key: "country", icon: FlagWavy, label: "Country", - getOptionIcon: (value) => ( - {value} - ), - getOptionLabel: (value) => COUNTRIES[value], + getOptionIcon: (value) => { + if (typeof value !== 'string') return null; + + return ( + {value} + ); + }, + getOptionLabel: (value) => { + if (typeof value !== 'string') return String(value); + return COUNTRIES[value] || value; + }, options: countries?.map(({ country, ...rest }) => ({ value: country, @@ -601,10 +606,14 @@ export function useAnalyticsFilters({ key: "continent", icon: MapPosition, label: "Continent", - getOptionIcon: (value) => ( - - ), - getOptionLabel: (value) => CONTINENTS[value], + getOptionIcon: (value) => { + if (typeof value !== 'string') return null; + return ; + }, + getOptionLabel: (value) => { + if (typeof value !== 'string') return String(value); + return CONTINENTS[value] || value; + }, options: continents?.map(({ continent, ...rest }) => ({ value: continent, @@ -616,13 +625,16 @@ export function useAnalyticsFilters({ key: "device", icon: MobilePhone, label: "Device", - getOptionIcon: (value) => ( - - ), + getOptionIcon: (value) => { + if (typeof value !== 'string') return null; + return ( + + ); + }, options: devices?.map(({ device, ...rest }) => ({ value: device, @@ -634,9 +646,10 @@ export function useAnalyticsFilters({ key: "browser", icon: Window, label: "Browser", - getOptionIcon: (value) => ( - - ), + getOptionIcon: (value) => { + if (typeof value !== 'string') return null; + return ; + }, options: browsers?.map(({ browser, ...rest }) => ({ value: browser, @@ -648,9 +661,10 @@ export function useAnalyticsFilters({ key: "os", icon: Cube, label: "OS", - getOptionIcon: (value) => ( - - ), + getOptionIcon: (value) => { + if (typeof value !== 'string') return null; + return ; + }, options: os?.map(({ os, ...rest }) => ({ value: os, @@ -682,9 +696,10 @@ export function useAnalyticsFilters({ key: "referer", icon: ReferredVia, label: "Referrer", - getOptionIcon: (value, _props) => ( - - ), + getOptionIcon: (value, _props) => { + if (typeof value !== 'string') return null; + return ; + }, options: referers?.map(({ referer, ...rest }) => ({ value: referer, @@ -699,9 +714,10 @@ export function useAnalyticsFilters({ key: "refererUrl", icon: ReferredVia, label: "Referrer URL", - getOptionIcon: (value, props) => ( - - ), + getOptionIcon: (value, props) => { + if (typeof value !== 'string') return null; + return ; + }, options: refererUrls?.map(({ refererUrl, ...rest }) => ({ value: refererUrl, @@ -731,9 +747,10 @@ export function useAnalyticsFilters({ key, icon: Icon, label: `UTM ${label}`, - getOptionIcon: (value) => ( - - ), + getOptionIcon: (value) => { + if (typeof value !== 'string') return null; + return ; + }, options: utmData[key]?.map((dt) => ({ value: dt[key], @@ -803,7 +820,6 @@ export function useAnalyticsFilters({ linkTags, folders, groups, - selectedTagIds, selectedCustomerId, countries, cities, @@ -821,6 +837,27 @@ export function useAnalyticsFilters({ const onSelect = useCallback( async (key, value) => { + if (Array.isArray(value)) { + if (value.length === 0) { + queryParams({ del: key, scroll: false }); + } else { + const currentParam = searchParamsObj[key]; + const isNegated = currentParam?.startsWith("-") ?? false; + + const newParam = isNegated + ? `-${value.join(",")}` + : value.join(","); + + queryParams({ + set: { [key]: newParam }, + del: "page", + scroll: false, + }); + } + + return; + } + if (key === "ai") { setStreaming(true); const prompt = value.replace("Ask AI ", ""); @@ -839,46 +876,80 @@ export function useAnalyticsFilters({ } } setStreaming(false); - } else { + } else if (key === "link") { queryParams({ - set: - key === "link" - ? { - domain: new URL(`https://${value}`).hostname, - key: new URL(`https://${value}`).pathname.slice(1) || "_root", - } - : key === "tagIds" - ? { - tagIds: selectedTagIds.concat(value).join(","), - } - : { - [key]: value, - }, + set: { + domain: new URL(`https://${value}`).hostname, + key: new URL(`https://${value}`).pathname.slice(1) || "_root", + }, del: "page", scroll: false, }); + } else { + const currentParam = searchParamsObj[key]; + + if (!currentParam) { + queryParams({ + set: { [key]: value }, + del: "page", + scroll: false, + }); + } else { + const parsed = parseFilterParam(currentParam); + + if (parsed && !parsed.values.includes(value)) { + const newValues = [...parsed.values, value]; + const newParam = parsed.operator.includes("NOT") + ? `-${newValues.join(",")}` + : newValues.join(","); + + queryParams({ + set: { [key]: newParam }, + del: "page", + scroll: false, + }); + } + } } }, - [queryParams, activeFilters, selectedTagIds], + [queryParams, activeFilters, searchParamsObj, parseFilterParam], ); const onRemove = useCallback( - (key, value) => - queryParams( - key === "tagIds" && - !(selectedTagIds.length === 1 && selectedTagIds[0] === value) - ? { - set: { - tagIds: selectedTagIds.filter((id) => id !== value).join(","), - }, - scroll: false, - } - : { - del: key === "link" ? ["domain", "key", "url"] : key, - scroll: false, - }, - ), - [queryParams, selectedTagIds], + (key, value) => { + if (key === "link") { + queryParams({ + del: ["domain", "key", "url"], + scroll: false, + }); + } else { + const currentParam = searchParamsObj[key]; + + if (!currentParam) return; + + const parsed = parseFilterParam(currentParam); + if (!parsed) { + queryParams({ del: key, scroll: false }); + return; + } + + const newValues = parsed.values.filter((v) => v !== value); + + if (newValues.length === 0) { + queryParams({ del: key, scroll: false }); + } else { + const newParam = parsed.operator.includes("NOT") + ? `-${newValues.join(",")}` + : newValues.join(","); + + queryParams({ + set: { [key]: newParam }, + scroll: false, + }); + } + } + }, + [queryParams, searchParamsObj, parseFilterParam], ); const onRemoveAll = useCallback( @@ -894,21 +965,55 @@ export function useAnalyticsFilters({ ); const onOpenFilter = useCallback( - (key) => - setRequestedFilters((rf) => (rf.includes(key) ? rf : [...rf, key])), + (key) => { + setRequestedFilters((rf) => (rf.includes(key) ? rf : [...rf, key])); + }, [], ); + const onToggleOperator = useCallback( + (key) => { + const currentParam = searchParamsObj[key]; + if (!currentParam) return; + + const isNegated = currentParam.startsWith("-"); + const cleanValue = isNegated ? currentParam.slice(1) : currentParam; + + const newParam = isNegated ? cleanValue : `-${cleanValue}`; + + queryParams({ + set: { [key]: newParam }, + del: "page", + scroll: false, + }); + }, + [searchParamsObj, queryParams], + ); + + const onRemoveFilter = useCallback( + (key) => { + if (key === "link") { + queryParams({ del: ["domain", "key", "url"], scroll: false }); + } else { + queryParams({ del: key, scroll: false }); + } + }, + [queryParams], + ); + const activeFiltersWithStreaming = useMemo( - () => [ - ...activeFilters, - ...(streaming && !activeFilters.length - ? Array.from({ length: 2 }, (_, i) => i).map((i) => ({ + () => { + return [ + ...activeFilters, + ...(streaming && !activeFilters.length + ? Array.from({ length: 2 }, (_, i) => i).map((i) => ({ key: "loader", - value: i, + values: [i], + operator: "IS" as const, })) - : []), - ], + : []), + ]; + }, [activeFilters, streaming], ); @@ -917,8 +1022,10 @@ export function useAnalyticsFilters({ activeFilters, onSelect, onRemove, + onRemoveFilter, onRemoveAll, onOpenFilter, + onToggleOperator, streaming, activeFiltersWithStreaming, }; diff --git a/apps/web/ui/links/use-link-filters.tsx b/apps/web/ui/links/use-link-filters.tsx index 7a982f2d376..3cf63ef9434 100644 --- a/apps/web/ui/links/use-link-filters.tsx +++ b/apps/web/ui/links/use-link-filters.tsx @@ -40,6 +40,7 @@ export function useLinkFilters() { icon: Tag, label: "Tag", multiple: true, + hideOperator: true, shouldFilter: !tagsAsync, getOptionIcon: (value, props) => { const tagColor = @@ -49,6 +50,11 @@ export function useLinkFilters() { ) : null; }, + getOptionLabel: (value, props) => { + if (props.option?.label) return props.option.label; + const tag = tags?.find(({ id }) => id === value); + return tag?.name ?? value; + }, options: tags?.map(({ id, name, color, count, hideDuringSearch }) => ({ value: id, @@ -111,10 +117,10 @@ export function useLinkFilters() { const { domain, tagIds, userId } = searchParamsObj; return [ ...(domain ? [{ key: "domain", value: domain }] : []), - ...(tagIds ? [{ key: "tagIds", value: selectedTagIds }] : []), + ...(tagIds ? [{ key: "tagIds", values: selectedTagIds }] : []), ...(userId ? [{ key: "userId", value: userId }] : []), ]; - }, [searchParamsObj]); + }, [searchParamsObj, selectedTagIds]); const onSelect = (key: string, value: any) => { if (key === "tagIds") { @@ -152,6 +158,12 @@ export function useLinkFilters() { } }; + const onRemoveFilter = (key: string) => { + queryParams({ + del: [key, "page"], + }); + }; + const onRemoveAll = () => { queryParams({ del: ["domain", "tagIds", "userId", "search"], @@ -163,6 +175,7 @@ export function useLinkFilters() { activeFilters, onSelect, onRemove, + onRemoveFilter, onRemoveAll, setSearch, setSelectedFilter, diff --git a/packages/tinybird/pipes/v3_count.pipe b/packages/tinybird/pipes/v3_count.pipe index 442ba99d723..88b06b2dbd3 100644 --- a/packages/tinybird/pipes/v3_count.pipe +++ b/packages/tinybird/pipes/v3_count.pipe @@ -22,18 +22,54 @@ SQL > }} AND deleted == 0 {% if defined(programId) %} AND program_id = {{ programId }} {% end %} - {% if defined(partnerId) %} AND partner_id = {{ partnerId }} {% end %} - {% if defined(groupId) %} AND partner_group_id = {{ groupId }} {% end %} - {% if defined(tenantId) %} AND tenant_id = {{ tenantId }} {% end %} - {% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }} - {% elif defined(folderId) %} AND folder_id = {{ folderId }} + {% if defined(partnerId) %} + {% if defined(partnerIdOperator) and String(partnerIdOperator) == 'NOT IN' %} + AND partner_id NOT IN {{ Array(partnerId, 'String') }} + {% else %} + AND partner_id IN {{ Array(partnerId, 'String') }} + {% end %} + {% end %} + {% if defined(groupId) %} + {% if defined(groupIdOperator) and String(groupIdOperator) == 'NOT IN' %} + AND partner_group_id NOT IN {{ Array(groupId, 'String') }} + {% else %} + AND partner_group_id IN {{ Array(groupId, 'String') }} + {% end %} + {% end %} + {% if defined(tenantId) %} + {% if defined(tenantIdOperator) and String(tenantIdOperator) == 'NOT IN' %} + AND tenant_id NOT IN {{ Array(tenantId, 'String') }} + {% else %} + AND tenant_id IN {{ Array(tenantId, 'String') }} + {% end %} + {% end %} + {% if defined(domain) %} + {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %} + AND domain NOT IN {{ Array(domain, 'String') }} + {% else %} + AND domain IN {{ Array(domain, 'String') }} + {% end %} {% end %} - {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %} {% if defined(tagIds) %} - AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %} + AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids) + {% else %} + AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% end %} + {% end %} + {% if defined(folderId) %} + {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %} + AND folder_id NOT IN {{ Array(folderId, 'String') }} + {% else %} + AND folder_id IN {{ Array(folderId, 'String') }} + {% end %} {% end %} {% if defined(root) %} - {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %} + {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %} + AND NOT (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% else %} + AND (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% end %} {% end %} @@ -78,24 +114,123 @@ SQL > {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled in the sales subquery, not here #} + {% end %} + {% end %} + {% end %} + {% end %} @@ -129,20 +264,19 @@ SQL > {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime(start) }} {% end %} @@ -158,13 +292,108 @@ SQL > {% elif operator == 'notEquals' %} AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }} {% elif operator == 'greaterThan' %} - AND JSONExtractString(metadata, {{ metadataKey }}) > {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) {% elif operator == 'lessThan' %} - AND JSONExtractString(metadata, {{ metadataKey }}) < {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) {% elif operator == 'greaterThanOrEqual' %} - AND JSONExtractString(metadata, {{ metadataKey }}) >= {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) {% elif operator == 'lessThanOrEqual' %} - AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% elif item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled in the sales subquery, not here #} {% end %} {% end %} {% end %} @@ -240,29 +469,21 @@ SQL > {% if defined(browser) %} AND se.browser = {{ browser }} {% end %} {% if defined(os) %} AND se.os = {{ os }} {% end %} {% if defined(referer) %} AND se.referer = {{ referer }} {% end %} - {% if defined(refererUrl) %} - AND splitByString('?', se.referer_url)[1] = {{ refererUrl }} - {% end %} + {% if defined(refererUrl) %} AND splitByString('?', se.referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND se.url - LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND se.url - LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND se.url LIKE concat( - '%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%' - ) + AND se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND se.url - LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND se.url - LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', se.url)[1] = {{ url }} {% end %} {% if defined(start) %} AND se.timestamp >= {{ DateTime(start) }} {% end %} @@ -278,13 +499,108 @@ SQL > {% elif operator == 'notEquals' %} AND JSONExtractString(se.metadata, {{ metadataKey }}) != {{ value }} {% elif operator == 'greaterThan' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) > {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) {% elif operator == 'lessThan' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) < {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) {% elif operator == 'greaterThanOrEqual' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) >= {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) {% elif operator == 'lessThanOrEqual' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) <= {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% elif item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND se.country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND se.city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND se.continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND se.device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND se.browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND se.os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND se.trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND se.referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', se.referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', se.url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND se.link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled after this subquery #} {% end %} {% end %} {% end %} @@ -292,7 +608,17 @@ SQL > ) AS typed WHERE true - {% if defined(saleType) %} AND typed.sale_type = {{ String(saleType) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') == 'saleType' %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if operator == 'IN' %} AND typed.sale_type IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND typed.sale_type NOT IN {{ Array(values, 'String') }} + {% end %} + {% end %} + {% end %} + {% end %} ) AS subquery diff --git a/packages/tinybird/pipes/v3_events.pipe b/packages/tinybird/pipes/v3_events.pipe index caf12c54e1b..36fc609bd8d 100644 --- a/packages/tinybird/pipes/v3_events.pipe +++ b/packages/tinybird/pipes/v3_events.pipe @@ -22,18 +22,54 @@ SQL > }} AND deleted == 0 {% if defined(programId) %} AND program_id = {{ programId }} {% end %} - {% if defined(partnerId) %} AND partner_id = {{ partnerId }} {% end %} - {% if defined(groupId) %} AND partner_group_id = {{ groupId }} {% end %} - {% if defined(tenantId) %} AND tenant_id = {{ tenantId }} {% end %} - {% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }} - {% elif defined(folderId) %} AND folder_id = {{ folderId }} + {% if defined(partnerId) %} + {% if defined(partnerIdOperator) and String(partnerIdOperator) == 'NOT IN' %} + AND partner_id NOT IN {{ Array(partnerId, 'String') }} + {% else %} + AND partner_id IN {{ Array(partnerId, 'String') }} + {% end %} + {% end %} + {% if defined(groupId) %} + {% if defined(groupIdOperator) and String(groupIdOperator) == 'NOT IN' %} + AND partner_group_id NOT IN {{ Array(groupId, 'String') }} + {% else %} + AND partner_group_id IN {{ Array(groupId, 'String') }} + {% end %} + {% end %} + {% if defined(tenantId) %} + {% if defined(tenantIdOperator) and String(tenantIdOperator) == 'NOT IN' %} + AND tenant_id NOT IN {{ Array(tenantId, 'String') }} + {% else %} + AND tenant_id IN {{ Array(tenantId, 'String') }} + {% end %} + {% end %} + {% if defined(domain) %} + {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %} + AND domain NOT IN {{ Array(domain, 'String') }} + {% else %} + AND domain IN {{ Array(domain, 'String') }} + {% end %} {% end %} - {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %} {% if defined(tagIds) %} - AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %} + AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids) + {% else %} + AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% end %} + {% end %} + {% if defined(folderId) %} + {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %} + AND folder_id NOT IN {{ Array(folderId, 'String') }} + {% else %} + AND folder_id IN {{ Array(folderId, 'String') }} + {% end %} {% end %} {% if defined(root) %} - {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %} + {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %} + AND NOT (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% else %} + AND (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% end %} {% end %} @@ -82,24 +118,130 @@ SQL > {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('operand', '').startswith('metadata.') %} + {% set metadataKey = item.get('operand', '').split('.')[1] %} + {% set operator = item.get('operator', 'equals') %} + {% set value = item.get('value', '') %} + {% if operator == 'equals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }} + {% elif operator == 'notEquals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }} + {% elif operator == 'greaterThan' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) + {% elif operator == 'lessThan' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) + {% elif operator == 'greaterThanOrEqual' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) + {% elif operator == 'lessThanOrEqual' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% elif item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {# domain is already filtered at workspace_links node level #} + {% elif field == 'tagIds' %} + {# tagIds is already filtered at workspace_links node level #} + {% elif field == 'folderId' %} + {# folderId is already filtered at workspace_links node level #} + {% elif field == 'root' %} + {# root is already filtered at workspace_links node level #} + {% elif field == 'saleType' %} + {# saleType only applies to sales events #} + {% end %} + {% end %} + {% end %} + {% end %} ORDER BY timestamp {% if order == 'asc' %} ASC {% else %} DESC {% end %} LIMIT {{ Int32(limit, 100) }} {% if defined(offset) %} OFFSET {{ Int32(offset, 0) }} {% end %} @@ -140,20 +282,19 @@ SQL > {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime(start) }} {% end %} @@ -169,13 +310,98 @@ SQL > {% elif operator == 'notEquals' %} AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }} {% elif operator == 'greaterThan' %} - AND JSONExtractString(metadata, {{ metadataKey }}) > {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) {% elif operator == 'lessThan' %} - AND JSONExtractString(metadata, {{ metadataKey }}) < {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) {% elif operator == 'greaterThanOrEqual' %} - AND JSONExtractString(metadata, {{ metadataKey }}) >= {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) {% elif operator == 'lessThanOrEqual' %} - AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% elif item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {# domain is already filtered at workspace_links node level #} + {% elif field == 'tagIds' %} + {# tagIds is already filtered at workspace_links node level #} + {% elif field == 'folderId' %} + {# folderId is already filtered at workspace_links node level #} + {% elif field == 'root' %} + {# root is already filtered at workspace_links node level #} + {% elif field == 'saleType' %} + {# saleType only applies to sales events #} {% end %} {% end %} {% end %} @@ -292,20 +518,117 @@ SQL > {% elif operator == 'notEquals' %} AND JSONExtractString(se.metadata, {{ metadataKey }}) != {{ value }} {% elif operator == 'greaterThan' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) > {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) {% elif operator == 'lessThan' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) < {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) {% elif operator == 'greaterThanOrEqual' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) >= {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) {% elif operator == 'lessThanOrEqual' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) <= {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% elif item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND se.country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND se.city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND se.continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND se.device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND se.browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND se.os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND se.trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND se.referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', se.referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', se.url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {# domain is already filtered at workspace_links node level #} + {% elif field == 'tagIds' %} + {# tagIds is already filtered at workspace_links node level #} + {% elif field == 'folderId' %} + {# folderId is already filtered at workspace_links node level #} + {% elif field == 'root' %} + {# root is already filtered at workspace_links node level #} + {% elif field == 'saleType' %} + {# saleType is filtered in the outer query where the computed sale_type is available #} {% end %} {% end %} {% end %} {% end %} ) AS t - WHERE true {% if defined(saleType) %} AND t.sale_type = {{ String(saleType) }} {% end %} + WHERE true + {% if defined(saleType) %} AND t.sale_type = {{ String(saleType) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') == 'saleType' %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if operator == 'IN' %} AND t.sale_type IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND t.sale_type NOT IN {{ Array(values, 'String') }} + {% end %} + {% end %} + {% end %} + {% end %} ORDER BY t.timestamp {% if order == 'asc' %} ASC {% else %} DESC {% end %} LIMIT {{ Int32(limit, 100) }} {% if defined(offset) %} OFFSET {{ Int32(offset, 0) }} {% end %} diff --git a/packages/tinybird/pipes/v3_group_by.pipe b/packages/tinybird/pipes/v3_group_by.pipe index 3e8926441fa..dae18e1f73e 100644 --- a/packages/tinybird/pipes/v3_group_by.pipe +++ b/packages/tinybird/pipes/v3_group_by.pipe @@ -22,18 +22,54 @@ SQL > }} AND deleted == 0 {% if defined(programId) %} AND program_id = {{ programId }} {% end %} - {% if defined(partnerId) %} AND partner_id = {{ partnerId }} {% end %} - {% if defined(groupId) %} AND partner_group_id = {{ groupId }} {% end %} - {% if defined(tenantId) %} AND tenant_id = {{ tenantId }} {% end %} - {% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }} - {% elif defined(folderId) %} AND folder_id = {{ folderId }} + {% if defined(partnerId) %} + {% if defined(partnerIdOperator) and String(partnerIdOperator) == 'NOT IN' %} + AND partner_id NOT IN {{ Array(partnerId, 'String') }} + {% else %} + AND partner_id IN {{ Array(partnerId, 'String') }} + {% end %} + {% end %} + {% if defined(groupId) %} + {% if defined(groupIdOperator) and String(groupIdOperator) == 'NOT IN' %} + AND partner_group_id NOT IN {{ Array(groupId, 'String') }} + {% else %} + AND partner_group_id IN {{ Array(groupId, 'String') }} + {% end %} + {% end %} + {% if defined(tenantId) %} + {% if defined(tenantIdOperator) and String(tenantIdOperator) == 'NOT IN' %} + AND tenant_id NOT IN {{ Array(tenantId, 'String') }} + {% else %} + AND tenant_id IN {{ Array(tenantId, 'String') }} + {% end %} + {% end %} + {% if defined(domain) %} + {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %} + AND domain NOT IN {{ Array(domain, 'String') }} + {% else %} + AND domain IN {{ Array(domain, 'String') }} + {% end %} {% end %} - {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %} {% if defined(tagIds) %} - AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %} + AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids) + {% else %} + AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% end %} + {% end %} + {% if defined(folderId) %} + {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %} + AND folder_id NOT IN {{ Array(folderId, 'String') }} + {% else %} + AND folder_id IN {{ Array(folderId, 'String') }} + {% end %} {% end %} {% if defined(root) %} - {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %} + {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %} + AND NOT (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% else %} + AND (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% end %} {% end %} @@ -123,24 +159,140 @@ SQL > {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('operand', '').startswith('metadata.') %} + {% set metadataKey = item.get('operand', '').split('.')[1] %} + {% set operator = item.get('operator', 'equals') %} + {% set value = item.get('value', '') %} + {% if operator == 'equals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }} + {% elif operator == 'notEquals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }} + {% elif operator == 'greaterThan' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) + {% elif operator == 'lessThan' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) + {% elif operator == 'greaterThanOrEqual' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) + {% elif operator == 'lessThanOrEqual' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% elif item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled in the sales subquery, not here #} + {% end %} + {% end %} + {% end %} + {% end %} GROUP BY groupByField {% if String(groupBy) == 'cities' %}, country, region @@ -226,20 +378,19 @@ SQL > {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime(start) }} {% end %} @@ -255,13 +406,108 @@ SQL > {% elif operator == 'notEquals' %} AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }} {% elif operator == 'greaterThan' %} - AND JSONExtractString(metadata, {{ metadataKey }}) > {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) {% elif operator == 'lessThan' %} - AND JSONExtractString(metadata, {{ metadataKey }}) < {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) {% elif operator == 'greaterThanOrEqual' %} - AND JSONExtractString(metadata, {{ metadataKey }}) >= {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) {% elif operator == 'lessThanOrEqual' %} - AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% elif item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled in the sales subquery, not here #} {% end %} {% end %} {% end %} @@ -387,20 +633,19 @@ SQL > {% if defined(referer) %} AND se.referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', se.referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND se.url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND se.url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND se.url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND se.url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND se.url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', se.url)[1] = {{ url }} {% end %} {% if defined(start) %} AND se.timestamp >= {{ DateTime(start) }} {% end %} @@ -416,13 +661,108 @@ SQL > {% elif operator == 'notEquals' %} AND JSONExtractString(se.metadata, {{ metadataKey }}) != {{ value }} {% elif operator == 'greaterThan' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) > {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) {% elif operator == 'lessThan' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) < {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) {% elif operator == 'greaterThanOrEqual' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) >= {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) {% elif operator == 'lessThanOrEqual' %} - AND JSONExtractString(se.metadata, {{ metadataKey }}) <= {{ value }} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% elif item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND se.country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND se.city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND se.continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND se.device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND se.browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND se.os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND se.trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND se.referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', se.referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', se.url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND se.link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled after this subquery #} {% end %} {% end %} {% end %} @@ -430,7 +770,17 @@ SQL > ) AS typed WHERE groupByField != '' AND groupByField != 'Unknown' - {% if defined(saleType) %} AND typed.sale_type = {{ String(saleType) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') == 'saleType' %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if operator == 'IN' %} AND typed.sale_type IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND typed.sale_type NOT IN {{ Array(values, 'String') }} + {% end %} + {% end %} + {% end %} + {% end %} GROUP BY groupByField {% if String(groupBy) == 'cities' %}, country, region diff --git a/packages/tinybird/pipes/v3_group_by_link_metadata.pipe b/packages/tinybird/pipes/v3_group_by_link_metadata.pipe index b0f277b3d17..b0a02fdaf9d 100644 --- a/packages/tinybird/pipes/v3_group_by_link_metadata.pipe +++ b/packages/tinybird/pipes/v3_group_by_link_metadata.pipe @@ -46,18 +46,54 @@ SQL > partner_id != '' ) {% if defined(programId) %} AND program_id = {{ programId }} {% end %} - {% if defined(partnerId) %} AND partner_id = {{ partnerId }} {% end %} - {% if defined(groupId) %} AND partner_group_id = {{ groupId }} {% end %} - {% if defined(tenantId) %} AND tenant_id = {{ tenantId }} {% end %} - {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %} - {% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }} - {% elif defined(folderId) %} AND folder_id = {{ folderId }} + {% if defined(partnerId) %} + {% if defined(partnerIdOperator) and String(partnerIdOperator) == 'NOT IN' %} + AND partner_id NOT IN {{ Array(partnerId, 'String') }} + {% else %} + AND partner_id IN {{ Array(partnerId, 'String') }} + {% end %} + {% end %} + {% if defined(groupId) %} + {% if defined(groupIdOperator) and String(groupIdOperator) == 'NOT IN' %} + AND partner_group_id NOT IN {{ Array(groupId, 'String') }} + {% else %} + AND partner_group_id IN {{ Array(groupId, 'String') }} + {% end %} + {% end %} + {% if defined(tenantId) %} + {% if defined(tenantIdOperator) and String(tenantIdOperator) == 'NOT IN' %} + AND tenant_id NOT IN {{ Array(tenantId, 'String') }} + {% else %} + AND tenant_id IN {{ Array(tenantId, 'String') }} + {% end %} + {% end %} + {% if defined(domain) %} + {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %} + AND domain NOT IN {{ Array(domain, 'String') }} + {% else %} + AND domain IN {{ Array(domain, 'String') }} + {% end %} {% end %} {% if defined(tagIds) %} - AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %} + AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids) + {% else %} + AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% end %} + {% end %} + {% if defined(folderId) %} + {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %} + AND folder_id NOT IN {{ Array(folderId, 'String') }} + {% else %} + AND folder_id IN {{ Array(folderId, 'String') }} + {% end %} {% end %} {% if defined(root) %} - {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %} + {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %} + AND NOT (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% else %} + AND (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% end %} {% end %} @@ -82,35 +118,99 @@ SQL > {% if defined(linkIds) %} AND ce.link_id IN {{ Array(linkIds, 'String') }} {% elif defined(linkId) %} AND ce.link_id = {{ linkId }} {% end %} - {% if defined(continent) %} AND continent = {{ continent }} {% end %} - {% if defined(country) %} AND country = {{ country }} {% end %} {% if defined(region) %} AND region = {{ region }} {% end %} - {% if defined(city) %} AND city = {{ city }} {% end %} - {% if defined(device) %} AND device = {{ device }} {% end %} - {% if defined(browser) %} AND browser = {{ browser }} {% end %} - {% if defined(os) %} AND os = {{ os }} {% end %} - {% if defined(trigger) %} AND trigger = {{ trigger }} {% end %} - {% if defined(referer) %} AND referer = {{ referer }} {% end %} - {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} - {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') - {% end %} - {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') - {% end %} - {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') - {% end %} - {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') - {% end %} - {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') - {% end %} - {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {# domain is already filtered at workspace_links node level #} + {% elif field == 'tagIds' %} + {# tagIds is already filtered at workspace_links node level #} + {% elif field == 'folderId' %} + {# folderId is already filtered at workspace_links node level #} + {% elif field == 'root' %} + {# root is already filtered at workspace_links node level #} + {% elif field == 'saleType' %} + {# saleType is handled in the sales subquery, not here #} + {% end %} + {% end %} + {% end %} + {% end %} GROUP BY wl.groupByField ORDER BY clicks DESC LIMIT 5000 @@ -145,28 +245,126 @@ SQL > {% if defined(device) %} AND device = {{ device }} {% end %} {% if defined(browser) %} AND browser = {{ browser }} {% end %} {% if defined(os) %} AND os = {{ os }} {% end %} - {% if defined(trigger) %} AND trigger = {{ trigger }} {% end %} {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND le.link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND le.link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND le.link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND le.link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND le.link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND le.link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND le.link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND le.link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled in the sales subquery, not here #} + {% end %} + {% end %} + {% end %} + {% end %} GROUP BY wl.groupByField ORDER BY leads DESC LIMIT 5000 @@ -238,34 +436,142 @@ SQL > {% if defined(device) %} AND se.device = {{ device }} {% end %} {% if defined(browser) %} AND se.browser = {{ browser }} {% end %} {% if defined(os) %} AND se.os = {{ os }} {% end %} - {% if defined(trigger) %} AND se.trigger = {{ trigger }} {% end %} {% if defined(referer) %} AND se.referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', se.referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND se.url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND se.url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND se.url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND se.url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND se.url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', se.url)[1] = {{ url }} {% end %} {% if defined(start) %} AND se.timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND se.timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND se.country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND se.city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND se.continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND se.device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND se.browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND se.os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND se.trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND se.referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', se.referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', se.url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND se.link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled after this subquery #} + {% end %} + {% end %} + {% end %} + {% end %} ) AS t JOIN workspace_links AS wl ON t.link_id = wl.link_id WHERE true - {% if defined(saleType) %} AND t.sale_type = {{ String(saleType) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') == 'saleType' %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if operator == 'IN' %} AND t.sale_type IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND t.sale_type NOT IN {{ Array(values, 'String') }} + {% end %} + {% end %} + {% end %} + {% end %} GROUP BY wl.groupByField ORDER BY saleAmount DESC LIMIT 5000 diff --git a/packages/tinybird/pipes/v3_timeseries.pipe b/packages/tinybird/pipes/v3_timeseries.pipe index fb290381d1d..3557e784dc6 100644 --- a/packages/tinybird/pipes/v3_timeseries.pipe +++ b/packages/tinybird/pipes/v3_timeseries.pipe @@ -95,18 +95,54 @@ SQL > }} AND deleted == 0 {% if defined(programId) %} AND program_id = {{ programId }} {% end %} - {% if defined(partnerId) %} AND partner_id = {{ partnerId }} {% end %} - {% if defined(groupId) %} AND partner_group_id = {{ groupId }} {% end %} - {% if defined(tenantId) %} AND tenant_id = {{ tenantId }} {% end %} - {% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }} - {% elif defined(folderId) %} AND folder_id = {{ folderId }} + {% if defined(partnerId) %} + {% if defined(partnerIdOperator) and String(partnerIdOperator) == 'NOT IN' %} + AND partner_id NOT IN {{ Array(partnerId, 'String') }} + {% else %} + AND partner_id IN {{ Array(partnerId, 'String') }} + {% end %} + {% end %} + {% if defined(groupId) %} + {% if defined(groupIdOperator) and String(groupIdOperator) == 'NOT IN' %} + AND partner_group_id NOT IN {{ Array(groupId, 'String') }} + {% else %} + AND partner_group_id IN {{ Array(groupId, 'String') }} + {% end %} + {% end %} + {% if defined(tenantId) %} + {% if defined(tenantIdOperator) and String(tenantIdOperator) == 'NOT IN' %} + AND tenant_id NOT IN {{ Array(tenantId, 'String') }} + {% else %} + AND tenant_id IN {{ Array(tenantId, 'String') }} + {% end %} + {% end %} + {% if defined(domain) %} + {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %} + AND domain NOT IN {{ Array(domain, 'String') }} + {% else %} + AND domain IN {{ Array(domain, 'String') }} + {% end %} {% end %} - {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %} {% if defined(tagIds) %} - AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %} + AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids) + {% else %} + AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != [] + {% end %} + {% end %} + {% if defined(folderId) %} + {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %} + AND folder_id NOT IN {{ Array(folderId, 'String') }} + {% else %} + AND folder_id IN {{ Array(folderId, 'String') }} + {% end %} {% end %} {% if defined(root) %} - {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %} + {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %} + AND NOT (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% else %} + AND (0{% for val in root %}{% if val == 'true' %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %}) + {% end %} {% end %} @@ -150,35 +186,109 @@ SQL > root ) %} AND link_id in (SELECT link_id from workspace_links) {% end %} - {% if defined(continent) %} AND continent = {{ continent }} {% end %} - {% if defined(country) %} AND country = {{ country }} {% end %} {% if defined(region) %} AND region = {{ region }} {% end %} - {% if defined(city) %} AND city = {{ city }} {% end %} - {% if defined(device) %} AND device = {{ device }} {% end %} - {% if defined(browser) %} AND browser = {{ browser }} {% end %} - {% if defined(os) %} AND os = {{ os }} {% end %} - {% if defined(trigger) %} AND trigger = {{ trigger }} {% end %} - {% if defined(referer) %} AND referer = {{ referer }} {% end %} - {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} - {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') - {% end %} - {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') - {% end %} - {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') - {% end %} - {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') - {% end %} - {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') - {% end %} - {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled in the sales subquery, not here #} + {% end %} + {% end %} + {% end %} + {% end %} GROUP BY interval ORDER BY interval @@ -239,24 +349,140 @@ SQL > {% if defined(referer) %} AND referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %} {% if defined(start) %} AND timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled in the sales subquery, not here #} + {% end %} + {% elif item.get('operand', '').startswith('metadata.') %} + {% set metadataKey = item.get('operand', '').split('.')[1] %} + {% set operator = item.get('operator', 'equals') %} + {% set value = item.get('value', '') %} + {% if operator == 'equals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }} + {% elif operator == 'notEquals' %} + AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }} + {% elif operator == 'greaterThan' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) + {% elif operator == 'lessThan' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) + {% elif operator == 'greaterThanOrEqual' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) + {% elif operator == 'lessThanOrEqual' %} + AND toFloat64OrNull(JSONExtractString(metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% end %} + {% end %} + {% end %} GROUP BY interval ORDER BY interval @@ -370,28 +596,154 @@ SQL > {% if defined(referer) %} AND se.referer = {{ referer }} {% end %} {% if defined(refererUrl) %} AND splitByString('?', se.referer_url)[1] = {{ refererUrl }} {% end %} {% if defined(utm_source) %} - AND se.url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') + AND se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%') {% end %} {% if defined(utm_medium) %} - AND se.url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') + AND se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%') {% end %} {% if defined(utm_campaign) %} - AND se.url - LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') + AND se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%') {% end %} {% if defined(utm_term) %} - AND se.url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') + AND se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%') {% end %} {% if defined(utm_content) %} - AND se.url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') + AND se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%') {% end %} {% if defined(url) %} AND splitByString('?', se.url)[1] = {{ url }} {% end %} {% if defined(start) %} AND se.timestamp >= {{ DateTime64(start) }} {% end %} {% if defined(end) %} AND se.timestamp <= {{ DateTime64(end) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') %} + {% set field = item.get('field', '') %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if field == 'country' %} + {% if operator == 'IN' %} AND se.country IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.country NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'city' %} + {% if operator == 'IN' %} AND se.city IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.city NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'continent' %} + {% if operator == 'IN' %} AND se.continent IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.continent NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'device' %} + {% if operator == 'IN' %} AND se.device IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.device NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'browser' %} + {% if operator == 'IN' %} AND se.browser IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.browser NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'os' %} + {% if operator == 'IN' %} AND se.os IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.os NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'trigger' %} + {% if operator == 'IN' %} AND se.trigger IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.trigger NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'referer' %} + {% if operator == 'IN' %} AND se.referer IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND se.referer NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'refererUrl' %} + {% if operator == 'IN' %} AND splitByString('?', se.referer_url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.referer_url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'url' %} + {% if operator == 'IN' %} AND splitByString('?', se.url)[1] IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND splitByString('?', se.url)[1] NOT IN {{ Array(values, 'String') }} + {% end %} + {% elif field == 'utm_source' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_medium' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_campaign' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_term' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'utm_content' %} + {% if operator == 'IN' %} + AND (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% elif operator == 'NOT IN' %} + AND NOT (0{% for val in values %} OR se.url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %}) + {% end %} + {% elif field == 'domain' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'tagIds' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != []) + {% end %} + {% elif field == 'folderId' %} + {% if operator == 'IN' %} AND se.link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% elif operator == 'NOT IN' %} AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }}) + {% end %} + {% elif field == 'root' %} + {% if operator == 'IN' %} + AND se.link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% elif operator == 'NOT IN' %} + AND se.link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %}) + {% end %} + {% elif field == 'saleType' %} + {# saleType is handled after this subquery #} + {% end %} + {% elif item.get('operand', '').startswith('metadata.') %} + {% set metadataKey = item.get('operand', '').split('.')[1] %} + {% set operator = item.get('operator', 'equals') %} + {% set value = item.get('value', '') %} + {% if operator == 'equals' %} + AND JSONExtractString(se.metadata, {{ metadataKey }}) = {{ value }} + {% elif operator == 'notEquals' %} + AND JSONExtractString(se.metadata, {{ metadataKey }}) != {{ value }} + {% elif operator == 'greaterThan' %} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) > toFloat64OrNull({{ value }}) + {% elif operator == 'lessThan' %} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) < toFloat64OrNull({{ value }}) + {% elif operator == 'greaterThanOrEqual' %} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) >= toFloat64OrNull({{ value }}) + {% elif operator == 'lessThanOrEqual' %} + AND toFloat64OrNull(JSONExtractString(se.metadata, {{ metadataKey }})) <= toFloat64OrNull({{ value }}) + {% end %} + {% end %} + {% end %} + {% end %} ) AS typed WHERE true - {% if defined(saleType) %} AND typed.sale_type = {{ String(saleType) }} {% end %} + {% if defined(filters) %} + {% for item in JSON(filters, '[]') %} + {% if item.get('field', '') == 'saleType' %} + {% set operator = item.get('operator', 'IN') %} + {% set values = item.get('values', []) %} + {% if operator == 'IN' %} AND typed.sale_type IN {{ Array(values, 'String') }} + {% elif operator == 'NOT IN' %} AND typed.sale_type NOT IN {{ Array(values, 'String') }} + {% end %} + {% end %} + {% end %} + {% end %} GROUP BY interval ORDER BY interval diff --git a/packages/ui/src/filter/filter-list.tsx b/packages/ui/src/filter/filter-list.tsx index 91a3e421df6..1ee95f93953 100644 --- a/packages/ui/src/filter/filter-list.tsx +++ b/packages/ui/src/filter/filter-list.tsx @@ -1,35 +1,79 @@ import { cn, truncate } from "@dub/utils"; +import { Command } from "cmdk"; import { X } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; -import Link from "next/link"; -import { ReactNode, isValidElement, useState } from "react"; +import { ReactNode, isValidElement, useState, useCallback } from "react"; import { AnimatedSizeContainer } from "../animated-size-container"; import { Combobox, ComboboxOption } from "../combobox"; import { useKeyboardShortcut } from "../hooks"; -import { Icon } from "../icons"; -import { Filter, FilterOption } from "./types"; +import { Check, Icon } from "../icons"; +import { Popover } from "../popover"; +import { Tooltip } from "../tooltip"; +import { Filter, FilterOption, ActiveFilterInput, FilterOperator, normalizeActiveFilter } from "./types"; type FilterListProps = { filters: Filter[]; - activeFilters?: { - key: Filter["key"]; - value: FilterOption["value"]; - }[]; + activeFilters?: ActiveFilterInput[]; onRemove: (key: string, value: FilterOption["value"]) => void; + onRemoveFilter?: (key: string) => void; onRemoveAll: () => void; - onSelect?: (key: string, value: FilterOption["value"]) => void; + onSelect?: (key: string, value: FilterOption["value"] | FilterOption["value"][]) => void; + onToggleOperator?: (key: string) => void; + isAdvancedFilter?: boolean; className?: string; }; +function getOperatorLabel(operator: FilterOperator): string { + switch (operator) { + case "IS": + case "IS_ONE_OF": + return "is"; + + case "IS_NOT": + case "IS_NOT_ONE_OF": + return "is not"; + + default: + return "is"; + } +} + +function pluralize(word: string, count: number): string { + if (count === 1) return word.toLowerCase(); + + const irregularPlurals: Record = { + 'country': 'countries', + 'city': 'cities', + 'category': 'categories', + 'os': 'OS', + }; + + const lowerWord = word.toLowerCase(); + if (irregularPlurals[lowerWord]) { + return irregularPlurals[lowerWord]; + } + + if (lowerWord.endsWith('y') && !['ay', 'ey', 'iy', 'oy', 'uy'].some(ending => lowerWord.endsWith(ending))) { + return lowerWord.slice(0, -1) + 'ies'; + } + + return lowerWord + 's'; +} + export function FilterList({ filters, activeFilters, onRemove, + onRemoveFilter, onRemoveAll, onSelect, + onToggleOperator, + isAdvancedFilter = false, className, }: FilterListProps) { useKeyboardShortcut("Escape", onRemoveAll, { priority: 1 }); + const normalizedFilters = activeFilters?.map(normalizeActiveFilter) ?? []; + return (
- {activeFilters?.map(({ key, value: filterValue }) => { + {normalizedFilters.map(({ key, values, operator }) => { if (key === "loader") { return ( { - const option = filter.options?.find((o) => - typeof o.value === "string" && typeof value === "string" - ? o.value.toLowerCase() === value.toLowerCase() - : o.value === value, - ); + const isSingleValue = values.length === 1; + const displayValues = values.slice(0, 3); - const OptionIcon = - option?.icon ?? - filter.getOptionIcon?.(value, { - key: filter.key, - option, - }) ?? - filter.icon; - - const optionLabel = - option?.label ?? - filter.getOptionLabel?.(value, { key: filter.key, option }) ?? - value; - - const optionPermalink = - option?.permalink ?? - filter.getOptionPermalink?.(value) ?? - null; - - const OptionDisplay = ({ - className, - }: { - className?: string; - }) => ( -
+ const displayLabel = isSingleValue + ? (() => { + const value = values[0]; + const option = filter.options?.find((o) => + typeof o.value === "string" && typeof value === "string" + ? o.value.toLowerCase() === value.toLowerCase() + : o.value === value, + ); + return option?.label ?? + filter.getOptionLabel?.(value, { key: filter.key, option }) ?? + String(value); + })() + : `${values.length} ${pluralize(filter.label, values.length)}`; + + const showMultipleIcons = ["country", "city"].includes(key); + + const OptionDisplay = ({ + className, + }: { + className?: string; + }) => { + let iconDisplay; + + if (isSingleValue) { + const value = values[0]; + const option = filter.options?.find((o) => + typeof o.value === "string" && typeof value === "string" + ? o.value.toLowerCase() === value.toLowerCase() + : o.value === value, + ); + + const OptionIcon = + option?.icon ?? + filter.getOptionIcon?.(value, { + key: filter.key, + option, + }) ?? + filter.icon; + + iconDisplay = ( {isReactNode(OptionIcon) ? ( OptionIcon @@ -111,120 +158,93 @@ export function FilterList({ )} - {optionPermalink ? ( - - {truncate(optionLabel, 30)} - - ) : ( - truncate(optionLabel, 30) - )} -
- ); + ); + } else if (showMultipleIcons) { + iconDisplay = ( +
+ {displayValues.map((value, idx) => { + const option = filter.options?.find((o) => + typeof o.value === "string" && typeof value === "string" + ? o.value.toLowerCase() === value.toLowerCase() + : o.value === value, + ); - return ( - - {/* Filter */} -
- - {isReactNode(filter.icon) ? ( - filter.icon - ) : ( - - )} - - {filter.label} -
+ const OptionIcon = + option?.icon ?? + filter.getOptionIcon?.(value, { + key: filter.key, + option, + }) ?? + filter.icon; - {/* is */} -
is
- - {/* Option */} -
- {!filter.options ? ( -
-
-
- ) : // show the filter list item dropdown if there's onSelect configured - // and the filter is not hidden in the main filter dropdown itself - onSelect && !filter.hideInFilterDropdown ? ( - (() => { - // Precompute options array once - const options: ComboboxOption[] = - filter.options?.map((opt): ComboboxOption => { - const optionIcon = - opt.icon ?? - filter.getOptionIcon?.(opt.value, { - key: filter.key, - option: opt, - }) ?? - filter.icon; - - return { - label: - opt.label ?? - filter.getOptionLabel?.(opt.value, { - key: filter.key, - option: opt, - }) ?? - String(opt.value), - value: String(opt.value), - icon: optionIcon, - }; - }) ?? []; - - // Find selected option from precomputed array - const selectedOption = options.find((opt) => - typeof opt.value === "string" && - typeof value === "string" - ? opt.value.toLowerCase() === - String(value).toLowerCase() - : opt.value === String(value), - ); - - return ( - - ); - })() + return ( + + {isReactNode(OptionIcon) ? ( + OptionIcon + ) : ( + + )} + + ); + })} +
+ ); + } else { + iconDisplay = ( + + {isReactNode(filter.icon) ? ( + filter.icon ) : ( - OptionDisplay({}) + )} -
+ + ); + } - {/* Remove */} - - + return ( +
+ {values.length > 3 && showMultipleIcons ? ( + + {iconDisplay} + + ) : ( + iconDisplay + )} + {truncate(displayLabel, 30)} +
); - }); + }; + + return ( + + ); })}
- {activeFilters?.length !== 0 && ( + {normalizedFilters.length !== 0 && ( + +
+ } + align="center" + > + + + ) : ( +
+ is +
+ )} + + { + setValueDropdownOpen(open); + if (!open) { + setSearch(""); + setInitialSelectedValues(new Set()); + } + }} + content={ +
+ + +
+ +
+
+ + {(() => { + const filteredOptions = filter.options?.filter(option => { + if (!search) return true; + const optionLabel = ( + option.label ?? + filter.getOptionLabel?.(option.value, { key: filter.key, option }) ?? + String(option.value) + ).toLowerCase(); + return optionLabel.includes(search.toLowerCase()); + }) ?? []; + + const selectedOptions = filteredOptions.filter(option => + initialSelectedValues.has(option.value) + ); + const unselectedOptions = filteredOptions.filter(option => + !initialSelectedValues.has(option.value) + ); + + const renderOption = (option: FilterOption) => { + const isSelected = values.includes(option.value); + const OptionIcon = + option.icon ?? + filter.getOptionIcon?.(option.value, { + key: filter.key, + option, + }) ?? + filter.icon; + + const optionLabel = + option.label ?? + filter.getOptionLabel?.(option.value, { key: filter.key, option }) ?? + String(option.value); + + return ( + { + toggleValue(option.value); + }} + onPointerDown={(e) => { + e.preventDefault(); + }} + value={optionLabel + option.value} + > + {(isAdvancedFilter || filter.multiple) && ( +
+ {isSelected && } +
+ )} + + {isReactNode(OptionIcon) ? OptionIcon : } + + {truncate(optionLabel, 48)} +
+ {(isAdvancedFilter || filter.multiple) ? ( + option.right + ) : ( + isSelected ? ( + + ) : ( + option.right + ) + )} +
+ + ); + }; + + return ( + <> + {selectedOptions.map(renderOption)} + + {(isAdvancedFilter || filter.multiple) && selectedOptions.length > 0 && unselectedOptions.length > 0 && ( + + )} + + {unselectedOptions.map(renderOption)} + + ); + })()} + +
+ + +
+ } + align="start" + > + + + + + + ); +} + const isReactNode = (element: any): element is ReactNode => isValidElement(element); diff --git a/packages/ui/src/filter/filter-select.tsx b/packages/ui/src/filter/filter-select.tsx index 9837cf26e01..299175e089d 100644 --- a/packages/ui/src/filter/filter-select.tsx +++ b/packages/ui/src/filter/filter-select.tsx @@ -18,20 +18,18 @@ import { useKeyboardShortcut, useMediaQuery } from "../hooks"; import { useScrollProgress } from "../hooks/use-scroll-progress"; import { Check, LoadingSpinner, Magic } from "../icons"; import { Popover } from "../popover"; -import { Filter, FilterOption } from "./types"; +import { Filter, FilterOption, ActiveFilterInput, normalizeActiveFilter } from "./types"; type FilterSelectProps = { filters: Filter[]; - onSelect: (key: string, value: FilterOption["value"]) => void; + onSelect: (key: string, value: FilterOption["value"] | FilterOption["value"][]) => void; onRemove: (key: string, value: FilterOption["value"]) => void; onOpenFilter?: (key: string) => void; onSearchChange?: (search: string) => void; onSelectedFilterChange?: (key: string | null) => void; - activeFilters?: { - key: Filter["key"]; - value: FilterOption["value"]; - }[]; + activeFilters?: ActiveFilterInput[]; askAI?: boolean; + isAdvancedFilter?: boolean; children?: ReactNode; emptyState?: ReactNode | Record; className?: string; @@ -46,6 +44,7 @@ export function FilterSelect({ onSelectedFilterChange, activeFilters, askAI, + isAdvancedFilter = false, children, emptyState, className, @@ -96,43 +95,49 @@ export function FilterSelect({ setSearch(""); setSelectedFilterKey(key); + onOpenFilter?.(key); - }, []); + }, [onOpenFilter]); const isOptionSelected = useCallback( (value: FilterOption["value"]) => { if (!selectedFilter || !activeFilters) return false; - const activeFilter = activeFilters.find( - ({ key }) => key === selectedFilterKey, - ); - - return ( - activeFilter?.value === value || - (activeFilter && - selectedFilter.multiple && - Array.isArray(activeFilter.value) && - activeFilter.value.includes(value)) - ); + const rawActiveFilter = activeFilters.find((filter) => filter.key === selectedFilterKey); + if (!rawActiveFilter) return false; + + const normalizedFilter = normalizeActiveFilter(rawActiveFilter); + return normalizedFilter.values.includes(value); }, - [selectedFilter], + [selectedFilter, activeFilters, selectedFilterKey], ); + const selectOption = useCallback( (value: FilterOption["value"]) => { if (selectedFilter) { - const isSelected = isOptionSelected(value); - - isSelected - ? onRemove(selectedFilter.key, value) - : onSelect(selectedFilter.key, value); - - if (!selectedFilter.multiple) setIsOpen(false); + const isSingleSelect = selectedFilter?.singleSelect || (!isAdvancedFilter && !selectedFilter?.multiple); + + if (isSingleSelect) { + const isSelected = isOptionSelected(value); + isSelected + ? onRemove(selectedFilter.key, value) + : onSelect(selectedFilter.key, value); + setIsOpen(false); + } else { + const isSelected = isOptionSelected(value); + if (isSelected) { + onRemove(selectedFilter.key, value); + } else { + onSelect(selectedFilter.key, value); + } + } } }, - [activeFilters, selectedFilter, askAI], + [selectedFilter, isOptionSelected, onSelect, onRemove, isAdvancedFilter], ); + useEffect(() => { onSearchChange?.(search); }, [search]); @@ -236,6 +241,7 @@ export function FilterSelect({ selectedFilter.options ?.filter((option) => !search || !option.hideDuringSearch) ?.map((option) => { + const isSingleSelect = selectedFilter?.singleSelect || (!isAdvancedFilter && !selectedFilter?.multiple); const isSelected = isOptionSelected(option.value); return ( @@ -243,9 +249,15 @@ export function FilterSelect({ key={option.value} filter={selectedFilter} option={option} + showCheckbox={!isSingleSelect && (isAdvancedFilter || selectedFilter?.multiple)} + isChecked={isSelected} right={ - isSelected ? ( - + isSingleSelect ? ( + isSelected ? ( + + ) : ( + option.right + ) ) : ( option.right ) @@ -378,11 +390,15 @@ function FilterButton({ filter, option, right, + showCheckbox, + isChecked, onSelect, }: { filter: Filter; option?: FilterOption; right?: ReactNode; + showCheckbox?: boolean; + isChecked?: boolean; onSelect: () => void; }) { const { isMobile } = useMediaQuery(); @@ -416,11 +432,19 @@ function FilterButton({ onSelect={onSelect} value={label + option?.value} > + {showCheckbox && ( +
+ {isChecked && } +
+ )} {isReactNode(Icon) ? Icon : } - {truncate(label, 48)} -
+ {truncate(label, 48)} +
{right}
diff --git a/packages/ui/src/filter/index.ts b/packages/ui/src/filter/index.ts index d407cd63c9d..5182f46c56a 100644 --- a/packages/ui/src/filter/index.ts +++ b/packages/ui/src/filter/index.ts @@ -4,3 +4,13 @@ import { FilterSelect } from "./filter-select"; const Filter = { Select: FilterSelect, List: FilterList }; export { Filter }; +export type { + FilterOperator, + ActiveFilter, + ActiveFilterInput, + LegacyActiveFilterSingular, + LegacyActiveFilterPlural, + Filter as FilterType, + FilterOption +} from "./types"; +export { normalizeActiveFilter } from "./types"; diff --git a/packages/ui/src/filter/types.ts b/packages/ui/src/filter/types.ts index fce15750ba5..d9dd7515749 100644 --- a/packages/ui/src/filter/types.ts +++ b/packages/ui/src/filter/types.ts @@ -1,3 +1,4 @@ +import { type FilterOperator } from "@dub/utils"; import { LucideIcon } from "lucide-react"; import { ComponentType, ReactNode, SVGProps } from "react"; @@ -6,6 +7,8 @@ type FilterIcon = | ReactNode | ComponentType>; +export type { FilterOperator }; + export type Filter = { key: string; icon: FilterIcon; @@ -15,6 +18,8 @@ export type Filter = { shouldFilter?: boolean; separatorAfter?: boolean; multiple?: boolean; + singleSelect?: boolean; // Force single-select behavior even if multiSelect is enabled globally + hideOperator?: boolean; // Hide the operator dropdown (is/is not) even when multiple is enabled getOptionIcon?: ( value: FilterOption["value"], props: { key: Filter["key"]; option?: FilterOption }, @@ -35,3 +40,60 @@ export type FilterOption = { data?: Record; permalink?: string; }; + +export type ActiveFilter = { + key: Filter["key"]; + values: FilterOption["value"][]; + operator: FilterOperator; +}; + +export type LegacyActiveFilterSingular = { + key: Filter["key"]; + value: FilterOption["value"]; +}; + +export type LegacyActiveFilterPlural = { + key: Filter["key"]; + values: FilterOption["value"][]; +}; + +export type ActiveFilterInput = + | ActiveFilter + | LegacyActiveFilterSingular + | LegacyActiveFilterPlural; + +/** + * Normalize active filter to the new format with operator support + * Handles backward compatibility with legacy formats: + * - { key, value } → { key, values: [value], operator: 'IS' } + * - { key, values } → { key, values, operator: 'IS' or 'IS_ONE_OF' } + * - { key, values, operator } → unchanged (already correct) + */ +export function normalizeActiveFilter(filter: ActiveFilterInput): ActiveFilter { + if ('operator' in filter && filter.operator && Array.isArray(filter.values)) { + return filter as ActiveFilter; + } + + if ('value' in filter && !('values' in filter)) { + return { + key: filter.key, + operator: 'IS' as FilterOperator, + values: [filter.value], + }; + } + + if (Array.isArray((filter as any).values) && (!('operator' in filter) || !filter.operator)) { + const values = (filter as LegacyActiveFilterPlural).values; + return { + key: filter.key, + operator: values.length > 1 ? 'IS_ONE_OF' : 'IS', + values: values, + }; + } + + return { + key: filter.key, + operator: 'IS', + values: [], + }; +} diff --git a/packages/utils/src/functions/index.ts b/packages/utils/src/functions/index.ts index 73deb553e90..f2a577251aa 100644 --- a/packages/utils/src/functions/index.ts +++ b/packages/utils/src/functions/index.ts @@ -24,6 +24,7 @@ export * from "./log"; export * from "./nanoid"; export * from "./nformatter"; export * from "./normalize-string"; +export * from "./parse-filter-value"; export * from "./pick"; export * from "./pluralize"; export * from "./pretty-print"; diff --git a/packages/utils/src/functions/parse-filter-value.ts b/packages/utils/src/functions/parse-filter-value.ts new file mode 100644 index 00000000000..9aea1826795 --- /dev/null +++ b/packages/utils/src/functions/parse-filter-value.ts @@ -0,0 +1,66 @@ +export type FilterOperator = "IS" | "IS_NOT" | "IS_ONE_OF" | "IS_NOT_ONE_OF"; +export type SQLOperator = "IN" | "NOT IN"; + +export interface ParsedFilter { + operator: FilterOperator; + sqlOperator: SQLOperator; + values: string[]; +} + +/** + * Parse filter value from URL format to structured filter + * + * Formats supported: + * - "US" → IS, values: ["US"], SQL: IN + * - "US,BR,FR" → IS_ONE_OF, values: ["US", "BR", "FR"], SQL: IN + * - "-US" → IS_NOT, values: ["US"], SQL: NOT IN + * - "-US,BR" → IS_NOT_ONE_OF, values: ["US", "BR"], SQL: NOT IN + * + * Note: All filters now use IN/NOT IN operators for consistency, + * even for single values. This simplifies SQL query generation. + * + * @param value - The filter value string (can include "-" prefix for negation) + * @returns Parsed filter with operator and values array + */ +export function parseFilterValue( + value: string | string[] | undefined, +): ParsedFilter | undefined { + if (!value) return undefined; + + if (Array.isArray(value)) { + return { + operator: value.length > 1 ? "IS_ONE_OF" : "IS", + sqlOperator: "IN", + values: value, + }; + } + + const isNegated = value.startsWith("-"); + const cleanValue = isNegated ? value.slice(1) : value; + const values = cleanValue.split(",").filter(Boolean); + + if (values.length === 0) return undefined; + + const operator: FilterOperator = isNegated + ? values.length > 1 + ? "IS_NOT_ONE_OF" + : "IS_NOT" + : values.length > 1 + ? "IS_ONE_OF" + : "IS"; + + const sqlOperator: SQLOperator = isNegated ? "NOT IN" : "IN"; + + return { operator, sqlOperator, values }; +} + +/** + * Build filter value string from parsed filter + * + * @param parsed - The parsed filter object + * @returns URL-formatted filter string + */ +export function buildFilterValue(parsed: ParsedFilter): string { + const joined = parsed.values.join(","); + return parsed.operator.includes("NOT") ? `-${joined}` : joined; +}