Tinybird performance and quarantined rows improvements, backfill performance updates (#2942)

Author: epipav

backend/src/database/migrations/U1742295164__addIndexesForSequinBackfills.sql

@@ -0,0 +1,3 @@
+create index if not exists "ix_activityRelations_updatedAt_activityId" on public."activityRelations" ("updatedAt", "activityId");
+create index if not exists "ix_members_updatedAt_id" on public.members ("updatedAt", id);
+create index if not exists "ix_organizations_updatedAt_id" on public.organizations ("updatedAt", id);

backend/src/database/migrations/V1742295164__addIndexesForSequinBackfills.sql

@@ -727,7 +727,14 @@ function post_up_scaffold() {
 }
 
 function down_scaffold() {
-    $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml down
+    local profile=''
+    if [[ -n "${WITH_NGINX}" ]]; then
+        profile='--profile nginx'
+    elif [[ -n "${WITH_INSIGHTS}" ]]; then
+        profile='--profile insights'
+    fi
+    
+    $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml ${profile} down
 }
 
 function scaffold_destroy() {
@@ -790,8 +797,15 @@ function service_start() {
 }
 
 function scaffold_destroy_confirmed() {
+    local profile=''
+    if [[ -n "${WITH_NGINX}" ]]; then
+        profile='--profile nginx'
+    elif [[ -n "${WITH_INSIGHTS}" ]]; then
+        profile='--profile insights'
+    fi
+
     kill_all_containers
-    $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml down
+    $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml ${profile} down
 
     # needed because if there are no volumes this might cause the script to exit
     set +eo pipefail

scripts/cli

@@ -75,12 +75,12 @@ export async function sendActivitiesToTinybird(activitiesRedisKey: string): Prom
     console.log(`Data sent to tinybird ->  ${JSON.stringify(response)}`)
   } catch (err) {
     if (axios.isAxiosError(err)) {
-      this.log.warn(
+      svc.log.warn(
         `Axios error occurred while sending activities to tinybird. ${err.response?.status} - ${err.response?.statusText}`,
       )
       throw new Error(`Sending data to tinybird failed with status: ${err.response?.status}`)
     } else {
-      this.log.error(`Unexpected error while sending data to tinybird: ${err}`)
+      svc.log.error(`Unexpected error while sending data to tinybird: ${err}`)
       throw new Error('An unexpected error occurred')
     }
   }

services/apps/script_executor_worker/src/activities/copy-activities-from-questdb-to-tinybird/index.ts

@@ -25,15 +25,9 @@ export const scheduleCopyActivitiesFromQuestdbToTinybird = async () => {
         },
         args: [
           {
-            batchSizePerRun: 50000,
+            batchSizePerRun: 5000,
             deleteIndexedEntities: false,
-            segmentIds: [
-              '7b7bf9ce-6051-4c97-85eb-35c790203a0c',
-              'f809d51c-68fa-401a-8aea-eb67989285c1',
-              'e2fa0373-7379-416f-a651-70b1fc1b89b9',
-              'e2c3321f-0d85-4a16-b603-66fd9f882a06',
-              '2cd61b6e-de33-4a51-bb17-fbe33cc0ac0c',
-            ],
+            segmentIds: process.env['CROWD_COPY_ACTIVITIES_TO_TINYBIRD_SEGMENT_IDS'].split(','),
           },
         ],
       },

services/apps/script_executor_worker/src/schedules/scheduleCopyActivitiesFromQuestdbToTinybird.ts

@@ -48,5 +48,6 @@ export async function copyActivitiesFromQuestdbToTinybird(
   await continueAsNew<typeof copyActivitiesFromQuestdbToTinybird>({
     batchSizePerRun: args.batchSizePerRun,
     latestSyncedActivityTimestamp: lastTimestamp,
+    segmentIds: args.segmentIds,
   })
 }

services/apps/script_executor_worker/src/workflows/copyActivitiesFromQuestdbToTinybird.ts

@@ -16,11 +16,11 @@ SCHEMA >
     `channel` String `json:$.channel` DEFAULT '',
     `url` String `json:$.url` DEFAULT '',
     `sentimentLabel` String `json:$.sentimentLabel` DEFAULT '',
-    `sentimentScore` Int8 `json:$.sentimentScore` DEFAULT -1,
-    `sentimentScoreMixed` Int8 `json:$.sentimentScoreMixed` DEFAULT -1,
-    `sentimentScoreNeutral` Int8 `json:$.sentimentScoreNeutral` DEFAULT -1,
-    `sentimentScoreNegative` Int8 `json:$.sentimentScoreNegative` DEFAULT -1,
-    `sentimentScorePositive` Int8 `json:$.sentimentScorePositive` DEFAULT -1,
+    `sentimentScore` Float32 `json:$.sentimentScore` DEFAULT -1,
+    `sentimentScoreMixed` Float32 `json:$.sentimentScoreMixed` DEFAULT -1,
+    `sentimentScoreNeutral` Float32 `json:$.sentimentScoreNeutral` DEFAULT -1,
+    `sentimentScoreNegative` Float32 `json:$.sentimentScoreNegative` DEFAULT -1,
+    `sentimentScorePositive` Float32 `json:$.sentimentScorePositive` DEFAULT -1,
     `gitIsMainBranch` UInt8 `json:$.gitIsMainBranch` DEFAULT 0,
     `gitIsIndirectFork` UInt8 `json:$.gitIsIndirectFork` DEFAULT 0,
     `gitLines` Int32 `json:$.gitLines` DEFAULT 0,

services/libs/tinybird/datasources/activities.datasource

@@ -0,0 +1,33 @@
+
+SCHEMA >
+    `id` String,
+    `type` LowCardinality(String),
+    `timestamp` DateTime,
+    `platform` LowCardinality(String),
+    `isContribution` UInt8,
+    `score` Int8,
+    `sourceId` String,
+    `createdAt` DateTime64(3),
+    `updatedAt` DateTime64(3),
+    `sourceParentId` String,
+    `attributes` String,
+    `title` String,
+    `body` String,
+    `channel` String,
+    `url` String,
+    `sentimentLabel` String,
+    `sentimentScore` Float32,
+    `sentimentScoreMixed` Float32,
+    `sentimentScoreNeutral` Float32,
+    `sentimentScoreNegative` Float32,
+    `sentimentScorePositive` Float32,
+    `gitIsMainBranch` UInt8,
+    `gitIsIndirectFork` UInt8,
+    `gitLines` Int32,
+    `gitInsertions` Int32,
+    `gitDeletions` Int32,
+    `gitIsMerge` UInt8
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYear(timestamp)"
+ENGINE_SORTING_KEY "timestamp, channel, platform, type, isContribution"

services/libs/tinybird/datasources/activities_sorted.datasource

@@ -0,0 +1,16 @@
+
+SCHEMA >
+    `id` String,
+    `timestamp` DateTime,
+    `platform` LowCardinality(String),
+    `type` LowCardinality(String),
+    `channel` String,
+    `isContribution` UInt8,
+    `updatedAt` DateTime64(3),
+    `memberId` String,
+    `organizationId` String,
+    `segmentId` String
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYear(timestamp)"
+ENGINE_SORTING_KEY "segmentId, timestamp, type, platform, memberId, organizationId"

services/libs/tinybird/datasources/activities_with_relations_sorted_ds.datasource

@@ -0,0 +1,18 @@
+
+SCHEMA >
+    `activityId` String,
+    `conversationId` String,
+    `createdAt` DateTime64(3),
+    `updatedAt` DateTime64(3),
+    `memberId` String,
+    `objectMemberId` String,
+    `objectMemberUsername` String,
+    `organizationId` String,
+    `parentId` String,
+    `platform` LowCardinality(String),
+    `segmentId` String,
+    `username` String
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYear(createdAt)"
+ENGINE_SORTING_KEY "segmentId, memberId, organizationId, activityId"