CI for Tinybird schemas

.github/workflows/tinybird-ci.yml

@@ -0,0 +1,224 @@
+name: Tinybird CI
+
+on:
+  pull_request:
+    paths:
+      - 'services/libs/tinybird/**'
+  workflow_dispatch:
+
+env:
+  DATA_PROJECT_DIR: services/libs/tinybird
+  GIT_DEPTH: 300
+  USE_LAST_PARTITION: true
+
+jobs:
+  check:
+    name: Datafiles checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Tinybird CLI
+        run: |
+          if [ -f "${{ env.DATA_PROJECT_DIR }}/requirements.txt" ]; then
+            pip install -r ${{ env.DATA_PROJECT_DIR }}/requirements.txt
+          else
+            pip install tinybird-cli
+          fi
+
+      - name: Get changed files
+        id: files
+        uses: tj-actions/changed-files@v42
+        with:
+          files: |
+            **/*.{datasource,incl,pipe}
+
+      - name: Check formatting
+        if: ${{ steps.files.outputs.any_changed == 'true' }}
+        shell: bash
+        run: |
+          for file in ${{ steps.files.outputs.all_changed_files }}; do
+            tb fmt --diff "$file"
+          done
+
+  deploy:
+    name: Deploy to CI Branch
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.DATA_PROJECT_DIR }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: ${{ env.GIT_DEPTH }}
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+          cache: pip
+
+      - name: Set environment variables
+        run: |
+          _ENV_FLAGS="${{ env.USE_LAST_PARTITION == 'true' && '--last-partition ' || '' }}--wait"
+          _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
+          GIT_BRANCH=${GITHUB_HEAD_REF}
+          echo "GIT_BRANCH=$GIT_BRANCH" >> $GITHUB_ENV
+          echo "_ENV_FLAGS=$_ENV_FLAGS" >> $GITHUB_ENV
+          echo "_NORMALIZED_BRANCH_NAME=$_NORMALIZED_BRANCH_NAME" >> $GITHUB_ENV
+          if [ -f .tinyenv ]; then grep -v '^#' .tinyenv >> $GITHUB_ENV; fi
+          echo >> $GITHUB_ENV
+
+      - name: Install Tinybird CLI
+        run: |
+          if [ -f "requirements.txt" ]; then
+            pip install -r requirements.txt
+          else
+            pip install tinybird-cli
+          fi
+
+      - name: Tinybird version
+        run: tb --version
+
+      - name: Check all the data files syntax
+        run: tb check
+
+      - name: Check auth
+        run: tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} auth info
+
+      - name: Try delete previous Branch
+        run: |
+          output=$(tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} branch ls)
+          BRANCH_NAME="tmp_ci_${_NORMALIZED_BRANCH_NAME}_${{ github.event.pull_request.number }}"
+          if echo "$output" | grep -q "\b$BRANCH_NAME\b"; then
+              tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} branch rm $BRANCH_NAME --yes
+          else
+              echo "Skipping clean up: The Branch '$BRANCH_NAME' does not exist."
+          fi
+
+      - name: Create new test Branch
+        run: |
+          tb \
+          --host ${{ secrets.TB_HOST }} \
+          --token ${{ secrets.TB_ADMIN_TOKEN }} \
+          branch create tmp_ci_${_NORMALIZED_BRANCH_NAME}_${{ github.event.pull_request.number }} \
+          ${_ENV_FLAGS}
+
+      - name: Deploy changes to the test Branch
+        run: |
+          source .tinyenv || true
+          DEPLOY_FILE=./deploy/${VERSION}/deploy.sh
+          if [ ! -f "$DEPLOY_FILE" ]; then
+            echo "$DEPLOY_FILE not found, running default tb deploy command"
+            tb deploy ${CI_FLAGS}
+            tb release ls
+          fi
+
+      - name: Custom deployment to the test Branch
+        run: |
+          source .tinyenv || true
+          DEPLOY_FILE=./deploy/${VERSION}/deploy.sh
+          if [ -f "$DEPLOY_FILE" ]; then
+            echo "$DEPLOY_FILE found"
+            if ! [ -x "$DEPLOY_FILE" ]; then
+              echo "Error: You do not have permission to execute '$DEPLOY_FILE'. Run:"
+              echo "> chmod +x $DEPLOY_FILE"
+              echo "and commit your changes"
+              exit 1
+            else
+              $DEPLOY_FILE
+            fi
+          fi
+
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+    needs:
+      - deploy
+    defaults:
+      run:
+        working-directory: ${{ env.DATA_PROJECT_DIR }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+          cache: pip
+
+      - name: Set environment variables
+        run: |
+          _ENV_FLAGS="--last-partition --wait"
+          _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
+          GIT_BRANCH=${GITHUB_HEAD_REF}
+          echo "GIT_BRANCH=$GIT_BRANCH" >> $GITHUB_ENV
+          echo "_ENV_FLAGS=$_ENV_FLAGS" >> $GITHUB_ENV
+          echo "_NORMALIZED_BRANCH_NAME=$_NORMALIZED_BRANCH_NAME" >> $GITHUB_ENV
+          if [ -f .tinyenv ]; then grep -v '^#' .tinyenv >> $GITHUB_ENV; fi
+          echo >> $GITHUB_ENV
+
+      - name: Install Tinybird CLI
+        run: |
+          if [ -f "requirements.txt" ]; then
+            pip install -r requirements.txt
+          else
+            pip install tinybird-cli
+          fi
+
+      - name: Tinybird version
+        run: tb --version
+
+      - name: Check auth
+        run: tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} auth info
+
+      - name: Use Branch
+        run: |
+          BRANCH_NAME="tmp_ci_${_NORMALIZED_BRANCH_NAME}_${{ github.event.pull_request.number }}"
+          tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} branch use $BRANCH_NAME
+
+      - name: Post deploy
+        run: |
+          POSTDEPLOY_FILE=./deploy/${VERSION}/postdeploy.sh
+          if [ -f "$POSTDEPLOY_FILE" ]; then
+            if ! [ -x "$POSTDEPLOY_FILE" ]; then
+              echo "Error: You do not have permission to execute '$POSTDEPLOY_FILE'. Run:"
+              echo "> chmod +x $POSTDEPLOY_FILE"
+              echo "and commit your changes"
+              exit 1
+            else
+              $POSTDEPLOY_FILE
+            fi
+          fi
+
+      - name: Get regression labels
+        id: regression_labels
+        uses: alrocar/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          label_key: regression
+
+      - name: Run pipe regression tests
+        run: |
+          source .tinyenv || true
+          echo ${{ steps.regression_labels.outputs.labels }}
+          REGRESSION_LABELS=$(echo "${{ steps.regression_labels.outputs.labels }}" | awk -F, '{for (i=1; i<=NF; i++) if ($i ~ /^--/) print $i}' ORS=',' | sed 's/,$//')
+          echo "Regression labels: ${REGRESSION_LABELS}"
+
+          CONFIG_FILE=./tests/regression.yaml
+          BASE_CMD="tb branch regression-tests"
+          LABELS_CMD="$(echo ${REGRESSION_LABELS} | tr , ' ')"
+          if [ -f ${CONFIG_FILE} ]; then
+              echo "Config file '${CONFIG_FILE}' found, adding pull request labels as options"
+              ${BASE_CMD} -f ${CONFIG_FILE} --wait ${LABELS_CMD}
+          else
+              echo "Config file not found at '${CONFIG_FILE}', running with default values"
+              ${BASE_CMD} coverage --wait ${LABELS_CMD}
+          fi

services/libs/tinybird/datasources/activities.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `id` String `json:$.id`,
     `type` LowCardinality(String) `json:$.type`,
@@ -28,7 +27,7 @@ SCHEMA >
     `gitDeletions` Int32 `json:$.gitDeletions` DEFAULT 0,
     `gitIsMerge` UInt8 `json:$.gitIsMerge` DEFAULT 0
 
-ENGINE "ReplacingMergeTree"
-ENGINE_PARTITION_KEY "toYear(createdAt)"
-ENGINE_SORTING_KEY "isContribution, platform, type, channel, sourceId, timestamp"
-ENGINE_VER "updatedAt"
+ENGINE ReplacingMergeTree
+ENGINE_PARTITION_KEY toYear(createdAt)
+ENGINE_SORTING_KEY isContribution, platform, type, channel, sourceId, timestamp
+ENGINE_VER updatedAt

services/libs/tinybird/datasources/activities_sorted.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `id` String,
     `type` LowCardinality(String),
@@ -28,6 +27,6 @@ SCHEMA >
     `gitDeletions` Int32,
     `gitIsMerge` UInt8
 
-ENGINE "MergeTree"
-ENGINE_PARTITION_KEY "toYear(timestamp)"
-ENGINE_SORTING_KEY "timestamp, channel, platform, type, isContribution"
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(timestamp)
+ENGINE_SORTING_KEY timestamp, channel, platform, type, isContribution

services/libs/tinybird/datasources/activities_with_relations_sorted_ds.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `id` String,
     `timestamp` DateTime,
@@ -11,6 +10,6 @@ SCHEMA >
     `organizationId` String,
     `segmentId` String
 
-ENGINE "MergeTree"
-ENGINE_PARTITION_KEY "toYear(timestamp)"
-ENGINE_SORTING_KEY "segmentId, timestamp, type, platform, memberId, organizationId"
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(timestamp)
+ENGINE_SORTING_KEY segmentId, timestamp, type, platform, memberId, organizationId

services/libs/tinybird/datasources/activityRelations.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `activityId` String `json:$.record.activityId`,
     `conversationId` String `json:$.record.conversationId` DEFAULT '',
@@ -13,7 +12,7 @@ SCHEMA >
     `segmentId` String `json:$.record.segmentId`,
     `username` String `json:$.record.username`
 
-ENGINE "ReplacingMergeTree"
-ENGINE_PARTITION_KEY "toYear(createdAt)"
-ENGINE_SORTING_KEY "segmentId, platform, activityId"
-ENGINE_VER "updatedAt"
+ENGINE ReplacingMergeTree
+ENGINE_PARTITION_KEY toYear(createdAt)
+ENGINE_SORTING_KEY segmentId, platform, activityId
+ENGINE_VER updatedAt

services/libs/tinybird/datasources/activityRelations_sorted.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `activityId` String,
     `conversationId` String,
@@ -13,6 +12,6 @@ SCHEMA >
     `segmentId` String,
     `username` String
 
-ENGINE "MergeTree"
-ENGINE_PARTITION_KEY "toYear(createdAt)"
-ENGINE_SORTING_KEY "segmentId, memberId, organizationId, activityId"
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(createdAt)
+ENGINE_SORTING_KEY segmentId, memberId, organizationId, activityId

services/libs/tinybird/datasources/collections.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `id` String `json:$.record.id`,
     `name` String `json:$.record.name`,
@@ -8,7 +7,7 @@ SCHEMA >
     `createdAt` DateTime64(3) `json:$.record.createdAt`,
     `updatedAt` DateTime64(3) `json:$.record.updatedAt`
 
-ENGINE "ReplacingMergeTree"
-ENGINE_PARTITION_KEY "toYear(createdAt)"
-ENGINE_SORTING_KEY "id"
-ENGINE_VER "updatedAt"
+ENGINE ReplacingMergeTree
+ENGINE_PARTITION_KEY toYear(createdAt)
+ENGINE_SORTING_KEY id
+ENGINE_VER updatedAt

services/libs/tinybird/datasources/collectionsInsightsProjects.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `id` String `json:$.record.id`,
     `collectionId` String `json:$.record.collectionId`,
@@ -7,7 +6,7 @@ SCHEMA >
     `createdAt` DateTime64(3) `json:$.record.createdAt`,
     `updatedAt` DateTime64(3) `json:$.record.updatedAt`
 
-ENGINE "ReplacingMergeTree"
-ENGINE_PARTITION_KEY "toYear(createdAt)"
-ENGINE_SORTING_KEY "insightsProjectId, collectionId"
-ENGINE_VER "updatedAt"
+ENGINE ReplacingMergeTree
+ENGINE_PARTITION_KEY toYear(createdAt)
+ENGINE_SORTING_KEY insightsProjectId, collectionId
+ENGINE_VER updatedAt

services/libs/tinybird/datasources/insightsProjects.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `id` String `json:$.record.id`,
     `name` String `json:$.record.name`,
@@ -16,7 +15,7 @@ SCHEMA >
     `widgets` Array(String) `json:$.record.widgets[:]` DEFAULT [],
     `repositories` Array(String) `json:$.record.repositories[:]` DEFAULT []
 
-ENGINE "ReplacingMergeTree"
-ENGINE_PARTITION_KEY "toYear(createdAt)"
-ENGINE_SORTING_KEY "id"
-ENGINE_VER "updatedAt"
+ENGINE ReplacingMergeTree
+ENGINE_PARTITION_KEY toYear(createdAt)
+ENGINE_SORTING_KEY id
+ENGINE_VER updatedAt

services/libs/tinybird/datasources/members.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `id` String `json:$.record.id`,
     `attributes` String `json:$.record.attributes` DEFAULT '',
@@ -14,7 +13,7 @@ SCHEMA >
     `displayName` String `json:$.record.displayName`,
     `score` Int32 `json:$.record.score` DEFAULT -1
 
-ENGINE "ReplacingMergeTree"
-ENGINE_PARTITION_KEY "toYear(createdAt)"
-ENGINE_SORTING_KEY "id"
-ENGINE_VER "updatedAt"
+ENGINE ReplacingMergeTree
+ENGINE_PARTITION_KEY toYear(createdAt)
+ENGINE_SORTING_KEY id
+ENGINE_VER updatedAt

services/libs/tinybird/datasources/members_sorted.datasource

@@ -1,4 +1,3 @@
-
 SCHEMA >
     `id` String,
     `attributes` String,
@@ -14,6 +13,6 @@ SCHEMA >
     `displayName` String,
     `score` Int32
 
-ENGINE "MergeTree"
-ENGINE_PARTITION_KEY "toYear(joinedAt)"
-ENGINE_SORTING_KEY "id"
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(joinedAt)
+ENGINE_SORTING_KEY id