-
Notifications
You must be signed in to change notification settings - Fork 740
224 lines (196 loc) · 7.3 KB
/
tinybird-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
name: Tinybird CI
on:
pull_request:
paths:
- 'services/libs/tinybird/**'
workflow_dispatch:
env:
DATA_PROJECT_DIR: services/libs/tinybird
GIT_DEPTH: 300
USE_LAST_PARTITION: true
jobs:
check:
name: Datafiles checks
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install Tinybird CLI
run: |
if [ -f "${{ env.DATA_PROJECT_DIR }}/requirements.txt" ]; then
pip install -r ${{ env.DATA_PROJECT_DIR }}/requirements.txt
else
pip install tinybird-cli
fi
- name: Get changed files
id: files
uses: tj-actions/changed-files@v42
with:
files: |
**/*.{datasource,incl,pipe}
- name: Check formatting
if: ${{ steps.files.outputs.any_changed == 'true' }}
shell: bash
run: |
for file in ${{ steps.files.outputs.all_changed_files }}; do
tb fmt --diff "$file"
done
deploy:
name: Deploy to CI Branch
runs-on: ubuntu-latest
defaults:
run:
working-directory: ${{ env.DATA_PROJECT_DIR }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: ${{ env.GIT_DEPTH }}
ref: ${{ github.event.pull_request.head.sha }}
- uses: actions/setup-python@v5
with:
python-version: "3.11"
architecture: "x64"
cache: pip
- name: Set environment variables
run: |
_ENV_FLAGS="${{ env.USE_LAST_PARTITION == 'true' && '--last-partition ' || '' }}--wait"
_NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
GIT_BRANCH=${GITHUB_HEAD_REF}
echo "GIT_BRANCH=$GIT_BRANCH" >> $GITHUB_ENV
echo "_ENV_FLAGS=$_ENV_FLAGS" >> $GITHUB_ENV
echo "_NORMALIZED_BRANCH_NAME=$_NORMALIZED_BRANCH_NAME" >> $GITHUB_ENV
if [ -f .tinyenv ]; then grep -v '^#' .tinyenv >> $GITHUB_ENV; fi
echo >> $GITHUB_ENV
- name: Install Tinybird CLI
run: |
if [ -f "requirements.txt" ]; then
pip install -r requirements.txt
else
pip install tinybird-cli
fi
- name: Tinybird version
run: tb --version
- name: Check all the data files syntax
run: tb check
- name: Check auth
run: tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} auth info
- name: Try delete previous Branch
run: |
output=$(tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} branch ls)
BRANCH_NAME="tmp_ci_${_NORMALIZED_BRANCH_NAME}_${{ github.event.pull_request.number }}"
if echo "$output" | grep -q "\b$BRANCH_NAME\b"; then
tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} branch rm $BRANCH_NAME --yes
else
echo "Skipping clean up: The Branch '$BRANCH_NAME' does not exist."
fi
- name: Create new test Branch
run: |
tb \
--host ${{ secrets.TB_HOST }} \
--token ${{ secrets.TB_ADMIN_TOKEN }} \
branch create tmp_ci_${_NORMALIZED_BRANCH_NAME}_${{ github.event.pull_request.number }} \
${_ENV_FLAGS}
- name: Deploy changes to the test Branch
run: |
source .tinyenv || true
DEPLOY_FILE=./deploy/${VERSION}/deploy.sh
if [ ! -f "$DEPLOY_FILE" ]; then
echo "$DEPLOY_FILE not found, running default tb deploy command"
tb deploy ${CI_FLAGS}
tb release ls
fi
- name: Custom deployment to the test Branch
run: |
source .tinyenv || true
DEPLOY_FILE=./deploy/${VERSION}/deploy.sh
if [ -f "$DEPLOY_FILE" ]; then
echo "$DEPLOY_FILE found"
if ! [ -x "$DEPLOY_FILE" ]; then
echo "Error: You do not have permission to execute '$DEPLOY_FILE'. Run:"
echo "> chmod +x $DEPLOY_FILE"
echo "and commit your changes"
exit 1
else
$DEPLOY_FILE
fi
fi
test:
name: Run tests
runs-on: ubuntu-latest
needs:
- deploy
defaults:
run:
working-directory: ${{ env.DATA_PROJECT_DIR }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- uses: actions/setup-python@v5
with:
python-version: "3.11"
architecture: "x64"
cache: pip
- name: Set environment variables
run: |
_ENV_FLAGS="--last-partition --wait"
_NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
GIT_BRANCH=${GITHUB_HEAD_REF}
echo "GIT_BRANCH=$GIT_BRANCH" >> $GITHUB_ENV
echo "_ENV_FLAGS=$_ENV_FLAGS" >> $GITHUB_ENV
echo "_NORMALIZED_BRANCH_NAME=$_NORMALIZED_BRANCH_NAME" >> $GITHUB_ENV
if [ -f .tinyenv ]; then grep -v '^#' .tinyenv >> $GITHUB_ENV; fi
echo >> $GITHUB_ENV
- name: Install Tinybird CLI
run: |
if [ -f "requirements.txt" ]; then
pip install -r requirements.txt
else
pip install tinybird-cli
fi
- name: Tinybird version
run: tb --version
- name: Check auth
run: tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} auth info
- name: Use Branch
run: |
BRANCH_NAME="tmp_ci_${_NORMALIZED_BRANCH_NAME}_${{ github.event.pull_request.number }}"
tb --host ${{ secrets.TB_HOST }} --token ${{ secrets.TB_ADMIN_TOKEN }} branch use $BRANCH_NAME
- name: Post deploy
run: |
POSTDEPLOY_FILE=./deploy/${VERSION}/postdeploy.sh
if [ -f "$POSTDEPLOY_FILE" ]; then
if ! [ -x "$POSTDEPLOY_FILE" ]; then
echo "Error: You do not have permission to execute '$POSTDEPLOY_FILE'. Run:"
echo "> chmod +x $POSTDEPLOY_FILE"
echo "and commit your changes"
exit 1
else
$POSTDEPLOY_FILE
fi
fi
- name: Get regression labels
id: regression_labels
uses: alrocar/[email protected]
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
label_key: regression
- name: Run pipe regression tests
run: |
source .tinyenv || true
echo ${{ steps.regression_labels.outputs.labels }}
REGRESSION_LABELS=$(echo "${{ steps.regression_labels.outputs.labels }}" | awk -F, '{for (i=1; i<=NF; i++) if ($i ~ /^--/) print $i}' ORS=',' | sed 's/,$//')
echo "Regression labels: ${REGRESSION_LABELS}"
CONFIG_FILE=./tests/regression.yaml
BASE_CMD="tb branch regression-tests"
LABELS_CMD="$(echo ${REGRESSION_LABELS} | tr , ' ')"
if [ -f ${CONFIG_FILE} ]; then
echo "Config file '${CONFIG_FILE}' found, adding pull request labels as options"
${BASE_CMD} -f ${CONFIG_FILE} --wait ${LABELS_CMD}
else
echo "Config file not found at '${CONFIG_FILE}', running with default values"
${BASE_CMD} coverage --wait ${LABELS_CMD}
fi