-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy path.gitlab-ci.yml
More file actions
245 lines (220 loc) · 8.23 KB
/
.gitlab-ci.yml
File metadata and controls
245 lines (220 loc) · 8.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
#file: noinspection YAMLSchemaValidation
---
include:
- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'
stages:
- 🧪 test
- 📋 lint
- 🏗 build
- 🚚 publish
- 📣 release
variables:
# Image
UV_CACHE_DIR: ".uv-cache"
UV_VERSION: "0.9.9"
PYTHON_VERSION: "3.9"
BASE_LAYER: "bookworm-slim"
# Task runner
TASK_BIN: "uv run task"
# Publishing
S3_SCHEMA_BUCKET_STAGE: metadata-resources-testing.data.bas.ac.uk
S3_SCHEMA_BUCKET_PROD: metadata-resources.data.bas.ac.uk
# Secrets (set in GitLab CI/CD settings)
# - SAFETY_API_KEY
# - https://start.1password.com/open/i?a=QSB6V7TUNVEOPPPWR6G7S2ARJ4&v=k34cpwfkqaxp2r56u4aklza6ni&i=swbuhnii4ego6qycyqknvtk7gi&h=magic.1password.eu
# - PYPI_TOKEN
# - create tokens at https://pypi.org/manage/account/
# - AWS IAM id/secret key for 'bas-gitlab-ci-bas-metadata-generator' user
# - AWS_ACCESS_KEY_ID: "[Sensitive]"
# - AWS_SECRET_ACCESS_KEY: "[Sensitive]"
image: ghcr.io/astral-sh/uv:$UV_VERSION-python$PYTHON_VERSION-$BASE_LAYER
.before_script_python: &before_script_python
- apt-get update
# setup UV
- uv --version
.before_script_testing: &before_script_testing
- *before_script_python
# setup app dep libs
- apt-get update
- apt-get install -y libxml2-utils
# setup app deps
- uv sync --all-groups
# Jobs
pytest:
stage: 🧪 test
needs: []
before_script:
- *before_script_testing
script:
- $TASK_BIN ci-test
coverage: '/Total coverage: \d+\.\d+/'
artifacts:
when: always
reports:
junit: test-results.xml
paths:
- htmlcov
expire_in: 1 month
rules:
-
changes:
- 'src/**/*.py'
- 'tests/**/*.py'
- 'pyproject.toml'
- 'uv.lock'
if: '$CI_COMMIT_BRANCH != "main" && $CI_COMMIT_TAG == null'
ruff:
stage: 📋 lint
needs: []
before_script:
- *before_script_testing
script:
- $TASK_BIN lint
- $TASK_BIN format --check
rules:
-
changes:
- '**/*.py'
if: '$CI_COMMIT_BRANCH != "main" && $CI_COMMIT_TAG == null'
safety:
stage: 📋 lint
needs: []
before_script:
- *before_script_testing
script:
- $TASK_BIN ci-safety
rules:
-
changes:
- '.safety-policy.yml'
- 'uv.lock'
if: '$CI_COMMIT_BRANCH != "main" && $CI_COMMIT_TAG == null'
markdown:
stage: 📋 lint
needs: []
before_script:
- *before_script_python
script:
- $TASK_BIN markdown
rules:
-
changes:
- 'DEVELOPING.md'
- 'README.md'
- 'CHANGELOG.md'
if: '$CI_COMMIT_BRANCH != "main" && $CI_COMMIT_TAG == null'
build:
stage: 🏗 build
needs: []
before_script:
- *before_script_python
script:
- $TASK_BIN build
artifacts:
paths:
- dist/
expire_in: 1 month
rules:
-
changes:
- '**/*.py'
- 'pyproject.toml'
- 'uv.lock'
if: '$CI_COMMIT_BRANCH != "main" || $CI_COMMIT_TAG != null'
publish:
stage: 🚚 publish
needs:
- job: build
artifacts: true
before_script:
- *before_script_python
script:
- $TASK_BIN publish --token $PYPI_TOKEN
rules:
- if: $CI_COMMIT_TAG
publish-schemas-stage:
stage: 🚚 publish
needs: []
image:
name: governmentpaas/awscli:latest
entrypoint: [""]
before_script: []
script:
- "aws s3 cp src/bas_metadata_library/schemas/dist/iso_19115_0_v4.json s3://$S3_SCHEMA_BUCKET_STAGE/bas-metadata-generator-configuration-schemas/v2/iso-19115-0-v4.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/iso_19115_2_v4.json s3://$S3_SCHEMA_BUCKET_STAGE/bas-metadata-generator-configuration-schemas/v2/iso-19115-2-v4.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/magic_discovery_v1.json s3://$S3_SCHEMA_BUCKET_STAGE/bas-metadata-generator-configuration-schemas/v2/magic-discovery-v1.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/magic_discovery_v2.json s3://$S3_SCHEMA_BUCKET_STAGE/bas-metadata-generator-configuration-schemas/v2/magic-discovery-v2.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/magic_administration_content_v1.json s3://$S3_SCHEMA_BUCKET_STAGE/bas-metadata-generator-configuration-schemas/v2/magic-administration-content-v1.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/magic_administration_encoding_v1.json s3://$S3_SCHEMA_BUCKET_STAGE/bas-metadata-generator-configuration-schemas/v2/magic-administration-encoding-v1.json"
rules:
- if: '$CI_COMMIT_BRANCH == "main" && $CI_COMMIT_TAG == null'
publish-schemas-prod:
stage: 🚚 publish
needs: []
image:
name: governmentpaas/awscli:latest
entrypoint: [""]
before_script: []
script:
- "aws s3 cp src/bas_metadata_library/schemas/dist/iso_19115_0_v4.json s3://$S3_SCHEMA_BUCKET_PROD/bas-metadata-generator-configuration-schemas/v2/iso-19115-0-v4.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/iso_19115_2_v4.json s3://$S3_SCHEMA_BUCKET_PROD/bas-metadata-generator-configuration-schemas/v2/iso-19115-2-v4.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/magic_discovery_v1.json s3://$S3_SCHEMA_BUCKET_PROD/bas-metadata-generator-configuration-schemas/v2/magic-discovery-v1.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/magic_discovery_v2.json s3://$S3_SCHEMA_BUCKET_PROD/bas-metadata-generator-configuration-schemas/v2/magic-discovery-v2.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/magic_administration_content_v1.json s3://$S3_SCHEMA_BUCKET_PROD/bas-metadata-generator-configuration-schemas/v2/magic-administration-content-v1.json"
- "aws s3 cp src/bas_metadata_library/schemas/dist/magic_administration_encoding_v1.json s3://$S3_SCHEMA_BUCKET_PROD/bas-metadata-generator-configuration-schemas/v2/magic-administration-encoding-v1.json"
rules:
- if: $CI_COMMIT_TAG
release:
stage: 📣 release
needs:
- job: publish
image: registry.gitlab.com/gitlab-org/release-cli:latest
before_script:
- apk add --no-cache curl jq
# for a string 'v0.8.13a1', strip the leading 'v' (e.g. 0.8.13a1)
- export TAG_NO_PREFIX=$(echo $CI_COMMIT_TAG | cut -c 2-)
# for a string 'v0.8.13a1', strip any modifiers (e.g. v0.8.13)
- export TAG_NO_SUFFIX=$(echo $CI_COMMIT_TAG | sed 's/[a-zA-Z].*$//')
# for a string 'v0.8.13', replace the patch component with '0' (e.g. v0.8.0)
- export TAG_MINOR=$(echo $TAG_NO_SUFFIX | sed 's/[0-9]$/0/')
- 'curl -s -H "Authorization: Bearer $PROJECT_ACCESS_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/milestones?title=$CI_COMMIT_TAG" > milestone_exact.json'
- 'curl -s -H "Authorization: Bearer $PROJECT_ACCESS_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/milestones?title=$TAG_MINOR" > milestone-minor.json'
- export MILESTONE_TITLE_EXACT=$(cat milestone_exact.json | jq -r ".[0] | .title") && rm milestone_exact.json
- export MILESTONE_TITLE_MINOR=$(cat milestone-minor.json | jq -r ".[0] | .title") && rm milestone-minor.json
- >
if [ "$MILESTONE_TITLE_EXACT" != "null" ]; then
export MILESTONE_TITLE=$MILESTONE_TITLE_EXACT
elif [ "$MILESTONE_TITLE_MINOR" != "null" ]; then
export MILESTONE_TITLE=$MILESTONE_TITLE_MINOR
else
export MILESTONE_TITLE=""
fi
- >
if echo "$CI_COMMIT_TAG" | grep -Eq 'a|b|rc'; then
echo "Prerelease." > changelog.txt
else
curl -s -L -O https://github.com/taiki-e/parse-changelog/releases/download/v0.6.8/parse-changelog-x86_64-unknown-linux-musl.tar.gz
tar -xzf parse-changelog-x86_64-unknown-linux-musl.tar.gz -C /usr/local/bin/
parse-changelog CHANGELOG.md "$TAG_NO_PREFIX" > changelog.txt
fi
# the release section cannot access variables defined in a script but can read from a file :|
- echo "$TAG_NO_PREFIX" > tag_no_prefix.txt
- echo "$MILESTONE_TITLE" > milestone_title.txt
script:
- echo 'releasing'
release:
name: $(cat tag_no_prefix.txt)
tag_name: $CI_COMMIT_TAG
milestones:
- $(cat milestone_title.txt)
description: $(cat changelog.txt)
assets:
links:
- name: README
url: '$CI_PROJECT_URL/-/blob/$CI_COMMIT_TAG/README.md'
link_type: runbook
- name: Python Package
url: 'https://pypi.org/project/bas-metadata-library/$(cat tag_no_prefix.txt)/'
link_type: package
rules:
- if: $CI_COMMIT_TAG