Skip to content

Commit 25c06e7

Browse files
borrrdenborrrden
authored
.NET prebuild logic and quality of life improvements (#200)
- Fix a bug in .NET test server locking over async boundaries - Deduplicate test artifacts so that Jenkins can store them side by side
1 parent 3d3e5ea commit 25c06e7

30 files changed

Lines changed: 384 additions & 270 deletions

‎environment/aws/topology_setup/test_server.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -213,15 +213,16 @@ def download(self) -> None:
213213
Raises:
214214
FileNotFoundError: If the test server package is not found on the latestbuilds server.
215215
"""
216+
url = f"https://latestbuilds.service.couchbase.com/builds/latestbuilds/{self.latestbuilds_path}"
217+
header(f"Downloading {url}")
216218
download_dir = DOWNLOADED_TEST_SERVER_DIR / self.platform / self.version
217219
download_dir.mkdir(parents=True, exist_ok=True)
218220
if (download_dir / ".downloaded").exists():
219221
click.secho("Already downloaded", fg="green")
220222
click.echo()
223+
self._downloaded = True
221224
return
222225

223-
url = f"https://latestbuilds.service.couchbase.com/builds/latestbuilds/{self.latestbuilds_path}"
224-
header(f"Downloading {url}")
225226
response = requests.head(url)
226227
if response.status_code == 404:
227228
raise FileNotFoundError(f"Test server not found at {url}")

‎jenkins/pipelines/dev_e2e/c/Jenkinsfile‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ pipeline {
3030
options {
3131
lock('mob-e2e-mac-01')
3232
}
33+
environment {
34+
TS_ARTIFACTS_DIR = 'macos'
35+
}
3336
steps {
3437
echo "Run macOS Test"
3538
timeout(time: 60, unit: 'MINUTES') {
@@ -42,8 +45,7 @@ pipeline {
4245
timeout(time: 5, unit: 'MINUTES') {
4346
sh 'jenkins/pipelines/dev_e2e/c/teardown.sh'
4447
}
45-
archiveArtifacts artifacts: 'tests/dev_e2e/session.log', fingerprint: true, allowEmptyArchive: true
46-
archiveArtifacts artifacts: 'tests/dev_e2e/http_log/*', fingerprint: true, allowEmptyArchive: true
48+
archiveArtifacts artifacts: 'tests/dev_e2e/macOS/**', fingerprint: true, allowEmptyArchive: true
4749
}
4850
}
4951
}
@@ -53,7 +55,8 @@ pipeline {
5355
lock('mob-e2e-iphone-01')
5456
}
5557
environment {
56-
KEYCHAIN_PASSWORD = credentials('mob-e2e-mac-01-keychain-password')
58+
KEYCHAIN_PASSWORD = credentials('mob-e2e-mac-01-keychain-password')
59+
TS_ARTIFACTS_DIR = 'ios'
5760
}
5861
steps {
5962
// Unlock keychain:
@@ -69,8 +72,7 @@ pipeline {
6972
timeout(time: 5, unit: 'MINUTES') {
7073
sh 'jenkins/pipelines/dev_e2e/c/teardown.sh'
7174
}
72-
archiveArtifacts artifacts: 'tests/dev_e2e/session.log', fingerprint: true, allowEmptyArchive: true
73-
archiveArtifacts artifacts: 'tests/dev_e2e/http_log/*', fingerprint: true, allowEmptyArchive: true
75+
archiveArtifacts artifacts: 'tests/dev_e2e/ios/**', fingerprint: true, allowEmptyArchive: true
7476
}
7577
}
7678
}
@@ -79,6 +81,9 @@ pipeline {
7981
options {
8082
lock('mob-e2e-android-01')
8183
}
84+
environment {
85+
TS_ARTIFACTS_DIR = 'android'
86+
}
8287
steps {
8388
echo "Run Android Test"
8489
timeout(time: 60, unit: 'MINUTES') {
@@ -91,8 +96,7 @@ pipeline {
9196
timeout(time: 5, unit: 'MINUTES') {
9297
sh 'jenkins/pipelines/dev_e2e/c/teardown.sh'
9398
}
94-
archiveArtifacts artifacts: 'tests/dev_e2e/session.log', fingerprint: true, allowEmptyArchive: true
95-
archiveArtifacts artifacts: 'tests/dev_e2e/http_log/*', fingerprint: true, allowEmptyArchive: true
99+
archiveArtifacts artifacts: 'tests/dev_e2e/android/**', fingerprint: true, allowEmptyArchive: true
96100
}
97101
}
98102
}
@@ -101,6 +105,9 @@ pipeline {
101105
options {
102106
lock('mob-e2e-deb-02')
103107
}
108+
environment {
109+
TS_ARTIFACTS_DIR = 'linux_x86_64'
110+
}
104111
steps {
105112
echo "Run linux Test"
106113
timeout(time: 60, unit: 'MINUTES') {
@@ -113,8 +120,7 @@ pipeline {
113120
timeout(time: 5, unit: 'MINUTES') {
114121
sh 'jenkins/pipelines/dev_e2e/c/teardown.sh'
115122
}
116-
archiveArtifacts artifacts: 'tests/dev_e2e/session.log', fingerprint: true, allowEmptyArchive: true
117-
archiveArtifacts artifacts: 'tests/dev_e2e/http_log/*', fingerprint: true, allowEmptyArchive: true
123+
archiveArtifacts artifacts: 'tests/dev_e2e/linux_x86_64/**', fingerprint: true, allowEmptyArchive: true
118124
}
119125
}
120126
}
@@ -134,8 +140,7 @@ pipeline {
134140
timeout(time: 5, unit: 'MINUTES') {
135141
pwsh 'jenkins\\pipelines\\dev_e2e\\c\\teardown.ps1'
136142
}
137-
archiveArtifacts artifacts: 'tests\\dev_e2e\\session.log', fingerprint: true, allowEmptyArchive: true
138-
archiveArtifacts artifacts: 'tests\\dev_e2e\\http_log\\*', fingerprint: true, allowEmptyArchive: true
143+
archiveArtifacts artifacts: 'tests\\dev_e2e\\windows\\**', fingerprint: true, allowEmptyArchive: true
139144
}
140145
}
141146
}

‎jenkins/pipelines/dev_e2e/c/run_test.ps1‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@ $ErrorActionPreference = "Stop"
99

1010
Import-Module $PSScriptRoot\..\..\shared\config.psm1 -Force
1111

12-
python -m venv venv
12+
Stop-Venv
13+
New-Venv venv
1314
.\venv\Scripts\activate
14-
pip install -r $AWS_ENVIRONMENT_DIR\requirements.txt
15+
trap { Stop-Venv; break }
16+
uv pip install -r $AWS_ENVIRONMENT_DIR\requirements.txt
1517
$python_args = @("windows", $Version, $DatasetVersion, $SgwVersion)
1618
if ($null -ne $PrivateKeyPath) {
1719
$python_args += "--private_key"
@@ -24,11 +26,14 @@ if($LASTEXITCODE -ne 0) {
2426
}
2527

2628
Push-Location $DEV_E2E_TESTS_DIR
27-
pip install -r requirements.txt
28-
pytest -v --no-header -W ignore::DeprecationWarning --config config.json
29-
$saved_exit = $LASTEXITCODE
30-
deactivate
31-
Pop-Location
29+
try {
30+
uv pip install -r requirements.txt
31+
pytest -v --no-header -W ignore::DeprecationWarning --config config.json
32+
$saved_exit = $LASTEXITCODE
33+
deactivate
34+
} finally {
35+
Pop-Location
36+
}
3237

3338
if($saved_exit -ne 0) {
3439
throw "Testing failed!"

‎jenkins/pipelines/dev_e2e/c/run_test.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ dataset_version=$2
2929
platform=$3
3030
sgw_version=$4
3131

32+
stop_venv
3233
create_venv venv
3334
source venv/bin/activate
35+
trap stop_venv EXIT
3436
pip install -r $AWS_ENVIRONMENT_DIR/requirements.txt
3537
if [ -n "$private_key_path" ]; then
3638
python3 $SCRIPT_DIR/setup_test.py $platform $cbl_version $dataset_version $sgw_version --private_key $private_key_path

‎jenkins/pipelines/dev_e2e/c/teardown.ps1‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,21 @@
11
$env:PYTHONPATH = "$PSScriptRoot\..\..\..\"
22
Import-Module $PSScriptRoot\..\..\shared\config.psm1 -Global
33
Push-Location $AWS_ENVIRONMENT_DIR
4-
python -m venv venv
4+
Move-Artifacts
5+
6+
function Cleanup {
7+
Stop-Venv
8+
Pop-Location
9+
}
10+
11+
trap {
12+
Cleanup
13+
break
14+
}
15+
16+
Stop-Venv
17+
New-Venv venv
518
.\venv\Scripts\activate
6-
pip install -r requirements.txt
19+
uv pip install -r requirements.txt
720
python .\stop_backend.py --topology topology_setup\topology.json
8-
Pop-Location
21+
Cleanup

‎jenkins/pipelines/dev_e2e/c/teardown.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@ set -euo pipefail
44

55
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
66
source $SCRIPT_DIR/../../shared/config.sh
7+
move_artifacts
78

89
export PYTHONPATH=$SCRIPT_DIR/../../../
910
pushd $AWS_ENVIRONMENT_DIR
11+
stop_venv
1012
create_venv venv
1113
source venv/bin/activate
12-
pip install -r requirements.txt
14+
trap stop_venv EXIT
15+
uv pip install -r requirements.txt
1316
python3 ./stop_backend.py --topology topology_setup/topology.json

‎jenkins/pipelines/dev_e2e/dotnet/Jenkinsfile‎

Lines changed: 79 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,47 @@ pipeline {
1919
if (params.CBL_BUILD == '') { error "CBL_BUILD is required" }
2020
if (params.CBL_DATASET_VERSION == '') { error "CBL_DATASET_VERSION is required" }
2121
if (params.SGW_VERSION == '') { error "SGW_VERSION is required" }
22-
currentBuild.displayName = "${params.CBL_VERSION}-${params.CBL_BUILD}-${CBL_EDITION}"
23-
currentBuild.description = "Dataset: ${params.CBL_DATASET_VERSION} / SGW: ${params.SGW_VERSION}"
22+
currentBuild.displayName = "${params.CBL_VERSION}-${params.CBL_BUILD}"
23+
currentBuild.description = "SGW: ${params.SGW_VERSION}"
2424
}
2525
}
2626
}
27-
stage('Desktop') {
27+
stage('Prebuild Servers') {
28+
steps {
29+
script {
30+
def platforms = [
31+
'dotnet_windows', 'dotnet_ios', 'dotnet_android', 'dotnet_macos'
32+
]
33+
def parallelBuilds = [:]
34+
for(p in platforms) {
35+
// Groovy stupidness. Need to set a local variable here
36+
// to avoid late binding (all jobs use dotnet_macos).
37+
def platform = p
38+
parallelBuilds[platform] = {
39+
build job: 'prebuild-test-server',
40+
parameters: [
41+
string(name: 'TS_PLATFORM', value: platform),
42+
string(name: 'CBL_VERSION', value: params.CBL_VERSION),
43+
string(name: 'CBL_BUILD', value: params.CBL_BUILD)
44+
],
45+
wait: true,
46+
propagate: true
47+
}
48+
}
49+
parallel parallelBuilds
50+
}
51+
}
52+
}
53+
stage('Run Tests') {
2854
parallel {
2955
stage("Windows") {
3056
agent { label 'mob-e2e-win-01' }
3157
options {
3258
lock("mob-e2e-win-01")
3359
}
60+
environment {
61+
TS_ARTIFACTS_DIR = 'windows'
62+
}
3463
steps {
3564
timeout(time: 60, unit: 'MINUTES') {
3665
pwsh "jenkins\\pipelines\\dev_e2e\\dotnet\\run_test.ps1 -Version ${params.CBL_VERSION}-${params.CBL_BUILD} -Dataset ${params.CBL_DATASET_VERSION} -SgwVersion ${params.SGW_VERSION} -PrivateKeyPath C:\\Users\\mob-e\\.ssh\\jborden.pem"
@@ -41,8 +70,7 @@ pipeline {
4170
timeout(time: 5, unit: 'MINUTES') {
4271
pwsh 'jenkins\\pipelines\\dev_e2e\\dotnet\\teardown.ps1'
4372
}
44-
archiveArtifacts artifacts: 'tests\\dev_e2e\\session.log', fingerprint: true, allowEmptyArchive: true
45-
archiveArtifacts artifacts: 'tests\\dev_e2e\\http_log\\*', fingerprint: true, allowEmptyArchive: true
73+
archiveArtifacts artifacts: 'tests\\dev_e2e\\windows\\**', fingerprint: true, allowEmptyArchive: true
4674
}
4775
}
4876
}
@@ -55,6 +83,7 @@ pipeline {
5583
KEYCHAIN_PASSWORD = credentials('mob-e2e-mac-01-keychain-password')
5684
DEVELOPER_DIR="/Applications/Xcode.app"
5785
MD_APPLE_SDK_ROOT="/Applications/Xcode.app"
86+
TS_ARTIFACTS_DIR = 'macos'
5887
}
5988
steps {
6089
sh 'security unlock-keychain -p ${KEYCHAIN_PASSWORD} ~/Library/Keychains/login.keychain-db'
@@ -67,56 +96,57 @@ pipeline {
6796
timeout(time: 5, unit: 'MINUTES') {
6897
sh 'jenkins/pipelines/dev_e2e/dotnet/teardown.sh'
6998
}
70-
archiveArtifacts artifacts: 'tests/dev_e2e/session.log', fingerprint: true, allowEmptyArchive: true
71-
archiveArtifacts artifacts: 'tests/dev_e2e/http_log/*', fingerprint: true, allowEmptyArchive: true
99+
archiveArtifacts artifacts: 'tests/dev_e2e/macos/**', fingerprint: true, allowEmptyArchive: true
72100
}
73101
}
74102
}
75-
}
76-
}
77-
stage('iOS') {
78-
options {
79-
lock("mob-e2e-iphone-01")
80-
}
81-
environment {
82-
KEYCHAIN_PASSWORD = credentials('mob-e2e-mac-01-keychain-password')
83-
DEVELOPER_DIR="/Applications/Xcode.app"
84-
MD_APPLE_SDK_ROOT="/Applications/Xcode.app"
85-
}
86-
agent { label 'mob-e2e-mac-01' }
87-
steps {
88-
sh 'security unlock-keychain -p ${KEYCHAIN_PASSWORD} ~/Library/Keychains/login.keychain-db'
89-
timeout(time: 60, unit: 'MINUTES') {
90-
sh "jenkins/pipelines/dev_e2e/dotnet/run_test.sh ${params.CBL_VERSION}-${params.CBL_BUILD} ${params.CBL_DATASET_VERSION} ios '${params.SGW_VERSION}' ~/.ssh/jborden.pem"
91-
}
92-
}
93-
post {
94-
always {
95-
timeout(time: 5, unit: 'MINUTES') {
96-
sh 'jenkins/pipelines/dev_e2e/dotnet/teardown.sh'
103+
stage('iOS') {
104+
options {
105+
lock("mob-e2e-iphone-01")
106+
}
107+
environment {
108+
KEYCHAIN_PASSWORD = credentials('mob-e2e-mac-01-keychain-password')
109+
DEVELOPER_DIR="/Applications/Xcode.app"
110+
MD_APPLE_SDK_ROOT="/Applications/Xcode.app"
111+
TS_ARTIFACTS_DIR = 'ios'
112+
}
113+
agent { label 'mob-e2e-mac-01' }
114+
steps {
115+
sh 'security unlock-keychain -p ${KEYCHAIN_PASSWORD} ~/Library/Keychains/login.keychain-db'
116+
timeout(time: 60, unit: 'MINUTES') {
117+
sh "jenkins/pipelines/dev_e2e/dotnet/run_test.sh ${params.CBL_VERSION}-${params.CBL_BUILD} ${params.CBL_DATASET_VERSION} ios '${params.SGW_VERSION}' ~/.ssh/jborden.pem"
118+
}
119+
}
120+
post {
121+
always {
122+
timeout(time: 5, unit: 'MINUTES') {
123+
sh 'jenkins/pipelines/dev_e2e/dotnet/teardown.sh'
124+
}
125+
archiveArtifacts artifacts: 'tests/dev_e2e/ios/**', fingerprint: true, allowEmptyArchive: true
126+
}
97127
}
98-
archiveArtifacts artifacts: 'tests/dev_e2e/session.log', fingerprint: true, allowEmptyArchive: true
99-
archiveArtifacts artifacts: 'tests/dev_e2e/http_log/*', fingerprint: true, allowEmptyArchive: true
100-
}
101-
}
102-
}
103-
stage('Android') {
104-
options {
105-
lock("mob-e2e-android-01")
106-
}
107-
agent { label 'mob-e2e-mac-01' }
108-
steps {
109-
timeout(time: 60, unit: 'MINUTES') {
110-
sh "jenkins/pipelines/dev_e2e/dotnet/run_test.sh ${params.CBL_VERSION}-${params.CBL_BUILD} ${params.CBL_DATASET_VERSION} android '${params.SGW_VERSION}' ~/.ssh/jborden.pem"
111128
}
112-
}
113-
post {
114-
always {
115-
timeout(time: 5, unit: 'MINUTES') {
116-
sh 'jenkins/pipelines/dev_e2e/dotnet/teardown.sh'
129+
stage('Android') {
130+
options {
131+
lock("mob-e2e-android-01")
132+
}
133+
agent { label 'mob-e2e-mac-01' }
134+
environment {
135+
TS_ARTIFACTS_DIR = 'android'
136+
}
137+
steps {
138+
timeout(time: 60, unit: 'MINUTES') {
139+
sh "jenkins/pipelines/dev_e2e/dotnet/run_test.sh ${params.CBL_VERSION}-${params.CBL_BUILD} ${params.CBL_DATASET_VERSION} android '${params.SGW_VERSION}' ~/.ssh/jborden.pem"
140+
}
141+
}
142+
post {
143+
always {
144+
timeout(time: 5, unit: 'MINUTES') {
145+
sh 'jenkins/pipelines/dev_e2e/dotnet/teardown.sh'
146+
}
147+
archiveArtifacts artifacts: 'tests/dev_e2e/android/**', fingerprint: true, allowEmptyArchive: true
148+
}
117149
}
118-
archiveArtifacts artifacts: 'tests/dev_e2e/session.log', fingerprint: true, allowEmptyArchive: true
119-
archiveArtifacts artifacts: 'tests/dev_e2e/http_log/*', fingerprint: true, allowEmptyArchive: true
120150
}
121151
}
122152
}

0 commit comments

Comments
 (0)