Skip to content

Commit 3db8832

Browse files
SevenannnSevenannn
authored
Test spice chat in e2e_test_spice_cli (spiceai#5447)
* Test spice chat in e2e_test_spice_cli * remove temporary changes
1 parent 89b251f commit 3db8832

3 files changed

Lines changed: 207 additions & 26 deletions

File tree

‎.github/workflows/e2e_test_spice_cli.yml‎

Lines changed: 94 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
}
4747
];
4848
return matrix;
49-
49+
5050
build:
5151
name: Build ${{ matrix.target.name }} CLI
5252
if: ${{ github.event.inputs.build_cli == 'true' || github.event.inputs.build_cli == null }}
@@ -81,7 +81,7 @@ jobs:
8181
run: |
8282
mv target/release/spice spice
8383
chmod +x spice
84-
84+
8585
- name: Move Spice(Windows)
8686
if: matrix.target.target_os == 'windows'
8787
shell: pwsh
@@ -129,7 +129,7 @@ jobs:
129129
with:
130130
name: build_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}
131131
path: ./build
132-
132+
133133
- name: download spice cli released version
134134
if: github.event.inputs.build_cli == 'false' && matrix.target.target_os != 'windows'
135135
run: |
@@ -141,7 +141,7 @@ jobs:
141141
tar -xzf "$output"
142142
rm "$output"
143143
chmod +x spice
144-
144+
145145
- name: download spice cli released version
146146
if: github.event.inputs.build_cli == 'false' && matrix.target.target_os == 'windows'
147147
run: |
@@ -152,7 +152,7 @@ jobs:
152152
Invoke-WebRequest -Uri $url -OutFile $output
153153
tar -xzf $output
154154
Remove-Item $output
155-
155+
156156
- name: Download a older spiced
157157
shell: pwsh
158158
if: matrix.target.target_os == 'windows'
@@ -183,14 +183,14 @@ jobs:
183183
- name: Run spice upgrade
184184
run: |
185185
spice upgrade
186-
186+
187187
- name: Check the temp directory (Windows)
188188
if: matrix.target.target_os == 'windows'
189189
run: |
190190
Write-Host "System temp directory: $env:TEMP"
191-
Get-ChildItem -Path $env:TEMP -Recurse |
192-
Where-Object { $_.Name -like "spice*" } |
193-
Select-Object FullName, Length, LastWriteTime |
191+
Get-ChildItem -Path $env:TEMP -Recurse |
192+
Where-Object { $_.Name -like "spice*" } |
193+
Select-Object FullName, Length, LastWriteTime |
194194
Format-Table -AutoSize
195195
196196
- name: Run spice upgrade again
@@ -202,9 +202,9 @@ jobs:
202202
if: matrix.target.target_os == 'windows'
203203
run: |
204204
Write-Host "System temp directory: $env:TEMP"
205-
Get-ChildItem -Path $env:TEMP -Recurse |
206-
Where-Object { $_.Name -like "spice*" } |
207-
Select-Object FullName, Length, LastWriteTime |
205+
Get-ChildItem -Path $env:TEMP -Recurse |
206+
Where-Object { $_.Name -like "spice*" } |
207+
Select-Object FullName, Length, LastWriteTime |
208208
Format-Table -AutoSize
209209
210210
test_spice_upgrade_without_runtime:
@@ -244,11 +244,11 @@ jobs:
244244
tar -xzf "$output"
245245
rm "$output"
246246
chmod +x spice
247-
247+
248248
- name: download spice cli released version
249249
if: github.event.inputs.build_cli == 'false' && matrix.target.target_os == 'windows'
250250
run: |
251-
New-Item -ItemType Directory -Force -Path "./build" > $null
251+
New-Item -ItemType Directory -Force -Path "./build" > $null
252252
Set-Location "./build"
253253
$url = "https://github.com/spiceai/spiceai/releases/download/v${{ env.REL_VERSION }}/spice.exe_windows_x86_64.tar.gz"
254254
$output = "spice.exe_windows_x86_64.tar.gz"
@@ -264,14 +264,14 @@ jobs:
264264
- name: Run spice upgrade
265265
run: |
266266
spice upgrade
267-
267+
268268
- name: Check the temp directory (Windows)
269269
if: matrix.target.target_os == 'windows'
270270
run: |
271271
Write-Host "System temp directory: $env:TEMP"
272-
Get-ChildItem -Path $env:TEMP -Recurse |
273-
Where-Object { $_.Name -like "spice*" } |
274-
Select-Object FullName, Length, LastWriteTime |
272+
Get-ChildItem -Path $env:TEMP -Recurse |
273+
Where-Object { $_.Name -like "spice*" } |
274+
Select-Object FullName, Length, LastWriteTime |
275275
Format-Table -AutoSize
276276
277277
- name: Run spice upgrade again
@@ -283,9 +283,9 @@ jobs:
283283
if: matrix.target.target_os == 'windows'
284284
run: |
285285
Write-Host "System temp directory: $env:TEMP"
286-
Get-ChildItem -Path $env:TEMP -Recurse |
287-
Where-Object { $_.Name -like "spice*" } |
288-
Select-Object FullName, Length, LastWriteTime |
286+
Get-ChildItem -Path $env:TEMP -Recurse |
287+
Where-Object { $_.Name -like "spice*" } |
288+
Select-Object FullName, Length, LastWriteTime |
289289
Format-Table -AutoSize
290290
291291
test_spice_install_without_runtime:
@@ -325,11 +325,11 @@ jobs:
325325
tar -xzf "$output"
326326
rm "$output"
327327
chmod +x spice
328-
328+
329329
- name: download spice cli released version
330330
if: github.event.inputs.build_cli == 'false' && matrix.target.target_os == 'windows'
331331
run: |
332-
New-Item -ItemType Directory -Force -Path "./build" > $null
332+
New-Item -ItemType Directory -Force -Path "./build" > $null
333333
Set-Location "./build"
334334
$url = "https://github.com/spiceai/spiceai/releases/download/v${{ env.REL_VERSION }}/spice.exe_windows_x86_64.tar.gz"
335335
$output = "spice.exe_windows_x86_64.tar.gz"
@@ -387,11 +387,11 @@ jobs:
387387
tar -xzf "$output"
388388
rm "$output"
389389
chmod +x spice
390-
390+
391391
- name: download spice cli released version
392392
if: github.event.inputs.build_cli == 'false' && matrix.target.target_os == 'windows'
393393
run: |
394-
New-Item -ItemType Directory -Force -Path "./build" > $null
394+
New-Item -ItemType Directory -Force -Path "./build" > $null
395395
Set-Location "./build"
396396
$url = "https://github.com/spiceai/spiceai/releases/download/v${{ env.REL_VERSION }}/spice.exe_windows_x86_64.tar.gz"
397397
$output = "spice.exe_windows_x86_64.tar.gz"
@@ -455,7 +455,7 @@ jobs:
455455
target_os: "darwin"
456456
target_arch: "aarch64"
457457
target_arch_go: "arm64"
458-
458+
459459
steps:
460460
- uses: actions/checkout@v4
461461

@@ -482,3 +482,71 @@ jobs:
482482
- name: run spice upgrade
483483
run: |
484484
spice upgrade
485+
486+
test_spice_chat:
487+
name: 'Test spice chat'
488+
if: github.event.inputs.build_cli == 'true'
489+
continue-on-error: true
490+
runs-on: ${{ matrix.target.runner }}
491+
env:
492+
REL_VERSION: ${{ github.event.inputs.rel_version || '1.0.2' }}
493+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
494+
needs:
495+
- build
496+
timeout-minutes: 5
497+
498+
strategy:
499+
matrix:
500+
target:
501+
- name: "macOS aarch64 (Apple Silicon)"
502+
runner: "macos-14"
503+
target_os: "darwin"
504+
target_arch: "aarch64"
505+
target_arch_go: "arm64"
506+
507+
steps:
508+
- uses: actions/checkout@v4
509+
510+
- name: download artifacts - build_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}
511+
if: github.event.inputs.build_cli != 'false'
512+
uses: actions/download-artifact@v4
513+
with:
514+
name: build_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}
515+
path: ./build
516+
517+
- name: Install spice
518+
uses: ./.github/actions/install-spice-cli
519+
with:
520+
build-path: ./build
521+
522+
- name: Init spice app
523+
run: |
524+
cp ./test/models/spicepod_openai.yml ./spicepod.yaml
525+
cat ./spicepod.yaml
526+
527+
- name: Start spice runtime
528+
env:
529+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
530+
SPICE_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
531+
run: |
532+
spice run &> spice.log &
533+
534+
- name: Wait for Spice runtime is ready
535+
timeout-minutes: 1
536+
run: |
537+
while [[ "$(curl -s http://localhost:8090/v1/ready)" != "ready" ]]; do sleep 1; done
538+
539+
- name: Install expect (macOS)
540+
if: matrix.target.target_os == 'darwin'
541+
run: |
542+
brew install expect
543+
544+
- name: Test `spice chat` error
545+
run: |
546+
./test/cli/spice_chat_error.exp
547+
548+
- name: Stop spice and check logs
549+
if: always()
550+
run: |
551+
killall spice || true
552+
cat spice.log

‎test/cli/spice_chat_error.exp‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/expect -f
2+
3+
# Test invalid --model error case
4+
proc test_invalid_model {} {
5+
set command "spice chat --model not_exist"
6+
7+
catch {exec sh -c "$command 2>&1"} output
8+
puts "Command output: $output"
9+
10+
# Check for expected error messages in the output
11+
if {[regexp {ERROR.*Requested model is not available.*not_exist} $output]} {
12+
puts "✓ Error message about unavailable model detected"
13+
} else {
14+
puts "✗ Missing error about unavailable model"
15+
exit 1
16+
}
17+
18+
# Check that available models are listed
19+
if {[regexp {INFO.*Available models} $output]} {
20+
puts "✓ Available models list is shown"
21+
} else {
22+
puts "✗ Available models list not shown"
23+
exit 1
24+
}
25+
26+
# Check that expected models are in the list
27+
if {[regexp {openai-gpt} $output]} {
28+
puts "✓ Found expected model 'openai-gpt' in the list"
29+
} else {
30+
puts "✗ Expected model 'openai-gpt' not found in the list"
31+
exit 1
32+
}
33+
34+
puts "✓ Invalid model test passed"
35+
}
36+
37+
# Test invalid --http-endpoint error case
38+
proc test_invalid_http_endpoint {} {
39+
set command "spice chat --http-endpoint https://data.spicea"
40+
41+
catch {exec sh -c "$command 2>&1"} output
42+
puts "Command output: $output"
43+
44+
# Check for expected error messages in the output
45+
if {[regexp {ERROR.*could not list models.*no such host} $output]} {
46+
puts "✓ Error message about invalid host detected"
47+
} else {
48+
puts "✗ Missing error about invalid host"
49+
exit 1
50+
}
51+
52+
# Check for the specific host name in the error
53+
if {[regexp {lookup data.spicea: no such host} $output]} {
54+
puts "✓ Invalid hostname correctly reported in error"
55+
} else {
56+
puts "✗ Invalid hostname not correctly reported"
57+
exit 1
58+
}
59+
60+
puts "✓ Invalid host test passed"
61+
}
62+
63+
# Test invalid --temperature error case
64+
proc test_invalid_temperature {} {
65+
set command "spice chat --temperature -1"
66+
67+
catch {exec sh -c "$command 2>&1"} output
68+
puts "Command output: $output"
69+
70+
# Check for expected error messages in the output
71+
if {[regexp {ERROR.*temperature must be greater than or equal to 0} $output]} {
72+
puts "✓ Error message about invalid temperature detected"
73+
} else {
74+
puts "✗ Missing error about invalid temperature"
75+
exit 1
76+
}
77+
78+
puts "✓ Invalid temperature test passed"
79+
}
80+
81+
# Test invalid --api-key error case
82+
proc test_invalid_api_key {} {
83+
set command "spice chat --http-endpoint https://data.spiceai.io --api-key \"wrong_key\""
84+
85+
catch {exec sh -c "$command 2>&1"} output
86+
puts "Command output: $output"
87+
88+
# Check for expected error messages in the output
89+
if {[regexp {ERROR.*could not list models.*unauthorized: invalid or missing Spice API key} $output]} {
90+
puts "✓ Error message about invalid API key detected"
91+
} else {
92+
puts "✗ Missing error about invalid API key"
93+
exit 1
94+
}
95+
96+
puts "✓ Invalid API key test passed"
97+
}
98+
99+
# Run the test
100+
test_invalid_model
101+
test_invalid_http_endpoint
102+
test_invalid_temperature
103+
test_invalid_api_key
104+
exit 0

‎test/cli/spicepod_openai.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
version: v1
2+
kind: Spicepod
3+
name: ai-test-spicepod
4+
5+
models:
6+
- from: openai:gpt-4o-mini
7+
name: openai-gpt
8+
params:
9+
openai_api_key: ${ secrets:SPICE_OPENAI_API_KEY }

0 commit comments

Comments
 (0)