SpiceQA #46
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: SpiceQA | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| spice_version: | |
| description: 'Spice.ai version to test' | |
| required: true | |
| default: 'v1.0.6' | |
| type: string | |
| jobs: | |
| test: | |
| name: Test ${{ matrix.recipe_name }} Recipe | |
| runs-on: ubuntu-latest | |
| environment: Production | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| recipe_name: | |
| - 'File Data Connector' | |
| - 'Dremio Data Connector' | |
| - 'Federated SQL Query' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Create output directory | |
| run: mkdir -p api_responses | |
| - name: Install Spice | |
| shell: bash | |
| run: | | |
| curl https://install.spiceai.org | /bin/bash | |
| echo "~/.spice/bin" >> $GITHUB_PATH | |
| - name: Verify Spice Installation | |
| shell: bash | |
| run: | | |
| spice version | |
| - name: Call Spice AI Completion API | |
| shell: bash | |
| env: | |
| SPICE_SPICEAI_API_KEY: ${{ secrets.SPICE_API_KEY }} | |
| run: | | |
| recipe_file_path="$(echo "${{ matrix.recipe_name }}" | tr '[:upper:]' '[:lower:]' | tr ' ' '_')" | |
| echo "RECIPE_FILE_PATH=$recipe_file_path" >> $GITHUB_ENV | |
| output_file="api_responses/${recipe_file_path}.log" | |
| chat_input="Install Spice.ai OSS using the spiceai docs. Add it to the PATH. Verify installation of version '${{ github.event.inputs.spice_version }}' by running the CLI command spice version. Use the '${{ matrix.recipe_name }}' Recipe to conduct testing. Ensure you execute all instructions in the recipe. Ensure you report on the spice sql query or spice chat results if the recipe includes them." | |
| spice chat --cloud --http-endpoint https://data.spiceai.io --model timmy "$chat_input" > "$output_file" 2>&1 | |
| # Output response for debugging | |
| echo "API Response saved to $output_file" | |
| # Check for errors | |
| if echo "$response" | grep -q "\"error\""; then | |
| echo "Error detected in API response" | |
| echo "$response" | |
| exit 1 | |
| fi | |
| - name: Upload response file | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: api-response-${{ env.RECIPE_FILE_PATH }} | |
| path: api_responses/${{ env.RECIPE_FILE_PATH }}.log | |
| - name: Check the spice-qa passing status | |
| shell: bash | |
| env: | |
| SPICE_SPICEAI_API_KEY: ${{ secrets.SPICE_API_KEY }} | |
| run: | | |
| read -r qa_stat qa_score < <( | |
| grep -oE '✅ QA passed \([0-9]+% confidence\)|❌ QA failed \([0-9]+% confidence\)' \ | |
| "api_responses/${{ env.RECIPE_FILE_PATH }}.log" \ | |
| | sed -E \ | |
| -e 's/✅ QA passed \(([0-9]+)% confidence\)/passed \1/' \ | |
| -e 's/❌ QA failed \(([0-9]+)% confidence\)/failed \1/' | |
| ) | |
| echo "QA Status: $qa_stat" | |
| echo "QA Score: $qa_score" | |
| if [[ "$qa_stat" != "passed" ]]; then | |
| exit 1 | |
| fi |