Add Data Format Validation CI/CD Workflow #1
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: BFCL Data Format Check | ||
| on: | ||
| pull_request: | ||
| branches: [ main ] | ||
| paths: | ||
| - 'berkeley-function-call-leaderboard/data/**/*.json' | ||
| jobs: | ||
| check-data-format: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.10' | ||
| - name: Install dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| pip install jsonschema | ||
| - name: Run format validation | ||
| id: validate | ||
| run: python berkeley-function-call-leaderboard/scripts/validate_data_format.py | ||
| - name: Process validation results | ||
| id: process_results | ||
| if: always() | ||
| run: | | ||
| if [ -f validation_results.json ]; then | ||
| echo "VALIDATION_RESULTS<<EOF" >> $GITHUB_ENV | ||
| # Added change: directly appending file contents | ||
| cat validation_results.json >> $GITHUB_ENV | ||
| echo "EOF" >> $GITHUB_ENV | ||
| fi | ||
| # New step: Calculate the total files checked using jq | ||
| - name: Get total files checked | ||
| id: file_count | ||
| if: always() | ||
| run: | | ||
| # Check if the file exists and use jq to compute the length | ||
| if [ -f validation_results.json ]; then | ||
| total_files=$(jq '.results | length' validation_results.json) | ||
| else | ||
| total_files=0 | ||
| fi | ||
| # Output the computed length so it can be used in later steps | ||
| echo "total_files=$total_files" >> $GITHUB_OUTPUT | ||
| - name: Comment on PR with results | ||
| if: github.event_name == 'pull_request' | ||
| uses: peter-evans/create-or-update-comment@v4 | ||
| with: | ||
| issue-number: ${{ github.event.pull_request.number }} | ||
| body: | | ||
| ## BFCL Data Format Check Results | ||
| ${{ fromJson(env.VALIDATION_RESULTS).all_success && '✅ All data files are correctly formatted!' || '❌ Data format validation failed!' }} | ||
| ### Summary | ||
| <<<<<<< HEAD | ||
| - Total files checked: ${{ steps.file_count.outputs.total_files }} | ||
| ======= | ||
| - Total files checked: ${{ length(fromJson(env.VALIDATION_RESULTS).results) }} | ||
| >>>>>>> 08733aa11e7e9b7e16b17602b94f2666b33a294b | ||
| ${{ !fromJson(env.VALIDATION_RESULTS).all_success && '### Failed Validations:' || '' }} | ||
| ${{ !fromJson(env.VALIDATION_RESULTS).all_success && join(fromJson(env.VALIDATION_RESULTS).results.*.errors, ' | ||
| ') || '' }} | ||
| ${{ !fromJson(env.VALIDATION_RESULTS).all_success && '### How to fix: | ||
| 1. Review the errors above for each failed file | ||
| 2. Ensure all data entries follow the required format for their respective types | ||
| 3. Update the files and push the changes' || '' }} | ||
| - name: Fail if validation failed | ||
| if: fromJson(env.VALIDATION_RESULTS).all_success != true | ||
| run: exit 1 | ||
