fix: use jq for kaggle push to avoid python quoting issues #14
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Push Kaggle Kernels | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - dev | |
| jobs: | |
| push-kernels: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install Kaggle CLI | |
| run: pip install kaggle | |
| - name: Configure Kaggle credentials | |
| env: | |
| KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_API_TOKEN }} | |
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | |
| KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }} | |
| run: | | |
| mkdir -p ~/.kaggle | |
| if [ -n "${KAGGLE_API_TOKEN}" ]; then | |
| printf '%s' "${KAGGLE_API_TOKEN}" > ~/.kaggle/access_token | |
| chmod 600 ~/.kaggle/access_token | |
| elif [ -n "${KAGGLE_KEY}" ] && [[ "${KAGGLE_KEY}" == KGAT_* ]]; then | |
| # Support users who placed modern API token in KAGGLE_KEY. | |
| printf '%s' "${KAGGLE_KEY}" > ~/.kaggle/access_token | |
| chmod 600 ~/.kaggle/access_token | |
| elif [ -n "${KAGGLE_USERNAME}" ] && [ -n "${KAGGLE_KEY}" ]; then | |
| cat > ~/.kaggle/kaggle.json <<EOF | |
| {"username":"${KAGGLE_USERNAME}","key":"${KAGGLE_KEY}"} | |
| EOF | |
| chmod 600 ~/.kaggle/kaggle.json | |
| else | |
| echo "No Kaggle credentials found. Set KAGGLE_API_TOKEN or KAGGLE_USERNAME/KAGGLE_KEY secrets." | |
| exit 1 | |
| fi | |
| - name: Verify Kaggle authentication | |
| env: | |
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | |
| run: kaggle kernels list --user "${KAGGLE_USERNAME}" > /dev/null | |
| - name: Push notebooks to Kaggle | |
| env: | |
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | |
| run: | | |
| set -e | |
| count=0 | |
| for metadata in *.kernel-metadata.json; do | |
| [ -f "$metadata" ] || continue | |
| count=$((count + 1)) | |
| kernel_id=$(jq -r '.id' "$metadata") | |
| code_file=$(jq -r '.code_file' "$metadata") | |
| echo "Pushing kernel using $metadata" | |
| echo "Kernel id: $kernel_id" | |
| echo "Expected URL: https://www.kaggle.com/code/$kernel_id" | |
| if [ ! -f "$code_file" ]; then | |
| echo "ERROR: code_file does not exist: $code_file" | |
| exit 1 | |
| fi | |
| # Stage notebook at repo root so Kaggle path resolution works for | |
| # both new and existing kernels. | |
| staged="__kaggle_push.ipynb" | |
| cp "$code_file" "$staged" | |
| jq --arg cf "$staged" '.code_file = $cf' "$metadata" > kernel-metadata.json | |
| max_retries=12 | |
| attempt=1 | |
| while true; do | |
| set +e | |
| push_output=$(kaggle kernels push -p . 2>&1) | |
| push_exit=$? | |
| set -e | |
| echo "$push_output" | |
| if [ "$push_exit" -eq 0 ]; then | |
| break | |
| fi | |
| if echo "$push_output" | grep -q "Maximum batch CPU session count"; then | |
| if [ "$attempt" -ge "$max_retries" ]; then | |
| echo "Kaggle CPU session limit persisted after ${max_retries} attempts" | |
| exit 1 | |
| fi | |
| echo "Kaggle CPU session limit reached, retrying in 90s (attempt ${attempt}/${max_retries})" | |
| attempt=$((attempt + 1)) | |
| sleep 90 | |
| else | |
| echo "Non-retryable push error for $kernel_id" | |
| exit "$push_exit" | |
| fi | |
| done | |
| kaggle kernels status "$kernel_id" || true | |
| rm -f "$staged" kernel-metadata.json | |
| done | |
| if [ "$count" -eq 0 ]; then | |
| echo "No *.kernel-metadata.json files found" | |
| exit 1 | |
| fi | |
| - name: Verify pushed kernels are listed | |
| env: | |
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | |
| run: | | |
| echo "Kernels visible for user: ${KAGGLE_USERNAME}" | |
| kaggle kernels list --user "${KAGGLE_USERNAME}" |