feat: use the array api in the scanner

.editorconfig

@@ -0,0 +1,39 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{json,toml,yml,gyp}]
+indent_style = space
+indent_size = 2
+
+[*.js]
+indent_style = space
+indent_size = 2
+
+[*.rs]
+indent_style = space
+indent_size = 4
+
+[*.{c,cc,h}]
+indent_style = space
+indent_size = 4
+
+[*.{py,pyi}]
+indent_style = space
+indent_size = 4
+
+[*.swift]
+indent_style = space
+indent_size = 4
+
+[*.go]
+indent_style = tab
+indent_size = 8
+
+[Makefile]
+indent_style = tab
+indent_size = 8

.gitattributes

@@ -1,10 +1,10 @@
-/src/** linguist-vendored
-/examples/* linguist-vendored
+* text eol=lf
 
-src/grammar.json linguist-generated
-src/node-types.json linguist-generated
+src/*.json linguist-generated
 src/parser.c linguist-generated
+src/tree_sitter/* linguist-generated
 
-src/grammar.json -diff
-src/node-types.json -diff
-src/parser.c -diff
+bindings/** linguist-generated
+binding.gyp linguist-generated
+setup.py linguist-generated
+Makefile linguist-generated

.github/workflows/ci.yml

@@ -1,32 +1,78 @@
 name: CI
 
 on:
-  pull_request:
-    branches:
-      - "**"
   push:
-    branches:
-      - "master"
+    branches: ["*"]
+    paths:
+      - grammar.js
+      - src/**
+      - test/**
+      - bindings/**
+      - binding.gyp
+  pull_request:
+    paths:
+      - grammar.js
+      - src/**
+      - test/**
+      - bindings/**
+      - binding.gyp
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
 jobs:
   test:
-    runs-on: ${{ matrix.os }}
+    name: Test parser
+    runs-on: ${{matrix.os}}
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
-        os: [macos-latest, ubuntu-latest]
+        os: [ubuntu-latest, windows-latest, macos-14]
+        include:
+          - examples:
+            - { repo: 'numpy/numpy', path: 'examples/numpy' }
+            - { repo: 'django/django', path: 'examples/django' }
+            - { repo: 'pallets/flask', path: 'examples/flask' }
+            - { repo: 'python/cpython', path: 'examples/python' }
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - name: Set up the repo
+        uses: tree-sitter/[email protected]
+        with:
+          node-version: ${{vars.NODE_VERSION}}
+      - name: Set up examples
+        uses: actions/checkout@v4
+        with:
+          repository: ${{matrix.examples.repo}}
+          path: ${{matrix.examples.path}}
+          clean: false
+      - name: Run tests
+        uses: tree-sitter/[email protected]
         with:
-          node-version: 18
-      - run: npm install
-      - run: npm test
-  test_windows:
-    runs-on: windows-latest
+          lint: true
+          test-library: ${{runner.os == 'Linux'}}
+          examples: |
+            examples/examples/**/*.py
+            !examples/cpython/Lib/test/badsyntax_3131.py
+            !examples/cpython/Lib/test/badsyntax_future8.py
+            !examples/cpython/Lib/test/test_compile.py
+            !examples/cpython/Tools/build/generate_re_casefix.py
+  fuzz:
+    name: Fuzz parser
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
         with:
-          node-version: 18
-      - run: npm install
-      - run: npm run-script test-windows
+          fetch-depth: 2
+      - name: Check for scanner changes
+        id: scanner-changes
+        run: |-
+          if git diff --quiet HEAD^ -- src/scanner.c; then
+            printf 'changed=false\n' >> "$GITHUB_OUTPUT"
+          else
+            printf 'changed=true\n' >> "$GITHUB_OUTPUT"
+          fi
+      - name: Fuzz parser
+        uses: tree-sitter/fuzz-action@v4
+        if: steps.scanner-changes.outputs.changed == 'true'

.github/workflows/release.yml

@@ -1,103 +1,19 @@
-name: Release
+name: Publish package
 
 on:
-  workflow_run:
-    workflows: ["CI"]
-    branches:
-      - master
-    types:
-      - completed
+  push:
+    tags: ["*"]
 
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Get previous commit SHA
-        id: get_previous_commit
-        run: |
-          LATEST_TAG=$(git describe --tags --abbrev=0)
-          if [[ -z "$LATEST_TAG" ]]; then
-            echo "No tag found. Failing..."
-            exit 1
-          fi
-          echo "latest_tag=${LATEST_TAG#v}" >> "$GITHUB_ENV" # Remove 'v' prefix from the tag
-
-      - name: Check if version changed and is greater than the previous
-        id: version_check
-        run: |
-          # Compare the current version with the version from the previous commit
-          PREVIOUS_NPM_VERSION=${{ env.latest_tag }}
-          CURRENT_NPM_VERSION=$(jq -r '.version' package.json)
-          CURRENT_CARGO_VERSION=$(awk -F '"' '/^version/ {print $2}' Cargo.toml)
-          if [[ "$CURRENT_NPM_VERSION" != "$CURRENT_CARGO_VERSION" ]]; then # Cargo.toml and package.json versions must match
-            echo "Mismatch: NPM version ($CURRENT_NPM_VERSION) and Cargo.toml version ($CURRENT_CARGO_VERSION)"
-            echo "version_changed=false" >> "$GITHUB_ENV"
-          else
-            if [[ "$PREVIOUS_NPM_VERSION" ==  "$CURRENT_NPM_VERSION" ]]; then
-              echo "version_changed=" >> "$GITHUB_ENV"
-            else
-              IFS='.' read -ra PREVIOUS_VERSION_PARTS <<< "$PREVIOUS_NPM_VERSION"
-              IFS='.' read -ra CURRENT_VERSION_PARTS <<< "$CURRENT_NPM_VERSION"
-              VERSION_CHANGED=false
-              for i in "${!PREVIOUS_VERSION_PARTS[@]}"; do
-                if [[ ${CURRENT_VERSION_PARTS[i]} -gt ${PREVIOUS_VERSION_PARTS[i]} ]]; then
-                  VERSION_CHANGED=true
-                  break
-                elif [[ ${CURRENT_VERSION_PARTS[i]} -lt ${PREVIOUS_VERSION_PARTS[i]} ]]; then
-                  break
-                fi
-              done
-
-              echo "version_changed=$VERSION_CHANGED" >> "$GITHUB_ENV"
-              echo "current_version=${CURRENT_NPM_VERSION}" >> "$GITHUB_ENV"
-            fi
-          fi
-
-      - name: Display result
-        run: |
-          echo "Version bump detected: ${{ env.version_changed }}"
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
 
-      - name: Fail if version is lower
-        if: env.version_changed == 'false'
-        run: exit 1
-
-      - name: Setup Node
-        if: env.version_changed == 'true'
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-          registry-url: "https://registry.npmjs.org"
-      - name: Publish to NPM
-        if: env.version_changed == 'true'
-        env:
-          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
-        run: npm publish
-
-      - name: Setup Rust
-        if: env.version_changed == 'true'
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - name: Publish to Crates.io
-        if: env.version_changed == 'true'
-        uses: katyo/publish-crates@v2
-        with:
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-
-      - name: Tag versions
-        if: env.version_changed == 'true'
-        run: |
-          git checkout master
-          git config user.name github-actions[bot]
-          git config user.email github-actions[bot]@users.noreply.github.com
-          git tag -d "v${{ env.current_version }}" || true
-          git push origin --delete "v${{ env.current_version }}" || true
-          git tag -a "v${{ env.current_version }}" -m "Version ${{ env.current_version }}"
-          git push origin "v${{ env.current_version }}"
+jobs:
+  npm:
+    uses: tree-sitter/workflows/.github/workflows/package-npm.yml@main
+    secrets:
+      NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
+  crates:
+    uses: tree-sitter/workflows/.github/workflows/package-crates.yml@main
+    secrets:
+      CARGO_REGISTRY_TOKEN: ${{secrets.CARGO_TOKEN}}

.gitignore

@@ -1,7 +1,36 @@
-Cargo.lock
-package-lock.json
-node_modules
-build
-*.log
-/examples/*/
+# Rust artifacts
+/Cargo.lock
 /target/
+
+# Node artifacts
+/build/
+/node_modules/
+
+# Swift artifacts
+/.build/
+
+# Python artifacts
+/dist/
+*.egg-info
+*.whl
+
+# Zig artifacts
+/zig-cache/
+/zig-out/
+
+# C artifacts
+*.a
+*.so
+*.so.*
+*.dylib
+*.dll
+*.pc
+
+# Example dirs
+/examples/*/
+
+# Grammar volatiles
+dsl.d.ts
+*.wasm
+*.obj
+*.o

.npmignore

@@ -1,6 +1,17 @@
-/test
-/examples
-/build
-/script
-/target
+bindings/c
+bindings/go
+bindings/python
 bindings/rust
+bindings/swift
+Cargo.toml
+Makefile
+examples
+pyproject.toml
+setup.py
+test
+.editorconfig
+.github
+.gitignore
+.gitattributes
+.gitmodules
+.npmignore