Writing a Rust Backend V1

.github/workflows/ci.yml

@@ -2,39 +2,74 @@ name: CI
 
 on:
   pull_request:
-    branches: [ main ]
+    branches: [main]
   push:
-    branches: [ main ]
+    branches: [main]
 
 jobs:
-  ci:
+  # Build native module on Linux (fast sanity check)
+  build-native:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: osgrep-core
+
+      - name: Install napi-rs CLI
+        run: npm install -g @napi-rs/cli
+
+      - name: Build native module
+        working-directory: osgrep-core
+        run: napi build --platform --release
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
         with:
-          version: 9
-          run_install: false
+          name: native-linux
+          path: osgrep-core/*.node
+
+  # TypeScript checks and tests
+  ci:
+    runs-on: ubuntu-latest
+    needs: build-native
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup bun
+        uses: oven-sh/setup-bun@v2
 
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '20'
-          cache: 'pnpm'
-          cache-dependency-path: 'pnpm-lock.yaml'
+
+      - name: Download native module
+        uses: actions/download-artifact@v4
+        with:
+          name: native-linux
+          path: osgrep-core
 
       - name: Install dependencies
-        run: pnpm install --frozen-lockfile
+        run: bun install
 
       - name: Check types
-        run: pnpm run typecheck
+        run: bun run typecheck
 
       - name: Run tests
-        run: pnpm run test
+        run: bun run test
 
       - name: Build
-        run: pnpm run build
-
+        run: bun run build

.github/workflows/release.yml

@@ -6,47 +6,113 @@ on:
       - 'v*.*.*'
   workflow_dispatch:
 
+env:
+  CARGO_INCREMENTAL: 0
+
 jobs:
-  publish-npm:
+  # =============================================================================
+  # Build native binaries for each platform
+  # =============================================================================
+  build-native:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - target: aarch64-apple-darwin
+            os: macos-latest
+          - target: x86_64-apple-darwin
+            os: macos-latest
+          - target: x86_64-unknown-linux-gnu
+            os: ubuntu-latest
+          - target: x86_64-unknown-linux-musl
+            os: ubuntu-latest
+            use-cross: true
+          - target: x86_64-pc-windows-msvc
+            os: windows-latest
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install dependencies (Linux musl)
+        if: matrix.use-cross
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y musl-tools
+
+      - name: Install napi-rs CLI
+        run: npm install -g @napi-rs/cli
+
+      - name: Build native module
+        working-directory: osgrep-core
+        run: napi build --platform --release --target ${{ matrix.target }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bindings-${{ matrix.target }}
+          path: osgrep-core/*.node
+          if-no-files-found: error
+
+  # =============================================================================
+  # Publish platform packages + meta package + osgrep
+  # =============================================================================
+  publish:
     runs-on: ubuntu-latest
+    needs: build-native
     permissions:
       contents: write
       id-token: write
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 9
-          run_install: false
-
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '20'
           registry-url: 'https://registry.npmjs.org'
-          cache: 'pnpm'
-          cache-dependency-path: 'pnpm-lock.yaml'
+
+      - name: Install napi-rs CLI
+        run: npm install -g @napi-rs/cli
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: osgrep-core/artifacts
+
+      - name: Move artifacts to osgrep-core
+        run: |
+          cd osgrep-core
+          for dir in artifacts/bindings-*/; do
+            mv "$dir"*.node . 2>/dev/null || true
+          done
+          rm -rf artifacts
+          ls -la *.node
 
       - name: Verify tag commit is on main
-        working-directory: .
         run: |
           git fetch origin main --depth=1
           if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then
             echo "Refusing to publish: tag commit is not on main"
             exit 1
           fi
 
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-
-      - name: Check types
-        run: pnpm run typecheck
-
       - name: Verify tag matches package.json version
         run: |
           TAG="${GITHUB_REF##*/}"
@@ -56,15 +122,36 @@ jobs:
             exit 1
           fi
 
-      - name: Build
-        run: pnpm build
+      - name: Publish osgrep-core platform packages
+        working-directory: osgrep-core
+        run: napi prepublish -t npm
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Publish osgrep-core meta package
+        working-directory: osgrep-core
+        run: npm publish --access public --provenance
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Install bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Install osgrep dependencies
+        run: bun install
+
+      - name: Check types
+        run: bun run typecheck
+
+      - name: Build osgrep
+        run: bun run build
 
-      - name: Publish to npm
-        run: pnpm publish --access public --provenance --no-git-checks
+      - name: Publish osgrep to npm
+        run: npm publish --access public --provenance
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-      - name: Create GitHub Release (tag only, no assets)
+      - name: Create GitHub Release
         uses: softprops/action-gh-release@v1
         with:
           tag_name: ${{ github.ref_name }}

.gitignore

@@ -1,28 +1,53 @@
 dist/
 node_modules/
+**/node_modules/
 *.tsbuildinfo
 *.tgz
+.DS_Store
+
+# Local osgrep data
+.osgrep/
+
+# Development files
+CLAUDE.md
+TODO.md
+AGENTS.md
 ADVANCED.md
-test-build.sh
-TEST_SUITE_PROMPT.md
-WATCH_MODE_PROMPT.md
 ENGINEERING_DIARY.md
 RELEASE_GUIDE.md
-osgrep/run-benchmark.sh
+TEST_SUITE_PROMPT.md
+WATCH_MODE_PROMPT.md
+AGENT_BENCHMARK.md
+latest_review.md
+plan.md
+
+# Benchmark & experiments (dev-only)
+benchmark/
+experiments/
+scripts/
+tools/
+public/
+test_skeleton.py
+test-build.sh
 run-agent-benchmark.sh
 setup-benchmark-repos.sh
-AGENT_BENCHMARK.md
 benchmark-results.json
-src/bench/benchmark-agent.ts
-src/bench/generate-benchmark-chart.ts
-src/.env.local
-CLAUDE.md
-.DS_Store
-latest_review.md
 benchmarks/*.log
-.osgrep/server.json
-.osgrep/cache/meta.lmdb-lock
+src/bench/
+
+# Env files
+src/.env.local
+.env*
 
+# Lockfiles (using bun)
+pnpm-lock.yaml
+package-lock.json
+yarn.lock
+
+bun.lockb
+# Native core (Rust / N-API) build outputs
+osgrep-core/target/
+**/target/
+osgrep-core/*.node
+osgrep-core/bench/
 .osgrep
-TODO.md
-AGENTS.md