diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7f0de95a..c7e15210 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,39 +2,74 @@ name: CI
on:
pull_request:
- branches: [ main ]
+ branches: [main]
push:
- branches: [ main ]
+ branches: [main]
jobs:
- ci:
+ # Build native module on Linux (fast sanity check)
+ build-native:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- - name: Setup pnpm
- uses: pnpm/action-setup@v4
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Cache Rust
+ uses: Swatinem/rust-cache@v2
+ with:
+ workspaces: osgrep-core
+
+ - name: Install napi-rs CLI
+ run: npm install -g @napi-rs/cli
+
+ - name: Build native module
+ working-directory: osgrep-core
+ run: napi build --platform --release
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v4
with:
- version: 9
- run_install: false
+ name: native-linux
+ path: osgrep-core/*.node
+
+ # TypeScript checks and tests
+ ci:
+ runs-on: ubuntu-latest
+ needs: build-native
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup bun
+ uses: oven-sh/setup-bun@v2
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '20'
- cache: 'pnpm'
- cache-dependency-path: 'pnpm-lock.yaml'
+
+ - name: Download native module
+ uses: actions/download-artifact@v4
+ with:
+ name: native-linux
+ path: osgrep-core
- name: Install dependencies
- run: pnpm install --frozen-lockfile
+ run: bun install
- name: Check types
- run: pnpm run typecheck
+ run: bun run typecheck
- name: Run tests
- run: pnpm run test
+ run: bun run test
- name: Build
- run: pnpm run build
-
+ run: bun run build
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 66336a7c..7343b46c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -6,34 +6,106 @@ on:
- 'v*.*.*'
workflow_dispatch:
+env:
+ CARGO_INCREMENTAL: 0
+
jobs:
- publish-npm:
+ # =============================================================================
+ # Build native binaries for each platform
+ # =============================================================================
+ build-native:
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - target: aarch64-apple-darwin
+ os: macos-latest
+ - target: x86_64-apple-darwin
+ os: macos-latest
+ - target: x86_64-unknown-linux-gnu
+ os: ubuntu-latest
+ - target: x86_64-unknown-linux-musl
+ os: ubuntu-latest
+ use-cross: true
+ - target: x86_64-pc-windows-msvc
+ os: windows-latest
+
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ targets: ${{ matrix.target }}
+
+ - name: Install dependencies (Linux musl)
+ if: matrix.use-cross
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y musl-tools
+
+ - name: Install napi-rs CLI
+ run: npm install -g @napi-rs/cli
+
+ - name: Build native module
+ working-directory: osgrep-core
+ run: napi build --platform --release --target ${{ matrix.target }}
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: bindings-${{ matrix.target }}
+ path: osgrep-core/*.node
+ if-no-files-found: error
+
+ # =============================================================================
+ # Publish platform packages + meta package + osgrep
+ # =============================================================================
+ publish:
runs-on: ubuntu-latest
+ needs: build-native
permissions:
contents: write
id-token: write
+
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- - name: Setup pnpm
- uses: pnpm/action-setup@v4
- with:
- version: 9
- run_install: false
-
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '20'
registry-url: 'https://registry.npmjs.org'
- cache: 'pnpm'
- cache-dependency-path: 'pnpm-lock.yaml'
+
+ - name: Install napi-rs CLI
+ run: npm install -g @napi-rs/cli
+
+ - name: Download all artifacts
+ uses: actions/download-artifact@v4
+ with:
+ path: osgrep-core/artifacts
+
+ - name: Move artifacts to osgrep-core
+ run: |
+ cd osgrep-core
+ for dir in artifacts/bindings-*/; do
+ mv "$dir"*.node . 2>/dev/null || true
+ done
+ rm -rf artifacts
+ ls -la *.node
- name: Verify tag commit is on main
- working-directory: .
run: |
git fetch origin main --depth=1
if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then
@@ -41,12 +113,6 @@ jobs:
exit 1
fi
- - name: Install dependencies
- run: pnpm install --frozen-lockfile
-
- - name: Check types
- run: pnpm run typecheck
-
- name: Verify tag matches package.json version
run: |
TAG="${GITHUB_REF##*/}"
@@ -56,15 +122,36 @@ jobs:
exit 1
fi
- - name: Build
- run: pnpm build
+ - name: Publish osgrep-core platform packages
+ working-directory: osgrep-core
+ run: napi prepublish -t npm
+ env:
+ NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+ - name: Publish osgrep-core meta package
+ working-directory: osgrep-core
+ run: npm publish --access public --provenance
+ env:
+ NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+ - name: Install bun
+ uses: oven-sh/setup-bun@v2
+
+ - name: Install osgrep dependencies
+ run: bun install
+
+ - name: Check types
+ run: bun run typecheck
+
+ - name: Build osgrep
+ run: bun run build
- - name: Publish to npm
- run: pnpm publish --access public --provenance --no-git-checks
+ - name: Publish osgrep to npm
+ run: npm publish --access public --provenance
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- - name: Create GitHub Release (tag only, no assets)
+ - name: Create GitHub Release
uses: softprops/action-gh-release@v1
with:
tag_name: ${{ github.ref_name }}
diff --git a/.gitignore b/.gitignore
index 9b85d444..e08a9c8d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,28 +1,53 @@
dist/
node_modules/
+**/node_modules/
*.tsbuildinfo
*.tgz
+.DS_Store
+
+# Local osgrep data
+.osgrep/
+
+# Development files
+CLAUDE.md
+TODO.md
+AGENTS.md
ADVANCED.md
-test-build.sh
-TEST_SUITE_PROMPT.md
-WATCH_MODE_PROMPT.md
ENGINEERING_DIARY.md
RELEASE_GUIDE.md
-osgrep/run-benchmark.sh
+TEST_SUITE_PROMPT.md
+WATCH_MODE_PROMPT.md
+AGENT_BENCHMARK.md
+latest_review.md
+plan.md
+
+# Benchmark & experiments (dev-only)
+benchmark/
+experiments/
+scripts/
+tools/
+public/
+test_skeleton.py
+test-build.sh
run-agent-benchmark.sh
setup-benchmark-repos.sh
-AGENT_BENCHMARK.md
benchmark-results.json
-src/bench/benchmark-agent.ts
-src/bench/generate-benchmark-chart.ts
-src/.env.local
-CLAUDE.md
-.DS_Store
-latest_review.md
benchmarks/*.log
-.osgrep/server.json
-.osgrep/cache/meta.lmdb-lock
+src/bench/
+
+# Env files
+src/.env.local
+.env*
+# Lockfiles (using bun)
+pnpm-lock.yaml
+package-lock.json
+yarn.lock
+
+bun.lockb
+# Native core (Rust / N-API) build outputs
+osgrep-core/target/
+**/target/
+osgrep-core/*.node
+osgrep-core/bench/
.osgrep
-TODO.md
-AGENTS.md
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index 6d69c9c6..00000000
--- a/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright 2025 Mixedbread AI (original mgrep work)
- Copyright 2025 Ryan D'Onofrio (osgrep modifications and enhancements)
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/NOTICE b/NOTICE
deleted file mode 100644
index 62b2f374..00000000
--- a/NOTICE
+++ /dev/null
@@ -1,33 +0,0 @@
-osgrep
-Copyright 2025 Ryan Donofrio
-
-This product includes software originally developed by MixedBread as mgrep.
-Original mgrep Copyright 2025 MixedBread
-https://github.com/mixedbread-ai/mgrep
-
-================================================================================
-
-MODIFICATIONS AND ENHANCEMENTS
-
-osgrep is a substantial modification and enhancement of the original mgrep
-codebase. The following major changes and additions have been made by
-Ryan Donofrio:
-
-- Complete rewrite to use local-only embeddings (removing remote API dependencies)
-- Integration with @huggingface/transformers for local model inference
-- Implementation of LanceDB for local vector storage
-- New local storage architecture and indexing system
-- Enhanced chunking and context management
-- Addition of watch mode for real-time index updates
-- New CLI commands and improved user experience
-- Comprehensive benchmarking and evaluation tools
-- Extensive test suite additions
-- Documentation and setup improvements
-
-
-================================================================================
-
-THIRD-PARTY DEPENDENCIES
-
-This software depends on various open-source libraries. See package.json for
-a complete list of dependencies and their respective licenses.
diff --git a/README.md b/README.md
index e88861c5..47008e29 100644
--- a/README.md
+++ b/README.md
@@ -20,11 +20,10 @@
Natural-language search that works like `grep`. Fast, local, and built for coding agents.
- **Semantic:** Finds concepts ("where do transactions get created?"), not just strings.
-- **Call Graph Tracing:** Map dependencies with `trace` to see who calls what.
- **Role Detection:** Distinguishes `ORCHESTRATION` (high-level logic) from `DEFINITION` (types/classes).
-- **Local & Private:** 100% local embeddings via `onnxruntime-node`.
+- **Local & Private:** 100% local embeddings via `onnxruntime`.
- **Auto-Isolated:** Each repository gets its own index automatically.
-- **Agent-Ready:** Native output with symbols, roles, and call graphs.
+- **Agent-Ready:** Native output with symbols and roles.
## Quick Start
@@ -50,26 +49,6 @@ Natural-language search that works like `grep`. Fast, local, and built for codin
**Your first search will automatically index the repository.** Each repository is automatically isolated with its own index. Switching between repos "just works" ā no manual configuration needed. If the background server is running (`osgrep serve`), search goes through the hot daemon; otherwise it falls back to on-demand indexing.
-4. **Trace** (Call Graph)
-
- ```bash
- osgrep trace "function_name"
- ```
-See who calls a function (upstream dependencies) and what it calls (downstream dependencies). Perfect for impact analysis and understanding code flow.
-
-To find the symbols in your code base:
- ```bash
- osgrep symbols
- ```
-
-In our public benchmarks, `osgrep` can save about 20% of your LLM tokens and deliver a 30% speedup.
-
-
-
-