Merge pull request #3 from AstraBert/chore/add-tests

chore: tests and CI

Author: AstraBert

.github/workflows/build.yaml

@@ -0,0 +1,42 @@
+name: CI - Build
+
+on:
+  pull_request:
+    paths:
+      - src/*.rs
+      - Cargo.toml
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Cache cargo index
+        uses: actions/cache@v4
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-cargo-git-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Cache cargo build
+        uses: actions/cache@v4
+        with:
+          path: target
+          key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Build
+        run: make build

.github/workflows/lint.yaml

@@ -0,0 +1,52 @@
+name: Lint
+
+on:
+  pull_request:
+    paths:
+      - src/*.rs
+      - Cargo.toml
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install protoc
+        uses: taiki-e/install-action@protoc
+
+      - name: Run clippy
+        run: make clippy
+
+  format:
+    name: Format
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt
+
+      - name: Check formatting
+        run: make format-check

.github/workflows/publish.yaml

@@ -0,0 +1,45 @@
+name: Publish
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read # for checkout
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # to be able to publish a GitHub release
+      issues: write # to be able to comment on released issues
+      pull-requests: write # to be able to comment on released pull requests
+      id-token: write # to enable use of OIDC for npm provenance
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Login to cargo
+        run: cargo login <<< $CARGO_API_TOKEN
+        env:
+          CARGO_API_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
+
+      - name: Publish to crates
+        run: cargo publish
+        env:
+          CARGO_API_TOKEN: ${{ secrets.CARGO_API_TOKEN }}

.github/workflows/test.yaml

@@ -0,0 +1,45 @@
+name: Test
+
+on:
+  pull_request:
+    paths:
+      - src/*.rs
+      - Cargo.toml
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  test:
+    name: Test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        rust: [stable]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ matrix.rust }}
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-
+
+      - name: Install protoc
+        uses: taiki-e/install-action@protoc
+
+      - name: Run tests
+        run: make test

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "rag-rs"
-version = "0.2.0-alpha"
+version = "0.2.1"
 edition = "2024"
 license-file = "LICENSE"
 readme = "README.md"
@@ -28,3 +28,4 @@ tower-http = {version = "0.6.2", features = ["fs", "cors"]}
 async-openai = { version = "0.32.3", features = ["responses", "chat-completion"] }
 pdf-extract = "0.10.0"
 cacache = { version = "13.1.0", features = ["tokio-runtime", "mmap"], default-features = false }
+tower = "0.5.3"

Cargo.toml

@@ -0,0 +1,33 @@
+.PHONY: build test clippy clippy-fix format format-check version-bump npm-publish
+
+build:
+	cargo build --target-dir target/
+
+test:
+	$(info ****************** running tests ******************)
+	rm -rf .rag-rs-cache/ && cargo test
+
+clippy:
+	$(info ****************** running clippy in check mode ******************)
+	cargo clippy
+
+clippy-fix:
+	$(info ****************** running clippy in fix mode ******************)
+	cargo clippy --fix --bin "rag-rs"
+
+format:
+	$(info ****************** running rustfmt in fix mode ******************)
+	cargo fmt
+
+format-check:
+	$(info ****************** running rustfmt in check mode ******************)
+	cargo fmt --check
+
+version-bump:
+	$(info ****************** bumping version in package.json and Cargo.toml ******************)
+	python3 scripts/version_bump.py
+
+npm-publish:
+	$(info ****************** login and publish to npm ******************)
+	$(info ****************** meant for manual usage ******************)
+	bash scripts/login_and_publish_to_npm.sh

Makefile

@@ -27,6 +27,12 @@ Install with cargo:
 cargo install rag-rs
 ```
 
+Install with npm:
+
+```bash
+npm install @cle-does-things/rag-rs@latest
+```
+
 ### `load` command
 
 Parse, chunk and embed the documents in a given directory, and upload them to a vector store.
@@ -125,7 +131,10 @@ rag-rs --qdrant-url http://localhost:6334 \
 To reach the first stable version, this software will first:
 
 - [X] Add a caching layer (v0.2.0-alpha)
-- [ ] Introduce thorough testing
-- [ ] Add a programmatic API along with the CLI app, possibly both in Rust and Python
-- [ ] Add an NPM-installable version
-- [ ] Add support for more text-based file formats, and possibly for more unstructured file formats
+- [X] Introduce thorough testing (v0.2.1)
+- [X] Add an NPM-installable version (v0.2.1)
+
+Moreover, for future releases, there will be:
+
+- [ ] A programmatic API along with the CLI app, possibly both in Rust and Python
+- [ ] Support for more text-based file formats, and possibly for more unstructured file formats

README.md

@@ -0,0 +1,38 @@
+{
+  "name": "@cle-does-things/rag-rs",
+  "version": "0.2.1",
+  "description": "A Rust-native implementation of the RAG stack",
+  "main": "start.js",
+  "directories": {
+    "test": "tests"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "postinstall": "node ./pre-install.js",
+    "uninstall": "node ./uninstall.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/AstraBert/rag-rs.git"
+  },
+  "keywords": [
+    "rag",
+    "llm",
+    "ai",
+    "rust"
+  ],
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/AstraBert/rag-rs/issues"
+  },
+  "files": [
+    "pre-install.js",
+    "start.js",
+    "uninstall.js",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}

package.json

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const path = require("path");
+const { exec } = require("child_process");
+const { homedir } = require("os");
+
+const cargoDir = path.join(homedir(), ".cargo");
+
+// check if directory exists
+if (fs.existsSync(cargoDir)) {
+  //   console.log("Cargo found.");
+} else {
+  const setCargo = 'PATH="/$HOME/.cargo/bin:${PATH}"';
+  console.log("Installing deps [cargo].");
+
+  exec(
+    `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && ${setCargo}`,
+    (error) => {
+      if (error) {
+        console.log(
+          "curl failed! Curl may not be installed on the OS. View https://curl.se/download.html to install.",
+        );
+        console.log(error);
+      }
+    },
+  );
+}
+
+const features = process.env.npm_config_features
+  ? `--features ${process.env.npm_config_features.replace(",", " ")}`
+  : "";
+
+console.log(`Installing and compiling rag-rs 0.2.1 ${features} ...`);
+exec(
+  `cargo install rag-rs --vers 0.2.1 ${features}`,
+  (error, stdout, stderr) => {
+    console.log(stdout);
+    if (error || stderr) {
+      console.log(error || stderr);
+    } else {
+      console.log("install finished!");
+    }
+  },
+);