Workflow files for static checking

.github/workflows/static-checks.yml

@@ -0,0 +1,71 @@
+name: Static Checks
+
+on: # Defines when the action should occur
+  push: # Will be triggered on every `push` event
+    branches: [ "master", "main" ]  # The branches that the `push` events have to happen on
+  pull_request: # Will be triggered on every `pull_request` event
+    branches: [ "master", "main" ]  # The branches that the `pull_request` events have to happen on
+
+env:
+  CARGO_TERM_COLOR: always  # This environment variable makes sure Cargo's output is always colored
+
+jobs:
+  static-checks-windows-linux: # The name of the job
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+    runs-on: ${{ matrix.os }}  # The type of machine to run the job on
+
+    steps: # The steps of the job
+      - uses: actions/checkout@v3  # Checks-out your repository under `$GITHUB_WORKSPACE`
+
+      - name: Set up Rust  # Sets up a Rust environment
+        uses: actions-rs/toolchain@v1  # GitHub action to install the Rust toolchain
+        with:
+          profile: minimal  # The type of installation to perform
+          toolchain: 1.72.0  # The version of the toolchain to install
+          override: true  # Whether to install the toolchain as the default
+      - name: Install clippy  # Install clippy for the repo
+        run: rustup component add clippy  # The command to install clippy
+
+      # Caching
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Build  # Build the project
+        run: cargo build --verbose  # The command to build the project
+
+      - name: Run tests  # Run the tests for the project
+        run: cargo test --verbose  # The command to run the tests for the project
+
+      - name: Run clippy  # Run clippy on the project
+        run: cargo clippy -- -D warnings  # The command to run clippy and fail on warnings
+
+  static-checks-macos:
+    strategy:
+      matrix:
+        args: ['', '-F metal']
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.72.0
+          override: true
+
+      - name: Install clippy
+        run: rustup component add clippy
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Build
+        run: cargo build --verbose ${{ matrix.args }}
+
+      - name: Run tests
+        run: cargo test --verbose ${{ matrix.args }}
+
+      - name: Run clippy
+        run: cargo clippy -- -D warnings

.gitignore

@@ -5,4 +5,5 @@ python/env/
 Cargo.lock
 .vscode
 models
-.env
+.env
+.idea

README.md

@@ -7,9 +7,14 @@ The llama2 model **runs on GPU** using `ggml-sys` crate with specific compilatio
 
 ### Using Docker (recommended way)
 
-The easiest way of getting started is using the official Docker container. Make sure you have `docker` and `docker-compose` installed on your machine (example install for [ubuntu20.04](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-20-04)).
+The easiest way of getting started is using the official Docker container. Make sure you have `docker`
+and `docker-compose` installed on your machine (example install
+for [ubuntu20.04](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-20-04)).
 
-`cria` provides two docker images : one for CPU only deployments and a second GPU accelerated image. To use GPU image, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.7 or higher.
+`cria` provides two docker images : one for CPU only deployments and a second GPU accelerated image. To use GPU image,
+you need to install
+the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).
+We also recommend using NVIDIA drivers with CUDA version 11.7 or higher.
 
 To deploy the `cria` gpu version using `docker-compose`:
 
@@ -20,7 +25,9 @@ git clone git@github.com:AmineDiro/cria.git
 cd cria/docker
 ```
 
-2. The api will load the model located in `/app/model.bin` by default. You should change the docker-compose file with ggml model path for docker to bind mount. You can also change environement variables for your specific config. Alternatively, the easiest way is to set `CRIA_MODEL_PATH` in a`docker/.env` :
+2. The api will load the model located in `/app/model.bin` by default. You should change the docker-compose file with
+   ggml model path for docker to bind mount. You can also change environement variables for your specific config.
+   Alternatively, the easiest way is to set `CRIA_MODEL_PATH` in a`docker/.env` :
 
 ```bash
 # .env
@@ -59,25 +66,27 @@ docker compose up -f docker-compose-gpu.yaml -d
    cargo b --release
    ```
 
-   - For `cuBLAS` (nvidia GPU ) acceleration use
-     ```bash
-     cargo b --release --features cublas
-     ```
-   - For `metal` acceleration use
-     ```bash
-     cargo b --release --features metal
-     ```
-     > ❗ NOTE: If you have issues building for GPU, checkout the building issues section
-
-3. Download GGML `.bin` LLama-2 quantized model (for example [llama-2-7b](https://huggingface.co/TheBloke/Llama-2-7B-GGML/tree/main))
+    - For `cuBLAS` (nvidia GPU ) acceleration use
+      ```bash
+      cargo b --release --features cublas
+      ```
+    - For `metal` acceleration use
+      ```bash
+      cargo b --release --features metal
+      ```
+      > ❗ NOTE: If you have issues building for GPU, checkout the building issues section
+
+3. Download GGML `.bin` LLama-2 quantized model (for
+   example [llama-2-7b](https://huggingface.co/TheBloke/Llama-2-7B-GGML/tree/main))
 4. Run API, use the `use-gpu` flag to offload model layers to your GPU
    ```bash
    ./target/cria -a llama --model {MODEL_BIN_PATH} --use-gpu --gpu-layers 32
    ```
 
 ## Command line arguments reference
 
-All the parameters can be passed as environment variables or command line arguments. Here is the reference for the command line arguments:
+All the parameters can be passed as environment variables or command line arguments. Here is the reference for the
+command line arguments:
 
 ```bash
 ./target/cria --help
@@ -102,7 +111,8 @@ Options:
   -h, --help                                         Print help
 ```
 
-For environment variables, just prefix the argument with `CRIA_` and use uppercase letters. For example, to set the model path, you can use `CRIA_MODEL` environment variable.
+For environment variables, just prefix the argument with `CRIA_` and use uppercase letters. For example, to set the
+model path, you can use `CRIA_MODEL` environment variable.
 
 There is a an example `docker/.env.sample` file in the project root directory.
 
@@ -139,33 +149,32 @@ import urllib3
 
 url = "http://localhost:3000/v1/completions"
 
-
 http = urllib3.PoolManager()
 response = http.request(
-    "POST",
-    url,
-    preload_content=False,
-    headers={
-        "Content-Type": "application/json",
-    },
-    body=json.dumps(
-        {
-            "prompt": "Morocco is a beautiful country situated in north africa.",
-            "temperature": 0.1,
-        }
-    ),
+	"POST",
+	url,
+	preload_content=False,
+	headers={
+		"Content-Type": "application/json",
+	},
+	body=json.dumps(
+		{
+			"prompt": "Morocco is a beautiful country situated in north africa.",
+			"temperature": 0.1,
+		}
+	),
 )
 
 client = sseclient.SSEClient(response)
 
 s = time.perf_counter()
 for event in client.events():
-    chunk = json.loads(event.data)
-    sys.stdout.write(chunk["choices"][0]["text"])
-    sys.stdout.flush()
+	chunk = json.loads(event.data)
+	sys.stdout.write(chunk["choices"][0]["text"])
+	sys.stdout.flush()
 e = time.perf_counter()
 
-print(f"\nGeneration from completion took {e-s:.2f} !")
+print(f"\nGeneration from completion took {e - s:.2f} !")
 
 ```
 
@@ -193,9 +202,9 @@ You can clearly see generation using my M1 GPU:
 - [x] Docker deployment on CPUs / GPU
 - [x] Metrics : Prometheus
 - [x] Implement a global request queue
-  - [x] For each response put an entry in a queue
-  - [x] Spawn a model in separate task reading from ringbuffer, get entry and put each token in response
-  - [x] Construct stream from flume resp_rx chan and stream responses to user.
+    - [x] For each response put an entry in a queue
+    - [x] Spawn a model in separate task reading from ringbuffer, get entry and put each token in response
+    - [x] Construct stream from flume resp_rx chan and stream responses to user.
 - [ ] BETTER ERRORS and http responses (deal with all the unwrapping)
 - [ ] Implement streaming chat completions SSE
 - [ ] Implement request batching

src/defaults.rs

@@ -1,37 +1,37 @@
 // llm defaults
-pub fn default_max_tokens() -> usize {
+pub const fn default_max_tokens() -> usize {
     256
 }
-pub fn default_temperature() -> f32 {
+pub const fn default_temperature() -> f32 {
     0.8
 }
-pub fn default_top_p() -> f32 {
+pub const fn default_top_p() -> f32 {
     0.95
 }
-pub fn default_stream() -> bool {
+pub const fn default_stream() -> bool {
     false
 }
-pub fn default_top_k() -> usize {
+pub const fn default_top_k() -> usize {
     40
 }
-pub fn default_repeat_penalty() -> f32 {
+pub const fn default_repeat_penalty() -> f32 {
     1.1
 }
-pub fn default_presence_penalty() -> f32 {
+pub const fn default_presence_penalty() -> f32 {
     0.0
 }
-pub fn default_frequence_penalty() -> f32 {
+pub const fn default_frequence_penalty() -> f32 {
     0.0
 }
-pub fn default_microstat_mode() -> usize {
+pub const fn default_microstat_mode() -> usize {
     0
 }
-pub fn default_microstat_tau() -> f32 {
+pub const fn default_microstat_tau() -> f32 {
     5.0
 }
-pub fn default_microstat_eta() -> f32 {
+pub const fn default_microstat_eta() -> f32 {
     0.1
 }
-pub fn default_echo() -> bool {
+pub const fn default_echo() -> bool {
     false
 }

src/inferer.rs

@@ -16,7 +16,7 @@ use crate::routes::embeddings::Embedding;
 use crate::routes::embeddings::EmbeddingRequest;
 
 fn stream_completion(
-    model: &Box<dyn Model>,
+    model: &dyn Model,
     request: CompletionRequest,
     request_tx: Sender<Result<StreamingResponse, InferenceError>>,
 ) {
@@ -41,7 +41,7 @@ fn stream_completion(
     if !prompt.is_empty() {
         session
             .feed_prompt(
-                model.as_ref(),
+                model,
                 llm::Prompt::Text(&prompt),
                 &mut Default::default(),
                 feed_prompt_callback::<_>(|r| match r {
@@ -66,7 +66,7 @@ fn stream_completion(
 
     while tokens_processed < maximum_token_count {
         let token = match session.infer_next_token(
-            model.as_ref(),
+            model,
             &llm::InferenceParameters {
                 sampler: sampler.clone(),
             },
@@ -97,7 +97,7 @@ fn stream_completion(
 }
 
 fn embed_string(
-    model: &Box<dyn Model>,
+    model: &dyn Model,
     session: &mut InferenceSession,
     vocab: &Tokenizer,
     query: &str,
@@ -122,16 +122,16 @@ fn embed_string(
 }
 
 fn stream_embedding(
-    model: &Box<dyn Model>,
+    model: &dyn Model,
     request: EmbeddingRequest,
     request_tx: Sender<Result<Embedding, InferenceError>>,
 ) {
     let mut session = model.start_session(Default::default());
     let vocab = model.tokenizer();
 
     for input in request.input {
-        let embd = embed_string(&model, &mut session, &vocab, &input);
-        let _res = request_tx
+        let embd = embed_string(model, &mut session, vocab, &input);
+        request_tx
             .send_timeout(Ok(embd), Duration::from_millis(10))
             .unwrap();
     }
@@ -143,13 +143,13 @@ pub fn inference_loop(model: Box<dyn Model>, rx_queue: Receiver<InferenceEvent>)
     while let Ok(inference_request) = rx_queue.recv() {
         match inference_request {
             InferenceEvent::CompletionEvent(request, request_tx) => {
-                stream_completion(&model, request, request_tx)
+                stream_completion(model.as_ref(), request, request_tx)
             }
             InferenceEvent::EmbeddingEvent(request, request_tx) => {
-                stream_embedding(&model, request, request_tx)
+                stream_embedding(model.as_ref(), request, request_tx)
             }
             InferenceEvent::ChatEvent(request, request_tx) => {
-                chat_completion(&model, request, request_tx)
+                chat_completion(model.as_ref(), request, request_tx)
             }
         }
     }

src/lib.rs

@@ -12,7 +12,6 @@ use std::convert::Infallible;
 use std::fmt;
 use std::marker::PhantomData;
 pub mod defaults;
-use defaults::*;
 pub mod config;
 use config::Config;
 
@@ -61,7 +60,7 @@ pub async fn run_webserver(config: Config) {
     let queue = RequestQueue::new(tx);
 
     tokio::task::spawn_blocking(move || {
-        let _ = inference_loop(model, rx);
+        inference_loop(model, rx);
     });
 
     tracing::info!(
@@ -72,7 +71,7 @@ pub async fn run_webserver(config: Config) {
     );
 
     let model_list = ModelList {
-        models: ["llama-2".into()],
+        models: [model_architecture.to_string()],
     };
 
     let app = Router::new()