Skip to content

Commit b59203e

Browse files
cbh778899cbh778899
authored
Add CLI (#60)
* add CLI Signed-off-by: cbh778899 <[email protected]> * remove setup executable Signed-off-by: cbh778899 <[email protected]> * fix environment variable not set bug Signed-off-by: cbh778899 <[email protected]> * add setup executable Signed-off-by: cbh778899 <[email protected]> * add readme for CLI Signed-off-by: cbh778899 <[email protected]> --------- Signed-off-by: cbh778899 <[email protected]>
1 parent c01ff0d commit b59203e

13 files changed

+1496
-15
lines changed

‎.dockerignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,6 @@ eslint.config.mjs
1212
LICENSE
1313
volumes
1414
docker-compose*
15-
Makefile
15+
Makefile
16+
setup
17+
generate_production_env.html

‎.env

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
APP_PORT=8000
2+
APP_EXPOSE_PORT=8000
23
ENG_ACCESS_PORT=8080
34
MODEL_SAVE_PATH=volumes/models
45
INFERENCE_ENG=llamacpp
56
INFERENCE_ENG_PORT=8080
6-
INFERENCE_ENG_VERSION=server--b1-2321a5e
7+
INFERENCE_ENG_VERSION=server--b1-27d4b7c
78
NUM_CPU_CORES=8.00
8-
NUM_THREADS_COUNT=8
9+
NUM_THREADS_COUNT=8.00
910
EMBEDDING_ENG=embedding_eng
1011
EMBEDDING_ENG_PORT=8081
1112
NUM_CPU_CORES_EMBEDDING=4.00
13+
NUM_THREAD_COUNTS_EMBEDDING=4.00
1214
LANGUAGE_MODEL_NAME=Phi3-mini-4k-instruct-Q4.gguf
13-
LANGUAGE_MODEL_URL=https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf?download=true
1415
EMBEDDING_MODEL_NAME=all-MiniLM-L6-v2-Q4_K_M-v2.gguf
15-
EMBEDDING_MODEL_URL=https://huggingface.co/aisuko/all-MiniLM-L6-v2-gguf/resolve/main/all-MiniLM-L6-v2-Q4_K_M-v2.gguf?download=true

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
node_modules
22
.git
33
volumes
4-
__pycache__
4+
__pycache__
5+
setup/setup

‎Makefile

+18-8
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
# project related
22
PROJECT_NAME:=voyager
3-
CONTAINER_NAME:=voyager:v0.1.0
3+
CONTAINER_NAME:=voyager:v0.2.0
44
APP_PORT:=8000
5+
APP_EXPOSE_PORT:=8000
56
# compose build related
67
ENV_FILE:=.env
78

@@ -10,15 +11,16 @@ MODEL_SAVE_PATH:=volumes/models
1011

1112
INFERENCE_ENG:=llamacpp
1213
INFERENCE_ENG_PORT:=8080
13-
INFERENCE_ENG_VERSION:=server--b1-2321a5e
14+
INFERENCE_ENG_VERSION:=server--b1-27d4b7c
1415
NUM_CPU_CORES:=8.00
15-
NUM_THREADS_COUNT:=8
16+
NUM_THREADS_COUNT:=8.00
1617

1718
EMBEDDING_ENG:=embedding_eng
1819
EMBEDDING_ENG_PORT:=8081
1920
NUM_CPU_CORES_EMBEDDING:=4.00
20-
LANGUAGE_MODEL_NAME:=Phi3-mini-4k-instruct-Q4.gguf
21-
LANGUAGE_MODEL_URL:=https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf?download=true
21+
NUM_THREAD_COUNTS_EMBEDDING:=4.00
22+
LANGUAGE_MODEL_NAME:=ft-smollm-135M-instruct-on-hf-ultrafeedback-f16.gguf
23+
LANGUAGE_MODEL_URL:=https://huggingface.co/aisuko/ft-smollm-135M-instruct-on-hf-ultrafeedback-gguf/resolve/main/ft-smollm-135M-instruct-on-hf-ultrafeedback-f16.gguf
2224
EMBEDDING_MODEL_NAME:=all-MiniLM-L6-v2-Q4_K_M-v2.gguf
2325
EMBEDDING_MODEL_URL:=https://huggingface.co/aisuko/all-MiniLM-L6-v2-gguf/resolve/main/all-MiniLM-L6-v2-Q4_K_M-v2.gguf?download=true
2426

@@ -35,6 +37,7 @@ run: build
3537
.PHONY: env
3638
env:
3739
@echo "APP_PORT=$(APP_PORT)"> $(ENV_FILE)
40+
@echo "APP_EXPOSE_PORT=$(APP_EXPOSE_PORT)">> $(ENV_FILE)
3841
@echo "ENG_ACCESS_PORT=$(ENG_ACCESS_PORT)">> $(ENV_FILE)
3942
@echo "MODEL_SAVE_PATH=$(MODEL_SAVE_PATH)">> $(ENV_FILE)
4043
@echo "INFERENCE_ENG=$(INFERENCE_ENG)">> $(ENV_FILE)
@@ -45,10 +48,9 @@ env:
4548
@echo "EMBEDDING_ENG=$(EMBEDDING_ENG)">> $(ENV_FILE)
4649
@echo "EMBEDDING_ENG_PORT=$(EMBEDDING_ENG_PORT)">> $(ENV_FILE)
4750
@echo "NUM_CPU_CORES_EMBEDDING=$(NUM_CPU_CORES_EMBEDDING)">> $(ENV_FILE)
51+
@echo "NUM_THREAD_COUNTS_EMBEDDING=$(NUM_THREAD_COUNTS_EMBEDDING)">> $(ENV_FILE)
4852
@echo "LANGUAGE_MODEL_NAME=$(LANGUAGE_MODEL_NAME)">> $(ENV_FILE)
49-
@echo "LANGUAGE_MODEL_URL=$(LANGUAGE_MODEL_URL)">> $(ENV_FILE)
5053
@echo "EMBEDDING_MODEL_NAME=$(EMBEDDING_MODEL_NAME)">> $(ENV_FILE)
51-
@echo "EMBEDDING_MODEL_URL=$(EMBEDDING_MODEL_URL)">> $(ENV_FILE)
5254

5355
.PHONY: model-prepare
5456
model-prepare:
@@ -83,4 +85,12 @@ stop:
8385

8486
.PHONY: pytest
8587
pytest:
86-
@python3 -m pytest -v
88+
@python3 -m pytest -v
89+
90+
#########################################################################################
91+
# setup
92+
93+
.PHONY: setup
94+
setup:
95+
gcc setup/setup.c -o setup/setup
96+
setup/setup

‎README.md

+12
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,18 @@ The project is OpenAI-like API service of SkywardAI ecosystem.
1313
https://github.com/user-attachments/assets/2b8f1ea7-0aca-44ea-b218-eff8e1769729
1414

1515

16+
### CLI 💥
17+
Introducing our new CLI tool!
18+
> Make sure you can normally run `make`, `docker compose`, `gcc`, `sh` in your host machine.
19+
20+
Simply run `make setup` in the root folder to compile & run the CLI tool.
21+
22+
Don't want to set? Directly go to `Save & Build` menu in it and use `Build and start the server` option to load the app in default settings.
23+
24+
No gcc compiler? You can choose to compile the file `/setup/setup.c` yourself.
25+
26+
Explore it yourself to find more settings!
27+
1628

1729
### Local Machine
1830
* Please make sure you installed `Node.js` on your local machine.

‎database/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ export async function initDB(force = false) {
4242
// create or re-open api key table
4343
await db.createEmptyTable(API_KEY_TABLE, new Schema([
4444
new Field("api_key", new Utf8()),
45-
new Field("usage", new Int32()),
45+
new Field("usage", new Int32())
4646
]), open_options);
4747
}
4848

‎docker-compose-adv.yaml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
services:
2+
llamacpp:
3+
container_name: ${INFERENCE_ENG}
4+
image: gclub/llama.cpp:${INFERENCE_ENG_VERSION}
5+
restart: always
6+
deploy: # https://github.com/compose-spec/compose-spec/blob/master/deploy.md
7+
resources:
8+
reservations:
9+
cpus: "${NUM_CPU_CORES}"
10+
volumes:
11+
- "${DOCKER_VOLUME_DIRECTORY:-.}/${MODEL_SAVE_PATH}:/models"
12+
expose:
13+
- ${ENG_ACCESS_PORT}
14+
ports:
15+
- ${INFERENCE_ENG_PORT}:${ENG_ACCESS_PORT}
16+
command: ["-m", "models/${LANGUAGE_MODEL_NAME}","-t","${NUM_THREADS_COUNT}","-c","8192"]
17+
18+
embedding_eng:
19+
container_name: ${EMBEDDING_ENG}
20+
image: gclub/llama.cpp:${INFERENCE_ENG_VERSION}
21+
restart: always
22+
deploy: # https://github.com/compose-spec/compose-spec/blob/master/deploy.md
23+
resources:
24+
reservations:
25+
cpus: "${NUM_CPU_CORES_EMBEDDING}"
26+
volumes:
27+
- "${DOCKER_VOLUME_DIRECTORY:-.}/${MODEL_SAVE_PATH}:/models"
28+
expose:
29+
- ${ENG_ACCESS_PORT}
30+
ports:
31+
- ${EMBEDDING_ENG_PORT}:${ENG_ACCESS_PORT}
32+
command: ["-m", "models/${EMBEDDING_MODEL_NAME}","--embeddings","--pooling","mean","-t","${NUM_THREAD_COUNTS_EMBEDDING}","-c","512"]
33+
34+
voyager:
35+
container_name: voyager
36+
restart: always
37+
build:
38+
dockerfile: setup/Dockerfile
39+
context: .
40+
expose:
41+
- ${APP_EXPOSE_PORT}
42+
ports:
43+
- ${APP_EXPOSE_PORT}:${APP_PORT}
44+
depends_on:
45+
- llamacpp
46+
- embedding_eng

‎setup/Dockerfile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM node:20.15.1-slim
2+
WORKDIR /app
3+
COPY . .
4+
5+
HEALTHCHECK --interval=300s --timeout=30s --start-period=5s --retries=3 CMD [ "node", "healthy-check.js" ]
6+
RUN npm install -g pnpm && pnpm install
7+
ENTRYPOINT [ "npm", "start" ]

‎setup/config.h

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#include "setup_types.h"
2+
3+
#ifndef SETUP_CONFIG_H
4+
#define SETUP_CONFIG_H
5+
6+
#define INFERENCE_MODEL_NAME "Phi3-mini-4k-instruct-Q4.gguf"
7+
#define INFERENCE_MODEL_URL "https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf"
8+
#define INFERENCE_CPU_CORES 8.00
9+
#define INFERENCE_THREAD_COUNTS 8.00
10+
11+
#define EMBEDDING_CPU_CORES 4.00
12+
#define EMBEDDING_THREAD_COUNTS 4.00
13+
14+
#define ALLOW_ORIGIN_NAME "*"
15+
16+
#define HTTPS_ENABLED 0
17+
#define HTTPS_CERT_PATH_HOST "*"
18+
#define HTTPS_CERT_PATH_CONTAINER "*"
19+
#define HTTPS_CERT_NAME "cert.pem"
20+
#define HTTPS_PRIVKEY_NAME "privkey.pem"
21+
#define HTTPS_CA_NAME "chain.pem"
22+
#define APP_EXPOSE_PORT "8000"
23+
24+
#define PLUGIN_ENABLED 0
25+
26+
#define SYSTEM_INSTRUCTION "*"
27+
28+
#define STATIC_API_KEY_ENABLED 0
29+
#define STATIC_API_KEY "*"
30+
31+
#define DEFAULT_DATASET_ENABLED 0
32+
#define DEFAULT_DATASET_NAME "production_dataset"
33+
34+
#define API_INDEX_DOC_ENABLED 1
35+
#define API_INDEX_STATS_ENABLED 1
36+
#define API_INDEX_HEALTHY_ENABLED 1
37+
#define API_INFERENCE_COMP_ENABLED 1
38+
#define API_INFERENCE_RAG_ENABLED 1
39+
#define API_TOKEN_ENABLED 1
40+
#define API_EMBEDDING_CALC_ENABLED 1
41+
#define API_EMBEDDING_DS_ENABLED 1
42+
#define API_VERSION_ENABLED 1
43+
44+
#define DEV_MODE_ENABLED 0
45+
46+
#endif

‎setup/default_config.h

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#include "setup_types.h"
2+
3+
#ifndef SETUP_DEFAULT_CONFIG_H
4+
#define SETUP_DEFAULT_CONFIG_H
5+
6+
#define DEFAULT_INFERENCE_MODEL_NAME MODEL_PHI_NAME
7+
#define DEFAULT_INFERENCE_MODEL_URL MODEL_PHI_URL
8+
#define DEFAULT_INFERENCE_CPU_CORES 8.00
9+
#define DEFAULT_INFERENCE_THREAD_COUNTS 8.00
10+
11+
#define DEFAULT_EMBEDDING_CPU_CORES 4.00
12+
#define DEFAULT_EMBEDDING_THREAD_COUNTS 4.00
13+
14+
#define DEFAULT_ALLOW_ORIGIN_NAME "*"
15+
16+
#define DEFAULT_HTTPS_ENABLED 0
17+
#define DEFAULT_HTTPS_CERT_PATH_HOST "*"
18+
#define DEFAULT_HTTPS_CERT_PATH_CONTAINER "*"
19+
#define DEFAULT_HTTPS_CERT_NAME "cert.pem"
20+
#define DEFAULT_HTTPS_PRIVKEY_NAME "privkey.pem"
21+
#define DEFAULT_HTTPS_CA_NAME "chain.pem"
22+
#define DEFAULT_APP_EXPOSE_PORT "8000"
23+
24+
#define DEFAULT_PLUGIN_ENABLED 0
25+
26+
#define DEFAULT_SYSTEM_INSTRUCTION "*"
27+
28+
#define DEFAULT_STATIC_API_KEY_ENABLED 0
29+
#define DEFAULT_STATIC_API_KEY "*"
30+
31+
#define DEFAULT_DEFAULT_DATASET_ENABLED 0
32+
#define DEFAULT_DEFAULT_DATASET_NAME "production_dataset"
33+
34+
#define DEFAULT_API_INDEX_DOC_ENABLED 1
35+
#define DEFAULT_API_INDEX_STATS_ENABLED 1
36+
#define DEFAULT_API_INDEX_HEALTHY_ENABLED 1
37+
#define DEFAULT_API_INFERENCE_COMP_ENABLED 1
38+
#define DEFAULT_API_INFERENCE_RAG_ENABLED 1
39+
#define DEFAULT_API_TOKEN_ENABLED 1
40+
#define DEFAULT_API_EMBEDDING_CALC_ENABLED 1
41+
#define DEFAULT_API_EMBEDDING_DS_ENABLED 1
42+
#define DEFAULT_API_VERSION_ENABLED 1
43+
44+
#define DEFAULT_DEV_MODE_ENABLED 0
45+
46+
#endif

0 commit comments

Comments
 (0)