Skip to content

Commit 07662b8

Browse files
authored
Merge pull request #15 from m5stack/dev
Dev
2 parents 39e9fa8 + 6e503a1 commit 07662b8

File tree

99 files changed

+5291
-894
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

99 files changed

+5291
-894
lines changed

.clang-format

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,5 +163,4 @@ StatementMacros:
163163
- QT_REQUIRE_VERSION
164164
TabWidth: 4
165165
UseCRLF: false
166-
UseTab: Never
167-
...
166+
UseTab: Never

.github/workflows/benchmark.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: Benchmark Test
2+
on:
3+
workflow_dispatch:
4+
push:
5+
branches:
6+
- dev
7+
jobs:
8+
build:
9+
runs-on: [self-hosted, linux, arm64]
10+
steps:
11+
- name: Checkout code
12+
uses: actions/checkout@v4
13+
14+
- name: Start Benchmark Test
15+
run: |
16+
echo "This job runs on a self-hosted runner!"
17+
echo "Running benchmark test..."
18+
python3 benchmark/benchmodulellm.py

benchmark/README.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
benchmodulellm can be used to test llm unit inference performance
2+
3+
Only the llm unit definition files (model json) are required.
4+
5+
If no model specified, it would benchmark default list. More model networks may be added later.
6+
7+
Usage
8+
```shell
9+
python benchmodulellm.py --host 192.168.20.100 --port 10001 --test-items default.yaml
10+
```

benchmark/RESULTS.md

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Results
2+
3+
## ModuleLLM (AX630C)
4+
5+
### LLM
6+
| model | ttft (ms) | avg-token/s | model version | llm version |
7+
|---------------------------------|------------|-------------|---------------|-------------|
8+
| qwen2.5-0.5B-prefill-20e | 359.8 | 10.32 | v0.2 | v1.8 |
9+
| qwen2.5-0.5B-p256-ax630c | 1126.19 | 10.30 | v0.4 | v1.8 |
10+
| qwen2.5-0.5B-Int4-ax630c | 442.95 | 12.52 | v0.4 | v1.8 |
11+
| qwen2.5-coder-0.5B-ax630c | 361.81 | 10.28 | v0.2 | v1.8 |
12+
| qwen2.5-1.5B-ax630c | 1029.41 | 3.59 | v0.3 | v1.8 |
13+
| qwen2.5-1.5B-p256-ax630c | 3056.54 | 3.57 | v0.4 | v1.8 |
14+
| qwen2.5-1.5B-Int4-ax630c | 1219.54 | 4.63 | v0.4 | v1.8 |
15+
| deepseek-r1-1.5B-ax630c | 1075.04 | 3.57 | v0.3 | v1.8 |
16+
| deepseek-r1-1.5B-p256-ax630c | 3056.86 | 3.57 | v0.4 | v1.8 |
17+
| llama3.2-1B-prefill-ax630c | 891.00 | 4.48 | v0.2 | v1.8 |
18+
| llama3.2-1B-p256-ax630c | 2601.11 | 4.49 | v0.4 | v1.8 |
19+
| openbuddy-llama3.2-1B-ax630c | 891.02 | 4.52 | v0.2 | v1.8 |
20+
21+
`The input text used by the llm test is "hello!“`
22+
23+
### VLM
24+
| model | ttft (ms) | avg-token/s | image encode (ms) | model version | vlm version |
25+
|---------------------------------|------------|-------------|-------------------|---------------|-------------|
26+
| internvl2.5-1B-364-ax630c | 1117.27 | 10.56 | 1164.61 | v0.4 | v1.7 |
27+
| smolvlm-256M-ax630c | 185.75 | 30.16 | 799.11 | v0.4 | v1.7 |
28+
| smolvlm-500M-ax630c | 365.69 | 13.14 | 838.30 | v0.4 | v1.7 |
29+
30+
`The image encoding test uses a jpg image with a size of 810*1080`
31+
32+
### STT
33+
| model | encode (ms) | avg-decode (ms) | model version | whisper version |
34+
|--------------------|-------------|-----------------|---------------|-----------------|
35+
| whisper-tiny | 248.0 | 32.54 | v0.4 | v1.7 |
36+
| whisper-base | 660.31 | 51.11 | v0.4 | v1.7 |
37+
| whisper-small | 1606.08 | 148.92 | v0.4 | v1.7 |
38+
39+
`The STT test uses a 30-second wav English audio`

benchmark/benchmodulellm.py

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
import argparse
2+
import os
3+
import sys
4+
5+
import yaml
6+
import logging
7+
8+
from pathlib import Path
9+
10+
from utils import LLMClient
11+
12+
FILE = Path(__file__).resolve()
13+
ROOT = FILE.parents[0]
14+
if str(ROOT) not in sys.path:
15+
sys.path.append(str(ROOT))
16+
ROOT = Path(os.path.relpath(ROOT, Path.cwd()))
17+
18+
logging.basicConfig(
19+
level=logging.INFO,
20+
format="%(asctime)s - %(levelname)s - %(message)s",
21+
datefmt="%Y-%m-%d %H:%M:%S",
22+
)
23+
24+
def parse_opt(known=False):
25+
"""
26+
Parse command-line options.
27+
"""
28+
parser = argparse.ArgumentParser()
29+
parser.add_argument("--host", type=str, default="127.0.0.1", help="ModuleLLM IP Address")
30+
parser.add_argument("--port", type=int, default=10001, help="ModuleLLM TCP Port")
31+
parser.add_argument("--test-items", type=str, default=ROOT / "default.yaml", help="testitems.yaml path")
32+
33+
args = parser.parse_known_args()[0] if known else parser.parse_args()
34+
35+
return args
36+
37+
def read_yaml(file_path):
38+
"""
39+
Read a YAML file and return its content.
40+
"""
41+
if not os.path.exists(file_path):
42+
logging.error(f"YAML file '{file_path}' does not exist.")
43+
sys.exit(1)
44+
45+
try:
46+
with open(file_path, "r") as file:
47+
data = yaml.safe_load(file)
48+
if data is None:
49+
logging.warning(f"YAML file '{file_path}' is empty.")
50+
return {}
51+
52+
logging.info(f"YAML file '{file_path}' read successfully.")
53+
54+
if "items" in data:
55+
return data["items"]
56+
else:
57+
logging.warning(f"'items' not found in YAML file.")
58+
return []
59+
except Exception as e:
60+
logging.error(f"Failed to read YAML file '{file_path}': {e}")
61+
sys.exit(1)
62+
63+
def write_yaml(file_path, data):
64+
"""
65+
Write data to a YAML file.
66+
"""
67+
try:
68+
with open(file_path, "w") as file:
69+
yaml.safe_dump(data, file)
70+
logging.info(f"YAML file '{file_path}' written successfully.")
71+
except Exception as e:
72+
logging.error(f"Failed to write YAML file '{file_path}': {e}")
73+
sys.exit(1)
74+
75+
def categorize_and_deduplicate(items):
76+
"""
77+
Categorize items by 'type' and remove duplicate 'model_name'.
78+
"""
79+
categorized = {}
80+
for item in items:
81+
item_type = item.get("type")
82+
model_name = item.get("model_name")
83+
if not item_type or not model_name:
84+
continue
85+
86+
if item_type not in categorized:
87+
categorized[item_type] = set()
88+
89+
categorized[item_type].add(model_name)
90+
91+
# Convert sets back to lists for easier usage
92+
return {key: list(value) for key, value in categorized.items()}
93+
94+
def main(opt):
95+
items = read_yaml(opt.test_items)
96+
if not items:
97+
logging.warning(f"No items found in YAML file '{opt.test_items}'.")
98+
return
99+
100+
categorized_items = categorize_and_deduplicate(items)
101+
102+
logging.info("Categorized items:")
103+
for item_type, models in categorized_items.items():
104+
logging.info(f"Type: {item_type}, Models: {models}")
105+
106+
if item_type == "llm":
107+
logging.info("Initializing LLMClient...")
108+
llm_client = LLMClient(opt.host, opt.port)
109+
110+
for model_name in models:
111+
logging.info(f"Testing model: {model_name}")
112+
input_text = "Tell me an adventure story."
113+
try:
114+
result = llm_client.test(model_name, input_text)
115+
logging.info(f"Test result for model '{model_name}': {result}")
116+
except Exception as e:
117+
logging.error(f"Error testing model '{model_name}': {e}")
118+
119+
del llm_client
120+
logging.info("LLMClient deleted successfully.")
121+
122+
return categorized_items
123+
124+
if __name__ == "__main__":
125+
opt = parse_opt()
126+
main(opt)

benchmark/default.yaml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
items:
2+
- model_name: qwen2.5-0.5B-p256-ax630c
3+
type: llm
4+
- model_name: internvl2.5-1B-364-ax630c
5+
type: vlm
6+
- model_name: whisper-tiny
7+
type: whisper
8+
- model_name: whisper-base
9+
type: whisper
10+
- model_name: whisper-small
11+
type: whisper
12+
- model_name: sherpa-ncnn-streaming-zipformer-20M-2023-02-17
13+
type: asr
14+
- model_name: sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23
15+
type: asr
16+
- model_name: sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01
17+
type: kws
18+
- model_name: sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01
19+
type: kws
20+
- model_name: melotts-zh-cn
21+
type: melotts
22+
- model_name: single_speaker_english_fast
23+
type: tts
24+
- model_name: single_speaker_fast
25+
type: tts
26+
- model_name: yolo11n
27+
type: yolo
28+
- model_name: yolo11n-seg
29+
type: yolo
30+
- model_name: yolo11n-pose
31+
type: yolo

benchmark/utils/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from .llm import LLMClient
2+
3+
__all__ = ["LLMClient"]

0 commit comments

Comments
 (0)