Skip to content

Commit 24048ee

Browse files
asteinhasteinh
authored
Merge pull request #5 from raws-labs/hardening/benchmark-validation
Harden clone-local benchmark validation
2 parents c3aa466 + 34fe2fd commit 24048ee

8 files changed

Lines changed: 508 additions & 125 deletions

File tree

‎.github/requirements-host-validation.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
11
numpy==2.3.5
22
rich==13.7.1
3+
click==8.3.1
4+
lz4==4.4.5
5+
onnx==1.20.1
6+
PyYAML==6.0.3

‎.github/workflows/host-validation.yml‎

Lines changed: 93 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: Host validation
22

3-
# Clone-local by design: no board access, compiler toolchains, or mutable
4-
# third_party/fetch.sh network checkouts are required by this workflow.
3+
# Hardware-free by design: no board access or cross compiler is required.
4+
# Host CMSIS parity uses only immutable, full-SHA source checkouts.
55

66
on:
77
push:
@@ -70,3 +70,94 @@ jobs:
7070
run: >-
7171
python -W error scripts/validate_accuracy.py
7272
results/summary.json
73+
74+
- name: Build and run host CMSIS-NN parity
75+
shell: bash
76+
run: |
77+
set -euo pipefail
78+
host_root="$RUNNER_TEMP/tigris-host-validation"
79+
compiler_revision="$(python -c \
80+
'import json; print(json.load(open("core-versions.json"))["compiler"]["commit"])')"
81+
runtime_revision="$(python -c \
82+
'import json; print(json.load(open("core-versions.json"))["runtime"]["commit"])')"
83+
cmsis_revision="$(python -c \
84+
'import json; print(json.load(open("cortex-m-deployability/results/summary.json"))["provenance"]["common"]["dependencies"]["CMSIS-NN"])')"
85+
86+
checkout_revision() {
87+
local repository="$1" destination="$2" revision="$3"
88+
git init "$destination"
89+
git -C "$destination" remote add origin "$repository"
90+
git -C "$destination" fetch --depth=1 origin "$revision"
91+
git -C "$destination" checkout --detach FETCH_HEAD
92+
test "$(git -C "$destination" rev-parse HEAD)" = "$revision"
93+
}
94+
checkout_revision https://github.com/raws-labs/tigris.git \
95+
"$host_root/compiler" "$compiler_revision"
96+
checkout_revision https://github.com/raws-labs/tigris-runtime.git \
97+
"$host_root/runtime" "$runtime_revision"
98+
checkout_revision https://github.com/ARM-software/CMSIS-NN.git \
99+
"$host_root/CMSIS-NN" "$cmsis_revision"
100+
101+
python -m pip install --no-deps "$host_root/compiler"
102+
HOST_ROOT="$host_root" python - <<'PY'
103+
import os
104+
from pathlib import Path
105+
106+
import numpy as np
107+
import onnx
108+
from onnx import TensorProto, helper, numpy_helper
109+
110+
root = Path(os.environ["HOST_ROOT"])
111+
scalar_f32 = lambda name, value: numpy_helper.from_array(
112+
np.array([value], dtype=np.float32), name)
113+
scalar_i8 = lambda name: numpy_helper.from_array(
114+
np.array([0], dtype=np.int8), name)
115+
inputs = helper.make_tensor_value_info(
116+
"input", TensorProto.FLOAT, [1, 1, 4, 4])
117+
outputs = helper.make_tensor_value_info(
118+
"output", TensorProto.FLOAT, [1, 1, 4, 4])
119+
initializers = [
120+
scalar_f32("input_scale", 0.25), scalar_i8("input_zp"),
121+
numpy_helper.from_array(
122+
np.array([[[[2]]]], dtype=np.int8), "weight"),
123+
scalar_f32("weight_scale", 0.25), scalar_i8("weight_zp"),
124+
scalar_f32("output_scale", 0.25), scalar_i8("output_zp"),
125+
]
126+
nodes = [
127+
helper.make_node(
128+
"QuantizeLinear", ["input", "input_scale", "input_zp"],
129+
["input_q"]),
130+
helper.make_node(
131+
"DequantizeLinear", ["input_q", "input_scale", "input_zp"],
132+
["input_dq"]),
133+
helper.make_node(
134+
"DequantizeLinear", ["weight", "weight_scale", "weight_zp"],
135+
["weight_dq"]),
136+
helper.make_node("Conv", ["input_dq", "weight_dq"], ["raw"]),
137+
helper.make_node(
138+
"QuantizeLinear", ["raw", "output_scale", "output_zp"],
139+
["output_q"]),
140+
helper.make_node(
141+
"DequantizeLinear", ["output_q", "output_scale", "output_zp"],
142+
["output"]),
143+
]
144+
model = helper.make_model(
145+
helper.make_graph(
146+
nodes, "host_cmsis_conv", [inputs], [outputs], initializers),
147+
opset_imports=[helper.make_opsetid("", 13)],
148+
)
149+
model.ir_version = 8
150+
onnx.checker.check_model(model)
151+
onnx.save(model, root / "host_cmsis_conv.onnx")
152+
PY
153+
tigris compile "$host_root/host_cmsis_conv.onnx" \
154+
--mem 4K --output "$host_root/host_cmsis_conv.tgrs"
155+
cmake -S cortex-m-deployability -B "$host_root/build" \
156+
-DCMAKE_BUILD_TYPE=Release \
157+
-DTIGRIS_HOST_VALIDATION=ON \
158+
-DTIGRIS_RUNTIME_ROOT="$host_root/runtime" \
159+
-DCMSIS_NN_DIR="$host_root/CMSIS-NN"
160+
cmake --build "$host_root/build" \
161+
--target host_cmsis_validate --parallel 2
162+
"$host_root/build/host_cmsis_validate" \
163+
"$host_root/host_cmsis_conv.tgrs"

‎cortex-m-deployability/CMakeLists.txt‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ set(TIGRIS_CODEGEN "${CMAKE_CURRENT_SOURCE_DIR}/../../tigris/.venv/bin/tigris"
1515
CACHE FILEPATH "TiGrIS codegen executable")
1616
set(TIGRIS_RUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../tigris-runtime"
1717
CACHE PATH "pinned TiGrIS runtime checkout")
18+
option(TIGRIS_HOST_VALIDATION
19+
"Build the hardware-free CMSIS-NN adapter parity executable" OFF)
1820
# Optimization level for ALL benchmark code (TiGrIS runtime, CMSIS-NN, harness, TFLM
1921
# glue). MUST be consistent across boards and frameworks for a fair comparison: TFLM's
2022
# own kernels build at -O2, and without this the clean-CMake path defaults to -O0 (no
@@ -25,10 +27,36 @@ project(cortexm_bench C CXX ASM)
2527
set(CMAKE_C_STANDARD 11)
2628

2729
# ---- vendored CMSIS locations (populated by third_party/fetch.sh) ----
28-
set(CMSIS_NN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/CMSIS-NN)
29-
set(CMSIS_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/CMSIS_6/CMSIS/Core)
30-
set(CMSIS_DEVICE_H7_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cmsis-device-h7)
31-
set(CMSIS_DEVICE_F4_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cmsis-device-f4)
30+
set(CMSIS_NN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/CMSIS-NN"
31+
CACHE PATH "CMSIS-NN source checkout")
32+
set(CMSIS_CORE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/CMSIS_6/CMSIS/Core"
33+
CACHE PATH "CMSIS-Core source directory")
34+
set(CMSIS_DEVICE_H7_DIR
35+
"${CMAKE_CURRENT_SOURCE_DIR}/third_party/cmsis-device-h7"
36+
CACHE PATH "STM32H7 CMSIS device checkout")
37+
set(CMSIS_DEVICE_F4_DIR
38+
"${CMAKE_CURRENT_SOURCE_DIR}/third_party/cmsis-device-f4"
39+
CACHE PATH "STM32F4 CMSIS device checkout")
40+
41+
if(TIGRIS_HOST_VALIDATION)
42+
if(NOT EXISTS ${TIGRIS_RUNTIME_ROOT}/CMakeLists.txt)
43+
message(FATAL_ERROR
44+
"TiGrIS runtime not found at ${TIGRIS_RUNTIME_ROOT}.\n"
45+
"Pass -DTIGRIS_RUNTIME_ROOT=<checkout>.")
46+
endif()
47+
add_subdirectory(third_party)
48+
add_subdirectory(${TIGRIS_RUNTIME_ROOT}
49+
${CMAKE_BINARY_DIR}/tigris_runtime)
50+
target_compile_definitions(tigris_runtime PUBLIC TIGRIS_HAS_CMSIS_NN)
51+
target_include_directories(tigris_runtime SYSTEM PRIVATE
52+
${CMSIS_NN_DIR}/Include)
53+
add_executable(host_cmsis_validate tests/host_cmsis_validate.c)
54+
target_compile_options(host_cmsis_validate PRIVATE
55+
-Wall -Wextra -Wpedantic -Werror)
56+
target_link_libraries(host_cmsis_validate PRIVATE
57+
tigris_runtime cmsis-nn)
58+
return()
59+
endif()
3260

3361
# ---- board: sets BOARD_MCU_FLAGS, BOARD_STARTUP/SYSTEM/BSP/LINKER_SCRIPT,
3462
# BOARD_DEFINES, BOARD_INCLUDES ----

‎cortex-m-deployability/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ int8 weights plus biases, measured from the committed plans.
4747

4848
| Framework | Kernel | Latency | Cycles | RAM (work. set) | Flash (firmware) |
4949
|---|---|---|---|---|---|
50-
| TiGrIS | cmsis_nn | 11.14 ms | 5.35 M | 25.5 KB | 118 KB |
50+
| TiGrIS | cmsis_nn | 11.14 ms | 5.35 M | 25.5 KB | 117 KB |
5151
| TFLM | cmsis_nn | 12.80 ms | 6.14 M | 22.2 KB | 176 KB |
5252
| TiGrIS | s8_ref | 81.96 ms | 39.34 M | 25.3 KB | 93 KB |
5353

@@ -57,7 +57,7 @@ int8 weights plus biases, measured from the committed plans.
5757
|---|---|---|---|---|---|
5858
| TiGrIS | cmsis_nn | 1.19 ms | 570 K | 11.6 KB | 370 KB |
5959
| TFLM | cmsis_nn | 1.16 ms | 558 K | 15.5 KB | 417 KB |
60-
| TiGrIS | s8_ref | 3.05 ms | 1.46 M | 11.4 KB | 346 KB |
60+
| TiGrIS | s8_ref | 3.05 ms | 1.46 M | 11.4 KB | 345 KB |
6161

6262
**Timeseries:**
6363

0 commit comments

Comments
 (0)