Skip to content

Commit a59ca0f

Browse files
victoryang00sakria9KikiSpaceBrian Zhao
committed
Enhance wasm with checkpoint and restore support (#2333)
- Add wasm_runtime_checkpoint/wasm_runtime_restore API - Support AOT and Classic Interpreter mode checkpoint and debug through OS signal, tested on windows/mac/linux aarch64/x64 - Static instrument the AOT to have the checkpoint and restore switches - Add sub extra library folder for implementing the ckpt-restore - Include extra dependency of yalantinglib Co-authored-by: Aibo Hu <[email protected]> Co-authored-by: kikispace <[email protected]> Co-authored-by: Brian Zhao <[email protected]> Signed-off-by: victoryang00 <[email protected]>
1 parent f1bad78 commit a59ca0f

File tree

5 files changed

+379
-0
lines changed

5 files changed

+379
-0
lines changed

‎.github/workflows/codeql_buildscript.sh

+277
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
#!/usr/bin/env bash
2+
3+
sudo apt update
4+
5+
sudo apt install -y build-essential cmake g++-multilib libgcc-11-dev lib32gcc-11-dev ccache ninja-build ccache
6+
7+
WAMR_DIR=${PWD}
8+
9+
# TODO: use pre-built llvm binary to build wamrc to
10+
# avoid static code analysing for llvm
11+
: '
12+
# build wamrc
13+
cd ${WAMR_DIR}/wamr-compiler
14+
./build_llvm.sh
15+
rm -fr build && mkdir build && cd build
16+
cmake ..
17+
make -j
18+
if [[ $? != 0 ]]; then
19+
echo "Failed to build wamrc!"
20+
exit 1;
21+
fi
22+
'
23+
24+
# build iwasm with default features enabled
25+
cd ${WAMR_DIR}/product-mini/platforms/linux
26+
rm -fr build && mkdir build && cd build
27+
cmake ..
28+
make -j
29+
if [[ $? != 0 ]]; then
30+
echo "Failed to build iwasm with default features enabled!"
31+
exit 1;
32+
fi
33+
34+
# build iwasm with default features enabled on x86_32
35+
cd ${WAMR_DIR}/product-mini/platforms/linux
36+
rm -fr build && mkdir build && cd build
37+
cmake .. -DWAMR_BUILD_TARGET=X86_32
38+
make -j
39+
if [[ $? != 0 ]]; then
40+
echo "Failed to build iwasm with default features enabled on x86_32!"
41+
exit 1;
42+
fi
43+
44+
# build iwasm with classic interpreter enabled
45+
cd ${WAMR_DIR}/product-mini/platforms/linux
46+
rm -rf build && mkdir build && cd build
47+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_INTERP=0
48+
make -j
49+
if [[ $? != 0 ]]; then
50+
echo "Failed to build iwasm with classic interpreter enabled!"
51+
exit 1;
52+
fi
53+
54+
# build iwasm with extra features enabled
55+
cd ${WAMR_DIR}/product-mini/platforms/linux
56+
rm -fr build && mkdir build && cd build
57+
cmake .. -DCMAKE_BUILD_TYPE=Debug \
58+
-DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \
59+
-DWAMR_BUILD_MULTI_MODULE=1 -DWAMR_BUILD_SIMD=1 \
60+
-DWAMR_BUILD_TAIL_CALL=1 -DWAMR_BUILD_REF_TYPES=1 \
61+
-DWAMR_BUILD_CUSTOM_NAME_SECTION=1 -DWAMR_BUILD_MEMORY_PROFILING=1 \
62+
-DWAMR_BUILD_PERF_PROFILING=1 -DWAMR_BUILD_DUMP_CALL_STACK=1 \
63+
-DWAMR_BUILD_LOAD_CUSTOM_SECTION=1
64+
make -j
65+
if [[ $? != 0 ]]; then
66+
echo "Failed to build wamrc iwasm with extra features enabled!"
67+
exit 1;
68+
fi
69+
70+
# build iwasm with global heap pool enabled
71+
cd ${WAMR_DIR}/product-mini/platforms/linux
72+
rm -fr build && mkdir build && cd build
73+
cmake .. -DCMAKE_BUILD_TYPE=Debug \
74+
-DWAMR_BUILD_ALLOC_WITH_USER_DATA=1 \
75+
-DWAMR_DISABLE_STACK_HW_BOUND_CHECK=1 \
76+
-DWAMR_BUILD_GLOBAL_HEAP_POOL=1 \
77+
-DWAMR_BUILD_GLOBAL_HEAP_SIZE=131072
78+
make -j
79+
if [[ $? != 0 ]]; then
80+
echo "Failed to build iwasm with global heap pool enabled!"
81+
exit 1;
82+
fi
83+
84+
# build iwasm with wasi-threads enabled
85+
cd ${WAMR_DIR}/product-mini/platforms/linux
86+
rm -fr build && mkdir build && cd build
87+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LIB_WASI_THREADS=1
88+
make -j
89+
if [[ $? != 0 ]]; then
90+
echo "Failed to build iwasm with wasi-threads enabled!"
91+
exit 1;
92+
fi
93+
94+
# build iwasm with GC enabled
95+
cd ${WAMR_DIR}/product-mini/platforms/linux
96+
rm -rf build && mkdir build && cd build
97+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_GC=1
98+
make -j
99+
if [[ $? != 0 ]]; then
100+
echo "Failed to build iwasm with GC enabled!"
101+
exit 1;
102+
fi
103+
104+
# build iwasm with exception handling enabled
105+
cd ${WAMR_DIR}/product-mini/platforms/linux
106+
rm -rf build && mkdir build && cd build
107+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_EXCE_HANDLING=1
108+
make -j
109+
if [[ $? != 0 ]]; then
110+
echo "Failed to build iwasm with exception handling enabled!"
111+
exit 1;
112+
fi
113+
114+
# build iwasm with memory64 enabled
115+
cd ${WAMR_DIR}/product-mini/platforms/linux
116+
rm -rf build && mkdir build && cd build
117+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MEMORY64=1
118+
make -j
119+
if [[ $? != 0 ]]; then
120+
echo "Failed to build iwasm with memory64 enabled!"
121+
exit 1;
122+
fi
123+
124+
# build iwasm with hardware boundary check disabled
125+
cd ${WAMR_DIR}/product-mini/platforms/linux
126+
rm -rf build && mkdir build && cd build
127+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_DISABLE_HW_BOUND_CHECK=1
128+
make -j
129+
if [[ $? != 0 ]]; then
130+
echo "Failed to build iwasm with hardware boundary check disabled!"
131+
exit 1;
132+
fi
133+
134+
# build iwasm with quick AOT entry disabled
135+
cd ${WAMR_DIR}/product-mini/platforms/linux
136+
rm -rf build && mkdir build && cd build
137+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_QUICK_AOT_ENTRY=0
138+
make -j
139+
if [[ $? != 0 ]]; then
140+
echo "Failed to build iwasm with quick AOT entry disabled!"
141+
exit 1;
142+
fi
143+
144+
# build iwasm with wakeup of blocking operations disabled
145+
cd ${WAMR_DIR}/product-mini/platforms/linux
146+
rm -rf build && mkdir build && cd build
147+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_DISABLE_WAKEUP_BLOCKING_OP=1
148+
make -j
149+
if [[ $? != 0 ]]; then
150+
echo "Failed to build iwasm with wakeup of blocking operations disabled!"
151+
exit 1;
152+
fi
153+
154+
# build iwasm with module instance context disabled
155+
cd ${WAMR_DIR}/product-mini/platforms/linux
156+
rm -rf build && mkdir build && cd build
157+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MODULE_INST_CONTEXT=0 \
158+
-DWAMR_BUILD_LIBC_BUILTIN=0 -DWAMR_BUILD_LIBC_WASI=0
159+
make -j
160+
if [[ $? != 0 ]]; then
161+
echo "Failed to build iwasm with module instance context disabled!"
162+
exit 1;
163+
fi
164+
165+
# build iwasm with libc-uvwasi enabled
166+
cd ${WAMR_DIR}/product-mini/platforms/linux
167+
rm -fr build && mkdir build && cd build
168+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LIBC_UVWASI=1
169+
make -j
170+
if [[ $? != 0 ]]; then
171+
echo "Failed to build iwasm with libc-uvwasi enabled!"
172+
exit 1;
173+
fi
174+
175+
# build iwasm with fast jit lazy mode enabled
176+
cd ${WAMR_DIR}/product-mini/platforms/linux
177+
rm -rf build && mkdir build && cd build
178+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_FAST_JIT_DUMP=1
179+
make -j
180+
if [[ $? != 0 ]]; then
181+
echo "Failed to build iwasm with fast jit lazy mode enabled!"
182+
exit 1;
183+
fi
184+
185+
# build iwasm with fast jit eager mode enabled
186+
cd ${WAMR_DIR}/product-mini/platforms/linux
187+
rm -rf build && mkdir build && cd build
188+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_FAST_JIT_DUMP=1
189+
make -j
190+
if [[ $? != 0 ]]; then
191+
echo "Failed to build iwasm with fast jit eager mode enabled!"
192+
exit 1;
193+
fi
194+
195+
# TODO: use pre-built llvm binary to build llvm-jit and multi-tier-jit
196+
: '
197+
# build iwasm with llvm jit lazy mode enabled
198+
cd ${WAMR_DIR}/product-mini/platforms/linux
199+
rm -rf build && mkdir build && cd build
200+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1
201+
make -j
202+
if [[ $? != 0 ]]; then
203+
echo "Failed to build llvm jit lazy mode enabled!"
204+
exit 1;
205+
fi
206+
207+
# build iwasm with llvm jit eager mode enabled
208+
cd ${WAMR_DIR}/product-mini/platforms/linux
209+
rm -rf build && mkdir build && cd build
210+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0
211+
make -j
212+
if [[ $? != 0 ]]; then
213+
echo "Failed to build llvm jit eager mode enabled!"
214+
exit 1;
215+
fi
216+
217+
# build iwasm with multi-tier jit enabled
218+
cd ${WAMR_DIR}/product-mini/platforms/linux
219+
rm -rf build && mkdir build && cd build
220+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \
221+
-DWAMR_BUILD_FAST_JIT_DUMP=1
222+
make -j
223+
if [[ $? != 0 ]]; then
224+
echo "Failed to build iwasm with multi-tier jit enabled!"
225+
exit 1;
226+
fi
227+
'
228+
229+
# build iwasm with wasm mini-loader enabled
230+
cd ${WAMR_DIR}/product-mini/platforms/linux
231+
rm -rf build && mkdir build && cd build
232+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MINI_LOADER=1
233+
make -j
234+
if [[ $? != 0 ]]; then
235+
echo "Failed to build with wasm mini-loader enabled!"
236+
exit 1;
237+
fi
238+
239+
# build iwasm with source debugging enabled
240+
cd ${WAMR_DIR}/product-mini/platforms/linux
241+
rm -rf build && mkdir build && cd build
242+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_DEBUG_INTERP=1 -DWAMR_BUILD_DEBUG_AOT=1
243+
make -j
244+
if [[ $? != 0 ]]; then
245+
echo "Failed to build iwasm with source debugging enabled!"
246+
exit 1;
247+
fi
248+
249+
# build iwasm with AOT static PGO enabled
250+
cd ${WAMR_DIR}/product-mini/platforms/linux
251+
rm -rf build && mkdir build && cd build
252+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_STATIC_PGO=1
253+
make -j
254+
if [[ $? != 0 ]]; then
255+
echo "Failed to build iwasm with AOT static PGO enabled!"
256+
exit 1;
257+
fi
258+
259+
# build iwasm with configurable bounds checks enabled
260+
cd ${WAMR_DIR}/product-mini/platforms/linux
261+
rm -rf build && mkdir build && cd build
262+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_CONFIGUABLE_BOUNDS_CHECKS=1
263+
make -j
264+
if [[ $? != 0 ]]; then
265+
echo "Failed to build iwasm with configurable bounds checks enabled!"
266+
exit 1;
267+
fi
268+
269+
# build iwasm with linux perf support enabled
270+
cd ${WAMR_DIR}/product-mini/platforms/linux/
271+
rm -rf build && mkdir build && cd build
272+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LINUX_PERF=1
273+
make -j
274+
if [[ $? != 0 ]]; then
275+
echo "Failed to build iwasm with linux perf support enabled!"
276+
exit 1;
277+
fi

‎.github/workflows/codeql_fail_on_error.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/env python3
2+
3+
import json
4+
import sys
5+
6+
# Return whether SARIF file contains error-level results
7+
def codeql_sarif_contain_error(filename):
8+
with open(filename, 'r') as f:
9+
s = json.load(f)
10+
11+
for run in s.get('runs', []):
12+
rules_metadata = run['tool']['driver']['rules']
13+
if not rules_metadata:
14+
rules_metadata = run['tool']['extensions'][0]['rules']
15+
16+
for res in run.get('results', []):
17+
if 'ruleIndex' in res:
18+
rule_index = res['ruleIndex']
19+
elif 'rule' in res and 'index' in res['rule']:
20+
rule_index = res['rule']['index']
21+
else:
22+
continue
23+
try:
24+
rule_level = rules_metadata[rule_index]['defaultConfiguration']['level']
25+
except IndexError as e:
26+
print(e, rule_index, len(rules_metadata))
27+
else:
28+
if rule_level == 'error':
29+
return True
30+
return False
31+
32+
if __name__ == "__main__":
33+
if codeql_sarif_contain_error(sys.argv[1]):
34+
sys.exit(1)

‎core/iwasm/compilation/aot_emit_function.c

+15
Original file line numberDiff line numberDiff line change
@@ -1420,6 +1420,21 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
14201420
/* Get param cell number */
14211421
param_cell_num = func_type->param_cell_num;
14221422

1423+
// #if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING !=
1424+
// 0)
1425+
// if (comp_ctx->enable_aux_stack_frame) {
1426+
// LLVMValueRef func_idx_const;
1427+
1428+
// if (!(func_idx_const = I32_CONST(func_idx))) {
1429+
// aot_set_last_error("llvm build const failed.");
1430+
// return false;
1431+
// }
1432+
// if (!call_aot_alloc_frame_func(comp_ctx, func_ctx,
1433+
// func_idx_const))
1434+
// return false;
1435+
// }
1436+
// #endif
1437+
14231438
/* Allocate memory for parameters.
14241439
* Parameters layout:
14251440
* - exec env

‎core/iwasm/include/wasm_export.h

+10
Original file line numberDiff line numberDiff line change
@@ -646,6 +646,16 @@ wasm_runtime_instantiate_ex(const wasm_module_t module,
646646
const InstantiationArgs *args, char *error_buf,
647647
uint32_t error_buf_size);
648648

649+
/**
650+
* Instantiate a WASM module, with specified instantiation arguments
651+
*
652+
* Same as wasm_runtime_instantiate, but it also allows overwriting maximum memory
653+
*/
654+
WASM_RUNTIME_API_EXTERN wasm_module_inst_t
655+
wasm_runtime_instantiate_ex(const wasm_module_t module,
656+
const InstantiationArgs *args,
657+
char *error_buf, uint32_t error_buf_size);
658+
649659
/**
650660
* Set the running mode of a WASM module instance, override the
651661
* default running mode of the runtime. Note that it only makes sense when

‎core/iwasm/libraries/ckpt-restore/get_func_name.py

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import subprocess
2+
3+
def get_func_name(func, file):
4+
cmd = ["wasm2wat", "--enable-all", file]
5+
grep_cmd = ["grep", "func"]
6+
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
7+
grep_process = subprocess.Popen(
8+
grep_cmd, stdin=process.stdout, stdout=subprocess.PIPE
9+
)
10+
process.stdout.close()
11+
output = grep_process.communicate()[0].decode("utf-8")
12+
output = output.split("\n")
13+
output1 = [
14+
x
15+
for x in output
16+
if not x.__contains__("(type (;")
17+
# and not x.__contains__("(import ")
18+
and not x.__contains__("(table (;")
19+
and not x.__contains__("global.get")
20+
]
21+
22+
import_count = len([
23+
x
24+
for x in output
25+
if x.__contains__("(import ")
26+
])
27+
28+
for i in range(len(output1)):
29+
if i == func:
30+
return output1[i].split(" ")[3], i - import_count
31+
32+
if __name__ == "__main__":
33+
import sys
34+
wasm = sys.argv[1]
35+
names = []
36+
aot_idxes = []
37+
for i in range(2, len(sys.argv)):
38+
idx = int(sys.argv[i])
39+
name, aot_idx = get_func_name(idx, wasm)
40+
names.append(name)
41+
aot_idxes.append(aot_idx)
42+
for i in range(len(names)):
43+
print(f"{names[i]} {aot_idxes[i]}")

0 commit comments

Comments
 (0)