Skip to content

Commit 5893f97

Browse files
authored
Merge pull request #3102 from veryl-lang/perf/aot-c-wide-coverage
perf(simulator): keep wide comb shapes AOT-native (128-bit shift, dynsel store, 65-128-bit dynamic-index store)
2 parents 7897816 + 9dfe3f4 commit 5893f97

2 files changed

Lines changed: 268 additions & 8 deletions

File tree

crates/simulator/src/backend/aot_c/emit.rs

Lines changed: 102 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2967,11 +2967,75 @@ pub fn emit_stmt(stmt: &ProtoStatement) -> Option<String> {
29672967
if a.dst.is_ff() {
29682968
return emit_event_ff_assign(a);
29692969
}
2970-
// A runtime-indexed bit-slice store into a comb target is emitted
2971-
// below (after the rhs is computed), for dst_width <= 64. Wider
2972-
// dynamic-select stores stay on Cranelift.
2973-
if a.dynamic_select.is_some() && (a.dst_width > 64 || a.dst_width == 0) {
2974-
return None;
2970+
// A runtime-indexed bit-slice store. A ≤64-bit dst is the scalar
2971+
// RMW below; a wide (>64-bit) dst is handled here because that path
2972+
// shifts the mask in u64 and so can't reach a field crossing the
2973+
// 64-bit word boundary (SIMT per-lane rows `logic<N, W>[idx] = v`).
2974+
if let Some(dyn_sel) = &a.dynamic_select {
2975+
if a.dst_width == 0 {
2976+
return None;
2977+
}
2978+
if a.dst_width > 64 {
2979+
// A static bit-select / rhs_select / sign-extend combined
2980+
// with the dynamic index isn't modelled here — bail those.
2981+
if a.select.is_some() || eff_rhs_select.is_some() || se_from.is_some() {
2982+
return None;
2983+
}
2984+
let ew = dyn_sel.elem_width;
2985+
let ne = dyn_sel.num_elements;
2986+
let win = dyn_sel.window;
2987+
let VarOffset::Comb(store_off) = a.dst else {
2988+
return None;
2989+
};
2990+
if store_off < 0 || ew == 0 || win == 0 || ne == 0 {
2991+
return None;
2992+
}
2993+
let nb = native_bytes(a.dst_width);
2994+
let nw = nb / 8;
2995+
if win > nb * 8 {
2996+
return None;
2997+
}
2998+
let dst = format!("(uint8_t*)(comb_values + {store_off:#x})");
2999+
let max_idx = ne - 1;
3000+
let idx = emit_expr(&dyn_sel.index_expr)?;
3001+
let mut pre = String::new();
3002+
// rhs value (masked to `win` below) as an nb-byte buffer.
3003+
let r = emit_wide_operand(eff_expr, nb, &mut pre)?;
3004+
let wmsk = next_wide_tmp(); // mask_range = fill_ones(win) << _sh
3005+
let keep = next_wide_tmp(); // widthmask & ~mask_range
3006+
let widm = next_wide_tmp(); // fill_ones(dst_width)
3007+
let srcsh = next_wide_tmp();
3008+
let newv = next_wide_tmp();
3009+
// Mirror AssignStatement::eval_step's dynamic_select +
3010+
// Value::assign(beg=_sh+win-1, end=_sh) EXACTLY, including
3011+
// the out-of-width spill (no final width clamp), so AOT and
3012+
// the interpreter stay byte-identical.
3013+
return Some(format!(
3014+
"{{ {pre}uint64_t _di_raw = (uint64_t)({idx}); \
3015+
uint64_t _di = _di_raw < {max_idx} ? _di_raw : {max_idx}; \
3016+
uint64_t _sh = _di * {ew}ull; \
3017+
uint64_t _w{wmsk}[{nw}]; \
3018+
vw_fill_ones((uint8_t*)_w{wmsk}, (const uint8_t*)0, {pkw}u); \
3019+
vw_shl((uint8_t*)_w{wmsk}, (const uint8_t*)_w{wmsk}, _sh, {nb}u); \
3020+
uint64_t _w{widm}[{nw}]; \
3021+
vw_fill_ones((uint8_t*)_w{widm}, (const uint8_t*)0, {pkd}u); \
3022+
uint64_t _w{keep}[{nw}]; \
3023+
vw_band_not((uint8_t*)_w{keep}, (const uint8_t*)_w{widm}, (const uint8_t*)_w{wmsk}, {nb}u); \
3024+
uint64_t _w{srcsh}[{nw}]; \
3025+
vw_copy((uint8_t*)_w{srcsh}, {src}, {nb}u); \
3026+
vw_apply_mask((uint8_t*)_w{srcsh}, (const uint8_t*)0, {pkw}u); \
3027+
vw_shl((uint8_t*)_w{srcsh}, (const uint8_t*)_w{srcsh}, _sh, {nb}u); \
3028+
vw_band((uint8_t*)_w{srcsh}, (const uint8_t*)_w{srcsh}, (const uint8_t*)_w{wmsk}, {nb}u); \
3029+
uint64_t _w{newv}[{nw}]; \
3030+
vw_band((uint8_t*)_w{newv}, {dst}, (const uint8_t*)_w{keep}, {nb}u); \
3031+
vw_bor((uint8_t*)_w{newv}, (const uint8_t*)_w{newv}, (const uint8_t*)_w{srcsh}, {nb}u); \
3032+
vw_copy({dst}, (const uint8_t*)_w{newv}, {nb}u); }}",
3033+
pkw = wpack(nb, win),
3034+
pkd = wpack(nb, a.dst_width),
3035+
src = r.addr,
3036+
dst = dst,
3037+
));
3038+
}
29753039
}
29763040
// Wide comb store via the wide-op helper table. Two cases route
29773041
// here: (a) dst_width > 128 (never a C scalar); (b) a 65-128-bit
@@ -3346,13 +3410,15 @@ pub fn emit_stmt(stmt: &ProtoStatement) -> Option<String> {
33463410
if a.dst_base.is_ff() {
33473411
return None; // handled above in event mode; else out of scope
33483412
}
3349-
// Wide (>128-bit) dynamic-indexed comb store via the wide-op
3413+
// Wide (>64-bit) dynamic-indexed comb store via the wide-op
33503414
// helper table. A `var` array written by runtime index inside
33513415
// always_ff whose ff_log_base_current_offset is None maps to the
33523416
// comb buffer, so eval_step writes DIRECTLY to `base + stride*idx`
33533417
// with no write-log push. Mirror that byte for byte (RMW for
3354-
// select, copy+mask for full). The 65-128 range still bails below.
3355-
if a.dst_width > 128 {
3418+
// select, copy+mask for full). Covers 65-128-bit elements too
3419+
// (native_bytes/vw_* are width-agnostic); the scalar path below
3420+
// then only ever sees a ≤64-bit dst.
3421+
if a.dst_width > 64 {
33563422
if a.dynamic_select.is_some() || a.rhs_select.is_some() {
33573423
return None;
33583424
}
@@ -4241,6 +4307,34 @@ fn emit_expr_inner(expr: &ProtoExpression, needs_clean: bool) -> Option<String>
42414307
// already 128-bit, so only both-narrow truncate; a left shift
42424308
// follows its left operand alone. Bail those (and signed wide,
42434309
// which the block below can't sign-extend to 128) to Cranelift.
4310+
// 65..128-bit unsigned shift-LEFT with a narrow (≤64-bit) left
4311+
// operand: `(uint64_t)xs << ys` truncates to 64 bits, so promote
4312+
// xs to __uint128_t first. Placed before the `wide_truncates`
4313+
// bail below, which would otherwise send this to the interpreter.
4314+
if expr_context.width > 64
4315+
&& expr_context.width <= 128
4316+
&& matches!(op, Op::LogicShiftL | Op::ArithShiftL)
4317+
&& x.width() <= 64
4318+
&& !expr_context.signed
4319+
{
4320+
let w = expr_context.width;
4321+
let xm = if x.width() >= 64 {
4322+
format!("((__uint128_t)((uint64_t)({xs})))")
4323+
} else {
4324+
format!(
4325+
"((__uint128_t)(((uint64_t)({xs})) & 0x{:x}ULL))",
4326+
width_mask(x.width())
4327+
)
4328+
};
4329+
let shifted = format!(
4330+
"((((__uint128_t)({ys})) >= {w}) ? (__uint128_t)0 : (({xm}) << ({ys})))"
4331+
);
4332+
return Some(if w < 128 {
4333+
mask_u128(&shifted, w)
4334+
} else {
4335+
shifted
4336+
});
4337+
}
42444338
let wide_truncates = match op {
42454339
Op::Add | Op::Sub | Op::Mul => x.width() <= 64 && y.width() <= 64,
42464340
Op::LogicShiftL | Op::ArithShiftL => x.width() <= 64,

crates/simulator/src/tests/simulation.rs

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1351,6 +1351,172 @@ fn wide_128_ff_validate_pool_agnostic() {
13511351
);
13521352
}
13531353

1354+
#[test]
1355+
fn probe_r4_wide_dynsel_store() {
1356+
// ROUND 4 (Vortex/SIMT): dynamic bit-select store into a WIDE (>64-bit)
1357+
// packed-row target `logic<N, W>[idx] = row` — aot_c_validate dual-runs
1358+
// cc vs Cranelift each cycle. Non-power-of-2 element width (ew=31) and a
1359+
// window that fits exactly ((ne-1)*ew + win == dst_width).
1360+
if !crate::backend::aot_c::cc_available() {
1361+
return;
1362+
}
1363+
let code = r#"
1364+
module Top (
1365+
idx: input logic<2>,
1366+
v: input logic<31>,
1367+
base: input logic<124>,
1368+
o: output logic<124>,
1369+
) {
1370+
var t: logic<4, 31>;
1371+
always_comb {
1372+
t = base;
1373+
t[idx] = v;
1374+
}
1375+
assign o = t;
1376+
}
1377+
"#;
1378+
let config = aot_native_validate_config();
1379+
let ir = analyze(code, &config);
1380+
assert!(
1381+
ir.whole_comb.is_some(),
1382+
"wide dynsel-store comb must be AOT-C-native"
1383+
);
1384+
let mut sim = Simulator::new(ir, None);
1385+
for idx in 0u64..4 {
1386+
sim.set("base", Value::new(0, 124, false));
1387+
sim.set("idx", Value::new(idx, 2, false));
1388+
sim.set("v", Value::new(0x7F, 31, false));
1389+
sim.step(&Event::Clock(VarId::SYNTHETIC));
1390+
// element `idx` = 0x7F (7 bits), rest 0.
1391+
let o = sim.get("o").unwrap();
1392+
let expect = Value::from_u128(0x7Fu128 << (31 * idx), 0, 124, false);
1393+
assert_eq!(o, expect, "idx={idx}");
1394+
}
1395+
}
1396+
1397+
#[test]
1398+
fn probe_r4_wide_assign_dynamic() {
1399+
// ROUND 4: dynamic-INDEX store into a 65..128-bit unpacked-array element
1400+
// `logic<70> [N]; arr[idx] = v`. aot_c_validate dual-runs cc vs Cranelift.
1401+
if !crate::backend::aot_c::cc_available() {
1402+
return;
1403+
}
1404+
let code = r#"
1405+
module Top (
1406+
clk: input clock,
1407+
idx: input logic<2>,
1408+
v: input logic<70>,
1409+
o: output logic<70>,
1410+
) {
1411+
var arr: logic<70> [4];
1412+
always_ff {
1413+
for i in 0..4 {
1414+
arr[i] = 70'h0;
1415+
}
1416+
arr[idx] = v;
1417+
}
1418+
assign o = arr[3];
1419+
}
1420+
"#;
1421+
let config = aot_native_validate_config();
1422+
let ir = analyze(code, &config);
1423+
assert!(
1424+
!ir.whole_events.is_empty(),
1425+
"wide AssignDynamic event must be AOT-C-native"
1426+
);
1427+
let mut sim = Simulator::new(ir, None);
1428+
let clk = sim.get_clock("clk").unwrap();
1429+
let bigv: u128 = (1u128 << 69) | (1u128 << 40) | 0xABCD;
1430+
for idx in 0u64..4 {
1431+
sim.set("idx", Value::new(idx, 2, false));
1432+
sim.set("v", Value::from_u128(bigv, 0, 70, false));
1433+
sim.step(&clk);
1434+
let o = sim.get("o").unwrap();
1435+
let expect = if idx == 3 {
1436+
Value::from_u128(bigv, 0, 70, false)
1437+
} else {
1438+
Value::new(0, 70, false)
1439+
};
1440+
assert_eq!(o, expect, "idx={idx}");
1441+
}
1442+
}
1443+
1444+
#[test]
1445+
fn probe_r4_shift_left_128() {
1446+
// ROUND 4: 65..128-bit unsigned shift-LEFT with a narrow (≤64-bit) left
1447+
// operand — `(a << s)` promoted to __uint128_t in a 128-bit context (a
1448+
// u64 `<<` would truncate). aot_c_validate dual-runs cc vs Cranelift.
1449+
if !crate::backend::aot_c::cc_available() {
1450+
return;
1451+
}
1452+
let code = r#"
1453+
module Top (
1454+
a: input logic<32>,
1455+
s: input logic<7>,
1456+
o: output logic<128>,
1457+
) {
1458+
assign o = (a << s) + 128'h1;
1459+
}
1460+
"#;
1461+
let config = aot_native_validate_config();
1462+
let ir = analyze(code, &config);
1463+
assert!(
1464+
ir.whole_comb.is_some(),
1465+
"128-bit shift comb must be AOT-C-native"
1466+
);
1467+
let mut sim = Simulator::new(ir, None);
1468+
for s in [0u64, 1, 31, 63, 64, 96, 100, 127] {
1469+
sim.set("a", Value::new(0xDEAD_BEEF, 32, false));
1470+
sim.set("s", Value::new(s, 7, false));
1471+
sim.step(&Event::Clock(VarId::SYNTHETIC));
1472+
let o = sim.get("o").unwrap();
1473+
let expect_val: u128 = (0xDEAD_BEEFu128 << s).wrapping_add(1);
1474+
// mask to 128 bits (u128 already), the shift drops bits >= 128.
1475+
let expect = Value::from_u128(expect_val, 0, 128, false);
1476+
assert_eq!(o, expect, "s={s}");
1477+
}
1478+
}
1479+
1480+
#[test]
1481+
fn probe_r4_wide_src_rhs_select() {
1482+
// ROUND 4: windowed copy `dst = wide_src[hi:lo]` — a static rhs_select of
1483+
// a >128-bit source into both wide (176-bit) and narrow (48-bit) targets.
1484+
// aot_c_validate dual-runs cc vs Cranelift.
1485+
if !crate::backend::aot_c::cc_available() {
1486+
return;
1487+
}
1488+
let code = r#"
1489+
module Top (
1490+
src: input logic<472>,
1491+
wide: output logic<176>,
1492+
narr: output logic<48>,
1493+
) {
1494+
assign wide = src[351:176];
1495+
assign narr = src[119:72];
1496+
}
1497+
"#;
1498+
let config = aot_native_validate_config();
1499+
let ir = analyze(code, &config);
1500+
assert!(
1501+
ir.whole_comb.is_some(),
1502+
"wide-src rhs_select comb must be AOT-C-native"
1503+
);
1504+
let mut sim = Simulator::new(ir, None);
1505+
// Distinctive payload: low word pattern + a set bit near the top window.
1506+
let mut src = Value::new(0, 472, false);
1507+
// set bits so both windows carry data: bit 72.. and 176..
1508+
src = Value::from_u128((0x1234_5678u128) << 72, 0, 472, false);
1509+
sim.set("src", src);
1510+
sim.step(&Event::Clock(VarId::SYNTHETIC));
1511+
// narr = src[119:72] = 0x1234_5678 low 48 bits.
1512+
assert_eq!(
1513+
sim.get("narr").unwrap(),
1514+
Value::new((0x1234_5678u64) & ((1u64 << 48) - 1), 48, false)
1515+
);
1516+
// wide = src[351:176] — the payload only reaches bit ~104, so window is 0.
1517+
assert_eq!(sim.get("wide").unwrap(), Value::new(0, 176, false));
1518+
}
1519+
13541520
#[test]
13551521
fn wide_256_reduce_and_nested_unary() {
13561522
// Wide unary REDUCTIONS (&a / |a / ^a → 1-bit) exercise

0 commit comments

Comments
 (0)