Fix panicking overflow when calculating table sizes (#13244)

Return an error instead of panicking in the same manner that OOM is
handled.

Author: alexcrichton

RELEASES.md

@@ -1,3 +1,14 @@
+## 44.0.1
+
+Released 2026-04-30.
+
+### Fixed
+
+* Panic when allocating a table exceeding the size of the host's address space.
+  [GHSA-p8xm-42r7-89xg](https://github.com/bytecodealliance/wasmtime/security/advisories/GHSA-p8xm-42r7-89xg)
+
+--------------------------------------------------------------------------------
+
 ## 44.0.0
 
 Released 2026-04-20.

crates/wasmtime/src/runtime/vm/table.rs

@@ -309,7 +309,9 @@ unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<TryVec<Option<T>
 
     let size = mem::size_of::<Option<T>>();
     let size = size.next_multiple_of(align);
-    let size = size.checked_mul(len).unwrap();
+    let size = size
+        .checked_mul(len)
+        .ok_or_else(|| format_err!("overflow calculating table allocation size"))?;
 
     let layout = Layout::from_size_align(size, align)?;
 

tests/misc_testsuite/memory64/table-too-big.wast

@@ -0,0 +1,17 @@
+;;! memory64 = true
+;;! hogs_memory = true
+;;! reference_types = true
+
+(assert_trap
+  (module (table i64 0x2000_0000_0000_0000 funcref))
+  "overflow calculating table allocation size")
+
+(module
+  (table i64 0 funcref)
+  (func (export "grow") (param i64) (result i64)
+    (table.grow 0 (ref.null func) (local.get 0))
+  )
+)
+
+(assert_trap (invoke "grow" (i64.const 0x2000_0000_0000_0000))
+  "failed to allocate")