Skip to content

Fixed length list support #1992

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 27 commits into from
Apr 28, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
a5eb177
support fixed length lists, part 1
cpetig Feb 2, 2025
6aeef59
more fixed size
cpetig Feb 2, 2025
c14d32d
fix tests for wit-parser
cpetig Feb 2, 2025
ef4b5e5
re-implement as two separate types (outside of WIT)
cpetig Feb 9, 2025
5d2e4d8
adapt the tests for the new type
cpetig Feb 9, 2025
bd6db9a
handle reading correctly
cpetig Feb 9, 2025
4b06d9d
wat parsing
cpetig Feb 9, 2025
c7b697c
generate fixed size lists in wit-smith
cpetig Feb 9, 2025
f1d6efe
proper rustfmt (whitespace change)
cpetig Feb 9, 2025
b5f54ee
implement Alex' suggetions, part one
cpetig Feb 11, 2025
36e2bd8
implement roundtrip tests
cpetig Feb 11, 2025
2abd0f8
limit the maximum size of a fixed list. to avoid memory explosion at …
cpetig Mar 9, 2025
0b8d062
separate list types, as proposed by Alex
cpetig Mar 9, 2025
4b30b54
post-rebase fixes
cpetig Apr 2, 2025
cca3733
fix the tests
cpetig Apr 2, 2025
879702b
Move local test to CLI test
alexcrichton Mar 12, 2025
1fc9e72
use same command line order as proposed by Alex
cpetig Apr 2, 2025
0bfcb28
prefer the name chosen by Alex
cpetig Apr 2, 2025
e7b5952
apply missing changes by Alex (I should have fetched before rebase)
cpetig Apr 2, 2025
aa5618f
Merge remote-tracking branch 'origin/main' into fixed-length-list
cpetig Apr 17, 2025
ba0d39f
Merge branch 'main' of github.com:bytecodealliance/wasm-tools into fi…
cpetig Apr 27, 2025
bc48583
limit the size of flattened fixed size lists, forbid empty ones
cpetig Apr 27, 2025
62fa3b7
test for zero and too high number
cpetig Apr 27, 2025
1a6ac1e
check for type mismatch during composition
cpetig Apr 27, 2025
bf43b10
clearly separate list from fixed-size-list in code
cpetig Apr 27, 2025
b74e6d1
separate list types in encoder as well
cpetig Apr 27, 2025
9669555
manually run rustfmt
cpetig Apr 27, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion crates/wasm-compose/src/encoding.rs
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,9 @@ impl<'a> TypeEncoder<'a> {
ComponentDefinedType::Record(r) => self.record(state, r),
ComponentDefinedType::Variant(v) => self.variant(state, v),
ComponentDefinedType::List(ty) => self.list(state, *ty),
ComponentDefinedType::FixedSizeList(ty, elements) => {
self.fixed_size_list(state, *ty, *elements)
}
ComponentDefinedType::Tuple(t) => self.tuple(state, t),
ComponentDefinedType::Flags(names) => Self::flags(&mut state.cur.encodable, names),
ComponentDefinedType::Enum(cases) => Self::enum_type(&mut state.cur.encodable, cases),
Expand Down Expand Up @@ -709,6 +712,23 @@ impl<'a> TypeEncoder<'a> {
index
}

fn fixed_size_list(
&self,
state: &mut TypeState<'a>,
ty: ct::ComponentValType,
elements: u32,
) -> u32 {
let ty = self.component_val_type(state, ty);
let index = state.cur.encodable.type_count();
state
.cur
.encodable
.ty()
.defined_type()
.fixed_size_list(ty, elements);
index
}

fn tuple(&self, state: &mut TypeState<'a>, tuple: &TupleType) -> u32 {
let types = tuple
.types
Expand Down Expand Up @@ -1228,7 +1248,9 @@ impl DependencyRegistrar<'_, '_> {
ComponentDefinedType::Primitive(_)
| ComponentDefinedType::Enum(_)
| ComponentDefinedType::Flags(_) => {}
ComponentDefinedType::List(t) | ComponentDefinedType::Option(t) => self.val_type(*t),
ComponentDefinedType::List(t)
| ComponentDefinedType::FixedSizeList(t, _)
| ComponentDefinedType::Option(t) => self.val_type(*t),
ComponentDefinedType::Own(r) | ComponentDefinedType::Borrow(r) => {
self.ty(ComponentAnyTypeId::Resource(*r))
}
Expand Down
7 changes: 7 additions & 0 deletions crates/wasm-encoder/src/component/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -593,6 +593,13 @@ impl ComponentDefinedTypeEncoder<'_> {
ty.into().encode(self.0);
}

/// Define a fixed size list type.
pub fn fixed_size_list(self, ty: impl Into<ComponentValType>, elements: u32) {
self.0.push(0x67);
ty.into().encode(self.0);
elements.encode(self.0);
}

/// Define a tuple type.
pub fn tuple<I, T>(self, types: I)
where
Expand Down
3 changes: 3 additions & 0 deletions crates/wasm-encoder/src/reencode/component.rs
Original file line number Diff line number Diff line change
Expand Up @@ -767,6 +767,9 @@ pub mod component_utils {
wasmparser::ComponentDefinedType::List(t) => {
defined.list(reencoder.component_val_type(t));
}
wasmparser::ComponentDefinedType::FixedSizeList(t, elements) => {
defined.fixed_size_list(reencoder.component_val_type(t), elements);
}
wasmparser::ComponentDefinedType::Tuple(t) => {
defined.tuple(t.iter().map(|t| reencoder.component_val_type(*t)));
}
Expand Down
11 changes: 11 additions & 0 deletions crates/wasm-wave/src/value/ty.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ pub struct Type(pub(super) TypeEnum);
pub(super) enum TypeEnum {
Simple(SimpleType),
List(Arc<ListType>),
FixedSizeList(Arc<ListType>, u32),
Record(Arc<RecordType>),
Tuple(Arc<TupleType>),
Variant(Arc<VariantType>),
Expand Down Expand Up @@ -55,6 +56,15 @@ impl Type {
Self(TypeEnum::List(Arc::new(ListType { element })))
}

/// Returns a list type with the given element type.
pub fn fixed_size_list(element_type: impl Into<Self>, elements: u32) -> Self {
let element = element_type.into();
Self(TypeEnum::FixedSizeList(
Arc::new(ListType { element }),
elements,
))
}

/// Returns a record type with the given field types. Returns None if
/// `fields` is empty.
pub fn record<T: Into<Box<str>>>(
Expand Down Expand Up @@ -189,6 +199,7 @@ impl WasmType for Type {
match self.0 {
TypeEnum::Simple(simple) => simple.0,
TypeEnum::List(_) => WasmTypeKind::List,
TypeEnum::FixedSizeList(_, _) => WasmTypeKind::FixedSizeList,
TypeEnum::Record(_) => WasmTypeKind::Record,
TypeEnum::Tuple(_) => WasmTypeKind::Tuple,
TypeEnum::Variant(_) => WasmTypeKind::Variant,
Expand Down
8 changes: 8 additions & 0 deletions crates/wasm-wave/src/value/wit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ impl<'a> TypeResolver<'a> {
TypeDefKind::Option(some_type) => self.resolve_option(some_type),
TypeDefKind::Result(result) => self.resolve_result(result),
TypeDefKind::List(element_type) => self.resolve_list(element_type),
TypeDefKind::FixedSizeList(element_type, elements) => {
self.resolve_fixed_size_list(element_type, *elements)
}
TypeDefKind::Type(Type::Bool) => Ok(value::Type::BOOL),
TypeDefKind::Type(Type::U8) => Ok(value::Type::U8),
TypeDefKind::Type(Type::U16) => Ok(value::Type::U16),
Expand Down Expand Up @@ -145,6 +148,11 @@ impl<'a> TypeResolver<'a> {
let element_type = self.resolve_type(*element_type)?;
Ok(value::Type::list(element_type))
}

fn resolve_fixed_size_list(&self, element_type: &Type, elements: u32) -> ValueResult {
let element_type = self.resolve_type(*element_type)?;
Ok(value::Type::fixed_size_list(element_type, elements))
}
}

#[cfg(test)]
Expand Down
2 changes: 2 additions & 0 deletions crates/wasm-wave/src/wasm/ty.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ pub enum WasmTypeKind {
Char,
String,
List,
FixedSizeList,
Record,
Tuple,
Variant,
Expand Down Expand Up @@ -48,6 +49,7 @@ impl std::fmt::Display for WasmTypeKind {
WasmTypeKind::Char => "char",
WasmTypeKind::String => "string",
WasmTypeKind::List => "list",
WasmTypeKind::FixedSizeList => "list<_,N>",
WasmTypeKind::Record => "record",
WasmTypeKind::Tuple => "tuple",
WasmTypeKind::Variant => "variant",
Expand Down
10 changes: 10 additions & 0 deletions crates/wasm-wave/src/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,16 @@ impl<W: Write> Writer<W> {
}
self.write_str("]")
}
WasmTypeKind::FixedSizeList => {
self.write_str("[")?;
for (idx, val) in val.unwrap_list().enumerate() {
if idx != 0 {
self.write_str(", ")?;
}
self.write_value(&*val)?;
}
self.write_str("]")
}
WasmTypeKind::Record => {
self.write_str("{")?;
let mut first = true;
Expand Down
5 changes: 5 additions & 0 deletions crates/wasmparser/src/features.rs
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,11 @@ define_wasm_features! {
/// Corresponds to the 📝 character in
/// <https://github.com/WebAssembly/component-model/blob/main/design/mvp/Explainer.md>.
pub cm_error_context: CM_ERROR_CONTEXT(1 << 30) = false;
/// Support for fixed size lists
///
/// Corresponds to the 🔧 character in
/// <https://github.com/WebAssembly/component-model/blob/main/design/mvp/Explainer.md>.
pub cm_fixed_size_list: CM_FIXED_SIZE_LIST(1 << 31) = false;
}
}

Expand Down
5 changes: 4 additions & 1 deletion crates/wasmparser/src/readers/component/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,8 @@ pub enum ComponentDefinedType<'a> {
Variant(Box<[VariantCase<'a>]>),
/// The type is a list of the given value type.
List(ComponentValType),
/// The type is a fixed size list of the given value type.
FixedSizeList(ComponentValType, u32),
/// The type is a tuple of the given value types.
Tuple(Box<[ComponentValType]>),
/// The type is flags with the given names.
Expand Down Expand Up @@ -503,8 +505,9 @@ impl<'a> ComponentDefinedType<'a> {
},
0x69 => ComponentDefinedType::Own(reader.read()?),
0x68 => ComponentDefinedType::Borrow(reader.read()?),
0x65 => ComponentDefinedType::Future(reader.read()?),
0x67 => ComponentDefinedType::FixedSizeList(reader.read()?, reader.read_var_u32()?),
0x66 => ComponentDefinedType::Stream(reader.read()?),
0x65 => ComponentDefinedType::Future(reader.read()?),
x => return reader.invalid_leading_byte(x, "component defined type"),
})
}
Expand Down
21 changes: 18 additions & 3 deletions crates/wasmparser/src/validator/component.rs
Original file line number Diff line number Diff line change
Expand Up @@ -724,9 +724,9 @@ impl ComponentState {
.map(|t| types.type_named_valtype(t, set))
.unwrap_or(true)
}
ComponentDefinedType::List(ty) | ComponentDefinedType::Option(ty) => {
types.type_named_valtype(ty, set)
}
ComponentDefinedType::List(ty)
| ComponentDefinedType::FixedSizeList(ty, _)
| ComponentDefinedType::Option(ty) => types.type_named_valtype(ty, set),

// The resource referred to by own/borrow must be named.
ComponentDefinedType::Own(id) | ComponentDefinedType::Borrow(id) => {
Expand Down Expand Up @@ -3475,6 +3475,21 @@ impl ComponentState {
crate::ComponentDefinedType::List(ty) => Ok(ComponentDefinedType::List(
self.create_component_val_type(ty, offset)?,
)),
crate::ComponentDefinedType::FixedSizeList(ty, elements) => {
if !self.features.cm_fixed_size_list() {
bail!(
offset,
"Fixed size lists require the component model fixed size list feature"
)
}
if elements < 1 {
bail!(offset, "Fixed size lists must have more than zero elements")
}
Ok(ComponentDefinedType::FixedSizeList(
self.create_component_val_type(ty, offset)?,
elements,
))
}
crate::ComponentDefinedType::Tuple(tys) => {
self.create_tuple_type(tys.as_ref(), types, offset)
}
Expand Down
35 changes: 28 additions & 7 deletions crates/wasmparser/src/validator/component_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1084,6 +1084,8 @@ pub enum ComponentDefinedType {
Variant(VariantType),
/// The type is a list.
List(ComponentValType),
/// The type is a fixed size list.
FixedSizeList(ComponentValType, u32),
/// The type is a tuple.
Tuple(TupleType),
/// The type is a set of flags.
Expand Down Expand Up @@ -1124,7 +1126,7 @@ impl TypeData for ComponentDefinedType {
Self::Record(r) => r.info,
Self::Variant(v) => v.info,
Self::Tuple(t) => t.info,
Self::List(ty) | Self::Option(ty) => ty.info(types),
Self::List(ty) | Self::FixedSizeList(ty, _) | Self::Option(ty) => ty.info(types),
Self::Result { ok, err } => {
let default = TypeInfo::new();
let mut info = ok.map(|ty| ty.type_info(types)).unwrap_or(default);
Expand Down Expand Up @@ -1153,7 +1155,7 @@ impl ComponentDefinedType {
| Self::Borrow(_)
| Self::Future(_)
| Self::Stream(_) => false,
Self::Option(ty) => ty.contains_ptr(types),
Self::Option(ty) | Self::FixedSizeList(ty, _) => ty.contains_ptr(types),
Self::Result { ok, err } => {
ok.map(|ty| ty.contains_ptr(types)).unwrap_or(false)
|| err.map(|ty| ty.contains_ptr(types)).unwrap_or(false)
Expand All @@ -1174,6 +1176,12 @@ impl ComponentDefinedType {
lowered_types,
),
Self::List(_) => lowered_types.push(ValType::I32) && lowered_types.push(ValType::I32),
Self::FixedSizeList(ty, length) => {
if *length as usize > lowered_types.max {
return false;
}
Comment on lines +1180 to +1182
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Upon further reflection of this I'm starting to have second thoughts about this approach. This input for example:

package a:b;

world x {
  type t = list<t2, 16>;
  type t2 = list<t3, 16>;
  type t3 = list<t4, 16>;
  type t4 = list<t5, 16>;
  type t5 = list<t6, 16>;
  type t6 = list<t7, 16>;
  type t7 = list<t8, 16>;
  type t8 = list<u32, 16>;
  import x: func(t: t);
}

when locally run as:

$ cargo run component embed --dummy foo.wit -t

takes ~30 seconds to produce the result. I think the reason is that this case isn't hit and then the recursion gets to go hog wild for a bit.

Perhaps an alternative fix would be to iterate from 0 to length but then check each iteration of the loop if the max has been reached and bail out if so? That way each level would check the break-out and once that happens it'd quickly exit.

Also if you're up for it I think that these would be good test cases to add to the test suite to ensure processing large or nested lists doesn't accidentally take forever

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, ... but the same problem shows with

package a:b;

world x {
  type t = tuple<t2,t2,t2,t2,t2,t2,t2,t2,t2,t2,t2,t2,t2,t2,t2,t2,>;
  type t2 = tuple<t3,t3,t3,t3,t3,t3,t3,t3,t3,t3,t3,t3,t3,t3,t3,t3,>;
  type t3 = tuple<t4,t4,t4,t4,t4,t4,t4,t4,t4,t4,t4,t4,t4,t4,t4,t4,>;
  type t4 = tuple<t5,t5,t5,t5,t5,t5,t5,t5,t5,t5,t5,t5,t5,t5,t5,t5,>;
  type t5 = tuple<t6,t6,t6,t6,t6,t6,t6,t6,t6,t6,t6,t6,t6,t6,t6,t6,>;
  type t6 = tuple<t7,t7,t7,t7,t7,t7,t7,t7,t7,t7,t7,t7,t7,t7,t7,t7,>;
  type t7 = tuple<t8,t8,t8,t8,t8,t8,t8,t8,t8,t8,t8,t8,t8,t8,t8,t8,>;
  type t8 = tuple<u32,u32,u32,u32,u32,u32,u32,u32,u32,u32,u32,u32,u32,u32,u32,u32,>;
  import x: func(t: t);
}

I will take a look into whether I can shrink the time needed to process this.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh that's a really good point! Ok in that case I'm going to go ahead and flag this for merge and follow-ups can handle fixes for cases like this.

(0..*length).all(|_n| ty.push_wasm_types(types, lowered_types))
}
Self::Tuple(t) => t
.types
.iter()
Expand Down Expand Up @@ -1248,6 +1256,7 @@ impl ComponentDefinedType {
ComponentDefinedType::Flags(_) => "flags",
ComponentDefinedType::Option(_) => "option",
ComponentDefinedType::List(_) => "list",
ComponentDefinedType::FixedSizeList(_, _) => "fixed size list",
ComponentDefinedType::Result { .. } => "result",
ComponentDefinedType::Own(_) => "own",
ComponentDefinedType::Borrow(_) => "borrow",
Expand Down Expand Up @@ -1985,7 +1994,9 @@ impl TypeAlloc {
}
}
}
ComponentDefinedType::List(ty) | ComponentDefinedType::Option(ty) => {
ComponentDefinedType::List(ty)
| ComponentDefinedType::FixedSizeList(ty, _)
| ComponentDefinedType::Option(ty) => {
self.free_variables_valtype(ty, set);
}
ComponentDefinedType::Result { ok, err } => {
Expand Down Expand Up @@ -2127,9 +2138,9 @@ impl TypeAlloc {
.map(|t| self.type_named_valtype(t, set))
.unwrap_or(true)
}
ComponentDefinedType::List(ty) | ComponentDefinedType::Option(ty) => {
self.type_named_valtype(ty, set)
}
ComponentDefinedType::List(ty)
| ComponentDefinedType::FixedSizeList(ty, _)
| ComponentDefinedType::Option(ty) => self.type_named_valtype(ty, set),

// own/borrow themselves don't have to be named, but the resource
// they refer to must be named.
Expand Down Expand Up @@ -2314,7 +2325,9 @@ where
}
}
}
ComponentDefinedType::List(ty) | ComponentDefinedType::Option(ty) => {
ComponentDefinedType::List(ty)
| ComponentDefinedType::FixedSizeList(ty, _)
| ComponentDefinedType::Option(ty) => {
any_changed |= self.remap_valtype(ty, map);
}
ComponentDefinedType::Result { ok, err } => {
Expand Down Expand Up @@ -3203,6 +3216,14 @@ impl<'a> SubtypeCx<'a> {
(Variant(_), b) => bail!(offset, "expected {}, found variant", b.desc()),
(List(a), List(b)) | (Option(a), Option(b)) => self.component_val_type(a, b, offset),
(List(_), b) => bail!(offset, "expected {}, found list", b.desc()),
(FixedSizeList(a, asize), FixedSizeList(b, bsize)) => {
if asize != bsize {
bail!(offset, "expected fixed size {bsize}, found size {asize}")
} else {
self.component_val_type(a, b, offset)
}
}
(FixedSizeList(_, _), b) => bail!(offset, "expected {}, found list", b.desc()),
(Option(_), b) => bail!(offset, "expected {}, found option", b.desc()),
(Tuple(a), Tuple(b)) => {
if a.types.len() != b.types.len() {
Expand Down
16 changes: 16 additions & 0 deletions crates/wasmprinter/src/component.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,19 @@ impl Printer<'_, '_> {
Ok(())
}

pub(crate) fn print_fixed_size_list_type(
&mut self,
state: &State,
element_ty: &ComponentValType,
elements: u32,
) -> Result<()> {
self.start_group("list ")?;
self.print_component_val_type(state, element_ty)?;
self.result.write_str(&format!(" {elements}"))?;
self.end_group()?;
Ok(())
}

pub(crate) fn print_tuple_type(
&mut self,
state: &State,
Expand Down Expand Up @@ -265,6 +278,9 @@ impl Printer<'_, '_> {
ComponentDefinedType::Record(fields) => self.print_record_type(state, fields)?,
ComponentDefinedType::Variant(cases) => self.print_variant_type(state, cases)?,
ComponentDefinedType::List(ty) => self.print_list_type(state, ty)?,
ComponentDefinedType::FixedSizeList(ty, elements) => {
self.print_fixed_size_list_type(state, ty, *elements)?
}
ComponentDefinedType::Tuple(tys) => self.print_tuple_type(state, tys)?,
ComponentDefinedType::Flags(names) => self.print_flag_or_enum_type("flags", names)?,
ComponentDefinedType::Enum(cases) => self.print_flag_or_enum_type("enum", cases)?,
Expand Down
3 changes: 3 additions & 0 deletions crates/wast/src/component/binary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@ fn encode_defined_type(encoder: ComponentDefinedTypeEncoder, ty: &ComponentDefin
ComponentDefinedType::List(l) => {
encoder.list(l.element.as_ref());
}
ComponentDefinedType::FixedSizeList(l) => {
encoder.fixed_size_list(l.element.as_ref(), l.elements);
}
ComponentDefinedType::Tuple(t) => {
encoder.tuple(t.fields.iter());
}
Expand Down
8 changes: 6 additions & 2 deletions crates/wast/src/component/expand.rs
Original file line number Diff line number Diff line change
Expand Up @@ -532,8 +532,12 @@ impl<'a> Expander<'a> {
}
}
}
ComponentDefinedType::List(t) => {
self.expand_component_val_ty(&mut t.element);
ComponentDefinedType::List(List { element: t })
| ComponentDefinedType::FixedSizeList(FixedSizeList {
element: t,
elements: _,
}) => {
self.expand_component_val_ty(t);
}
ComponentDefinedType::Tuple(t) => {
for field in t.fields.iter_mut() {
Expand Down
8 changes: 6 additions & 2 deletions crates/wast/src/component/resolve.rs
Original file line number Diff line number Diff line change
Expand Up @@ -547,8 +547,12 @@ impl<'a> Resolver<'a> {
}
}
}
ComponentDefinedType::List(l) => {
self.component_val_type(&mut l.element)?;
ComponentDefinedType::List(List { element: t })
| ComponentDefinedType::FixedSizeList(FixedSizeList {
element: t,
elements: _,
}) => {
self.component_val_type(t)?;
}
ComponentDefinedType::Tuple(t) => {
for field in t.fields.iter_mut() {
Expand Down
Loading