fix: Guard SizeSkeleton::compute against stack overflow

compiler/rustc_middle/src/ty/layout.rs

@@ -332,9 +332,24 @@ impl<'tcx> SizeSkeleton<'tcx> {
         ty: Ty<'tcx>,
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
+    ) -> Result<SizeSkeleton<'tcx>, &'tcx LayoutError<'tcx>> {
+        Self::compute_inner(ty, tcx, typing_env, 0)
+    }
+
+    fn compute_inner(
+        ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        typing_env: ty::TypingEnv<'tcx>,
+        depth: usize,
     ) -> Result<SizeSkeleton<'tcx>, &'tcx LayoutError<'tcx>> {
         debug_assert!(!ty.has_non_region_infer());
 
+        // Bail out if we've recursed too deeply (issue #156137); a cyclic type
+        // alias can otherwise blow the stack here.
+        if !tcx.recursion_limit().value_within_limit(depth) {
+            return Err(tcx.arena.alloc(LayoutError::Unknown(ty)));
+        }
+
         // First try computing a static layout.
         let err = match tcx.layout_of(typing_env.as_query_input(ty)) {
             Ok(layout) => {
@@ -407,7 +422,7 @@ impl<'tcx> SizeSkeleton<'tcx> {
                     return Ok(SizeSkeleton::Known(Size::from_bytes(0), None));
                 }
 
-                match SizeSkeleton::compute(inner, tcx, typing_env)? {
+                match SizeSkeleton::compute_inner(inner, tcx, typing_env, depth + 1)? {
                     // This may succeed because the multiplication of two types may overflow
                     // but a single size of a nested array will not.
                     SizeSkeleton::Known(s, a) => {
@@ -434,10 +449,9 @@ impl<'tcx> SizeSkeleton<'tcx> {
                 // Get a zero-sized variant or a pointer newtype.
                 let zero_or_ptr_variant = |i| {
                     let i = VariantIdx::from_usize(i);
-                    let fields =
-                        def.variant(i).fields.iter().map(|field| {
-                            SizeSkeleton::compute(field.ty(tcx, args), tcx, typing_env)
-                        });
+                    let fields = def.variant(i).fields.iter().map(|field| {
+                        SizeSkeleton::compute_inner(field.ty(tcx, args), tcx, typing_env, depth + 1)
+                    });
                     let mut ptr = None;
                     for field in fields {
                         let field = field?;
@@ -487,13 +501,13 @@ impl<'tcx> SizeSkeleton<'tcx> {
                 if ty == normalized {
                     Err(err)
                 } else {
-                    SizeSkeleton::compute(normalized, tcx, typing_env)
+                    SizeSkeleton::compute_inner(normalized, tcx, typing_env, depth + 1)
                 }
             }
 
             ty::Pat(base, pat) => {
                 // Pattern types are always the same size as their base.
-                let base = SizeSkeleton::compute(base, tcx, typing_env);
+                let base = SizeSkeleton::compute_inner(base, tcx, typing_env, depth + 1);
                 match *pat {
                     ty::PatternKind::Range { .. } | ty::PatternKind::Or(_) => base,
                     // But in the case of `!null` patterns we need to note that in the

tests/ui/transmute/recursive-size-skeleton.rs

@@ -0,0 +1,24 @@
+// Regression test for issue #156137.
+// Computing the `SizeSkeleton` of a type whose layout depends on itself
+// through a normalizing type alias used to recurse without bound and
+// blow the stack. We now bail out via the recursion limit and emit a
+// regular error instead of ICE-ing.
+
+use std::mem::transmute;
+
+trait Trait {
+    type Assoc;
+}
+impl<T> Trait for T {
+    type Assoc = T;
+}
+type Alias<T> = <T as Trait>::Assoc;
+
+pub struct Thing<T: ?Sized>(*const T, Alias<Thing<T>>);
+
+pub fn weird<T: ?Sized>(value: Thing<T>) {
+    let _: i32 = unsafe { transmute(value) };
+    //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
+}
+
+fn main() {}

tests/ui/transmute/recursive-size-skeleton.stderr

@@ -0,0 +1,12 @@
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/recursive-size-skeleton.rs:20:27
+   |
+LL |     let _: i32 = unsafe { transmute(value) };
+   |                           ^^^^^^^^^
+   |
+   = note: source type: `Thing<T>` (the type `*const T` has an unknown layout)
+   = note: target type: `i32` (32 bits)
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0512`.