rustc_ty_utils/
layout.rs

1use hir::def_id::DefId;
2use rustc_abi::Integer::{I8, I32};
3use rustc_abi::Primitive::{self, Float, Int, Pointer};
4use rustc_abi::{
5    AddressSpace, BackendRepr, FIRST_VARIANT, FieldIdx, FieldsShape, HasDataLayout, Layout,
6    LayoutCalculatorError, LayoutData, Niche, ReprOptions, Scalar, Size, StructKind, TagEncoding,
7    VariantIdx, Variants, WrappingRange,
8};
9use rustc_hashes::Hash64;
10use rustc_index::IndexVec;
11use rustc_middle::bug;
12use rustc_middle::query::Providers;
13use rustc_middle::ty::layout::{
14    FloatExt, HasTyCtxt, IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout,
15};
16use rustc_middle::ty::print::with_no_trimmed_paths;
17use rustc_middle::ty::{
18    self, AdtDef, CoroutineArgsExt, EarlyBinder, PseudoCanonicalInput, Ty, TyCtxt, TypeVisitableExt,
19};
20use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo};
21use rustc_span::{Symbol, sym};
22use tracing::{debug, instrument};
23use {rustc_abi as abi, rustc_hir as hir};
24
25use crate::errors::{NonPrimitiveSimdType, OversizedSimdType, ZeroLengthSimdType};
26
27mod invariant;
28
29pub(crate) fn provide(providers: &mut Providers) {
30    *providers = Providers { layout_of, ..*providers };
31}
32
33#[instrument(skip(tcx, query), level = "debug")]
34fn layout_of<'tcx>(
35    tcx: TyCtxt<'tcx>,
36    query: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>,
37) -> Result<TyAndLayout<'tcx>, &'tcx LayoutError<'tcx>> {
38    let PseudoCanonicalInput { typing_env, value: ty } = query;
39    debug!(?ty);
40
41    // Optimization: We convert to TypingMode::PostAnalysis and convert opaque types in
42    // the where bounds to their hidden types. This reduces overall uncached invocations
43    // of `layout_of` and is thus a small performance improvement.
44    let typing_env = typing_env.with_post_analysis_normalized(tcx);
45    let unnormalized_ty = ty;
46
47    // FIXME: We might want to have two different versions of `layout_of`:
48    // One that can be called after typecheck has completed and can use
49    // `normalize_erasing_regions` here and another one that can be called
50    // before typecheck has completed and uses `try_normalize_erasing_regions`.
51    let ty = match tcx.try_normalize_erasing_regions(typing_env, ty) {
52        Ok(t) => t,
53        Err(normalization_error) => {
54            return Err(tcx
55                .arena
56                .alloc(LayoutError::NormalizationFailure(ty, normalization_error)));
57        }
58    };
59
60    if ty != unnormalized_ty {
61        // Ensure this layout is also cached for the normalized type.
62        return tcx.layout_of(typing_env.as_query_input(ty));
63    }
64
65    let cx = LayoutCx::new(tcx, typing_env);
66
67    let layout = layout_of_uncached(&cx, ty)?;
68    let layout = TyAndLayout { ty, layout };
69
70    // If we are running with `-Zprint-type-sizes`, maybe record layouts
71    // for dumping later.
72    if cx.tcx().sess.opts.unstable_opts.print_type_sizes {
73        record_layout_for_printing(&cx, layout);
74    }
75
76    invariant::layout_sanity_check(&cx, &layout);
77
78    Ok(layout)
79}
80
81fn error<'tcx>(cx: &LayoutCx<'tcx>, err: LayoutError<'tcx>) -> &'tcx LayoutError<'tcx> {
82    cx.tcx().arena.alloc(err)
83}
84
85fn map_error<'tcx>(
86    cx: &LayoutCx<'tcx>,
87    ty: Ty<'tcx>,
88    err: LayoutCalculatorError<TyAndLayout<'tcx>>,
89) -> &'tcx LayoutError<'tcx> {
90    let err = match err {
91        LayoutCalculatorError::SizeOverflow => {
92            // This is sometimes not a compile error in `check` builds.
93            // See `tests/ui/limits/huge-enum.rs` for an example.
94            LayoutError::SizeOverflow(ty)
95        }
96        LayoutCalculatorError::UnexpectedUnsized(field) => {
97            // This is sometimes not a compile error if there are trivially false where clauses.
98            // See `tests/ui/layout/trivial-bounds-sized.rs` for an example.
99            assert!(field.layout.is_unsized(), "invalid layout error {err:#?}");
100            if cx.typing_env.param_env.caller_bounds().is_empty() {
101                cx.tcx().dcx().delayed_bug(format!(
102                    "encountered unexpected unsized field in layout of {ty:?}: {field:#?}"
103                ));
104            }
105            LayoutError::Unknown(ty)
106        }
107        LayoutCalculatorError::EmptyUnion => {
108            // This is always a compile error.
109            let guar =
110                cx.tcx().dcx().delayed_bug(format!("computed layout of empty union: {ty:?}"));
111            LayoutError::ReferencesError(guar)
112        }
113        LayoutCalculatorError::ReprConflict => {
114            // packed enums are the only known trigger of this, but others might arise
115            let guar = cx
116                .tcx()
117                .dcx()
118                .delayed_bug(format!("computed impossible repr (packed enum?): {ty:?}"));
119            LayoutError::ReferencesError(guar)
120        }
121        LayoutCalculatorError::ZeroLengthSimdType => {
122            // Can't be caught in typeck if the array length is generic.
123            cx.tcx().dcx().emit_fatal(ZeroLengthSimdType { ty })
124        }
125        LayoutCalculatorError::OversizedSimdType { max_lanes } => {
126            // Can't be caught in typeck if the array length is generic.
127            cx.tcx().dcx().emit_fatal(OversizedSimdType { ty, max_lanes })
128        }
129        LayoutCalculatorError::NonPrimitiveSimdType(field) => {
130            // This error isn't caught in typeck, e.g., if
131            // the element type of the vector is generic.
132            cx.tcx().dcx().emit_fatal(NonPrimitiveSimdType { ty, e_ty: field.ty })
133        }
134    };
135    error(cx, err)
136}
137
138fn extract_const_value<'tcx>(
139    cx: &LayoutCx<'tcx>,
140    ty: Ty<'tcx>,
141    ct: ty::Const<'tcx>,
142) -> Result<ty::Value<'tcx>, &'tcx LayoutError<'tcx>> {
143    match ct.kind() {
144        ty::ConstKind::Value(cv) => Ok(cv),
145        ty::ConstKind::Param(_) | ty::ConstKind::Expr(_) => {
146            if !ct.has_param() {
147                bug!("failed to normalize const, but it is not generic: {ct:?}");
148            }
149            Err(error(cx, LayoutError::TooGeneric(ty)))
150        }
151        ty::ConstKind::Unevaluated(_) => {
152            let err = if ct.has_param() {
153                LayoutError::TooGeneric(ty)
154            } else {
155                // This case is reachable with unsatisfiable predicates and GCE (which will
156                // cause anon consts to inherit the unsatisfiable predicates). For example
157                // if we have an unsatisfiable `u8: Trait` bound, then it's not a compile
158                // error to mention `[u8; <u8 as Trait>::CONST]`, but we can't compute its
159                // layout.
160                LayoutError::Unknown(ty)
161            };
162            Err(error(cx, err))
163        }
164        ty::ConstKind::Infer(_)
165        | ty::ConstKind::Bound(..)
166        | ty::ConstKind::Placeholder(_)
167        | ty::ConstKind::Error(_) => {
168            // `ty::ConstKind::Error` is handled at the top of `layout_of_uncached`
169            // (via `ty.error_reported()`).
170            bug!("layout_of: unexpected const: {ct:?}");
171        }
172    }
173}
174
175fn layout_of_uncached<'tcx>(
176    cx: &LayoutCx<'tcx>,
177    ty: Ty<'tcx>,
178) -> Result<Layout<'tcx>, &'tcx LayoutError<'tcx>> {
179    // Types that reference `ty::Error` pessimistically don't have a meaningful layout.
180    // The only side-effect of this is possibly worse diagnostics in case the layout
181    // was actually computable (like if the `ty::Error` showed up only in a `PhantomData`).
182    if let Err(guar) = ty.error_reported() {
183        return Err(error(cx, LayoutError::ReferencesError(guar)));
184    }
185
186    let tcx = cx.tcx();
187
188    // layout of `async_drop_in_place<T>::{closure}` in case,
189    // when T is a coroutine, contains this internal coroutine's ref
190
191    let dl = cx.data_layout();
192    let map_layout = |result: Result<_, _>| match result {
193        Ok(layout) => Ok(tcx.mk_layout(layout)),
194        Err(err) => Err(map_error(cx, ty, err)),
195    };
196    let scalar_unit = |value: Primitive| {
197        let size = value.size(dl);
198        assert!(size.bits() <= 128);
199        Scalar::Initialized { value, valid_range: WrappingRange::full(size) }
200    };
201    let scalar = |value: Primitive| tcx.mk_layout(LayoutData::scalar(cx, scalar_unit(value)));
202
203    let univariant = |tys: &[Ty<'tcx>], kind| {
204        let fields = tys.iter().map(|ty| cx.layout_of(*ty)).try_collect::<IndexVec<_, _>>()?;
205        let repr = ReprOptions::default();
206        map_layout(cx.calc.univariant(&fields, &repr, kind))
207    };
208    debug_assert!(!ty.has_non_region_infer());
209
210    Ok(match *ty.kind() {
211        ty::Pat(ty, pat) => {
212            let layout = cx.layout_of(ty)?.layout;
213            let mut layout = LayoutData::clone(&layout.0);
214            match *pat {
215                ty::PatternKind::Range { start, end } => {
216                    if let BackendRepr::Scalar(scalar) | BackendRepr::ScalarPair(scalar, _) =
217                        &mut layout.backend_repr
218                    {
219                        scalar.valid_range_mut().start = extract_const_value(cx, ty, start)?
220                            .try_to_bits(tcx, cx.typing_env)
221                            .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;
222
223                        scalar.valid_range_mut().end = extract_const_value(cx, ty, end)?
224                            .try_to_bits(tcx, cx.typing_env)
225                            .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;
226
227                        // FIXME(pattern_types): create implied bounds from pattern types in signatures
228                        // that require that the range end is >= the range start so that we can't hit
229                        // this error anymore without first having hit a trait solver error.
230                        // Very fuzzy on the details here, but pattern types are an internal impl detail,
231                        // so we can just go with this for now
232                        if scalar.is_signed() {
233                            let range = scalar.valid_range_mut();
234                            let start = layout.size.sign_extend(range.start);
235                            let end = layout.size.sign_extend(range.end);
236                            if end < start {
237                                let guar = tcx.dcx().err(format!(
238                                    "pattern type ranges cannot wrap: {start}..={end}"
239                                ));
240
241                                return Err(error(cx, LayoutError::ReferencesError(guar)));
242                            }
243                        } else {
244                            let range = scalar.valid_range_mut();
245                            if range.end < range.start {
246                                let guar = tcx.dcx().err(format!(
247                                    "pattern type ranges cannot wrap: {}..={}",
248                                    range.start, range.end
249                                ));
250
251                                return Err(error(cx, LayoutError::ReferencesError(guar)));
252                            }
253                        };
254
255                        let niche = Niche {
256                            offset: Size::ZERO,
257                            value: scalar.primitive(),
258                            valid_range: scalar.valid_range(cx),
259                        };
260
261                        layout.largest_niche = Some(niche);
262                    } else {
263                        bug!("pattern type with range but not scalar layout: {ty:?}, {layout:?}")
264                    }
265                }
266                ty::PatternKind::Or(variants) => match *variants[0] {
267                    ty::PatternKind::Range { .. } => {
268                        if let BackendRepr::Scalar(scalar) = &mut layout.backend_repr {
269                            let variants: Result<Vec<_>, _> = variants
270                                .iter()
271                                .map(|pat| match *pat {
272                                    ty::PatternKind::Range { start, end } => Ok((
273                                        extract_const_value(cx, ty, start)
274                                            .unwrap()
275                                            .try_to_bits(tcx, cx.typing_env)
276                                            .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?,
277                                        extract_const_value(cx, ty, end)
278                                            .unwrap()
279                                            .try_to_bits(tcx, cx.typing_env)
280                                            .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?,
281                                    )),
282                                    ty::PatternKind::Or(_) => {
283                                        unreachable!("mixed or patterns are not allowed")
284                                    }
285                                })
286                                .collect();
287                            let mut variants = variants?;
288                            if !scalar.is_signed() {
289                                let guar = tcx.dcx().err(format!(
290                                    "only signed integer base types are allowed for or-pattern pattern types at present"
291                                ));
292
293                                return Err(error(cx, LayoutError::ReferencesError(guar)));
294                            }
295                            variants.sort();
296                            if variants.len() != 2 {
297                                let guar = tcx
298                                .dcx()
299                                .err(format!("the only or-pattern types allowed are two range patterns that are directly connected at their overflow site"));
300
301                                return Err(error(cx, LayoutError::ReferencesError(guar)));
302                            }
303
304                            // first is the one starting at the signed in range min
305                            let mut first = variants[0];
306                            let mut second = variants[1];
307                            if second.0
308                                == layout.size.truncate(layout.size.signed_int_min() as u128)
309                            {
310                                (second, first) = (first, second);
311                            }
312
313                            if layout.size.sign_extend(first.1) >= layout.size.sign_extend(second.0)
314                            {
315                                let guar = tcx.dcx().err(format!(
316                                    "only non-overlapping pattern type ranges are allowed at present"
317                                ));
318
319                                return Err(error(cx, LayoutError::ReferencesError(guar)));
320                            }
321                            if layout.size.signed_int_max() as u128 != second.1 {
322                                let guar = tcx.dcx().err(format!(
323                                    "one pattern needs to end at `{ty}::MAX`, but was {} instead",
324                                    second.1
325                                ));
326
327                                return Err(error(cx, LayoutError::ReferencesError(guar)));
328                            }
329
330                            // Now generate a wrapping range (which aren't allowed in surface syntax).
331                            scalar.valid_range_mut().start = second.0;
332                            scalar.valid_range_mut().end = first.1;
333
334                            let niche = Niche {
335                                offset: Size::ZERO,
336                                value: scalar.primitive(),
337                                valid_range: scalar.valid_range(cx),
338                            };
339
340                            layout.largest_niche = Some(niche);
341                        } else {
342                            bug!(
343                                "pattern type with range but not scalar layout: {ty:?}, {layout:?}"
344                            )
345                        }
346                    }
347                    ty::PatternKind::Or(..) => bug!("patterns cannot have nested or patterns"),
348                },
349            }
350            tcx.mk_layout(layout)
351        }
352
353        // Basic scalars.
354        ty::Bool => tcx.mk_layout(LayoutData::scalar(
355            cx,
356            Scalar::Initialized {
357                value: Int(I8, false),
358                valid_range: WrappingRange { start: 0, end: 1 },
359            },
360        )),
361        ty::Char => tcx.mk_layout(LayoutData::scalar(
362            cx,
363            Scalar::Initialized {
364                value: Int(I32, false),
365                valid_range: WrappingRange { start: 0, end: 0x10FFFF },
366            },
367        )),
368        ty::Int(ity) => scalar(Int(abi::Integer::from_int_ty(dl, ity), true)),
369        ty::Uint(ity) => scalar(Int(abi::Integer::from_uint_ty(dl, ity), false)),
370        ty::Float(fty) => scalar(Float(abi::Float::from_float_ty(fty))),
371        ty::FnPtr(..) => {
372            let mut ptr = scalar_unit(Pointer(dl.instruction_address_space));
373            ptr.valid_range_mut().start = 1;
374            tcx.mk_layout(LayoutData::scalar(cx, ptr))
375        }
376
377        // The never type.
378        ty::Never => tcx.mk_layout(LayoutData::never_type(cx)),
379
380        // Potentially-wide pointers.
381        ty::Ref(_, pointee, _) | ty::RawPtr(pointee, _) => {
382            let mut data_ptr = scalar_unit(Pointer(AddressSpace::DATA));
383            if !ty.is_raw_ptr() {
384                data_ptr.valid_range_mut().start = 1;
385            }
386
387            if pointee.is_sized(tcx, cx.typing_env) {
388                return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
389            }
390
391            let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type() {
392                let pointee_metadata = Ty::new_projection(tcx, metadata_def_id, [pointee]);
393                let metadata_ty =
394                    match tcx.try_normalize_erasing_regions(cx.typing_env, pointee_metadata) {
395                        Ok(metadata_ty) => metadata_ty,
396                        Err(mut err) => {
397                            // Usually `<Ty as Pointee>::Metadata` can't be normalized because
398                            // its struct tail cannot be normalized either, so try to get a
399                            // more descriptive layout error here, which will lead to less confusing
400                            // diagnostics.
401                            //
402                            // We use the raw struct tail function here to get the first tail
403                            // that is an alias, which is likely the cause of the normalization
404                            // error.
405                            match tcx.try_normalize_erasing_regions(
406                                cx.typing_env,
407                                tcx.struct_tail_raw(pointee, |ty| ty, || {}),
408                            ) {
409                                Ok(_) => {}
410                                Err(better_err) => {
411                                    err = better_err;
412                                }
413                            }
414                            return Err(error(cx, LayoutError::NormalizationFailure(pointee, err)));
415                        }
416                    };
417
418                let metadata_layout = cx.layout_of(metadata_ty)?;
419                // If the metadata is a 1-zst, then the pointer is thin.
420                if metadata_layout.is_1zst() {
421                    return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
422                }
423
424                let BackendRepr::Scalar(metadata) = metadata_layout.backend_repr else {
425                    return Err(error(cx, LayoutError::Unknown(pointee)));
426                };
427
428                metadata
429            } else {
430                let unsized_part = tcx.struct_tail_for_codegen(pointee, cx.typing_env);
431
432                match unsized_part.kind() {
433                    ty::Foreign(..) => {
434                        return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
435                    }
436                    ty::Slice(_) | ty::Str => scalar_unit(Int(dl.ptr_sized_integer(), false)),
437                    ty::Dynamic(..) => {
438                        let mut vtable = scalar_unit(Pointer(AddressSpace::DATA));
439                        vtable.valid_range_mut().start = 1;
440                        vtable
441                    }
442                    _ => {
443                        return Err(error(cx, LayoutError::Unknown(pointee)));
444                    }
445                }
446            };
447
448            // Effectively a (ptr, meta) tuple.
449            tcx.mk_layout(LayoutData::scalar_pair(cx, data_ptr, metadata))
450        }
451
452        ty::Dynamic(_, _, ty::DynStar) => {
453            let mut data = scalar_unit(Pointer(AddressSpace::DATA));
454            data.valid_range_mut().start = 0;
455            let mut vtable = scalar_unit(Pointer(AddressSpace::DATA));
456            vtable.valid_range_mut().start = 1;
457            tcx.mk_layout(LayoutData::scalar_pair(cx, data, vtable))
458        }
459
460        // Arrays and slices.
461        ty::Array(element, count) => {
462            let count = extract_const_value(cx, ty, count)?
463                .try_to_target_usize(tcx)
464                .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;
465
466            let element = cx.layout_of(element)?;
467            map_layout(cx.calc.array_like(&element, Some(count)))?
468        }
469        ty::Slice(element) => {
470            let element = cx.layout_of(element)?;
471            map_layout(cx.calc.array_like(&element, None).map(|mut layout| {
472                // a randomly chosen value to distinguish slices
473                layout.randomization_seed = Hash64::new(0x2dcba99c39784102);
474                layout
475            }))?
476        }
477        ty::Str => {
478            let element = scalar(Int(I8, false));
479            map_layout(cx.calc.array_like(&element, None).map(|mut layout| {
480                // another random value
481                layout.randomization_seed = Hash64::new(0xc1325f37d127be22);
482                layout
483            }))?
484        }
485
486        // Odd unit types.
487        ty::FnDef(..) | ty::Dynamic(_, _, ty::Dyn) | ty::Foreign(..) => {
488            let sized = matches!(ty.kind(), ty::FnDef(..));
489            tcx.mk_layout(LayoutData::unit(cx, sized))
490        }
491
492        ty::Coroutine(def_id, args) => {
493            use rustc_middle::ty::layout::PrimitiveExt as _;
494
495            let info = tcx.coroutine_layout(def_id, args)?;
496
497            let local_layouts = info
498                .field_tys
499                .iter()
500                .map(|local| {
501                    let field_ty = EarlyBinder::bind(local.ty);
502                    let uninit_ty = Ty::new_maybe_uninit(tcx, field_ty.instantiate(tcx, args));
503                    cx.spanned_layout_of(uninit_ty, local.source_info.span)
504                })
505                .try_collect::<IndexVec<_, _>>()?;
506
507            let prefix_layouts = args
508                .as_coroutine()
509                .prefix_tys()
510                .iter()
511                .map(|ty| cx.layout_of(ty))
512                .try_collect::<IndexVec<_, _>>()?;
513
514            let layout = cx
515                .calc
516                .coroutine(
517                    &local_layouts,
518                    prefix_layouts,
519                    &info.variant_fields,
520                    &info.storage_conflicts,
521                    |tag| TyAndLayout {
522                        ty: tag.primitive().to_ty(tcx),
523                        layout: tcx.mk_layout(LayoutData::scalar(cx, tag)),
524                    },
525                )
526                .map(|mut layout| {
527                    // this is similar to how ReprOptions populates its field_shuffle_seed
528                    layout.randomization_seed = tcx.def_path_hash(def_id).0.to_smaller_hash();
529                    debug!("coroutine layout ({:?}): {:#?}", ty, layout);
530                    layout
531                });
532            map_layout(layout)?
533        }
534
535        ty::Closure(_, args) => univariant(args.as_closure().upvar_tys(), StructKind::AlwaysSized)?,
536
537        ty::CoroutineClosure(_, args) => {
538            univariant(args.as_coroutine_closure().upvar_tys(), StructKind::AlwaysSized)?
539        }
540
541        ty::Tuple(tys) => {
542            let kind =
543                if tys.len() == 0 { StructKind::AlwaysSized } else { StructKind::MaybeUnsized };
544
545            univariant(tys, kind)?
546        }
547
548        // SIMD vector types.
549        ty::Adt(def, args) if def.repr().simd() => {
550            // Supported SIMD vectors are ADTs with a single array field:
551            //
552            // * #[repr(simd)] struct S([T; 4])
553            //
554            // where T is a primitive scalar (integer/float/pointer).
555            let Some(ty::Array(e_ty, e_len)) = def
556                .is_struct()
557                .then(|| &def.variant(FIRST_VARIANT).fields)
558                .filter(|fields| fields.len() == 1)
559                .map(|fields| *fields[FieldIdx::ZERO].ty(tcx, args).kind())
560            else {
561                // Invalid SIMD types should have been caught by typeck by now.
562                let guar = tcx.dcx().delayed_bug("#[repr(simd)] was applied to an invalid ADT");
563                return Err(error(cx, LayoutError::ReferencesError(guar)));
564            };
565
566            let e_len = extract_const_value(cx, ty, e_len)?
567                .try_to_target_usize(tcx)
568                .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;
569
570            let e_ly = cx.layout_of(e_ty)?;
571
572            map_layout(cx.calc.simd_type(e_ly, e_len, def.repr().packed()))?
573        }
574
575        // ADTs.
576        ty::Adt(def, args) => {
577            // Cache the field layouts.
578            let variants = def
579                .variants()
580                .iter()
581                .map(|v| {
582                    v.fields
583                        .iter()
584                        .map(|field| cx.layout_of(field.ty(tcx, args)))
585                        .try_collect::<IndexVec<_, _>>()
586                })
587                .try_collect::<IndexVec<VariantIdx, _>>()?;
588
589            if def.is_union() {
590                if def.repr().pack.is_some() && def.repr().align.is_some() {
591                    let guar = tcx.dcx().span_delayed_bug(
592                        tcx.def_span(def.did()),
593                        "union cannot be packed and aligned",
594                    );
595                    return Err(error(cx, LayoutError::ReferencesError(guar)));
596                }
597
598                return map_layout(cx.calc.layout_of_union(&def.repr(), &variants));
599            }
600
601            // UnsafeCell and UnsafePinned both disable niche optimizations
602            let is_special_no_niche = def.is_unsafe_cell() || def.is_unsafe_pinned();
603
604            let get_discriminant_type =
605                |min, max| abi::Integer::repr_discr(tcx, ty, &def.repr(), min, max);
606
607            let discriminants_iter = || {
608                def.is_enum()
609                    .then(|| def.discriminants(tcx).map(|(v, d)| (v, d.val as i128)))
610                    .into_iter()
611                    .flatten()
612            };
613
614            let dont_niche_optimize_enum = def.repr().inhibit_enum_layout_opt()
615                || def
616                    .variants()
617                    .iter_enumerated()
618                    .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()));
619
620            let maybe_unsized = def.is_struct()
621                && def.non_enum_variant().tail_opt().is_some_and(|last_field| {
622                    let typing_env = ty::TypingEnv::post_analysis(tcx, def.did());
623                    !tcx.type_of(last_field.did).instantiate_identity().is_sized(tcx, typing_env)
624                });
625
626            let layout = cx
627                .calc
628                .layout_of_struct_or_enum(
629                    &def.repr(),
630                    &variants,
631                    def.is_enum(),
632                    is_special_no_niche,
633                    tcx.layout_scalar_valid_range(def.did()),
634                    get_discriminant_type,
635                    discriminants_iter(),
636                    dont_niche_optimize_enum,
637                    !maybe_unsized,
638                )
639                .map_err(|err| map_error(cx, ty, err))?;
640
641            if !maybe_unsized && layout.is_unsized() {
642                bug!("got unsized layout for type that cannot be unsized {ty:?}: {layout:#?}");
643            }
644
645            // If the struct tail is sized and can be unsized, check that unsizing doesn't move the fields around.
646            if cfg!(debug_assertions)
647                && maybe_unsized
648                && def.non_enum_variant().tail().ty(tcx, args).is_sized(tcx, cx.typing_env)
649            {
650                let mut variants = variants;
651                let tail_replacement = cx.layout_of(Ty::new_slice(tcx, tcx.types.u8)).unwrap();
652                *variants[FIRST_VARIANT].raw.last_mut().unwrap() = tail_replacement;
653
654                let Ok(unsized_layout) = cx.calc.layout_of_struct_or_enum(
655                    &def.repr(),
656                    &variants,
657                    def.is_enum(),
658                    is_special_no_niche,
659                    tcx.layout_scalar_valid_range(def.did()),
660                    get_discriminant_type,
661                    discriminants_iter(),
662                    dont_niche_optimize_enum,
663                    !maybe_unsized,
664                ) else {
665                    bug!("failed to compute unsized layout of {ty:?}");
666                };
667
668                let FieldsShape::Arbitrary { offsets: sized_offsets, .. } = &layout.fields else {
669                    bug!("unexpected FieldsShape for sized layout of {ty:?}: {:?}", layout.fields);
670                };
671                let FieldsShape::Arbitrary { offsets: unsized_offsets, .. } =
672                    &unsized_layout.fields
673                else {
674                    bug!(
675                        "unexpected FieldsShape for unsized layout of {ty:?}: {:?}",
676                        unsized_layout.fields
677                    );
678                };
679
680                let (sized_tail, sized_fields) = sized_offsets.raw.split_last().unwrap();
681                let (unsized_tail, unsized_fields) = unsized_offsets.raw.split_last().unwrap();
682
683                if sized_fields != unsized_fields {
684                    bug!("unsizing {ty:?} changed field order!\n{layout:?}\n{unsized_layout:?}");
685                }
686
687                if sized_tail < unsized_tail {
688                    bug!("unsizing {ty:?} moved tail backwards!\n{layout:?}\n{unsized_layout:?}");
689                }
690            }
691
692            tcx.mk_layout(layout)
693        }
694
695        ty::UnsafeBinder(bound_ty) => {
696            let ty = tcx.instantiate_bound_regions_with_erased(bound_ty.into());
697            cx.layout_of(ty)?.layout
698        }
699
700        // Types with no meaningful known layout.
701        ty::Param(_) | ty::Placeholder(..) => {
702            return Err(error(cx, LayoutError::TooGeneric(ty)));
703        }
704
705        ty::Alias(..) => {
706            // NOTE(eddyb) `layout_of` query should've normalized these away,
707            // if that was possible, so there's no reason to try again here.
708            let err = if ty.has_param() {
709                LayoutError::TooGeneric(ty)
710            } else {
711                // This is only reachable with unsatisfiable predicates. For example, if we have
712                // `u8: Iterator`, then we can't compute the layout of `<u8 as Iterator>::Item`.
713                LayoutError::Unknown(ty)
714            };
715            return Err(error(cx, err));
716        }
717
718        ty::Bound(..) | ty::CoroutineWitness(..) | ty::Infer(_) | ty::Error(_) => {
719            // `ty::Error` is handled at the top of this function.
720            bug!("layout_of: unexpected type `{ty}`")
721        }
722    })
723}
724
725fn record_layout_for_printing<'tcx>(cx: &LayoutCx<'tcx>, layout: TyAndLayout<'tcx>) {
726    // Ignore layouts that are done with non-empty environments or
727    // non-monomorphic layouts, as the user only wants to see the stuff
728    // resulting from the final codegen session.
729    if layout.ty.has_non_region_param() || !cx.typing_env.param_env.caller_bounds().is_empty() {
730        return;
731    }
732
733    // (delay format until we actually need it)
734    let record = |kind, packed, opt_discr_size, variants| {
735        let type_desc = with_no_trimmed_paths!(format!("{}", layout.ty));
736        cx.tcx().sess.code_stats.record_type_size(
737            kind,
738            type_desc,
739            layout.align.abi,
740            layout.size,
741            packed,
742            opt_discr_size,
743            variants,
744        );
745    };
746
747    match *layout.ty.kind() {
748        ty::Adt(adt_def, _) => {
749            debug!("print-type-size t: `{:?}` process adt", layout.ty);
750            let adt_kind = adt_def.adt_kind();
751            let adt_packed = adt_def.repr().pack.is_some();
752            let (variant_infos, opt_discr_size) = variant_info_for_adt(cx, layout, adt_def);
753            record(adt_kind.into(), adt_packed, opt_discr_size, variant_infos);
754        }
755
756        ty::Coroutine(def_id, args) => {
757            debug!("print-type-size t: `{:?}` record coroutine", layout.ty);
758            // Coroutines always have a begin/poisoned/end state with additional suspend points
759            let (variant_infos, opt_discr_size) =
760                variant_info_for_coroutine(cx, layout, def_id, args);
761            record(DataTypeKind::Coroutine, false, opt_discr_size, variant_infos);
762        }
763
764        ty::Closure(..) => {
765            debug!("print-type-size t: `{:?}` record closure", layout.ty);
766            record(DataTypeKind::Closure, false, None, vec![]);
767        }
768
769        _ => {
770            debug!("print-type-size t: `{:?}` skip non-nominal", layout.ty);
771        }
772    };
773}
774
775fn variant_info_for_adt<'tcx>(
776    cx: &LayoutCx<'tcx>,
777    layout: TyAndLayout<'tcx>,
778    adt_def: AdtDef<'tcx>,
779) -> (Vec<VariantInfo>, Option<Size>) {
780    let build_variant_info = |n: Option<Symbol>, flds: &[Symbol], layout: TyAndLayout<'tcx>| {
781        let mut min_size = Size::ZERO;
782        let field_info: Vec<_> = flds
783            .iter()
784            .enumerate()
785            .map(|(i, &name)| {
786                let field_layout = layout.field(cx, i);
787                let offset = layout.fields.offset(i);
788                min_size = min_size.max(offset + field_layout.size);
789                FieldInfo {
790                    kind: FieldKind::AdtField,
791                    name,
792                    offset: offset.bytes(),
793                    size: field_layout.size.bytes(),
794                    align: field_layout.align.abi.bytes(),
795                    type_name: None,
796                }
797            })
798            .collect();
799
800        VariantInfo {
801            name: n,
802            kind: if layout.is_unsized() { SizeKind::Min } else { SizeKind::Exact },
803            align: layout.align.abi.bytes(),
804            size: if min_size.bytes() == 0 { layout.size.bytes() } else { min_size.bytes() },
805            fields: field_info,
806        }
807    };
808
809    match layout.variants {
810        Variants::Empty => (vec![], None),
811
812        Variants::Single { index } => {
813            debug!("print-type-size `{:#?}` variant {}", layout, adt_def.variant(index).name);
814            let variant_def = &adt_def.variant(index);
815            let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect();
816            (vec![build_variant_info(Some(variant_def.name), &fields, layout)], None)
817        }
818
819        Variants::Multiple { tag, ref tag_encoding, .. } => {
820            debug!(
821                "print-type-size `{:#?}` adt general variants def {}",
822                layout.ty,
823                adt_def.variants().len()
824            );
825            let variant_infos: Vec<_> = adt_def
826                .variants()
827                .iter_enumerated()
828                .map(|(i, variant_def)| {
829                    let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect();
830                    build_variant_info(Some(variant_def.name), &fields, layout.for_variant(cx, i))
831                })
832                .collect();
833
834            (
835                variant_infos,
836                match tag_encoding {
837                    TagEncoding::Direct => Some(tag.size(cx)),
838                    _ => None,
839                },
840            )
841        }
842    }
843}
844
845fn variant_info_for_coroutine<'tcx>(
846    cx: &LayoutCx<'tcx>,
847    layout: TyAndLayout<'tcx>,
848    def_id: DefId,
849    args: ty::GenericArgsRef<'tcx>,
850) -> (Vec<VariantInfo>, Option<Size>) {
851    use itertools::Itertools;
852
853    let Variants::Multiple { tag, ref tag_encoding, tag_field, .. } = layout.variants else {
854        return (vec![], None);
855    };
856
857    let coroutine = cx.tcx().coroutine_layout(def_id, args).unwrap();
858    let upvar_names = cx.tcx().closure_saved_names_of_captured_variables(def_id);
859
860    let mut upvars_size = Size::ZERO;
861    let upvar_fields: Vec<_> = args
862        .as_coroutine()
863        .upvar_tys()
864        .iter()
865        .zip_eq(upvar_names)
866        .enumerate()
867        .map(|(field_idx, (_, name))| {
868            let field_layout = layout.field(cx, field_idx);
869            let offset = layout.fields.offset(field_idx);
870            upvars_size = upvars_size.max(offset + field_layout.size);
871            FieldInfo {
872                kind: FieldKind::Upvar,
873                name: *name,
874                offset: offset.bytes(),
875                size: field_layout.size.bytes(),
876                align: field_layout.align.abi.bytes(),
877                type_name: None,
878            }
879        })
880        .collect();
881
882    let mut variant_infos: Vec<_> = coroutine
883        .variant_fields
884        .iter_enumerated()
885        .map(|(variant_idx, variant_def)| {
886            let variant_layout = layout.for_variant(cx, variant_idx);
887            let mut variant_size = Size::ZERO;
888            let fields = variant_def
889                .iter()
890                .enumerate()
891                .map(|(field_idx, local)| {
892                    let field_name = coroutine.field_names[*local];
893                    let field_layout = variant_layout.field(cx, field_idx);
894                    let offset = variant_layout.fields.offset(field_idx);
895                    // The struct is as large as the last field's end
896                    variant_size = variant_size.max(offset + field_layout.size);
897                    FieldInfo {
898                        kind: FieldKind::CoroutineLocal,
899                        name: field_name.unwrap_or(Symbol::intern(&format!(
900                            ".coroutine_field{}",
901                            local.as_usize()
902                        ))),
903                        offset: offset.bytes(),
904                        size: field_layout.size.bytes(),
905                        align: field_layout.align.abi.bytes(),
906                        // Include the type name if there is no field name, or if the name is the
907                        // __awaitee placeholder symbol which means a child future being `.await`ed.
908                        type_name: (field_name.is_none() || field_name == Some(sym::__awaitee))
909                            .then(|| Symbol::intern(&field_layout.ty.to_string())),
910                    }
911                })
912                .chain(upvar_fields.iter().copied())
913                .collect();
914
915            // If the variant has no state-specific fields, then it's the size of the upvars.
916            if variant_size == Size::ZERO {
917                variant_size = upvars_size;
918            }
919
920            // This `if` deserves some explanation.
921            //
922            // The layout code has a choice of where to place the discriminant of this coroutine.
923            // If the discriminant of the coroutine is placed early in the layout (before the
924            // variant's own fields), then it'll implicitly be counted towards the size of the
925            // variant, since we use the maximum offset to calculate size.
926            //    (side-note: I know this is a bit problematic given upvars placement, etc).
927            //
928            // This is important, since the layout printing code always subtracts this discriminant
929            // size from the variant size if the struct is "enum"-like, so failing to account for it
930            // will either lead to numerical underflow, or an underreported variant size...
931            //
932            // However, if the discriminant is placed past the end of the variant, then we need
933            // to factor in the size of the discriminant manually. This really should be refactored
934            // better, but this "works" for now.
935            if layout.fields.offset(tag_field) >= variant_size {
936                variant_size += match tag_encoding {
937                    TagEncoding::Direct => tag.size(cx),
938                    _ => Size::ZERO,
939                };
940            }
941
942            VariantInfo {
943                name: Some(Symbol::intern(&ty::CoroutineArgs::variant_name(variant_idx))),
944                kind: SizeKind::Exact,
945                size: variant_size.bytes(),
946                align: variant_layout.align.abi.bytes(),
947                fields,
948            }
949        })
950        .collect();
951
952    // The first three variants are hardcoded to be `UNRESUMED`, `RETURNED` and `POISONED`.
953    // We will move the `RETURNED` and `POISONED` elements to the end so we
954    // are left with a sorting order according to the coroutines yield points:
955    // First `Unresumed`, then the `SuspendN` followed by `Returned` and `Panicked` (POISONED).
956    let end_states = variant_infos.drain(1..=2);
957    let end_states: Vec<_> = end_states.collect();
958    variant_infos.extend(end_states);
959
960    (
961        variant_infos,
962        match tag_encoding {
963            TagEncoding::Direct => Some(tag.size(cx)),
964            _ => None,
965        },
966    )
967}