rustc_codegen_llvm/
intrinsic.rs

1use std::assert_matches::assert_matches;
2use std::cmp::Ordering;
3
4use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size};
5use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
6use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
7use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
8use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
9use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
10use rustc_codegen_ssa::traits::*;
11use rustc_hir as hir;
12use rustc_middle::mir::BinOp;
13use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
14use rustc_middle::ty::{self, GenericArgsRef, Ty};
15use rustc_middle::{bug, span_bug};
16use rustc_span::{Span, Symbol, sym};
17use rustc_symbol_mangling::mangle_internal_symbol;
18use rustc_target::spec::{HasTargetSpec, PanicStrategy};
19use tracing::debug;
20
21use crate::abi::FnAbiLlvmExt;
22use crate::builder::Builder;
23use crate::context::CodegenCx;
24use crate::llvm::{self, Metadata};
25use crate::type_::Type;
26use crate::type_of::LayoutLlvmExt;
27use crate::va_arg::emit_va_arg;
28use crate::value::Value;
29
30fn get_simple_intrinsic<'ll>(
31    cx: &CodegenCx<'ll, '_>,
32    name: Symbol,
33) -> Option<(&'ll Type, &'ll Value)> {
34    let llvm_name = match name {
35        sym::sqrtf16 => "llvm.sqrt.f16",
36        sym::sqrtf32 => "llvm.sqrt.f32",
37        sym::sqrtf64 => "llvm.sqrt.f64",
38        sym::sqrtf128 => "llvm.sqrt.f128",
39
40        sym::powif16 => "llvm.powi.f16.i32",
41        sym::powif32 => "llvm.powi.f32.i32",
42        sym::powif64 => "llvm.powi.f64.i32",
43        sym::powif128 => "llvm.powi.f128.i32",
44
45        sym::sinf16 => "llvm.sin.f16",
46        sym::sinf32 => "llvm.sin.f32",
47        sym::sinf64 => "llvm.sin.f64",
48        sym::sinf128 => "llvm.sin.f128",
49
50        sym::cosf16 => "llvm.cos.f16",
51        sym::cosf32 => "llvm.cos.f32",
52        sym::cosf64 => "llvm.cos.f64",
53        sym::cosf128 => "llvm.cos.f128",
54
55        sym::powf16 => "llvm.pow.f16",
56        sym::powf32 => "llvm.pow.f32",
57        sym::powf64 => "llvm.pow.f64",
58        sym::powf128 => "llvm.pow.f128",
59
60        sym::expf16 => "llvm.exp.f16",
61        sym::expf32 => "llvm.exp.f32",
62        sym::expf64 => "llvm.exp.f64",
63        sym::expf128 => "llvm.exp.f128",
64
65        sym::exp2f16 => "llvm.exp2.f16",
66        sym::exp2f32 => "llvm.exp2.f32",
67        sym::exp2f64 => "llvm.exp2.f64",
68        sym::exp2f128 => "llvm.exp2.f128",
69
70        sym::logf16 => "llvm.log.f16",
71        sym::logf32 => "llvm.log.f32",
72        sym::logf64 => "llvm.log.f64",
73        sym::logf128 => "llvm.log.f128",
74
75        sym::log10f16 => "llvm.log10.f16",
76        sym::log10f32 => "llvm.log10.f32",
77        sym::log10f64 => "llvm.log10.f64",
78        sym::log10f128 => "llvm.log10.f128",
79
80        sym::log2f16 => "llvm.log2.f16",
81        sym::log2f32 => "llvm.log2.f32",
82        sym::log2f64 => "llvm.log2.f64",
83        sym::log2f128 => "llvm.log2.f128",
84
85        sym::fmaf16 => "llvm.fma.f16",
86        sym::fmaf32 => "llvm.fma.f32",
87        sym::fmaf64 => "llvm.fma.f64",
88        sym::fmaf128 => "llvm.fma.f128",
89
90        sym::fmuladdf16 => "llvm.fmuladd.f16",
91        sym::fmuladdf32 => "llvm.fmuladd.f32",
92        sym::fmuladdf64 => "llvm.fmuladd.f64",
93        sym::fmuladdf128 => "llvm.fmuladd.f128",
94
95        sym::fabsf16 => "llvm.fabs.f16",
96        sym::fabsf32 => "llvm.fabs.f32",
97        sym::fabsf64 => "llvm.fabs.f64",
98        sym::fabsf128 => "llvm.fabs.f128",
99
100        sym::minnumf16 => "llvm.minnum.f16",
101        sym::minnumf32 => "llvm.minnum.f32",
102        sym::minnumf64 => "llvm.minnum.f64",
103        sym::minnumf128 => "llvm.minnum.f128",
104
105        sym::minimumf16 => "llvm.minimum.f16",
106        sym::minimumf32 => "llvm.minimum.f32",
107        sym::minimumf64 => "llvm.minimum.f64",
108        // There are issues on x86_64 and aarch64 with the f128 variant,
109        // let's instead use the instrinsic fallback body.
110        // sym::minimumf128 => "llvm.minimum.f128",
111        sym::maxnumf16 => "llvm.maxnum.f16",
112        sym::maxnumf32 => "llvm.maxnum.f32",
113        sym::maxnumf64 => "llvm.maxnum.f64",
114        sym::maxnumf128 => "llvm.maxnum.f128",
115
116        sym::maximumf16 => "llvm.maximum.f16",
117        sym::maximumf32 => "llvm.maximum.f32",
118        sym::maximumf64 => "llvm.maximum.f64",
119        // There are issues on x86_64 and aarch64 with the f128 variant,
120        // let's instead use the instrinsic fallback body.
121        // sym::maximumf128 => "llvm.maximum.f128",
122        sym::copysignf16 => "llvm.copysign.f16",
123        sym::copysignf32 => "llvm.copysign.f32",
124        sym::copysignf64 => "llvm.copysign.f64",
125        sym::copysignf128 => "llvm.copysign.f128",
126
127        sym::floorf16 => "llvm.floor.f16",
128        sym::floorf32 => "llvm.floor.f32",
129        sym::floorf64 => "llvm.floor.f64",
130        sym::floorf128 => "llvm.floor.f128",
131
132        sym::ceilf16 => "llvm.ceil.f16",
133        sym::ceilf32 => "llvm.ceil.f32",
134        sym::ceilf64 => "llvm.ceil.f64",
135        sym::ceilf128 => "llvm.ceil.f128",
136
137        sym::truncf16 => "llvm.trunc.f16",
138        sym::truncf32 => "llvm.trunc.f32",
139        sym::truncf64 => "llvm.trunc.f64",
140        sym::truncf128 => "llvm.trunc.f128",
141
142        // We could use any of `rint`, `nearbyint`, or `roundeven`
143        // for this -- they are all identical in semantics when
144        // assuming the default FP environment.
145        // `rint` is what we used for $forever.
146        sym::round_ties_even_f16 => "llvm.rint.f16",
147        sym::round_ties_even_f32 => "llvm.rint.f32",
148        sym::round_ties_even_f64 => "llvm.rint.f64",
149        sym::round_ties_even_f128 => "llvm.rint.f128",
150
151        sym::roundf16 => "llvm.round.f16",
152        sym::roundf32 => "llvm.round.f32",
153        sym::roundf64 => "llvm.round.f64",
154        sym::roundf128 => "llvm.round.f128",
155
156        sym::ptr_mask => "llvm.ptrmask",
157
158        _ => return None,
159    };
160    Some(cx.get_intrinsic(llvm_name))
161}
162
163impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
164    fn codegen_intrinsic_call(
165        &mut self,
166        instance: ty::Instance<'tcx>,
167        args: &[OperandRef<'tcx, &'ll Value>],
168        result: PlaceRef<'tcx, &'ll Value>,
169        span: Span,
170    ) -> Result<(), ty::Instance<'tcx>> {
171        let tcx = self.tcx;
172        let callee_ty = instance.ty(tcx, self.typing_env());
173
174        let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else {
175            bug!("expected fn item type, found {}", callee_ty);
176        };
177
178        let sig = callee_ty.fn_sig(tcx);
179        let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig);
180        let arg_tys = sig.inputs();
181        let ret_ty = sig.output();
182        let name = tcx.item_name(def_id);
183
184        let llret_ty = self.layout_of(ret_ty).llvm_type(self);
185
186        let simple = get_simple_intrinsic(self, name);
187        let llval = match name {
188            _ if simple.is_some() => {
189                let (simple_ty, simple_fn) = simple.unwrap();
190                self.call(
191                    simple_ty,
192                    None,
193                    None,
194                    simple_fn,
195                    &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
196                    None,
197                    Some(instance),
198                )
199            }
200            sym::is_val_statically_known => {
201                let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx);
202                let kind = self.type_kind(intrinsic_type);
203                let intrinsic_name = match kind {
204                    TypeKind::Pointer | TypeKind::Integer => {
205                        Some(format!("llvm.is.constant.{intrinsic_type:?}"))
206                    }
207                    // LLVM float types' intrinsic names differ from their type names.
208                    TypeKind::Half => Some(format!("llvm.is.constant.f16")),
209                    TypeKind::Float => Some(format!("llvm.is.constant.f32")),
210                    TypeKind::Double => Some(format!("llvm.is.constant.f64")),
211                    TypeKind::FP128 => Some(format!("llvm.is.constant.f128")),
212                    _ => None,
213                };
214                if let Some(intrinsic_name) = intrinsic_name {
215                    self.call_intrinsic(&intrinsic_name, &[args[0].immediate()])
216                } else {
217                    self.const_bool(false)
218                }
219            }
220            sym::select_unpredictable => {
221                let cond = args[0].immediate();
222                assert_eq!(args[1].layout, args[2].layout);
223                let select = |bx: &mut Self, true_val, false_val| {
224                    let result = bx.select(cond, true_val, false_val);
225                    bx.set_unpredictable(&result);
226                    result
227                };
228                match (args[1].val, args[2].val) {
229                    (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
230                        assert!(true_val.llextra.is_none());
231                        assert!(false_val.llextra.is_none());
232                        assert_eq!(true_val.align, false_val.align);
233                        let ptr = select(self, true_val.llval, false_val.llval);
234                        let selected =
235                            OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
236                        selected.store(self, result);
237                        return Ok(());
238                    }
239                    (OperandValue::Immediate(_), OperandValue::Immediate(_))
240                    | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
241                        let true_val = args[1].immediate_or_packed_pair(self);
242                        let false_val = args[2].immediate_or_packed_pair(self);
243                        select(self, true_val, false_val)
244                    }
245                    (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
246                    _ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
247                }
248            }
249            sym::catch_unwind => {
250                catch_unwind_intrinsic(
251                    self,
252                    args[0].immediate(),
253                    args[1].immediate(),
254                    args[2].immediate(),
255                    result,
256                );
257                return Ok(());
258            }
259            sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]),
260            sym::va_copy => {
261                self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()])
262            }
263            sym::va_arg => {
264                match result.layout.backend_repr {
265                    BackendRepr::Scalar(scalar) => {
266                        match scalar.primitive() {
267                            Primitive::Int(..) => {
268                                if self.cx().size_of(ret_ty).bytes() < 4 {
269                                    // `va_arg` should not be called on an integer type
270                                    // less than 4 bytes in length. If it is, promote
271                                    // the integer to an `i32` and truncate the result
272                                    // back to the smaller type.
273                                    let promoted_result = emit_va_arg(self, args[0], tcx.types.i32);
274                                    self.trunc(promoted_result, llret_ty)
275                                } else {
276                                    emit_va_arg(self, args[0], ret_ty)
277                                }
278                            }
279                            Primitive::Float(Float::F16) => {
280                                bug!("the va_arg intrinsic does not work with `f16`")
281                            }
282                            Primitive::Float(Float::F64) | Primitive::Pointer(_) => {
283                                emit_va_arg(self, args[0], ret_ty)
284                            }
285                            // `va_arg` should never be used with the return type f32.
286                            Primitive::Float(Float::F32) => {
287                                bug!("the va_arg intrinsic does not work with `f32`")
288                            }
289                            Primitive::Float(Float::F128) => {
290                                bug!("the va_arg intrinsic does not work with `f128`")
291                            }
292                        }
293                    }
294                    _ => bug!("the va_arg intrinsic does not work with non-scalar types"),
295                }
296            }
297
298            sym::volatile_load | sym::unaligned_volatile_load => {
299                let ptr = args[0].immediate();
300                let load = self.volatile_load(result.layout.llvm_type(self), ptr);
301                let align = if name == sym::unaligned_volatile_load {
302                    1
303                } else {
304                    result.layout.align.abi.bytes() as u32
305                };
306                unsafe {
307                    llvm::LLVMSetAlignment(load, align);
308                }
309                if !result.layout.is_zst() {
310                    self.store_to_place(load, result.val);
311                }
312                return Ok(());
313            }
314            sym::volatile_store => {
315                let dst = args[0].deref(self.cx());
316                args[1].val.volatile_store(self, dst);
317                return Ok(());
318            }
319            sym::unaligned_volatile_store => {
320                let dst = args[0].deref(self.cx());
321                args[1].val.unaligned_volatile_store(self, dst);
322                return Ok(());
323            }
324            sym::prefetch_read_data
325            | sym::prefetch_write_data
326            | sym::prefetch_read_instruction
327            | sym::prefetch_write_instruction => {
328                let (rw, cache_type) = match name {
329                    sym::prefetch_read_data => (0, 1),
330                    sym::prefetch_write_data => (1, 1),
331                    sym::prefetch_read_instruction => (0, 0),
332                    sym::prefetch_write_instruction => (1, 0),
333                    _ => bug!(),
334                };
335                self.call_intrinsic(
336                    "llvm.prefetch",
337                    &[
338                        args[0].immediate(),
339                        self.const_i32(rw),
340                        args[1].immediate(),
341                        self.const_i32(cache_type),
342                    ],
343                )
344            }
345            sym::carrying_mul_add => {
346                let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
347
348                let wide_llty = self.type_ix(size.bits() * 2);
349                let args = args.as_array().unwrap();
350                let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
351
352                let wide = if signed {
353                    let prod = self.unchecked_smul(a, b);
354                    let acc = self.unchecked_sadd(prod, c);
355                    self.unchecked_sadd(acc, d)
356                } else {
357                    let prod = self.unchecked_umul(a, b);
358                    let acc = self.unchecked_uadd(prod, c);
359                    self.unchecked_uadd(acc, d)
360                };
361
362                let narrow_llty = self.type_ix(size.bits());
363                let low = self.trunc(wide, narrow_llty);
364                let bits_const = self.const_uint(wide_llty, size.bits());
365                // No need for ashr when signed; LLVM changes it to lshr anyway.
366                let high = self.lshr(wide, bits_const);
367                // FIXME: could be `trunc nuw`, even for signed.
368                let high = self.trunc(high, narrow_llty);
369
370                let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false);
371                let pair = self.const_poison(pair_llty);
372                let pair = self.insert_value(pair, low, 0);
373                let pair = self.insert_value(pair, high, 1);
374                pair
375            }
376            sym::ctlz
377            | sym::ctlz_nonzero
378            | sym::cttz
379            | sym::cttz_nonzero
380            | sym::ctpop
381            | sym::bswap
382            | sym::bitreverse
383            | sym::rotate_left
384            | sym::rotate_right
385            | sym::saturating_add
386            | sym::saturating_sub => {
387                let ty = arg_tys[0];
388                if !ty.is_integral() {
389                    tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
390                        span,
391                        name,
392                        ty,
393                    });
394                    return Ok(());
395                }
396                let (size, signed) = ty.int_size_and_signed(self.tcx);
397                let width = size.bits();
398                match name {
399                    sym::ctlz | sym::cttz => {
400                        let y = self.const_bool(false);
401                        let ret = self.call_intrinsic(
402                            &format!("llvm.{name}.i{width}"),
403                            &[args[0].immediate(), y],
404                        );
405
406                        self.intcast(ret, llret_ty, false)
407                    }
408                    sym::ctlz_nonzero => {
409                        let y = self.const_bool(true);
410                        let llvm_name = &format!("llvm.ctlz.i{width}");
411                        let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
412                        self.intcast(ret, llret_ty, false)
413                    }
414                    sym::cttz_nonzero => {
415                        let y = self.const_bool(true);
416                        let llvm_name = &format!("llvm.cttz.i{width}");
417                        let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
418                        self.intcast(ret, llret_ty, false)
419                    }
420                    sym::ctpop => {
421                        let ret = self.call_intrinsic(
422                            &format!("llvm.ctpop.i{width}"),
423                            &[args[0].immediate()],
424                        );
425                        self.intcast(ret, llret_ty, false)
426                    }
427                    sym::bswap => {
428                        if width == 8 {
429                            args[0].immediate() // byte swap a u8/i8 is just a no-op
430                        } else {
431                            self.call_intrinsic(
432                                &format!("llvm.bswap.i{width}"),
433                                &[args[0].immediate()],
434                            )
435                        }
436                    }
437                    sym::bitreverse => self.call_intrinsic(
438                        &format!("llvm.bitreverse.i{width}"),
439                        &[args[0].immediate()],
440                    ),
441                    sym::rotate_left | sym::rotate_right => {
442                        let is_left = name == sym::rotate_left;
443                        let val = args[0].immediate();
444                        let raw_shift = args[1].immediate();
445                        // rotate = funnel shift with first two args the same
446                        let llvm_name =
447                            &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
448
449                        // llvm expects shift to be the same type as the values, but rust
450                        // always uses `u32`.
451                        let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
452
453                        self.call_intrinsic(llvm_name, &[val, val, raw_shift])
454                    }
455                    sym::saturating_add | sym::saturating_sub => {
456                        let is_add = name == sym::saturating_add;
457                        let lhs = args[0].immediate();
458                        let rhs = args[1].immediate();
459                        let llvm_name = &format!(
460                            "llvm.{}{}.sat.i{}",
461                            if signed { 's' } else { 'u' },
462                            if is_add { "add" } else { "sub" },
463                            width
464                        );
465                        self.call_intrinsic(llvm_name, &[lhs, rhs])
466                    }
467                    _ => bug!(),
468                }
469            }
470
471            sym::raw_eq => {
472                use BackendRepr::*;
473                let tp_ty = fn_args.type_at(0);
474                let layout = self.layout_of(tp_ty).layout;
475                let use_integer_compare = match layout.backend_repr() {
476                    Scalar(_) | ScalarPair(_, _) => true,
477                    SimdVector { .. } => false,
478                    Memory { .. } => {
479                        // For rusty ABIs, small aggregates are actually passed
480                        // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
481                        // so we re-use that same threshold here.
482                        layout.size() <= self.data_layout().pointer_size * 2
483                    }
484                };
485
486                let a = args[0].immediate();
487                let b = args[1].immediate();
488                if layout.size().bytes() == 0 {
489                    self.const_bool(true)
490                } else if use_integer_compare {
491                    let integer_ty = self.type_ix(layout.size().bits());
492                    let a_val = self.load(integer_ty, a, layout.align().abi);
493                    let b_val = self.load(integer_ty, b, layout.align().abi);
494                    self.icmp(IntPredicate::IntEQ, a_val, b_val)
495                } else {
496                    let n = self.const_usize(layout.size().bytes());
497                    let cmp = self.call_intrinsic("memcmp", &[a, b, n]);
498                    match self.cx.sess().target.arch.as_ref() {
499                        "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)),
500                        _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)),
501                    }
502                }
503            }
504
505            sym::compare_bytes => {
506                // Here we assume that the `memcmp` provided by the target is a NOP for size 0.
507                let cmp = self.call_intrinsic(
508                    "memcmp",
509                    &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
510                );
511                // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`.
512                self.sext(cmp, self.type_ix(32))
513            }
514
515            sym::black_box => {
516                args[0].val.store(self, result);
517                let result_val_span = [result.val.llval];
518                // We need to "use" the argument in some way LLVM can't introspect, and on
519                // targets that support it we can typically leverage inline assembly to do
520                // this. LLVM's interpretation of inline assembly is that it's, well, a black
521                // box. This isn't the greatest implementation since it probably deoptimizes
522                // more than we want, but it's so far good enough.
523                //
524                // For zero-sized types, the location pointed to by the result may be
525                // uninitialized. Do not "use" the result in this case; instead just clobber
526                // the memory.
527                let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
528                    ("~{memory}", &[])
529                } else {
530                    ("r,~{memory}", &result_val_span)
531                };
532                crate::asm::inline_asm_call(
533                    self,
534                    "",
535                    constraint,
536                    inputs,
537                    self.type_void(),
538                    &[],
539                    true,
540                    false,
541                    llvm::AsmDialect::Att,
542                    &[span],
543                    false,
544                    None,
545                    None,
546                )
547                .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
548
549                // We have copied the value to `result` already.
550                return Ok(());
551            }
552
553            _ if name.as_str().starts_with("simd_") => {
554                // Unpack non-power-of-2 #[repr(packed, simd)] arguments.
555                // This gives them the expected layout of a regular #[repr(simd)] vector.
556                let mut loaded_args = Vec::new();
557                for (ty, arg) in arg_tys.iter().zip(args) {
558                    loaded_args.push(
559                        // #[repr(packed, simd)] vectors are passed like arrays (as references,
560                        // with reduced alignment and no padding) rather than as immediates.
561                        // We can use a vector load to fix the layout and turn the argument
562                        // into an immediate.
563                        if ty.is_simd()
564                            && let OperandValue::Ref(place) = arg.val
565                        {
566                            let (size, elem_ty) = ty.simd_size_and_type(self.tcx());
567                            let elem_ll_ty = match elem_ty.kind() {
568                                ty::Float(f) => self.type_float_from_ty(*f),
569                                ty::Int(i) => self.type_int_from_ty(*i),
570                                ty::Uint(u) => self.type_uint_from_ty(*u),
571                                ty::RawPtr(_, _) => self.type_ptr(),
572                                _ => unreachable!(),
573                            };
574                            let loaded =
575                                self.load_from_place(self.type_vector(elem_ll_ty, size), place);
576                            OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
577                        } else {
578                            *arg
579                        },
580                    );
581                }
582
583                let llret_ty = if ret_ty.is_simd()
584                    && let BackendRepr::Memory { .. } = self.layout_of(ret_ty).layout.backend_repr
585                {
586                    let (size, elem_ty) = ret_ty.simd_size_and_type(self.tcx());
587                    let elem_ll_ty = match elem_ty.kind() {
588                        ty::Float(f) => self.type_float_from_ty(*f),
589                        ty::Int(i) => self.type_int_from_ty(*i),
590                        ty::Uint(u) => self.type_uint_from_ty(*u),
591                        ty::RawPtr(_, _) => self.type_ptr(),
592                        _ => unreachable!(),
593                    };
594                    self.type_vector(elem_ll_ty, size)
595                } else {
596                    llret_ty
597                };
598
599                match generic_simd_intrinsic(
600                    self,
601                    name,
602                    callee_ty,
603                    fn_args,
604                    &loaded_args,
605                    ret_ty,
606                    llret_ty,
607                    span,
608                ) {
609                    Ok(llval) => llval,
610                    // If there was an error, just skip this invocation... we'll abort compilation
611                    // anyway, but we can keep codegen'ing to find more errors.
612                    Err(()) => return Ok(()),
613                }
614            }
615
616            _ => {
617                debug!("unknown intrinsic '{}' -- falling back to default body", name);
618                // Call the fallback body instead of generating the intrinsic code
619                return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
620            }
621        };
622
623        if result.layout.ty.is_bool() {
624            OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
625                .val
626                .store(self, result);
627        } else if !result.layout.ty.is_unit() {
628            self.store_to_place(llval, result.val);
629        }
630        Ok(())
631    }
632
633    fn abort(&mut self) {
634        self.call_intrinsic("llvm.trap", &[]);
635    }
636
637    fn assume(&mut self, val: Self::Value) {
638        if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
639            self.call_intrinsic("llvm.assume", &[val]);
640        }
641    }
642
643    fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
644        if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
645            self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)])
646        } else {
647            cond
648        }
649    }
650
651    fn type_test(&mut self, pointer: Self::Value, typeid: Self::Metadata) -> Self::Value {
652        // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time
653        // optimization pass replaces calls to this intrinsic with code to test type membership.
654        let typeid = self.get_metadata_value(typeid);
655        self.call_intrinsic("llvm.type.test", &[pointer, typeid])
656    }
657
658    fn type_checked_load(
659        &mut self,
660        llvtable: &'ll Value,
661        vtable_byte_offset: u64,
662        typeid: &'ll Metadata,
663    ) -> Self::Value {
664        let typeid = self.get_metadata_value(typeid);
665        let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
666        let type_checked_load =
667            self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]);
668        self.extract_value(type_checked_load, 0)
669    }
670
671    fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
672        self.call_intrinsic("llvm.va_start", &[va_list])
673    }
674
675    fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
676        self.call_intrinsic("llvm.va_end", &[va_list])
677    }
678}
679
680fn catch_unwind_intrinsic<'ll, 'tcx>(
681    bx: &mut Builder<'_, 'll, 'tcx>,
682    try_func: &'ll Value,
683    data: &'ll Value,
684    catch_func: &'ll Value,
685    dest: PlaceRef<'tcx, &'ll Value>,
686) {
687    if bx.sess().panic_strategy() == PanicStrategy::Abort {
688        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
689        bx.call(try_func_ty, None, None, try_func, &[data], None, None);
690        // Return 0 unconditionally from the intrinsic call;
691        // we can never unwind.
692        OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
693    } else if wants_msvc_seh(bx.sess()) {
694        codegen_msvc_try(bx, try_func, data, catch_func, dest);
695    } else if wants_wasm_eh(bx.sess()) {
696        codegen_wasm_try(bx, try_func, data, catch_func, dest);
697    } else if bx.sess().target.os == "emscripten" {
698        codegen_emcc_try(bx, try_func, data, catch_func, dest);
699    } else {
700        codegen_gnu_try(bx, try_func, data, catch_func, dest);
701    }
702}
703
704// MSVC's definition of the `rust_try` function.
705//
706// This implementation uses the new exception handling instructions in LLVM
707// which have support in LLVM for SEH on MSVC targets. Although these
708// instructions are meant to work for all targets, as of the time of this
709// writing, however, LLVM does not recommend the usage of these new instructions
710// as the old ones are still more optimized.
711fn codegen_msvc_try<'ll, 'tcx>(
712    bx: &mut Builder<'_, 'll, 'tcx>,
713    try_func: &'ll Value,
714    data: &'ll Value,
715    catch_func: &'ll Value,
716    dest: PlaceRef<'tcx, &'ll Value>,
717) {
718    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
719        bx.set_personality_fn(bx.eh_personality());
720
721        let normal = bx.append_sibling_block("normal");
722        let catchswitch = bx.append_sibling_block("catchswitch");
723        let catchpad_rust = bx.append_sibling_block("catchpad_rust");
724        let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
725        let caught = bx.append_sibling_block("caught");
726
727        let try_func = llvm::get_param(bx.llfn(), 0);
728        let data = llvm::get_param(bx.llfn(), 1);
729        let catch_func = llvm::get_param(bx.llfn(), 2);
730
731        // We're generating an IR snippet that looks like:
732        //
733        //   declare i32 @rust_try(%try_func, %data, %catch_func) {
734        //      %slot = alloca i8*
735        //      invoke %try_func(%data) to label %normal unwind label %catchswitch
736        //
737        //   normal:
738        //      ret i32 0
739        //
740        //   catchswitch:
741        //      %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller
742        //
743        //   catchpad_rust:
744        //      %tok = catchpad within %cs [%type_descriptor, 8, %slot]
745        //      %ptr = load %slot
746        //      call %catch_func(%data, %ptr)
747        //      catchret from %tok to label %caught
748        //
749        //   catchpad_foreign:
750        //      %tok = catchpad within %cs [null, 64, null]
751        //      call %catch_func(%data, null)
752        //      catchret from %tok to label %caught
753        //
754        //   caught:
755        //      ret i32 1
756        //   }
757        //
758        // This structure follows the basic usage of throw/try/catch in LLVM.
759        // For example, compile this C++ snippet to see what LLVM generates:
760        //
761        //      struct rust_panic {
762        //          rust_panic(const rust_panic&);
763        //          ~rust_panic();
764        //
765        //          void* x[2];
766        //      };
767        //
768        //      int __rust_try(
769        //          void (*try_func)(void*),
770        //          void *data,
771        //          void (*catch_func)(void*, void*) noexcept
772        //      ) {
773        //          try {
774        //              try_func(data);
775        //              return 0;
776        //          } catch(rust_panic& a) {
777        //              catch_func(data, &a);
778        //              return 1;
779        //          } catch(...) {
780        //              catch_func(data, NULL);
781        //              return 1;
782        //          }
783        //      }
784        //
785        // More information can be found in libstd's seh.rs implementation.
786        let ptr_size = bx.tcx().data_layout.pointer_size;
787        let ptr_align = bx.tcx().data_layout.pointer_align.abi;
788        let slot = bx.alloca(ptr_size, ptr_align);
789        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
790        bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
791
792        bx.switch_to_block(normal);
793        bx.ret(bx.const_i32(0));
794
795        bx.switch_to_block(catchswitch);
796        let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
797
798        // We can't use the TypeDescriptor defined in libpanic_unwind because it
799        // might be in another DLL and the SEH encoding only supports specifying
800        // a TypeDescriptor from the current module.
801        //
802        // However this isn't an issue since the MSVC runtime uses string
803        // comparison on the type name to match TypeDescriptors rather than
804        // pointer equality.
805        //
806        // So instead we generate a new TypeDescriptor in each module that uses
807        // `try` and let the linker merge duplicate definitions in the same
808        // module.
809        //
810        // When modifying, make sure that the type_name string exactly matches
811        // the one used in library/panic_unwind/src/seh.rs.
812        let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
813        let type_name = bx.const_bytes(b"rust_panic\0");
814        let type_info =
815            bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
816        let tydesc = bx.declare_global(
817            &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
818            bx.val_ty(type_info),
819        );
820
821        llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
822        if bx.cx.tcx.sess.target.supports_comdat() {
823            llvm::SetUniqueComdat(bx.llmod, tydesc);
824        }
825        llvm::set_initializer(tydesc, type_info);
826
827        // The flag value of 8 indicates that we are catching the exception by
828        // reference instead of by value. We can't use catch by value because
829        // that requires copying the exception object, which we don't support
830        // since our exception object effectively contains a Box.
831        //
832        // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang
833        bx.switch_to_block(catchpad_rust);
834        let flags = bx.const_i32(8);
835        let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
836        let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
837        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
838        bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
839        bx.catch_ret(&funclet, caught);
840
841        // The flag value of 64 indicates a "catch-all".
842        bx.switch_to_block(catchpad_foreign);
843        let flags = bx.const_i32(64);
844        let null = bx.const_null(bx.type_ptr());
845        let funclet = bx.catch_pad(cs, &[null, flags, null]);
846        bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None);
847        bx.catch_ret(&funclet, caught);
848
849        bx.switch_to_block(caught);
850        bx.ret(bx.const_i32(1));
851    });
852
853    // Note that no invoke is used here because by definition this function
854    // can't panic (that's what it's catching).
855    let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
856    OperandValue::Immediate(ret).store(bx, dest);
857}
858
859// WASM's definition of the `rust_try` function.
860fn codegen_wasm_try<'ll, 'tcx>(
861    bx: &mut Builder<'_, 'll, 'tcx>,
862    try_func: &'ll Value,
863    data: &'ll Value,
864    catch_func: &'ll Value,
865    dest: PlaceRef<'tcx, &'ll Value>,
866) {
867    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
868        bx.set_personality_fn(bx.eh_personality());
869
870        let normal = bx.append_sibling_block("normal");
871        let catchswitch = bx.append_sibling_block("catchswitch");
872        let catchpad = bx.append_sibling_block("catchpad");
873        let caught = bx.append_sibling_block("caught");
874
875        let try_func = llvm::get_param(bx.llfn(), 0);
876        let data = llvm::get_param(bx.llfn(), 1);
877        let catch_func = llvm::get_param(bx.llfn(), 2);
878
879        // We're generating an IR snippet that looks like:
880        //
881        //   declare i32 @rust_try(%try_func, %data, %catch_func) {
882        //      %slot = alloca i8*
883        //      invoke %try_func(%data) to label %normal unwind label %catchswitch
884        //
885        //   normal:
886        //      ret i32 0
887        //
888        //   catchswitch:
889        //      %cs = catchswitch within none [%catchpad] unwind to caller
890        //
891        //   catchpad:
892        //      %tok = catchpad within %cs [null]
893        //      %ptr = call @llvm.wasm.get.exception(token %tok)
894        //      %sel = call @llvm.wasm.get.ehselector(token %tok)
895        //      call %catch_func(%data, %ptr)
896        //      catchret from %tok to label %caught
897        //
898        //   caught:
899        //      ret i32 1
900        //   }
901        //
902        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
903        bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
904
905        bx.switch_to_block(normal);
906        bx.ret(bx.const_i32(0));
907
908        bx.switch_to_block(catchswitch);
909        let cs = bx.catch_switch(None, None, &[catchpad]);
910
911        bx.switch_to_block(catchpad);
912        let null = bx.const_null(bx.type_ptr());
913        let funclet = bx.catch_pad(cs, &[null]);
914
915        let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[funclet.cleanuppad()]);
916        let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[funclet.cleanuppad()]);
917
918        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
919        bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
920        bx.catch_ret(&funclet, caught);
921
922        bx.switch_to_block(caught);
923        bx.ret(bx.const_i32(1));
924    });
925
926    // Note that no invoke is used here because by definition this function
927    // can't panic (that's what it's catching).
928    let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
929    OperandValue::Immediate(ret).store(bx, dest);
930}
931
932// Definition of the standard `try` function for Rust using the GNU-like model
933// of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke`
934// instructions).
935//
936// This codegen is a little surprising because we always call a shim
937// function instead of inlining the call to `invoke` manually here. This is done
938// because in LLVM we're only allowed to have one personality per function
939// definition. The call to the `try` intrinsic is being inlined into the
940// function calling it, and that function may already have other personality
941// functions in play. By calling a shim we're guaranteed that our shim will have
942// the right personality function.
943fn codegen_gnu_try<'ll, 'tcx>(
944    bx: &mut Builder<'_, 'll, 'tcx>,
945    try_func: &'ll Value,
946    data: &'ll Value,
947    catch_func: &'ll Value,
948    dest: PlaceRef<'tcx, &'ll Value>,
949) {
950    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
951        // Codegens the shims described above:
952        //
953        //   bx:
954        //      invoke %try_func(%data) normal %normal unwind %catch
955        //
956        //   normal:
957        //      ret 0
958        //
959        //   catch:
960        //      (%ptr, _) = landingpad
961        //      call %catch_func(%data, %ptr)
962        //      ret 1
963        let then = bx.append_sibling_block("then");
964        let catch = bx.append_sibling_block("catch");
965
966        let try_func = llvm::get_param(bx.llfn(), 0);
967        let data = llvm::get_param(bx.llfn(), 1);
968        let catch_func = llvm::get_param(bx.llfn(), 2);
969        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
970        bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
971
972        bx.switch_to_block(then);
973        bx.ret(bx.const_i32(0));
974
975        // Type indicator for the exception being thrown.
976        //
977        // The first value in this tuple is a pointer to the exception object
978        // being thrown. The second value is a "selector" indicating which of
979        // the landing pad clauses the exception's type had been matched to.
980        // rust_try ignores the selector.
981        bx.switch_to_block(catch);
982        let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
983        let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
984        let tydesc = bx.const_null(bx.type_ptr());
985        bx.add_clause(vals, tydesc);
986        let ptr = bx.extract_value(vals, 0);
987        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
988        bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None);
989        bx.ret(bx.const_i32(1));
990    });
991
992    // Note that no invoke is used here because by definition this function
993    // can't panic (that's what it's catching).
994    let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
995    OperandValue::Immediate(ret).store(bx, dest);
996}
997
998// Variant of codegen_gnu_try used for emscripten where Rust panics are
999// implemented using C++ exceptions. Here we use exceptions of a specific type
1000// (`struct rust_panic`) to represent Rust panics.
1001fn codegen_emcc_try<'ll, 'tcx>(
1002    bx: &mut Builder<'_, 'll, 'tcx>,
1003    try_func: &'ll Value,
1004    data: &'ll Value,
1005    catch_func: &'ll Value,
1006    dest: PlaceRef<'tcx, &'ll Value>,
1007) {
1008    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1009        // Codegens the shims described above:
1010        //
1011        //   bx:
1012        //      invoke %try_func(%data) normal %normal unwind %catch
1013        //
1014        //   normal:
1015        //      ret 0
1016        //
1017        //   catch:
1018        //      (%ptr, %selector) = landingpad
1019        //      %rust_typeid = @llvm.eh.typeid.for(@_ZTI10rust_panic)
1020        //      %is_rust_panic = %selector == %rust_typeid
1021        //      %catch_data = alloca { i8*, i8 }
1022        //      %catch_data[0] = %ptr
1023        //      %catch_data[1] = %is_rust_panic
1024        //      call %catch_func(%data, %catch_data)
1025        //      ret 1
1026        let then = bx.append_sibling_block("then");
1027        let catch = bx.append_sibling_block("catch");
1028
1029        let try_func = llvm::get_param(bx.llfn(), 0);
1030        let data = llvm::get_param(bx.llfn(), 1);
1031        let catch_func = llvm::get_param(bx.llfn(), 2);
1032        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1033        bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
1034
1035        bx.switch_to_block(then);
1036        bx.ret(bx.const_i32(0));
1037
1038        // Type indicator for the exception being thrown.
1039        //
1040        // The first value in this tuple is a pointer to the exception object
1041        // being thrown. The second value is a "selector" indicating which of
1042        // the landing pad clauses the exception's type had been matched to.
1043        bx.switch_to_block(catch);
1044        let tydesc = bx.eh_catch_typeinfo();
1045        let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1046        let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2);
1047        bx.add_clause(vals, tydesc);
1048        bx.add_clause(vals, bx.const_null(bx.type_ptr()));
1049        let ptr = bx.extract_value(vals, 0);
1050        let selector = bx.extract_value(vals, 1);
1051
1052        // Check if the typeid we got is the one for a Rust panic.
1053        let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]);
1054        let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
1055        let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
1056
1057        // We need to pass two values to catch_func (ptr and is_rust_panic), so
1058        // create an alloca and pass a pointer to that.
1059        let ptr_size = bx.tcx().data_layout.pointer_size;
1060        let ptr_align = bx.tcx().data_layout.pointer_align.abi;
1061        let i8_align = bx.tcx().data_layout.i8_align.abi;
1062        // Required in order for there to be no padding between the fields.
1063        assert!(i8_align <= ptr_align);
1064        let catch_data = bx.alloca(2 * ptr_size, ptr_align);
1065        bx.store(ptr, catch_data, ptr_align);
1066        let catch_data_1 = bx.inbounds_ptradd(catch_data, bx.const_usize(ptr_size.bytes()));
1067        bx.store(is_rust_panic, catch_data_1, i8_align);
1068
1069        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1070        bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None, None);
1071        bx.ret(bx.const_i32(1));
1072    });
1073
1074    // Note that no invoke is used here because by definition this function
1075    // can't panic (that's what it's catching).
1076    let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1077    OperandValue::Immediate(ret).store(bx, dest);
1078}
1079
1080// Helper function to give a Block to a closure to codegen a shim function.
1081// This is currently primarily used for the `try` intrinsic functions above.
1082fn gen_fn<'a, 'll, 'tcx>(
1083    cx: &'a CodegenCx<'ll, 'tcx>,
1084    name: &str,
1085    rust_fn_sig: ty::PolyFnSig<'tcx>,
1086    codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1087) -> (&'ll Type, &'ll Value) {
1088    let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1089    let llty = fn_abi.llvm_type(cx);
1090    let llfn = cx.declare_fn(name, fn_abi, None);
1091    cx.set_frame_pointer_type(llfn);
1092    cx.apply_target_cpu_attr(llfn);
1093    // FIXME(eddyb) find a nicer way to do this.
1094    llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1095    let llbb = Builder::append_block(cx, llfn, "entry-block");
1096    let bx = Builder::build(cx, llbb);
1097    codegen(bx);
1098    (llty, llfn)
1099}
1100
1101// Helper function used to get a handle to the `__rust_try` function used to
1102// catch exceptions.
1103//
1104// This function is only generated once and is then cached.
1105fn get_rust_try_fn<'a, 'll, 'tcx>(
1106    cx: &'a CodegenCx<'ll, 'tcx>,
1107    codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1108) -> (&'ll Type, &'ll Value) {
1109    if let Some(llfn) = cx.rust_try_fn.get() {
1110        return llfn;
1111    }
1112
1113    // Define the type up front for the signature of the rust_try function.
1114    let tcx = cx.tcx;
1115    let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1116    // `unsafe fn(*mut i8) -> ()`
1117    let try_fn_ty = Ty::new_fn_ptr(
1118        tcx,
1119        ty::Binder::dummy(tcx.mk_fn_sig(
1120            [i8p],
1121            tcx.types.unit,
1122            false,
1123            hir::Safety::Unsafe,
1124            ExternAbi::Rust,
1125        )),
1126    );
1127    // `unsafe fn(*mut i8, *mut i8) -> ()`
1128    let catch_fn_ty = Ty::new_fn_ptr(
1129        tcx,
1130        ty::Binder::dummy(tcx.mk_fn_sig(
1131            [i8p, i8p],
1132            tcx.types.unit,
1133            false,
1134            hir::Safety::Unsafe,
1135            ExternAbi::Rust,
1136        )),
1137    );
1138    // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32`
1139    let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig(
1140        [try_fn_ty, i8p, catch_fn_ty],
1141        tcx.types.i32,
1142        false,
1143        hir::Safety::Unsafe,
1144        ExternAbi::Rust,
1145    ));
1146    let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1147    cx.rust_try_fn.set(Some(rust_try));
1148    rust_try
1149}
1150
1151fn generic_simd_intrinsic<'ll, 'tcx>(
1152    bx: &mut Builder<'_, 'll, 'tcx>,
1153    name: Symbol,
1154    callee_ty: Ty<'tcx>,
1155    fn_args: GenericArgsRef<'tcx>,
1156    args: &[OperandRef<'tcx, &'ll Value>],
1157    ret_ty: Ty<'tcx>,
1158    llret_ty: &'ll Type,
1159    span: Span,
1160) -> Result<&'ll Value, ()> {
1161    macro_rules! return_error {
1162        ($diag: expr) => {{
1163            bx.sess().dcx().emit_err($diag);
1164            return Err(());
1165        }};
1166    }
1167
1168    macro_rules! require {
1169        ($cond: expr, $diag: expr) => {
1170            if !$cond {
1171                return_error!($diag);
1172            }
1173        };
1174    }
1175
1176    macro_rules! require_simd {
1177        ($ty: expr, $variant:ident) => {{
1178            require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
1179            $ty.simd_size_and_type(bx.tcx())
1180        }};
1181    }
1182
1183    /// Returns the bitwidth of the `$ty` argument if it is an `Int` or `Uint` type.
1184    macro_rules! require_int_or_uint_ty {
1185        ($ty: expr, $diag: expr) => {
1186            match $ty {
1187                ty::Int(i) => i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()),
1188                ty::Uint(i) => {
1189                    i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
1190                }
1191                _ => {
1192                    return_error!($diag);
1193                }
1194            }
1195        };
1196    }
1197
1198    /// Converts a vector mask, where each element has a bit width equal to the data elements it is used with,
1199    /// down to an i1 based mask that can be used by llvm intrinsics.
1200    ///
1201    /// The rust simd semantics are that each element should either consist of all ones or all zeroes,
1202    /// but this information is not available to llvm. Truncating the vector effectively uses the lowest bit,
1203    /// but codegen for several targets is better if we consider the highest bit by shifting.
1204    ///
1205    /// For x86 SSE/AVX targets this is beneficial since most instructions with mask parameters only consider the highest bit.
1206    /// So even though on llvm level we have an additional shift, in the final assembly there is no shift or truncate and
1207    /// instead the mask can be used as is.
1208    ///
1209    /// For aarch64 and other targets there is a benefit because a mask from the sign bit can be more
1210    /// efficiently converted to an all ones / all zeroes mask by comparing whether each element is negative.
1211    fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
1212        bx: &mut Builder<'a, 'll, 'tcx>,
1213        i_xn: &'ll Value,
1214        in_elem_bitwidth: u64,
1215        in_len: u64,
1216    ) -> &'ll Value {
1217        // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
1218        let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
1219        let shift_indices = vec![shift_idx; in_len as _];
1220        let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
1221        // Truncate vector to an <i1 x N>
1222        bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
1223    }
1224
1225    let tcx = bx.tcx();
1226    let sig = tcx.normalize_erasing_late_bound_regions(bx.typing_env(), callee_ty.fn_sig(tcx));
1227    let arg_tys = sig.inputs();
1228
1229    // Sanity-check: all vector arguments must be immediates.
1230    if cfg!(debug_assertions) {
1231        for (ty, arg) in arg_tys.iter().zip(args) {
1232            if ty.is_simd() {
1233                assert_matches!(arg.val, OperandValue::Immediate(_));
1234            }
1235        }
1236    }
1237
1238    if name == sym::simd_select_bitmask {
1239        let (len, _) = require_simd!(arg_tys[1], SimdArgument);
1240
1241        let expected_int_bits = len.max(8).next_power_of_two();
1242        let expected_bytes = len.div_ceil(8);
1243
1244        let mask_ty = arg_tys[0];
1245        let mask = match mask_ty.kind() {
1246            ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1247            ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1248            ty::Array(elem, len)
1249                if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1250                    && len
1251                        .try_to_target_usize(bx.tcx)
1252                        .expect("expected monomorphic const in codegen")
1253                        == expected_bytes =>
1254            {
1255                let place = PlaceRef::alloca(bx, args[0].layout);
1256                args[0].val.store(bx, place);
1257                let int_ty = bx.type_ix(expected_bytes * 8);
1258                bx.load(int_ty, place.val.llval, Align::ONE)
1259            }
1260            _ => return_error!(InvalidMonomorphization::InvalidBitmask {
1261                span,
1262                name,
1263                mask_ty,
1264                expected_int_bits,
1265                expected_bytes
1266            }),
1267        };
1268
1269        let i1 = bx.type_i1();
1270        let im = bx.type_ix(len);
1271        let i1xn = bx.type_vector(i1, len);
1272        let m_im = bx.trunc(mask, im);
1273        let m_i1s = bx.bitcast(m_im, i1xn);
1274        return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1275    }
1276
1277    // every intrinsic below takes a SIMD vector as its first argument
1278    let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput);
1279    let in_ty = arg_tys[0];
1280
1281    let comparison = match name {
1282        sym::simd_eq => Some(BinOp::Eq),
1283        sym::simd_ne => Some(BinOp::Ne),
1284        sym::simd_lt => Some(BinOp::Lt),
1285        sym::simd_le => Some(BinOp::Le),
1286        sym::simd_gt => Some(BinOp::Gt),
1287        sym::simd_ge => Some(BinOp::Ge),
1288        _ => None,
1289    };
1290
1291    if let Some(cmp_op) = comparison {
1292        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1293
1294        require!(
1295            in_len == out_len,
1296            InvalidMonomorphization::ReturnLengthInputType {
1297                span,
1298                name,
1299                in_len,
1300                in_ty,
1301                ret_ty,
1302                out_len
1303            }
1304        );
1305        require!(
1306            bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
1307            InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
1308        );
1309
1310        return Ok(compare_simd_types(
1311            bx,
1312            args[0].immediate(),
1313            args[1].immediate(),
1314            in_elem,
1315            llret_ty,
1316            cmp_op,
1317        ));
1318    }
1319
1320    if name == sym::simd_shuffle_const_generic {
1321        let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch();
1322        let n = idx.len() as u64;
1323
1324        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1325        require!(
1326            out_len == n,
1327            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1328        );
1329        require!(
1330            in_elem == out_ty,
1331            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1332        );
1333
1334        let total_len = in_len * 2;
1335
1336        let indices: Option<Vec<_>> = idx
1337            .iter()
1338            .enumerate()
1339            .map(|(arg_idx, val)| {
1340                let idx = val.unwrap_leaf().to_i32();
1341                if idx >= i32::try_from(total_len).unwrap() {
1342                    bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
1343                        span,
1344                        name,
1345                        arg_idx: arg_idx as u64,
1346                        total_len: total_len.into(),
1347                    });
1348                    None
1349                } else {
1350                    Some(bx.const_i32(idx))
1351                }
1352            })
1353            .collect();
1354        let Some(indices) = indices else {
1355            return Ok(bx.const_null(llret_ty));
1356        };
1357
1358        return Ok(bx.shuffle_vector(
1359            args[0].immediate(),
1360            args[1].immediate(),
1361            bx.const_vector(&indices),
1362        ));
1363    }
1364
1365    if name == sym::simd_shuffle {
1366        // Make sure this is actually a SIMD vector.
1367        let idx_ty = args[2].layout.ty;
1368        let n: u64 = if idx_ty.is_simd()
1369            && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
1370        {
1371            idx_ty.simd_size_and_type(bx.cx.tcx).0
1372        } else {
1373            return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
1374        };
1375
1376        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1377        require!(
1378            out_len == n,
1379            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1380        );
1381        require!(
1382            in_elem == out_ty,
1383            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1384        );
1385
1386        let total_len = u128::from(in_len) * 2;
1387
1388        // Check that the indices are in-bounds.
1389        let indices = args[2].immediate();
1390        for i in 0..n {
1391            let val = bx.const_get_elt(indices, i as u64);
1392            let idx = bx
1393                .const_to_opt_u128(val, true)
1394                .unwrap_or_else(|| bug!("typeck should have already ensured that these are const"));
1395            if idx >= total_len {
1396                return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1397                    span,
1398                    name,
1399                    arg_idx: i,
1400                    total_len,
1401                });
1402            }
1403        }
1404
1405        return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
1406    }
1407
1408    if name == sym::simd_insert || name == sym::simd_insert_dyn {
1409        require!(
1410            in_elem == arg_tys[2],
1411            InvalidMonomorphization::InsertedType {
1412                span,
1413                name,
1414                in_elem,
1415                in_ty,
1416                out_ty: arg_tys[2]
1417            }
1418        );
1419
1420        let index_imm = if name == sym::simd_insert {
1421            let idx = bx
1422                .const_to_opt_u128(args[1].immediate(), false)
1423                .expect("typeck should have ensure that this is a const");
1424            if idx >= in_len.into() {
1425                return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1426                    span,
1427                    name,
1428                    arg_idx: 1,
1429                    total_len: in_len.into(),
1430                });
1431            }
1432            bx.const_i32(idx as i32)
1433        } else {
1434            args[1].immediate()
1435        };
1436
1437        return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
1438    }
1439    if name == sym::simd_extract || name == sym::simd_extract_dyn {
1440        require!(
1441            ret_ty == in_elem,
1442            InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1443        );
1444        let index_imm = if name == sym::simd_extract {
1445            let idx = bx
1446                .const_to_opt_u128(args[1].immediate(), false)
1447                .expect("typeck should have ensure that this is a const");
1448            if idx >= in_len.into() {
1449                return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1450                    span,
1451                    name,
1452                    arg_idx: 1,
1453                    total_len: in_len.into(),
1454                });
1455            }
1456            bx.const_i32(idx as i32)
1457        } else {
1458            args[1].immediate()
1459        };
1460
1461        return Ok(bx.extract_element(args[0].immediate(), index_imm));
1462    }
1463
1464    if name == sym::simd_select {
1465        let m_elem_ty = in_elem;
1466        let m_len = in_len;
1467        let (v_len, _) = require_simd!(arg_tys[1], SimdArgument);
1468        require!(
1469            m_len == v_len,
1470            InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
1471        );
1472        let in_elem_bitwidth = require_int_or_uint_ty!(
1473            m_elem_ty.kind(),
1474            InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1475        );
1476        let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
1477        return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1478    }
1479
1480    if name == sym::simd_bitmask {
1481        // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a vector mask and
1482        // returns one bit for each lane (which must all be `0` or `!0`) in the form of either:
1483        // * an unsigned integer
1484        // * an array of `u8`
1485        // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
1486        //
1487        // The bit order of the result depends on the byte endianness, LSB-first for little
1488        // endian and MSB-first for big endian.
1489        let expected_int_bits = in_len.max(8).next_power_of_two();
1490        let expected_bytes = in_len.div_ceil(8);
1491
1492        // Integer vector <i{in_bitwidth} x in_len>:
1493        let in_elem_bitwidth = require_int_or_uint_ty!(
1494            in_elem.kind(),
1495            InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
1496        );
1497
1498        let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
1499        // Bitcast <i1 x N> to iN:
1500        let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
1501
1502        match ret_ty.kind() {
1503            ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
1504                // Zero-extend iN to the bitmask type:
1505                return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
1506            }
1507            ty::Array(elem, len)
1508                if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1509                    && len
1510                        .try_to_target_usize(bx.tcx)
1511                        .expect("expected monomorphic const in codegen")
1512                        == expected_bytes =>
1513            {
1514                // Zero-extend iN to the array length:
1515                let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
1516
1517                // Convert the integer to a byte array
1518                let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
1519                bx.store(ze, ptr, Align::ONE);
1520                let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
1521                return Ok(bx.load(array_ty, ptr, Align::ONE));
1522            }
1523            _ => return_error!(InvalidMonomorphization::CannotReturn {
1524                span,
1525                name,
1526                ret_ty,
1527                expected_int_bits,
1528                expected_bytes
1529            }),
1530        }
1531    }
1532
1533    fn simd_simple_float_intrinsic<'ll, 'tcx>(
1534        name: Symbol,
1535        in_elem: Ty<'_>,
1536        in_ty: Ty<'_>,
1537        in_len: u64,
1538        bx: &mut Builder<'_, 'll, 'tcx>,
1539        span: Span,
1540        args: &[OperandRef<'tcx, &'ll Value>],
1541    ) -> Result<&'ll Value, ()> {
1542        macro_rules! return_error {
1543            ($diag: expr) => {{
1544                bx.sess().dcx().emit_err($diag);
1545                return Err(());
1546            }};
1547        }
1548
1549        let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
1550            let elem_ty = bx.cx.type_float_from_ty(*f);
1551            match f.bit_width() {
1552                16 => ("f16", elem_ty),
1553                32 => ("f32", elem_ty),
1554                64 => ("f64", elem_ty),
1555                128 => ("f128", elem_ty),
1556                _ => return_error!(InvalidMonomorphization::FloatingPointVector {
1557                    span,
1558                    name,
1559                    f_ty: *f,
1560                    in_ty,
1561                }),
1562            }
1563        } else {
1564            return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
1565        };
1566
1567        let vec_ty = bx.type_vector(elem_ty, in_len);
1568
1569        let (intr_name, fn_ty) = match name {
1570            sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
1571            sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
1572            sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
1573            sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
1574            sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
1575            sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
1576            sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
1577            sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
1578            sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
1579            sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
1580            sym::simd_relaxed_fma => ("fmuladd", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
1581            sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
1582            sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
1583            sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
1584            sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
1585            _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
1586        };
1587        let llvm_name = &format!("llvm.{intr_name}.v{in_len}{elem_ty_str}");
1588        let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty);
1589        let c = bx.call(
1590            fn_ty,
1591            None,
1592            None,
1593            f,
1594            &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
1595            None,
1596            None,
1597        );
1598        Ok(c)
1599    }
1600
1601    if std::matches!(
1602        name,
1603        sym::simd_ceil
1604            | sym::simd_fabs
1605            | sym::simd_fcos
1606            | sym::simd_fexp2
1607            | sym::simd_fexp
1608            | sym::simd_flog10
1609            | sym::simd_flog2
1610            | sym::simd_flog
1611            | sym::simd_floor
1612            | sym::simd_fma
1613            | sym::simd_fsin
1614            | sym::simd_fsqrt
1615            | sym::simd_relaxed_fma
1616            | sym::simd_round
1617            | sym::simd_trunc
1618    ) {
1619        return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
1620    }
1621
1622    // FIXME: use:
1623    //  https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182
1624    //  https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81
1625    fn llvm_vector_str(bx: &Builder<'_, '_, '_>, elem_ty: Ty<'_>, vec_len: u64) -> String {
1626        match *elem_ty.kind() {
1627            ty::Int(v) => format!(
1628                "v{}i{}",
1629                vec_len,
1630                // Normalize to prevent crash if v: IntTy::Isize
1631                v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
1632            ),
1633            ty::Uint(v) => format!(
1634                "v{}i{}",
1635                vec_len,
1636                // Normalize to prevent crash if v: UIntTy::Usize
1637                v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
1638            ),
1639            ty::Float(v) => format!("v{}f{}", vec_len, v.bit_width()),
1640            ty::RawPtr(_, _) => format!("v{}p0", vec_len),
1641            _ => unreachable!(),
1642        }
1643    }
1644
1645    fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
1646        let elem_ty = match *elem_ty.kind() {
1647            ty::Int(v) => cx.type_int_from_ty(v),
1648            ty::Uint(v) => cx.type_uint_from_ty(v),
1649            ty::Float(v) => cx.type_float_from_ty(v),
1650            ty::RawPtr(_, _) => cx.type_ptr(),
1651            _ => unreachable!(),
1652        };
1653        cx.type_vector(elem_ty, vec_len)
1654    }
1655
1656    if name == sym::simd_gather {
1657        // simd_gather(values: <N x T>, pointers: <N x *_ T>,
1658        //             mask: <N x i{M}>) -> <N x T>
1659        // * N: number of elements in the input vectors
1660        // * T: type of the element to load
1661        // * M: any integer width is supported, will be truncated to i1
1662
1663        // All types must be simd vector types
1664
1665        // The second argument must be a simd vector with an element type that's a pointer
1666        // to the element type of the first argument
1667        let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1668        let (out_len, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
1669        // The element type of the third argument must be a signed integer type of any width:
1670        let (out_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
1671        require_simd!(ret_ty, SimdReturn);
1672
1673        // Of the same length:
1674        require!(
1675            in_len == out_len,
1676            InvalidMonomorphization::SecondArgumentLength {
1677                span,
1678                name,
1679                in_len,
1680                in_ty,
1681                arg_ty: arg_tys[1],
1682                out_len
1683            }
1684        );
1685        require!(
1686            in_len == out_len2,
1687            InvalidMonomorphization::ThirdArgumentLength {
1688                span,
1689                name,
1690                in_len,
1691                in_ty,
1692                arg_ty: arg_tys[2],
1693                out_len: out_len2
1694            }
1695        );
1696
1697        // The return type must match the first argument type
1698        require!(
1699            ret_ty == in_ty,
1700            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
1701        );
1702
1703        require!(
1704            matches!(
1705                *element_ty1.kind(),
1706                ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
1707            ),
1708            InvalidMonomorphization::ExpectedElementType {
1709                span,
1710                name,
1711                expected_element: element_ty1,
1712                second_arg: arg_tys[1],
1713                in_elem,
1714                in_ty,
1715                mutability: ExpectedPointerMutability::Not,
1716            }
1717        );
1718
1719        let mask_elem_bitwidth = require_int_or_uint_ty!(
1720            element_ty2.kind(),
1721            InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1722        );
1723
1724        // Alignment of T, must be a constant integer value:
1725        let alignment_ty = bx.type_i32();
1726        let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1727
1728        // Truncate the mask vector to a vector of i1s:
1729        let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
1730        let mask_ty = bx.type_vector(bx.type_i1(), in_len);
1731
1732        // Type of the vector of pointers:
1733        let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
1734        let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len);
1735
1736        // Type of the vector of elements:
1737        let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
1738        let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len);
1739
1740        let llvm_intrinsic =
1741            format!("llvm.masked.gather.{llvm_elem_vec_str}.{llvm_pointer_vec_str}");
1742        let fn_ty = bx.type_func(
1743            &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty],
1744            llvm_elem_vec_ty,
1745        );
1746        let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1747        let v = bx.call(
1748            fn_ty,
1749            None,
1750            None,
1751            f,
1752            &[args[1].immediate(), alignment, mask, args[0].immediate()],
1753            None,
1754            None,
1755        );
1756        return Ok(v);
1757    }
1758
1759    if name == sym::simd_masked_load {
1760        // simd_masked_load(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T>
1761        // * N: number of elements in the input vectors
1762        // * T: type of the element to load
1763        // * M: any integer width is supported, will be truncated to i1
1764        // Loads contiguous elements from memory behind `pointer`, but only for
1765        // those lanes whose `mask` bit is enabled.
1766        // The memory addresses corresponding to the “off” lanes are not accessed.
1767
1768        // The element type of the "mask" argument must be a signed integer type of any width
1769        let mask_ty = in_ty;
1770        let (mask_len, mask_elem) = (in_len, in_elem);
1771
1772        // The second argument must be a pointer matching the element type
1773        let pointer_ty = arg_tys[1];
1774
1775        // The last argument is a passthrough vector providing values for disabled lanes
1776        let values_ty = arg_tys[2];
1777        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1778
1779        require_simd!(ret_ty, SimdReturn);
1780
1781        // Of the same length:
1782        require!(
1783            values_len == mask_len,
1784            InvalidMonomorphization::ThirdArgumentLength {
1785                span,
1786                name,
1787                in_len: mask_len,
1788                in_ty: mask_ty,
1789                arg_ty: values_ty,
1790                out_len: values_len
1791            }
1792        );
1793
1794        // The return type must match the last argument type
1795        require!(
1796            ret_ty == values_ty,
1797            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
1798        );
1799
1800        require!(
1801            matches!(
1802                *pointer_ty.kind(),
1803                ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
1804            ),
1805            InvalidMonomorphization::ExpectedElementType {
1806                span,
1807                name,
1808                expected_element: values_elem,
1809                second_arg: pointer_ty,
1810                in_elem: values_elem,
1811                in_ty: values_ty,
1812                mutability: ExpectedPointerMutability::Not,
1813            }
1814        );
1815
1816        let m_elem_bitwidth = require_int_or_uint_ty!(
1817            mask_elem.kind(),
1818            InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1819        );
1820
1821        let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1822        let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
1823
1824        // Alignment of T, must be a constant integer value:
1825        let alignment_ty = bx.type_i32();
1826        let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1827
1828        let llvm_pointer = bx.type_ptr();
1829
1830        // Type of the vector of elements:
1831        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1832        let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
1833
1834        let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0");
1835        let fn_ty = bx
1836            .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty);
1837        let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1838        let v = bx.call(
1839            fn_ty,
1840            None,
1841            None,
1842            f,
1843            &[args[1].immediate(), alignment, mask, args[2].immediate()],
1844            None,
1845            None,
1846        );
1847        return Ok(v);
1848    }
1849
1850    if name == sym::simd_masked_store {
1851        // simd_masked_store(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> ()
1852        // * N: number of elements in the input vectors
1853        // * T: type of the element to load
1854        // * M: any integer width is supported, will be truncated to i1
1855        // Stores contiguous elements to memory behind `pointer`, but only for
1856        // those lanes whose `mask` bit is enabled.
1857        // The memory addresses corresponding to the “off” lanes are not accessed.
1858
1859        // The element type of the "mask" argument must be a signed integer type of any width
1860        let mask_ty = in_ty;
1861        let (mask_len, mask_elem) = (in_len, in_elem);
1862
1863        // The second argument must be a pointer matching the element type
1864        let pointer_ty = arg_tys[1];
1865
1866        // The last argument specifies the values to store to memory
1867        let values_ty = arg_tys[2];
1868        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1869
1870        // Of the same length:
1871        require!(
1872            values_len == mask_len,
1873            InvalidMonomorphization::ThirdArgumentLength {
1874                span,
1875                name,
1876                in_len: mask_len,
1877                in_ty: mask_ty,
1878                arg_ty: values_ty,
1879                out_len: values_len
1880            }
1881        );
1882
1883        // The second argument must be a mutable pointer type matching the element type
1884        require!(
1885            matches!(
1886                *pointer_ty.kind(),
1887                ty::RawPtr(p_ty, p_mutbl)
1888                    if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
1889            ),
1890            InvalidMonomorphization::ExpectedElementType {
1891                span,
1892                name,
1893                expected_element: values_elem,
1894                second_arg: pointer_ty,
1895                in_elem: values_elem,
1896                in_ty: values_ty,
1897                mutability: ExpectedPointerMutability::Mut,
1898            }
1899        );
1900
1901        let m_elem_bitwidth = require_int_or_uint_ty!(
1902            mask_elem.kind(),
1903            InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1904        );
1905
1906        let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1907        let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
1908
1909        // Alignment of T, must be a constant integer value:
1910        let alignment_ty = bx.type_i32();
1911        let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1912
1913        let ret_t = bx.type_void();
1914
1915        let llvm_pointer = bx.type_ptr();
1916
1917        // Type of the vector of elements:
1918        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1919        let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
1920
1921        let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0");
1922        let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t);
1923        let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1924        let v = bx.call(
1925            fn_ty,
1926            None,
1927            None,
1928            f,
1929            &[args[2].immediate(), args[1].immediate(), alignment, mask],
1930            None,
1931            None,
1932        );
1933        return Ok(v);
1934    }
1935
1936    if name == sym::simd_scatter {
1937        // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
1938        //             mask: <N x i{M}>) -> ()
1939        // * N: number of elements in the input vectors
1940        // * T: type of the element to load
1941        // * M: any integer width is supported, will be truncated to i1
1942
1943        // All types must be simd vector types
1944        // The second argument must be a simd vector with an element type that's a pointer
1945        // to the element type of the first argument
1946        let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1947        let (element_len1, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
1948        let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
1949
1950        // Of the same length:
1951        require!(
1952            in_len == element_len1,
1953            InvalidMonomorphization::SecondArgumentLength {
1954                span,
1955                name,
1956                in_len,
1957                in_ty,
1958                arg_ty: arg_tys[1],
1959                out_len: element_len1
1960            }
1961        );
1962        require!(
1963            in_len == element_len2,
1964            InvalidMonomorphization::ThirdArgumentLength {
1965                span,
1966                name,
1967                in_len,
1968                in_ty,
1969                arg_ty: arg_tys[2],
1970                out_len: element_len2
1971            }
1972        );
1973
1974        require!(
1975            matches!(
1976                *element_ty1.kind(),
1977                ty::RawPtr(p_ty, p_mutbl)
1978                    if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
1979            ),
1980            InvalidMonomorphization::ExpectedElementType {
1981                span,
1982                name,
1983                expected_element: element_ty1,
1984                second_arg: arg_tys[1],
1985                in_elem,
1986                in_ty,
1987                mutability: ExpectedPointerMutability::Mut,
1988            }
1989        );
1990
1991        // The element type of the third argument must be an integer type of any width:
1992        let mask_elem_bitwidth = require_int_or_uint_ty!(
1993            element_ty2.kind(),
1994            InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1995        );
1996
1997        // Alignment of T, must be a constant integer value:
1998        let alignment_ty = bx.type_i32();
1999        let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
2000
2001        // Truncate the mask vector to a vector of i1s:
2002        let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2003        let mask_ty = bx.type_vector(bx.type_i1(), in_len);
2004
2005        let ret_t = bx.type_void();
2006
2007        // Type of the vector of pointers:
2008        let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2009        let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len);
2010
2011        // Type of the vector of elements:
2012        let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2013        let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len);
2014
2015        let llvm_intrinsic =
2016            format!("llvm.masked.scatter.{llvm_elem_vec_str}.{llvm_pointer_vec_str}");
2017        let fn_ty =
2018            bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t);
2019        let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2020        let v = bx.call(
2021            fn_ty,
2022            None,
2023            None,
2024            f,
2025            &[args[0].immediate(), args[1].immediate(), alignment, mask],
2026            None,
2027            None,
2028        );
2029        return Ok(v);
2030    }
2031
2032    macro_rules! arith_red {
2033        ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
2034         $identity:expr) => {
2035            if name == sym::$name {
2036                require!(
2037                    ret_ty == in_elem,
2038                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2039                );
2040                return match in_elem.kind() {
2041                    ty::Int(_) | ty::Uint(_) => {
2042                        let r = bx.$integer_reduce(args[0].immediate());
2043                        if $ordered {
2044                            // if overflow occurs, the result is the
2045                            // mathematical result modulo 2^n:
2046                            Ok(bx.$op(args[1].immediate(), r))
2047                        } else {
2048                            Ok(bx.$integer_reduce(args[0].immediate()))
2049                        }
2050                    }
2051                    ty::Float(f) => {
2052                        let acc = if $ordered {
2053                            // ordered arithmetic reductions take an accumulator
2054                            args[1].immediate()
2055                        } else {
2056                            // unordered arithmetic reductions use the identity accumulator
2057                            match f.bit_width() {
2058                                32 => bx.const_real(bx.type_f32(), $identity),
2059                                64 => bx.const_real(bx.type_f64(), $identity),
2060                                v => return_error!(
2061                                    InvalidMonomorphization::UnsupportedSymbolOfSize {
2062                                        span,
2063                                        name,
2064                                        symbol: sym::$name,
2065                                        in_ty,
2066                                        in_elem,
2067                                        size: v,
2068                                        ret_ty
2069                                    }
2070                                ),
2071                            }
2072                        };
2073                        Ok(bx.$float_reduce(acc, args[0].immediate()))
2074                    }
2075                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2076                        span,
2077                        name,
2078                        symbol: sym::$name,
2079                        in_ty,
2080                        in_elem,
2081                        ret_ty
2082                    }),
2083                };
2084            }
2085        };
2086    }
2087
2088    arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
2089    arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
2090    arith_red!(
2091        simd_reduce_add_unordered: vector_reduce_add,
2092        vector_reduce_fadd_reassoc,
2093        false,
2094        add,
2095        -0.0
2096    );
2097    arith_red!(
2098        simd_reduce_mul_unordered: vector_reduce_mul,
2099        vector_reduce_fmul_reassoc,
2100        false,
2101        mul,
2102        1.0
2103    );
2104
2105    macro_rules! minmax_red {
2106        ($name:ident: $int_red:ident, $float_red:ident) => {
2107            if name == sym::$name {
2108                require!(
2109                    ret_ty == in_elem,
2110                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2111                );
2112                return match in_elem.kind() {
2113                    ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
2114                    ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
2115                    ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
2116                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2117                        span,
2118                        name,
2119                        symbol: sym::$name,
2120                        in_ty,
2121                        in_elem,
2122                        ret_ty
2123                    }),
2124                };
2125            }
2126        };
2127    }
2128
2129    minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
2130    minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
2131
2132    macro_rules! bitwise_red {
2133        ($name:ident : $red:ident, $boolean:expr) => {
2134            if name == sym::$name {
2135                let input = if !$boolean {
2136                    require!(
2137                        ret_ty == in_elem,
2138                        InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2139                    );
2140                    args[0].immediate()
2141                } else {
2142                    let bitwidth = match in_elem.kind() {
2143                        ty::Int(i) => {
2144                            i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
2145                        }
2146                        ty::Uint(i) => {
2147                            i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
2148                        }
2149                        _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2150                            span,
2151                            name,
2152                            symbol: sym::$name,
2153                            in_ty,
2154                            in_elem,
2155                            ret_ty
2156                        }),
2157                    };
2158
2159                    vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
2160                };
2161                return match in_elem.kind() {
2162                    ty::Int(_) | ty::Uint(_) => {
2163                        let r = bx.$red(input);
2164                        Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
2165                    }
2166                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2167                        span,
2168                        name,
2169                        symbol: sym::$name,
2170                        in_ty,
2171                        in_elem,
2172                        ret_ty
2173                    }),
2174                };
2175            }
2176        };
2177    }
2178
2179    bitwise_red!(simd_reduce_and: vector_reduce_and, false);
2180    bitwise_red!(simd_reduce_or: vector_reduce_or, false);
2181    bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
2182    bitwise_red!(simd_reduce_all: vector_reduce_and, true);
2183    bitwise_red!(simd_reduce_any: vector_reduce_or, true);
2184
2185    if name == sym::simd_cast_ptr {
2186        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2187        require!(
2188            in_len == out_len,
2189            InvalidMonomorphization::ReturnLengthInputType {
2190                span,
2191                name,
2192                in_len,
2193                in_ty,
2194                ret_ty,
2195                out_len
2196            }
2197        );
2198
2199        match in_elem.kind() {
2200            ty::RawPtr(p_ty, _) => {
2201                let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2202                    bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2203                });
2204                require!(
2205                    metadata.is_unit(),
2206                    InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
2207                );
2208            }
2209            _ => {
2210                return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2211            }
2212        }
2213        match out_elem.kind() {
2214            ty::RawPtr(p_ty, _) => {
2215                let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2216                    bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2217                });
2218                require!(
2219                    metadata.is_unit(),
2220                    InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
2221                );
2222            }
2223            _ => {
2224                return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2225            }
2226        }
2227
2228        return Ok(args[0].immediate());
2229    }
2230
2231    if name == sym::simd_expose_provenance {
2232        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2233        require!(
2234            in_len == out_len,
2235            InvalidMonomorphization::ReturnLengthInputType {
2236                span,
2237                name,
2238                in_len,
2239                in_ty,
2240                ret_ty,
2241                out_len
2242            }
2243        );
2244
2245        match in_elem.kind() {
2246            ty::RawPtr(_, _) => {}
2247            _ => {
2248                return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2249            }
2250        }
2251        match out_elem.kind() {
2252            ty::Uint(ty::UintTy::Usize) => {}
2253            _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
2254        }
2255
2256        return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
2257    }
2258
2259    if name == sym::simd_with_exposed_provenance {
2260        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2261        require!(
2262            in_len == out_len,
2263            InvalidMonomorphization::ReturnLengthInputType {
2264                span,
2265                name,
2266                in_len,
2267                in_ty,
2268                ret_ty,
2269                out_len
2270            }
2271        );
2272
2273        match in_elem.kind() {
2274            ty::Uint(ty::UintTy::Usize) => {}
2275            _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
2276        }
2277        match out_elem.kind() {
2278            ty::RawPtr(_, _) => {}
2279            _ => {
2280                return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2281            }
2282        }
2283
2284        return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
2285    }
2286
2287    if name == sym::simd_cast || name == sym::simd_as {
2288        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2289        require!(
2290            in_len == out_len,
2291            InvalidMonomorphization::ReturnLengthInputType {
2292                span,
2293                name,
2294                in_len,
2295                in_ty,
2296                ret_ty,
2297                out_len
2298            }
2299        );
2300        // casting cares about nominal type, not just structural type
2301        if in_elem == out_elem {
2302            return Ok(args[0].immediate());
2303        }
2304
2305        #[derive(Copy, Clone)]
2306        enum Sign {
2307            Unsigned,
2308            Signed,
2309        }
2310        use Sign::*;
2311
2312        enum Style {
2313            Float,
2314            Int(Sign),
2315            Unsupported,
2316        }
2317
2318        let (in_style, in_width) = match in_elem.kind() {
2319            // vectors of pointer-sized integers should've been
2320            // disallowed before here, so this unwrap is safe.
2321            ty::Int(i) => (
2322                Style::Int(Signed),
2323                i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2324            ),
2325            ty::Uint(u) => (
2326                Style::Int(Unsigned),
2327                u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2328            ),
2329            ty::Float(f) => (Style::Float, f.bit_width()),
2330            _ => (Style::Unsupported, 0),
2331        };
2332        let (out_style, out_width) = match out_elem.kind() {
2333            ty::Int(i) => (
2334                Style::Int(Signed),
2335                i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2336            ),
2337            ty::Uint(u) => (
2338                Style::Int(Unsigned),
2339                u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2340            ),
2341            ty::Float(f) => (Style::Float, f.bit_width()),
2342            _ => (Style::Unsupported, 0),
2343        };
2344
2345        match (in_style, out_style) {
2346            (Style::Int(sign), Style::Int(_)) => {
2347                return Ok(match in_width.cmp(&out_width) {
2348                    Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
2349                    Ordering::Equal => args[0].immediate(),
2350                    Ordering::Less => match sign {
2351                        Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
2352                        Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
2353                    },
2354                });
2355            }
2356            (Style::Int(Sign::Signed), Style::Float) => {
2357                return Ok(bx.sitofp(args[0].immediate(), llret_ty));
2358            }
2359            (Style::Int(Sign::Unsigned), Style::Float) => {
2360                return Ok(bx.uitofp(args[0].immediate(), llret_ty));
2361            }
2362            (Style::Float, Style::Int(sign)) => {
2363                return Ok(match (sign, name == sym::simd_as) {
2364                    (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
2365                    (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
2366                    (_, true) => bx.cast_float_to_int(
2367                        matches!(sign, Sign::Signed),
2368                        args[0].immediate(),
2369                        llret_ty,
2370                    ),
2371                });
2372            }
2373            (Style::Float, Style::Float) => {
2374                return Ok(match in_width.cmp(&out_width) {
2375                    Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
2376                    Ordering::Equal => args[0].immediate(),
2377                    Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
2378                });
2379            }
2380            _ => { /* Unsupported. Fallthrough. */ }
2381        }
2382        return_error!(InvalidMonomorphization::UnsupportedCast {
2383            span,
2384            name,
2385            in_ty,
2386            in_elem,
2387            ret_ty,
2388            out_elem
2389        });
2390    }
2391    macro_rules! arith_binary {
2392        ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2393            $(if name == sym::$name {
2394                match in_elem.kind() {
2395                    $($(ty::$p(_))|* => {
2396                        return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
2397                    })*
2398                    _ => {},
2399                }
2400                return_error!(
2401                    InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2402                );
2403            })*
2404        }
2405    }
2406    arith_binary! {
2407        simd_add: Uint, Int => add, Float => fadd;
2408        simd_sub: Uint, Int => sub, Float => fsub;
2409        simd_mul: Uint, Int => mul, Float => fmul;
2410        simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
2411        simd_rem: Uint => urem, Int => srem, Float => frem;
2412        simd_shl: Uint, Int => shl;
2413        simd_shr: Uint => lshr, Int => ashr;
2414        simd_and: Uint, Int => and;
2415        simd_or: Uint, Int => or;
2416        simd_xor: Uint, Int => xor;
2417        simd_fmax: Float => maxnum;
2418        simd_fmin: Float => minnum;
2419
2420    }
2421    macro_rules! arith_unary {
2422        ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2423            $(if name == sym::$name {
2424                match in_elem.kind() {
2425                    $($(ty::$p(_))|* => {
2426                        return Ok(bx.$call(args[0].immediate()))
2427                    })*
2428                    _ => {},
2429                }
2430                return_error!(
2431                    InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2432                );
2433            })*
2434        }
2435    }
2436    arith_unary! {
2437        simd_neg: Int => neg, Float => fneg;
2438    }
2439
2440    // Unary integer intrinsics
2441    if matches!(
2442        name,
2443        sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz
2444    ) {
2445        let vec_ty = bx.cx.type_vector(
2446            match *in_elem.kind() {
2447                ty::Int(i) => bx.cx.type_int_from_ty(i),
2448                ty::Uint(i) => bx.cx.type_uint_from_ty(i),
2449                _ => return_error!(InvalidMonomorphization::UnsupportedOperation {
2450                    span,
2451                    name,
2452                    in_ty,
2453                    in_elem
2454                }),
2455            },
2456            in_len as u64,
2457        );
2458        let intrinsic_name = match name {
2459            sym::simd_bswap => "bswap",
2460            sym::simd_bitreverse => "bitreverse",
2461            sym::simd_ctlz => "ctlz",
2462            sym::simd_ctpop => "ctpop",
2463            sym::simd_cttz => "cttz",
2464            _ => unreachable!(),
2465        };
2466        let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
2467        let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,);
2468
2469        return match name {
2470            // byte swap is no-op for i8/u8
2471            sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
2472            sym::simd_ctlz | sym::simd_cttz => {
2473                // for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison`
2474                let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
2475                let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
2476                let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2477                Ok(bx.call(
2478                    fn_ty,
2479                    None,
2480                    None,
2481                    f,
2482                    &[args[0].immediate(), dont_poison_on_zero],
2483                    None,
2484                    None,
2485                ))
2486            }
2487            sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
2488                // simple unary argument cases
2489                let fn_ty = bx.type_func(&[vec_ty], vec_ty);
2490                let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2491                Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
2492            }
2493            _ => unreachable!(),
2494        };
2495    }
2496
2497    if name == sym::simd_arith_offset {
2498        // This also checks that the first operand is a ptr type.
2499        let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
2500            span_bug!(span, "must be called with a vector of pointer types as first argument")
2501        });
2502        let layout = bx.layout_of(pointee);
2503        let ptrs = args[0].immediate();
2504        // The second argument must be a ptr-sized integer.
2505        // (We don't care about the signedness, this is wrapping anyway.)
2506        let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx());
2507        if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
2508            span_bug!(
2509                span,
2510                "must be called with a vector of pointer-sized integers as second argument"
2511            );
2512        }
2513        let offsets = args[1].immediate();
2514
2515        return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets]));
2516    }
2517
2518    if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
2519        let lhs = args[0].immediate();
2520        let rhs = args[1].immediate();
2521        let is_add = name == sym::simd_saturating_add;
2522        let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
2523        let (signed, elem_width, elem_ty) = match *in_elem.kind() {
2524            ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
2525            ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
2526            _ => {
2527                return_error!(InvalidMonomorphization::ExpectedVectorElementType {
2528                    span,
2529                    name,
2530                    expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1,
2531                    vector_type: arg_tys[0]
2532                });
2533            }
2534        };
2535        let llvm_intrinsic = &format!(
2536            "llvm.{}{}.sat.v{}i{}",
2537            if signed { 's' } else { 'u' },
2538            if is_add { "add" } else { "sub" },
2539            in_len,
2540            elem_width
2541        );
2542        let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
2543
2544        let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty);
2545        let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2546        let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None, None);
2547        return Ok(v);
2548    }
2549
2550    span_bug!(span, "unknown SIMD intrinsic");
2551}