1use std::assert_matches::assert_matches;
2use std::cmp::Ordering;
3
4use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size};
5use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
6use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
7use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
8use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
9use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
10use rustc_codegen_ssa::traits::*;
11use rustc_hir as hir;
12use rustc_middle::mir::BinOp;
13use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
14use rustc_middle::ty::{self, GenericArgsRef, Ty};
15use rustc_middle::{bug, span_bug};
16use rustc_span::{Span, Symbol, sym};
17use rustc_symbol_mangling::mangle_internal_symbol;
18use rustc_target::spec::{HasTargetSpec, PanicStrategy};
19use tracing::debug;
20
21use crate::abi::FnAbiLlvmExt;
22use crate::builder::Builder;
23use crate::context::CodegenCx;
24use crate::llvm::{self, Metadata};
25use crate::type_::Type;
26use crate::type_of::LayoutLlvmExt;
27use crate::va_arg::emit_va_arg;
28use crate::value::Value;
29
30fn get_simple_intrinsic<'ll>(
31 cx: &CodegenCx<'ll, '_>,
32 name: Symbol,
33) -> Option<(&'ll Type, &'ll Value)> {
34 let llvm_name = match name {
35 sym::sqrtf16 => "llvm.sqrt.f16",
36 sym::sqrtf32 => "llvm.sqrt.f32",
37 sym::sqrtf64 => "llvm.sqrt.f64",
38 sym::sqrtf128 => "llvm.sqrt.f128",
39
40 sym::powif16 => "llvm.powi.f16.i32",
41 sym::powif32 => "llvm.powi.f32.i32",
42 sym::powif64 => "llvm.powi.f64.i32",
43 sym::powif128 => "llvm.powi.f128.i32",
44
45 sym::sinf16 => "llvm.sin.f16",
46 sym::sinf32 => "llvm.sin.f32",
47 sym::sinf64 => "llvm.sin.f64",
48 sym::sinf128 => "llvm.sin.f128",
49
50 sym::cosf16 => "llvm.cos.f16",
51 sym::cosf32 => "llvm.cos.f32",
52 sym::cosf64 => "llvm.cos.f64",
53 sym::cosf128 => "llvm.cos.f128",
54
55 sym::powf16 => "llvm.pow.f16",
56 sym::powf32 => "llvm.pow.f32",
57 sym::powf64 => "llvm.pow.f64",
58 sym::powf128 => "llvm.pow.f128",
59
60 sym::expf16 => "llvm.exp.f16",
61 sym::expf32 => "llvm.exp.f32",
62 sym::expf64 => "llvm.exp.f64",
63 sym::expf128 => "llvm.exp.f128",
64
65 sym::exp2f16 => "llvm.exp2.f16",
66 sym::exp2f32 => "llvm.exp2.f32",
67 sym::exp2f64 => "llvm.exp2.f64",
68 sym::exp2f128 => "llvm.exp2.f128",
69
70 sym::logf16 => "llvm.log.f16",
71 sym::logf32 => "llvm.log.f32",
72 sym::logf64 => "llvm.log.f64",
73 sym::logf128 => "llvm.log.f128",
74
75 sym::log10f16 => "llvm.log10.f16",
76 sym::log10f32 => "llvm.log10.f32",
77 sym::log10f64 => "llvm.log10.f64",
78 sym::log10f128 => "llvm.log10.f128",
79
80 sym::log2f16 => "llvm.log2.f16",
81 sym::log2f32 => "llvm.log2.f32",
82 sym::log2f64 => "llvm.log2.f64",
83 sym::log2f128 => "llvm.log2.f128",
84
85 sym::fmaf16 => "llvm.fma.f16",
86 sym::fmaf32 => "llvm.fma.f32",
87 sym::fmaf64 => "llvm.fma.f64",
88 sym::fmaf128 => "llvm.fma.f128",
89
90 sym::fmuladdf16 => "llvm.fmuladd.f16",
91 sym::fmuladdf32 => "llvm.fmuladd.f32",
92 sym::fmuladdf64 => "llvm.fmuladd.f64",
93 sym::fmuladdf128 => "llvm.fmuladd.f128",
94
95 sym::fabsf16 => "llvm.fabs.f16",
96 sym::fabsf32 => "llvm.fabs.f32",
97 sym::fabsf64 => "llvm.fabs.f64",
98 sym::fabsf128 => "llvm.fabs.f128",
99
100 sym::minnumf16 => "llvm.minnum.f16",
101 sym::minnumf32 => "llvm.minnum.f32",
102 sym::minnumf64 => "llvm.minnum.f64",
103 sym::minnumf128 => "llvm.minnum.f128",
104
105 sym::minimumf16 => "llvm.minimum.f16",
106 sym::minimumf32 => "llvm.minimum.f32",
107 sym::minimumf64 => "llvm.minimum.f64",
108 sym::maxnumf16 => "llvm.maxnum.f16",
112 sym::maxnumf32 => "llvm.maxnum.f32",
113 sym::maxnumf64 => "llvm.maxnum.f64",
114 sym::maxnumf128 => "llvm.maxnum.f128",
115
116 sym::maximumf16 => "llvm.maximum.f16",
117 sym::maximumf32 => "llvm.maximum.f32",
118 sym::maximumf64 => "llvm.maximum.f64",
119 sym::copysignf16 => "llvm.copysign.f16",
123 sym::copysignf32 => "llvm.copysign.f32",
124 sym::copysignf64 => "llvm.copysign.f64",
125 sym::copysignf128 => "llvm.copysign.f128",
126
127 sym::floorf16 => "llvm.floor.f16",
128 sym::floorf32 => "llvm.floor.f32",
129 sym::floorf64 => "llvm.floor.f64",
130 sym::floorf128 => "llvm.floor.f128",
131
132 sym::ceilf16 => "llvm.ceil.f16",
133 sym::ceilf32 => "llvm.ceil.f32",
134 sym::ceilf64 => "llvm.ceil.f64",
135 sym::ceilf128 => "llvm.ceil.f128",
136
137 sym::truncf16 => "llvm.trunc.f16",
138 sym::truncf32 => "llvm.trunc.f32",
139 sym::truncf64 => "llvm.trunc.f64",
140 sym::truncf128 => "llvm.trunc.f128",
141
142 sym::round_ties_even_f16 => "llvm.rint.f16",
147 sym::round_ties_even_f32 => "llvm.rint.f32",
148 sym::round_ties_even_f64 => "llvm.rint.f64",
149 sym::round_ties_even_f128 => "llvm.rint.f128",
150
151 sym::roundf16 => "llvm.round.f16",
152 sym::roundf32 => "llvm.round.f32",
153 sym::roundf64 => "llvm.round.f64",
154 sym::roundf128 => "llvm.round.f128",
155
156 sym::ptr_mask => "llvm.ptrmask",
157
158 _ => return None,
159 };
160 Some(cx.get_intrinsic(llvm_name))
161}
162
163impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
164 fn codegen_intrinsic_call(
165 &mut self,
166 instance: ty::Instance<'tcx>,
167 args: &[OperandRef<'tcx, &'ll Value>],
168 result: PlaceRef<'tcx, &'ll Value>,
169 span: Span,
170 ) -> Result<(), ty::Instance<'tcx>> {
171 let tcx = self.tcx;
172 let callee_ty = instance.ty(tcx, self.typing_env());
173
174 let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else {
175 bug!("expected fn item type, found {}", callee_ty);
176 };
177
178 let sig = callee_ty.fn_sig(tcx);
179 let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig);
180 let arg_tys = sig.inputs();
181 let ret_ty = sig.output();
182 let name = tcx.item_name(def_id);
183
184 let llret_ty = self.layout_of(ret_ty).llvm_type(self);
185
186 let simple = get_simple_intrinsic(self, name);
187 let llval = match name {
188 _ if simple.is_some() => {
189 let (simple_ty, simple_fn) = simple.unwrap();
190 self.call(
191 simple_ty,
192 None,
193 None,
194 simple_fn,
195 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
196 None,
197 Some(instance),
198 )
199 }
200 sym::is_val_statically_known => {
201 let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx);
202 let kind = self.type_kind(intrinsic_type);
203 let intrinsic_name = match kind {
204 TypeKind::Pointer | TypeKind::Integer => {
205 Some(format!("llvm.is.constant.{intrinsic_type:?}"))
206 }
207 TypeKind::Half => Some(format!("llvm.is.constant.f16")),
209 TypeKind::Float => Some(format!("llvm.is.constant.f32")),
210 TypeKind::Double => Some(format!("llvm.is.constant.f64")),
211 TypeKind::FP128 => Some(format!("llvm.is.constant.f128")),
212 _ => None,
213 };
214 if let Some(intrinsic_name) = intrinsic_name {
215 self.call_intrinsic(&intrinsic_name, &[args[0].immediate()])
216 } else {
217 self.const_bool(false)
218 }
219 }
220 sym::select_unpredictable => {
221 let cond = args[0].immediate();
222 assert_eq!(args[1].layout, args[2].layout);
223 let select = |bx: &mut Self, true_val, false_val| {
224 let result = bx.select(cond, true_val, false_val);
225 bx.set_unpredictable(&result);
226 result
227 };
228 match (args[1].val, args[2].val) {
229 (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
230 assert!(true_val.llextra.is_none());
231 assert!(false_val.llextra.is_none());
232 assert_eq!(true_val.align, false_val.align);
233 let ptr = select(self, true_val.llval, false_val.llval);
234 let selected =
235 OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
236 selected.store(self, result);
237 return Ok(());
238 }
239 (OperandValue::Immediate(_), OperandValue::Immediate(_))
240 | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
241 let true_val = args[1].immediate_or_packed_pair(self);
242 let false_val = args[2].immediate_or_packed_pair(self);
243 select(self, true_val, false_val)
244 }
245 (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
246 _ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
247 }
248 }
249 sym::catch_unwind => {
250 catch_unwind_intrinsic(
251 self,
252 args[0].immediate(),
253 args[1].immediate(),
254 args[2].immediate(),
255 result,
256 );
257 return Ok(());
258 }
259 sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]),
260 sym::va_copy => {
261 self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()])
262 }
263 sym::va_arg => {
264 match result.layout.backend_repr {
265 BackendRepr::Scalar(scalar) => {
266 match scalar.primitive() {
267 Primitive::Int(..) => {
268 if self.cx().size_of(ret_ty).bytes() < 4 {
269 let promoted_result = emit_va_arg(self, args[0], tcx.types.i32);
274 self.trunc(promoted_result, llret_ty)
275 } else {
276 emit_va_arg(self, args[0], ret_ty)
277 }
278 }
279 Primitive::Float(Float::F16) => {
280 bug!("the va_arg intrinsic does not work with `f16`")
281 }
282 Primitive::Float(Float::F64) | Primitive::Pointer(_) => {
283 emit_va_arg(self, args[0], ret_ty)
284 }
285 Primitive::Float(Float::F32) => {
287 bug!("the va_arg intrinsic does not work with `f32`")
288 }
289 Primitive::Float(Float::F128) => {
290 bug!("the va_arg intrinsic does not work with `f128`")
291 }
292 }
293 }
294 _ => bug!("the va_arg intrinsic does not work with non-scalar types"),
295 }
296 }
297
298 sym::volatile_load | sym::unaligned_volatile_load => {
299 let ptr = args[0].immediate();
300 let load = self.volatile_load(result.layout.llvm_type(self), ptr);
301 let align = if name == sym::unaligned_volatile_load {
302 1
303 } else {
304 result.layout.align.abi.bytes() as u32
305 };
306 unsafe {
307 llvm::LLVMSetAlignment(load, align);
308 }
309 if !result.layout.is_zst() {
310 self.store_to_place(load, result.val);
311 }
312 return Ok(());
313 }
314 sym::volatile_store => {
315 let dst = args[0].deref(self.cx());
316 args[1].val.volatile_store(self, dst);
317 return Ok(());
318 }
319 sym::unaligned_volatile_store => {
320 let dst = args[0].deref(self.cx());
321 args[1].val.unaligned_volatile_store(self, dst);
322 return Ok(());
323 }
324 sym::prefetch_read_data
325 | sym::prefetch_write_data
326 | sym::prefetch_read_instruction
327 | sym::prefetch_write_instruction => {
328 let (rw, cache_type) = match name {
329 sym::prefetch_read_data => (0, 1),
330 sym::prefetch_write_data => (1, 1),
331 sym::prefetch_read_instruction => (0, 0),
332 sym::prefetch_write_instruction => (1, 0),
333 _ => bug!(),
334 };
335 self.call_intrinsic(
336 "llvm.prefetch",
337 &[
338 args[0].immediate(),
339 self.const_i32(rw),
340 args[1].immediate(),
341 self.const_i32(cache_type),
342 ],
343 )
344 }
345 sym::carrying_mul_add => {
346 let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
347
348 let wide_llty = self.type_ix(size.bits() * 2);
349 let args = args.as_array().unwrap();
350 let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
351
352 let wide = if signed {
353 let prod = self.unchecked_smul(a, b);
354 let acc = self.unchecked_sadd(prod, c);
355 self.unchecked_sadd(acc, d)
356 } else {
357 let prod = self.unchecked_umul(a, b);
358 let acc = self.unchecked_uadd(prod, c);
359 self.unchecked_uadd(acc, d)
360 };
361
362 let narrow_llty = self.type_ix(size.bits());
363 let low = self.trunc(wide, narrow_llty);
364 let bits_const = self.const_uint(wide_llty, size.bits());
365 let high = self.lshr(wide, bits_const);
367 let high = self.trunc(high, narrow_llty);
369
370 let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false);
371 let pair = self.const_poison(pair_llty);
372 let pair = self.insert_value(pair, low, 0);
373 let pair = self.insert_value(pair, high, 1);
374 pair
375 }
376 sym::ctlz
377 | sym::ctlz_nonzero
378 | sym::cttz
379 | sym::cttz_nonzero
380 | sym::ctpop
381 | sym::bswap
382 | sym::bitreverse
383 | sym::rotate_left
384 | sym::rotate_right
385 | sym::saturating_add
386 | sym::saturating_sub => {
387 let ty = arg_tys[0];
388 if !ty.is_integral() {
389 tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
390 span,
391 name,
392 ty,
393 });
394 return Ok(());
395 }
396 let (size, signed) = ty.int_size_and_signed(self.tcx);
397 let width = size.bits();
398 match name {
399 sym::ctlz | sym::cttz => {
400 let y = self.const_bool(false);
401 let ret = self.call_intrinsic(
402 &format!("llvm.{name}.i{width}"),
403 &[args[0].immediate(), y],
404 );
405
406 self.intcast(ret, llret_ty, false)
407 }
408 sym::ctlz_nonzero => {
409 let y = self.const_bool(true);
410 let llvm_name = &format!("llvm.ctlz.i{width}");
411 let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
412 self.intcast(ret, llret_ty, false)
413 }
414 sym::cttz_nonzero => {
415 let y = self.const_bool(true);
416 let llvm_name = &format!("llvm.cttz.i{width}");
417 let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
418 self.intcast(ret, llret_ty, false)
419 }
420 sym::ctpop => {
421 let ret = self.call_intrinsic(
422 &format!("llvm.ctpop.i{width}"),
423 &[args[0].immediate()],
424 );
425 self.intcast(ret, llret_ty, false)
426 }
427 sym::bswap => {
428 if width == 8 {
429 args[0].immediate() } else {
431 self.call_intrinsic(
432 &format!("llvm.bswap.i{width}"),
433 &[args[0].immediate()],
434 )
435 }
436 }
437 sym::bitreverse => self.call_intrinsic(
438 &format!("llvm.bitreverse.i{width}"),
439 &[args[0].immediate()],
440 ),
441 sym::rotate_left | sym::rotate_right => {
442 let is_left = name == sym::rotate_left;
443 let val = args[0].immediate();
444 let raw_shift = args[1].immediate();
445 let llvm_name =
447 &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
448
449 let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
452
453 self.call_intrinsic(llvm_name, &[val, val, raw_shift])
454 }
455 sym::saturating_add | sym::saturating_sub => {
456 let is_add = name == sym::saturating_add;
457 let lhs = args[0].immediate();
458 let rhs = args[1].immediate();
459 let llvm_name = &format!(
460 "llvm.{}{}.sat.i{}",
461 if signed { 's' } else { 'u' },
462 if is_add { "add" } else { "sub" },
463 width
464 );
465 self.call_intrinsic(llvm_name, &[lhs, rhs])
466 }
467 _ => bug!(),
468 }
469 }
470
471 sym::raw_eq => {
472 use BackendRepr::*;
473 let tp_ty = fn_args.type_at(0);
474 let layout = self.layout_of(tp_ty).layout;
475 let use_integer_compare = match layout.backend_repr() {
476 Scalar(_) | ScalarPair(_, _) => true,
477 SimdVector { .. } => false,
478 Memory { .. } => {
479 layout.size() <= self.data_layout().pointer_size * 2
483 }
484 };
485
486 let a = args[0].immediate();
487 let b = args[1].immediate();
488 if layout.size().bytes() == 0 {
489 self.const_bool(true)
490 } else if use_integer_compare {
491 let integer_ty = self.type_ix(layout.size().bits());
492 let a_val = self.load(integer_ty, a, layout.align().abi);
493 let b_val = self.load(integer_ty, b, layout.align().abi);
494 self.icmp(IntPredicate::IntEQ, a_val, b_val)
495 } else {
496 let n = self.const_usize(layout.size().bytes());
497 let cmp = self.call_intrinsic("memcmp", &[a, b, n]);
498 match self.cx.sess().target.arch.as_ref() {
499 "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)),
500 _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)),
501 }
502 }
503 }
504
505 sym::compare_bytes => {
506 let cmp = self.call_intrinsic(
508 "memcmp",
509 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
510 );
511 self.sext(cmp, self.type_ix(32))
513 }
514
515 sym::black_box => {
516 args[0].val.store(self, result);
517 let result_val_span = [result.val.llval];
518 let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
528 ("~{memory}", &[])
529 } else {
530 ("r,~{memory}", &result_val_span)
531 };
532 crate::asm::inline_asm_call(
533 self,
534 "",
535 constraint,
536 inputs,
537 self.type_void(),
538 &[],
539 true,
540 false,
541 llvm::AsmDialect::Att,
542 &[span],
543 false,
544 None,
545 None,
546 )
547 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
548
549 return Ok(());
551 }
552
553 _ if name.as_str().starts_with("simd_") => {
554 let mut loaded_args = Vec::new();
557 for (ty, arg) in arg_tys.iter().zip(args) {
558 loaded_args.push(
559 if ty.is_simd()
564 && let OperandValue::Ref(place) = arg.val
565 {
566 let (size, elem_ty) = ty.simd_size_and_type(self.tcx());
567 let elem_ll_ty = match elem_ty.kind() {
568 ty::Float(f) => self.type_float_from_ty(*f),
569 ty::Int(i) => self.type_int_from_ty(*i),
570 ty::Uint(u) => self.type_uint_from_ty(*u),
571 ty::RawPtr(_, _) => self.type_ptr(),
572 _ => unreachable!(),
573 };
574 let loaded =
575 self.load_from_place(self.type_vector(elem_ll_ty, size), place);
576 OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
577 } else {
578 *arg
579 },
580 );
581 }
582
583 let llret_ty = if ret_ty.is_simd()
584 && let BackendRepr::Memory { .. } = self.layout_of(ret_ty).layout.backend_repr
585 {
586 let (size, elem_ty) = ret_ty.simd_size_and_type(self.tcx());
587 let elem_ll_ty = match elem_ty.kind() {
588 ty::Float(f) => self.type_float_from_ty(*f),
589 ty::Int(i) => self.type_int_from_ty(*i),
590 ty::Uint(u) => self.type_uint_from_ty(*u),
591 ty::RawPtr(_, _) => self.type_ptr(),
592 _ => unreachable!(),
593 };
594 self.type_vector(elem_ll_ty, size)
595 } else {
596 llret_ty
597 };
598
599 match generic_simd_intrinsic(
600 self,
601 name,
602 callee_ty,
603 fn_args,
604 &loaded_args,
605 ret_ty,
606 llret_ty,
607 span,
608 ) {
609 Ok(llval) => llval,
610 Err(()) => return Ok(()),
613 }
614 }
615
616 _ => {
617 debug!("unknown intrinsic '{}' -- falling back to default body", name);
618 return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
620 }
621 };
622
623 if result.layout.ty.is_bool() {
624 OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
625 .val
626 .store(self, result);
627 } else if !result.layout.ty.is_unit() {
628 self.store_to_place(llval, result.val);
629 }
630 Ok(())
631 }
632
633 fn abort(&mut self) {
634 self.call_intrinsic("llvm.trap", &[]);
635 }
636
637 fn assume(&mut self, val: Self::Value) {
638 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
639 self.call_intrinsic("llvm.assume", &[val]);
640 }
641 }
642
643 fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
644 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
645 self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)])
646 } else {
647 cond
648 }
649 }
650
651 fn type_test(&mut self, pointer: Self::Value, typeid: Self::Metadata) -> Self::Value {
652 let typeid = self.get_metadata_value(typeid);
655 self.call_intrinsic("llvm.type.test", &[pointer, typeid])
656 }
657
658 fn type_checked_load(
659 &mut self,
660 llvtable: &'ll Value,
661 vtable_byte_offset: u64,
662 typeid: &'ll Metadata,
663 ) -> Self::Value {
664 let typeid = self.get_metadata_value(typeid);
665 let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
666 let type_checked_load =
667 self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]);
668 self.extract_value(type_checked_load, 0)
669 }
670
671 fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
672 self.call_intrinsic("llvm.va_start", &[va_list])
673 }
674
675 fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
676 self.call_intrinsic("llvm.va_end", &[va_list])
677 }
678}
679
680fn catch_unwind_intrinsic<'ll, 'tcx>(
681 bx: &mut Builder<'_, 'll, 'tcx>,
682 try_func: &'ll Value,
683 data: &'ll Value,
684 catch_func: &'ll Value,
685 dest: PlaceRef<'tcx, &'ll Value>,
686) {
687 if bx.sess().panic_strategy() == PanicStrategy::Abort {
688 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
689 bx.call(try_func_ty, None, None, try_func, &[data], None, None);
690 OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
693 } else if wants_msvc_seh(bx.sess()) {
694 codegen_msvc_try(bx, try_func, data, catch_func, dest);
695 } else if wants_wasm_eh(bx.sess()) {
696 codegen_wasm_try(bx, try_func, data, catch_func, dest);
697 } else if bx.sess().target.os == "emscripten" {
698 codegen_emcc_try(bx, try_func, data, catch_func, dest);
699 } else {
700 codegen_gnu_try(bx, try_func, data, catch_func, dest);
701 }
702}
703
704fn codegen_msvc_try<'ll, 'tcx>(
712 bx: &mut Builder<'_, 'll, 'tcx>,
713 try_func: &'ll Value,
714 data: &'ll Value,
715 catch_func: &'ll Value,
716 dest: PlaceRef<'tcx, &'ll Value>,
717) {
718 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
719 bx.set_personality_fn(bx.eh_personality());
720
721 let normal = bx.append_sibling_block("normal");
722 let catchswitch = bx.append_sibling_block("catchswitch");
723 let catchpad_rust = bx.append_sibling_block("catchpad_rust");
724 let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
725 let caught = bx.append_sibling_block("caught");
726
727 let try_func = llvm::get_param(bx.llfn(), 0);
728 let data = llvm::get_param(bx.llfn(), 1);
729 let catch_func = llvm::get_param(bx.llfn(), 2);
730
731 let ptr_size = bx.tcx().data_layout.pointer_size;
787 let ptr_align = bx.tcx().data_layout.pointer_align.abi;
788 let slot = bx.alloca(ptr_size, ptr_align);
789 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
790 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
791
792 bx.switch_to_block(normal);
793 bx.ret(bx.const_i32(0));
794
795 bx.switch_to_block(catchswitch);
796 let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
797
798 let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
813 let type_name = bx.const_bytes(b"rust_panic\0");
814 let type_info =
815 bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
816 let tydesc = bx.declare_global(
817 &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
818 bx.val_ty(type_info),
819 );
820
821 llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
822 if bx.cx.tcx.sess.target.supports_comdat() {
823 llvm::SetUniqueComdat(bx.llmod, tydesc);
824 }
825 llvm::set_initializer(tydesc, type_info);
826
827 bx.switch_to_block(catchpad_rust);
834 let flags = bx.const_i32(8);
835 let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
836 let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
837 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
838 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
839 bx.catch_ret(&funclet, caught);
840
841 bx.switch_to_block(catchpad_foreign);
843 let flags = bx.const_i32(64);
844 let null = bx.const_null(bx.type_ptr());
845 let funclet = bx.catch_pad(cs, &[null, flags, null]);
846 bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None);
847 bx.catch_ret(&funclet, caught);
848
849 bx.switch_to_block(caught);
850 bx.ret(bx.const_i32(1));
851 });
852
853 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
856 OperandValue::Immediate(ret).store(bx, dest);
857}
858
859fn codegen_wasm_try<'ll, 'tcx>(
861 bx: &mut Builder<'_, 'll, 'tcx>,
862 try_func: &'ll Value,
863 data: &'ll Value,
864 catch_func: &'ll Value,
865 dest: PlaceRef<'tcx, &'ll Value>,
866) {
867 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
868 bx.set_personality_fn(bx.eh_personality());
869
870 let normal = bx.append_sibling_block("normal");
871 let catchswitch = bx.append_sibling_block("catchswitch");
872 let catchpad = bx.append_sibling_block("catchpad");
873 let caught = bx.append_sibling_block("caught");
874
875 let try_func = llvm::get_param(bx.llfn(), 0);
876 let data = llvm::get_param(bx.llfn(), 1);
877 let catch_func = llvm::get_param(bx.llfn(), 2);
878
879 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
903 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
904
905 bx.switch_to_block(normal);
906 bx.ret(bx.const_i32(0));
907
908 bx.switch_to_block(catchswitch);
909 let cs = bx.catch_switch(None, None, &[catchpad]);
910
911 bx.switch_to_block(catchpad);
912 let null = bx.const_null(bx.type_ptr());
913 let funclet = bx.catch_pad(cs, &[null]);
914
915 let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[funclet.cleanuppad()]);
916 let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[funclet.cleanuppad()]);
917
918 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
919 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
920 bx.catch_ret(&funclet, caught);
921
922 bx.switch_to_block(caught);
923 bx.ret(bx.const_i32(1));
924 });
925
926 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
929 OperandValue::Immediate(ret).store(bx, dest);
930}
931
932fn codegen_gnu_try<'ll, 'tcx>(
944 bx: &mut Builder<'_, 'll, 'tcx>,
945 try_func: &'ll Value,
946 data: &'ll Value,
947 catch_func: &'ll Value,
948 dest: PlaceRef<'tcx, &'ll Value>,
949) {
950 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
951 let then = bx.append_sibling_block("then");
964 let catch = bx.append_sibling_block("catch");
965
966 let try_func = llvm::get_param(bx.llfn(), 0);
967 let data = llvm::get_param(bx.llfn(), 1);
968 let catch_func = llvm::get_param(bx.llfn(), 2);
969 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
970 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
971
972 bx.switch_to_block(then);
973 bx.ret(bx.const_i32(0));
974
975 bx.switch_to_block(catch);
982 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
983 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
984 let tydesc = bx.const_null(bx.type_ptr());
985 bx.add_clause(vals, tydesc);
986 let ptr = bx.extract_value(vals, 0);
987 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
988 bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None);
989 bx.ret(bx.const_i32(1));
990 });
991
992 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
995 OperandValue::Immediate(ret).store(bx, dest);
996}
997
998fn codegen_emcc_try<'ll, 'tcx>(
1002 bx: &mut Builder<'_, 'll, 'tcx>,
1003 try_func: &'ll Value,
1004 data: &'ll Value,
1005 catch_func: &'ll Value,
1006 dest: PlaceRef<'tcx, &'ll Value>,
1007) {
1008 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1009 let then = bx.append_sibling_block("then");
1027 let catch = bx.append_sibling_block("catch");
1028
1029 let try_func = llvm::get_param(bx.llfn(), 0);
1030 let data = llvm::get_param(bx.llfn(), 1);
1031 let catch_func = llvm::get_param(bx.llfn(), 2);
1032 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1033 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
1034
1035 bx.switch_to_block(then);
1036 bx.ret(bx.const_i32(0));
1037
1038 bx.switch_to_block(catch);
1044 let tydesc = bx.eh_catch_typeinfo();
1045 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1046 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2);
1047 bx.add_clause(vals, tydesc);
1048 bx.add_clause(vals, bx.const_null(bx.type_ptr()));
1049 let ptr = bx.extract_value(vals, 0);
1050 let selector = bx.extract_value(vals, 1);
1051
1052 let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]);
1054 let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
1055 let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
1056
1057 let ptr_size = bx.tcx().data_layout.pointer_size;
1060 let ptr_align = bx.tcx().data_layout.pointer_align.abi;
1061 let i8_align = bx.tcx().data_layout.i8_align.abi;
1062 assert!(i8_align <= ptr_align);
1064 let catch_data = bx.alloca(2 * ptr_size, ptr_align);
1065 bx.store(ptr, catch_data, ptr_align);
1066 let catch_data_1 = bx.inbounds_ptradd(catch_data, bx.const_usize(ptr_size.bytes()));
1067 bx.store(is_rust_panic, catch_data_1, i8_align);
1068
1069 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1070 bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None, None);
1071 bx.ret(bx.const_i32(1));
1072 });
1073
1074 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1077 OperandValue::Immediate(ret).store(bx, dest);
1078}
1079
1080fn gen_fn<'a, 'll, 'tcx>(
1083 cx: &'a CodegenCx<'ll, 'tcx>,
1084 name: &str,
1085 rust_fn_sig: ty::PolyFnSig<'tcx>,
1086 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1087) -> (&'ll Type, &'ll Value) {
1088 let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1089 let llty = fn_abi.llvm_type(cx);
1090 let llfn = cx.declare_fn(name, fn_abi, None);
1091 cx.set_frame_pointer_type(llfn);
1092 cx.apply_target_cpu_attr(llfn);
1093 llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1095 let llbb = Builder::append_block(cx, llfn, "entry-block");
1096 let bx = Builder::build(cx, llbb);
1097 codegen(bx);
1098 (llty, llfn)
1099}
1100
1101fn get_rust_try_fn<'a, 'll, 'tcx>(
1106 cx: &'a CodegenCx<'ll, 'tcx>,
1107 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1108) -> (&'ll Type, &'ll Value) {
1109 if let Some(llfn) = cx.rust_try_fn.get() {
1110 return llfn;
1111 }
1112
1113 let tcx = cx.tcx;
1115 let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1116 let try_fn_ty = Ty::new_fn_ptr(
1118 tcx,
1119 ty::Binder::dummy(tcx.mk_fn_sig(
1120 [i8p],
1121 tcx.types.unit,
1122 false,
1123 hir::Safety::Unsafe,
1124 ExternAbi::Rust,
1125 )),
1126 );
1127 let catch_fn_ty = Ty::new_fn_ptr(
1129 tcx,
1130 ty::Binder::dummy(tcx.mk_fn_sig(
1131 [i8p, i8p],
1132 tcx.types.unit,
1133 false,
1134 hir::Safety::Unsafe,
1135 ExternAbi::Rust,
1136 )),
1137 );
1138 let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig(
1140 [try_fn_ty, i8p, catch_fn_ty],
1141 tcx.types.i32,
1142 false,
1143 hir::Safety::Unsafe,
1144 ExternAbi::Rust,
1145 ));
1146 let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1147 cx.rust_try_fn.set(Some(rust_try));
1148 rust_try
1149}
1150
1151fn generic_simd_intrinsic<'ll, 'tcx>(
1152 bx: &mut Builder<'_, 'll, 'tcx>,
1153 name: Symbol,
1154 callee_ty: Ty<'tcx>,
1155 fn_args: GenericArgsRef<'tcx>,
1156 args: &[OperandRef<'tcx, &'ll Value>],
1157 ret_ty: Ty<'tcx>,
1158 llret_ty: &'ll Type,
1159 span: Span,
1160) -> Result<&'ll Value, ()> {
1161 macro_rules! return_error {
1162 ($diag: expr) => {{
1163 bx.sess().dcx().emit_err($diag);
1164 return Err(());
1165 }};
1166 }
1167
1168 macro_rules! require {
1169 ($cond: expr, $diag: expr) => {
1170 if !$cond {
1171 return_error!($diag);
1172 }
1173 };
1174 }
1175
1176 macro_rules! require_simd {
1177 ($ty: expr, $variant:ident) => {{
1178 require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
1179 $ty.simd_size_and_type(bx.tcx())
1180 }};
1181 }
1182
1183 macro_rules! require_int_or_uint_ty {
1185 ($ty: expr, $diag: expr) => {
1186 match $ty {
1187 ty::Int(i) => i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()),
1188 ty::Uint(i) => {
1189 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
1190 }
1191 _ => {
1192 return_error!($diag);
1193 }
1194 }
1195 };
1196 }
1197
1198 fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
1212 bx: &mut Builder<'a, 'll, 'tcx>,
1213 i_xn: &'ll Value,
1214 in_elem_bitwidth: u64,
1215 in_len: u64,
1216 ) -> &'ll Value {
1217 let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
1219 let shift_indices = vec![shift_idx; in_len as _];
1220 let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
1221 bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
1223 }
1224
1225 let tcx = bx.tcx();
1226 let sig = tcx.normalize_erasing_late_bound_regions(bx.typing_env(), callee_ty.fn_sig(tcx));
1227 let arg_tys = sig.inputs();
1228
1229 if cfg!(debug_assertions) {
1231 for (ty, arg) in arg_tys.iter().zip(args) {
1232 if ty.is_simd() {
1233 assert_matches!(arg.val, OperandValue::Immediate(_));
1234 }
1235 }
1236 }
1237
1238 if name == sym::simd_select_bitmask {
1239 let (len, _) = require_simd!(arg_tys[1], SimdArgument);
1240
1241 let expected_int_bits = len.max(8).next_power_of_two();
1242 let expected_bytes = len.div_ceil(8);
1243
1244 let mask_ty = arg_tys[0];
1245 let mask = match mask_ty.kind() {
1246 ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1247 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1248 ty::Array(elem, len)
1249 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1250 && len
1251 .try_to_target_usize(bx.tcx)
1252 .expect("expected monomorphic const in codegen")
1253 == expected_bytes =>
1254 {
1255 let place = PlaceRef::alloca(bx, args[0].layout);
1256 args[0].val.store(bx, place);
1257 let int_ty = bx.type_ix(expected_bytes * 8);
1258 bx.load(int_ty, place.val.llval, Align::ONE)
1259 }
1260 _ => return_error!(InvalidMonomorphization::InvalidBitmask {
1261 span,
1262 name,
1263 mask_ty,
1264 expected_int_bits,
1265 expected_bytes
1266 }),
1267 };
1268
1269 let i1 = bx.type_i1();
1270 let im = bx.type_ix(len);
1271 let i1xn = bx.type_vector(i1, len);
1272 let m_im = bx.trunc(mask, im);
1273 let m_i1s = bx.bitcast(m_im, i1xn);
1274 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1275 }
1276
1277 let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput);
1279 let in_ty = arg_tys[0];
1280
1281 let comparison = match name {
1282 sym::simd_eq => Some(BinOp::Eq),
1283 sym::simd_ne => Some(BinOp::Ne),
1284 sym::simd_lt => Some(BinOp::Lt),
1285 sym::simd_le => Some(BinOp::Le),
1286 sym::simd_gt => Some(BinOp::Gt),
1287 sym::simd_ge => Some(BinOp::Ge),
1288 _ => None,
1289 };
1290
1291 if let Some(cmp_op) = comparison {
1292 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1293
1294 require!(
1295 in_len == out_len,
1296 InvalidMonomorphization::ReturnLengthInputType {
1297 span,
1298 name,
1299 in_len,
1300 in_ty,
1301 ret_ty,
1302 out_len
1303 }
1304 );
1305 require!(
1306 bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
1307 InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
1308 );
1309
1310 return Ok(compare_simd_types(
1311 bx,
1312 args[0].immediate(),
1313 args[1].immediate(),
1314 in_elem,
1315 llret_ty,
1316 cmp_op,
1317 ));
1318 }
1319
1320 if name == sym::simd_shuffle_const_generic {
1321 let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch();
1322 let n = idx.len() as u64;
1323
1324 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1325 require!(
1326 out_len == n,
1327 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1328 );
1329 require!(
1330 in_elem == out_ty,
1331 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1332 );
1333
1334 let total_len = in_len * 2;
1335
1336 let indices: Option<Vec<_>> = idx
1337 .iter()
1338 .enumerate()
1339 .map(|(arg_idx, val)| {
1340 let idx = val.unwrap_leaf().to_i32();
1341 if idx >= i32::try_from(total_len).unwrap() {
1342 bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
1343 span,
1344 name,
1345 arg_idx: arg_idx as u64,
1346 total_len: total_len.into(),
1347 });
1348 None
1349 } else {
1350 Some(bx.const_i32(idx))
1351 }
1352 })
1353 .collect();
1354 let Some(indices) = indices else {
1355 return Ok(bx.const_null(llret_ty));
1356 };
1357
1358 return Ok(bx.shuffle_vector(
1359 args[0].immediate(),
1360 args[1].immediate(),
1361 bx.const_vector(&indices),
1362 ));
1363 }
1364
1365 if name == sym::simd_shuffle {
1366 let idx_ty = args[2].layout.ty;
1368 let n: u64 = if idx_ty.is_simd()
1369 && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
1370 {
1371 idx_ty.simd_size_and_type(bx.cx.tcx).0
1372 } else {
1373 return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
1374 };
1375
1376 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1377 require!(
1378 out_len == n,
1379 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1380 );
1381 require!(
1382 in_elem == out_ty,
1383 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1384 );
1385
1386 let total_len = u128::from(in_len) * 2;
1387
1388 let indices = args[2].immediate();
1390 for i in 0..n {
1391 let val = bx.const_get_elt(indices, i as u64);
1392 let idx = bx
1393 .const_to_opt_u128(val, true)
1394 .unwrap_or_else(|| bug!("typeck should have already ensured that these are const"));
1395 if idx >= total_len {
1396 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1397 span,
1398 name,
1399 arg_idx: i,
1400 total_len,
1401 });
1402 }
1403 }
1404
1405 return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
1406 }
1407
1408 if name == sym::simd_insert || name == sym::simd_insert_dyn {
1409 require!(
1410 in_elem == arg_tys[2],
1411 InvalidMonomorphization::InsertedType {
1412 span,
1413 name,
1414 in_elem,
1415 in_ty,
1416 out_ty: arg_tys[2]
1417 }
1418 );
1419
1420 let index_imm = if name == sym::simd_insert {
1421 let idx = bx
1422 .const_to_opt_u128(args[1].immediate(), false)
1423 .expect("typeck should have ensure that this is a const");
1424 if idx >= in_len.into() {
1425 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1426 span,
1427 name,
1428 arg_idx: 1,
1429 total_len: in_len.into(),
1430 });
1431 }
1432 bx.const_i32(idx as i32)
1433 } else {
1434 args[1].immediate()
1435 };
1436
1437 return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
1438 }
1439 if name == sym::simd_extract || name == sym::simd_extract_dyn {
1440 require!(
1441 ret_ty == in_elem,
1442 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1443 );
1444 let index_imm = if name == sym::simd_extract {
1445 let idx = bx
1446 .const_to_opt_u128(args[1].immediate(), false)
1447 .expect("typeck should have ensure that this is a const");
1448 if idx >= in_len.into() {
1449 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1450 span,
1451 name,
1452 arg_idx: 1,
1453 total_len: in_len.into(),
1454 });
1455 }
1456 bx.const_i32(idx as i32)
1457 } else {
1458 args[1].immediate()
1459 };
1460
1461 return Ok(bx.extract_element(args[0].immediate(), index_imm));
1462 }
1463
1464 if name == sym::simd_select {
1465 let m_elem_ty = in_elem;
1466 let m_len = in_len;
1467 let (v_len, _) = require_simd!(arg_tys[1], SimdArgument);
1468 require!(
1469 m_len == v_len,
1470 InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
1471 );
1472 let in_elem_bitwidth = require_int_or_uint_ty!(
1473 m_elem_ty.kind(),
1474 InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1475 );
1476 let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
1477 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1478 }
1479
1480 if name == sym::simd_bitmask {
1481 let expected_int_bits = in_len.max(8).next_power_of_two();
1490 let expected_bytes = in_len.div_ceil(8);
1491
1492 let in_elem_bitwidth = require_int_or_uint_ty!(
1494 in_elem.kind(),
1495 InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
1496 );
1497
1498 let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
1499 let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
1501
1502 match ret_ty.kind() {
1503 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
1504 return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
1506 }
1507 ty::Array(elem, len)
1508 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1509 && len
1510 .try_to_target_usize(bx.tcx)
1511 .expect("expected monomorphic const in codegen")
1512 == expected_bytes =>
1513 {
1514 let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
1516
1517 let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
1519 bx.store(ze, ptr, Align::ONE);
1520 let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
1521 return Ok(bx.load(array_ty, ptr, Align::ONE));
1522 }
1523 _ => return_error!(InvalidMonomorphization::CannotReturn {
1524 span,
1525 name,
1526 ret_ty,
1527 expected_int_bits,
1528 expected_bytes
1529 }),
1530 }
1531 }
1532
1533 fn simd_simple_float_intrinsic<'ll, 'tcx>(
1534 name: Symbol,
1535 in_elem: Ty<'_>,
1536 in_ty: Ty<'_>,
1537 in_len: u64,
1538 bx: &mut Builder<'_, 'll, 'tcx>,
1539 span: Span,
1540 args: &[OperandRef<'tcx, &'ll Value>],
1541 ) -> Result<&'ll Value, ()> {
1542 macro_rules! return_error {
1543 ($diag: expr) => {{
1544 bx.sess().dcx().emit_err($diag);
1545 return Err(());
1546 }};
1547 }
1548
1549 let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
1550 let elem_ty = bx.cx.type_float_from_ty(*f);
1551 match f.bit_width() {
1552 16 => ("f16", elem_ty),
1553 32 => ("f32", elem_ty),
1554 64 => ("f64", elem_ty),
1555 128 => ("f128", elem_ty),
1556 _ => return_error!(InvalidMonomorphization::FloatingPointVector {
1557 span,
1558 name,
1559 f_ty: *f,
1560 in_ty,
1561 }),
1562 }
1563 } else {
1564 return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
1565 };
1566
1567 let vec_ty = bx.type_vector(elem_ty, in_len);
1568
1569 let (intr_name, fn_ty) = match name {
1570 sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
1571 sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
1572 sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
1573 sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
1574 sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
1575 sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
1576 sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
1577 sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
1578 sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
1579 sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
1580 sym::simd_relaxed_fma => ("fmuladd", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
1581 sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
1582 sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
1583 sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
1584 sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
1585 _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
1586 };
1587 let llvm_name = &format!("llvm.{intr_name}.v{in_len}{elem_ty_str}");
1588 let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty);
1589 let c = bx.call(
1590 fn_ty,
1591 None,
1592 None,
1593 f,
1594 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
1595 None,
1596 None,
1597 );
1598 Ok(c)
1599 }
1600
1601 if std::matches!(
1602 name,
1603 sym::simd_ceil
1604 | sym::simd_fabs
1605 | sym::simd_fcos
1606 | sym::simd_fexp2
1607 | sym::simd_fexp
1608 | sym::simd_flog10
1609 | sym::simd_flog2
1610 | sym::simd_flog
1611 | sym::simd_floor
1612 | sym::simd_fma
1613 | sym::simd_fsin
1614 | sym::simd_fsqrt
1615 | sym::simd_relaxed_fma
1616 | sym::simd_round
1617 | sym::simd_trunc
1618 ) {
1619 return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
1620 }
1621
1622 fn llvm_vector_str(bx: &Builder<'_, '_, '_>, elem_ty: Ty<'_>, vec_len: u64) -> String {
1626 match *elem_ty.kind() {
1627 ty::Int(v) => format!(
1628 "v{}i{}",
1629 vec_len,
1630 v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
1632 ),
1633 ty::Uint(v) => format!(
1634 "v{}i{}",
1635 vec_len,
1636 v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
1638 ),
1639 ty::Float(v) => format!("v{}f{}", vec_len, v.bit_width()),
1640 ty::RawPtr(_, _) => format!("v{}p0", vec_len),
1641 _ => unreachable!(),
1642 }
1643 }
1644
1645 fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
1646 let elem_ty = match *elem_ty.kind() {
1647 ty::Int(v) => cx.type_int_from_ty(v),
1648 ty::Uint(v) => cx.type_uint_from_ty(v),
1649 ty::Float(v) => cx.type_float_from_ty(v),
1650 ty::RawPtr(_, _) => cx.type_ptr(),
1651 _ => unreachable!(),
1652 };
1653 cx.type_vector(elem_ty, vec_len)
1654 }
1655
1656 if name == sym::simd_gather {
1657 let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1668 let (out_len, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
1669 let (out_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
1671 require_simd!(ret_ty, SimdReturn);
1672
1673 require!(
1675 in_len == out_len,
1676 InvalidMonomorphization::SecondArgumentLength {
1677 span,
1678 name,
1679 in_len,
1680 in_ty,
1681 arg_ty: arg_tys[1],
1682 out_len
1683 }
1684 );
1685 require!(
1686 in_len == out_len2,
1687 InvalidMonomorphization::ThirdArgumentLength {
1688 span,
1689 name,
1690 in_len,
1691 in_ty,
1692 arg_ty: arg_tys[2],
1693 out_len: out_len2
1694 }
1695 );
1696
1697 require!(
1699 ret_ty == in_ty,
1700 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
1701 );
1702
1703 require!(
1704 matches!(
1705 *element_ty1.kind(),
1706 ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
1707 ),
1708 InvalidMonomorphization::ExpectedElementType {
1709 span,
1710 name,
1711 expected_element: element_ty1,
1712 second_arg: arg_tys[1],
1713 in_elem,
1714 in_ty,
1715 mutability: ExpectedPointerMutability::Not,
1716 }
1717 );
1718
1719 let mask_elem_bitwidth = require_int_or_uint_ty!(
1720 element_ty2.kind(),
1721 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1722 );
1723
1724 let alignment_ty = bx.type_i32();
1726 let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1727
1728 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
1730 let mask_ty = bx.type_vector(bx.type_i1(), in_len);
1731
1732 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
1734 let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len);
1735
1736 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
1738 let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len);
1739
1740 let llvm_intrinsic =
1741 format!("llvm.masked.gather.{llvm_elem_vec_str}.{llvm_pointer_vec_str}");
1742 let fn_ty = bx.type_func(
1743 &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty],
1744 llvm_elem_vec_ty,
1745 );
1746 let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1747 let v = bx.call(
1748 fn_ty,
1749 None,
1750 None,
1751 f,
1752 &[args[1].immediate(), alignment, mask, args[0].immediate()],
1753 None,
1754 None,
1755 );
1756 return Ok(v);
1757 }
1758
1759 if name == sym::simd_masked_load {
1760 let mask_ty = in_ty;
1770 let (mask_len, mask_elem) = (in_len, in_elem);
1771
1772 let pointer_ty = arg_tys[1];
1774
1775 let values_ty = arg_tys[2];
1777 let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1778
1779 require_simd!(ret_ty, SimdReturn);
1780
1781 require!(
1783 values_len == mask_len,
1784 InvalidMonomorphization::ThirdArgumentLength {
1785 span,
1786 name,
1787 in_len: mask_len,
1788 in_ty: mask_ty,
1789 arg_ty: values_ty,
1790 out_len: values_len
1791 }
1792 );
1793
1794 require!(
1796 ret_ty == values_ty,
1797 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
1798 );
1799
1800 require!(
1801 matches!(
1802 *pointer_ty.kind(),
1803 ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
1804 ),
1805 InvalidMonomorphization::ExpectedElementType {
1806 span,
1807 name,
1808 expected_element: values_elem,
1809 second_arg: pointer_ty,
1810 in_elem: values_elem,
1811 in_ty: values_ty,
1812 mutability: ExpectedPointerMutability::Not,
1813 }
1814 );
1815
1816 let m_elem_bitwidth = require_int_or_uint_ty!(
1817 mask_elem.kind(),
1818 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1819 );
1820
1821 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1822 let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
1823
1824 let alignment_ty = bx.type_i32();
1826 let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1827
1828 let llvm_pointer = bx.type_ptr();
1829
1830 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1832 let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
1833
1834 let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0");
1835 let fn_ty = bx
1836 .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty);
1837 let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1838 let v = bx.call(
1839 fn_ty,
1840 None,
1841 None,
1842 f,
1843 &[args[1].immediate(), alignment, mask, args[2].immediate()],
1844 None,
1845 None,
1846 );
1847 return Ok(v);
1848 }
1849
1850 if name == sym::simd_masked_store {
1851 let mask_ty = in_ty;
1861 let (mask_len, mask_elem) = (in_len, in_elem);
1862
1863 let pointer_ty = arg_tys[1];
1865
1866 let values_ty = arg_tys[2];
1868 let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1869
1870 require!(
1872 values_len == mask_len,
1873 InvalidMonomorphization::ThirdArgumentLength {
1874 span,
1875 name,
1876 in_len: mask_len,
1877 in_ty: mask_ty,
1878 arg_ty: values_ty,
1879 out_len: values_len
1880 }
1881 );
1882
1883 require!(
1885 matches!(
1886 *pointer_ty.kind(),
1887 ty::RawPtr(p_ty, p_mutbl)
1888 if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
1889 ),
1890 InvalidMonomorphization::ExpectedElementType {
1891 span,
1892 name,
1893 expected_element: values_elem,
1894 second_arg: pointer_ty,
1895 in_elem: values_elem,
1896 in_ty: values_ty,
1897 mutability: ExpectedPointerMutability::Mut,
1898 }
1899 );
1900
1901 let m_elem_bitwidth = require_int_or_uint_ty!(
1902 mask_elem.kind(),
1903 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1904 );
1905
1906 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1907 let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
1908
1909 let alignment_ty = bx.type_i32();
1911 let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1912
1913 let ret_t = bx.type_void();
1914
1915 let llvm_pointer = bx.type_ptr();
1916
1917 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1919 let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
1920
1921 let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0");
1922 let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t);
1923 let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1924 let v = bx.call(
1925 fn_ty,
1926 None,
1927 None,
1928 f,
1929 &[args[2].immediate(), args[1].immediate(), alignment, mask],
1930 None,
1931 None,
1932 );
1933 return Ok(v);
1934 }
1935
1936 if name == sym::simd_scatter {
1937 let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1947 let (element_len1, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
1948 let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
1949
1950 require!(
1952 in_len == element_len1,
1953 InvalidMonomorphization::SecondArgumentLength {
1954 span,
1955 name,
1956 in_len,
1957 in_ty,
1958 arg_ty: arg_tys[1],
1959 out_len: element_len1
1960 }
1961 );
1962 require!(
1963 in_len == element_len2,
1964 InvalidMonomorphization::ThirdArgumentLength {
1965 span,
1966 name,
1967 in_len,
1968 in_ty,
1969 arg_ty: arg_tys[2],
1970 out_len: element_len2
1971 }
1972 );
1973
1974 require!(
1975 matches!(
1976 *element_ty1.kind(),
1977 ty::RawPtr(p_ty, p_mutbl)
1978 if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
1979 ),
1980 InvalidMonomorphization::ExpectedElementType {
1981 span,
1982 name,
1983 expected_element: element_ty1,
1984 second_arg: arg_tys[1],
1985 in_elem,
1986 in_ty,
1987 mutability: ExpectedPointerMutability::Mut,
1988 }
1989 );
1990
1991 let mask_elem_bitwidth = require_int_or_uint_ty!(
1993 element_ty2.kind(),
1994 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1995 );
1996
1997 let alignment_ty = bx.type_i32();
1999 let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
2000
2001 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2003 let mask_ty = bx.type_vector(bx.type_i1(), in_len);
2004
2005 let ret_t = bx.type_void();
2006
2007 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2009 let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len);
2010
2011 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2013 let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len);
2014
2015 let llvm_intrinsic =
2016 format!("llvm.masked.scatter.{llvm_elem_vec_str}.{llvm_pointer_vec_str}");
2017 let fn_ty =
2018 bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t);
2019 let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2020 let v = bx.call(
2021 fn_ty,
2022 None,
2023 None,
2024 f,
2025 &[args[0].immediate(), args[1].immediate(), alignment, mask],
2026 None,
2027 None,
2028 );
2029 return Ok(v);
2030 }
2031
2032 macro_rules! arith_red {
2033 ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
2034 $identity:expr) => {
2035 if name == sym::$name {
2036 require!(
2037 ret_ty == in_elem,
2038 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2039 );
2040 return match in_elem.kind() {
2041 ty::Int(_) | ty::Uint(_) => {
2042 let r = bx.$integer_reduce(args[0].immediate());
2043 if $ordered {
2044 Ok(bx.$op(args[1].immediate(), r))
2047 } else {
2048 Ok(bx.$integer_reduce(args[0].immediate()))
2049 }
2050 }
2051 ty::Float(f) => {
2052 let acc = if $ordered {
2053 args[1].immediate()
2055 } else {
2056 match f.bit_width() {
2058 32 => bx.const_real(bx.type_f32(), $identity),
2059 64 => bx.const_real(bx.type_f64(), $identity),
2060 v => return_error!(
2061 InvalidMonomorphization::UnsupportedSymbolOfSize {
2062 span,
2063 name,
2064 symbol: sym::$name,
2065 in_ty,
2066 in_elem,
2067 size: v,
2068 ret_ty
2069 }
2070 ),
2071 }
2072 };
2073 Ok(bx.$float_reduce(acc, args[0].immediate()))
2074 }
2075 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2076 span,
2077 name,
2078 symbol: sym::$name,
2079 in_ty,
2080 in_elem,
2081 ret_ty
2082 }),
2083 };
2084 }
2085 };
2086 }
2087
2088 arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
2089 arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
2090 arith_red!(
2091 simd_reduce_add_unordered: vector_reduce_add,
2092 vector_reduce_fadd_reassoc,
2093 false,
2094 add,
2095 -0.0
2096 );
2097 arith_red!(
2098 simd_reduce_mul_unordered: vector_reduce_mul,
2099 vector_reduce_fmul_reassoc,
2100 false,
2101 mul,
2102 1.0
2103 );
2104
2105 macro_rules! minmax_red {
2106 ($name:ident: $int_red:ident, $float_red:ident) => {
2107 if name == sym::$name {
2108 require!(
2109 ret_ty == in_elem,
2110 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2111 );
2112 return match in_elem.kind() {
2113 ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
2114 ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
2115 ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
2116 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2117 span,
2118 name,
2119 symbol: sym::$name,
2120 in_ty,
2121 in_elem,
2122 ret_ty
2123 }),
2124 };
2125 }
2126 };
2127 }
2128
2129 minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
2130 minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
2131
2132 macro_rules! bitwise_red {
2133 ($name:ident : $red:ident, $boolean:expr) => {
2134 if name == sym::$name {
2135 let input = if !$boolean {
2136 require!(
2137 ret_ty == in_elem,
2138 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2139 );
2140 args[0].immediate()
2141 } else {
2142 let bitwidth = match in_elem.kind() {
2143 ty::Int(i) => {
2144 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
2145 }
2146 ty::Uint(i) => {
2147 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
2148 }
2149 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2150 span,
2151 name,
2152 symbol: sym::$name,
2153 in_ty,
2154 in_elem,
2155 ret_ty
2156 }),
2157 };
2158
2159 vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
2160 };
2161 return match in_elem.kind() {
2162 ty::Int(_) | ty::Uint(_) => {
2163 let r = bx.$red(input);
2164 Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
2165 }
2166 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2167 span,
2168 name,
2169 symbol: sym::$name,
2170 in_ty,
2171 in_elem,
2172 ret_ty
2173 }),
2174 };
2175 }
2176 };
2177 }
2178
2179 bitwise_red!(simd_reduce_and: vector_reduce_and, false);
2180 bitwise_red!(simd_reduce_or: vector_reduce_or, false);
2181 bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
2182 bitwise_red!(simd_reduce_all: vector_reduce_and, true);
2183 bitwise_red!(simd_reduce_any: vector_reduce_or, true);
2184
2185 if name == sym::simd_cast_ptr {
2186 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2187 require!(
2188 in_len == out_len,
2189 InvalidMonomorphization::ReturnLengthInputType {
2190 span,
2191 name,
2192 in_len,
2193 in_ty,
2194 ret_ty,
2195 out_len
2196 }
2197 );
2198
2199 match in_elem.kind() {
2200 ty::RawPtr(p_ty, _) => {
2201 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2202 bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2203 });
2204 require!(
2205 metadata.is_unit(),
2206 InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
2207 );
2208 }
2209 _ => {
2210 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2211 }
2212 }
2213 match out_elem.kind() {
2214 ty::RawPtr(p_ty, _) => {
2215 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2216 bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2217 });
2218 require!(
2219 metadata.is_unit(),
2220 InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
2221 );
2222 }
2223 _ => {
2224 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2225 }
2226 }
2227
2228 return Ok(args[0].immediate());
2229 }
2230
2231 if name == sym::simd_expose_provenance {
2232 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2233 require!(
2234 in_len == out_len,
2235 InvalidMonomorphization::ReturnLengthInputType {
2236 span,
2237 name,
2238 in_len,
2239 in_ty,
2240 ret_ty,
2241 out_len
2242 }
2243 );
2244
2245 match in_elem.kind() {
2246 ty::RawPtr(_, _) => {}
2247 _ => {
2248 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2249 }
2250 }
2251 match out_elem.kind() {
2252 ty::Uint(ty::UintTy::Usize) => {}
2253 _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
2254 }
2255
2256 return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
2257 }
2258
2259 if name == sym::simd_with_exposed_provenance {
2260 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2261 require!(
2262 in_len == out_len,
2263 InvalidMonomorphization::ReturnLengthInputType {
2264 span,
2265 name,
2266 in_len,
2267 in_ty,
2268 ret_ty,
2269 out_len
2270 }
2271 );
2272
2273 match in_elem.kind() {
2274 ty::Uint(ty::UintTy::Usize) => {}
2275 _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
2276 }
2277 match out_elem.kind() {
2278 ty::RawPtr(_, _) => {}
2279 _ => {
2280 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2281 }
2282 }
2283
2284 return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
2285 }
2286
2287 if name == sym::simd_cast || name == sym::simd_as {
2288 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2289 require!(
2290 in_len == out_len,
2291 InvalidMonomorphization::ReturnLengthInputType {
2292 span,
2293 name,
2294 in_len,
2295 in_ty,
2296 ret_ty,
2297 out_len
2298 }
2299 );
2300 if in_elem == out_elem {
2302 return Ok(args[0].immediate());
2303 }
2304
2305 #[derive(Copy, Clone)]
2306 enum Sign {
2307 Unsigned,
2308 Signed,
2309 }
2310 use Sign::*;
2311
2312 enum Style {
2313 Float,
2314 Int(Sign),
2315 Unsupported,
2316 }
2317
2318 let (in_style, in_width) = match in_elem.kind() {
2319 ty::Int(i) => (
2322 Style::Int(Signed),
2323 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2324 ),
2325 ty::Uint(u) => (
2326 Style::Int(Unsigned),
2327 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2328 ),
2329 ty::Float(f) => (Style::Float, f.bit_width()),
2330 _ => (Style::Unsupported, 0),
2331 };
2332 let (out_style, out_width) = match out_elem.kind() {
2333 ty::Int(i) => (
2334 Style::Int(Signed),
2335 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2336 ),
2337 ty::Uint(u) => (
2338 Style::Int(Unsigned),
2339 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2340 ),
2341 ty::Float(f) => (Style::Float, f.bit_width()),
2342 _ => (Style::Unsupported, 0),
2343 };
2344
2345 match (in_style, out_style) {
2346 (Style::Int(sign), Style::Int(_)) => {
2347 return Ok(match in_width.cmp(&out_width) {
2348 Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
2349 Ordering::Equal => args[0].immediate(),
2350 Ordering::Less => match sign {
2351 Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
2352 Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
2353 },
2354 });
2355 }
2356 (Style::Int(Sign::Signed), Style::Float) => {
2357 return Ok(bx.sitofp(args[0].immediate(), llret_ty));
2358 }
2359 (Style::Int(Sign::Unsigned), Style::Float) => {
2360 return Ok(bx.uitofp(args[0].immediate(), llret_ty));
2361 }
2362 (Style::Float, Style::Int(sign)) => {
2363 return Ok(match (sign, name == sym::simd_as) {
2364 (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
2365 (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
2366 (_, true) => bx.cast_float_to_int(
2367 matches!(sign, Sign::Signed),
2368 args[0].immediate(),
2369 llret_ty,
2370 ),
2371 });
2372 }
2373 (Style::Float, Style::Float) => {
2374 return Ok(match in_width.cmp(&out_width) {
2375 Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
2376 Ordering::Equal => args[0].immediate(),
2377 Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
2378 });
2379 }
2380 _ => { }
2381 }
2382 return_error!(InvalidMonomorphization::UnsupportedCast {
2383 span,
2384 name,
2385 in_ty,
2386 in_elem,
2387 ret_ty,
2388 out_elem
2389 });
2390 }
2391 macro_rules! arith_binary {
2392 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2393 $(if name == sym::$name {
2394 match in_elem.kind() {
2395 $($(ty::$p(_))|* => {
2396 return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
2397 })*
2398 _ => {},
2399 }
2400 return_error!(
2401 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2402 );
2403 })*
2404 }
2405 }
2406 arith_binary! {
2407 simd_add: Uint, Int => add, Float => fadd;
2408 simd_sub: Uint, Int => sub, Float => fsub;
2409 simd_mul: Uint, Int => mul, Float => fmul;
2410 simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
2411 simd_rem: Uint => urem, Int => srem, Float => frem;
2412 simd_shl: Uint, Int => shl;
2413 simd_shr: Uint => lshr, Int => ashr;
2414 simd_and: Uint, Int => and;
2415 simd_or: Uint, Int => or;
2416 simd_xor: Uint, Int => xor;
2417 simd_fmax: Float => maxnum;
2418 simd_fmin: Float => minnum;
2419
2420 }
2421 macro_rules! arith_unary {
2422 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2423 $(if name == sym::$name {
2424 match in_elem.kind() {
2425 $($(ty::$p(_))|* => {
2426 return Ok(bx.$call(args[0].immediate()))
2427 })*
2428 _ => {},
2429 }
2430 return_error!(
2431 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2432 );
2433 })*
2434 }
2435 }
2436 arith_unary! {
2437 simd_neg: Int => neg, Float => fneg;
2438 }
2439
2440 if matches!(
2442 name,
2443 sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz
2444 ) {
2445 let vec_ty = bx.cx.type_vector(
2446 match *in_elem.kind() {
2447 ty::Int(i) => bx.cx.type_int_from_ty(i),
2448 ty::Uint(i) => bx.cx.type_uint_from_ty(i),
2449 _ => return_error!(InvalidMonomorphization::UnsupportedOperation {
2450 span,
2451 name,
2452 in_ty,
2453 in_elem
2454 }),
2455 },
2456 in_len as u64,
2457 );
2458 let intrinsic_name = match name {
2459 sym::simd_bswap => "bswap",
2460 sym::simd_bitreverse => "bitreverse",
2461 sym::simd_ctlz => "ctlz",
2462 sym::simd_ctpop => "ctpop",
2463 sym::simd_cttz => "cttz",
2464 _ => unreachable!(),
2465 };
2466 let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
2467 let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,);
2468
2469 return match name {
2470 sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
2472 sym::simd_ctlz | sym::simd_cttz => {
2473 let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
2475 let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
2476 let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2477 Ok(bx.call(
2478 fn_ty,
2479 None,
2480 None,
2481 f,
2482 &[args[0].immediate(), dont_poison_on_zero],
2483 None,
2484 None,
2485 ))
2486 }
2487 sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
2488 let fn_ty = bx.type_func(&[vec_ty], vec_ty);
2490 let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2491 Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
2492 }
2493 _ => unreachable!(),
2494 };
2495 }
2496
2497 if name == sym::simd_arith_offset {
2498 let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
2500 span_bug!(span, "must be called with a vector of pointer types as first argument")
2501 });
2502 let layout = bx.layout_of(pointee);
2503 let ptrs = args[0].immediate();
2504 let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx());
2507 if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
2508 span_bug!(
2509 span,
2510 "must be called with a vector of pointer-sized integers as second argument"
2511 );
2512 }
2513 let offsets = args[1].immediate();
2514
2515 return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets]));
2516 }
2517
2518 if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
2519 let lhs = args[0].immediate();
2520 let rhs = args[1].immediate();
2521 let is_add = name == sym::simd_saturating_add;
2522 let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
2523 let (signed, elem_width, elem_ty) = match *in_elem.kind() {
2524 ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
2525 ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
2526 _ => {
2527 return_error!(InvalidMonomorphization::ExpectedVectorElementType {
2528 span,
2529 name,
2530 expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1,
2531 vector_type: arg_tys[0]
2532 });
2533 }
2534 };
2535 let llvm_intrinsic = &format!(
2536 "llvm.{}{}.sat.v{}i{}",
2537 if signed { 's' } else { 'u' },
2538 if is_add { "add" } else { "sub" },
2539 in_len,
2540 elem_width
2541 );
2542 let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
2543
2544 let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty);
2545 let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2546 let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None, None);
2547 return Ok(v);
2548 }
2549
2550 span_bug!(span, "unknown SIMD intrinsic");
2551}