rustc_codegen_llvm/
asm.rs

1use std::assert_matches::assert_matches;
2
3use rustc_abi::{BackendRepr, Float, Integer, Primitive, Scalar};
4use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
5use rustc_codegen_ssa::mir::operand::OperandValue;
6use rustc_codegen_ssa::traits::*;
7use rustc_data_structures::fx::FxHashMap;
8use rustc_middle::ty::Instance;
9use rustc_middle::ty::layout::TyAndLayout;
10use rustc_middle::{bug, span_bug};
11use rustc_span::{Pos, Span, Symbol, sym};
12use rustc_target::asm::*;
13use smallvec::SmallVec;
14use tracing::debug;
15
16use crate::builder::Builder;
17use crate::common::Funclet;
18use crate::context::CodegenCx;
19use crate::llvm::ToLlvmBool;
20use crate::type_::Type;
21use crate::type_of::LayoutLlvmExt;
22use crate::value::Value;
23use crate::{attributes, llvm};
24
25impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
26    fn codegen_inline_asm(
27        &mut self,
28        template: &[InlineAsmTemplatePiece],
29        operands: &[InlineAsmOperandRef<'tcx, Self>],
30        options: InlineAsmOptions,
31        line_spans: &[Span],
32        instance: Instance<'_>,
33        dest: Option<Self::BasicBlock>,
34        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
35    ) {
36        let asm_arch = self.tcx.sess.asm_arch.unwrap();
37
38        // Collect the types of output operands
39        let mut constraints = vec![];
40        let mut clobbers = vec![];
41        let mut output_types = vec![];
42        let mut op_idx = FxHashMap::default();
43        let mut clobbered_x87 = false;
44        for (idx, op) in operands.iter().enumerate() {
45            match *op {
46                InlineAsmOperandRef::Out { reg, late, place } => {
47                    let is_target_supported = |reg_class: InlineAsmRegClass| {
48                        for &(_, feature) in reg_class.supported_types(asm_arch, true) {
49                            if let Some(feature) = feature {
50                                if self
51                                    .tcx
52                                    .asm_target_features(instance.def_id())
53                                    .contains(&feature)
54                                {
55                                    return true;
56                                }
57                            } else {
58                                // Register class is unconditionally supported
59                                return true;
60                            }
61                        }
62                        false
63                    };
64
65                    let mut layout = None;
66                    let ty = if let Some(ref place) = place {
67                        layout = Some(&place.layout);
68                        llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)
69                    } else if matches!(
70                        reg.reg_class(),
71                        InlineAsmRegClass::X86(
72                            X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
73                        )
74                    ) {
75                        // Special handling for x87/mmx registers: we always
76                        // clobber the whole set if one register is marked as
77                        // clobbered. This is due to the way LLVM handles the
78                        // FP stack in inline assembly.
79                        if !clobbered_x87 {
80                            clobbered_x87 = true;
81                            clobbers.push("~{st}".to_string());
82                            for i in 1..=7 {
83                                clobbers.push(format!("~{{st({})}}", i));
84                            }
85                        }
86                        continue;
87                    } else if !is_target_supported(reg.reg_class())
88                        || reg.reg_class().is_clobber_only(asm_arch, true)
89                    {
90                        // We turn discarded outputs into clobber constraints
91                        // if the target feature needed by the register class is
92                        // disabled. This is necessary otherwise LLVM will try
93                        // to actually allocate a register for the dummy output.
94                        assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_));
95                        clobbers.push(format!("~{}", reg_to_llvm(reg, None)));
96                        continue;
97                    } else {
98                        // If the output is discarded, we don't really care what
99                        // type is used. We're just using this to tell LLVM to
100                        // reserve the register.
101                        dummy_output_type(self.cx, reg.reg_class())
102                    };
103                    output_types.push(ty);
104                    op_idx.insert(idx, constraints.len());
105                    let prefix = if late { "=" } else { "=&" };
106                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout)));
107                }
108                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
109                    let layout = if let Some(ref out_place) = out_place {
110                        &out_place.layout
111                    } else {
112                        // LLVM required tied operands to have the same type,
113                        // so we just use the type of the input.
114                        &in_value.layout
115                    };
116                    let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);
117                    output_types.push(ty);
118                    op_idx.insert(idx, constraints.len());
119                    let prefix = if late { "=" } else { "=&" };
120                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
121                }
122                _ => {}
123            }
124        }
125
126        // Collect input operands
127        let mut inputs = vec![];
128        for (idx, op) in operands.iter().enumerate() {
129            match *op {
130                InlineAsmOperandRef::In { reg, value } => {
131                    let llval = llvm_fixup_input(
132                        self,
133                        value.immediate(),
134                        reg.reg_class(),
135                        &value.layout,
136                        instance,
137                    );
138                    inputs.push(llval);
139                    op_idx.insert(idx, constraints.len());
140                    constraints.push(reg_to_llvm(reg, Some(&value.layout)));
141                }
142                InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => {
143                    let value = llvm_fixup_input(
144                        self,
145                        in_value.immediate(),
146                        reg.reg_class(),
147                        &in_value.layout,
148                        instance,
149                    );
150                    inputs.push(value);
151
152                    // In the case of fixed registers, we have the choice of
153                    // either using a tied operand or duplicating the constraint.
154                    // We prefer the latter because it matches the behavior of
155                    // Clang.
156                    if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) {
157                        constraints.push(reg_to_llvm(reg, Some(&in_value.layout)));
158                    } else {
159                        constraints.push(format!("{}", op_idx[&idx]));
160                    }
161                }
162                InlineAsmOperandRef::SymFn { instance } => {
163                    inputs.push(self.cx.get_fn(instance));
164                    op_idx.insert(idx, constraints.len());
165                    constraints.push("s".to_string());
166                }
167                InlineAsmOperandRef::SymStatic { def_id } => {
168                    inputs.push(self.cx.get_static(def_id));
169                    op_idx.insert(idx, constraints.len());
170                    constraints.push("s".to_string());
171                }
172                _ => {}
173            }
174        }
175
176        // Build the template string
177        let mut labels = vec![];
178        let mut template_str = String::new();
179        for piece in template {
180            match *piece {
181                InlineAsmTemplatePiece::String(ref s) => {
182                    if s.contains('$') {
183                        for c in s.chars() {
184                            if c == '$' {
185                                template_str.push_str("$$");
186                            } else {
187                                template_str.push(c);
188                            }
189                        }
190                    } else {
191                        template_str.push_str(s)
192                    }
193                }
194                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
195                    match operands[operand_idx] {
196                        InlineAsmOperandRef::In { reg, .. }
197                        | InlineAsmOperandRef::Out { reg, .. }
198                        | InlineAsmOperandRef::InOut { reg, .. } => {
199                            let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
200                            if let Some(modifier) = modifier {
201                                template_str.push_str(&format!(
202                                    "${{{}:{}}}",
203                                    op_idx[&operand_idx], modifier
204                                ));
205                            } else {
206                                template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx]));
207                            }
208                        }
209                        InlineAsmOperandRef::Const { ref string } => {
210                            // Const operands get injected directly into the template
211                            template_str.push_str(string);
212                        }
213                        InlineAsmOperandRef::SymFn { .. }
214                        | InlineAsmOperandRef::SymStatic { .. } => {
215                            // Only emit the raw symbol name
216                            template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
217                        }
218                        InlineAsmOperandRef::Label { label } => {
219                            template_str.push_str(&format!("${{{}:l}}", constraints.len()));
220                            constraints.push("!i".to_owned());
221                            labels.push(label);
222                        }
223                    }
224                }
225            }
226        }
227
228        constraints.append(&mut clobbers);
229        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
230            match asm_arch {
231                InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC | InlineAsmArch::Arm => {
232                    constraints.push("~{cc}".to_string());
233                }
234                InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
235                    constraints.extend_from_slice(&[
236                        "~{dirflag}".to_string(),
237                        "~{fpsr}".to_string(),
238                        "~{flags}".to_string(),
239                    ]);
240                }
241                InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
242                    constraints.extend_from_slice(&[
243                        "~{vtype}".to_string(),
244                        "~{vl}".to_string(),
245                        "~{vxsat}".to_string(),
246                        "~{vxrm}".to_string(),
247                    ]);
248                }
249                InlineAsmArch::Avr => {
250                    constraints.push("~{sreg}".to_string());
251                }
252                InlineAsmArch::Nvptx64 => {}
253                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
254                InlineAsmArch::Hexagon => {}
255                InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
256                    constraints.extend_from_slice(&[
257                        "~{$fcc0}".to_string(),
258                        "~{$fcc1}".to_string(),
259                        "~{$fcc2}".to_string(),
260                        "~{$fcc3}".to_string(),
261                        "~{$fcc4}".to_string(),
262                        "~{$fcc5}".to_string(),
263                        "~{$fcc6}".to_string(),
264                        "~{$fcc7}".to_string(),
265                    ]);
266                }
267                InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
268                InlineAsmArch::S390x => {
269                    constraints.push("~{cc}".to_string());
270                }
271                InlineAsmArch::Sparc | InlineAsmArch::Sparc64 => {
272                    // In LLVM, ~{icc} represents icc and xcc in 64-bit code.
273                    // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/SparcRegisterInfo.td#L64
274                    constraints.push("~{icc}".to_string());
275                    constraints.push("~{fcc0}".to_string());
276                    constraints.push("~{fcc1}".to_string());
277                    constraints.push("~{fcc2}".to_string());
278                    constraints.push("~{fcc3}".to_string());
279                }
280                InlineAsmArch::SpirV => {}
281                InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
282                InlineAsmArch::Bpf => {}
283                InlineAsmArch::Msp430 => {
284                    constraints.push("~{sr}".to_string());
285                }
286                InlineAsmArch::M68k => {
287                    constraints.push("~{ccr}".to_string());
288                }
289                InlineAsmArch::CSKY => {
290                    constraints.push("~{psr}".to_string());
291                }
292            }
293        }
294        if !options.contains(InlineAsmOptions::NOMEM) {
295            // This is actually ignored by LLVM, but it's probably best to keep
296            // it just in case. LLVM instead uses the ReadOnly/ReadNone
297            // attributes on the call instruction to optimize.
298            constraints.push("~{memory}".to_string());
299        }
300        let volatile = !options.contains(InlineAsmOptions::PURE);
301        let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
302        let output_type = match &output_types[..] {
303            [] => self.type_void(),
304            [ty] => ty,
305            tys => self.type_struct(tys, false),
306        };
307        let dialect = match asm_arch {
308            InlineAsmArch::X86 | InlineAsmArch::X86_64
309                if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
310            {
311                llvm::AsmDialect::Intel
312            }
313            _ => llvm::AsmDialect::Att,
314        };
315        let result = inline_asm_call(
316            self,
317            &template_str,
318            &constraints.join(","),
319            &inputs,
320            output_type,
321            &labels,
322            volatile,
323            alignstack,
324            dialect,
325            line_spans,
326            options.contains(InlineAsmOptions::MAY_UNWIND),
327            dest,
328            catch_funclet,
329        )
330        .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
331
332        let mut attrs = SmallVec::<[_; 2]>::new();
333        if options.contains(InlineAsmOptions::PURE) {
334            if options.contains(InlineAsmOptions::NOMEM) {
335                attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx));
336            } else if options.contains(InlineAsmOptions::READONLY) {
337                attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx));
338            }
339            attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
340        } else if options.contains(InlineAsmOptions::NOMEM) {
341            attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
342        } else {
343            // LLVM doesn't have an attribute to represent ReadOnly + SideEffect
344        }
345        attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
346
347        // Write results to outputs. We need to do this for all possible control flow.
348        //
349        // Note that `dest` maybe populated with unreachable_block when asm goto with outputs
350        // is used (because we need to codegen callbr which always needs a destination), so
351        // here we use the NORETURN option to determine if `dest` should be used.
352        for block in (if options.contains(InlineAsmOptions::NORETURN) { None } else { Some(dest) })
353            .into_iter()
354            .chain(labels.iter().copied().map(Some))
355        {
356            if let Some(block) = block {
357                self.switch_to_block(block);
358            }
359
360            for (idx, op) in operands.iter().enumerate() {
361                if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
362                | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
363                {
364                    let value = if output_types.len() == 1 {
365                        result
366                    } else {
367                        self.extract_value(result, op_idx[&idx] as u64)
368                    };
369                    let value =
370                        llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);
371                    OperandValue::Immediate(value).store(self, place);
372                }
373            }
374        }
375    }
376}
377
378impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
379    fn codegen_global_asm(
380        &mut self,
381        template: &[InlineAsmTemplatePiece],
382        operands: &[GlobalAsmOperandRef<'tcx>],
383        options: InlineAsmOptions,
384        _line_spans: &[Span],
385    ) {
386        let asm_arch = self.tcx.sess.asm_arch.unwrap();
387
388        // Build the template string
389        let mut template_str = String::new();
390
391        // On X86 platforms there are two assembly syntaxes. Rust uses intel by default,
392        // but AT&T can be specified explicitly.
393        if matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) {
394            if options.contains(InlineAsmOptions::ATT_SYNTAX) {
395                template_str.push_str(".att_syntax\n")
396            } else {
397                template_str.push_str(".intel_syntax\n")
398            }
399        }
400
401        for piece in template {
402            match *piece {
403                InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
404                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
405                    match operands[operand_idx] {
406                        GlobalAsmOperandRef::Const { ref string } => {
407                            // Const operands get injected directly into the
408                            // template. Note that we don't need to escape $
409                            // here unlike normal inline assembly.
410                            template_str.push_str(string);
411                        }
412                        GlobalAsmOperandRef::SymFn { instance } => {
413                            let llval = self.get_fn(instance);
414                            self.add_compiler_used_global(llval);
415                            let symbol = llvm::build_string(|s| unsafe {
416                                llvm::LLVMRustGetMangledName(llval, s);
417                            })
418                            .expect("symbol is not valid UTF-8");
419                            template_str.push_str(&symbol);
420                        }
421                        GlobalAsmOperandRef::SymStatic { def_id } => {
422                            let llval = self
423                                .renamed_statics
424                                .borrow()
425                                .get(&def_id)
426                                .copied()
427                                .unwrap_or_else(|| self.get_static(def_id));
428                            self.add_compiler_used_global(llval);
429                            let symbol = llvm::build_string(|s| unsafe {
430                                llvm::LLVMRustGetMangledName(llval, s);
431                            })
432                            .expect("symbol is not valid UTF-8");
433                            template_str.push_str(&symbol);
434                        }
435                    }
436                }
437            }
438        }
439
440        // Just to play it safe, if intel was used, reset the assembly syntax to att.
441        if matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
442            && !options.contains(InlineAsmOptions::ATT_SYNTAX)
443        {
444            template_str.push_str("\n.att_syntax\n");
445        }
446
447        llvm::append_module_inline_asm(self.llmod, template_str.as_bytes());
448    }
449
450    fn mangled_name(&self, instance: Instance<'tcx>) -> String {
451        let llval = self.get_fn(instance);
452        llvm::build_string(|s| unsafe {
453            llvm::LLVMRustGetMangledName(llval, s);
454        })
455        .expect("symbol is not valid UTF-8")
456    }
457}
458
459pub(crate) fn inline_asm_call<'ll>(
460    bx: &mut Builder<'_, 'll, '_>,
461    asm: &str,
462    cons: &str,
463    inputs: &[&'ll Value],
464    output: &'ll llvm::Type,
465    labels: &[&'ll llvm::BasicBlock],
466    volatile: bool,
467    alignstack: bool,
468    dia: llvm::AsmDialect,
469    line_spans: &[Span],
470    unwind: bool,
471    dest: Option<&'ll llvm::BasicBlock>,
472    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
473) -> Option<&'ll Value> {
474    let argtys = inputs
475        .iter()
476        .map(|v| {
477            debug!("Asm Input Type: {:?}", *v);
478            bx.cx.val_ty(*v)
479        })
480        .collect::<Vec<_>>();
481
482    debug!("Asm Output Type: {:?}", output);
483    let fty = bx.cx.type_func(&argtys, output);
484
485    // Ask LLVM to verify that the constraints are well-formed.
486    let constraints_ok = unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr(), cons.len()) };
487    debug!("constraint verification result: {:?}", constraints_ok);
488    if !constraints_ok {
489        // LLVM has detected an issue with our constraints, so bail out.
490        return None;
491    }
492
493    let v = unsafe {
494        llvm::LLVMGetInlineAsm(
495            fty,
496            asm.as_ptr(),
497            asm.len(),
498            cons.as_ptr(),
499            cons.len(),
500            volatile.to_llvm_bool(),
501            alignstack.to_llvm_bool(),
502            dia,
503            unwind.to_llvm_bool(),
504        )
505    };
506
507    let call = if !labels.is_empty() {
508        assert!(catch_funclet.is_none());
509        bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None)
510    } else if let Some((catch, funclet)) = catch_funclet {
511        bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None)
512    } else {
513        bx.call(fty, None, None, v, inputs, None, None)
514    };
515
516    // Store mark in a metadata node so we can map LLVM errors
517    // back to source locations. See #17552.
518    let key = "srcloc";
519    let kind = bx.get_md_kind_id(key);
520
521    // `srcloc` contains one 64-bit integer for each line of assembly code,
522    // where the lower 32 bits hold the lo byte position and the upper 32 bits
523    // hold the hi byte position.
524    let mut srcloc = vec![];
525    if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
526        // LLVM inserts an extra line to add the ".intel_syntax", so add
527        // a dummy srcloc entry for it.
528        //
529        // Don't do this if we only have 1 line span since that may be
530        // due to the asm template string coming from a macro. LLVM will
531        // default to the first srcloc for lines that don't have an
532        // associated srcloc.
533        srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0)));
534    }
535    srcloc.extend(line_spans.iter().map(|span| {
536        llvm::LLVMValueAsMetadata(
537            bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
538        )
539    }));
540    let md = unsafe { llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()) };
541    let md = bx.get_metadata_value(md);
542    llvm::LLVMSetMetadata(call, kind, md);
543
544    Some(call)
545}
546
547/// If the register is an xmm/ymm/zmm register then return its index.
548fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
549    use X86InlineAsmReg::*;
550    match reg {
551        InlineAsmReg::X86(reg) if reg as u32 >= xmm0 as u32 && reg as u32 <= xmm15 as u32 => {
552            Some(reg as u32 - xmm0 as u32)
553        }
554        InlineAsmReg::X86(reg) if reg as u32 >= ymm0 as u32 && reg as u32 <= ymm15 as u32 => {
555            Some(reg as u32 - ymm0 as u32)
556        }
557        InlineAsmReg::X86(reg) if reg as u32 >= zmm0 as u32 && reg as u32 <= zmm31 as u32 => {
558            Some(reg as u32 - zmm0 as u32)
559        }
560        _ => None,
561    }
562}
563
564/// If the register is an AArch64 integer register then return its index.
565fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
566    match reg {
567        InlineAsmReg::AArch64(r) => r.reg_index(),
568        _ => None,
569    }
570}
571
572/// If the register is an AArch64 vector register then return its index.
573fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
574    match reg {
575        InlineAsmReg::AArch64(reg) => reg.vreg_index(),
576        _ => None,
577    }
578}
579
580/// Converts a register class to an LLVM constraint code.
581fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
582    use InlineAsmRegClass::*;
583    match reg {
584        // For vector registers LLVM wants the register name to match the type size.
585        InlineAsmRegOrRegClass::Reg(reg) => {
586            if let Some(idx) = xmm_reg_index(reg) {
587                let class = if let Some(layout) = layout {
588                    match layout.size.bytes() {
589                        64 => 'z',
590                        32 => 'y',
591                        _ => 'x',
592                    }
593                } else {
594                    // We use f32 as the type for discarded outputs
595                    'x'
596                };
597                format!("{{{}mm{}}}", class, idx)
598            } else if let Some(idx) = a64_reg_index(reg) {
599                let class = if let Some(layout) = layout {
600                    match layout.size.bytes() {
601                        8 => 'x',
602                        _ => 'w',
603                    }
604                } else {
605                    // We use i32 as the type for discarded outputs
606                    'w'
607                };
608                if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
609                    // LLVM doesn't recognize x30. use lr instead.
610                    "{lr}".to_string()
611                } else {
612                    format!("{{{}{}}}", class, idx)
613                }
614            } else if let Some(idx) = a64_vreg_index(reg) {
615                let class = if let Some(layout) = layout {
616                    match layout.size.bytes() {
617                        16 => 'q',
618                        8 => 'd',
619                        4 => 's',
620                        2 => 'h',
621                        1 => 'd', // We fixup i8 to i8x8
622                        _ => unreachable!(),
623                    }
624                } else {
625                    // We use i64x2 as the type for discarded outputs
626                    'q'
627                };
628                format!("{{{}{}}}", class, idx)
629            } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
630                // LLVM doesn't recognize r14
631                "{lr}".to_string()
632            } else {
633                format!("{{{}}}", reg.name())
634            }
635        }
636        // The constraints can be retrieved from
637        // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
638        InlineAsmRegOrRegClass::RegClass(reg) => match reg {
639            AArch64(AArch64InlineAsmRegClass::reg) => "r",
640            AArch64(AArch64InlineAsmRegClass::vreg) => "w",
641            AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
642            AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
643            Arm(ArmInlineAsmRegClass::reg) => "r",
644            Arm(ArmInlineAsmRegClass::sreg)
645            | Arm(ArmInlineAsmRegClass::dreg_low16)
646            | Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
647            Arm(ArmInlineAsmRegClass::sreg_low16)
648            | Arm(ArmInlineAsmRegClass::dreg_low8)
649            | Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
650            Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w",
651            Hexagon(HexagonInlineAsmRegClass::reg) => "r",
652            Hexagon(HexagonInlineAsmRegClass::preg) => unreachable!("clobber-only"),
653            LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
654            LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
655            Mips(MipsInlineAsmRegClass::reg) => "r",
656            Mips(MipsInlineAsmRegClass::freg) => "f",
657            Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
658            Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
659            Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
660            PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
661            PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
662            PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
663            PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
664            PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => {
665                unreachable!("clobber-only")
666            }
667            RiscV(RiscVInlineAsmRegClass::reg) => "r",
668            RiscV(RiscVInlineAsmRegClass::freg) => "f",
669            RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
670            X86(X86InlineAsmRegClass::reg) => "r",
671            X86(X86InlineAsmRegClass::reg_abcd) => "Q",
672            X86(X86InlineAsmRegClass::reg_byte) => "q",
673            X86(X86InlineAsmRegClass::xmm_reg) | X86(X86InlineAsmRegClass::ymm_reg) => "x",
674            X86(X86InlineAsmRegClass::zmm_reg) => "v",
675            X86(X86InlineAsmRegClass::kreg) => "^Yk",
676            X86(
677                X86InlineAsmRegClass::x87_reg
678                | X86InlineAsmRegClass::mmx_reg
679                | X86InlineAsmRegClass::kreg0
680                | X86InlineAsmRegClass::tmm_reg,
681            ) => unreachable!("clobber-only"),
682            Wasm(WasmInlineAsmRegClass::local) => "r",
683            Bpf(BpfInlineAsmRegClass::reg) => "r",
684            Bpf(BpfInlineAsmRegClass::wreg) => "w",
685            Avr(AvrInlineAsmRegClass::reg) => "r",
686            Avr(AvrInlineAsmRegClass::reg_upper) => "d",
687            Avr(AvrInlineAsmRegClass::reg_pair) => "r",
688            Avr(AvrInlineAsmRegClass::reg_iw) => "w",
689            Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
690            S390x(S390xInlineAsmRegClass::reg) => "r",
691            S390x(S390xInlineAsmRegClass::reg_addr) => "a",
692            S390x(S390xInlineAsmRegClass::freg) => "f",
693            S390x(S390xInlineAsmRegClass::vreg) => "v",
694            S390x(S390xInlineAsmRegClass::areg) => {
695                unreachable!("clobber-only")
696            }
697            Sparc(SparcInlineAsmRegClass::reg) => "r",
698            Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"),
699            Msp430(Msp430InlineAsmRegClass::reg) => "r",
700            M68k(M68kInlineAsmRegClass::reg) => "r",
701            M68k(M68kInlineAsmRegClass::reg_addr) => "a",
702            M68k(M68kInlineAsmRegClass::reg_data) => "d",
703            CSKY(CSKYInlineAsmRegClass::reg) => "r",
704            CSKY(CSKYInlineAsmRegClass::freg) => "f",
705            SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
706            Err => unreachable!(),
707        }
708        .to_string(),
709    }
710}
711
712/// Converts a modifier into LLVM's equivalent modifier.
713fn modifier_to_llvm(
714    arch: InlineAsmArch,
715    reg: InlineAsmRegClass,
716    modifier: Option<char>,
717) -> Option<char> {
718    use InlineAsmRegClass::*;
719    // The modifiers can be retrieved from
720    // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
721    match reg {
722        AArch64(AArch64InlineAsmRegClass::reg) => modifier,
723        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
724            if modifier == Some('v') {
725                None
726            } else {
727                modifier
728            }
729        }
730        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
731        Arm(ArmInlineAsmRegClass::reg) => None,
732        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
733        Arm(ArmInlineAsmRegClass::dreg)
734        | Arm(ArmInlineAsmRegClass::dreg_low16)
735        | Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
736        Arm(ArmInlineAsmRegClass::qreg)
737        | Arm(ArmInlineAsmRegClass::qreg_low8)
738        | Arm(ArmInlineAsmRegClass::qreg_low4) => {
739            if modifier.is_none() {
740                Some('q')
741            } else {
742                modifier
743            }
744        }
745        Hexagon(_) => None,
746        LoongArch(_) => None,
747        Mips(_) => None,
748        Nvptx(_) => None,
749        PowerPC(_) => None,
750        RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
751        RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
752        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
753            None if arch == InlineAsmArch::X86_64 => Some('q'),
754            None => Some('k'),
755            Some('l') => Some('b'),
756            Some('h') => Some('h'),
757            Some('x') => Some('w'),
758            Some('e') => Some('k'),
759            Some('r') => Some('q'),
760            _ => unreachable!(),
761        },
762        X86(X86InlineAsmRegClass::reg_byte) => None,
763        X86(reg @ X86InlineAsmRegClass::xmm_reg)
764        | X86(reg @ X86InlineAsmRegClass::ymm_reg)
765        | X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
766            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
767            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
768            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
769            (_, Some('x')) => Some('x'),
770            (_, Some('y')) => Some('t'),
771            (_, Some('z')) => Some('g'),
772            _ => unreachable!(),
773        },
774        X86(X86InlineAsmRegClass::kreg) => None,
775        X86(
776            X86InlineAsmRegClass::x87_reg
777            | X86InlineAsmRegClass::mmx_reg
778            | X86InlineAsmRegClass::kreg0
779            | X86InlineAsmRegClass::tmm_reg,
780        ) => unreachable!("clobber-only"),
781        Wasm(WasmInlineAsmRegClass::local) => None,
782        Bpf(_) => None,
783        Avr(AvrInlineAsmRegClass::reg_pair)
784        | Avr(AvrInlineAsmRegClass::reg_iw)
785        | Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
786            Some('h') => Some('B'),
787            Some('l') => Some('A'),
788            _ => None,
789        },
790        Avr(_) => None,
791        S390x(_) => None,
792        Sparc(_) => None,
793        Msp430(_) => None,
794        SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
795        M68k(_) => None,
796        CSKY(_) => None,
797        Err => unreachable!(),
798    }
799}
800
801/// Type to use for outputs that are discarded. It doesn't really matter what
802/// the type is, as long as it is valid for the constraint code.
803fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
804    use InlineAsmRegClass::*;
805    match reg {
806        AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
807        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
808            cx.type_vector(cx.type_i64(), 2)
809        }
810        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
811        Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
812        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
813        Arm(ArmInlineAsmRegClass::dreg)
814        | Arm(ArmInlineAsmRegClass::dreg_low16)
815        | Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
816        Arm(ArmInlineAsmRegClass::qreg)
817        | Arm(ArmInlineAsmRegClass::qreg_low8)
818        | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2),
819        Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
820        Hexagon(HexagonInlineAsmRegClass::preg) => unreachable!("clobber-only"),
821        LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
822        LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
823        Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
824        Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
825        Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
826        Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
827        Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
828        PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
829        PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
830        PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
831        PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
832        PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => {
833            unreachable!("clobber-only")
834        }
835        RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
836        RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
837        RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
838        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
839        X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
840        X86(X86InlineAsmRegClass::xmm_reg)
841        | X86(X86InlineAsmRegClass::ymm_reg)
842        | X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
843        X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
844        X86(
845            X86InlineAsmRegClass::x87_reg
846            | X86InlineAsmRegClass::mmx_reg
847            | X86InlineAsmRegClass::kreg0
848            | X86InlineAsmRegClass::tmm_reg,
849        ) => unreachable!("clobber-only"),
850        Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
851        Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
852        Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
853        Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
854        Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
855        Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
856        Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
857        Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
858        S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
859        S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
860        S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
861        S390x(S390xInlineAsmRegClass::areg) => {
862            unreachable!("clobber-only")
863        }
864        Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),
865        Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"),
866        Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
867        M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
868        M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
869        M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
870        CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
871        CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
872        SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
873        Err => unreachable!(),
874    }
875}
876
877/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
878/// the equivalent integer type.
879fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
880    let dl = &cx.tcx.data_layout;
881    match scalar.primitive() {
882        Primitive::Int(Integer::I8, _) => cx.type_i8(),
883        Primitive::Int(Integer::I16, _) => cx.type_i16(),
884        Primitive::Int(Integer::I32, _) => cx.type_i32(),
885        Primitive::Int(Integer::I64, _) => cx.type_i64(),
886        Primitive::Float(Float::F16) => cx.type_f16(),
887        Primitive::Float(Float::F32) => cx.type_f32(),
888        Primitive::Float(Float::F64) => cx.type_f64(),
889        Primitive::Float(Float::F128) => cx.type_f128(),
890        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
891        Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
892        _ => unreachable!(),
893    }
894}
895
896fn any_target_feature_enabled(
897    cx: &CodegenCx<'_, '_>,
898    instance: Instance<'_>,
899    features: &[Symbol],
900) -> bool {
901    let enabled = cx.tcx.asm_target_features(instance.def_id());
902    features.iter().any(|feat| enabled.contains(feat))
903}
904
905/// Fix up an input value to work around LLVM bugs.
906fn llvm_fixup_input<'ll, 'tcx>(
907    bx: &mut Builder<'_, 'll, 'tcx>,
908    mut value: &'ll Value,
909    reg: InlineAsmRegClass,
910    layout: &TyAndLayout<'tcx>,
911    instance: Instance<'_>,
912) -> &'ll Value {
913    use InlineAsmRegClass::*;
914    let dl = &bx.tcx.data_layout;
915    match (reg, layout.backend_repr) {
916        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
917            if let Primitive::Int(Integer::I8, _) = s.primitive() {
918                let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
919                bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
920            } else {
921                value
922            }
923        }
924        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
925            if s.primitive() != Primitive::Float(Float::F128) =>
926        {
927            let elem_ty = llvm_asm_scalar_type(bx.cx, s);
928            let count = 16 / layout.size.bytes();
929            let vec_ty = bx.cx.type_vector(elem_ty, count);
930            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
931            if let Primitive::Pointer(_) = s.primitive() {
932                let t = bx.type_from_integer(dl.ptr_sized_integer());
933                value = bx.ptrtoint(value, t);
934            }
935            bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
936        }
937        (
938            AArch64(AArch64InlineAsmRegClass::vreg_low16),
939            BackendRepr::SimdVector { element, count },
940        ) if layout.size.bytes() == 8 => {
941            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
942            let vec_ty = bx.cx.type_vector(elem_ty, count);
943            let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
944            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
945        }
946        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
947            if s.primitive() == Primitive::Float(Float::F64) =>
948        {
949            bx.bitcast(value, bx.cx.type_i64())
950        }
951        (
952            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
953            BackendRepr::SimdVector { .. },
954        ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
955        (
956            X86(
957                X86InlineAsmRegClass::xmm_reg
958                | X86InlineAsmRegClass::ymm_reg
959                | X86InlineAsmRegClass::zmm_reg,
960            ),
961            BackendRepr::Scalar(s),
962        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
963            && s.primitive() == Primitive::Float(Float::F128) =>
964        {
965            bx.bitcast(value, bx.type_vector(bx.type_i32(), 4))
966        }
967        (
968            X86(
969                X86InlineAsmRegClass::xmm_reg
970                | X86InlineAsmRegClass::ymm_reg
971                | X86InlineAsmRegClass::zmm_reg,
972            ),
973            BackendRepr::Scalar(s),
974        ) if s.primitive() == Primitive::Float(Float::F16) => {
975            let value = bx.insert_element(
976                bx.const_undef(bx.type_vector(bx.type_f16(), 8)),
977                value,
978                bx.const_usize(0),
979            );
980            bx.bitcast(value, bx.type_vector(bx.type_i16(), 8))
981        }
982        (
983            X86(
984                X86InlineAsmRegClass::xmm_reg
985                | X86InlineAsmRegClass::ymm_reg
986                | X86InlineAsmRegClass::zmm_reg,
987            ),
988            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
989        ) if element.primitive() == Primitive::Float(Float::F16) => {
990            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
991        }
992        (
993            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
994            BackendRepr::Scalar(s),
995        ) => {
996            if let Primitive::Int(Integer::I32, _) = s.primitive() {
997                bx.bitcast(value, bx.cx.type_f32())
998            } else {
999                value
1000            }
1001        }
1002        (
1003            Arm(
1004                ArmInlineAsmRegClass::dreg
1005                | ArmInlineAsmRegClass::dreg_low8
1006                | ArmInlineAsmRegClass::dreg_low16,
1007            ),
1008            BackendRepr::Scalar(s),
1009        ) => {
1010            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1011                bx.bitcast(value, bx.cx.type_f64())
1012            } else {
1013                value
1014            }
1015        }
1016        (
1017            Arm(
1018                ArmInlineAsmRegClass::dreg
1019                | ArmInlineAsmRegClass::dreg_low8
1020                | ArmInlineAsmRegClass::dreg_low16
1021                | ArmInlineAsmRegClass::qreg
1022                | ArmInlineAsmRegClass::qreg_low4
1023                | ArmInlineAsmRegClass::qreg_low8,
1024            ),
1025            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1026        ) if element.primitive() == Primitive::Float(Float::F16) => {
1027            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1028        }
1029        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1030            if s.primitive() == Primitive::Float(Float::F16) =>
1031        {
1032            // Smaller floats are always "NaN-boxed" inside larger floats on LoongArch.
1033            let value = bx.bitcast(value, bx.type_i16());
1034            let value = bx.zext(value, bx.type_i32());
1035            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1036            bx.bitcast(value, bx.type_f32())
1037        }
1038        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1039            match s.primitive() {
1040                // MIPS only supports register-length arithmetics.
1041                Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
1042                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_i32()),
1043                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_i64()),
1044                _ => value,
1045            }
1046        }
1047        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1048            if s.primitive() == Primitive::Float(Float::F16)
1049                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1050        {
1051            // Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.
1052            let value = bx.bitcast(value, bx.type_i16());
1053            let value = bx.zext(value, bx.type_i32());
1054            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1055            bx.bitcast(value, bx.type_f32())
1056        }
1057        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1058            if s.primitive() == Primitive::Float(Float::F32) =>
1059        {
1060            let value = bx.insert_element(
1061                bx.const_undef(bx.type_vector(bx.type_f32(), 4)),
1062                value,
1063                bx.const_usize(0),
1064            );
1065            bx.bitcast(value, bx.type_vector(bx.type_f32(), 4))
1066        }
1067        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1068            if s.primitive() == Primitive::Float(Float::F64) =>
1069        {
1070            let value = bx.insert_element(
1071                bx.const_undef(bx.type_vector(bx.type_f64(), 2)),
1072                value,
1073                bx.const_usize(0),
1074            );
1075            bx.bitcast(value, bx.type_vector(bx.type_f64(), 2))
1076        }
1077        _ => value,
1078    }
1079}
1080
1081/// Fix up an output value to work around LLVM bugs.
1082fn llvm_fixup_output<'ll, 'tcx>(
1083    bx: &mut Builder<'_, 'll, 'tcx>,
1084    mut value: &'ll Value,
1085    reg: InlineAsmRegClass,
1086    layout: &TyAndLayout<'tcx>,
1087    instance: Instance<'_>,
1088) -> &'ll Value {
1089    use InlineAsmRegClass::*;
1090    match (reg, layout.backend_repr) {
1091        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1092            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1093                bx.extract_element(value, bx.const_i32(0))
1094            } else {
1095                value
1096            }
1097        }
1098        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1099            if s.primitive() != Primitive::Float(Float::F128) =>
1100        {
1101            value = bx.extract_element(value, bx.const_i32(0));
1102            if let Primitive::Pointer(_) = s.primitive() {
1103                value = bx.inttoptr(value, layout.llvm_type(bx.cx));
1104            }
1105            value
1106        }
1107        (
1108            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1109            BackendRepr::SimdVector { element, count },
1110        ) if layout.size.bytes() == 8 => {
1111            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1112            let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
1113            let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
1114            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1115        }
1116        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1117            if s.primitive() == Primitive::Float(Float::F64) =>
1118        {
1119            bx.bitcast(value, bx.cx.type_f64())
1120        }
1121        (
1122            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1123            BackendRepr::SimdVector { .. },
1124        ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
1125        (
1126            X86(
1127                X86InlineAsmRegClass::xmm_reg
1128                | X86InlineAsmRegClass::ymm_reg
1129                | X86InlineAsmRegClass::zmm_reg,
1130            ),
1131            BackendRepr::Scalar(s),
1132        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1133            && s.primitive() == Primitive::Float(Float::F128) =>
1134        {
1135            bx.bitcast(value, bx.type_f128())
1136        }
1137        (
1138            X86(
1139                X86InlineAsmRegClass::xmm_reg
1140                | X86InlineAsmRegClass::ymm_reg
1141                | X86InlineAsmRegClass::zmm_reg,
1142            ),
1143            BackendRepr::Scalar(s),
1144        ) if s.primitive() == Primitive::Float(Float::F16) => {
1145            let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8));
1146            bx.extract_element(value, bx.const_usize(0))
1147        }
1148        (
1149            X86(
1150                X86InlineAsmRegClass::xmm_reg
1151                | X86InlineAsmRegClass::ymm_reg
1152                | X86InlineAsmRegClass::zmm_reg,
1153            ),
1154            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1155        ) if element.primitive() == Primitive::Float(Float::F16) => {
1156            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1157        }
1158        (
1159            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1160            BackendRepr::Scalar(s),
1161        ) => {
1162            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1163                bx.bitcast(value, bx.cx.type_i32())
1164            } else {
1165                value
1166            }
1167        }
1168        (
1169            Arm(
1170                ArmInlineAsmRegClass::dreg
1171                | ArmInlineAsmRegClass::dreg_low8
1172                | ArmInlineAsmRegClass::dreg_low16,
1173            ),
1174            BackendRepr::Scalar(s),
1175        ) => {
1176            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1177                bx.bitcast(value, bx.cx.type_i64())
1178            } else {
1179                value
1180            }
1181        }
1182        (
1183            Arm(
1184                ArmInlineAsmRegClass::dreg
1185                | ArmInlineAsmRegClass::dreg_low8
1186                | ArmInlineAsmRegClass::dreg_low16
1187                | ArmInlineAsmRegClass::qreg
1188                | ArmInlineAsmRegClass::qreg_low4
1189                | ArmInlineAsmRegClass::qreg_low8,
1190            ),
1191            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1192        ) if element.primitive() == Primitive::Float(Float::F16) => {
1193            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1194        }
1195        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1196            if s.primitive() == Primitive::Float(Float::F16) =>
1197        {
1198            let value = bx.bitcast(value, bx.type_i32());
1199            let value = bx.trunc(value, bx.type_i16());
1200            bx.bitcast(value, bx.type_f16())
1201        }
1202        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1203            match s.primitive() {
1204                // MIPS only supports register-length arithmetics.
1205                Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
1206                Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
1207                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_f32()),
1208                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_f64()),
1209                _ => value,
1210            }
1211        }
1212        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1213            if s.primitive() == Primitive::Float(Float::F16)
1214                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1215        {
1216            let value = bx.bitcast(value, bx.type_i32());
1217            let value = bx.trunc(value, bx.type_i16());
1218            bx.bitcast(value, bx.type_f16())
1219        }
1220        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1221            if s.primitive() == Primitive::Float(Float::F32) =>
1222        {
1223            let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4));
1224            bx.extract_element(value, bx.const_usize(0))
1225        }
1226        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1227            if s.primitive() == Primitive::Float(Float::F64) =>
1228        {
1229            let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2));
1230            bx.extract_element(value, bx.const_usize(0))
1231        }
1232        _ => value,
1233    }
1234}
1235
1236/// Output type to use for llvm_fixup_output.
1237fn llvm_fixup_output_type<'ll, 'tcx>(
1238    cx: &CodegenCx<'ll, 'tcx>,
1239    reg: InlineAsmRegClass,
1240    layout: &TyAndLayout<'tcx>,
1241    instance: Instance<'_>,
1242) -> &'ll Type {
1243    use InlineAsmRegClass::*;
1244    match (reg, layout.backend_repr) {
1245        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1246            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1247                cx.type_vector(cx.type_i8(), 8)
1248            } else {
1249                layout.llvm_type(cx)
1250            }
1251        }
1252        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1253            if s.primitive() != Primitive::Float(Float::F128) =>
1254        {
1255            let elem_ty = llvm_asm_scalar_type(cx, s);
1256            let count = 16 / layout.size.bytes();
1257            cx.type_vector(elem_ty, count)
1258        }
1259        (
1260            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1261            BackendRepr::SimdVector { element, count },
1262        ) if layout.size.bytes() == 8 => {
1263            let elem_ty = llvm_asm_scalar_type(cx, element);
1264            cx.type_vector(elem_ty, count * 2)
1265        }
1266        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1267            if s.primitive() == Primitive::Float(Float::F64) =>
1268        {
1269            cx.type_i64()
1270        }
1271        (
1272            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1273            BackendRepr::SimdVector { .. },
1274        ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
1275        (
1276            X86(
1277                X86InlineAsmRegClass::xmm_reg
1278                | X86InlineAsmRegClass::ymm_reg
1279                | X86InlineAsmRegClass::zmm_reg,
1280            ),
1281            BackendRepr::Scalar(s),
1282        ) if cx.sess().asm_arch == Some(InlineAsmArch::X86)
1283            && s.primitive() == Primitive::Float(Float::F128) =>
1284        {
1285            cx.type_vector(cx.type_i32(), 4)
1286        }
1287        (
1288            X86(
1289                X86InlineAsmRegClass::xmm_reg
1290                | X86InlineAsmRegClass::ymm_reg
1291                | X86InlineAsmRegClass::zmm_reg,
1292            ),
1293            BackendRepr::Scalar(s),
1294        ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8),
1295        (
1296            X86(
1297                X86InlineAsmRegClass::xmm_reg
1298                | X86InlineAsmRegClass::ymm_reg
1299                | X86InlineAsmRegClass::zmm_reg,
1300            ),
1301            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1302        ) if element.primitive() == Primitive::Float(Float::F16) => {
1303            cx.type_vector(cx.type_i16(), count)
1304        }
1305        (
1306            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1307            BackendRepr::Scalar(s),
1308        ) => {
1309            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1310                cx.type_f32()
1311            } else {
1312                layout.llvm_type(cx)
1313            }
1314        }
1315        (
1316            Arm(
1317                ArmInlineAsmRegClass::dreg
1318                | ArmInlineAsmRegClass::dreg_low8
1319                | ArmInlineAsmRegClass::dreg_low16,
1320            ),
1321            BackendRepr::Scalar(s),
1322        ) => {
1323            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1324                cx.type_f64()
1325            } else {
1326                layout.llvm_type(cx)
1327            }
1328        }
1329        (
1330            Arm(
1331                ArmInlineAsmRegClass::dreg
1332                | ArmInlineAsmRegClass::dreg_low8
1333                | ArmInlineAsmRegClass::dreg_low16
1334                | ArmInlineAsmRegClass::qreg
1335                | ArmInlineAsmRegClass::qreg_low4
1336                | ArmInlineAsmRegClass::qreg_low8,
1337            ),
1338            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1339        ) if element.primitive() == Primitive::Float(Float::F16) => {
1340            cx.type_vector(cx.type_i16(), count)
1341        }
1342        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1343            if s.primitive() == Primitive::Float(Float::F16) =>
1344        {
1345            cx.type_f32()
1346        }
1347        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1348            match s.primitive() {
1349                // MIPS only supports register-length arithmetics.
1350                Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1351                Primitive::Float(Float::F32) => cx.type_i32(),
1352                Primitive::Float(Float::F64) => cx.type_i64(),
1353                _ => layout.llvm_type(cx),
1354            }
1355        }
1356        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1357            if s.primitive() == Primitive::Float(Float::F16)
1358                && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
1359        {
1360            cx.type_f32()
1361        }
1362        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1363            if s.primitive() == Primitive::Float(Float::F32) =>
1364        {
1365            cx.type_vector(cx.type_f32(), 4)
1366        }
1367        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1368            if s.primitive() == Primitive::Float(Float::F64) =>
1369        {
1370            cx.type_vector(cx.type_f64(), 2)
1371        }
1372        _ => layout.llvm_type(cx),
1373    }
1374}
rustc_codegen_llvm/asm.rs

rustc_codegen_llvm/
asm.rs
