rustc_mir_transform/
lower_intrinsics.rs

1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14        let local_decls = &body.local_decls;
15        for block in body.basic_blocks.as_mut() {
16            let terminator = block.terminator.as_mut().unwrap();
17            if let TerminatorKind::Call { func, args, destination, target, .. } =
18                &mut terminator.kind
19                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20                && let Some(intrinsic) = tcx.intrinsic(def_id)
21            {
22                match intrinsic.name {
23                    sym::unreachable => {
24                        terminator.kind = TerminatorKind::Unreachable;
25                    }
26                    sym::ub_checks => {
27                        let target = target.unwrap();
28                        block.statements.push(Statement::new(
29                            terminator.source_info,
30                            StatementKind::Assign(Box::new((
31                                *destination,
32                                Rvalue::NullaryOp(NullOp::UbChecks, tcx.types.bool),
33                            ))),
34                        ));
35                        terminator.kind = TerminatorKind::Goto { target };
36                    }
37                    sym::contract_checks => {
38                        let target = target.unwrap();
39                        block.statements.push(Statement::new(
40                            terminator.source_info,
41                            StatementKind::Assign(Box::new((
42                                *destination,
43                                Rvalue::NullaryOp(NullOp::ContractChecks, tcx.types.bool),
44                            ))),
45                        ));
46                        terminator.kind = TerminatorKind::Goto { target };
47                    }
48                    sym::forget => {
49                        let target = target.unwrap();
50                        block.statements.push(Statement::new(
51                            terminator.source_info,
52                            StatementKind::Assign(Box::new((
53                                *destination,
54                                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
55                                    span: terminator.source_info.span,
56                                    user_ty: None,
57                                    const_: Const::zero_sized(tcx.types.unit),
58                                }))),
59                            ))),
60                        ));
61                        terminator.kind = TerminatorKind::Goto { target };
62                    }
63                    sym::copy_nonoverlapping => {
64                        let target = target.unwrap();
65                        let Ok([src, dst, count]) = take_array(args) else {
66                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
67                        };
68                        block.statements.push(Statement::new(
69                            terminator.source_info,
70                            StatementKind::Intrinsic(Box::new(
71                                NonDivergingIntrinsic::CopyNonOverlapping(
72                                    rustc_middle::mir::CopyNonOverlapping {
73                                        src: src.node,
74                                        dst: dst.node,
75                                        count: count.node,
76                                    },
77                                ),
78                            )),
79                        ));
80                        terminator.kind = TerminatorKind::Goto { target };
81                    }
82                    sym::assume => {
83                        let target = target.unwrap();
84                        let Ok([arg]) = take_array(args) else {
85                            bug!("Wrong arguments for assume intrinsic");
86                        };
87                        block.statements.push(Statement::new(
88                            terminator.source_info,
89                            StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
90                                arg.node,
91                            ))),
92                        ));
93                        terminator.kind = TerminatorKind::Goto { target };
94                    }
95                    sym::wrapping_add
96                    | sym::wrapping_sub
97                    | sym::wrapping_mul
98                    | sym::three_way_compare
99                    | sym::unchecked_add
100                    | sym::unchecked_sub
101                    | sym::unchecked_mul
102                    | sym::unchecked_div
103                    | sym::unchecked_rem
104                    | sym::unchecked_shl
105                    | sym::unchecked_shr => {
106                        let target = target.unwrap();
107                        let Ok([lhs, rhs]) = take_array(args) else {
108                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
109                        };
110                        let bin_op = match intrinsic.name {
111                            sym::wrapping_add => BinOp::Add,
112                            sym::wrapping_sub => BinOp::Sub,
113                            sym::wrapping_mul => BinOp::Mul,
114                            sym::three_way_compare => BinOp::Cmp,
115                            sym::unchecked_add => BinOp::AddUnchecked,
116                            sym::unchecked_sub => BinOp::SubUnchecked,
117                            sym::unchecked_mul => BinOp::MulUnchecked,
118                            sym::unchecked_div => BinOp::Div,
119                            sym::unchecked_rem => BinOp::Rem,
120                            sym::unchecked_shl => BinOp::ShlUnchecked,
121                            sym::unchecked_shr => BinOp::ShrUnchecked,
122                            _ => bug!("unexpected intrinsic"),
123                        };
124                        block.statements.push(Statement::new(
125                            terminator.source_info,
126                            StatementKind::Assign(Box::new((
127                                *destination,
128                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
129                            ))),
130                        ));
131                        terminator.kind = TerminatorKind::Goto { target };
132                    }
133                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
134                        let target = target.unwrap();
135                        let Ok([lhs, rhs]) = take_array(args) else {
136                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
137                        };
138                        let bin_op = match intrinsic.name {
139                            sym::add_with_overflow => BinOp::AddWithOverflow,
140                            sym::sub_with_overflow => BinOp::SubWithOverflow,
141                            sym::mul_with_overflow => BinOp::MulWithOverflow,
142                            _ => bug!("unexpected intrinsic"),
143                        };
144                        block.statements.push(Statement::new(
145                            terminator.source_info,
146                            StatementKind::Assign(Box::new((
147                                *destination,
148                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
149                            ))),
150                        ));
151                        terminator.kind = TerminatorKind::Goto { target };
152                    }
153                    sym::size_of | sym::align_of => {
154                        let target = target.unwrap();
155                        let tp_ty = generic_args.type_at(0);
156                        let null_op = match intrinsic.name {
157                            sym::size_of => NullOp::SizeOf,
158                            sym::align_of => NullOp::AlignOf,
159                            _ => bug!("unexpected intrinsic"),
160                        };
161                        block.statements.push(Statement::new(
162                            terminator.source_info,
163                            StatementKind::Assign(Box::new((
164                                *destination,
165                                Rvalue::NullaryOp(null_op, tp_ty),
166                            ))),
167                        ));
168                        terminator.kind = TerminatorKind::Goto { target };
169                    }
170                    sym::read_via_copy => {
171                        let Ok([arg]) = take_array(args) else {
172                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
173                        };
174                        let derefed_place = if let Some(place) = arg.node.place()
175                            && let Some(local) = place.as_local()
176                        {
177                            tcx.mk_place_deref(local.into())
178                        } else {
179                            span_bug!(
180                                terminator.source_info.span,
181                                "Only passing a local is supported"
182                            );
183                        };
184                        // Add new statement at the end of the block that does the read, and patch
185                        // up the terminator.
186                        block.statements.push(Statement::new(
187                            terminator.source_info,
188                            StatementKind::Assign(Box::new((
189                                *destination,
190                                Rvalue::Use(Operand::Copy(derefed_place)),
191                            ))),
192                        ));
193                        terminator.kind = match *target {
194                            None => {
195                                // No target means this read something uninhabited,
196                                // so it must be unreachable.
197                                TerminatorKind::Unreachable
198                            }
199                            Some(target) => TerminatorKind::Goto { target },
200                        }
201                    }
202                    sym::write_via_move => {
203                        let target = target.unwrap();
204                        let Ok([ptr, val]) = take_array(args) else {
205                            span_bug!(
206                                terminator.source_info.span,
207                                "Wrong number of arguments for write_via_move intrinsic",
208                            );
209                        };
210                        let derefed_place = if let Some(place) = ptr.node.place()
211                            && let Some(local) = place.as_local()
212                        {
213                            tcx.mk_place_deref(local.into())
214                        } else {
215                            span_bug!(
216                                terminator.source_info.span,
217                                "Only passing a local is supported"
218                            );
219                        };
220                        block.statements.push(Statement::new(
221                            terminator.source_info,
222                            StatementKind::Assign(Box::new((derefed_place, Rvalue::Use(val.node)))),
223                        ));
224                        terminator.kind = TerminatorKind::Goto { target };
225                    }
226                    sym::discriminant_value => {
227                        let target = target.unwrap();
228                        let Ok([arg]) = take_array(args) else {
229                            span_bug!(
230                                terminator.source_info.span,
231                                "Wrong arguments for discriminant_value intrinsic"
232                            );
233                        };
234                        let arg = arg.node.place().unwrap();
235                        let arg = tcx.mk_place_deref(arg);
236                        block.statements.push(Statement::new(
237                            terminator.source_info,
238                            StatementKind::Assign(Box::new((
239                                *destination,
240                                Rvalue::Discriminant(arg),
241                            ))),
242                        ));
243                        terminator.kind = TerminatorKind::Goto { target };
244                    }
245                    sym::offset => {
246                        let target = target.unwrap();
247                        let Ok([ptr, delta]) = take_array(args) else {
248                            span_bug!(
249                                terminator.source_info.span,
250                                "Wrong number of arguments for offset intrinsic",
251                            );
252                        };
253                        block.statements.push(Statement::new(
254                            terminator.source_info,
255                            StatementKind::Assign(Box::new((
256                                *destination,
257                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
258                            ))),
259                        ));
260                        terminator.kind = TerminatorKind::Goto { target };
261                    }
262                    sym::slice_get_unchecked => {
263                        let target = target.unwrap();
264                        let Ok([ptrish, index]) = take_array(args) else {
265                            span_bug!(
266                                terminator.source_info.span,
267                                "Wrong number of arguments for {intrinsic:?}",
268                            );
269                        };
270
271                        let place = ptrish.node.place().unwrap();
272                        assert!(!place.is_indirect());
273                        let updated_place = place.project_deeper(
274                            &[
275                                ProjectionElem::Deref,
276                                ProjectionElem::Index(
277                                    index.node.place().unwrap().as_local().unwrap(),
278                                ),
279                            ],
280                            tcx,
281                        );
282
283                        let ret_ty = generic_args.type_at(0);
284                        let rvalue = match *ret_ty.kind() {
285                            ty::RawPtr(_, Mutability::Not) => {
286                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
287                            }
288                            ty::RawPtr(_, Mutability::Mut) => {
289                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
290                            }
291                            ty::Ref(region, _, Mutability::Not) => {
292                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
293                            }
294                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
295                                region,
296                                BorrowKind::Mut { kind: MutBorrowKind::Default },
297                                updated_place,
298                            ),
299                            _ => bug!("Unknown return type {ret_ty:?}"),
300                        };
301
302                        block.statements.push(Statement::new(
303                            terminator.source_info,
304                            StatementKind::Assign(Box::new((*destination, rvalue))),
305                        ));
306                        terminator.kind = TerminatorKind::Goto { target };
307                    }
308                    sym::transmute | sym::transmute_unchecked => {
309                        let dst_ty = destination.ty(local_decls, tcx).ty;
310                        let Ok([arg]) = take_array(args) else {
311                            span_bug!(
312                                terminator.source_info.span,
313                                "Wrong number of arguments for transmute intrinsic",
314                            );
315                        };
316
317                        // Always emit the cast, even if we transmute to an uninhabited type,
318                        // because that lets CTFE and codegen generate better error messages
319                        // when such a transmute actually ends up reachable.
320                        block.statements.push(Statement::new(
321                            terminator.source_info,
322                            StatementKind::Assign(Box::new((
323                                *destination,
324                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
325                            ))),
326                        ));
327                        if let Some(target) = *target {
328                            terminator.kind = TerminatorKind::Goto { target };
329                        } else {
330                            terminator.kind = TerminatorKind::Unreachable;
331                        }
332                    }
333                    sym::aggregate_raw_ptr => {
334                        let Ok([data, meta]) = take_array(args) else {
335                            span_bug!(
336                                terminator.source_info.span,
337                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
338                            );
339                        };
340                        let target = target.unwrap();
341                        let pointer_ty = generic_args.type_at(0);
342                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
343                            AggregateKind::RawPtr(*pointee_ty, *mutability)
344                        } else {
345                            span_bug!(
346                                terminator.source_info.span,
347                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
348                            );
349                        };
350                        let fields = [data.node, meta.node];
351                        block.statements.push(Statement::new(
352                            terminator.source_info,
353                            StatementKind::Assign(Box::new((
354                                *destination,
355                                Rvalue::Aggregate(Box::new(kind), fields.into()),
356                            ))),
357                        ));
358                        terminator.kind = TerminatorKind::Goto { target };
359                    }
360                    sym::ptr_metadata => {
361                        let Ok([ptr]) = take_array(args) else {
362                            span_bug!(
363                                terminator.source_info.span,
364                                "Wrong number of arguments for ptr_metadata intrinsic",
365                            );
366                        };
367                        let target = target.unwrap();
368                        block.statements.push(Statement::new(
369                            terminator.source_info,
370                            StatementKind::Assign(Box::new((
371                                *destination,
372                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
373                            ))),
374                        ));
375                        terminator.kind = TerminatorKind::Goto { target };
376                    }
377                    _ => {}
378                }
379            }
380        }
381    }
382
383    fn is_required(&self) -> bool {
384        true
385    }
386}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
