miri/intrinsics/
mod.rs

1#![warn(clippy::arithmetic_side_effects)]
2
3mod atomic;
4mod simd;
5
6use rand::Rng;
7use rustc_abi::Size;
8use rustc_apfloat::{Float, Round};
9use rustc_middle::mir;
10use rustc_middle::ty::{self, FloatTy};
11use rustc_span::{Symbol, sym};
12
13use self::atomic::EvalContextExt as _;
14use self::helpers::{ToHost, ToSoft, check_intrinsic_arg_count};
15use self::simd::EvalContextExt as _;
16use crate::math::apply_random_float_error_to_imm;
17use crate::*;
18
19impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
20pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
21    fn call_intrinsic(
22        &mut self,
23        instance: ty::Instance<'tcx>,
24        args: &[OpTy<'tcx>],
25        dest: &PlaceTy<'tcx>,
26        ret: Option<mir::BasicBlock>,
27        unwind: mir::UnwindAction,
28    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
29        let this = self.eval_context_mut();
30
31        // Force use of fallback body, if available.
32        if this.machine.force_intrinsic_fallback
33            && !this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden
34        {
35            return interp_ok(Some(ty::Instance {
36                def: ty::InstanceKind::Item(instance.def_id()),
37                args: instance.args,
38            }));
39        }
40
41        // See if the core engine can handle this intrinsic.
42        if this.eval_intrinsic(instance, args, dest, ret)? {
43            return interp_ok(None);
44        }
45        let intrinsic_name = this.tcx.item_name(instance.def_id());
46        let intrinsic_name = intrinsic_name.as_str();
47
48        // FIXME: avoid allocating memory
49        let dest = this.force_allocation(dest)?;
50
51        match this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, &dest, ret)? {
52            EmulateItemResult::NotSupported => {
53                // We haven't handled the intrinsic, let's see if we can use a fallback body.
54                if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
55                    throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
56                }
57                let intrinsic_fallback_is_spec = Symbol::intern("intrinsic_fallback_is_spec");
58                if this
59                    .tcx
60                    .get_attrs_by_path(instance.def_id(), &[sym::miri, intrinsic_fallback_is_spec])
61                    .next()
62                    .is_none()
63                {
64                    throw_unsup_format!(
65                        "Miri can only use intrinsic fallback bodies that exactly reflect the specification: they fully check for UB and are as non-deterministic as possible. After verifying that `{intrinsic_name}` does so, add the `#[miri::intrinsic_fallback_is_spec]` attribute to it; also ping @rust-lang/miri when you do that"
66                    );
67                }
68                interp_ok(Some(ty::Instance {
69                    def: ty::InstanceKind::Item(instance.def_id()),
70                    args: instance.args,
71                }))
72            }
73            EmulateItemResult::NeedsReturn => {
74                trace!("{:?}", this.dump_place(&dest.clone().into()));
75                this.return_to_block(ret)?;
76                interp_ok(None)
77            }
78            EmulateItemResult::NeedsUnwind => {
79                // Jump to the unwind block to begin unwinding.
80                this.unwind_to_block(unwind)?;
81                interp_ok(None)
82            }
83            EmulateItemResult::AlreadyJumped => interp_ok(None),
84        }
85    }
86
87    /// Emulates a Miri-supported intrinsic (not supported by the core engine).
88    /// Returns `Ok(true)` if the intrinsic was handled.
89    fn emulate_intrinsic_by_name(
90        &mut self,
91        intrinsic_name: &str,
92        generic_args: ty::GenericArgsRef<'tcx>,
93        args: &[OpTy<'tcx>],
94        dest: &MPlaceTy<'tcx>,
95        ret: Option<mir::BasicBlock>,
96    ) -> InterpResult<'tcx, EmulateItemResult> {
97        let this = self.eval_context_mut();
98
99        if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
100            return this.emulate_atomic_intrinsic(name, generic_args, args, dest);
101        }
102        if let Some(name) = intrinsic_name.strip_prefix("simd_") {
103            return this.emulate_simd_intrinsic(name, generic_args, args, dest);
104        }
105
106        match intrinsic_name {
107            // Basic control flow
108            "abort" => {
109                throw_machine_stop!(TerminationInfo::Abort(
110                    "the program aborted execution".to_owned()
111                ));
112            }
113            "catch_unwind" => {
114                this.handle_catch_unwind(args, dest, ret)?;
115                // This pushed a stack frame, don't jump to `ret`.
116                return interp_ok(EmulateItemResult::AlreadyJumped);
117            }
118
119            // Raw memory accesses
120            "volatile_load" => {
121                let [place] = check_intrinsic_arg_count(args)?;
122                let place = this.deref_pointer(place)?;
123                this.copy_op(&place, dest)?;
124            }
125            "volatile_store" => {
126                let [place, dest] = check_intrinsic_arg_count(args)?;
127                let place = this.deref_pointer(place)?;
128                this.copy_op(dest, &place)?;
129            }
130
131            "volatile_set_memory" => {
132                let [ptr, val_byte, count] = check_intrinsic_arg_count(args)?;
133                this.write_bytes_intrinsic(ptr, val_byte, count, "volatile_set_memory")?;
134            }
135
136            // Memory model / provenance manipulation
137            "ptr_mask" => {
138                let [ptr, mask] = check_intrinsic_arg_count(args)?;
139
140                let ptr = this.read_pointer(ptr)?;
141                let mask = this.read_target_usize(mask)?;
142
143                let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask);
144
145                this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
146            }
147
148            // We want to return either `true` or `false` at random, or else something like
149            // ```
150            // if !is_val_statically_known(0) { unreachable_unchecked(); }
151            // ```
152            // Would not be considered UB, or the other way around (`is_val_statically_known(0)`).
153            "is_val_statically_known" => {
154                let [_arg] = check_intrinsic_arg_count(args)?;
155                // FIXME: should we check for validity here? It's tricky because we do not have a
156                // place. Codegen does not seem to set any attributes like `noundef` for intrinsic
157                // calls, so we don't *have* to do anything.
158                let branch: bool = this.machine.rng.get_mut().random();
159                this.write_scalar(Scalar::from_bool(branch), dest)?;
160            }
161
162            "sqrtf32" => {
163                let [f] = check_intrinsic_arg_count(args)?;
164                let f = this.read_scalar(f)?.to_f32()?;
165                // Sqrt is specified to be fully precise.
166                let res = math::sqrt(f);
167                let res = this.adjust_nan(res, &[f]);
168                this.write_scalar(res, dest)?;
169            }
170            "sqrtf64" => {
171                let [f] = check_intrinsic_arg_count(args)?;
172                let f = this.read_scalar(f)?.to_f64()?;
173                // Sqrt is specified to be fully precise.
174                let res = math::sqrt(f);
175                let res = this.adjust_nan(res, &[f]);
176                this.write_scalar(res, dest)?;
177            }
178
179            #[rustfmt::skip]
180            | "sinf32"
181            | "cosf32"
182            | "expf32"
183            | "exp2f32"
184            | "logf32"
185            | "log10f32"
186            | "log2f32"
187            => {
188                let [f] = check_intrinsic_arg_count(args)?;
189                let f = this.read_scalar(f)?.to_f32()?;
190                // Using host floats (but it's fine, these operations do not have
191                // guaranteed precision).
192                let host = f.to_host();
193                let res = match intrinsic_name {
194                    "sinf32" => host.sin(),
195                    "cosf32" => host.cos(),
196                    "expf32" => host.exp(),
197                    "exp2f32" => host.exp2(),
198                    "logf32" => host.ln(),
199                    "log10f32" => host.log10(),
200                    "log2f32" => host.log2(),
201                    _ => bug!(),
202                };
203                let res = res.to_soft();
204                // Apply a relative error of 16ULP to introduce some non-determinism
205                // simulating imprecise implementations and optimizations.
206                // FIXME: temporarily disabled as it breaks std tests.
207                // let res = apply_random_float_error_ulp(
208                //     this,
209                //     res,
210                //     4, // log2(16)
211                // );
212                let res = this.adjust_nan(res, &[f]);
213                this.write_scalar(res, dest)?;
214            }
215            #[rustfmt::skip]
216            | "sinf64"
217            | "cosf64"
218            | "expf64"
219            | "exp2f64"
220            | "logf64"
221            | "log10f64"
222            | "log2f64"
223            => {
224                let [f] = check_intrinsic_arg_count(args)?;
225                let f = this.read_scalar(f)?.to_f64()?;
226                // Using host floats (but it's fine, these operations do not have
227                // guaranteed precision).
228                let host = f.to_host();
229                let res = match intrinsic_name {
230                    "sinf64" => host.sin(),
231                    "cosf64" => host.cos(),
232                    "expf64" => host.exp(),
233                    "exp2f64" => host.exp2(),
234                    "logf64" => host.ln(),
235                    "log10f64" => host.log10(),
236                    "log2f64" => host.log2(),
237                    _ => bug!(),
238                };
239                let res = res.to_soft();
240                // Apply a relative error of 16ULP to introduce some non-determinism
241                // simulating imprecise implementations and optimizations.
242                // FIXME: temporarily disabled as it breaks std tests.
243                // let res = apply_random_float_error_ulp(
244                //     this,
245                //     res,
246                //     4, // log2(16)
247                // );
248                let res = this.adjust_nan(res, &[f]);
249                this.write_scalar(res, dest)?;
250            }
251
252            "fmaf32" => {
253                let [a, b, c] = check_intrinsic_arg_count(args)?;
254                let a = this.read_scalar(a)?.to_f32()?;
255                let b = this.read_scalar(b)?.to_f32()?;
256                let c = this.read_scalar(c)?.to_f32()?;
257                // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11
258                let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft();
259                let res = this.adjust_nan(res, &[a, b, c]);
260                this.write_scalar(res, dest)?;
261            }
262            "fmaf64" => {
263                let [a, b, c] = check_intrinsic_arg_count(args)?;
264                let a = this.read_scalar(a)?.to_f64()?;
265                let b = this.read_scalar(b)?.to_f64()?;
266                let c = this.read_scalar(c)?.to_f64()?;
267                // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11
268                let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft();
269                let res = this.adjust_nan(res, &[a, b, c]);
270                this.write_scalar(res, dest)?;
271            }
272
273            "fmuladdf32" => {
274                let [a, b, c] = check_intrinsic_arg_count(args)?;
275                let a = this.read_scalar(a)?.to_f32()?;
276                let b = this.read_scalar(b)?.to_f32()?;
277                let c = this.read_scalar(c)?.to_f32()?;
278                let fuse: bool = this.machine.rng.get_mut().random();
279                let res = if fuse {
280                    // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11
281                    a.to_host().mul_add(b.to_host(), c.to_host()).to_soft()
282                } else {
283                    ((a * b).value + c).value
284                };
285                let res = this.adjust_nan(res, &[a, b, c]);
286                this.write_scalar(res, dest)?;
287            }
288            "fmuladdf64" => {
289                let [a, b, c] = check_intrinsic_arg_count(args)?;
290                let a = this.read_scalar(a)?.to_f64()?;
291                let b = this.read_scalar(b)?.to_f64()?;
292                let c = this.read_scalar(c)?.to_f64()?;
293                let fuse: bool = this.machine.rng.get_mut().random();
294                let res = if fuse {
295                    // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11
296                    a.to_host().mul_add(b.to_host(), c.to_host()).to_soft()
297                } else {
298                    ((a * b).value + c).value
299                };
300                let res = this.adjust_nan(res, &[a, b, c]);
301                this.write_scalar(res, dest)?;
302            }
303
304            "powf32" => {
305                // FIXME: apply random relative error but without altering behaviour of powf
306                let [f1, f2] = check_intrinsic_arg_count(args)?;
307                let f1 = this.read_scalar(f1)?.to_f32()?;
308                let f2 = this.read_scalar(f2)?.to_f32()?;
309                // Using host floats (but it's fine, this operation does not have guaranteed precision).
310                let res = f1.to_host().powf(f2.to_host()).to_soft();
311                let res = this.adjust_nan(res, &[f1, f2]);
312                this.write_scalar(res, dest)?;
313            }
314            "powf64" => {
315                // FIXME: apply random relative error but without altering behaviour of powf
316                let [f1, f2] = check_intrinsic_arg_count(args)?;
317                let f1 = this.read_scalar(f1)?.to_f64()?;
318                let f2 = this.read_scalar(f2)?.to_f64()?;
319                // Using host floats (but it's fine, this operation does not have guaranteed precision).
320                let res = f1.to_host().powf(f2.to_host()).to_soft();
321                let res = this.adjust_nan(res, &[f1, f2]);
322                this.write_scalar(res, dest)?;
323            }
324
325            "powif32" => {
326                // FIXME: apply random relative error but without altering behaviour of powi
327                let [f, i] = check_intrinsic_arg_count(args)?;
328                let f = this.read_scalar(f)?.to_f32()?;
329                let i = this.read_scalar(i)?.to_i32()?;
330                // Using host floats (but it's fine, this operation does not have guaranteed precision).
331                let res = f.to_host().powi(i).to_soft();
332                let res = this.adjust_nan(res, &[f]);
333                this.write_scalar(res, dest)?;
334            }
335            "powif64" => {
336                // FIXME: apply random relative error but without altering behaviour of powi
337                let [f, i] = check_intrinsic_arg_count(args)?;
338                let f = this.read_scalar(f)?.to_f64()?;
339                let i = this.read_scalar(i)?.to_i32()?;
340                // Using host floats (but it's fine, this operation does not have guaranteed precision).
341                let res = f.to_host().powi(i).to_soft();
342                let res = this.adjust_nan(res, &[f]);
343                this.write_scalar(res, dest)?;
344            }
345
346            #[rustfmt::skip]
347            | "fadd_fast"
348            | "fsub_fast"
349            | "fmul_fast"
350            | "fdiv_fast"
351            | "frem_fast"
352            => {
353                let [a, b] = check_intrinsic_arg_count(args)?;
354                let a = this.read_immediate(a)?;
355                let b = this.read_immediate(b)?;
356                let op = match intrinsic_name {
357                    "fadd_fast" => mir::BinOp::Add,
358                    "fsub_fast" => mir::BinOp::Sub,
359                    "fmul_fast" => mir::BinOp::Mul,
360                    "fdiv_fast" => mir::BinOp::Div,
361                    "frem_fast" => mir::BinOp::Rem,
362                    _ => bug!(),
363                };
364                let float_finite = |x: &ImmTy<'tcx>| -> InterpResult<'tcx, bool> {
365                    let ty::Float(fty) = x.layout.ty.kind() else {
366                        bug!("float_finite: non-float input type {}", x.layout.ty)
367                    };
368                    interp_ok(match fty {
369                        FloatTy::F16 => x.to_scalar().to_f16()?.is_finite(),
370                        FloatTy::F32 => x.to_scalar().to_f32()?.is_finite(),
371                        FloatTy::F64 => x.to_scalar().to_f64()?.is_finite(),
372                        FloatTy::F128 => x.to_scalar().to_f128()?.is_finite(),
373                    })
374                };
375                match (float_finite(&a)?, float_finite(&b)?) {
376                    (false, false) => throw_ub_format!(
377                        "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
378                    ),
379                    (false, _) => throw_ub_format!(
380                        "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
381                    ),
382                    (_, false) => throw_ub_format!(
383                        "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
384                    ),
385                    _ => {}
386                }
387                let res = this.binary_op(op, &a, &b)?;
388                // This cannot be a NaN so we also don't have to apply any non-determinism.
389                // (Also, `binary_op` already called `generate_nan` if needed.)
390                if !float_finite(&res)? {
391                    throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result");
392                }
393                // Apply a relative error of 4ULP to simulate non-deterministic precision loss
394                // due to optimizations.
395                let res = apply_random_float_error_to_imm(this, res, 2 /* log2(4) */)?;
396                this.write_immediate(*res, dest)?;
397            }
398
399            "float_to_int_unchecked" => {
400                let [val] = check_intrinsic_arg_count(args)?;
401                let val = this.read_immediate(val)?;
402
403                let res = this
404                    .float_to_int_checked(&val, dest.layout, Round::TowardZero)?
405                    .ok_or_else(|| {
406                        err_ub_format!(
407                            "`float_to_int_unchecked` intrinsic called on {val} which cannot be represented in target type `{:?}`",
408                            dest.layout.ty
409                        )
410                    })?;
411
412                this.write_immediate(*res, dest)?;
413            }
414
415            // Other
416            "breakpoint" => {
417                let [] = check_intrinsic_arg_count(args)?;
418                // normally this would raise a SIGTRAP, which aborts if no debugger is connected
419                throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
420            }
421
422            _ => return interp_ok(EmulateItemResult::NotSupported),
423        }
424
425        interp_ok(EmulateItemResult::NeedsReturn)
426    }
427}