1#![warn(clippy::arithmetic_side_effects)]
2
3mod atomic;
4mod simd;
5
6use rand::Rng;
7use rustc_abi::Size;
8use rustc_apfloat::{Float, Round};
9use rustc_middle::mir;
10use rustc_middle::ty::{self, FloatTy};
11use rustc_span::{Symbol, sym};
12
13use self::atomic::EvalContextExt as _;
14use self::helpers::{ToHost, ToSoft, check_intrinsic_arg_count};
15use self::simd::EvalContextExt as _;
16use crate::math::apply_random_float_error_to_imm;
17use crate::*;
18
19impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
20pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
21 fn call_intrinsic(
22 &mut self,
23 instance: ty::Instance<'tcx>,
24 args: &[OpTy<'tcx>],
25 dest: &PlaceTy<'tcx>,
26 ret: Option<mir::BasicBlock>,
27 unwind: mir::UnwindAction,
28 ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
29 let this = self.eval_context_mut();
30
31 if this.machine.force_intrinsic_fallback
33 && !this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden
34 {
35 return interp_ok(Some(ty::Instance {
36 def: ty::InstanceKind::Item(instance.def_id()),
37 args: instance.args,
38 }));
39 }
40
41 if this.eval_intrinsic(instance, args, dest, ret)? {
43 return interp_ok(None);
44 }
45 let intrinsic_name = this.tcx.item_name(instance.def_id());
46 let intrinsic_name = intrinsic_name.as_str();
47
48 let dest = this.force_allocation(dest)?;
50
51 match this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, &dest, ret)? {
52 EmulateItemResult::NotSupported => {
53 if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
55 throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
56 }
57 let intrinsic_fallback_is_spec = Symbol::intern("intrinsic_fallback_is_spec");
58 if this
59 .tcx
60 .get_attrs_by_path(instance.def_id(), &[sym::miri, intrinsic_fallback_is_spec])
61 .next()
62 .is_none()
63 {
64 throw_unsup_format!(
65 "Miri can only use intrinsic fallback bodies that exactly reflect the specification: they fully check for UB and are as non-deterministic as possible. After verifying that `{intrinsic_name}` does so, add the `#[miri::intrinsic_fallback_is_spec]` attribute to it; also ping @rust-lang/miri when you do that"
66 );
67 }
68 interp_ok(Some(ty::Instance {
69 def: ty::InstanceKind::Item(instance.def_id()),
70 args: instance.args,
71 }))
72 }
73 EmulateItemResult::NeedsReturn => {
74 trace!("{:?}", this.dump_place(&dest.clone().into()));
75 this.return_to_block(ret)?;
76 interp_ok(None)
77 }
78 EmulateItemResult::NeedsUnwind => {
79 this.unwind_to_block(unwind)?;
81 interp_ok(None)
82 }
83 EmulateItemResult::AlreadyJumped => interp_ok(None),
84 }
85 }
86
87 fn emulate_intrinsic_by_name(
90 &mut self,
91 intrinsic_name: &str,
92 generic_args: ty::GenericArgsRef<'tcx>,
93 args: &[OpTy<'tcx>],
94 dest: &MPlaceTy<'tcx>,
95 ret: Option<mir::BasicBlock>,
96 ) -> InterpResult<'tcx, EmulateItemResult> {
97 let this = self.eval_context_mut();
98
99 if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
100 return this.emulate_atomic_intrinsic(name, generic_args, args, dest);
101 }
102 if let Some(name) = intrinsic_name.strip_prefix("simd_") {
103 return this.emulate_simd_intrinsic(name, generic_args, args, dest);
104 }
105
106 match intrinsic_name {
107 "abort" => {
109 throw_machine_stop!(TerminationInfo::Abort(
110 "the program aborted execution".to_owned()
111 ));
112 }
113 "catch_unwind" => {
114 this.handle_catch_unwind(args, dest, ret)?;
115 return interp_ok(EmulateItemResult::AlreadyJumped);
117 }
118
119 "volatile_load" => {
121 let [place] = check_intrinsic_arg_count(args)?;
122 let place = this.deref_pointer(place)?;
123 this.copy_op(&place, dest)?;
124 }
125 "volatile_store" => {
126 let [place, dest] = check_intrinsic_arg_count(args)?;
127 let place = this.deref_pointer(place)?;
128 this.copy_op(dest, &place)?;
129 }
130
131 "volatile_set_memory" => {
132 let [ptr, val_byte, count] = check_intrinsic_arg_count(args)?;
133 this.write_bytes_intrinsic(ptr, val_byte, count, "volatile_set_memory")?;
134 }
135
136 "ptr_mask" => {
138 let [ptr, mask] = check_intrinsic_arg_count(args)?;
139
140 let ptr = this.read_pointer(ptr)?;
141 let mask = this.read_target_usize(mask)?;
142
143 let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask);
144
145 this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
146 }
147
148 "is_val_statically_known" => {
154 let [_arg] = check_intrinsic_arg_count(args)?;
155 let branch: bool = this.machine.rng.get_mut().random();
159 this.write_scalar(Scalar::from_bool(branch), dest)?;
160 }
161
162 "sqrtf32" => {
163 let [f] = check_intrinsic_arg_count(args)?;
164 let f = this.read_scalar(f)?.to_f32()?;
165 let res = math::sqrt(f);
167 let res = this.adjust_nan(res, &[f]);
168 this.write_scalar(res, dest)?;
169 }
170 "sqrtf64" => {
171 let [f] = check_intrinsic_arg_count(args)?;
172 let f = this.read_scalar(f)?.to_f64()?;
173 let res = math::sqrt(f);
175 let res = this.adjust_nan(res, &[f]);
176 this.write_scalar(res, dest)?;
177 }
178
179 #[rustfmt::skip]
180 | "sinf32"
181 | "cosf32"
182 | "expf32"
183 | "exp2f32"
184 | "logf32"
185 | "log10f32"
186 | "log2f32"
187 => {
188 let [f] = check_intrinsic_arg_count(args)?;
189 let f = this.read_scalar(f)?.to_f32()?;
190 let host = f.to_host();
193 let res = match intrinsic_name {
194 "sinf32" => host.sin(),
195 "cosf32" => host.cos(),
196 "expf32" => host.exp(),
197 "exp2f32" => host.exp2(),
198 "logf32" => host.ln(),
199 "log10f32" => host.log10(),
200 "log2f32" => host.log2(),
201 _ => bug!(),
202 };
203 let res = res.to_soft();
204 let res = this.adjust_nan(res, &[f]);
213 this.write_scalar(res, dest)?;
214 }
215 #[rustfmt::skip]
216 | "sinf64"
217 | "cosf64"
218 | "expf64"
219 | "exp2f64"
220 | "logf64"
221 | "log10f64"
222 | "log2f64"
223 => {
224 let [f] = check_intrinsic_arg_count(args)?;
225 let f = this.read_scalar(f)?.to_f64()?;
226 let host = f.to_host();
229 let res = match intrinsic_name {
230 "sinf64" => host.sin(),
231 "cosf64" => host.cos(),
232 "expf64" => host.exp(),
233 "exp2f64" => host.exp2(),
234 "logf64" => host.ln(),
235 "log10f64" => host.log10(),
236 "log2f64" => host.log2(),
237 _ => bug!(),
238 };
239 let res = res.to_soft();
240 let res = this.adjust_nan(res, &[f]);
249 this.write_scalar(res, dest)?;
250 }
251
252 "fmaf32" => {
253 let [a, b, c] = check_intrinsic_arg_count(args)?;
254 let a = this.read_scalar(a)?.to_f32()?;
255 let b = this.read_scalar(b)?.to_f32()?;
256 let c = this.read_scalar(c)?.to_f32()?;
257 let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft();
259 let res = this.adjust_nan(res, &[a, b, c]);
260 this.write_scalar(res, dest)?;
261 }
262 "fmaf64" => {
263 let [a, b, c] = check_intrinsic_arg_count(args)?;
264 let a = this.read_scalar(a)?.to_f64()?;
265 let b = this.read_scalar(b)?.to_f64()?;
266 let c = this.read_scalar(c)?.to_f64()?;
267 let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft();
269 let res = this.adjust_nan(res, &[a, b, c]);
270 this.write_scalar(res, dest)?;
271 }
272
273 "fmuladdf32" => {
274 let [a, b, c] = check_intrinsic_arg_count(args)?;
275 let a = this.read_scalar(a)?.to_f32()?;
276 let b = this.read_scalar(b)?.to_f32()?;
277 let c = this.read_scalar(c)?.to_f32()?;
278 let fuse: bool = this.machine.rng.get_mut().random();
279 let res = if fuse {
280 a.to_host().mul_add(b.to_host(), c.to_host()).to_soft()
282 } else {
283 ((a * b).value + c).value
284 };
285 let res = this.adjust_nan(res, &[a, b, c]);
286 this.write_scalar(res, dest)?;
287 }
288 "fmuladdf64" => {
289 let [a, b, c] = check_intrinsic_arg_count(args)?;
290 let a = this.read_scalar(a)?.to_f64()?;
291 let b = this.read_scalar(b)?.to_f64()?;
292 let c = this.read_scalar(c)?.to_f64()?;
293 let fuse: bool = this.machine.rng.get_mut().random();
294 let res = if fuse {
295 a.to_host().mul_add(b.to_host(), c.to_host()).to_soft()
297 } else {
298 ((a * b).value + c).value
299 };
300 let res = this.adjust_nan(res, &[a, b, c]);
301 this.write_scalar(res, dest)?;
302 }
303
304 "powf32" => {
305 let [f1, f2] = check_intrinsic_arg_count(args)?;
307 let f1 = this.read_scalar(f1)?.to_f32()?;
308 let f2 = this.read_scalar(f2)?.to_f32()?;
309 let res = f1.to_host().powf(f2.to_host()).to_soft();
311 let res = this.adjust_nan(res, &[f1, f2]);
312 this.write_scalar(res, dest)?;
313 }
314 "powf64" => {
315 let [f1, f2] = check_intrinsic_arg_count(args)?;
317 let f1 = this.read_scalar(f1)?.to_f64()?;
318 let f2 = this.read_scalar(f2)?.to_f64()?;
319 let res = f1.to_host().powf(f2.to_host()).to_soft();
321 let res = this.adjust_nan(res, &[f1, f2]);
322 this.write_scalar(res, dest)?;
323 }
324
325 "powif32" => {
326 let [f, i] = check_intrinsic_arg_count(args)?;
328 let f = this.read_scalar(f)?.to_f32()?;
329 let i = this.read_scalar(i)?.to_i32()?;
330 let res = f.to_host().powi(i).to_soft();
332 let res = this.adjust_nan(res, &[f]);
333 this.write_scalar(res, dest)?;
334 }
335 "powif64" => {
336 let [f, i] = check_intrinsic_arg_count(args)?;
338 let f = this.read_scalar(f)?.to_f64()?;
339 let i = this.read_scalar(i)?.to_i32()?;
340 let res = f.to_host().powi(i).to_soft();
342 let res = this.adjust_nan(res, &[f]);
343 this.write_scalar(res, dest)?;
344 }
345
346 #[rustfmt::skip]
347 | "fadd_fast"
348 | "fsub_fast"
349 | "fmul_fast"
350 | "fdiv_fast"
351 | "frem_fast"
352 => {
353 let [a, b] = check_intrinsic_arg_count(args)?;
354 let a = this.read_immediate(a)?;
355 let b = this.read_immediate(b)?;
356 let op = match intrinsic_name {
357 "fadd_fast" => mir::BinOp::Add,
358 "fsub_fast" => mir::BinOp::Sub,
359 "fmul_fast" => mir::BinOp::Mul,
360 "fdiv_fast" => mir::BinOp::Div,
361 "frem_fast" => mir::BinOp::Rem,
362 _ => bug!(),
363 };
364 let float_finite = |x: &ImmTy<'tcx>| -> InterpResult<'tcx, bool> {
365 let ty::Float(fty) = x.layout.ty.kind() else {
366 bug!("float_finite: non-float input type {}", x.layout.ty)
367 };
368 interp_ok(match fty {
369 FloatTy::F16 => x.to_scalar().to_f16()?.is_finite(),
370 FloatTy::F32 => x.to_scalar().to_f32()?.is_finite(),
371 FloatTy::F64 => x.to_scalar().to_f64()?.is_finite(),
372 FloatTy::F128 => x.to_scalar().to_f128()?.is_finite(),
373 })
374 };
375 match (float_finite(&a)?, float_finite(&b)?) {
376 (false, false) => throw_ub_format!(
377 "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
378 ),
379 (false, _) => throw_ub_format!(
380 "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
381 ),
382 (_, false) => throw_ub_format!(
383 "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
384 ),
385 _ => {}
386 }
387 let res = this.binary_op(op, &a, &b)?;
388 if !float_finite(&res)? {
391 throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result");
392 }
393 let res = apply_random_float_error_to_imm(this, res, 2 )?;
396 this.write_immediate(*res, dest)?;
397 }
398
399 "float_to_int_unchecked" => {
400 let [val] = check_intrinsic_arg_count(args)?;
401 let val = this.read_immediate(val)?;
402
403 let res = this
404 .float_to_int_checked(&val, dest.layout, Round::TowardZero)?
405 .ok_or_else(|| {
406 err_ub_format!(
407 "`float_to_int_unchecked` intrinsic called on {val} which cannot be represented in target type `{:?}`",
408 dest.layout.ty
409 )
410 })?;
411
412 this.write_immediate(*res, dest)?;
413 }
414
415 "breakpoint" => {
417 let [] = check_intrinsic_arg_count(args)?;
418 throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
420 }
421
422 _ => return interp_ok(EmulateItemResult::NotSupported),
423 }
424
425 interp_ok(EmulateItemResult::NeedsReturn)
426 }
427}