miri/intrinsics/
atomic.rs

1use rustc_middle::mir::BinOp;
2use rustc_middle::ty::AtomicOrdering;
3use rustc_middle::{mir, ty};
4
5use self::helpers::check_intrinsic_arg_count;
6use crate::*;
7
8pub enum AtomicOp {
9    /// The `bool` indicates whether the result of the operation should be negated (`UnOp::Not`,
10    /// must be a boolean-typed operation).
11    MirOp(mir::BinOp, bool),
12    Max,
13    Min,
14}
15
16impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
17pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
18    /// Calls the atomic intrinsic `intrinsic`; the `atomic_` prefix has already been removed.
19    /// Returns `Ok(true)` if the intrinsic was handled.
20    fn emulate_atomic_intrinsic(
21        &mut self,
22        intrinsic_name: &str,
23        generic_args: ty::GenericArgsRef<'tcx>,
24        args: &[OpTy<'tcx>],
25        dest: &MPlaceTy<'tcx>,
26    ) -> InterpResult<'tcx, EmulateItemResult> {
27        let this = self.eval_context_mut();
28
29        let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect();
30
31        fn read_ord(ord: &str) -> AtomicReadOrd {
32            match ord {
33                "seqcst" => AtomicReadOrd::SeqCst,
34                "acquire" => AtomicReadOrd::Acquire,
35                "relaxed" => AtomicReadOrd::Relaxed,
36                _ => panic!("invalid read ordering `{ord}`"),
37            }
38        }
39
40        fn read_ord_const_generic(o: AtomicOrdering) -> AtomicReadOrd {
41            match o {
42                AtomicOrdering::SeqCst => AtomicReadOrd::SeqCst,
43                AtomicOrdering::Acquire => AtomicReadOrd::Acquire,
44                AtomicOrdering::Relaxed => AtomicReadOrd::Relaxed,
45                _ => panic!("invalid read ordering `{o:?}`"),
46            }
47        }
48
49        fn write_ord(ord: &str) -> AtomicWriteOrd {
50            match ord {
51                "seqcst" => AtomicWriteOrd::SeqCst,
52                "release" => AtomicWriteOrd::Release,
53                "relaxed" => AtomicWriteOrd::Relaxed,
54                _ => panic!("invalid write ordering `{ord}`"),
55            }
56        }
57
58        fn rw_ord(ord: &str) -> AtomicRwOrd {
59            match ord {
60                "seqcst" => AtomicRwOrd::SeqCst,
61                "acqrel" => AtomicRwOrd::AcqRel,
62                "acquire" => AtomicRwOrd::Acquire,
63                "release" => AtomicRwOrd::Release,
64                "relaxed" => AtomicRwOrd::Relaxed,
65                _ => panic!("invalid read-write ordering `{ord}`"),
66            }
67        }
68
69        fn fence_ord(ord: &str) -> AtomicFenceOrd {
70            match ord {
71                "seqcst" => AtomicFenceOrd::SeqCst,
72                "acqrel" => AtomicFenceOrd::AcqRel,
73                "acquire" => AtomicFenceOrd::Acquire,
74                "release" => AtomicFenceOrd::Release,
75                _ => panic!("invalid fence ordering `{ord}`"),
76            }
77        }
78
79        match &*intrinsic_structure {
80            // New-style intrinsics that use const generics
81            ["load"] => {
82                let ordering = generic_args.const_at(1).to_value();
83                let ordering =
84                    ordering.valtree.unwrap_branch()[0].unwrap_leaf().to_atomic_ordering();
85                this.atomic_load(args, dest, read_ord_const_generic(ordering))?;
86            }
87
88            // Old-style intrinsics that have the ordering in the intrinsic name
89            ["store", ord] => this.atomic_store(args, write_ord(ord))?,
90
91            ["fence", ord] => this.atomic_fence_intrinsic(args, fence_ord(ord))?,
92            ["singlethreadfence", ord] => this.compiler_fence_intrinsic(args, fence_ord(ord))?,
93
94            ["xchg", ord] => this.atomic_exchange(args, dest, rw_ord(ord))?,
95            ["cxchg", ord1, ord2] =>
96                this.atomic_compare_exchange(args, dest, rw_ord(ord1), read_ord(ord2))?,
97            ["cxchgweak", ord1, ord2] =>
98                this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1), read_ord(ord2))?,
99
100            ["or", ord] =>
101                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord))?,
102            ["xor", ord] =>
103                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord))?,
104            ["and", ord] =>
105                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord))?,
106            ["nand", ord] =>
107                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord))?,
108            ["xadd", ord] =>
109                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord))?,
110            ["xsub", ord] =>
111                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord))?,
112            ["min", ord] => {
113                // Later we will use the type to indicate signed vs unsigned,
114                // so make sure it matches the intrinsic name.
115                assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
116                this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord))?;
117            }
118            ["umin", ord] => {
119                // Later we will use the type to indicate signed vs unsigned,
120                // so make sure it matches the intrinsic name.
121                assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
122                this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord))?;
123            }
124            ["max", ord] => {
125                // Later we will use the type to indicate signed vs unsigned,
126                // so make sure it matches the intrinsic name.
127                assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
128                this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord))?;
129            }
130            ["umax", ord] => {
131                // Later we will use the type to indicate signed vs unsigned,
132                // so make sure it matches the intrinsic name.
133                assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
134                this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord))?;
135            }
136
137            _ => return interp_ok(EmulateItemResult::NotSupported),
138        }
139        interp_ok(EmulateItemResult::NeedsReturn)
140    }
141}
142
143impl<'tcx> EvalContextPrivExt<'tcx> for MiriInterpCx<'tcx> {}
144trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> {
145    fn atomic_load(
146        &mut self,
147        args: &[OpTy<'tcx>],
148        dest: &MPlaceTy<'tcx>,
149        atomic: AtomicReadOrd,
150    ) -> InterpResult<'tcx> {
151        let this = self.eval_context_mut();
152
153        let [place] = check_intrinsic_arg_count(args)?;
154        let place = this.deref_pointer(place)?;
155
156        // Perform atomic load.
157        let val = this.read_scalar_atomic(&place, atomic)?;
158        // Perform regular store.
159        this.write_scalar(val, dest)?;
160        interp_ok(())
161    }
162
163    fn atomic_store(&mut self, args: &[OpTy<'tcx>], atomic: AtomicWriteOrd) -> InterpResult<'tcx> {
164        let this = self.eval_context_mut();
165
166        let [place, val] = check_intrinsic_arg_count(args)?;
167        let place = this.deref_pointer(place)?;
168
169        // Perform regular load.
170        let val = this.read_scalar(val)?;
171        // Perform atomic store.
172        this.write_scalar_atomic(val, &place, atomic)?;
173        interp_ok(())
174    }
175
176    fn compiler_fence_intrinsic(
177        &mut self,
178        args: &[OpTy<'tcx>],
179        atomic: AtomicFenceOrd,
180    ) -> InterpResult<'tcx> {
181        let [] = check_intrinsic_arg_count(args)?;
182        let _ = atomic;
183        // FIXME, FIXME(GenMC): compiler fences are currently ignored (also ignored in GenMC mode)
184        interp_ok(())
185    }
186
187    fn atomic_fence_intrinsic(
188        &mut self,
189        args: &[OpTy<'tcx>],
190        atomic: AtomicFenceOrd,
191    ) -> InterpResult<'tcx> {
192        let this = self.eval_context_mut();
193        let [] = check_intrinsic_arg_count(args)?;
194        this.atomic_fence(atomic)?;
195        interp_ok(())
196    }
197
198    fn atomic_rmw_op(
199        &mut self,
200        args: &[OpTy<'tcx>],
201        dest: &MPlaceTy<'tcx>,
202        atomic_op: AtomicOp,
203        atomic: AtomicRwOrd,
204    ) -> InterpResult<'tcx> {
205        let this = self.eval_context_mut();
206
207        let [place, rhs] = check_intrinsic_arg_count(args)?;
208        let place = this.deref_pointer(place)?;
209        let rhs = this.read_immediate(rhs)?;
210
211        if !place.layout.ty.is_integral() && !place.layout.ty.is_raw_ptr() {
212            span_bug!(
213                this.cur_span(),
214                "atomic arithmetic operations only work on integer and raw pointer types",
215            );
216        }
217        if rhs.layout.ty != place.layout.ty {
218            span_bug!(this.cur_span(), "atomic arithmetic operation type mismatch");
219        }
220
221        let old = match atomic_op {
222            AtomicOp::Min =>
223                this.atomic_min_max_scalar(&place, rhs, /* min */ true, atomic)?,
224            AtomicOp::Max =>
225                this.atomic_min_max_scalar(&place, rhs, /* min */ false, atomic)?,
226            AtomicOp::MirOp(op, not) =>
227                this.atomic_rmw_op_immediate(&place, &rhs, op, not, atomic)?,
228        };
229        this.write_immediate(*old, dest)?; // old value is returned
230        interp_ok(())
231    }
232
233    fn atomic_exchange(
234        &mut self,
235        args: &[OpTy<'tcx>],
236        dest: &MPlaceTy<'tcx>,
237        atomic: AtomicRwOrd,
238    ) -> InterpResult<'tcx> {
239        let this = self.eval_context_mut();
240
241        let [place, new] = check_intrinsic_arg_count(args)?;
242        let place = this.deref_pointer(place)?;
243        let new = this.read_scalar(new)?;
244
245        let old = this.atomic_exchange_scalar(&place, new, atomic)?;
246        this.write_scalar(old, dest)?; // old value is returned
247        interp_ok(())
248    }
249
250    fn atomic_compare_exchange_impl(
251        &mut self,
252        args: &[OpTy<'tcx>],
253        dest: &MPlaceTy<'tcx>,
254        success: AtomicRwOrd,
255        fail: AtomicReadOrd,
256        can_fail_spuriously: bool,
257    ) -> InterpResult<'tcx> {
258        let this = self.eval_context_mut();
259
260        let [place, expect_old, new] = check_intrinsic_arg_count(args)?;
261        let place = this.deref_pointer(place)?;
262        let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
263        let new = this.read_scalar(new)?;
264
265        let old = this.atomic_compare_exchange_scalar(
266            &place,
267            &expect_old,
268            new,
269            success,
270            fail,
271            can_fail_spuriously,
272        )?;
273
274        // Return old value.
275        this.write_immediate(old, dest)?;
276        interp_ok(())
277    }
278
279    fn atomic_compare_exchange(
280        &mut self,
281        args: &[OpTy<'tcx>],
282        dest: &MPlaceTy<'tcx>,
283        success: AtomicRwOrd,
284        fail: AtomicReadOrd,
285    ) -> InterpResult<'tcx> {
286        self.atomic_compare_exchange_impl(args, dest, success, fail, false)
287    }
288
289    fn atomic_compare_exchange_weak(
290        &mut self,
291        args: &[OpTy<'tcx>],
292        dest: &MPlaceTy<'tcx>,
293        success: AtomicRwOrd,
294        fail: AtomicReadOrd,
295    ) -> InterpResult<'tcx> {
296        self.atomic_compare_exchange_impl(args, dest, success, fail, true)
297    }
298}
miri/intrinsics/atomic.rs

miri/intrinsics/
atomic.rs
