1use std::sync::atomic::Ordering::Relaxed;
2
3use either::{Left, Right};
4use rustc_abi::{self as abi, BackendRepr};
5use rustc_hir::def::DefKind;
6use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo};
7use rustc_middle::mir::{self, ConstAlloc, ConstValue};
8use rustc_middle::query::TyCtxtAt;
9use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
10use rustc_middle::ty::print::with_no_trimmed_paths;
11use rustc_middle::ty::{self, Ty, TyCtxt};
12use rustc_middle::{bug, throw_inval};
13use rustc_span::def_id::LocalDefId;
14use rustc_span::{DUMMY_SP, Span};
15use tracing::{debug, instrument, trace};
16
17use super::{CanAccessMutGlobal, CompileTimeInterpCx, CompileTimeMachine};
18use crate::const_eval::CheckAlignment;
19use crate::interpret::{
20 CtfeValidationMode, GlobalId, Immediate, InternKind, InternResult, InterpCx, InterpErrorKind,
21 InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, StackPopCleanup, create_static_alloc,
22 eval_nullary_intrinsic, intern_const_alloc_recursive, interp_ok, throw_exhaust,
23};
24use crate::{CTRL_C_RECEIVED, errors};
25
26#[instrument(level = "trace", skip(ecx, body))]
28fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
29 ecx: &mut CompileTimeInterpCx<'tcx>,
30 cid: GlobalId<'tcx>,
31 body: &'tcx mir::Body<'tcx>,
32) -> InterpResult<'tcx, R> {
33 let tcx = *ecx.tcx;
34 assert!(
35 cid.promoted.is_some()
36 || matches!(
37 ecx.tcx.def_kind(cid.instance.def_id()),
38 DefKind::Const
39 | DefKind::Static { .. }
40 | DefKind::ConstParam
41 | DefKind::AnonConst
42 | DefKind::InlineConst
43 | DefKind::AssocConst
44 ),
45 "Unexpected DefKind: {:?}",
46 ecx.tcx.def_kind(cid.instance.def_id())
47 );
48 let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
49 assert!(layout.is_sized());
50
51 let intern_kind = if cid.promoted.is_some() {
52 InternKind::Promoted
53 } else {
54 match tcx.static_mutability(cid.instance.def_id()) {
55 Some(m) => InternKind::Static(m),
56 None => InternKind::Constant,
57 }
58 };
59
60 let ret = if let InternKind::Static(_) = intern_kind {
61 create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
62 } else {
63 ecx.allocate(layout, MemoryKind::Stack)?
64 };
65
66 trace!(
67 "eval_body_using_ecx: pushing stack frame for global: {}{}",
68 with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
69 cid.promoted.map_or_else(String::new, |p| format!("::{p:?}"))
70 );
71
72 ecx.push_stack_frame_raw(
75 cid.instance,
76 body,
77 &ret.clone().into(),
78 StackPopCleanup::Root { cleanup: false },
79 )?;
80 ecx.storage_live_for_always_live_locals()?;
81
82 while ecx.step()? {
84 if CTRL_C_RECEIVED.load(Relaxed) {
85 throw_exhaust!(Interrupted);
86 }
87 }
88
89 let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
91
92 const_validate_mplace(ecx, &ret, cid)?;
94
95 match intern_result {
99 Ok(()) => {}
100 Err(InternResult::FoundDanglingPointer) => {
101 throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
102 ecx.tcx
103 .dcx()
104 .emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
105 )));
106 }
107 Err(InternResult::FoundBadMutablePointer) => {
108 throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
109 ecx.tcx
110 .dcx()
111 .emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
112 )));
113 }
114 }
115
116 interp_ok(R::make_result(ret, ecx))
117}
118
119pub(crate) fn mk_eval_cx_to_read_const_val<'tcx>(
130 tcx: TyCtxt<'tcx>,
131 root_span: Span,
132 typing_env: ty::TypingEnv<'tcx>,
133 can_access_mut_global: CanAccessMutGlobal,
134) -> CompileTimeInterpCx<'tcx> {
135 debug!("mk_eval_cx: {:?}", typing_env);
136 InterpCx::new(
137 tcx,
138 root_span,
139 typing_env,
140 CompileTimeMachine::new(can_access_mut_global, CheckAlignment::No),
141 )
142}
143
144pub fn mk_eval_cx_for_const_val<'tcx>(
147 tcx: TyCtxtAt<'tcx>,
148 typing_env: ty::TypingEnv<'tcx>,
149 val: mir::ConstValue<'tcx>,
150 ty: Ty<'tcx>,
151) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
152 let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, typing_env, CanAccessMutGlobal::No);
153 let op = ecx.const_val_to_op(val, ty, None).discard_err()?;
155 Some((ecx, op))
156}
157
158#[instrument(skip(ecx), level = "debug")]
165pub(super) fn op_to_const<'tcx>(
166 ecx: &CompileTimeInterpCx<'tcx>,
167 op: &OpTy<'tcx>,
168 for_diagnostics: bool,
169) -> ConstValue<'tcx> {
170 if op.layout.is_zst() {
172 return ConstValue::ZeroSized;
173 }
174
175 let force_as_immediate = match op.layout.backend_repr {
181 BackendRepr::Scalar(abi::Scalar::Initialized { .. }) => true,
182 _ => false,
190 };
191 let immediate = if force_as_immediate {
192 match ecx.read_immediate(op).report_err() {
193 Ok(imm) => Right(imm),
194 Err(err) => {
195 if for_diagnostics {
196 op.as_mplace_or_imm()
198 } else {
199 panic!("normalization works on validated constants: {err:?}")
200 }
201 }
202 }
203 } else {
204 op.as_mplace_or_imm()
205 };
206
207 debug!(?immediate);
208
209 match immediate {
210 Left(ref mplace) => {
211 let (prov, offset) = mplace.ptr().into_parts();
213 let alloc_id = prov.expect("cannot have `fake` place for non-ZST type").alloc_id();
214 ConstValue::Indirect { alloc_id, offset }
215 }
216 Right(imm) => match *imm {
218 Immediate::Scalar(x) => ConstValue::Scalar(x),
219 Immediate::ScalarPair(a, b) => {
220 debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
221 let pointee_ty = imm.layout.ty.builtin_deref(false).unwrap(); debug_assert!(
226 matches!(
227 ecx.tcx.struct_tail_for_codegen(pointee_ty, ecx.typing_env()).kind(),
228 ty::Str | ty::Slice(..),
229 ),
230 "`ConstValue::Slice` is for slice-tailed types only, but got {}",
231 imm.layout.ty,
232 );
233 let msg = "`op_to_const` on an immediate scalar pair must only be used on slice references to the beginning of an actual allocation";
234 let (prov, offset) = a.to_pointer(ecx).expect(msg).into_parts();
236 let alloc_id = prov.expect(msg).alloc_id();
237 let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
238 assert!(offset == abi::Size::ZERO, "{}", msg);
239 let meta = b.to_target_usize(ecx).expect(msg);
240 ConstValue::Slice { data, meta }
241 }
242 Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
243 },
244 }
245}
246
247#[instrument(skip(tcx), level = "debug", ret)]
248pub(crate) fn turn_into_const_value<'tcx>(
249 tcx: TyCtxt<'tcx>,
250 constant: ConstAlloc<'tcx>,
251 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
252) -> ConstValue<'tcx> {
253 let cid = key.value;
254 let def_id = cid.instance.def.def_id();
255 let is_static = tcx.is_static(def_id);
256 let ecx = mk_eval_cx_to_read_const_val(
258 tcx,
259 tcx.def_span(key.value.instance.def_id()),
260 key.typing_env,
261 CanAccessMutGlobal::from(is_static),
262 );
263
264 let mplace = ecx.raw_const_to_mplace(constant).expect(
265 "can only fail if layout computation failed, \
266 which should have given a good error before ever invoking this function",
267 );
268 assert!(
269 !is_static || cid.promoted.is_some(),
270 "the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead"
271 );
272
273 op_to_const(&ecx, &mplace.into(), false)
275}
276
277#[instrument(skip(tcx), level = "debug")]
278pub fn eval_to_const_value_raw_provider<'tcx>(
279 tcx: TyCtxt<'tcx>,
280 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
281) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> {
282 if let ty::InstanceKind::Intrinsic(def_id) = key.value.instance.def {
285 let ty = key.value.instance.ty(tcx, key.typing_env);
286 let ty::FnDef(_, args) = ty.kind() else {
287 bug!("intrinsic with type {:?}", ty);
288 };
289 return eval_nullary_intrinsic(tcx, key.typing_env, def_id, args).report_err().map_err(
290 |error| {
291 let span = tcx.def_span(def_id);
292
293 super::report(
294 tcx,
295 error.into_kind(),
296 span,
297 || (span, vec![]),
298 |span, _| errors::NullaryIntrinsicError { span },
299 )
300 },
301 );
302 }
303
304 tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
305}
306
307#[instrument(skip(tcx), level = "debug")]
308pub fn eval_static_initializer_provider<'tcx>(
309 tcx: TyCtxt<'tcx>,
310 def_id: LocalDefId,
311) -> ::rustc_middle::mir::interpret::EvalStaticInitializerRawResult<'tcx> {
312 assert!(tcx.is_static(def_id.to_def_id()));
313
314 let instance = ty::Instance::mono(tcx, def_id.to_def_id());
315 let cid = rustc_middle::mir::interpret::GlobalId { instance, promoted: None };
316 eval_in_interpreter(tcx, cid, ty::TypingEnv::fully_monomorphized())
317}
318
319pub trait InterpretationResult<'tcx> {
320 fn make_result(
324 mplace: MPlaceTy<'tcx>,
325 ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
326 ) -> Self;
327}
328
329impl<'tcx> InterpretationResult<'tcx> for ConstAlloc<'tcx> {
330 fn make_result(
331 mplace: MPlaceTy<'tcx>,
332 _ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
333 ) -> Self {
334 ConstAlloc { alloc_id: mplace.ptr().provenance.unwrap().alloc_id(), ty: mplace.layout.ty }
335 }
336}
337
338#[instrument(skip(tcx), level = "debug")]
339pub fn eval_to_allocation_raw_provider<'tcx>(
340 tcx: TyCtxt<'tcx>,
341 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
342) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
343 assert!(key.value.promoted.is_some() || !tcx.is_static(key.value.instance.def_id()));
346 debug_assert_eq!(key.typing_env.typing_mode, ty::TypingMode::PostAnalysis);
349 if cfg!(debug_assertions) {
350 let instance = with_no_trimmed_paths!(key.value.instance.to_string());
356 trace!("const eval: {:?} ({})", key, instance);
357 }
358
359 eval_in_interpreter(tcx, key.value, key.typing_env)
360}
361
362fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
363 tcx: TyCtxt<'tcx>,
364 cid: GlobalId<'tcx>,
365 typing_env: ty::TypingEnv<'tcx>,
366) -> Result<R, ErrorHandled> {
367 let def = cid.instance.def.def_id();
368 let is_static = tcx.is_static(def);
369
370 let mut ecx = InterpCx::new(
371 tcx,
372 tcx.def_span(def),
373 typing_env,
374 CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
379 );
380 let res = ecx.load_mir(cid.instance.def, cid.promoted);
381 res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
382 .report_err()
383 .map_err(|error| report_eval_error(&ecx, cid, error))
384}
385
386#[inline(always)]
387fn const_validate_mplace<'tcx>(
388 ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
389 mplace: &MPlaceTy<'tcx>,
390 cid: GlobalId<'tcx>,
391) -> Result<(), ErrorHandled> {
392 let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
393 let mut ref_tracking = RefTracking::new(mplace.clone());
394 let mut inner = false;
395 while let Some((mplace, path)) = ref_tracking.next() {
396 let mode = match ecx.tcx.static_mutability(cid.instance.def_id()) {
397 _ if cid.promoted.is_some() => CtfeValidationMode::Promoted,
398 Some(mutbl) => CtfeValidationMode::Static { mutbl }, None => {
400 CtfeValidationMode::Const { allow_immutable_unsafe_cell: !inner }
404 }
405 };
406 ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)
407 .report_err()
408 .map_err(|error| report_validation_error(&ecx, cid, error, alloc_id))?;
411 inner = true;
412 }
413
414 Ok(())
415}
416
417#[inline(never)]
418fn report_eval_error<'tcx>(
419 ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
420 cid: GlobalId<'tcx>,
421 error: InterpErrorInfo<'tcx>,
422) -> ErrorHandled {
423 let (error, backtrace) = error.into_parts();
424 backtrace.print_backtrace();
425
426 let (kind, instance) = if ecx.tcx.is_static(cid.instance.def_id()) {
427 ("static", String::new())
428 } else {
429 let instance = &cid.instance;
433 if !instance.args.is_empty() {
434 let instance = with_no_trimmed_paths!(instance.to_string());
435 ("const_with_path", instance)
436 } else {
437 ("const", String::new())
438 }
439 };
440
441 super::report(
442 *ecx.tcx,
443 error,
444 DUMMY_SP,
445 || super::get_span_and_frames(ecx.tcx, ecx.stack()),
446 |span, frames| errors::ConstEvalError {
447 span,
448 error_kind: kind,
449 instance,
450 frame_notes: frames,
451 },
452 )
453}
454
455#[inline(never)]
456fn report_validation_error<'tcx>(
457 ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
458 cid: GlobalId<'tcx>,
459 error: InterpErrorInfo<'tcx>,
460 alloc_id: AllocId,
461) -> ErrorHandled {
462 if !matches!(error.kind(), InterpErrorKind::UndefinedBehavior(_)) {
463 return report_eval_error(ecx, cid, error);
465 }
466
467 let (error, backtrace) = error.into_parts();
468 backtrace.print_backtrace();
469
470 let bytes = ecx.print_alloc_bytes_for_diagnostics(alloc_id);
471 let info = ecx.get_alloc_info(alloc_id);
472 let raw_bytes =
473 errors::RawBytesNote { size: info.size.bytes(), align: info.align.bytes(), bytes };
474
475 crate::const_eval::report(
476 *ecx.tcx,
477 error,
478 DUMMY_SP,
479 || crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()),
480 move |span, frames| errors::ValidationFailure { span, ub_note: (), frames, raw_bytes },
481 )
482}