use rustc_hir::def::DefKind;
use rustc_hir::{LangItem, CRATE_HIR_ID};
use rustc_middle::mir;
use rustc_middle::mir::interpret::PointerArithmetic;
use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout};
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_session::lint::builtin::INVALID_ALIGNMENT;
use std::borrow::Borrow;
use std::hash::Hash;
use std::ops::ControlFlow;
use rustc_data_structures::fx::FxIndexMap;
use rustc_data_structures::fx::IndexEntry;
use std::fmt;
use rustc_ast::Mutability;
use rustc_hir::def_id::DefId;
use rustc_middle::mir::AssertMessage;
use rustc_session::Limit;
use rustc_span::symbol::{sym, Symbol};
use rustc_target::abi::{Align, Size};
use rustc_target::spec::abi::Abi as CallAbi;
use crate::interpret::{
self, compile_time_machine, AllocId, ConstAllocation, FnVal, Frame, ImmTy, InterpCx,
InterpResult, OpTy, PlaceTy, Pointer, Scalar,
};
use super::error::*;
pub struct CompileTimeInterpreter<'mir, 'tcx> {
pub(super) steps_remaining: usize,
pub(super) stack: Vec<Frame<'mir, 'tcx, AllocId, ()>>,
pub(super) can_access_statics: bool,
pub(super) check_alignment: CheckAlignment,
}
#[derive(Copy, Clone)]
pub enum CheckAlignment {
No,
Error,
FutureIncompat,
}
impl CheckAlignment {
pub fn should_check(&self) -> bool {
match self {
CheckAlignment::No => false,
CheckAlignment::Error | CheckAlignment::FutureIncompat => true,
}
}
}
impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> {
pub(crate) fn new(
const_eval_limit: Limit,
can_access_statics: bool,
check_alignment: CheckAlignment,
) -> Self {
CompileTimeInterpreter {
steps_remaining: const_eval_limit.0,
stack: Vec::new(),
can_access_statics,
check_alignment,
}
}
}
impl<K: Hash + Eq, V> interpret::AllocMap<K, V> for FxIndexMap<K, V> {
#[inline(always)]
fn contains_key<Q: ?Sized + Hash + Eq>(&mut self, k: &Q) -> bool
where
K: Borrow<Q>,
{
FxIndexMap::contains_key(self, k)
}
#[inline(always)]
fn insert(&mut self, k: K, v: V) -> Option<V> {
FxIndexMap::insert(self, k, v)
}
#[inline(always)]
fn remove<Q: ?Sized + Hash + Eq>(&mut self, k: &Q) -> Option<V>
where
K: Borrow<Q>,
{
FxIndexMap::remove(self, k)
}
#[inline(always)]
fn filter_map_collect<T>(&self, mut f: impl FnMut(&K, &V) -> Option<T>) -> Vec<T> {
self.iter().filter_map(move |(k, v)| f(k, &*v)).collect()
}
#[inline(always)]
fn get_or<E>(&self, k: K, vacant: impl FnOnce() -> Result<V, E>) -> Result<&V, E> {
match self.get(&k) {
Some(v) => Ok(v),
None => {
vacant()?;
bug!("The CTFE machine shouldn't ever need to extend the alloc_map when reading")
}
}
}
#[inline(always)]
fn get_mut_or<E>(&mut self, k: K, vacant: impl FnOnce() -> Result<V, E>) -> Result<&mut V, E> {
match self.entry(k) {
IndexEntry::Occupied(e) => Ok(e.into_mut()),
IndexEntry::Vacant(e) => {
let v = vacant()?;
Ok(e.insert(v))
}
}
}
}
pub(crate) type CompileTimeEvalContext<'mir, 'tcx> =
InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>;
#[derive(Debug, PartialEq, Eq, Copy, Clone)]
pub enum MemoryKind {
Heap,
}
impl fmt::Display for MemoryKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
MemoryKind::Heap => write!(f, "heap allocation"),
}
}
}
impl interpret::MayLeak for MemoryKind {
#[inline(always)]
fn may_leak(self) -> bool {
match self {
MemoryKind::Heap => false,
}
}
}
impl interpret::MayLeak for ! {
#[inline(always)]
fn may_leak(self) -> bool {
self
}
}
impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
fn hook_special_const_fn(
&mut self,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx>],
dest: &PlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
let def_id = instance.def_id();
if Some(def_id) == self.tcx.lang_items().panic_display()
|| Some(def_id) == self.tcx.lang_items().begin_panic_fn()
{
assert!(args.len() == 1);
let mut msg_place = self.deref_operand(&args[0])?;
while msg_place.layout.ty.is_ref() {
msg_place = self.deref_operand(&msg_place.into())?;
}
let msg = Symbol::intern(self.read_str(&msg_place)?);
let span = self.find_closest_untracked_caller_location();
let (file, line, col) = self.location_triple_for_span(span);
return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into());
} else if Some(def_id) == self.tcx.lang_items().panic_fmt() {
let const_def_id = self.tcx.require_lang_item(LangItem::ConstPanicFmt, None);
let new_instance = ty::Instance::resolve(
*self.tcx,
ty::ParamEnv::reveal_all(),
const_def_id,
instance.substs,
)
.unwrap()
.unwrap();
return Ok(Some(new_instance));
} else if Some(def_id) == self.tcx.lang_items().align_offset_fn() {
match self.align_offset(instance, args, dest, ret)? {
ControlFlow::Continue(()) => return Ok(Some(instance)),
ControlFlow::Break(()) => return Ok(None),
}
}
Ok(Some(instance))
}
fn align_offset(
&mut self,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx>],
dest: &PlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, ControlFlow<()>> {
assert_eq!(args.len(), 2);
let ptr = self.read_pointer(&args[0])?;
let target_align = self.read_scalar(&args[1])?.to_target_usize(self)?;
if !target_align.is_power_of_two() {
throw_ub_format!("`align_offset` called with non-power-of-two align: {}", target_align);
}
match self.ptr_try_get_alloc_id(ptr) {
Ok((alloc_id, offset, _extra)) => {
let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id);
if target_align <= alloc_align.bytes() {
let addr = ImmTy::from_uint(offset.bytes(), args[0].layout).into();
let align = ImmTy::from_uint(target_align, args[1].layout).into();
let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?;
self.eval_fn_call(
FnVal::Instance(instance),
(CallAbi::Rust, fn_abi),
&[addr, align],
false,
dest,
ret,
mir::UnwindAction::Unreachable,
)?;
Ok(ControlFlow::Break(()))
} else {
let usize_max = Scalar::from_target_usize(self.target_usize_max(), self);
self.write_scalar(usize_max, dest)?;
self.return_to_block(ret)?;
Ok(ControlFlow::Break(()))
}
}
Err(_addr) => {
Ok(ControlFlow::Continue(()))
}
}
}
fn guaranteed_cmp(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, u8> {
Ok(match (a, b) {
(Scalar::Int { .. }, Scalar::Int { .. }) => {
if a == b {
1
} else {
0
}
}
(Scalar::Int(int), ptr @ Scalar::Ptr(..))
| (ptr @ Scalar::Ptr(..), Scalar::Int(int))
if int.is_null() && !self.scalar_may_be_null(ptr)? =>
{
0
}
(Scalar::Int { .. }, Scalar::Ptr(..)) | (Scalar::Ptr(..), Scalar::Int { .. }) => 2,
(Scalar::Ptr(..), Scalar::Ptr(..)) => 2,
})
}
}
impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, 'tcx> {
compile_time_machine!(<'mir, 'tcx>);
type MemoryKind = MemoryKind;
const PANIC_ON_ALLOC_FAIL: bool = false; #[inline(always)]
fn enforce_alignment(ecx: &InterpCx<'mir, 'tcx, Self>) -> CheckAlignment {
ecx.machine.check_alignment
}
#[inline(always)]
fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>, layout: TyAndLayout<'tcx>) -> bool {
ecx.tcx.sess.opts.unstable_opts.extra_const_ub_checks || layout.abi.is_uninhabited()
}
fn alignment_check_failed(
ecx: &InterpCx<'mir, 'tcx, Self>,
has: Align,
required: Align,
check: CheckAlignment,
) -> InterpResult<'tcx, ()> {
let err = err_ub!(AlignmentCheckFailed { has, required }).into();
match check {
CheckAlignment::Error => Err(err),
CheckAlignment::No => span_bug!(
ecx.cur_span(),
"`alignment_check_failed` called when no alignment check requested"
),
CheckAlignment::FutureIncompat => {
let err = ConstEvalErr::new(ecx, err, None);
ecx.tcx.struct_span_lint_hir(
INVALID_ALIGNMENT,
ecx.stack().iter().find_map(|frame| frame.lint_root()).unwrap_or(CRATE_HIR_ID),
err.span,
err.error.to_string(),
|db| {
err.decorate(db, |_| {});
db
},
);
Ok(())
}
}
}
fn load_mir(
ecx: &InterpCx<'mir, 'tcx, Self>,
instance: ty::InstanceDef<'tcx>,
) -> InterpResult<'tcx, &'tcx mir::Body<'tcx>> {
match instance {
ty::InstanceDef::Item(def) => {
if ecx.tcx.is_ctfe_mir_available(def) {
Ok(ecx.tcx.mir_for_ctfe(def))
} else if ecx.tcx.def_kind(def) == DefKind::AssocConst {
let guar = ecx.tcx.sess.delay_span_bug(
rustc_span::DUMMY_SP,
"This is likely a const item that is missing from its impl",
);
throw_inval!(AlreadyReported(guar));
} else {
let path = ecx.tcx.def_path_str(def);
bug!("trying to call extern function `{path}` at compile-time");
}
}
_ => Ok(ecx.tcx.instance_mir(instance)),
}
}
fn find_mir_or_eval_fn(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
instance: ty::Instance<'tcx>,
_abi: CallAbi,
args: &[OpTy<'tcx>],
dest: &PlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
_unwind: mir::UnwindAction, ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> {
debug!("find_mir_or_eval_fn: {:?}", instance);
if let ty::InstanceDef::Item(def) = instance.def {
if !ecx.tcx.is_const_fn_raw(def) {
if !ecx.tcx.is_const_default_method(def) {
throw_unsup_format!("calling non-const function `{}`", instance)
}
}
let Some(new_instance) = ecx.hook_special_const_fn(instance, args, dest, ret)? else {
return Ok(None);
};
if new_instance != instance {
return Ok(Self::find_mir_or_eval_fn(
ecx,
new_instance,
_abi,
args,
dest,
ret,
_unwind,
)?
.map(|(body, _instance)| (body, instance)));
}
}
Ok(Some((ecx.load_mir(instance.def, None)?, instance)))
}
fn call_intrinsic(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx>],
dest: &PlaceTy<'tcx, Self::Provenance>,
target: Option<mir::BasicBlock>,
_unwind: mir::UnwindAction,
) -> InterpResult<'tcx> {
if ecx.emulate_intrinsic(instance, args, dest, target)? {
return Ok(());
}
let intrinsic_name = ecx.tcx.item_name(instance.def_id());
let Some(ret) = target else {
throw_unsup_format!("intrinsic `{intrinsic_name}` is not supported at compile-time");
};
match intrinsic_name {
sym::ptr_guaranteed_cmp => {
let a = ecx.read_scalar(&args[0])?;
let b = ecx.read_scalar(&args[1])?;
let cmp = ecx.guaranteed_cmp(a, b)?;
ecx.write_scalar(Scalar::from_u8(cmp), dest)?;
}
sym::const_allocate => {
let size = ecx.read_scalar(&args[0])?.to_target_usize(ecx)?;
let align = ecx.read_scalar(&args[1])?.to_target_usize(ecx)?;
let align = match Align::from_bytes(align) {
Ok(a) => a,
Err(err) => throw_ub_format!("align has to be a power of 2, {}", err),
};
let ptr = ecx.allocate_ptr(
Size::from_bytes(size as u64),
align,
interpret::MemoryKind::Machine(MemoryKind::Heap),
)?;
ecx.write_pointer(ptr, dest)?;
}
sym::const_deallocate => {
let ptr = ecx.read_pointer(&args[0])?;
let size = ecx.read_scalar(&args[1])?.to_target_usize(ecx)?;
let align = ecx.read_scalar(&args[2])?.to_target_usize(ecx)?;
let size = Size::from_bytes(size);
let align = match Align::from_bytes(align) {
Ok(a) => a,
Err(err) => throw_ub_format!("align has to be a power of 2, {}", err),
};
let (alloc_id, _, _) = ecx.ptr_get_alloc_id(ptr)?;
let is_allocated_in_another_const = matches!(
ecx.tcx.try_get_global_alloc(alloc_id),
Some(interpret::GlobalAlloc::Memory(_))
);
if !is_allocated_in_another_const {
ecx.deallocate_ptr(
ptr,
Some((size, align)),
interpret::MemoryKind::Machine(MemoryKind::Heap),
)?;
}
}
_ => {
throw_unsup_format!(
"intrinsic `{intrinsic_name}` is not supported at compile-time"
);
}
}
ecx.go_to_block(ret);
Ok(())
}
fn assert_panic(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
msg: &AssertMessage<'tcx>,
_unwind: mir::UnwindAction,
) -> InterpResult<'tcx> {
use rustc_middle::mir::AssertKind::*;
let eval_to_int =
|op| ecx.read_immediate(&ecx.eval_operand(op, None)?).map(|x| x.to_const_int());
let err = match msg {
BoundsCheck { len, index } => {
let len = eval_to_int(len)?;
let index = eval_to_int(index)?;
BoundsCheck { len, index }
}
Overflow(op, l, r) => Overflow(*op, eval_to_int(l)?, eval_to_int(r)?),
OverflowNeg(op) => OverflowNeg(eval_to_int(op)?),
DivisionByZero(op) => DivisionByZero(eval_to_int(op)?),
RemainderByZero(op) => RemainderByZero(eval_to_int(op)?),
ResumedAfterReturn(generator_kind) => ResumedAfterReturn(*generator_kind),
ResumedAfterPanic(generator_kind) => ResumedAfterPanic(*generator_kind),
MisalignedPointerDereference { ref required, ref found } => {
MisalignedPointerDereference {
required: eval_to_int(required)?,
found: eval_to_int(found)?,
}
}
};
Err(ConstEvalErrKind::AssertFailure(err).into())
}
fn abort(_ecx: &mut InterpCx<'mir, 'tcx, Self>, msg: String) -> InterpResult<'tcx, !> {
Err(ConstEvalErrKind::Abort(msg).into())
}
fn binary_ptr_op(
ecx: &InterpCx<'mir, 'tcx, Self>,
bin_op: mir::BinOp,
left: &ImmTy<'tcx>,
right: &ImmTy<'tcx>,
) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> {
if bin_op == mir::BinOp::Offset {
let ptr = left.to_scalar().to_pointer(ecx)?;
let offset_count = right.to_scalar().to_target_isize(ecx)?;
let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty;
let offset_ptr = ecx.ptr_offset_inbounds(ptr, pointee_ty, offset_count)?;
return Ok((Scalar::from_maybe_pointer(offset_ptr, ecx), false, left.layout.ty));
}
throw_unsup_format!("pointer arithmetic or comparison is not supported at compile-time");
}
fn increment_const_eval_counter(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
if ecx.machine.steps_remaining == 0 {
return Ok(());
}
ecx.machine.steps_remaining -= 1;
if ecx.machine.steps_remaining == 0 {
throw_exhaust!(StepLimitReached)
}
Ok(())
}
#[inline(always)]
fn expose_ptr(
_ecx: &mut InterpCx<'mir, 'tcx, Self>,
_ptr: Pointer<AllocId>,
) -> InterpResult<'tcx> {
throw_unsup_format!("exposing pointers is not possible at compile-time")
}
#[inline(always)]
fn init_frame_extra(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
frame: Frame<'mir, 'tcx>,
) -> InterpResult<'tcx, Frame<'mir, 'tcx>> {
if !ecx.recursion_limit.value_within_limit(ecx.stack().len() + 1) {
throw_exhaust!(StackFrameLimitReached)
} else {
Ok(frame)
}
}
#[inline(always)]
fn stack<'a>(
ecx: &'a InterpCx<'mir, 'tcx, Self>,
) -> &'a [Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>] {
&ecx.machine.stack
}
#[inline(always)]
fn stack_mut<'a>(
ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
) -> &'a mut Vec<Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>> {
&mut ecx.machine.stack
}
fn before_access_global(
_tcx: TyCtxt<'tcx>,
machine: &Self,
alloc_id: AllocId,
alloc: ConstAllocation<'tcx>,
static_def_id: Option<DefId>,
is_write: bool,
) -> InterpResult<'tcx> {
let alloc = alloc.inner();
if is_write {
match alloc.mutability {
Mutability::Not => Err(err_ub!(WriteToReadOnly(alloc_id)).into()),
Mutability::Mut => Err(ConstEvalErrKind::ModifiedGlobal.into()),
}
} else {
if machine.can_access_statics {
Ok(())
} else if static_def_id.is_some() {
Err(ConstEvalErrKind::ConstAccessesStatic.into())
} else {
assert_eq!(alloc.mutability, Mutability::Not);
Ok(())
}
}
}
}