use std::collections::hash_map::Entry;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_hir::def::DefKind;
use rustc_hir::def_id::{DefId, CRATE_DEF_INDEX, LOCAL_CRATE};
use rustc_hir::definitions::DefPathDataName;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::middle::exported_symbols::SymbolExportLevel;
use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, Linkage, Visibility};
use rustc_middle::mir::mono::{InstantiationMode, MonoItem};
use rustc_middle::ty::print::characteristic_def_id_of_type;
use rustc_middle::ty::{self, DefIdTree, InstanceDef, TyCtxt};
use rustc_span::symbol::Symbol;
use super::PartitioningCx;
use crate::monomorphize::collector::InliningMap;
use crate::monomorphize::partitioning::merging;
use crate::monomorphize::partitioning::{
MonoItemPlacement, Partitioner, PostInliningPartitioning, PreInliningPartitioning,
};
pub struct DefaultPartitioning;
impl<'tcx> Partitioner<'tcx> for DefaultPartitioning {
fn place_root_mono_items(
&mut self,
cx: &PartitioningCx<'_, 'tcx>,
mono_items: &mut dyn Iterator<Item = MonoItem<'tcx>>,
) -> PreInliningPartitioning<'tcx> {
let mut roots = FxHashSet::default();
let mut codegen_units = FxHashMap::default();
let is_incremental_build = cx.tcx.sess.opts.incremental.is_some();
let mut internalization_candidates = FxHashSet::default();
let export_generics =
cx.tcx.sess.opts.share_generics() && cx.tcx.local_crate_exports_generics();
let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx);
let cgu_name_cache = &mut FxHashMap::default();
for mono_item in mono_items {
match mono_item.instantiation_mode(cx.tcx) {
InstantiationMode::GloballyShared { .. } => {}
InstantiationMode::LocalCopy => continue,
}
let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item);
let is_volatile = is_incremental_build && mono_item.is_generic_fn();
let codegen_unit_name = match characteristic_def_id {
Some(def_id) => compute_codegen_unit_name(
cx.tcx,
cgu_name_builder,
def_id,
is_volatile,
cgu_name_cache,
),
None => fallback_cgu_name(cgu_name_builder),
};
let codegen_unit = codegen_units
.entry(codegen_unit_name)
.or_insert_with(|| CodegenUnit::new(codegen_unit_name));
let mut can_be_internalized = true;
let (linkage, visibility) = mono_item_linkage_and_visibility(
cx.tcx,
&mono_item,
&mut can_be_internalized,
export_generics,
);
if visibility == Visibility::Hidden && can_be_internalized {
internalization_candidates.insert(mono_item);
}
codegen_unit.items_mut().insert(mono_item, (linkage, visibility));
roots.insert(mono_item);
}
if codegen_units.is_empty() {
let codegen_unit_name = fallback_cgu_name(cgu_name_builder);
codegen_units.insert(codegen_unit_name, CodegenUnit::new(codegen_unit_name));
}
PreInliningPartitioning {
codegen_units: codegen_units
.into_iter()
.map(|(_, codegen_unit)| codegen_unit)
.collect(),
roots,
internalization_candidates,
}
}
fn merge_codegen_units(
&mut self,
cx: &PartitioningCx<'_, 'tcx>,
initial_partitioning: &mut PreInliningPartitioning<'tcx>,
) {
merging::merge_codegen_units(cx, initial_partitioning);
}
fn place_inlined_mono_items(
&mut self,
cx: &PartitioningCx<'_, 'tcx>,
initial_partitioning: PreInliningPartitioning<'tcx>,
) -> PostInliningPartitioning<'tcx> {
let mut new_partitioning = Vec::new();
let mut mono_item_placements = FxHashMap::default();
let PreInliningPartitioning {
codegen_units: initial_cgus,
roots,
internalization_candidates,
} = initial_partitioning;
let single_codegen_unit = initial_cgus.len() == 1;
for old_codegen_unit in initial_cgus {
let mut reachable = FxHashSet::default();
for root in old_codegen_unit.items().keys() {
follow_inlining(*root, cx.inlining_map, &mut reachable);
}
let mut new_codegen_unit = CodegenUnit::new(old_codegen_unit.name());
for mono_item in reachable {
if let Some(linkage) = old_codegen_unit.items().get(&mono_item) {
new_codegen_unit.items_mut().insert(mono_item, *linkage);
} else {
if roots.contains(&mono_item) {
bug!(
"GloballyShared mono-item inlined into other CGU: \
{:?}",
mono_item
);
}
new_codegen_unit
.items_mut()
.insert(mono_item, (Linkage::Internal, Visibility::Default));
}
if !single_codegen_unit {
match mono_item_placements.entry(mono_item) {
Entry::Occupied(e) => {
let placement = e.into_mut();
debug_assert!(match *placement {
MonoItemPlacement::SingleCgu { cgu_name } => {
cgu_name != new_codegen_unit.name()
}
MonoItemPlacement::MultipleCgus => true,
});
*placement = MonoItemPlacement::MultipleCgus;
}
Entry::Vacant(e) => {
e.insert(MonoItemPlacement::SingleCgu {
cgu_name: new_codegen_unit.name(),
});
}
}
}
}
new_partitioning.push(new_codegen_unit);
}
return PostInliningPartitioning {
codegen_units: new_partitioning,
mono_item_placements,
internalization_candidates,
};
fn follow_inlining<'tcx>(
mono_item: MonoItem<'tcx>,
inlining_map: &InliningMap<'tcx>,
visited: &mut FxHashSet<MonoItem<'tcx>>,
) {
if !visited.insert(mono_item) {
return;
}
inlining_map.with_inlining_candidates(mono_item, |target| {
follow_inlining(target, inlining_map, visited);
});
}
}
fn internalize_symbols(
&mut self,
cx: &PartitioningCx<'_, 'tcx>,
partitioning: &mut PostInliningPartitioning<'tcx>,
) {
if partitioning.codegen_units.len() == 1 {
for cgu in &mut partitioning.codegen_units {
for candidate in &partitioning.internalization_candidates {
cgu.items_mut().insert(*candidate, (Linkage::Internal, Visibility::Default));
}
}
return;
}
let mut accessor_map: FxHashMap<MonoItem<'tcx>, Vec<MonoItem<'tcx>>> = Default::default();
cx.inlining_map.iter_accesses(|accessor, accessees| {
for accessee in accessees {
accessor_map.entry(*accessee).or_default().push(accessor);
}
});
let mono_item_placements = &partitioning.mono_item_placements;
for cgu in &mut partitioning.codegen_units {
let home_cgu = MonoItemPlacement::SingleCgu { cgu_name: cgu.name() };
for (accessee, linkage_and_visibility) in cgu.items_mut() {
if !partitioning.internalization_candidates.contains(accessee) {
continue;
}
debug_assert_eq!(mono_item_placements[accessee], home_cgu);
if let Some(accessors) = accessor_map.get(accessee) {
if accessors
.iter()
.filter_map(|accessor| {
mono_item_placements.get(accessor)
})
.any(|placement| *placement != home_cgu)
{
continue;
}
}
*linkage_and_visibility = (Linkage::Internal, Visibility::Default);
}
}
}
}
fn characteristic_def_id_of_mono_item<'tcx>(
tcx: TyCtxt<'tcx>,
mono_item: MonoItem<'tcx>,
) -> Option<DefId> {
match mono_item {
MonoItem::Fn(instance) => {
let def_id = match instance.def {
ty::InstanceDef::Item(def) => def.did,
ty::InstanceDef::VtableShim(..)
| ty::InstanceDef::ReifyShim(..)
| ty::InstanceDef::FnPtrShim(..)
| ty::InstanceDef::ClosureOnceShim { .. }
| ty::InstanceDef::Intrinsic(..)
| ty::InstanceDef::DropGlue(..)
| ty::InstanceDef::Virtual(..)
| ty::InstanceDef::CloneShim(..) => return None,
};
if tcx.trait_of_item(def_id).is_some() {
let self_ty = instance.substs.type_at(0);
return characteristic_def_id_of_type(self_ty).or(Some(def_id));
}
if let Some(impl_def_id) = tcx.impl_of_method(def_id) {
if tcx.sess.opts.incremental.is_some()
&& tcx.trait_id_of_impl(impl_def_id) == tcx.lang_items().drop_trait()
{
return None;
}
let impl_self_ty = tcx.subst_and_normalize_erasing_regions(
instance.substs,
ty::ParamEnv::reveal_all(),
tcx.type_of(impl_def_id),
);
if let Some(def_id) = characteristic_def_id_of_type(impl_self_ty) {
return Some(def_id);
}
}
Some(def_id)
}
MonoItem::Static(def_id) => Some(def_id),
MonoItem::GlobalAsm(hir_id) => Some(tcx.hir().local_def_id(hir_id).to_def_id()),
}
}
fn compute_codegen_unit_name(
tcx: TyCtxt<'_>,
name_builder: &mut CodegenUnitNameBuilder<'_>,
def_id: DefId,
volatile: bool,
cache: &mut CguNameCache,
) -> Symbol {
let mut current_def_id = def_id;
let mut cgu_def_id = None;
loop {
if current_def_id.index == CRATE_DEF_INDEX {
if cgu_def_id.is_none() {
cgu_def_id = Some(DefId { krate: def_id.krate, index: CRATE_DEF_INDEX });
}
break;
} else if tcx.def_kind(current_def_id) == DefKind::Mod {
if cgu_def_id.is_none() {
cgu_def_id = Some(current_def_id);
}
} else {
cgu_def_id = None;
}
current_def_id = tcx.parent(current_def_id).unwrap();
}
let cgu_def_id = cgu_def_id.unwrap();
*cache.entry((cgu_def_id, volatile)).or_insert_with(|| {
let def_path = tcx.def_path(cgu_def_id);
let components = def_path.data.iter().map(|part| match part.data.name() {
DefPathDataName::Named(name) => name,
DefPathDataName::Anon { .. } => unreachable!(),
});
let volatile_suffix = volatile.then_some("volatile");
name_builder.build_cgu_name(def_path.krate, components, volatile_suffix)
})
}
fn fallback_cgu_name(name_builder: &mut CodegenUnitNameBuilder<'_>) -> Symbol {
name_builder.build_cgu_name(LOCAL_CRATE, &["fallback"], Some("cgu"))
}
fn mono_item_linkage_and_visibility(
tcx: TyCtxt<'tcx>,
mono_item: &MonoItem<'tcx>,
can_be_internalized: &mut bool,
export_generics: bool,
) -> (Linkage, Visibility) {
if let Some(explicit_linkage) = mono_item.explicit_linkage(tcx) {
return (explicit_linkage, Visibility::Default);
}
let vis = mono_item_visibility(tcx, mono_item, can_be_internalized, export_generics);
(Linkage::External, vis)
}
type CguNameCache = FxHashMap<(DefId, bool), Symbol>;
fn mono_item_visibility(
tcx: TyCtxt<'tcx>,
mono_item: &MonoItem<'tcx>,
can_be_internalized: &mut bool,
export_generics: bool,
) -> Visibility {
let instance = match mono_item {
MonoItem::Fn(instance) => instance,
MonoItem::Static(def_id) => {
return if tcx.is_reachable_non_generic(*def_id) {
*can_be_internalized = false;
default_visibility(tcx, *def_id, false)
} else {
Visibility::Hidden
};
}
MonoItem::GlobalAsm(hir_id) => {
let def_id = tcx.hir().local_def_id(*hir_id);
return if tcx.is_reachable_non_generic(def_id) {
*can_be_internalized = false;
default_visibility(tcx, def_id.to_def_id(), false)
} else {
Visibility::Hidden
};
}
};
let def_id = match instance.def {
InstanceDef::Item(def) => def.did,
InstanceDef::DropGlue(def_id, Some(_)) => def_id,
InstanceDef::VtableShim(..)
| InstanceDef::ReifyShim(..)
| InstanceDef::FnPtrShim(..)
| InstanceDef::Virtual(..)
| InstanceDef::Intrinsic(..)
| InstanceDef::ClosureOnceShim { .. }
| InstanceDef::DropGlue(..)
| InstanceDef::CloneShim(..) => return Visibility::Hidden,
};
if tcx.lang_items().start_fn() == Some(def_id) {
*can_be_internalized = false;
return Visibility::Hidden;
}
let is_generic = instance.substs.non_erasable_generics().next().is_some();
if !def_id.is_local() {
return if export_generics && is_generic {
*can_be_internalized = false;
default_visibility(tcx, def_id, true)
} else {
Visibility::Hidden
};
}
if is_generic {
if export_generics {
if tcx.is_unreachable_local_definition(def_id) {
Visibility::Hidden
} else {
*can_be_internalized = false;
default_visibility(tcx, def_id, true)
}
} else {
Visibility::Hidden
}
} else {
if tcx.is_reachable_non_generic(def_id) {
*can_be_internalized = false;
debug_assert!(!is_generic);
return default_visibility(tcx, def_id, false);
}
let attrs = tcx.codegen_fn_attrs(def_id);
if attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL) {
*can_be_internalized = false;
}
Visibility::Hidden
}
}
fn default_visibility(tcx: TyCtxt<'_>, id: DefId, is_generic: bool) -> Visibility {
if !tcx.sess.target.default_hidden_visibility {
return Visibility::Default;
}
if is_generic {
return Visibility::Hidden;
}
if !id.is_local() {
return Visibility::Hidden;
}
match tcx.reachable_non_generics(id.krate).get(&id) {
Some(SymbolExportLevel::C) => Visibility::Default,
_ => Visibility::Hidden,
}
}