use super::graph;
use graph::{BasicCoverageBlock, BcbBranch, CoverageGraph, TraverseCoverageGraphWithLoops};
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::graph::WithNumNodes;
use rustc_index::bit_set::BitSet;
use rustc_index::IndexVec;
use rustc_middle::mir::coverage::*;
use std::fmt::{self, Debug};
#[derive(Clone)]
pub(super) enum BcbCounter {
Counter { id: CounterId },
Expression { id: ExpressionId },
}
impl BcbCounter {
fn is_expression(&self) -> bool {
matches!(self, Self::Expression { .. })
}
pub(super) fn as_term(&self) -> CovTerm {
match *self {
BcbCounter::Counter { id, .. } => CovTerm::Counter(id),
BcbCounter::Expression { id, .. } => CovTerm::Expression(id),
}
}
}
impl Debug for BcbCounter {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Counter { id, .. } => write!(fmt, "Counter({:?})", id.index()),
Self::Expression { id } => write!(fmt, "Expression({:?})", id.index()),
}
}
}
pub(super) struct CoverageCounters {
next_counter_id: CounterId,
bcb_counters: IndexVec<BasicCoverageBlock, Option<BcbCounter>>,
bcb_edge_counters: FxHashMap<(BasicCoverageBlock, BasicCoverageBlock), BcbCounter>,
bcb_has_incoming_edge_counters: BitSet<BasicCoverageBlock>,
expressions: IndexVec<ExpressionId, Expression>,
}
impl CoverageCounters {
pub(super) fn new(basic_coverage_blocks: &CoverageGraph) -> Self {
let num_bcbs = basic_coverage_blocks.num_nodes();
Self {
next_counter_id: CounterId::START,
bcb_counters: IndexVec::from_elem_n(None, num_bcbs),
bcb_edge_counters: FxHashMap::default(),
bcb_has_incoming_edge_counters: BitSet::new_empty(num_bcbs),
expressions: IndexVec::new(),
}
}
pub fn make_bcb_counters(
&mut self,
basic_coverage_blocks: &CoverageGraph,
bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool,
) {
MakeBcbCounters::new(self, basic_coverage_blocks).make_bcb_counters(bcb_has_coverage_spans)
}
fn make_counter(&mut self) -> BcbCounter {
let id = self.next_counter();
BcbCounter::Counter { id }
}
fn make_expression(&mut self, lhs: CovTerm, op: Op, rhs: CovTerm) -> BcbCounter {
let id = self.expressions.push(Expression { lhs, op, rhs });
BcbCounter::Expression { id }
}
fn next_counter(&mut self) -> CounterId {
let next = self.next_counter_id;
self.next_counter_id = self.next_counter_id + 1;
next
}
pub(super) fn num_counters(&self) -> usize {
self.next_counter_id.as_usize()
}
#[cfg(test)]
pub(super) fn num_expressions(&self) -> usize {
self.expressions.len()
}
fn set_bcb_counter(&mut self, bcb: BasicCoverageBlock, counter_kind: BcbCounter) -> CovTerm {
assert!(
counter_kind.is_expression() || !self.bcb_has_incoming_edge_counters.contains(bcb),
"attempt to add a `Counter` to a BCB target with existing incoming edge counters"
);
let term = counter_kind.as_term();
if let Some(replaced) = self.bcb_counters[bcb].replace(counter_kind) {
bug!(
"attempt to set a BasicCoverageBlock coverage counter more than once; \
{bcb:?} already had counter {replaced:?}",
);
} else {
term
}
}
fn set_bcb_edge_counter(
&mut self,
from_bcb: BasicCoverageBlock,
to_bcb: BasicCoverageBlock,
counter_kind: BcbCounter,
) -> CovTerm {
if let Some(node_counter) = self.bcb_counter(to_bcb) && !node_counter.is_expression() {
bug!(
"attempt to add an incoming edge counter from {from_bcb:?} \
when the target BCB already has {node_counter:?}"
);
}
self.bcb_has_incoming_edge_counters.insert(to_bcb);
let term = counter_kind.as_term();
if let Some(replaced) = self.bcb_edge_counters.insert((from_bcb, to_bcb), counter_kind) {
bug!(
"attempt to set an edge counter more than once; from_bcb: \
{from_bcb:?} already had counter {replaced:?}",
);
} else {
term
}
}
pub(super) fn bcb_counter(&self, bcb: BasicCoverageBlock) -> Option<&BcbCounter> {
self.bcb_counters[bcb].as_ref()
}
pub(super) fn bcb_node_counters(
&self,
) -> impl Iterator<Item = (BasicCoverageBlock, &BcbCounter)> {
self.bcb_counters
.iter_enumerated()
.filter_map(|(bcb, counter_kind)| Some((bcb, counter_kind.as_ref()?)))
}
pub(super) fn bcb_edge_counters(
&self,
) -> impl Iterator<Item = (BasicCoverageBlock, BasicCoverageBlock, &BcbCounter)> {
self.bcb_edge_counters
.iter()
.map(|(&(from_bcb, to_bcb), counter_kind)| (from_bcb, to_bcb, counter_kind))
}
pub(super) fn take_expressions(&mut self) -> IndexVec<ExpressionId, Expression> {
std::mem::take(&mut self.expressions)
}
}
struct MakeBcbCounters<'a> {
coverage_counters: &'a mut CoverageCounters,
basic_coverage_blocks: &'a CoverageGraph,
}
impl<'a> MakeBcbCounters<'a> {
fn new(
coverage_counters: &'a mut CoverageCounters,
basic_coverage_blocks: &'a CoverageGraph,
) -> Self {
Self { coverage_counters, basic_coverage_blocks }
}
fn make_bcb_counters(&mut self, bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool) {
debug!("make_bcb_counters(): adding a counter or expression to each BasicCoverageBlock");
let mut traversal = TraverseCoverageGraphWithLoops::new(&self.basic_coverage_blocks);
while let Some(bcb) = traversal.next() {
if bcb_has_coverage_spans(bcb) {
debug!("{:?} has at least one coverage span. Get or make its counter", bcb);
let branching_counter_operand = self.get_or_make_counter_operand(bcb);
if self.bcb_needs_branch_counters(bcb) {
self.make_branch_counters(&traversal, bcb, branching_counter_operand);
}
} else {
debug!(
"{:?} does not have any coverage spans. A counter will only be added if \
and when a covered BCB has an expression dependency.",
bcb,
);
}
}
assert!(
traversal.is_complete(),
"`TraverseCoverageGraphWithLoops` missed some `BasicCoverageBlock`s: {:?}",
traversal.unvisited(),
);
}
fn make_branch_counters(
&mut self,
traversal: &TraverseCoverageGraphWithLoops<'_>,
branching_bcb: BasicCoverageBlock,
branching_counter_operand: CovTerm,
) {
let branches = self.bcb_branches(branching_bcb);
debug!(
"{:?} has some branch(es) without counters:\n {}",
branching_bcb,
branches
.iter()
.map(|branch| { format!("{:?}: {:?}", branch, self.branch_counter(branch)) })
.collect::<Vec<_>>()
.join("\n "),
);
let expression_branch = self.choose_preferred_expression_branch(traversal, &branches);
let mut some_sumup_counter_operand = None;
for branch in branches {
if branch != expression_branch {
let branch_counter_operand = if branch.is_only_path_to_target() {
debug!(
" {:?} has only one incoming edge (from {:?}), so adding a \
counter",
branch, branching_bcb
);
self.get_or_make_counter_operand(branch.target_bcb)
} else {
debug!(" {:?} has multiple incoming edges, so adding an edge counter", branch);
self.get_or_make_edge_counter_operand(branching_bcb, branch.target_bcb)
};
if let Some(sumup_counter_operand) =
some_sumup_counter_operand.replace(branch_counter_operand)
{
let intermediate_expression = self.coverage_counters.make_expression(
branch_counter_operand,
Op::Add,
sumup_counter_operand,
);
debug!(" [new intermediate expression: {:?}]", intermediate_expression);
let intermediate_expression_operand = intermediate_expression.as_term();
some_sumup_counter_operand.replace(intermediate_expression_operand);
}
}
}
let sumup_counter_operand =
some_sumup_counter_operand.expect("sumup_counter_operand should have a value");
debug!(
"Making an expression for the selected expression_branch: {:?} \
(expression_branch predecessors: {:?})",
expression_branch,
self.bcb_predecessors(expression_branch.target_bcb),
);
let expression = self.coverage_counters.make_expression(
branching_counter_operand,
Op::Subtract,
sumup_counter_operand,
);
debug!("{:?} gets an expression: {:?}", expression_branch, expression);
let bcb = expression_branch.target_bcb;
if expression_branch.is_only_path_to_target() {
self.coverage_counters.set_bcb_counter(bcb, expression);
} else {
self.coverage_counters.set_bcb_edge_counter(branching_bcb, bcb, expression);
}
}
#[instrument(level = "debug", skip(self))]
fn get_or_make_counter_operand(&mut self, bcb: BasicCoverageBlock) -> CovTerm {
if let Some(counter_kind) = &self.coverage_counters.bcb_counters[bcb] {
debug!("{bcb:?} already has a counter: {counter_kind:?}");
return counter_kind.as_term();
}
let one_path_to_target = self.bcb_has_one_path_to_target(bcb);
if one_path_to_target || self.bcb_predecessors(bcb).contains(&bcb) {
let counter_kind = self.coverage_counters.make_counter();
if one_path_to_target {
debug!("{bcb:?} gets a new counter: {counter_kind:?}");
} else {
debug!(
"{bcb:?} has itself as its own predecessor. It can't be part of its own \
Expression sum, so it will get its own new counter: {counter_kind:?}. \
(Note, the compiled code will generate an infinite loop.)",
);
}
return self.coverage_counters.set_bcb_counter(bcb, counter_kind);
}
let _sumup_debug_span = debug_span!("(preparing sum-up expression)").entered();
let mut predecessors = self.bcb_predecessors(bcb).to_owned().into_iter();
let first_edge_counter_operand =
self.get_or_make_edge_counter_operand(predecessors.next().unwrap(), bcb);
let mut some_sumup_edge_counter_operand = None;
for predecessor in predecessors {
let edge_counter_operand = self.get_or_make_edge_counter_operand(predecessor, bcb);
if let Some(sumup_edge_counter_operand) =
some_sumup_edge_counter_operand.replace(edge_counter_operand)
{
let intermediate_expression = self.coverage_counters.make_expression(
sumup_edge_counter_operand,
Op::Add,
edge_counter_operand,
);
debug!("new intermediate expression: {intermediate_expression:?}");
let intermediate_expression_operand = intermediate_expression.as_term();
some_sumup_edge_counter_operand.replace(intermediate_expression_operand);
}
}
let counter_kind = self.coverage_counters.make_expression(
first_edge_counter_operand,
Op::Add,
some_sumup_edge_counter_operand.unwrap(),
);
drop(_sumup_debug_span);
debug!("{bcb:?} gets a new counter (sum of predecessor counters): {counter_kind:?}");
self.coverage_counters.set_bcb_counter(bcb, counter_kind)
}
#[instrument(level = "debug", skip(self))]
fn get_or_make_edge_counter_operand(
&mut self,
from_bcb: BasicCoverageBlock,
to_bcb: BasicCoverageBlock,
) -> CovTerm {
let successors = self.bcb_successors(from_bcb).iter();
if successors.len() == 1 {
return self.get_or_make_counter_operand(from_bcb);
}
if let Some(counter_kind) =
self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb))
{
debug!("Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}");
return counter_kind.as_term();
}
let counter_kind = self.coverage_counters.make_counter();
debug!("Edge {from_bcb:?}->{to_bcb:?} gets a new counter: {counter_kind:?}");
self.coverage_counters.set_bcb_edge_counter(from_bcb, to_bcb, counter_kind)
}
fn choose_preferred_expression_branch(
&self,
traversal: &TraverseCoverageGraphWithLoops<'_>,
branches: &[BcbBranch],
) -> BcbBranch {
let good_reloop_branch = self.find_good_reloop_branch(traversal, &branches);
if let Some(reloop_branch) = good_reloop_branch {
assert!(self.branch_has_no_counter(&reloop_branch));
debug!("Selecting reloop branch {reloop_branch:?} to get an expression");
reloop_branch
} else {
let &branch_without_counter =
branches.iter().find(|&branch| self.branch_has_no_counter(branch)).expect(
"needs_branch_counters was `true` so there should be at least one \
branch",
);
debug!(
"Selecting any branch={:?} that still needs a counter, to get the \
`Expression` because there was no `reloop_branch`, or it already had a \
counter",
branch_without_counter
);
branch_without_counter
}
}
fn find_good_reloop_branch(
&self,
traversal: &TraverseCoverageGraphWithLoops<'_>,
branches: &[BcbBranch],
) -> Option<BcbBranch> {
for reloop_bcbs in traversal.reloop_bcbs_per_loop() {
let mut all_branches_exit_this_loop = true;
for &branch in branches {
let is_reloop_branch = reloop_bcbs.iter().any(|&reloop_bcb| {
self.basic_coverage_blocks.dominates(branch.target_bcb, reloop_bcb)
});
if is_reloop_branch {
all_branches_exit_this_loop = false;
if self.branch_has_no_counter(&branch) {
return Some(branch);
}
} else {
}
}
if !all_branches_exit_this_loop {
debug!("All reloop branches had counters; skip checking the other loops");
return None;
}
}
None
}
#[inline]
fn bcb_predecessors(&self, bcb: BasicCoverageBlock) -> &[BasicCoverageBlock] {
&self.basic_coverage_blocks.predecessors[bcb]
}
#[inline]
fn bcb_successors(&self, bcb: BasicCoverageBlock) -> &[BasicCoverageBlock] {
&self.basic_coverage_blocks.successors[bcb]
}
#[inline]
fn bcb_branches(&self, from_bcb: BasicCoverageBlock) -> Vec<BcbBranch> {
self.bcb_successors(from_bcb)
.iter()
.map(|&to_bcb| BcbBranch::from_to(from_bcb, to_bcb, &self.basic_coverage_blocks))
.collect::<Vec<_>>()
}
fn bcb_needs_branch_counters(&self, bcb: BasicCoverageBlock) -> bool {
let branch_needs_a_counter = |branch: &BcbBranch| self.branch_has_no_counter(branch);
let branches = self.bcb_branches(bcb);
branches.len() > 1 && branches.iter().any(branch_needs_a_counter)
}
fn branch_has_no_counter(&self, branch: &BcbBranch) -> bool {
self.branch_counter(branch).is_none()
}
fn branch_counter(&self, branch: &BcbBranch) -> Option<&BcbCounter> {
let to_bcb = branch.target_bcb;
if let Some(from_bcb) = branch.edge_from_bcb {
self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb))
} else {
self.coverage_counters.bcb_counters[to_bcb].as_ref()
}
}
#[inline]
fn bcb_has_one_path_to_target(&self, bcb: BasicCoverageBlock) -> bool {
self.bcb_predecessors(bcb).len() <= 1
}
}