[EarlyIfConversion] Determine if branch is predictable using new APIs.

Author: mgudim

llvm/include/llvm/CodeGen/MachineLoopInfo.h

@@ -83,15 +83,17 @@ class MachineLoop : public LoopBase<MachineBasicBlock, MachineLoop> {
  /// ExcludeReg can be used to exclude the given register from the check
  /// i.e. when we're considering hoisting it's definition but not hoisted it
  /// yet
- bool isLoopInvariant(MachineInstr &I, const Register ExcludeReg = 0) const;
+ bool isLoopInvariant(const MachineInstr &I, const Register ExcludeReg = 0,
+ unsigned RecursionDepth = 1) const;
 
  void dump() const;
 
 private:
  friend class LoopInfoBase<MachineBasicBlock, MachineLoop>;
 
  /// Returns true if the given physreg has no defs inside the loop.
- bool isLoopInvariantImplicitPhysReg(Register Reg) const;
+ bool isLoopInvariantImplicitPhysReg(Register Reg, Register ExcludeReg,
+ unsigned RecursionDepth = 1) const;
 
  explicit MachineLoop(MachineBasicBlock *MBB)
  : LoopBase<MachineBasicBlock, MachineLoop>(MBB) {}

llvm/include/llvm/CodeGen/TargetInstrInfo.h

@@ -45,6 +45,7 @@ class InstrItineraryData;
 class LiveIntervals;
 class LiveVariables;
 class MachineLoop;
+class MachineLoopInfo;
 class MachineMemOperand;
 class MachineModuleInfo;
 class MachineRegisterInfo;
@@ -655,6 +656,17 @@ class TargetInstrInfo : public MCInstrInfo {
  return true;
  }
 
+ // Same as above but also if IsPredictable is non-null set IsPredictable to
+ // "true" if target considers this branch to be predictable and to false
+ // otherwise.
+ virtual bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
+ MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond,
+ bool *IsPredictable, const MachineLoopInfo *MLI,
+ bool AllowModify = false) const {
+ return analyzeBranch(MBB, TBB, FBB, Cond, AllowModify);
+ }
+
  /// Represents a predicate at the MachineFunction level. The control flow a
  /// MachineBranchPredicate represents is:
  ///

llvm/lib/CodeGen/EarlyIfConversion.cpp

@@ -83,6 +83,7 @@ class SSAIfConv {
  const TargetInstrInfo *TII;
  const TargetRegisterInfo *TRI;
  MachineRegisterInfo *MRI;
+ const MachineLoopInfo *MLI;
 
 public:
  /// The block containing the conditional branch.
@@ -121,6 +122,8 @@ class SSAIfConv {
 
  /// The branch condition determined by analyzeBranch.
  SmallVector<MachineOperand, 4> Cond;
+ /// Is branch predictable as determined by analyzeBranch.
+ bool IsPredictableBranch = false;
 
 private:
  /// Instructions in Head that define values used by the conditional blocks.
@@ -163,14 +166,15 @@ class SSAIfConv {
  void rewritePHIOperands();
 
 public:
- SSAIfConv(MachineFunction &MF) {
+ SSAIfConv(MachineFunction &MF, const MachineLoopInfo *MLI) {
  TII = MF.getSubtarget().getInstrInfo();
  TRI = MF.getSubtarget().getRegisterInfo();
  MRI = &MF.getRegInfo();
  LiveRegUnits.clear();
  LiveRegUnits.setUniverse(TRI->getNumRegUnits());
  ClobberedRegUnits.clear();
  ClobberedRegUnits.resize(TRI->getNumRegUnits());
+ this->MLI = MLI;
  }
 
  /// canConvertIf - If the sub-CFG headed by MBB can be if-converted,
@@ -484,7 +488,7 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
 
  // The branch we're looking to eliminate must be analyzable.
  Cond.clear();
- if (TII->analyzeBranch(*Head, TBB, FBB, Cond)) {
+ if (TII->analyzeBranch(*Head, TBB, FBB, Cond, &IsPredictableBranch, MLI)) {
  LLVM_DEBUG(dbgs() << "Branch not analyzable.\n");
  return false;
  }
@@ -874,33 +878,7 @@ bool EarlyIfConverter::shouldConvertIf(SSAIfConv &IfConv) {
  // Do not try to if-convert if the condition has a high chance of being
  // predictable.
  MachineLoop *CurrentLoop = Loops->getLoopFor(IfConv.Head);
- // If the condition is in a loop, consider it predictable if the condition
- // itself or all its operands are loop-invariant. E.g. this considers a load
- // from a loop-invariant address predictable; we were unable to prove that it
- // doesn't alias any of the memory-writes in the loop, but it is likely to
- // read to same value multiple times.
- if (CurrentLoop && any_of(IfConv.Cond, [&](MachineOperand &MO) {
- if (!MO.isReg() || !MO.isUse())
- return false;
- Register Reg = MO.getReg();
- if (Register::isPhysicalRegister(Reg))
- return false;
-
- MachineInstr *Def = MRI->getVRegDef(Reg);
- return CurrentLoop->isLoopInvariant(*Def) ||
- all_of(Def->operands(), [&](MachineOperand &Op) {
- if (Op.isImm())
- return true;
- if (!MO.isReg() || !MO.isUse())
- return false;
- Register Reg = MO.getReg();
- if (Register::isPhysicalRegister(Reg))
- return false;
-
- MachineInstr *Def = MRI->getVRegDef(Reg);
- return CurrentLoop->isLoopInvariant(*Def);
- });
- }))
+ if (CurrentLoop && IfConv.IsPredictableBranch)
  return false;
 
  if (!MinInstr)
@@ -1096,7 +1074,7 @@ bool EarlyIfConverter::runOnMachineFunction(MachineFunction &MF) {
  MinInstr = nullptr;
 
  bool Changed = false;
- SSAIfConv IfConv(MF);
+ SSAIfConv IfConv(MF, Loops);
 
  // Visit blocks in dominator tree post-order. The post-order enables nested
  // if-conversion in a single pass. The tryConvertIf() function may erase
@@ -1232,7 +1210,7 @@ bool EarlyIfPredicator::runOnMachineFunction(MachineFunction &MF) {
  MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
 
  bool Changed = false;
- SSAIfConv IfConv(MF);
+ SSAIfConv IfConv(MF, Loops);
 
  // Visit blocks in dominator tree post-order. The post-order enables nested
  // if-conversion in a single pass. The tryConvertIf() function may erase

llvm/lib/CodeGen/MachineLoopInfo.cpp

@@ -226,7 +226,8 @@ MDNode *MachineLoop::getLoopID() const {
  return LoopID;
 }
 
-bool MachineLoop::isLoopInvariantImplicitPhysReg(Register Reg) const {
+bool MachineLoop::isLoopInvariantImplicitPhysReg(
+ Register Reg, Register ExcludeReg, unsigned RecursionDepth) const {
  MachineFunction *MF = getHeader()->getParent();
  MachineRegisterInfo *MRI = &MF->getRegInfo();
 
@@ -238,15 +239,20 @@ bool MachineLoop::isLoopInvariantImplicitPhysReg(Register Reg) const {
  ->shouldAnalyzePhysregInMachineLoopInfo(Reg))
  return false;
 
- return !llvm::any_of(
- MRI->def_instructions(Reg),
- [this](const MachineInstr &MI) { return this->contains(&MI); });
+ return !llvm::any_of(MRI->def_instructions(Reg), [=](const MachineInstr &MI) {
+ return (this->contains(&MI) &&
+ !isLoopInvariant(MI, ExcludeReg, RecursionDepth - 1));
+ });
 }
 
-bool MachineLoop::isLoopInvariant(MachineInstr &I,
- const Register ExcludeReg) const {
- MachineFunction *MF = I.getParent()->getParent();
- MachineRegisterInfo *MRI = &MF->getRegInfo();
+bool MachineLoop::isLoopInvariant(const MachineInstr &I,
+ const Register ExcludeReg,
+ unsigned RecursionDepth) const {
+ if (RecursionDepth == 0)
+ return false;
+
+ const MachineFunction *MF = I.getParent()->getParent();
+ const MachineRegisterInfo *MRI = &MF->getRegInfo();
  const TargetSubtargetInfo &ST = MF->getSubtarget();
  const TargetRegisterInfo *TRI = ST.getRegisterInfo();
  const TargetInstrInfo *TII = ST.getInstrInfo();
@@ -271,7 +277,7 @@ bool MachineLoop::isLoopInvariant(MachineInstr &I,
  // it could get allocated to something with a def during allocation.
  // However, if the physreg is known to always be caller saved/restored
  // then this use is safe to hoist.
- if (!isLoopInvariantImplicitPhysReg(Reg) &&
+ if (!isLoopInvariantImplicitPhysReg(Reg, ExcludeReg, RecursionDepth) &&
  !(TRI->isCallerPreservedPhysReg(Reg.asMCReg(), *I.getMF())) &&
  !TII->isIgnorableUse(MO))
  return false;
@@ -293,9 +299,11 @@ bool MachineLoop::isLoopInvariant(MachineInstr &I,
  assert(MRI->getVRegDef(Reg) &&
  "Machine instr not mapped for this vreg?!");
 
- // If the loop contains the definition of an operand, then the instruction
- // isn't loop invariant.
- if (contains(MRI->getVRegDef(Reg)))
+ // If the loop contains the definition of an operand, then it must be loop
+ // invariant
+ MachineInstr *VRegDefMI = MRI->getVRegDef(Reg);
+ if (contains(VRegDefMI) &&
+ !isLoopInvariant(*VRegDefMI, ExcludeReg, RecursionDepth - 1))
  return false;
  }
 

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

@@ -29,6 +29,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
@@ -329,12 +330,27 @@ void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
  .addImm(16);
 }
 
-// Branch analysis.
+bool AArch64InstrInfo::isCondBranchPredictable(
+ const MachineInstr &CondBr, const MachineLoopInfo &MLI) const {
+ MachineLoop *Loop = MLI.getLoopFor(CondBr.getParent());
+ if (!Loop)
+ return false;
+ return Loop->isLoopInvariant(CondBr, /*ExcludeReg=*/0, /*RecursionDepth=*/2);
+}
+
 bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  MachineBasicBlock *&TBB,
  MachineBasicBlock *&FBB,
  SmallVectorImpl<MachineOperand> &Cond,
  bool AllowModify) const {
+ return analyzeBranch(MBB, TBB, FBB, Cond, nullptr, nullptr, AllowModify);
+}
+
+// Branch analysis.
+bool AArch64InstrInfo::analyzeBranch(
+ MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond, bool *IsPredictable,
+ const MachineLoopInfo *MLI, bool AllowModify) const {
  // If the block has no terminators, it just falls into the block after it.
  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
  if (I == MBB.end())
@@ -362,6 +378,8 @@ bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  if (isCondBranchOpcode(LastOpc)) {
  // Block ends with fall-through condbranch.
  parseCondBranch(LastInst, TBB, Cond);
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(*LastInst, *MLI);
  return false;
  }
  return true; // Can't handle indirect branch.
@@ -404,6 +422,8 @@ bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  if (isCondBranchOpcode(LastOpc)) {
  // Block ends with fall-through condbranch.
  parseCondBranch(LastInst, TBB, Cond);
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(*LastInst, *MLI);
  return false;
  }
  return true; // Can't handle indirect branch.
@@ -420,6 +440,8 @@ bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
  parseCondBranch(SecondLastInst, TBB, Cond);
  FBB = LastInst->getOperand(0).getMBB();
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(*SecondLastInst, *MLI);
  return false;
  }
 

llvm/lib/Target/AArch64/AArch64InstrInfo.h

@@ -384,10 +384,17 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
  MachineBasicBlock &RestoreBB, const DebugLoc &DL,
  int64_t BrOffset, RegScavenger *RS) const override;
 
+ bool isCondBranchPredictable(const MachineInstr &CondBr,
+ const MachineLoopInfo &MLI) const;
  bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
  MachineBasicBlock *&FBB,
  SmallVectorImpl<MachineOperand> &Cond,
  bool AllowModify = false) const override;
+ bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
+ MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond, bool *IsPredictable,
+ const MachineLoopInfo *MLI,
+ bool AllowModify = false) const override;
  bool analyzeBranchPredicate(MachineBasicBlock &MBB,
  MachineBranchPredicate &MBP,
  bool AllowModify) const override;

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
@@ -1255,13 +1256,28 @@ MCInst PPCInstrInfo::getNop() const {
 }
 
 // Branch analysis.
-// Note: If the condition register is set to CTR or CTR8 then this is a
-// BDNZ (imm == 1) or BDZ (imm == 0) branch.
+bool PPCInstrInfo::isCondBranchPredictable(const MachineInstr &CondBr,
+ const MachineLoopInfo &MLI) const {
+ MachineLoop *Loop = MLI.getLoopFor(CondBr.getParent());
+ if (!Loop)
+ return false;
+ return Loop->isLoopInvariant(CondBr, /*ExcludeReg=*/0, /*RecursionDepth=*/2);
+}
+
 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  MachineBasicBlock *&TBB,
  MachineBasicBlock *&FBB,
  SmallVectorImpl<MachineOperand> &Cond,
  bool AllowModify) const {
+ return analyzeBranch(MBB, TBB, FBB, Cond, nullptr, nullptr, AllowModify);
+}
+
+// Note: If the condition register is set to CTR or CTR8 then this is a
+// BDNZ (imm == 1) or BDZ (imm == 0) branch.
+bool PPCInstrInfo::analyzeBranch(
+ MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond, bool *IsPredictable,
+ const MachineLoopInfo *MLI, bool AllowModify) const {
  bool isPPC64 = Subtarget.isPPC64();
 
  // If the block has no terminators, it just falls into the block after it.
@@ -1303,6 +1319,8 @@ bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  TBB = LastInst.getOperand(2).getMBB();
  Cond.push_back(LastInst.getOperand(0));
  Cond.push_back(LastInst.getOperand(1));
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(LastInst, *MLI);
  return false;
  } else if (LastInst.getOpcode() == PPC::BC) {
  if (!LastInst.getOperand(1).isMBB())
@@ -1311,6 +1329,8 @@ bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  TBB = LastInst.getOperand(1).getMBB();
  Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
  Cond.push_back(LastInst.getOperand(0));
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(LastInst, *MLI);
  return false;
  } else if (LastInst.getOpcode() == PPC::BCn) {
  if (!LastInst.getOperand(1).isMBB())
@@ -1319,6 +1339,8 @@ bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  TBB = LastInst.getOperand(1).getMBB();
  Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
  Cond.push_back(LastInst.getOperand(0));
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(LastInst, *MLI);
  return false;
  } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
  LastInst.getOpcode() == PPC::BDNZ) {
@@ -1365,6 +1387,8 @@ bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  Cond.push_back(SecondLastInst.getOperand(0));
  Cond.push_back(SecondLastInst.getOperand(1));
  FBB = LastInst.getOperand(0).getMBB();
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(SecondLastInst, *MLI);
  return false;
  } else if (SecondLastInst.getOpcode() == PPC::BC &&
  LastInst.getOpcode() == PPC::B) {
@@ -1375,6 +1399,8 @@ bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
  Cond.push_back(SecondLastInst.getOperand(0));
  FBB = LastInst.getOperand(0).getMBB();
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(SecondLastInst, *MLI);
  return false;
  } else if (SecondLastInst.getOpcode() == PPC::BCn &&
  LastInst.getOpcode() == PPC::B) {
@@ -1385,6 +1411,8 @@ bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
  Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
  Cond.push_back(SecondLastInst.getOperand(0));
  FBB = LastInst.getOperand(0).getMBB();
+ if (IsPredictable && MLI)
+ *IsPredictable = isCondBranchPredictable(SecondLastInst, *MLI);
  return false;
  } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
  SecondLastInst.getOpcode() == PPC::BDNZ) &&

llvm/lib/Target/PowerPC/PPCInstrInfo.h

@@ -432,10 +432,17 @@ class PPCInstrInfo : public PPCGenInstrInfo {
 
 
  // Branch analysis.
+ bool isCondBranchPredictable(const MachineInstr &CondBr,
+ const MachineLoopInfo &MLI) const;
  bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
  MachineBasicBlock *&FBB,
  SmallVectorImpl<MachineOperand> &Cond,
  bool AllowModify) const override;
+ bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
+ MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond, bool *IsPredictable,
+ const MachineLoopInfo *MLI,
+ bool AllowModify) const override;
  unsigned removeBranch(MachineBasicBlock &MBB,
  int *BytesRemoved = nullptr) const override;
  unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,