[RDF] Use hash-based containers, cache extra information

This improves performance.

Author: Krzysztof Parzyszek

llvm/include/llvm/CodeGen/RDFLiveness.h

@@ -18,6 +18,8 @@
 #include "llvm/MC/LaneBitmask.h"
 #include <map>
 #include <set>
+#include <unordered_map>
+#include <unordered_set>
 #include <utility>
 
 namespace llvm {
@@ -28,6 +30,30 @@ class MachineDominatorTree;
 class MachineRegisterInfo;
 class TargetRegisterInfo;
 
+} // namespace llvm
+
+namespace llvm {
+namespace rdf {
+namespace detail {
+
+using NodeRef = std::pair<NodeId, LaneBitmask>;
+
+} // namespace detail
+} // namespace rdf
+} // namespace llvm
+
+namespace std {
+
+template <> struct hash<llvm::rdf::detail::NodeRef> {
+ std::size_t operator()(llvm::rdf::detail::NodeRef R) const {
+ return std::hash<llvm::rdf::NodeId>{}(R.first) ^
+ std::hash<llvm::LaneBitmask::Type>{}(R.second.getAsInteger());
+ }
+};
+
+} // namespace std
+
+namespace llvm {
 namespace rdf {
 
  struct Liveness {
@@ -46,10 +72,9 @@ namespace rdf {
  std::map<MachineBasicBlock*,RegisterAggr> Map;
  };
 
- using NodeRef = std::pair<NodeId, LaneBitmask>;
- using NodeRefSet = std::set<NodeRef>;
- // RegisterId in RefMap must be normalized.
- using RefMap = std::map<RegisterId, NodeRefSet>;
+ using NodeRef = detail::NodeRef;
+ using NodeRefSet = std::unordered_set<NodeRef>;
+ using RefMap = std::unordered_map<RegisterId, NodeRefSet>;
 
  Liveness(MachineRegisterInfo &mri, const DataFlowGraph &g)
  : DFG(g), TRI(g.getTRI()), PRI(g.getPRI()), MDT(g.getDT()),
@@ -110,15 +135,14 @@ namespace rdf {
  // Cache of mapping from node ids (for RefNodes) to the containing
  // basic blocks. Not computing it each time for each node reduces
  // the liveness calculation time by a large fraction.
- using NodeBlockMap = DenseMap<NodeId, MachineBasicBlock *>;
- NodeBlockMap NBMap;
+ DenseMap<NodeId, MachineBasicBlock *> NBMap;
 
  // Phi information:
  //
  // RealUseMap
  // map: NodeId -> (map: RegisterId -> NodeRefSet)
  // phi id -> (map: register -> set of reached non-phi uses)
- std::map<NodeId, RefMap> RealUseMap;
+ DenseMap<NodeId, RefMap> RealUseMap;
 
  // Inverse iterated dominance frontier.
  std::map<MachineBasicBlock*,std::set<MachineBasicBlock*>> IIDF;

llvm/include/llvm/CodeGen/RDFRegisters.h

@@ -91,6 +91,11 @@ namespace rdf {
  bool operator< (const RegisterRef &RR) const {
  return Reg < RR.Reg || (Reg == RR.Reg && Mask < RR.Mask);
  }
+
+ size_t hash() const {
+ return std::hash<RegisterId>{}(Reg) ^
+ std::hash<LaneBitmask::Type>{}(Mask.getAsInteger());
+ }
  };
 
 
@@ -171,10 +176,15 @@ namespace rdf {
  : Units(pri.getTRI().getNumRegUnits()), PRI(pri) {}
  RegisterAggr(const RegisterAggr &RG) = default;
 
+ unsigned count() const { return Units.count(); }
  bool empty() const { return Units.none(); }
  bool hasAliasOf(RegisterRef RR) const;
  bool hasCoverOf(RegisterRef RR) const;
 
+ bool operator==(const RegisterAggr &A) const {
+ return DenseMapInfo<BitVector>::isEqual(Units, A.Units);
+ }
+
  static bool isCoverOf(RegisterRef RA, RegisterRef RB,
  const PhysicalRegisterInfo &PRI) {
  return RegisterAggr(PRI).insert(RA).hasCoverOf(RB);
@@ -191,6 +201,10 @@ namespace rdf {
  RegisterRef clearIn(RegisterRef RR) const;
  RegisterRef makeRegRef() const;
 
+ size_t hash() const {
+ return DenseMapInfo<BitVector>::getHashValue(Units);
+ }
+
  void print(raw_ostream &OS) const;
 
  struct rr_iterator {
@@ -244,9 +258,26 @@ namespace rdf {
  LaneBitmask Mask;
  };
  raw_ostream &operator<< (raw_ostream &OS, const PrintLaneMaskOpt &P);
-
 } // end namespace rdf
 
 } // end namespace llvm
 
+namespace std {
+ template <> struct hash<llvm::rdf::RegisterRef> {
+ size_t operator()(llvm::rdf::RegisterRef A) const {
+ return A.hash();
+ }
+ };
+ template <> struct hash<llvm::rdf::RegisterAggr> {
+ size_t operator()(const llvm::rdf::RegisterAggr &A) const {
+ return A.hash();
+ }
+ };
+ template <> struct equal_to<llvm::rdf::RegisterAggr> {
+ bool operator()(const llvm::rdf::RegisterAggr &A,
+ const llvm::rdf::RegisterAggr &B) const {
+ return A == B;
+ }
+ };
+}
 #endif // LLVM_LIB_TARGET_HEXAGON_RDFREGISTERS_H

llvm/lib/CodeGen/RDFLiveness.cpp

@@ -47,6 +47,7 @@
 #include <cstdint>
 #include <iterator>
 #include <map>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -476,7 +477,7 @@ void Liveness::computePhiInfo() {
  // phi use -> (map: reaching phi -> set of registers defined in between)
  std::map<NodeId,std::map<NodeId,RegisterAggr>> PhiUp;
  std::vector<NodeId> PhiUQ; // Work list of phis for upward propagation.
- std::map<NodeId,RegisterAggr> PhiDRs; // Phi -> registers defined by it.
+ std::unordered_map<NodeId,RegisterAggr> PhiDRs; // Phi -> registers defined by it.
 
  // Go over all phis.
  for (NodeAddr<PhiNode*> PhiA : Phis) {
@@ -652,6 +653,23 @@ void Liveness::computePhiInfo() {
  // is covered, or until reaching the final phi. Only assume that the
  // reference reaches the phi in the latter case.
 
+ // The operation "clearIn" can be expensive. For a given set of intervening
+ // defs, cache the result of subtracting these defs from a given register
+ // ref.
+ using SubMap = std::unordered_map<RegisterRef, RegisterRef>;
+ std::unordered_map<RegisterAggr, SubMap> Subs;
+ auto ClearIn = [] (RegisterRef RR, const RegisterAggr &Mid, SubMap &SM) {
+ if (Mid.empty())
+ return RR;
+ auto F = SM.find(RR);
+ if (F != SM.end())
+ return F->second;
+ RegisterRef S = Mid.clearIn(RR);
+ SM.insert({RR, S});
+ return S;
+ };
+
+ // Go over all phis.
  for (unsigned i = 0; i < PhiUQ.size(); ++i) {
  auto PA = DFG.addr<PhiNode*>(PhiUQ[i]);
  NodeList PUs = PA.Addr->members_if(DFG.IsRef<NodeAttrs::Use>, DFG);
@@ -663,13 +681,13 @@ void Liveness::computePhiInfo() {
  for (const std::pair<const NodeId, RegisterAggr> &P : PUM) {
  bool Changed = false;
  const RegisterAggr &MidDefs = P.second;
-
  // Collect the set PropUp of uses that are reached by the current
  // phi PA, and are not covered by any intervening def between the
  // currently visited use UA and the upward phi P.
 
  if (MidDefs.hasCoverOf(UR))
  continue;
+ SubMap &SM = Subs[MidDefs];
 
  // General algorithm:
  // for each (R,U) : U is use node of R, U is reached by PA
@@ -689,7 +707,7 @@ void Liveness::computePhiInfo() {
  LaneBitmask M = R.Mask & V.second;
  if (M.none())
  continue;
- if (RegisterRef SS = MidDefs.clearIn(RegisterRef(R.Reg, M))) {
+ if (RegisterRef SS = ClearIn(RegisterRef(R.Reg, M), MidDefs, SM)) {
  NodeRefSet &RS = RealUseMap[P.first][SS.Reg];
  Changed |= RS.insert({V.first,SS.Mask}).second;
  }