[TTI] Use MemIntrinsicCostAttributes for getExpandCompressMemoryOpCost

[arcbbb]

• Following [TTI] Use MemIntrinsicCostAttributes for getMaskedMemoryOpCost #168029. This is a step toward a unified interface for masked/gather-scatter/strided/expand-compress cost modeling.
• Replace the ad-hoc parameter list with a single attributes object.

API change:

- InstructionCost getExpandCompressMemoryOpCost(Opcode, DataTy, - VariableMask, Alignment, - CostKind, Inst); + InstructionCost getExpandCompressMemoryOpCost(MemIntrinsicCostAttributes, + CostKind); 

Notes:

• NFCI intended: callers populate MemIntrinsicCostAttributes with same information as before.

[llvmbot]

@llvm/pr-subscribers-llvm-analysis

@llvm/pr-subscribers-backend-risc-v

Author: Shih-Po Hung (arcbbb)

Changes

• Following #168029. This is a step toward a unified interface for masked/gather-scatter/strided/expand-compress cost modeling.
• Replace the ad-hoc parameter list with a single attributes object.

API change:

- InstructionCost getExpandCompressMemoryOpCost(Opcode, DataTy, - VariableMask, Alignment, - CostKind, Inst); + InstructionCost getExpandCompressMemoryOpCost(MemIntrinsicCostAttributes, + CostKind); 

Notes:

• NFCI intended: callers populate MemIntrinsicCostAttributes with same information as before.

---

Full diff: https://github.com/llvm/llvm-project/pull/168677.diff

6 Files Affected:

• (modified) llvm/include/llvm/Analysis/TargetTransformInfo.h (+18-10)
• (modified) llvm/include/llvm/Analysis/TargetTransformInfoImpl.h (+3-3)
• (modified) llvm/include/llvm/CodeGen/BasicTTIImpl.h (+11-9)
• (modified) llvm/lib/Analysis/TargetTransformInfo.cpp (+3-4)
• (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp (+9-4)
• (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h (+2-3)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index a65e4667ab76c..b25c285881586 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -125,12 +125,20 @@ struct HardwareLoopInfo { /// Information for memory intrinsic cost model. class MemIntrinsicCostAttributes { + /// Optional context instruction, if one exists, e.g. the + /// load/store to transform to the intrinsic. + const Instruction *I = nullptr; + /// Vector type of the data to be loaded or stored. Type *DataTy = nullptr; /// ID of the memory intrinsic. Intrinsic::ID IID; + /// True when the memory access is predicated with a mask + /// that is not a compile-time constant. + bool VariableMask = true; + /// Address space of the pointer. unsigned AddressSpace = 0; @@ -143,8 +151,16 @@ class MemIntrinsicCostAttributes { : DataTy(DataTy), IID(Id), AddressSpace(AddressSpace), Alignment(Alignment) {} + LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, + bool VariableMask, Align Alignment, + const Instruction *I = nullptr) + : I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask), + Alignment(Alignment) {} + Intrinsic::ID getID() const { return IID; } + const Instruction *getInst() const { return I; } Type *getDataType() const { return DataTy; } + bool getVariableMask() const { return VariableMask; } unsigned getAddressSpace() const { return AddressSpace; } Align getAlignment() const { return Alignment; } }; @@ -1600,17 +1616,9 @@ class TargetTransformInfo { const Instruction *I = nullptr) const; /// \return The cost of Expand Load or Compress Store operation - /// \p Opcode - is a type of memory access Load or Store - /// \p Src - a vector type of the data to be loaded or stored - /// \p VariableMask - true when the memory access is predicated with a mask - /// that is not a compile-time constant - /// \p Alignment - alignment of single element - /// \p I - the optional original context instruction, if one exists, e.g. the - /// load/store to transform or the call to the gather/scatter intrinsic LLVM_ABI InstructionCost getExpandCompressMemoryOpCost( - unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput, - const Instruction *I = nullptr) const; + const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const; /// \return The cost of strided memory operations. /// \p Opcode - is a type of memory access Load or Store diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index d8e35748f53e5..86a785f700eab 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -855,9 +855,9 @@ class TargetTransformInfoImplBase { return 1; } - virtual InstructionCost getExpandCompressMemoryOpCost( - unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, const Instruction *I = nullptr) const { + virtual InstructionCost + getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { return 1; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index cb389ae74ef46..a34094088bd44 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1580,10 +1580,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> { } InstructionCost - getExpandCompressMemoryOpCost(unsigned Opcode, Type *DataTy, - bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) const override { + getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const override { + unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload + ? Instruction::Load + : Instruction::Store; + Type *DataTy = MICA.getDataType(); + bool VariableMask = MICA.getVariableMask(); + Align Alignment = MICA.getAlignment(); // Treat expand load/compress store as gather/scatter operation. // TODO: implement more precise cost estimation for these intrinsics. return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask, @@ -1964,15 +1968,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> { const Value *Mask = Args[2]; Align Alignment = I->getParamAlign(1).valueOrOne(); return thisT()->getExpandCompressMemoryOpCost( - Instruction::Store, Data->getType(), !isa<Constant>(Mask), Alignment, - CostKind, I); + {IID, Data->getType(), !isa<Constant>(Mask), Alignment, I}, CostKind); } case Intrinsic::masked_expandload: { const Value *Mask = Args[1]; Align Alignment = I->getParamAlign(0).valueOrOne(); - return thisT()->getExpandCompressMemoryOpCost(Instruction::Load, RetTy, - !isa<Constant>(Mask), - Alignment, CostKind, I); + return thisT()->getExpandCompressMemoryOpCost( + {IID, RetTy, !isa<Constant>(Mask), Alignment, I}, CostKind); } case Intrinsic::experimental_vp_strided_store: { const Value *Data = Args[0]; diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index 45369f0ffe137..6c9e5131908b8 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -1201,10 +1201,9 @@ InstructionCost TargetTransformInfo::getGatherScatterOpCost( } InstructionCost TargetTransformInfo::getExpandCompressMemoryOpCost( - unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, const Instruction *I) const { - InstructionCost Cost = TTIImpl->getExpandCompressMemoryOpCost( - Opcode, DataTy, VariableMask, Alignment, CostKind, I); + const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + InstructionCost Cost = TTIImpl->getExpandCompressMemoryOpCost(MICA, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 1a1a93a9cb178..4788a428d7e64 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1146,15 +1146,20 @@ InstructionCost RISCVTTIImpl::getGatherScatterOpCost( } InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost( - unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, const Instruction *I) const { + const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload + ? Instruction::Load + : Instruction::Store; + Type *DataTy = MICA.getDataType(); + bool VariableMask = MICA.getVariableMask(); + Align Alignment = MICA.getAlignment(); bool IsLegal = (Opcode == Instruction::Store && isLegalMaskedCompressStore(DataTy, Alignment)) || (Opcode == Instruction::Load && isLegalMaskedExpandLoad(DataTy, Alignment)); if (!IsLegal || CostKind != TTI::TCK_RecipThroughput) - return BaseT::getExpandCompressMemoryOpCost(Opcode, DataTy, VariableMask, - Alignment, CostKind, I); + return BaseT::getExpandCompressMemoryOpCost(MICA, CostKind); // Example compressstore sequence: // vsetivli zero, 8, e32, m2, ta, ma (ignored) // vcompress.vm v10, v8, v0 diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h index 39c1173e2986c..85470371721eb 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h @@ -197,9 +197,8 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> { const Instruction *I) const override; InstructionCost - getExpandCompressMemoryOpCost(unsigned Opcode, Type *Src, bool VariableMask, - Align Alignment, TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) const override; + getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const override; InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, 

[github-actions]

🐧 Linux x64 Test Results

• 186655 tests passed
• 4888 tests skipped

[lukel97]

LGTM

[fhahn]

LGTM, thanks!