[GlobalISel][AArch64] Tail call libcalls.

This tries to allow libcalls to be tail called, using a similar method to DAG where the type is checked to make sure they match, and if so the backend, through lowerCall checks that the tailcall is valid for all arguments.

Author: davemgreen

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

@@ -288,11 +288,14 @@ class LegalizerHelper {
 
  // Implements floating-point environment read/write via library function call.
  LegalizeResult createGetStateLibcall(MachineIRBuilder &MIRBuilder,
- MachineInstr &MI);
+ MachineInstr &MI,
+ LostDebugLocObserver &LocObserver);
  LegalizeResult createSetStateLibcall(MachineIRBuilder &MIRBuilder,
- MachineInstr &MI);
+ MachineInstr &MI,
+ LostDebugLocObserver &LocObserver);
  LegalizeResult createResetStateLibcall(MachineIRBuilder &MIRBuilder,
- MachineInstr &MI);
+ MachineInstr &MI,
+ LostDebugLocObserver &LocObserver);
 
 public:
  /// Return the alignment to use for a stack temporary object with the given
@@ -440,13 +443,15 @@ class LegalizerHelper {
 LegalizerHelper::LegalizeResult
 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
  const CallLowering::ArgInfo &Result,
- ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC);
+ ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC,
+ LostDebugLocObserver &LocObserver, MachineInstr *MI = nullptr);
 
 /// Helper function that creates the given libcall.
 LegalizerHelper::LegalizeResult
 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
  const CallLowering::ArgInfo &Result,
- ArrayRef<CallLowering::ArgInfo> Args);
+ ArrayRef<CallLowering::ArgInfo> Args,
+ LostDebugLocObserver &LocObserver, MachineInstr *MI = nullptr);
 
 /// Create a libcall to memcpy et al.
 LegalizerHelper::LegalizeResult

llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

@@ -35,6 +35,7 @@ extern cl::opt<bool> DisableGISelLegalityCheck;
 class MachineFunction;
 class raw_ostream;
 class LegalizerHelper;
+class LostDebugLocObserver;
 class MachineInstr;
 class MachineRegisterInfo;
 class MCInstrInfo;
@@ -1288,8 +1289,8 @@ class LegalizerInfo {
  const MachineRegisterInfo &MRI) const;
 
  /// Called for instructions with the Custom LegalizationAction.
- virtual bool legalizeCustom(LegalizerHelper &Helper,
- MachineInstr &MI) const {
+ virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const {
  llvm_unreachable("must implement this if custom action is used");
  }
 

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

@@ -149,7 +149,8 @@ LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
  return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
  case Custom:
  LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
- return LI.legalizeCustom(*this, MI) ? Legalized : UnableToLegalize;
+ return LI.legalizeCustom(*this, MI, LocObserver) ? Legalized
+ : UnableToLegalize;
  default:
  LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
  return UnableToLegalize;
@@ -567,7 +568,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
 
 /// True if an instruction is in tail position in its caller. Intended for
 /// legalizing libcalls as tail calls when possible.
-static bool isLibCallInTailPosition(MachineInstr &MI,
+static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result,
+ MachineInstr &MI,
  const TargetInstrInfo &TII,
  MachineRegisterInfo &MRI) {
  MachineBasicBlock &MBB = *MI.getParent();
@@ -596,17 +598,12 @@ static bool isLibCallInTailPosition(MachineInstr &MI,
  // RET_ReallyLR implicit $x0
  auto Next = next_nodbg(MI.getIterator(), MBB.instr_end());
  if (Next != MBB.instr_end() && Next->isCopy()) {
- switch (MI.getOpcode()) {
- default:
- llvm_unreachable("unsupported opcode");
- case TargetOpcode::G_BZERO:
+ if (MI.getOpcode() == TargetOpcode::G_BZERO)
  return false;
- case TargetOpcode::G_MEMCPY:
- case TargetOpcode::G_MEMMOVE:
- case TargetOpcode::G_MEMSET:
- break;
- }
 
+ // For MEMCPY/MOMMOVE/MEMSET these will be the first use (the dst), as the
+ // mempy/etc routines return the same parameter. For other it will be the
+ // returned value.
  Register VReg = MI.getOperand(0).getReg();
  if (!VReg.isVirtual() || VReg != Next->getOperand(1).getReg())
  return false;
@@ -622,7 +619,7 @@ static bool isLibCallInTailPosition(MachineInstr &MI,
  if (Ret->getNumImplicitOperands() != 1)
  return false;
 
- if (PReg != Ret->getOperand(0).getReg())
+ if (!Ret->getOperand(0).isReg() || PReg != Ret->getOperand(0).getReg())
  return false;
 
  // Skip over the COPY that we just validated.
@@ -639,42 +636,73 @@ LegalizerHelper::LegalizeResult
 llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
  const CallLowering::ArgInfo &Result,
  ArrayRef<CallLowering::ArgInfo> Args,
- const CallingConv::ID CC) {
+ const CallingConv::ID CC, LostDebugLocObserver &LocObserver,
+ MachineInstr *MI) {
  auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
 
  CallLowering::CallLoweringInfo Info;
  Info.CallConv = CC;
  Info.Callee = MachineOperand::CreateES(Name);
  Info.OrigRet = Result;
+ if (MI)
+ Info.IsTailCall =
+ (Result.Ty->isVoidTy() ||
+ Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) &&
+ isLibCallInTailPosition(Result, *MI, MIRBuilder.getTII(),
+ *MIRBuilder.getMRI());
+
  std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
  if (!CLI.lowerCall(MIRBuilder, Info))
  return LegalizerHelper::UnableToLegalize;
 
+ if (MI && Info.LoweredTailCall) {
+ assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
+
+ // Check debug locations before removing the return.
+ LocObserver.checkpoint(true);
+
+ // We must have a return following the call (or debug insts) to get past
+ // isLibCallInTailPosition.
+ do {
+ MachineInstr *Next = MI->getNextNode();
+ assert(Next &&
+ (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
+ "Expected instr following MI to be return or debug inst?");
+ // We lowered a tail call, so the call is now the return from the block.
+ // Delete the old return.
+ Next->eraseFromParent();
+ } while (MI->getNextNode());
+
+ // We expect to lose the debug location from the return.
+ LocObserver.checkpoint(false);
+ }
  return LegalizerHelper::Legalized;
 }
 
 LegalizerHelper::LegalizeResult
 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
  const CallLowering::ArgInfo &Result,
- ArrayRef<CallLowering::ArgInfo> Args) {
+ ArrayRef<CallLowering::ArgInfo> Args,
+ LostDebugLocObserver &LocObserver, MachineInstr *MI) {
  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
  const char *Name = TLI.getLibcallName(Libcall);
  const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall);
- return createLibcall(MIRBuilder, Name, Result, Args, CC);
+ return createLibcall(MIRBuilder, Name, Result, Args, CC, LocObserver, MI);
 }
 
 // Useful for libcalls where all operands have the same type.
 static LegalizerHelper::LegalizeResult
 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
- Type *OpType) {
+ Type *OpType, LostDebugLocObserver &LocObserver) {
  auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
 
  // FIXME: What does the original arg index mean here?
  SmallVector<CallLowering::ArgInfo, 3> Args;
  for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))
  Args.push_back({MO.getReg(), OpType, 0});
  return createLibcall(MIRBuilder, Libcall,
- {MI.getOperand(0).getReg(), OpType, 0}, Args);
+ {MI.getOperand(0).getReg(), OpType, 0}, Args,
+ LocObserver, &MI);
 }
 
 LegalizerHelper::LegalizeResult
@@ -733,8 +761,9 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
  Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
  Info.Callee = MachineOperand::CreateES(Name);
  Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0);
- Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() &&
- isLibCallInTailPosition(MI, MIRBuilder.getTII(), MRI);
+ Info.IsTailCall =
+ MI.getOperand(MI.getNumOperands() - 1).getImm() &&
+ isLibCallInTailPosition(Info.OrigRet, MI, MIRBuilder.getTII(), MRI);
 
  std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
  if (!CLI.lowerCall(MIRBuilder, Info))
@@ -789,11 +818,11 @@ static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
 
 static LegalizerHelper::LegalizeResult
 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
- Type *FromType) {
+ Type *FromType, LostDebugLocObserver &LocObserver) {
  RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
- return createLibcall(MIRBuilder, Libcall,
-   {MI.getOperand(0).getReg(), ToType, 0},
-  {{MI.getOperand(1).getReg(), FromType, 0}});
+ return createLibcall(
+ MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType, 0},
+ {{MI.getOperand(1).getReg(), FromType, 0}}, LocObserver, &MI);
 }
 
 static RTLIB::Libcall
@@ -829,7 +858,8 @@ getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) {
 //
 LegalizerHelper::LegalizeResult
 LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder,
- MachineInstr &MI) {
+ MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) {
  const DataLayout &DL = MIRBuilder.getDataLayout();
  auto &MF = MIRBuilder.getMF();
  auto &MRI = *MIRBuilder.getMRI();
@@ -850,7 +880,8 @@ LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder,
  auto Res =
  createLibcall(MIRBuilder, RTLibcall,
  CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
- CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}));
+ CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}),
+ LocObserver, nullptr);
  if (Res != LegalizerHelper::Legalized)
  return Res;
 
@@ -867,7 +898,8 @@ LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder,
 // content of memory region.
 LegalizerHelper::LegalizeResult
 LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder,
- MachineInstr &MI) {
+ MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) {
  const DataLayout &DL = MIRBuilder.getDataLayout();
  auto &MF = MIRBuilder.getMF();
  auto &MRI = *MIRBuilder.getMRI();
@@ -892,7 +924,8 @@ LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder,
  RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
  return createLibcall(MIRBuilder, RTLibcall,
  CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
- CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}));
+ CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}),
+ LocObserver, nullptr);
 }
 
 // The function is used to legalize operations that set default environment
@@ -902,7 +935,8 @@ LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder,
 // it is not true, the target must provide custom lowering.
 LegalizerHelper::LegalizeResult
 LegalizerHelper::createResetStateLibcall(MachineIRBuilder &MIRBuilder,
- MachineInstr &MI) {
+ MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) {
  const DataLayout &DL = MIRBuilder.getDataLayout();
  auto &MF = MIRBuilder.getMF();
  auto &Ctx = MF.getFunction().getContext();
@@ -919,7 +953,8 @@ LegalizerHelper::createResetStateLibcall(MachineIRBuilder &MIRBuilder,
  RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
  return createLibcall(MIRBuilder, RTLibcall,
  CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
- CallLowering::ArgInfo({ Dest.getReg(), StatePtrTy, 0}));
+ CallLowering::ArgInfo({Dest.getReg(), StatePtrTy, 0}),
+ LocObserver, &MI);
 }
 
 LegalizerHelper::LegalizeResult
@@ -938,7 +973,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
  LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
  unsigned Size = LLTy.getSizeInBits();
  Type *HLTy = IntegerType::get(Ctx, Size);
- auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
+ auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy, LocObserver);
  if (Status != Legalized)
  return Status;
  break;
@@ -974,7 +1009,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
  LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
  return UnableToLegalize;
  }
- auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
+ auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy, LocObserver);
  if (Status != Legalized)
  return Status;
  break;
@@ -985,7 +1020,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
  Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg()));
  if (!FromTy || !ToTy)
  return UnableToLegalize;
- LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy );
+ LegalizeResult Status =
+ conversionLibcall(MI, MIRBuilder, ToTy, FromTy, LocObserver);
  if (Status != Legalized)
  return Status;
  break;
@@ -1000,7 +1036,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
  LegalizeResult Status = conversionLibcall(
  MI, MIRBuilder,
  ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
- FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
+ FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
+ LocObserver);
  if (Status != Legalized)
  return Status;
  break;
@@ -1015,7 +1052,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
  LegalizeResult Status = conversionLibcall(
  MI, MIRBuilder,
  ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
- FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
+ FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
+ LocObserver);
  if (Status != Legalized)
  return Status;
  break;
@@ -1032,19 +1070,20 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
  return Result;
  }
  case TargetOpcode::G_GET_FPMODE: {
- LegalizeResult Result = createGetStateLibcall(MIRBuilder, MI);
+ LegalizeResult Result = createGetStateLibcall(MIRBuilder, MI, LocObserver);
  if (Result != Legalized)
  return Result;
  break;
  }
  case TargetOpcode::G_SET_FPMODE: {
- LegalizeResult Result = createSetStateLibcall(MIRBuilder, MI);
+ LegalizeResult Result = createSetStateLibcall(MIRBuilder, MI, LocObserver);
  if (Result != Legalized)
  return Result;
  break;
  }
  case TargetOpcode::G_RESET_FPMODE: {
- LegalizeResult Result = createResetStateLibcall(MIRBuilder, MI);
+ LegalizeResult Result =
+ createResetStateLibcall(MIRBuilder, MI, LocObserver);
  if (Result != Legalized)
  return Result;
  break;

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

@@ -1133,8 +1133,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
  verify(*ST.getInstrInfo());
 }
 
-bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
- MachineInstr &MI) const {
+bool AArch64LegalizerInfo::legalizeCustom(
+ LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const {
  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
  GISelChangeObserver &Observer = Helper.Observer;

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h

@@ -28,7 +28,8 @@ class AArch64LegalizerInfo : public LegalizerInfo {
 public:
  AArch64LegalizerInfo(const AArch64Subtarget &ST);
 
- bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override;
+ bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const override;
 
  bool legalizeIntrinsic(LegalizerHelper &Helper,
  MachineInstr &MI) const override;

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

@@ -1996,8 +1996,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
  verify(*ST.getInstrInfo());
 }
 
-bool AMDGPULegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
- MachineInstr &MI) const {
+bool AMDGPULegalizerInfo::legalizeCustom(
+ LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();
 

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h

@@ -35,7 +35,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
  AMDGPULegalizerInfo(const GCNSubtarget &ST,
  const GCNTargetMachine &TM);
 
- bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override;
+ bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const override;
 
  Register getSegmentAperture(unsigned AddrSpace,
  MachineRegisterInfo &MRI,