[clang][bytecode] Allocate IntegralAP and Floating types using an allocator (#144246)

Both `APInt` and `APFloat` will heap-allocate memory themselves using the system allocator when the size of their data exceeds 64 bits. This is why clang has `APNumericStorage`, which allocates its memory using an allocator (via `ASTContext`) instead. Calling `getValue()` on an ast node like that will then create a new `APInt`/`APFloat` , which will copy the data (in the `APFloat` case, we even copy it twice). That's sad but whatever. In the bytecode interpreter, we have a similar problem. Large integers and floating-point values are placement-new allocated into the `InterpStack` (or into the bytecode, which is a `vector<std::byte>`). When we then later interrupt interpretation, we don't run the destructor for all items on the stack, which means we leak the memory the `APInt`/`APFloat` (which backs the `IntegralAP`/`Floating` the interpreter uses). Fix this by using an approach similar to the one used in the AST. Add an allocator to `InterpState`, which is used for temporaries and local values. Those values will be freed at the end of interpretation. For global variables, we need to promote the values to global lifetime, which we do via `InitGlobal` and `FinishInitGlobal` ops. Interestingly, this results in a slight _improvement_ in compile times: https://llvm-compile-time-tracker.com/compare.php?from=6bfcdda9b1ddf0900f82f7e30cb5e3253a791d50&to=88d1d899127b408f0fb0f385c2c58e6283195049&stat=instructions:u (but don't ask me why). Fixes #139012

Author: tbaederr

clang/lib/AST/ByteCode/Compiler.cpp

@@ -748,7 +748,8 @@ bool Compiler<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
  if (DiscardResult)
  return true;
 
- return this->emitConstFloat(E->getValue(), E);
+ APFloat F = E->getValue();
+ return this->emitFloat(F, E);
 }
 
 template <class Emitter>
@@ -4185,8 +4186,10 @@ bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
  nullptr, E);
  case PT_MemberPtr:
  return this->emitNullMemberPtr(0, nullptr, E);
- case PT_Float:
- return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
+ case PT_Float: {
+ APFloat F = APFloat::getZero(Ctx.getFloatSemantics(QT));
+ return this->emitFloat(F, E);
+ }
  case PT_FixedPoint: {
  auto Sem = Ctx.getASTContext().getFixedPointSemantics(E->getType());
  return this->emitConstFixedPoint(FixedPoint::zero(Sem), E);
@@ -4674,10 +4677,7 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
  if (!visitInitializer(Init))
  return false;
 
- if (!this->emitFinishInit(Init))
- return false;
-
- return this->emitPopPtr(Init);
+ return this->emitFinishInitGlobal(Init);
  };
 
  DeclScope<Emitter> LocalScope(this, VD);
@@ -4698,51 +4698,45 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
  return false;
 
  return !Init || (checkDecl() && initGlobal(*GlobalIndex));
- } else {
- InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
-
- if (VarT) {
- unsigned Offset = this->allocateLocalPrimitive(
- VD, *VarT, VD->getType().isConstQualified(), nullptr,
- ScopeKind::Block, IsConstexprUnknown);
- if (Init) {
- // If this is a toplevel declaration, create a scope for the
- // initializer.
- if (Toplevel) {
- LocalScope<Emitter> Scope(this);
- if (!this->visit(Init))
- return false;
- return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
- } else {
- if (!this->visit(Init))
- return false;
- return this->emitSetLocal(*VarT, Offset, VD);
- }
- }
- } else {
- if (std::optional<unsigned> Offset =
- this->allocateLocal(VD, VD->getType(), nullptr, ScopeKind::Block,
- IsConstexprUnknown)) {
- if (!Init)
- return true;
+ }
+ // Local variables.
+ InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
 
- if (!this->emitGetPtrLocal(*Offset, Init))
+ if (VarT) {
+ unsigned Offset = this->allocateLocalPrimitive(
+ VD, *VarT, VD->getType().isConstQualified(), nullptr, ScopeKind::Block,
+ IsConstexprUnknown);
+ if (Init) {
+ // If this is a toplevel declaration, create a scope for the
+ // initializer.
+ if (Toplevel) {
+ LocalScope<Emitter> Scope(this);
+ if (!this->visit(Init))
  return false;
-
- if (!visitInitializer(Init))
+ return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
+ } else {
+ if (!this->visit(Init))
  return false;
+ return this->emitSetLocal(*VarT, Offset, VD);
+ }
+ }
+ } else {
+ if (std::optional<unsigned> Offset = this->allocateLocal(
+ VD, VD->getType(), nullptr, ScopeKind::Block, IsConstexprUnknown)) {
+ if (!Init)
+ return true;
 
-  if (!this->emitFinishInit(Init))
-  return false;
+ if (!this->emitGetPtrLocal(*Offset, Init))
+ return false;
 
- return this->emitPopPtr(Init);
- }
- return false;
+ if (!visitInitializer(Init))
+ return false;
+
+ return this->emitFinishInitPop(Init);
  }
- return true;
+ return false;
  }
-
- return false;
+ return true;
 }
 
 template <class Emitter>
@@ -4751,8 +4745,10 @@ bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
  assert(!DiscardResult);
  if (Val.isInt())
  return this->emitConst(Val.getInt(), ValType, E);
- else if (Val.isFloat())
- return this->emitConstFloat(Val.getFloat(), E);
+ else if (Val.isFloat()) {
+ APFloat F = Val.getFloat();
+ return this->emitFloat(F, E);
+ }
 
  if (Val.isLValue()) {
  if (Val.isNullPointer())
@@ -6133,8 +6129,10 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
  const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
  if (!this->emitLoadFloat(E))
  return false;
- if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
+ APFloat F(TargetSemantics, 1);
+ if (!this->emitFloat(F, E))
  return false;
+
  if (!this->emitAddf(getFPOptions(E), E))
  return false;
  if (!this->emitStoreFloat(E))
@@ -6176,8 +6174,10 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
  const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
  if (!this->emitLoadFloat(E))
  return false;
- if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
+ APFloat F(TargetSemantics, 1);
+ if (!this->emitFloat(F, E))
  return false;
+
  if (!this->emitSubf(getFPOptions(E), E))
  return false;
  if (!this->emitStoreFloat(E))
@@ -6953,6 +6953,20 @@ bool Compiler<Emitter>::emitDummyPtr(const DeclTy &D, const Expr *E) {
  return true;
 }
 
+template <class Emitter>
+bool Compiler<Emitter>::emitFloat(const APFloat &F, const Expr *E) {
+ assert(!DiscardResult && "Should've been checked before");
+
+ if (Floating::singleWord(F.getSemantics()))
+ return this->emitConstFloat(Floating(F), E);
+
+ APInt I = F.bitcastToAPInt();
+ return this->emitConstFloat(
+ Floating(const_cast<uint64_t *>(I.getRawData()),
+ llvm::APFloatBase::SemanticsToEnum(F.getSemantics())),
+ E);
+}
+
 // This function is constexpr if and only if To, From, and the types of
 // all subobjects of To and From are types T such that...
 // (3.1) - is_union_v<T> is false;

clang/lib/AST/ByteCode/Compiler.h

@@ -391,6 +391,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
  bool emitRecordDestruction(const Record *R, SourceInfo Loc);
  bool emitDestruction(const Descriptor *Desc, SourceInfo Loc);
  bool emitDummyPtr(const DeclTy &D, const Expr *E);
+ bool emitFloat(const APFloat &F, const Expr *E);
  unsigned collectBaseOffset(const QualType BaseType,
  const QualType DerivedType);
  bool emitLambdaStaticInvokerBody(const CXXMethodDecl *MD);

clang/lib/AST/ByteCode/Descriptor.cpp

@@ -368,7 +368,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
  bool IsTemporary, bool IsConst, UnknownSize)
  : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark),
  MDSize(MD.value_or(0)),
- AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)),
+ AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), PrimT(Type),
  IsConst(IsConst), IsMutable(false), IsTemporary(IsTemporary),
  IsArray(true), CtorFn(getCtorArrayPrim(Type)),
  DtorFn(getDtorArrayPrim(Type)), MoveFn(getMoveArrayPrim(Type)) {

clang/lib/AST/ByteCode/Disasm.cpp

@@ -50,34 +50,56 @@ inline static std::string printArg(Program &P, CodePtr &OpPC) {
 }
 
 template <> inline std::string printArg<Floating>(Program &P, CodePtr &OpPC) {
- auto F = Floating::deserialize(*OpPC);
- OpPC += align(F.bytesToSerialize());
+ auto Sem = Floating::deserializeSemantics(*OpPC);
 
- std::string Result;
- llvm::raw_string_ostream SS(Result);
- SS << F;
- return Result;
+ unsigned BitWidth = llvm::APFloatBase::semanticsSizeInBits(
+ llvm::APFloatBase::EnumToSemantics(Sem));
+ auto Memory =
+ std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
+ Floating Result(Memory.get(), Sem);
+ Floating::deserialize(*OpPC, &Result);
+
+ OpPC += align(Result.bytesToSerialize());
+
+ std::string S;
+ llvm::raw_string_ostream SS(S);
+ SS << Result;
+ return S;
 }
 
 template <>
 inline std::string printArg<IntegralAP<false>>(Program &P, CodePtr &OpPC) {
- auto F = IntegralAP<false>::deserialize(*OpPC);
- OpPC += align(F.bytesToSerialize());
-
- std::string Result;
- llvm::raw_string_ostream SS(Result);
- SS << F;
- return Result;
+ using T = IntegralAP<false>;
+ unsigned BitWidth = T::deserializeSize(*OpPC);
+ auto Memory =
+ std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
+
+ T Result(Memory.get(), BitWidth);
+ T::deserialize(*OpPC, &Result);
+
+ OpPC += Result.bytesToSerialize();
+ std::string Str;
+ llvm::raw_string_ostream SS(Str);
+ SS << Result;
+ return Str;
 }
+
 template <>
 inline std::string printArg<IntegralAP<true>>(Program &P, CodePtr &OpPC) {
- auto F = IntegralAP<true>::deserialize(*OpPC);
- OpPC += align(F.bytesToSerialize());
+ using T = IntegralAP<true>;
+ unsigned BitWidth = T::deserializeSize(*OpPC);
+ auto Memory =
+ std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
 
- std::string Result;
- llvm::raw_string_ostream SS(Result);
- SS << F;
- return Result;
+ T Result(Memory.get(), BitWidth);
+ T::deserialize(*OpPC, &Result);
+
+ std::string Str;
+ llvm::raw_string_ostream SS(Str);
+ SS << Result;
+
+ OpPC += Result.bytesToSerialize();
+ return Str;
 }
 
 template <> inline std::string printArg<FixedPoint>(Program &P, CodePtr &OpPC) {