summaryrefslogtreecommitdiff
path: root/lib/CodeGen/CGLoopInfo.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/CGLoopInfo.cpp')
-rw-r--r--lib/CodeGen/CGLoopInfo.cpp129
1 files changed, 113 insertions, 16 deletions
diff --git a/lib/CodeGen/CGLoopInfo.cpp b/lib/CodeGen/CGLoopInfo.cpp
index 21e2b8dd8c31b..fd0a9c773a2ef 100644
--- a/lib/CodeGen/CGLoopInfo.cpp
+++ b/lib/CodeGen/CGLoopInfo.cpp
@@ -10,8 +10,8 @@
#include "CGLoopInfo.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Attr.h"
-#include "clang/Sema/LoopHint.h"
#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instructions.h"
@@ -21,14 +21,17 @@ using namespace llvm;
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
const llvm::DebugLoc &StartLoc,
- const llvm::DebugLoc &EndLoc) {
+ const llvm::DebugLoc &EndLoc, MDNode *&AccGroup) {
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
+ Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
+ Attrs.PipelineInitiationInterval == 0 &&
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
- Attrs.DistributeEnable == LoopAttributes::Unspecified &&
- !StartLoc && !EndLoc)
+ Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
+ Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
+ !EndLoc)
return nullptr;
SmallVector<Metadata *, 4> Args;
@@ -61,7 +64,7 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Setting interleave.count
+ // Setting unroll.count
if (Attrs.UnrollCount > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
ConstantAsMetadata::get(ConstantInt::get(
@@ -69,6 +72,14 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
Args.push_back(MDNode::get(Ctx, Vals));
}
+ // Setting unroll_and_jam.count
+ if (Attrs.UnrollAndJamCount > 0) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
// Setting vectorize.enable
if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
@@ -91,6 +102,19 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
Args.push_back(MDNode::get(Ctx, Vals));
}
+ // Setting unroll_and_jam.full or unroll_and_jam.disable
+ if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) {
+ std::string Name;
+ if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable)
+ Name = "llvm.loop.unroll_and_jam.enable";
+ else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full)
+ Name = "llvm.loop.unroll_and_jam.full";
+ else
+ Name = "llvm.loop.unroll_and_jam.disable";
+ Metadata *Vals[] = {MDString::get(Ctx, Name)};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
ConstantAsMetadata::get(ConstantInt::get(
@@ -99,6 +123,28 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
Args.push_back(MDNode::get(Ctx, Vals));
}
+ if (Attrs.IsParallel) {
+ AccGroup = MDNode::getDistinct(Ctx, {});
+ Args.push_back(MDNode::get(
+ Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
+ }
+
+ if (Attrs.PipelineDisabled) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.pipeline.disable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ if (Attrs.PipelineInitiationInterval > 0) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
// Set the first operand to itself.
MDNode *LoopID = MDNode::get(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
@@ -107,24 +153,31 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
LoopAttributes::LoopAttributes(bool IsParallel)
: IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
- UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
- InterleaveCount(0), UnrollCount(0),
- DistributeEnable(LoopAttributes::Unspecified) {}
+ UnrollEnable(LoopAttributes::Unspecified),
+ UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
+ InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
+ DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
+ PipelineInitiationInterval(0) {}
void LoopAttributes::clear() {
IsParallel = false;
VectorizeWidth = 0;
InterleaveCount = 0;
UnrollCount = 0;
+ UnrollAndJamCount = 0;
VectorizeEnable = LoopAttributes::Unspecified;
UnrollEnable = LoopAttributes::Unspecified;
+ UnrollAndJamEnable = LoopAttributes::Unspecified;
DistributeEnable = LoopAttributes::Unspecified;
+ PipelineDisabled = false;
+ PipelineInitiationInterval = 0;
}
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
: LoopID(nullptr), Header(Header), Attrs(Attrs) {
- LoopID = createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc);
+ LoopID =
+ createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc, AccGroup);
}
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
@@ -191,12 +244,20 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::Unroll:
setUnrollState(LoopAttributes::Disable);
break;
+ case LoopHintAttr::UnrollAndJam:
+ setUnrollAndJamState(LoopAttributes::Disable);
+ break;
case LoopHintAttr::Distribute:
setDistributeState(false);
break;
+ case LoopHintAttr::PipelineDisabled:
+ setPipelineDisabled(true);
+ break;
case LoopHintAttr::UnrollCount:
+ case LoopHintAttr::UnrollAndJamCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
+ case LoopHintAttr::PipelineInitiationInterval:
llvm_unreachable("Options cannot be disabled.");
break;
}
@@ -210,12 +271,18 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::Unroll:
setUnrollState(LoopAttributes::Enable);
break;
+ case LoopHintAttr::UnrollAndJam:
+ setUnrollAndJamState(LoopAttributes::Enable);
+ break;
case LoopHintAttr::Distribute:
setDistributeState(true);
break;
case LoopHintAttr::UnrollCount:
+ case LoopHintAttr::UnrollAndJamCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
+ case LoopHintAttr::PipelineDisabled:
+ case LoopHintAttr::PipelineInitiationInterval:
llvm_unreachable("Options cannot enabled.");
break;
}
@@ -229,10 +296,14 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
setVectorizeEnable(true);
break;
case LoopHintAttr::Unroll:
+ case LoopHintAttr::UnrollAndJam:
case LoopHintAttr::UnrollCount:
+ case LoopHintAttr::UnrollAndJamCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
case LoopHintAttr::Distribute:
+ case LoopHintAttr::PipelineDisabled:
+ case LoopHintAttr::PipelineInitiationInterval:
llvm_unreachable("Options cannot be used to assume mem safety.");
break;
}
@@ -242,12 +313,18 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::Unroll:
setUnrollState(LoopAttributes::Full);
break;
+ case LoopHintAttr::UnrollAndJam:
+ setUnrollAndJamState(LoopAttributes::Full);
+ break;
case LoopHintAttr::Vectorize:
case LoopHintAttr::Interleave:
case LoopHintAttr::UnrollCount:
+ case LoopHintAttr::UnrollAndJamCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
case LoopHintAttr::Distribute:
+ case LoopHintAttr::PipelineDisabled:
+ case LoopHintAttr::PipelineInitiationInterval:
llvm_unreachable("Options cannot be used with 'full' hint.");
break;
}
@@ -263,10 +340,18 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::UnrollCount:
setUnrollCount(ValueInt);
break;
+ case LoopHintAttr::UnrollAndJamCount:
+ setUnrollAndJamCount(ValueInt);
+ break;
+ case LoopHintAttr::PipelineInitiationInterval:
+ setPipelineInitiationInterval(ValueInt);
+ break;
case LoopHintAttr::Unroll:
+ case LoopHintAttr::UnrollAndJam:
case LoopHintAttr::Vectorize:
case LoopHintAttr::Interleave:
case LoopHintAttr::Distribute:
+ case LoopHintAttr::PipelineDisabled:
llvm_unreachable("Options cannot be assigned a value.");
break;
}
@@ -284,6 +369,21 @@ void LoopInfoStack::pop() {
}
void LoopInfoStack::InsertHelper(Instruction *I) const {
+ if (I->mayReadOrWriteMemory()) {
+ SmallVector<Metadata *, 4> AccessGroups;
+ for (const LoopInfo &AL : Active) {
+ // Here we assume that every loop that has an access group is parallel.
+ if (MDNode *Group = AL.getAccessGroup())
+ AccessGroups.push_back(Group);
+ }
+ MDNode *UnionMD = nullptr;
+ if (AccessGroups.size() == 1)
+ UnionMD = cast<MDNode>(AccessGroups[0]);
+ else if (AccessGroups.size() >= 2)
+ UnionMD = MDNode::get(I->getContext(), AccessGroups);
+ I->setMetadata("llvm.access.group", UnionMD);
+ }
+
if (!hasInfo())
return;
@@ -291,15 +391,12 @@ void LoopInfoStack::InsertHelper(Instruction *I) const {
if (!L.getLoopID())
return;
- if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) {
- for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i)
- if (TI->getSuccessor(i) == L.getHeader()) {
- TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
+ if (I->isTerminator()) {
+ for (BasicBlock *Succ : successors(I))
+ if (Succ == L.getHeader()) {
+ I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
break;
}
return;
}
-
- if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory())
- I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID());
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-12 15:55:17 +0000