summaryrefslogtreecommitdiff
path: root/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp')
-rw-r--r--contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp65
1 files changed, 51 insertions, 14 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp
index b2bc42bfa013..e4b184eb8798 100644
--- a/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -218,6 +218,7 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
if (Attrs.VectorizeEnable == LoopAttributes::Disable)
Enabled = false;
else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
+ Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
Enabled = true;
@@ -251,6 +252,22 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
Args.push_back(TempNode.get());
Args.append(LoopProperties.begin(), LoopProperties.end());
+ // Setting vectorize.predicate
+ bool IsVectorPredicateEnabled = false;
+ if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
+ Attrs.VectorizeEnable != LoopAttributes::Disable &&
+ Attrs.VectorizeWidth < 1) {
+
+ IsVectorPredicateEnabled =
+ (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
+
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
+ IsVectorPredicateEnabled))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
// Setting vectorize.width
if (Attrs.VectorizeWidth > 0) {
Metadata *Vals[] = {
@@ -269,14 +286,18 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Setting vectorize.enable
- if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
- Metadata *Vals[] = {
- MDString::get(Ctx, "llvm.loop.vectorize.enable"),
- ConstantAsMetadata::get(ConstantInt::get(
- llvm::Type::getInt1Ty(Ctx),
- (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
- Args.push_back(MDNode::get(Ctx, Vals));
+ // vectorize.enable is set if:
+ // 1) loop hint vectorize.enable is set, or
+ // 2) it is implied when vectorize.predicate is set, or
+ // 3) it is implied when vectorize.width is set.
+ if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
+ IsVectorPredicateEnabled ||
+ Attrs.VectorizeWidth > 1 ) {
+ bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
+ Args.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), AttrVal))}));
}
if (FollowupHasTransforms)
@@ -411,7 +432,8 @@ MDNode *LoopInfo::createMetadata(
LoopAttributes::LoopAttributes(bool IsParallel)
: IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
UnrollEnable(LoopAttributes::Unspecified),
- UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
+ UnrollAndJamEnable(LoopAttributes::Unspecified),
+ VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
PipelineInitiationInterval(0) {}
@@ -425,6 +447,7 @@ void LoopAttributes::clear() {
VectorizeEnable = LoopAttributes::Unspecified;
UnrollEnable = LoopAttributes::Unspecified;
UnrollAndJamEnable = LoopAttributes::Unspecified;
+ VectorizePredicateEnable = LoopAttributes::Unspecified;
DistributeEnable = LoopAttributes::Unspecified;
PipelineDisabled = false;
PipelineInitiationInterval = 0;
@@ -446,6 +469,7 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
Attrs.PipelineInitiationInterval == 0 &&
+ Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
@@ -480,6 +504,7 @@ void LoopInfo::finish() {
BeforeJam.InterleaveCount = Attrs.InterleaveCount;
BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
BeforeJam.DistributeEnable = Attrs.DistributeEnable;
+ BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
switch (Attrs.UnrollEnable) {
case LoopAttributes::Unspecified:
@@ -495,6 +520,7 @@ void LoopInfo::finish() {
break;
}
+ AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
AfterJam.UnrollCount = Attrs.UnrollCount;
AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
@@ -516,6 +542,7 @@ void LoopInfo::finish() {
// add it manually.
SmallVector<Metadata *, 1> BeforeLoopProperties;
if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
+ BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
BeforeLoopProperties.push_back(
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
@@ -537,8 +564,9 @@ void LoopInfo::finish() {
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
const llvm::DebugLoc &EndLoc) {
- Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
- Active.empty() ? nullptr : &Active.back()));
+ Active.emplace_back(
+ new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
+ Active.empty() ? nullptr : Active.back().get()));
// Clear the attributes so nested loops do not inherit them.
StagedAttrs.clear();
}
@@ -603,6 +631,9 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::UnrollAndJam:
setUnrollAndJamState(LoopAttributes::Disable);
break;
+ case LoopHintAttr::VectorizePredicate:
+ setVectorizePredicateState(LoopAttributes::Disable);
+ break;
case LoopHintAttr::Distribute:
setDistributeState(false);
break;
@@ -630,6 +661,9 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::UnrollAndJam:
setUnrollAndJamState(LoopAttributes::Enable);
break;
+ case LoopHintAttr::VectorizePredicate:
+ setVectorizePredicateState(LoopAttributes::Enable);
+ break;
case LoopHintAttr::Distribute:
setDistributeState(true);
break;
@@ -653,6 +687,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
break;
case LoopHintAttr::Unroll:
case LoopHintAttr::UnrollAndJam:
+ case LoopHintAttr::VectorizePredicate:
case LoopHintAttr::UnrollCount:
case LoopHintAttr::UnrollAndJamCount:
case LoopHintAttr::VectorizeWidth:
@@ -681,6 +716,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::Distribute:
case LoopHintAttr::PipelineDisabled:
case LoopHintAttr::PipelineInitiationInterval:
+ case LoopHintAttr::VectorizePredicate:
llvm_unreachable("Options cannot be used with 'full' hint.");
break;
}
@@ -704,6 +740,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
break;
case LoopHintAttr::Unroll:
case LoopHintAttr::UnrollAndJam:
+ case LoopHintAttr::VectorizePredicate:
case LoopHintAttr::Vectorize:
case LoopHintAttr::Interleave:
case LoopHintAttr::Distribute:
@@ -721,16 +758,16 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
void LoopInfoStack::pop() {
assert(!Active.empty() && "No active loops to pop");
- Active.back().finish();
+ Active.back()->finish();
Active.pop_back();
}
void LoopInfoStack::InsertHelper(Instruction *I) const {
if (I->mayReadOrWriteMemory()) {
SmallVector<Metadata *, 4> AccessGroups;
- for (const LoopInfo &AL : Active) {
+ for (const auto &AL : Active) {
// Here we assume that every loop that has an access group is parallel.
- if (MDNode *Group = AL.getAccessGroup())
+ if (MDNode *Group = AL->getAccessGroup())
AccessGroups.push_back(Group);
}
MDNode *UnionMD = nullptr;
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-01 18:21:36 +0000