summaryrefslogtreecommitdiff
path: root/llvm/lib/IR/AutoUpgrade.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/IR/AutoUpgrade.cpp')
-rw-r--r--llvm/lib/IR/AutoUpgrade.cpp376
1 files changed, 320 insertions, 56 deletions
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 75594f90c926..7b9c55ff30a5 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -82,6 +82,26 @@ static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
return true;
}
+static bool UpgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
+ Function *&NewFn) {
+ if (F->getReturnType()->getScalarType()->isBFloatTy())
+ return false;
+
+ rename(F);
+ NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+ return true;
+}
+
+static bool UpgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
+ Function *&NewFn) {
+ if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
+ return false;
+
+ rename(F);
+ NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+ return true;
+}
+
static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
// All of the intrinsics matches below should be marked with which llvm
// version started autoupgrading them. At some point in the future we would
@@ -488,6 +508,33 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
NewFn);
+ if (Name == "avx512bf16.cvtne2ps2bf16.128") // Added in 9.0
+ return UpgradeX86BF16Intrinsic(
+ F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128, NewFn);
+ if (Name == "avx512bf16.cvtne2ps2bf16.256") // Added in 9.0
+ return UpgradeX86BF16Intrinsic(
+ F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256, NewFn);
+ if (Name == "avx512bf16.cvtne2ps2bf16.512") // Added in 9.0
+ return UpgradeX86BF16Intrinsic(
+ F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512, NewFn);
+ if (Name == "avx512bf16.mask.cvtneps2bf16.128") // Added in 9.0
+ return UpgradeX86BF16Intrinsic(
+ F, Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128, NewFn);
+ if (Name == "avx512bf16.cvtneps2bf16.256") // Added in 9.0
+ return UpgradeX86BF16Intrinsic(
+ F, Intrinsic::x86_avx512bf16_cvtneps2bf16_256, NewFn);
+ if (Name == "avx512bf16.cvtneps2bf16.512") // Added in 9.0
+ return UpgradeX86BF16Intrinsic(
+ F, Intrinsic::x86_avx512bf16_cvtneps2bf16_512, NewFn);
+ if (Name == "avx512bf16.dpbf16ps.128") // Added in 9.0
+ return UpgradeX86BF16DPIntrinsic(
+ F, Intrinsic::x86_avx512bf16_dpbf16ps_128, NewFn);
+ if (Name == "avx512bf16.dpbf16ps.256") // Added in 9.0
+ return UpgradeX86BF16DPIntrinsic(
+ F, Intrinsic::x86_avx512bf16_dpbf16ps_256, NewFn);
+ if (Name == "avx512bf16.dpbf16ps.512") // Added in 9.0
+ return UpgradeX86BF16DPIntrinsic(
+ F, Intrinsic::x86_avx512bf16_dpbf16ps_512, NewFn);
// frcz.ss/sd may need to have an argument dropped. Added in 3.2
if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
@@ -536,7 +583,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
// Quickly eliminate it, if it's not a candidate.
StringRef Name = F->getName();
- if (Name.size() <= 8 || !Name.startswith("llvm."))
+ if (Name.size() <= 7 || !Name.startswith("llvm."))
return false;
Name = Name.substr(5); // Strip off "llvm."
@@ -558,6 +605,59 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
F->arg_begin()->getType());
return true;
}
+ if (Name == "aarch64.sve.bfdot.lane") {
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::aarch64_sve_bfdot_lane_v2);
+ return true;
+ }
+ if (Name == "aarch64.sve.bfmlalb.lane") {
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::aarch64_sve_bfmlalb_lane_v2);
+ return true;
+ }
+ if (Name == "aarch64.sve.bfmlalt.lane") {
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::aarch64_sve_bfmlalt_lane_v2);
+ return true;
+ }
+ static const Regex LdRegex("^aarch64\\.sve\\.ld[234](.nxv[a-z0-9]+|$)");
+ if (LdRegex.match(Name)) {
+ Type *ScalarTy =
+ dyn_cast<VectorType>(F->getReturnType())->getElementType();
+ ElementCount EC =
+ dyn_cast<VectorType>(F->arg_begin()->getType())->getElementCount();
+ Type *Ty = VectorType::get(ScalarTy, EC);
+ Intrinsic::ID ID =
+ StringSwitch<Intrinsic::ID>(Name)
+ .StartsWith("aarch64.sve.ld2", Intrinsic::aarch64_sve_ld2_sret)
+ .StartsWith("aarch64.sve.ld3", Intrinsic::aarch64_sve_ld3_sret)
+ .StartsWith("aarch64.sve.ld4", Intrinsic::aarch64_sve_ld4_sret)
+ .Default(Intrinsic::not_intrinsic);
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Ty);
+ return true;
+ }
+ if (Name.startswith("aarch64.sve.tuple.get")) {
+ Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_extract, Tys);
+ return true;
+ }
+ if (Name.startswith("aarch64.sve.tuple.set")) {
+ auto Args = F->getFunctionType()->params();
+ Type *Tys[] = {Args[0], Args[2], Args[1]};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_insert, Tys);
+ return true;
+ }
+ static const Regex CreateTupleRegex(
+ "^aarch64\\.sve\\.tuple\\.create[234](.nxv[a-z0-9]+|$)");
+ if (CreateTupleRegex.match(Name)) {
+ auto Args = F->getFunctionType()->params();
+ Type *Tys[] = {F->getReturnType(), Args[1]};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_insert, Tys);
+ return true;
+ }
if (Name.startswith("arm.neon.vclz")) {
Type* args[2] = {
F->arg_begin()->getType(),
@@ -808,6 +908,13 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
}
break;
}
+ case 'f':
+ if (Name.startswith("flt.rounds")) {
+ rename(F);
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::get_rounding);
+ return true;
+ }
+ break;
case 'i':
case 'l': {
bool IsLifetimeStart = Name.startswith("lifetime.start");
@@ -993,9 +1100,9 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
}
} else if (Name.startswith("ptr.annotation.") && F->arg_size() == 4) {
rename(F);
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::ptr_annotation,
- F->arg_begin()->getType());
+ NewFn = Intrinsic::getDeclaration(
+ F->getParent(), Intrinsic::ptr_annotation,
+ {F->arg_begin()->getType(), F->getArg(1)->getType()});
return true;
}
break;
@@ -1010,8 +1117,9 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
case 'v': {
if (Name == "var.annotation" && F->arg_size() == 4) {
rename(F);
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::var_annotation);
+ NewFn = Intrinsic::getDeclaration(
+ F->getParent(), Intrinsic::var_annotation,
+ {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
return true;
}
break;
@@ -1040,7 +1148,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
Name, F->getParent());
// The new function may also need remangling.
- if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F))
+ if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
NewFn = *Result;
return true;
}
@@ -1048,7 +1156,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
// Remangle our intrinsic since we upgrade the mangling
auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
- if (Result != None) {
+ if (Result != std::nullopt) {
NewFn = *Result;
return true;
}
@@ -1131,7 +1239,7 @@ static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
Idxs[l + i] = Idx + l;
}
- Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));
+ Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
}
// Bitcast back to a 64-bit element type.
@@ -1165,7 +1273,7 @@ static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
Idxs[l + i] = Idx + l;
}
- Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));
+ Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
}
// Bitcast back to a 64-bit element type.
@@ -1185,8 +1293,8 @@ static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
int Indices[4];
for (unsigned i = 0; i != NumElts; ++i)
Indices[i] = i;
- Mask = Builder.CreateShuffleVector(
- Mask, Mask, makeArrayRef(Indices, NumElts), "extract");
+ Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
+ "extract");
}
return Mask;
@@ -1260,9 +1368,8 @@ static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
}
}
- Value *Align = Builder.CreateShuffleVector(Op1, Op0,
- makeArrayRef(Indices, NumElts),
- "palignr");
+ Value *Align = Builder.CreateShuffleVector(
+ Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
return EmitX86Select(Builder, Mask, Align, Passthru);
}
@@ -1452,7 +1559,7 @@ static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
llvm::PointerType::getUnqual(Data->getType()));
const Align Alignment =
Aligned
- ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)
+ ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
: Align(1);
// If the mask is all ones just emit a regular store.
@@ -1474,8 +1581,9 @@ static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
const Align Alignment =
Aligned
- ? Align(Passthru->getType()->getPrimitiveSizeInBits().getFixedSize() /
- 8)
+ ? Align(
+ Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
+ 8)
: Align(1);
// If the mask is all ones just emit a regular store.
@@ -1957,13 +2065,17 @@ static Value *UpgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
/// Upgrade a call to an old intrinsic. All argument and return casting must be
/// provided to seamlessly integrate with existing context.
void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
- Function *F = CI->getCalledFunction();
+ // Note dyn_cast to Function is not quite the same as getCalledFunction, which
+ // checks the callee's function type matches. It's likely we need to handle
+ // type changes here.
+ Function *F = dyn_cast<Function>(CI->getCalledOperand());
+ if (!F)
+ return;
+
LLVMContext &C = CI->getContext();
IRBuilder<> Builder(C);
Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
- assert(F && "Intrinsic call is not direct?");
-
if (!NewFn) {
// Get the Function's name.
StringRef Name = F->getName();
@@ -2024,7 +2136,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
"cast");
StoreInst *SI = Builder.CreateAlignedStore(
Arg1, BC,
- Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
+ Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
SI->setMetadata(M->getMDKindID("nontemporal"), Node);
// Remove intrinsic.
@@ -2164,14 +2276,13 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
// First extract half of each vector. This gives better codegen than
// doing it in a single shuffle.
- LHS = Builder.CreateShuffleVector(LHS, LHS,
- makeArrayRef(Indices, NumElts / 2));
- RHS = Builder.CreateShuffleVector(RHS, RHS,
- makeArrayRef(Indices, NumElts / 2));
+ LHS =
+ Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
+ RHS =
+ Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
// Concat the vectors.
// NOTE: Operands have to be swapped to match intrinsic definition.
- Rep = Builder.CreateShuffleVector(RHS, LHS,
- makeArrayRef(Indices, NumElts));
+ Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
Rep = Builder.CreateBitCast(Rep, CI->getType());
} else if (IsX86 && Name == "avx512.kand.w") {
Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
@@ -3365,7 +3476,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
Ptr, PointerType::getUnqual(CI->getType()), "cast");
LoadInst *LI = Builder.CreateAlignedLoad(
CI->getType(), BC,
- Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
+ Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
LI->setMetadata(M->getMDKindID("nontemporal"), Node);
Rep = LI;
} else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
@@ -3824,21 +3935,29 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
}
// This must be an upgrade from a named to a literal struct.
- auto *OldST = cast<StructType>(CI->getType());
- assert(OldST != NewFn->getReturnType() && "Return type must have changed");
- assert(OldST->getNumElements() ==
- cast<StructType>(NewFn->getReturnType())->getNumElements() &&
- "Must have same number of elements");
+ if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
+ assert(OldST != NewFn->getReturnType() &&
+ "Return type must have changed");
+ assert(OldST->getNumElements() ==
+ cast<StructType>(NewFn->getReturnType())->getNumElements() &&
+ "Must have same number of elements");
- SmallVector<Value *> Args(CI->args());
- Value *NewCI = Builder.CreateCall(NewFn, Args);
- Value *Res = PoisonValue::get(OldST);
- for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
- Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
- Res = Builder.CreateInsertValue(Res, Elem, Idx);
+ SmallVector<Value *> Args(CI->args());
+ Value *NewCI = Builder.CreateCall(NewFn, Args);
+ Value *Res = PoisonValue::get(OldST);
+ for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
+ Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
+ Res = Builder.CreateInsertValue(Res, Elem, Idx);
+ }
+ CI->replaceAllUsesWith(Res);
+ CI->eraseFromParent();
+ return;
}
- CI->replaceAllUsesWith(Res);
- CI->eraseFromParent();
+
+ // We're probably about to produce something invalid. Let the verifier catch
+ // it instead of dying here.
+ CI->setCalledOperand(
+ ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
return;
};
CallInst *NewCall = nullptr;
@@ -3858,6 +3977,94 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
NewCall = Builder.CreateCall(NewFn, Args);
break;
}
+ case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
+ case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
+ case Intrinsic::aarch64_sve_bfdot_lane_v2: {
+ LLVMContext &Ctx = F->getParent()->getContext();
+ SmallVector<Value *, 4> Args(CI->args());
+ Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
+ cast<ConstantInt>(Args[3])->getZExtValue());
+ NewCall = Builder.CreateCall(NewFn, Args);
+ break;
+ }
+ case Intrinsic::aarch64_sve_ld3_sret:
+ case Intrinsic::aarch64_sve_ld4_sret:
+ case Intrinsic::aarch64_sve_ld2_sret: {
+ StringRef Name = F->getName();
+ Name = Name.substr(5);
+ unsigned N = StringSwitch<unsigned>(Name)
+ .StartsWith("aarch64.sve.ld2", 2)
+ .StartsWith("aarch64.sve.ld3", 3)
+ .StartsWith("aarch64.sve.ld4", 4)
+ .Default(0);
+ ScalableVectorType *RetTy =
+ dyn_cast<ScalableVectorType>(F->getReturnType());
+ unsigned MinElts = RetTy->getMinNumElements() / N;
+ SmallVector<Value *, 2> Args(CI->args());
+ Value *NewLdCall = Builder.CreateCall(NewFn, Args);
+ Value *Ret = llvm::PoisonValue::get(RetTy);
+ for (unsigned I = 0; I < N; I++) {
+ Value *Idx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
+ Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
+ Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
+ }
+ NewCall = dyn_cast<CallInst>(Ret);
+ break;
+ }
+
+ case Intrinsic::vector_extract: {
+ StringRef Name = F->getName();
+ Name = Name.substr(5); // Strip llvm
+ if (!Name.startswith("aarch64.sve.tuple.get")) {
+ DefaultCase();
+ return;
+ }
+ ScalableVectorType *RetTy =
+ dyn_cast<ScalableVectorType>(F->getReturnType());
+ unsigned MinElts = RetTy->getMinNumElements();
+ unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
+ Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
+ NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
+ break;
+ }
+
+ case Intrinsic::vector_insert: {
+ StringRef Name = F->getName();
+ Name = Name.substr(5);
+ if (!Name.startswith("aarch64.sve.tuple")) {
+ DefaultCase();
+ return;
+ }
+ if (Name.startswith("aarch64.sve.tuple.set")) {
+ unsigned I = dyn_cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
+ ScalableVectorType *Ty =
+ dyn_cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
+ Value *NewIdx =
+ ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
+ NewCall = Builder.CreateCall(
+ NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
+ break;
+ }
+ if (Name.startswith("aarch64.sve.tuple.create")) {
+ unsigned N = StringSwitch<unsigned>(Name)
+ .StartsWith("aarch64.sve.tuple.create2", 2)
+ .StartsWith("aarch64.sve.tuple.create3", 3)
+ .StartsWith("aarch64.sve.tuple.create4", 4)
+ .Default(0);
+ assert(N > 1 && "Create is expected to be between 2-4");
+ ScalableVectorType *RetTy =
+ dyn_cast<ScalableVectorType>(F->getReturnType());
+ Value *Ret = llvm::PoisonValue::get(RetTy);
+ unsigned MinElts = RetTy->getMinNumElements() / N;
+ for (unsigned I = 0; I < N; I++) {
+ Value *Idx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
+ Value *V = CI->getArgOperand(I);
+ Ret = Builder.CreateInsertVector(RetTy, Ret, V, Idx);
+ }
+ NewCall = dyn_cast<CallInst>(Ret);
+ }
+ break;
+ }
case Intrinsic::arm_neon_bfdot:
case Intrinsic::arm_neon_bfmmla:
@@ -3946,13 +4153,17 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
case Intrinsic::var_annotation:
// Upgrade from versions that lacked the annotation attribute argument.
- assert(CI->arg_size() == 4 &&
- "Before LLVM 12.0 this intrinsic took four arguments");
+ if (CI->arg_size() != 4) {
+ DefaultCase();
+ return;
+ }
// Create a new call with an added null annotation attribute argument.
NewCall = Builder.CreateCall(
NewFn,
{CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
+ NewCall->takeName(CI);
+ CI->replaceAllUsesWith(NewCall);
CI->eraseFromParent();
return;
@@ -4054,6 +4265,43 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
return;
}
+ case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
+ case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
+ case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
+ case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
+ case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
+ case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
+ SmallVector<Value *, 4> Args(CI->args());
+ unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
+ if (NewFn->getIntrinsicID() ==
+ Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
+ Args[1] = Builder.CreateBitCast(
+ Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
+
+ NewCall = Builder.CreateCall(NewFn, Args);
+ Value *Res = Builder.CreateBitCast(
+ NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
+
+ NewCall->takeName(CI);
+ CI->replaceAllUsesWith(Res);
+ CI->eraseFromParent();
+ return;
+ }
+ case Intrinsic::x86_avx512bf16_dpbf16ps_128:
+ case Intrinsic::x86_avx512bf16_dpbf16ps_256:
+ case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
+ SmallVector<Value *, 4> Args(CI->args());
+ unsigned NumElts =
+ cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
+ Args[1] = Builder.CreateBitCast(
+ Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
+ Args[2] = Builder.CreateBitCast(
+ Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
+
+ NewCall = Builder.CreateCall(NewFn, Args);
+ break;
+ }
+
case Intrinsic::thread_pointer: {
NewCall = Builder.CreateCall(NewFn, {});
break;
@@ -4391,26 +4639,34 @@ bool llvm::UpgradeModuleFlags(Module &M) {
MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
if (!ID)
continue;
+ auto SetBehavior = [&](Module::ModFlagBehavior B) {
+ Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt32Ty(M.getContext()), B)),
+ MDString::get(M.getContext(), ID->getString()),
+ Op->getOperand(2)};
+ ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
+ Changed = true;
+ };
+
if (ID->getString() == "Objective-C Image Info Version")
HasObjCFlag = true;
if (ID->getString() == "Objective-C Class Properties")
HasClassProperties = true;
- // Upgrade PIC/PIE Module Flags. The module flag behavior for these two
- // field was Error and now they are Max.
- if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") {
+ // Upgrade PIC from Error/Max to Min.
+ if (ID->getString() == "PIC Level") {
if (auto *Behavior =
mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
- if (Behavior->getLimitedValue() == Module::Error) {
- Type *Int32Ty = Type::getInt32Ty(M.getContext());
- Metadata *Ops[3] = {
- ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)),
- MDString::get(M.getContext(), ID->getString()),
- Op->getOperand(2)};
- ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
- Changed = true;
- }
+ uint64_t V = Behavior->getLimitedValue();
+ if (V == Module::Error || V == Module::Max)
+ SetBehavior(Module::Min);
}
}
+ // Upgrade "PIE Level" from Error to Max.
+ if (ID->getString() == "PIE Level")
+ if (auto *Behavior =
+ mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
+ if (Behavior->getLimitedValue() == Module::Error)
+ SetBehavior(Module::Max);
// Upgrade branch protection and return address signing module flags. The
// module flag behavior for these fields were Error and now they are Min.
@@ -4639,6 +4895,14 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
return DL.empty() ? std::string("G1") : (DL + "-G1").str();
}
+ if (T.isRISCV64()) {
+ // Make i32 a native type for 64-bit RISC-V.
+ auto I = DL.find("-n64-");
+ if (I != StringRef::npos)
+ return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
+ return DL.str();
+ }
+
std::string Res = DL.str();
if (!T.isX86())
return Res;
generated by cgit v1.3 (git 2.53.0) at 2026-05-13 22:33:25 +0000