summaryrefslogtreecommitdiff
path: root/lib/CodeGen
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen')
-rw-r--r--lib/CodeGen/CGBuiltin.cpp8
-rw-r--r--lib/CodeGen/CGExpr.cpp4
-rw-r--r--lib/CodeGen/CGExprCXX.cpp26
-rw-r--r--lib/CodeGen/CGOpenMPRuntime.cpp143
-rw-r--r--lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp31
-rw-r--r--lib/CodeGen/CGStmtOpenMP.cpp169
-rw-r--r--lib/CodeGen/CodeGenFunction.h19
7 files changed, 308 insertions, 92 deletions
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index 609987c4fa4c8..ba54f8342f1b5 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -8143,12 +8143,18 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
case X86::BI__builtin_ia32_storesd128_mask: {
return EmitX86MaskedStore(*this, Ops, 16);
}
+ case X86::BI__builtin_ia32_vpopcntb_128:
case X86::BI__builtin_ia32_vpopcntd_128:
case X86::BI__builtin_ia32_vpopcntq_128:
+ case X86::BI__builtin_ia32_vpopcntw_128:
+ case X86::BI__builtin_ia32_vpopcntb_256:
case X86::BI__builtin_ia32_vpopcntd_256:
case X86::BI__builtin_ia32_vpopcntq_256:
+ case X86::BI__builtin_ia32_vpopcntw_256:
+ case X86::BI__builtin_ia32_vpopcntb_512:
case X86::BI__builtin_ia32_vpopcntd_512:
- case X86::BI__builtin_ia32_vpopcntq_512: {
+ case X86::BI__builtin_ia32_vpopcntq_512:
+ case X86::BI__builtin_ia32_vpopcntw_512: {
llvm::Type *ResultType = ConvertType(E->getType());
llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
return Builder.CreateCall(F, Ops);
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index 90eeddf5cc0b6..c7dc8337e19e0 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -570,7 +570,7 @@ static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
- TCK == TCK_UpcastToVirtualBase;
+ TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
}
bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
@@ -578,7 +578,7 @@ bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
(TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
- TCK == TCK_UpcastToVirtualBase);
+ TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
}
bool CodeGenFunction::sanitizePerformTypeCheck() const {
diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp
index 0749b0ac46a78..c32f1e5415da9 100644
--- a/lib/CodeGen/CGExprCXX.cpp
+++ b/lib/CodeGen/CGExprCXX.cpp
@@ -2056,6 +2056,15 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
// Get the vtable pointer.
Address ThisPtr = CGF.EmitLValue(E).getAddress();
+ QualType SrcRecordTy = E->getType();
+
+ // C++ [class.cdtor]p4:
+ // If the operand of typeid refers to the object under construction or
+ // destruction and the static type of the operand is neither the constructor
+ // or destructor’s class nor one of its bases, the behavior is undefined.
+ CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(),
+ ThisPtr.getPointer(), SrcRecordTy);
+
// C++ [expr.typeid]p2:
// If the glvalue expression is obtained by applying the unary * operator to
// a pointer and the pointer is a null pointer value, the typeid expression
@@ -2064,7 +2073,6 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
// However, this paragraph's intent is not clear. We choose a very generous
// interpretation which implores us to consider comma operators, conditional
// operators, parentheses and other such constructs.
- QualType SrcRecordTy = E->getType();
if (CGF.CGM.getCXXABI().shouldTypeidBeNullChecked(
isGLValueFromPointerDeref(E), SrcRecordTy)) {
llvm::BasicBlock *BadTypeidBlock =
@@ -2127,10 +2135,6 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
CGM.EmitExplicitCastExprType(DCE, this);
QualType DestTy = DCE->getTypeAsWritten();
- if (DCE->isAlwaysNull())
- if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy))
- return T;
-
QualType SrcTy = DCE->getSubExpr()->getType();
// C++ [expr.dynamic.cast]p7:
@@ -2151,6 +2155,18 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
DestRecordTy = DestTy->castAs<ReferenceType>()->getPointeeType();
}
+ // C++ [class.cdtor]p5:
+ // If the operand of the dynamic_cast refers to the object under
+ // construction or destruction and the static type of the operand is not a
+ // pointer to or object of the constructor or destructor’s own class or one
+ // of its bases, the dynamic_cast results in undefined behavior.
+ EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr.getPointer(),
+ SrcRecordTy);
+
+ if (DCE->isAlwaysNull())
+ if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy))
+ return T;
+
assert(SrcRecordTy->isRecordType() && "source type must be a record type!");
// C++ [expr.dynamic.cast]p4:
diff --git a/lib/CodeGen/CGOpenMPRuntime.cpp b/lib/CodeGen/CGOpenMPRuntime.cpp
index 5db29eb6004d7..fa38ee80bf41e 100644
--- a/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -4175,14 +4175,23 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
LValue SrcBase;
- if (!Data.FirstprivateVars.empty()) {
+ bool IsTargetTask =
+ isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
+ isOpenMPTargetExecutionDirective(D.getDirectiveKind());
+ // For target-based directives skip 3 firstprivate arrays BasePointersArray,
+ // PointersArray and SizesArray. The original variables for these arrays are
+ // not captured and we get their addresses explicitly.
+ if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
+ (IsTargetTask && Data.FirstprivateVars.size() > 3)) {
SrcBase = CGF.MakeAddrLValue(
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
SharedsTy);
}
- CodeGenFunction::CGCapturedStmtInfo CapturesInfo(
- cast<CapturedStmt>(*D.getAssociatedStmt()));
+ OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
+ ? OMPD_taskloop
+ : OMPD_task;
+ CodeGenFunction::CGCapturedStmtInfo CapturesInfo(*D.getCapturedStmt(Kind));
FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
for (auto &&Pair : Privates) {
auto *VD = Pair.second.PrivateCopy;
@@ -4192,14 +4201,27 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
if (auto *Elem = Pair.second.PrivateElemInit) {
auto *OriginalVD = Pair.second.Original;
- auto *SharedField = CapturesInfo.lookup(OriginalVD);
- auto SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
- SharedRefLValue = CGF.MakeAddrLValue(
- Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
- SharedRefLValue.getType(),
- LValueBaseInfo(AlignmentSource::Decl),
- SharedRefLValue.getTBAAInfo());
+ // Check if the variable is the target-based BasePointersArray,
+ // PointersArray or SizesArray.
+ LValue SharedRefLValue;
QualType Type = OriginalVD->getType();
+ if (IsTargetTask && isa<ImplicitParamDecl>(OriginalVD) &&
+ isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
+ cast<CapturedDecl>(OriginalVD->getDeclContext())->getNumParams() ==
+ 0 &&
+ isa<TranslationUnitDecl>(
+ cast<CapturedDecl>(OriginalVD->getDeclContext())
+ ->getDeclContext())) {
+ SharedRefLValue =
+ CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
+ } else {
+ auto *SharedField = CapturesInfo.lookup(OriginalVD);
+ SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
+ SharedRefLValue = CGF.MakeAddrLValue(
+ Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
+ SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
+ SharedRefLValue.getTBAAInfo());
+ }
if (Type->isArrayType()) {
// Initialize firstprivate array.
if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
@@ -4400,8 +4422,10 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
}
KmpTaskTQTy = SavedKmpTaskloopTQTy;
} else {
- assert(D.getDirectiveKind() == OMPD_task &&
- "Expected taskloop or task directive");
+ assert((D.getDirectiveKind() == OMPD_task ||
+ isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
+ isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
+ "Expected taskloop, task or target directive");
if (SavedKmpTaskTQTy.isNull()) {
SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
@@ -7417,8 +7441,8 @@ void CGOpenMPRuntime::emitTargetDataCalls(
// Generate the code for the opening of the data environment. Capture all the
// arguments of the runtime call by reference because they are used in the
// closing of the region.
- auto &&BeginThenGen = [&D, Device, &Info, &CodeGen](CodeGenFunction &CGF,
- PrePostActionTy &) {
+ auto &&BeginThenGen = [this, &D, Device, &Info,
+ &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
// Fill up the arrays with all the mapped variables.
MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
MappableExprsHandler::MapValuesArrayTy Pointers;
@@ -7454,8 +7478,7 @@ void CGOpenMPRuntime::emitTargetDataCalls(
llvm::Value *OffloadingArgs[] = {
DeviceID, PointerNum, BasePointersArrayArg,
PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
- auto &RT = CGF.CGM.getOpenMPRuntime();
- CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_begin),
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_begin),
OffloadingArgs);
// If device pointer privatization is required, emit the body of the region
@@ -7465,7 +7488,8 @@ void CGOpenMPRuntime::emitTargetDataCalls(
};
// Generate code for the closing of the data region.
- auto &&EndThenGen = [Device, &Info](CodeGenFunction &CGF, PrePostActionTy &) {
+ auto &&EndThenGen = [this, Device, &Info](CodeGenFunction &CGF,
+ PrePostActionTy &) {
assert(Info.isValid() && "Invalid data environment closing arguments.");
llvm::Value *BasePointersArrayArg = nullptr;
@@ -7490,8 +7514,7 @@ void CGOpenMPRuntime::emitTargetDataCalls(
llvm::Value *OffloadingArgs[] = {
DeviceID, PointerNum, BasePointersArrayArg,
PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
- auto &RT = CGF.CGM.getOpenMPRuntime();
- CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_end),
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_end),
OffloadingArgs);
};
@@ -7543,25 +7566,11 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
isa<OMPTargetUpdateDirective>(D)) &&
"Expecting either target enter, exit data, or update directives.");
+ CodeGenFunction::OMPTargetDataInfo InputInfo;
+ llvm::Value *MapTypesArray = nullptr;
// Generate the code for the opening of the data environment.
- auto &&ThenGen = [&D, Device](CodeGenFunction &CGF, PrePostActionTy &) {
- // Fill up the arrays with all the mapped variables.
- MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
- MappableExprsHandler::MapValuesArrayTy Pointers;
- MappableExprsHandler::MapValuesArrayTy Sizes;
- MappableExprsHandler::MapFlagsArrayTy MapTypes;
-
- // Get map clause information.
- MappableExprsHandler MEHandler(D, CGF);
- MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
-
- // Fill up the arrays and create the arguments.
- TargetDataInfo Info;
- emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
- emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
- Info.PointersArray, Info.SizesArray,
- Info.MapTypesArray, Info);
-
+ auto &&ThenGen = [this, &D, Device, &InputInfo,
+ &MapTypesArray](CodeGenFunction &CGF, PrePostActionTy &) {
// Emit device ID if any.
llvm::Value *DeviceID = nullptr;
if (Device) {
@@ -7572,13 +7581,16 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
}
// Emit the number of elements in the offloading arrays.
- auto *PointerNum = CGF.Builder.getInt32(BasePointers.size());
+ llvm::Constant *PointerNum =
+ CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
- llvm::Value *OffloadingArgs[] = {
- DeviceID, PointerNum, Info.BasePointersArray,
- Info.PointersArray, Info.SizesArray, Info.MapTypesArray};
+ llvm::Value *OffloadingArgs[] = {DeviceID,
+ PointerNum,
+ InputInfo.BasePointersArray.getPointer(),
+ InputInfo.PointersArray.getPointer(),
+ InputInfo.SizesArray.getPointer(),
+ MapTypesArray};
- auto &RT = CGF.CGM.getOpenMPRuntime();
// Select the right runtime function call for each expected standalone
// directive.
const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
@@ -7600,18 +7612,47 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
: OMPRTL__tgt_target_data_update;
break;
}
- CGF.EmitRuntimeCall(RT.createRuntimeFunction(RTLFn), OffloadingArgs);
+ CGF.EmitRuntimeCall(createRuntimeFunction(RTLFn), OffloadingArgs);
};
- // In the event we get an if clause, we don't have to take any action on the
- // else side.
- auto &&ElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
+ auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray](
+ CodeGenFunction &CGF, PrePostActionTy &) {
+ // Fill up the arrays with all the mapped variables.
+ MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
+ MappableExprsHandler::MapValuesArrayTy Pointers;
+ MappableExprsHandler::MapValuesArrayTy Sizes;
+ MappableExprsHandler::MapFlagsArrayTy MapTypes;
- if (IfCond) {
- emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
- } else {
- RegionCodeGenTy ThenGenRCG(ThenGen);
- ThenGenRCG(CGF);
+ // Get map clause information.
+ MappableExprsHandler MEHandler(D, CGF);
+ MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
+
+ TargetDataInfo Info;
+ // Fill up the arrays and create the arguments.
+ emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
+ emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
+ Info.PointersArray, Info.SizesArray,
+ Info.MapTypesArray, Info);
+ InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
+ InputInfo.BasePointersArray =
+ Address(Info.BasePointersArray, CGM.getPointerAlign());
+ InputInfo.PointersArray =
+ Address(Info.PointersArray, CGM.getPointerAlign());
+ InputInfo.SizesArray =
+ Address(Info.SizesArray, CGM.getPointerAlign());
+ MapTypesArray = Info.MapTypesArray;
+ if (D.hasClausesOfKind<OMPDependClause>())
+ CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
+ else
+ emitInlinedDirective(CGF, OMPD_target_update, ThenGen);
+ };
+
+ if (IfCond)
+ emitOMPIfClause(CGF, IfCond, TargetThenGen,
+ [](CodeGenFunction &CGF, PrePostActionTy &) {});
+ else {
+ RegionCodeGenTy ThenRCG(TargetThenGen);
+ ThenRCG(CGF);
}
}
diff --git a/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp b/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
index b5fc8d308067e..7b2993cfd38db 100644
--- a/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
+++ b/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
@@ -33,10 +33,11 @@ enum OpenMPRTLFunctionNVPTX {
/// \brief Call to void __kmpc_spmd_kernel_deinit();
OMPRTL_NVPTX__kmpc_spmd_kernel_deinit,
/// \brief Call to void __kmpc_kernel_prepare_parallel(void
- /// *outlined_function, void ***args, kmp_int32 nArgs);
+ /// *outlined_function, void ***args, kmp_int32 nArgs, int16_t
+ /// IsOMPRuntimeInitialized);
OMPRTL_NVPTX__kmpc_kernel_prepare_parallel,
/// \brief Call to bool __kmpc_kernel_parallel(void **outlined_function, void
- /// ***args);
+ /// ***args, int16_t IsOMPRuntimeInitialized);
OMPRTL_NVPTX__kmpc_kernel_parallel,
/// \brief Call to void __kmpc_kernel_end_parallel();
OMPRTL_NVPTX__kmpc_kernel_end_parallel,
@@ -521,7 +522,9 @@ void CGOpenMPRuntimeNVPTX::emitWorkerLoop(CodeGenFunction &CGF,
// Set up shared arguments
Address SharedArgs =
CGF.CreateDefaultAlignTempAlloca(CGF.Int8PtrPtrTy, "shared_args");
- llvm::Value *Args[] = {WorkFn.getPointer(), SharedArgs.getPointer()};
+ // TODO: Optimize runtime initialization and pass in correct value.
+ llvm::Value *Args[] = {WorkFn.getPointer(), SharedArgs.getPointer(),
+ /*RequiresOMPRuntime=*/Bld.getInt16(1)};
llvm::Value *Ret = CGF.EmitRuntimeCall(
createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_parallel), Args);
Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus);
@@ -637,18 +640,21 @@ CGOpenMPRuntimeNVPTX::createNVPTXRuntimeFunction(unsigned Function) {
}
case OMPRTL_NVPTX__kmpc_kernel_prepare_parallel: {
/// Build void __kmpc_kernel_prepare_parallel(
- /// void *outlined_function, void ***args, kmp_int32 nArgs);
+ /// void *outlined_function, void ***args, kmp_int32 nArgs, int16_t
+ /// IsOMPRuntimeInitialized);
llvm::Type *TypeParams[] = {CGM.Int8PtrTy,
- CGM.Int8PtrPtrTy->getPointerTo(0), CGM.Int32Ty};
+ CGM.Int8PtrPtrTy->getPointerTo(0), CGM.Int32Ty,
+ CGM.Int16Ty};
llvm::FunctionType *FnTy =
llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_prepare_parallel");
break;
}
case OMPRTL_NVPTX__kmpc_kernel_parallel: {
- /// Build bool __kmpc_kernel_parallel(void **outlined_function, void ***args);
+ /// Build bool __kmpc_kernel_parallel(void **outlined_function, void
+ /// ***args, int16_t IsOMPRuntimeInitialized);
llvm::Type *TypeParams[] = {CGM.Int8PtrPtrTy,
- CGM.Int8PtrPtrTy->getPointerTo(0)};
+ CGM.Int8PtrPtrTy->getPointerTo(0), CGM.Int16Ty};
llvm::Type *RetTy = CGM.getTypes().ConvertType(CGM.getContext().BoolTy);
llvm::FunctionType *FnTy =
llvm::FunctionType::get(RetTy, TypeParams, /*isVarArg*/ false);
@@ -949,8 +955,10 @@ void CGOpenMPRuntimeNVPTX::emitGenericParallelCall(
CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy,
"shared_args");
llvm::Value *SharedArgsPtr = SharedArgs.getPointer();
+ // TODO: Optimize runtime initialization and pass in correct value.
llvm::Value *Args[] = {ID, SharedArgsPtr,
- Bld.getInt32(CapturedVars.size())};
+ Bld.getInt32(CapturedVars.size()),
+ /*RequiresOMPRuntime=*/Bld.getInt16(1)};
CGF.EmitRuntimeCall(
createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_prepare_parallel),
@@ -970,9 +978,10 @@ void CGOpenMPRuntimeNVPTX::emitGenericParallelCall(
Idx++;
}
} else {
- llvm::Value *Args[] = {ID,
- llvm::ConstantPointerNull::get(CGF.VoidPtrPtrTy->getPointerTo(0)),
- /*nArgs=*/Bld.getInt32(0)};
+ // TODO: Optimize runtime initialization and pass in correct value.
+ llvm::Value *Args[] = {
+ ID, llvm::ConstantPointerNull::get(CGF.VoidPtrPtrTy->getPointerTo(0)),
+ /*nArgs=*/Bld.getInt32(0), /*RequiresOMPRuntime=*/Bld.getInt16(1)};
CGF.EmitRuntimeCall(
createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_prepare_parallel),
Args);
diff --git a/lib/CodeGen/CGStmtOpenMP.cpp b/lib/CodeGen/CGStmtOpenMP.cpp
index f04d28ed0d4a9..f9861735832bf 100644
--- a/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/lib/CodeGen/CGStmtOpenMP.cpp
@@ -2907,6 +2907,151 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
TaskGen(*this, OutlinedFn, Data);
}
+static ImplicitParamDecl *
+createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
+ QualType Ty, CapturedDecl *CD) {
+ auto *OrigVD = ImplicitParamDecl::Create(
+ C, CD, SourceLocation(), /*Id=*/nullptr, Ty, ImplicitParamDecl::Other);
+ auto *OrigRef =
+ DeclRefExpr::Create(C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
+ /*RefersToEnclosingVariableOrCapture=*/false,
+ SourceLocation(), Ty, VK_LValue);
+ auto *PrivateVD = ImplicitParamDecl::Create(
+ C, CD, SourceLocation(), /*Id=*/nullptr, Ty, ImplicitParamDecl::Other);
+ auto *PrivateRef = DeclRefExpr::Create(
+ C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
+ /*RefersToEnclosingVariableOrCapture=*/false, SourceLocation(), Ty,
+ VK_LValue);
+ QualType ElemType = C.getBaseElementType(Ty);
+ auto *InitVD =
+ ImplicitParamDecl::Create(C, CD, SourceLocation(), /*Id=*/nullptr,
+ ElemType, ImplicitParamDecl::Other);
+ auto *InitRef =
+ DeclRefExpr::Create(C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
+ /*RefersToEnclosingVariableOrCapture=*/false,
+ SourceLocation(), ElemType, VK_LValue);
+ PrivateVD->setInitStyle(VarDecl::CInit);
+ PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
+ InitRef, /*BasePath=*/nullptr,
+ VK_RValue));
+ Data.FirstprivateVars.emplace_back(OrigRef);
+ Data.FirstprivateCopies.emplace_back(PrivateRef);
+ Data.FirstprivateInits.emplace_back(InitRef);
+ return OrigVD;
+}
+
+void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
+ const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
+ OMPTargetDataInfo &InputInfo) {
+ // Emit outlined function for task construct.
+ auto CS = S.getCapturedStmt(OMPD_task);
+ auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
+ auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
+ auto *I = CS->getCapturedDecl()->param_begin();
+ auto *PartId = std::next(I);
+ auto *TaskT = std::next(I, 4);
+ OMPTaskDataTy Data;
+ // The task is not final.
+ Data.Final.setInt(/*IntVal=*/false);
+ // Get list of firstprivate variables.
+ for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
+ auto IRef = C->varlist_begin();
+ auto IElemInitRef = C->inits().begin();
+ for (auto *IInit : C->private_copies()) {
+ Data.FirstprivateVars.push_back(*IRef);
+ Data.FirstprivateCopies.push_back(IInit);
+ Data.FirstprivateInits.push_back(*IElemInitRef);
+ ++IRef;
+ ++IElemInitRef;
+ }
+ }
+ OMPPrivateScope TargetScope(*this);
+ VarDecl *BPVD = nullptr;
+ VarDecl *PVD = nullptr;
+ VarDecl *SVD = nullptr;
+ if (InputInfo.NumberOfTargetItems > 0) {
+ auto *CD = CapturedDecl::Create(
+ getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
+ llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
+ QualType BaseAndPointersType = getContext().getConstantArrayType(
+ getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
+ /*IndexTypeQuals=*/0);
+ BPVD = createImplicitFirstprivateForType(getContext(), Data,
+ BaseAndPointersType, CD);
+ PVD = createImplicitFirstprivateForType(getContext(), Data,
+ BaseAndPointersType, CD);
+ QualType SizesType = getContext().getConstantArrayType(
+ getContext().getSizeType(), ArrSize, ArrayType::Normal,
+ /*IndexTypeQuals=*/0);
+ SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD);
+ TargetScope.addPrivate(
+ BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
+ TargetScope.addPrivate(PVD,
+ [&InputInfo]() { return InputInfo.PointersArray; });
+ TargetScope.addPrivate(SVD,
+ [&InputInfo]() { return InputInfo.SizesArray; });
+ }
+ (void)TargetScope.Privatize();
+ // Build list of dependences.
+ for (const auto *C : S.getClausesOfKind<OMPDependClause>())
+ for (auto *IRef : C->varlists())
+ Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
+ auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
+ &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
+ // Set proper addresses for generated private copies.
+ OMPPrivateScope Scope(CGF);
+ if (!Data.FirstprivateVars.empty()) {
+ enum { PrivatesParam = 2, CopyFnParam = 3 };
+ auto *CopyFn = CGF.Builder.CreateLoad(
+ CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
+ auto *PrivatesPtr = CGF.Builder.CreateLoad(
+ CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
+ // Map privates.
+ llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
+ llvm::SmallVector<llvm::Value *, 16> CallArgs;
+ CallArgs.push_back(PrivatesPtr);
+ for (auto *E : Data.FirstprivateVars) {
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ Address PrivatePtr =
+ CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
+ ".firstpriv.ptr.addr");
+ PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
+ CallArgs.push_back(PrivatePtr.getPointer());
+ }
+ CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
+ CopyFn, CallArgs);
+ for (auto &&Pair : PrivatePtrs) {
+ Address Replacement(CGF.Builder.CreateLoad(Pair.second),
+ CGF.getContext().getDeclAlign(Pair.first));
+ Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
+ }
+ }
+ // Privatize all private variables except for in_reduction items.
+ (void)Scope.Privatize();
+ InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
+ CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
+ InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
+ CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
+ InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
+ CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
+
+ Action.Enter(CGF);
+ OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true,
+ /*EmitPreInitStmt=*/false);
+ BodyGen(CGF);
+ };
+ auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
+ S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
+ Data.NumberOfParts);
+ llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
+ IntegerLiteral IfCond(getContext(), TrueOrFalse,
+ getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
+ SourceLocation());
+
+ CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, OutlinedFn,
+ SharedsTy, CapturedStruct, &IfCond, Data);
+}
+
void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
// Emit outlined function for task construct.
auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
@@ -4252,14 +4397,8 @@ void CodeGenFunction::EmitOMPTargetEnterDataDirective(
if (auto *C = S.getSingleClause<OMPDeviceClause>())
Device = C->getDevice();
- auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
- PrePostActionTy &) {
- CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
- Device);
- };
OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_enter_data,
- CodeGen);
+ CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
}
void CodeGenFunction::EmitOMPTargetExitDataDirective(
@@ -4279,14 +4418,8 @@ void CodeGenFunction::EmitOMPTargetExitDataDirective(
if (auto *C = S.getSingleClause<OMPDeviceClause>())
Device = C->getDevice();
- auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
- PrePostActionTy &) {
- CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
- Device);
- };
OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_exit_data,
- CodeGen);
+ CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
}
static void emitTargetParallelRegion(CodeGenFunction &CGF,
@@ -4585,12 +4718,6 @@ void CodeGenFunction::EmitOMPTargetUpdateDirective(
if (auto *C = S.getSingleClause<OMPDeviceClause>())
Device = C->getDevice();
- auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
- PrePostActionTy &) {
- CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
- Device);
- };
OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_update,
- CodeGen);
+ CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
}
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index cd62d00dfb536..dd4c2e43ef64b 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -2371,7 +2371,10 @@ public:
/// object within its lifetime.
TCK_UpcastToVirtualBase,
/// Checking the value assigned to a _Nonnull pointer. Must not be null.
- TCK_NonnullAssign
+ TCK_NonnullAssign,
+ /// Checking the operand of a dynamic_cast or a typeid expression. Must be
+ /// null or an object within its lifetime.
+ TCK_DynamicOperation
};
/// Determine whether the pointer type check \p TCK permits null pointers.
@@ -2820,6 +2823,20 @@ public:
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
const RegionCodeGenTy &BodyGen,
const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
+ struct OMPTargetDataInfo {
+ Address BasePointersArray = Address::invalid();
+ Address PointersArray = Address::invalid();
+ Address SizesArray = Address::invalid();
+ unsigned NumberOfTargetItems = 0;
+ explicit OMPTargetDataInfo() = default;
+ OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
+ Address SizesArray, unsigned NumberOfTargetItems)
+ : BasePointersArray(BasePointersArray), PointersArray(PointersArray),
+ SizesArray(SizesArray), NumberOfTargetItems(NumberOfTargetItems) {}
+ };
+ void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
+ const RegionCodeGenTy &BodyGen,
+ OMPTargetDataInfo &InputInfo);
void EmitOMPParallelDirective(const OMPParallelDirective &S);
void EmitOMPSimdDirective(const OMPSimdDirective &S);
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-02 01:06:08 +0000