diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h')
| -rw-r--r-- | contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h | 248 |
1 files changed, 171 insertions, 77 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h index 3f842ce96407..8159f5e8b790 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -15,12 +15,15 @@ #include "CGValue.h" #include "clang/AST/DeclOpenMP.h" +#include "clang/AST/GlobalDecl.h" #include "clang/AST/Type.h" #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/SourceLocation.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" +#include "llvm/Frontend/OpenMP/OMPConstants.h" #include "llvm/IR/Function.h" #include "llvm/IR/ValueHandle.h" @@ -36,7 +39,6 @@ class Value; namespace clang { class Expr; -class GlobalDecl; class OMPDependClause; class OMPExecutableDirective; class OMPLoopDirective; @@ -211,6 +213,43 @@ public: ~DisableAutoDeclareTargetRAII(); }; + /// Manages list of nontemporal decls for the specified directive. + class NontemporalDeclsRAII { + CodeGenModule &CGM; + const bool NeedToPush; + + public: + NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S); + ~NontemporalDeclsRAII(); + }; + + /// Maps the expression for the lastprivate variable to the global copy used + /// to store new value because original variables are not mapped in inner + /// parallel regions. Only private copies are captured but we need also to + /// store private copy in shared address. + /// Also, stores the expression for the private loop counter and it + /// threaprivate name. + struct LastprivateConditionalData { + llvm::SmallDenseMap<CanonicalDeclPtr<const Decl>, SmallString<16>> + DeclToUniqeName; + LValue IVLVal; + SmallString<16> IVName; + /// True if original lvalue for loop counter can be used in codegen (simd + /// region or simd only mode) and no need to create threadprivate + /// references. + bool UseOriginalIV = false; + }; + /// Manages list of lastprivate conditional decls for the specified directive. + class LastprivateConditionalRAII { + CodeGenModule &CGM; + const bool NeedToPush; + + public: + LastprivateConditionalRAII(CodeGenFunction &CGF, + const OMPExecutableDirective &S, LValue IVLVal); + ~LastprivateConditionalRAII(); + }; + protected: CodeGenModule &CGM; StringRef FirstSeparator, Separator; @@ -241,17 +280,6 @@ protected: bool IsOffloadEntry, const RegionCodeGenTy &CodeGen); - /// Emits code for OpenMP 'if' clause using specified \a CodeGen - /// function. Here is the logic: - /// if (Cond) { - /// ThenGen(); - /// } else { - /// ElseGen(); - /// } - void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, - const RegionCodeGenTy &ThenGen, - const RegionCodeGenTy &ElseGen); - /// Emits object of ident_t type with info for source location. /// \param Flags Flags for OpenMP location. /// @@ -291,6 +319,17 @@ protected: /// default location. virtual unsigned getDefaultLocationReserved2Flags() const { return 0; } + /// Tries to emit declare variant function for \p OldGD from \p NewGD. + /// \param OrigAddr LLVM IR value for \p OldGD. + /// \param IsForDefinition true, if requested emission for the definition of + /// \p OldGD. + /// \returns true, was able to emit a definition function for \p OldGD, which + /// points to \p NewGD. + virtual bool tryEmitDeclareVariant(const GlobalDecl &NewGD, + const GlobalDecl &OldGD, + llvm::GlobalValue *OrigAddr, + bool IsForDefinition); + /// Returns default flags for the barriers depending on the directive, for /// which this barier is going to be emitted. static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind); @@ -345,6 +384,14 @@ private: SmallVector<const OMPDeclareReductionDecl *, 4>> FunctionUDRMapTy; FunctionUDRMapTy FunctionUDRMap; + /// Map from the user-defined mapper declaration to its corresponding + /// functions. + llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap; + /// Map of functions and their local user-defined mappers. + using FunctionUDMMapTy = + llvm::DenseMap<llvm::Function *, + SmallVector<const OMPDeclareMapperDecl *, 4>>; + FunctionUDMMapTy FunctionUDMMap; /// Type kmp_critical_name, originally defined as typedef kmp_int32 /// kmp_critical_name[8]; llvm::ArrayType *KmpCriticalNameTy; @@ -392,29 +439,10 @@ private: /// // (function or global) /// char *name; // Name of the function or global. /// size_t size; // Size of the entry info (0 if it a function). + /// int32_t flags; + /// int32_t reserved; /// }; QualType TgtOffloadEntryQTy; - /// struct __tgt_device_image{ - /// void *ImageStart; // Pointer to the target code start. - /// void *ImageEnd; // Pointer to the target code end. - /// // We also add the host entries to the device image, as it may be useful - /// // for the target runtime to have access to that information. - /// __tgt_offload_entry *EntriesBegin; // Begin of the table with all - /// // the entries. - /// __tgt_offload_entry *EntriesEnd; // End of the table with all the - /// // entries (non inclusive). - /// }; - QualType TgtDeviceImageQTy; - /// struct __tgt_bin_desc{ - /// int32_t NumDevices; // Number of devices supported. - /// __tgt_device_image *DeviceImages; // Arrays of device images - /// // (one per device). - /// __tgt_offload_entry *EntriesBegin; // Begin of the table with all the - /// // entries. - /// __tgt_offload_entry *EntriesEnd; // End of the table with all the - /// // entries (non inclusive). - /// }; - QualType TgtBinaryDescriptorQTy; /// Entity that registers the offloading constants that were emitted so /// far. class OffloadEntriesInfoManagerTy { @@ -626,8 +654,8 @@ private: OffloadEntriesInfoManagerTy OffloadEntriesInfoManager; bool ShouldMarkAsGlobal = true; - /// List of the emitted functions. - llvm::StringSet<> AlreadyEmittedTargetFunctions; + /// List of the emitted declarations. + llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls; /// List of the global variables with their addresses that should not be /// emitted for the target. llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables; @@ -636,6 +664,22 @@ private: /// must be emitted. llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables; + /// Mapping of the original functions to their variants and original global + /// decl. + llvm::MapVector<CanonicalDeclPtr<const FunctionDecl>, + std::pair<GlobalDecl, GlobalDecl>> + DeferredVariantFunction; + + using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>; + /// Stack for list of declarations in current context marked as nontemporal. + /// The set is the union of all current stack elements. + llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack; + + /// Stack for list of addresses of declarations in current context marked as + /// lastprivate conditional. The set is the union of all current stack + /// elements. + llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack; + /// Flag for keeping track of weather a requires unified_shared_memory /// directive is present. bool HasRequiresUnifiedSharedMemory = false; @@ -647,14 +691,6 @@ private: /// Device routines are specific to the bool HasEmittedDeclareTargetRegion = false; - /// Creates and registers offloading binary descriptor for the current - /// compilation unit. The function that does the registration is returned. - llvm::Function *createOffloadingBinaryDescriptorRegistration(); - - /// Creates all the offload entries in the current compilation unit - /// along with the associated metadata. - void createOffloadEntriesAndInfoMetadata(); - /// Loads all the offload entries information from the host IR /// metadata. void loadOffloadInfoMetadata(); @@ -662,12 +698,6 @@ private: /// Returns __tgt_offload_entry type. QualType getTgtOffloadEntryQTy(); - /// Returns __tgt_device_image type. - QualType getTgtDeviceImageQTy(); - - /// Returns __tgt_bin_desc type. - QualType getTgtBinaryDescriptorQTy(); - /// Start scanning from statement \a S and and emit all target regions /// found along the way. /// \param S Starting statement. @@ -738,6 +768,14 @@ private: llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc); + /// Emit the array initialization or deletion portion for user-defined mapper + /// code generation. + void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF, + llvm::Value *Handle, llvm::Value *BasePtr, + llvm::Value *Ptr, llvm::Value *Size, + llvm::Value *MapType, CharUnits ElementSize, + llvm::BasicBlock *ExitBB, bool IsInit); + struct TaskResultTy { llvm::Value *NewTask = nullptr; llvm::Function *TaskEntry = nullptr; @@ -777,12 +815,34 @@ private: /// default. virtual unsigned getDefaultFirstprivateAddressSpace() const { return 0; } + /// Emit code that pushes the trip count of loops associated with constructs + /// 'target teams distribute' and 'teams distribute parallel for'. + /// \param SizeEmitter Emits the int64 value for the number of iterations of + /// the associated loop. + void emitTargetNumIterationsCall( + CodeGenFunction &CGF, const OMPExecutableDirective &D, + llvm::Value *DeviceID, + llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, + const OMPLoopDirective &D)> + SizeEmitter); + public: explicit CGOpenMPRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM, ".", ".") {} virtual ~CGOpenMPRuntime() {} virtual void clear(); + /// Emits code for OpenMP 'if' clause using specified \a CodeGen + /// function. Here is the logic: + /// if (Cond) { + /// ThenGen(); + /// } else { + /// ElseGen(); + /// } + void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, + const RegionCodeGenTy &ThenGen, + const RegionCodeGenTy &ElseGen); + /// Checks if the \p Body is the \a CompoundStmt and returns its child /// statement iff there is only one that is not evaluatable at the compile /// time. @@ -798,6 +858,10 @@ public: virtual std::pair<llvm::Function *, llvm::Function *> getUserDefinedReduction(const OMPDeclareReductionDecl *D); + /// Emit the function for the user defined mapper construct. + void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, + CodeGenFunction *CGF = nullptr); + /// Emits outlined function for the specified OpenMP parallel directive /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, /// kmp_int32 BoundID, struct context_vars*). @@ -1106,7 +1170,7 @@ public: /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. virtual void emitProcBindClause(CodeGenFunction &CGF, - OpenMPProcBindClauseKind ProcBind, + llvm::omp::ProcBindKind ProcBind, SourceLocation Loc); /// Returns address of the threadprivate variable for the current @@ -1394,15 +1458,6 @@ public: bool IsOffloadEntry, const RegionCodeGenTy &CodeGen); - /// Emit code that pushes the trip count of loops associated with constructs - /// 'target teams distribute' and 'teams distribute parallel for'. - /// \param SizeEmitter Emits the int64 value for the number of iterations of - /// the associated loop. - virtual void emitTargetNumIterationsCall( - CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *Device, - const llvm::function_ref<llvm::Value *( - CodeGenFunction &CGF, const OMPLoopDirective &D)> &SizeEmitter); - /// Emit the target offloading code associated with \a D. The emitted /// code attempts offloading the execution to the device, an the event of /// a failure it executes the host version outlined in \a OutlinedFn. @@ -1413,11 +1468,15 @@ public: /// directive, or null if no if clause is used. /// \param Device Expression evaluated in device clause associated with the /// target directive, or null if no device clause is used. - virtual void emitTargetCall(CodeGenFunction &CGF, - const OMPExecutableDirective &D, - llvm::Function *OutlinedFn, - llvm::Value *OutlinedFnID, const Expr *IfCond, - const Expr *Device); + /// \param SizeEmitter Callback to emit number of iterations for loop-based + /// directives. + virtual void + emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, + llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, + const Expr *IfCond, const Expr *Device, + llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, + const OMPLoopDirective &D)> + SizeEmitter); /// Emit the target regions enclosed in \a GD function definition or /// the function itself in case it is a valid device function. Returns true if @@ -1449,10 +1508,9 @@ public: /// requires directives was used in the current module. llvm::Function *emitRequiresDirectiveRegFun(); - /// Creates the offloading descriptor in the event any target region - /// was emitted in the current module and return the function that registers - /// it. - virtual llvm::Function *emitRegistrationFunction(); + /// Creates all the offload entries in the current compilation unit + /// along with the associated metadata. + void createOffloadEntriesAndInfoMetadata(); /// Emits code for teams call of the \a OutlinedFn with /// variables captured in a record which address is stored in \a @@ -1626,6 +1684,43 @@ public: /// Return whether the unified_shared_memory has been specified. bool hasRequiresUnifiedSharedMemory() const; + + /// Emits the definition of the declare variant function. + virtual bool emitDeclareVariant(GlobalDecl GD, bool IsForDefinition); + + /// Checks if the \p VD variable is marked as nontemporal declaration in + /// current context. + bool isNontemporalDecl(const ValueDecl *VD) const; + + /// Initializes global counter for lastprivate conditional. + virtual void + initLastprivateConditionalCounter(CodeGenFunction &CGF, + const OMPExecutableDirective &S); + + /// Checks if the provided \p LVal is lastprivate conditional and emits the + /// code to update the value of the original variable. + /// \code + /// lastprivate(conditional: a) + /// ... + /// <type> a; + /// lp_a = ...; + /// #pragma omp critical(a) + /// if (last_iv_a <= iv) { + /// last_iv_a = iv; + /// global_a = lp_a; + /// } + /// \endcode + virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, + const Expr *LHS); + + /// Gets the address of the global copy used for lastprivate conditional + /// update, if any. + /// \param PrivLVal LValue for the private copy. + /// \param VD Original lastprivate declaration. + virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, + LValue PrivLVal, + const VarDecl *VD, + SourceLocation Loc); }; /// Class supports emissionof SIMD-only code. @@ -1854,7 +1949,7 @@ public: /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. void emitProcBindClause(CodeGenFunction &CGF, - OpenMPProcBindClauseKind ProcBind, + llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override; /// Returns address of the threadprivate variable for the current @@ -2097,9 +2192,13 @@ public: /// directive, or null if no if clause is used. /// \param Device Expression evaluated in device clause associated with the /// target directive, or null if no device clause is used. - void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, - llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, - const Expr *IfCond, const Expr *Device) override; + void + emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, + llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, + const Expr *IfCond, const Expr *Device, + llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, + const OMPLoopDirective &D)> + SizeEmitter) override; /// Emit the target regions enclosed in \a GD function definition or /// the function itself in case it is a valid device function. Returns true if @@ -2117,11 +2216,6 @@ public: /// \param GD Global to scan. bool emitTargetGlobal(GlobalDecl GD) override; - /// Creates the offloading descriptor in the event any target region - /// was emitted in the current module and return the function that registers - /// it. - llvm::Function *emitRegistrationFunction() override; - /// Emits code for teams call of the \a OutlinedFn with /// variables captured in a record which address is stored in \a /// CapturedStruct. |