diff options
Diffstat (limited to 'contrib/llvm-project')
220 files changed, 5012 insertions, 7382 deletions
diff --git a/contrib/llvm-project/clang/include/clang/AST/ASTContext.h b/contrib/llvm-project/clang/include/clang/AST/ASTContext.h index ce47d54e44b0..ae69a68608b7 100644 --- a/contrib/llvm-project/clang/include/clang/AST/ASTContext.h +++ b/contrib/llvm-project/clang/include/clang/AST/ASTContext.h @@ -538,6 +538,9 @@ private: /// need them (like static local vars). llvm::MapVector<const NamedDecl *, unsigned> MangleNumbers; llvm::MapVector<const VarDecl *, unsigned> StaticLocalNumbers; + /// Mapping the associated device lambda mangling number if present. + mutable llvm::DenseMap<const CXXRecordDecl *, unsigned> + DeviceLambdaManglingNumbers; /// Mapping that stores parameterIndex values for ParmVarDecls when /// that value exceeds the bitfield size of ParmVarDeclBits.ParameterIndex. diff --git a/contrib/llvm-project/clang/include/clang/AST/Decl.h b/contrib/llvm-project/clang/include/clang/AST/Decl.h index 47c282f0a63d..1c5827b9c3a4 100644 --- a/contrib/llvm-project/clang/include/clang/AST/Decl.h +++ b/contrib/llvm-project/clang/include/clang/AST/Decl.h @@ -1276,15 +1276,12 @@ public: EvaluatedStmt *getEvaluatedStmt() const; /// Attempt to evaluate the value of the initializer attached to this - /// declaration, and produce notes explaining why it cannot be evaluated. - /// Returns a pointer to the value if evaluation succeeded, 0 otherwise. + /// declaration, and produce notes explaining why it cannot be evaluated or is + /// not a constant expression. Returns a pointer to the value if evaluation + /// succeeded, 0 otherwise. APValue *evaluateValue() const; + APValue *evaluateValue(SmallVectorImpl<PartialDiagnosticAt> &Notes) const; -private: - APValue *evaluateValueImpl(SmallVectorImpl<PartialDiagnosticAt> &Notes, - bool IsConstantInitialization) const; - -public: /// Return the already-evaluated value of this variable's /// initializer, or NULL if the value is not yet known. Returns pointer /// to untyped APValue if the value could not be evaluated. diff --git a/contrib/llvm-project/clang/include/clang/AST/DeclCXX.h b/contrib/llvm-project/clang/include/clang/AST/DeclCXX.h index e32101bb2276..89006b1cfa7f 100644 --- a/contrib/llvm-project/clang/include/clang/AST/DeclCXX.h +++ b/contrib/llvm-project/clang/include/clang/AST/DeclCXX.h @@ -1735,6 +1735,12 @@ public: getLambdaData().HasKnownInternalLinkage = HasKnownInternalLinkage; } + /// Set the device side mangling number. + void setDeviceLambdaManglingNumber(unsigned Num) const; + + /// Retrieve the device side mangling number. + unsigned getDeviceLambdaManglingNumber() const; + /// Returns the inheritance model used for this record. MSInheritanceModel getMSInheritanceModel() const; diff --git a/contrib/llvm-project/clang/include/clang/AST/Expr.h b/contrib/llvm-project/clang/include/clang/AST/Expr.h index a44d06967431..52f8f18af205 100644 --- a/contrib/llvm-project/clang/include/clang/AST/Expr.h +++ b/contrib/llvm-project/clang/include/clang/AST/Expr.h @@ -699,8 +699,7 @@ public: /// notes will be produced if the expression is not a constant expression. bool EvaluateAsInitializer(APValue &Result, const ASTContext &Ctx, const VarDecl *VD, - SmallVectorImpl<PartialDiagnosticAt> &Notes, - bool IsConstantInitializer) const; + SmallVectorImpl<PartialDiagnosticAt> &Notes) const; /// EvaluateWithSubstitution - Evaluate an expression as if from the context /// of a call to the given function with the given arguments, inside an diff --git a/contrib/llvm-project/clang/include/clang/AST/Mangle.h b/contrib/llvm-project/clang/include/clang/AST/Mangle.h index 0e8d6dd53d8a..7b6495d85eb6 100644 --- a/contrib/llvm-project/clang/include/clang/AST/Mangle.h +++ b/contrib/llvm-project/clang/include/clang/AST/Mangle.h @@ -96,6 +96,9 @@ public: virtual bool shouldMangleCXXName(const NamedDecl *D) = 0; virtual bool shouldMangleStringLiteral(const StringLiteral *SL) = 0; + virtual bool isDeviceMangleContext() const { return false; } + virtual void setDeviceMangleContext(bool) {} + // FIXME: consider replacing raw_ostream & with something like SmallString &. void mangleName(GlobalDecl GD, raw_ostream &); virtual void mangleCXXName(GlobalDecl GD, raw_ostream &) = 0; diff --git a/contrib/llvm-project/clang/include/clang/AST/MangleNumberingContext.h b/contrib/llvm-project/clang/include/clang/AST/MangleNumberingContext.h index f1ca6a05dbaf..eb33759682d6 100644 --- a/contrib/llvm-project/clang/include/clang/AST/MangleNumberingContext.h +++ b/contrib/llvm-project/clang/include/clang/AST/MangleNumberingContext.h @@ -52,6 +52,11 @@ public: /// this context. virtual unsigned getManglingNumber(const TagDecl *TD, unsigned MSLocalManglingNumber) = 0; + + /// Retrieve the mangling number of a new lambda expression with the + /// given call operator within the device context. No device number is + /// assigned if there's no device numbering context is associated. + virtual unsigned getDeviceManglingNumber(const CXXMethodDecl *) { return 0; } }; } // end namespace clang diff --git a/contrib/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h b/contrib/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h index 505ea700fd0e..7870cea198a7 100644 --- a/contrib/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h +++ b/contrib/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h @@ -186,6 +186,9 @@ public: /// code, e.g., implicit constructors and destructors. bool shouldVisitImplicitCode() const { return false; } + /// Return whether this visitor should recurse into lambda body + bool shouldVisitLambdaBody() const { return true; } + /// Return whether this visitor should traverse post-order. bool shouldTraversePostOrder() const { return false; } @@ -2057,6 +2060,15 @@ bool RecursiveASTVisitor<Derived>::TraverseFunctionHelper(FunctionDecl *D) { // by clang. (!D->isDefaulted() || getDerived().shouldVisitImplicitCode()); + if (const auto *MD = dyn_cast<CXXMethodDecl>(D)) { + if (const CXXRecordDecl *RD = MD->getParent()) { + if (RD->isLambda() && + declaresSameEntity(RD->getLambdaCallOperator(), MD)) { + VisitBody = VisitBody && getDerived().shouldVisitLambdaBody(); + } + } + } + if (VisitBody) { TRY_TO(TraverseStmt(D->getBody())); // Function body. } diff --git a/contrib/llvm-project/clang/include/clang/ASTMatchers/ASTMatchers.h b/contrib/llvm-project/clang/include/clang/ASTMatchers/ASTMatchers.h index 6f6dfab59a39..031fa4682c3a 100644 --- a/contrib/llvm-project/clang/include/clang/ASTMatchers/ASTMatchers.h +++ b/contrib/llvm-project/clang/include/clang/ASTMatchers/ASTMatchers.h @@ -344,7 +344,7 @@ extern const internal::VariadicAllOfMatcher<Decl> decl; /// int number = 42; /// auto [foo, bar] = std::make_pair{42, 42}; /// \endcode -extern const internal::VariadicAllOfMatcher<DecompositionDecl> +extern const internal::VariadicDynCastAllOfMatcher<Decl, DecompositionDecl> decompositionDecl; /// Matches a declaration of a linkage specification. diff --git a/contrib/llvm-project/clang/include/clang/Basic/CodeGenOptions.def b/contrib/llvm-project/clang/include/clang/Basic/CodeGenOptions.def index 5c8af65326ed..9d53b5b923bb 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/CodeGenOptions.def +++ b/contrib/llvm-project/clang/include/clang/Basic/CodeGenOptions.def @@ -266,6 +266,9 @@ CODEGENOPT(VectorizeLoop , 1, 0) ///< Run loop vectorizer. CODEGENOPT(VectorizeSLP , 1, 0) ///< Run SLP vectorizer. CODEGENOPT(ProfileSampleAccurate, 1, 0) ///< Sample profile is accurate. +/// Treat loops as finite: language, always, never. +ENUM_CODEGENOPT(FiniteLoops, FiniteLoopsKind, 2, FiniteLoopsKind::Language) + /// Attempt to use register sized accesses to bit-fields in structures, when /// possible. CODEGENOPT(UseRegisterSizedBitfieldAccess , 1, 0) diff --git a/contrib/llvm-project/clang/include/clang/Basic/CodeGenOptions.h b/contrib/llvm-project/clang/include/clang/Basic/CodeGenOptions.h index 73d41e3293c6..c550817f0f69 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/CodeGenOptions.h +++ b/contrib/llvm-project/clang/include/clang/Basic/CodeGenOptions.h @@ -140,6 +140,12 @@ public: All, // Keep all frame pointers. }; + enum FiniteLoopsKind { + Language, // Not specified, use language standard. + Always, // All loops are assumed to be finite. + Never, // No loop is assumed to be finite. + }; + /// The code model to use (-mcmodel). std::string CodeModel; diff --git a/contrib/llvm-project/clang/include/clang/Driver/Options.td b/contrib/llvm-project/clang/include/clang/Driver/Options.td index 42c5319041d0..817798926650 100644 --- a/contrib/llvm-project/clang/include/clang/Driver/Options.td +++ b/contrib/llvm-project/clang/include/clang/Driver/Options.td @@ -1147,7 +1147,7 @@ def fprofile_update_EQ : Joined<["-"], "fprofile-update=">, defm pseudo_probe_for_profiling : BoolFOption<"pseudo-probe-for-profiling", CodeGenOpts<"PseudoProbeForProfiling">, DefaultFalse, PosFlag<SetTrue, [], "Emit">, NegFlag<SetFalse, [], "Do not emit">, - BothFlags<[NoXarchOption, CC1Option], " pseudo probes for sample profiler">>; + BothFlags<[NoXarchOption, CC1Option], " pseudo probes for sample profiling">>; def forder_file_instrumentation : Flag<["-"], "forder-file-instrumentation">, Group<f_Group>, Flags<[CC1Option, CoreOption]>, HelpText<"Generate instrumented code to collect order file into default.profraw file (overridden by '=' form of option or LLVM_PROFILE_FILE env var)">; @@ -2410,6 +2410,11 @@ def fno_unroll_loops : Flag<["-"], "fno-unroll-loops">, Group<f_Group>, defm reroll_loops : BoolFOption<"reroll-loops", CodeGenOpts<"RerollLoops">, DefaultFalse, PosFlag<SetTrue, [CC1Option], "Turn on loop reroller">, NegFlag<SetFalse>>; +def ffinite_loops: Flag<["-"], "ffinite-loops">, Group<f_Group>, + HelpText<"Assume all loops are finite.">, Flags<[CC1Option]>; +def fno_finite_loops: Flag<["-"], "fno-finite-loops">, Group<f_Group>, + HelpText<"Do not assume that any loop is finite.">, Flags<[CC1Option]>; + def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>, HelpText<"Process trigraph sequences">, Flags<[CC1Option]>; def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>, diff --git a/contrib/llvm-project/clang/include/clang/Lex/VariadicMacroSupport.h b/contrib/llvm-project/clang/include/clang/Lex/VariadicMacroSupport.h index 989e0ac703c9..119f02201fc6 100644 --- a/contrib/llvm-project/clang/include/clang/Lex/VariadicMacroSupport.h +++ b/contrib/llvm-project/clang/include/clang/Lex/VariadicMacroSupport.h @@ -39,17 +39,14 @@ namespace clang { assert(Ident__VA_ARGS__->isPoisoned() && "__VA_ARGS__ should be poisoned " "outside an ISO C/C++ variadic " "macro definition!"); - assert( - !Ident__VA_OPT__ || - (Ident__VA_OPT__->isPoisoned() && "__VA_OPT__ should be poisoned!")); + assert(Ident__VA_OPT__->isPoisoned() && "__VA_OPT__ should be poisoned!"); } /// Client code should call this function just before the Preprocessor is /// about to Lex tokens from the definition of a variadic (ISO C/C++) macro. void enterScope() { Ident__VA_ARGS__->setIsPoisoned(false); - if (Ident__VA_OPT__) - Ident__VA_OPT__->setIsPoisoned(false); + Ident__VA_OPT__->setIsPoisoned(false); } /// Client code should call this function as soon as the Preprocessor has @@ -58,8 +55,7 @@ namespace clang { /// (might be explicitly called, and then reinvoked via the destructor). void exitScope() { Ident__VA_ARGS__->setIsPoisoned(true); - if (Ident__VA_OPT__) - Ident__VA_OPT__->setIsPoisoned(true); + Ident__VA_OPT__->setIsPoisoned(true); } ~VariadicMacroScopeGuard() { exitScope(); } diff --git a/contrib/llvm-project/clang/include/clang/Sema/Sema.h b/contrib/llvm-project/clang/include/clang/Sema/Sema.h index 7f7c84eb1b1d..2530a2776373 100644 --- a/contrib/llvm-project/clang/include/clang/Sema/Sema.h +++ b/contrib/llvm-project/clang/include/clang/Sema/Sema.h @@ -6558,7 +6558,7 @@ public: /// Number lambda for linkage purposes if necessary. void handleLambdaNumbering( CXXRecordDecl *Class, CXXMethodDecl *Method, - Optional<std::tuple<unsigned, bool, Decl *>> Mangling = None); + Optional<std::tuple<bool, unsigned, unsigned, Decl *>> Mangling = None); /// Endow the lambda scope info with the relevant properties. void buildLambdaScope(sema::LambdaScopeInfo *LSI, @@ -11948,8 +11948,8 @@ public: /// if (diagIfOpenMPDeviceCode(Loc, diag::err_vla_unsupported)) /// return ExprError(); /// // Otherwise, continue parsing as normal. - SemaDiagnosticBuilder diagIfOpenMPDeviceCode(SourceLocation Loc, - unsigned DiagID); + SemaDiagnosticBuilder + diagIfOpenMPDeviceCode(SourceLocation Loc, unsigned DiagID, FunctionDecl *FD); /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current /// context is "used as host code". @@ -11965,17 +11965,19 @@ public: /// return ExprError(); /// // Otherwise, continue parsing as normal. SemaDiagnosticBuilder diagIfOpenMPHostCode(SourceLocation Loc, - unsigned DiagID); + unsigned DiagID, FunctionDecl *FD); - SemaDiagnosticBuilder targetDiag(SourceLocation Loc, unsigned DiagID); + SemaDiagnosticBuilder targetDiag(SourceLocation Loc, unsigned DiagID, + FunctionDecl *FD = nullptr); SemaDiagnosticBuilder targetDiag(SourceLocation Loc, - const PartialDiagnostic &PD) { - return targetDiag(Loc, PD.getDiagID()) << PD; + const PartialDiagnostic &PD, + FunctionDecl *FD = nullptr) { + return targetDiag(Loc, PD.getDiagID(), FD) << PD; } /// Check if the expression is allowed to be used in expressions for the /// offloading devices. - void checkDeviceDecl(const ValueDecl *D, SourceLocation Loc); + void checkDeviceDecl(ValueDecl *D, SourceLocation Loc); enum CUDAFunctionTarget { CFT_Device, diff --git a/contrib/llvm-project/clang/lib/AST/ASTImporter.cpp b/contrib/llvm-project/clang/lib/AST/ASTImporter.cpp index 085c50c0667b..0d723fbbcd8c 100644 --- a/contrib/llvm-project/clang/lib/AST/ASTImporter.cpp +++ b/contrib/llvm-project/clang/lib/AST/ASTImporter.cpp @@ -2848,6 +2848,8 @@ ExpectedDecl ASTNodeImporter::VisitRecordDecl(RecordDecl *D) { return CDeclOrErr.takeError(); D2CXX->setLambdaMangling(DCXX->getLambdaManglingNumber(), *CDeclOrErr, DCXX->hasKnownLambdaInternalLinkage()); + D2CXX->setDeviceLambdaManglingNumber( + DCXX->getDeviceLambdaManglingNumber()); } else if (DCXX->isInjectedClassName()) { // We have to be careful to do a similar dance to the one in // Sema::ActOnStartCXXMemberDeclarations diff --git a/contrib/llvm-project/clang/lib/AST/CXXABI.h b/contrib/llvm-project/clang/lib/AST/CXXABI.h index 31cb36918726..ca9424bcb7a4 100644 --- a/contrib/llvm-project/clang/lib/AST/CXXABI.h +++ b/contrib/llvm-project/clang/lib/AST/CXXABI.h @@ -22,8 +22,9 @@ class ASTContext; class CXXConstructorDecl; class DeclaratorDecl; class Expr; -class MemberPointerType; +class MangleContext; class MangleNumberingContext; +class MemberPointerType; /// Implements C++ ABI-specific semantic analysis functions. class CXXABI { @@ -75,6 +76,8 @@ public: /// Creates an instance of a C++ ABI class. CXXABI *CreateItaniumCXXABI(ASTContext &Ctx); CXXABI *CreateMicrosoftCXXABI(ASTContext &Ctx); +std::unique_ptr<MangleNumberingContext> +createItaniumNumberingContext(MangleContext *); } #endif diff --git a/contrib/llvm-project/clang/lib/AST/Decl.cpp b/contrib/llvm-project/clang/lib/AST/Decl.cpp index feb9b0645ebc..10cfe145b3f0 100644 --- a/contrib/llvm-project/clang/lib/AST/Decl.cpp +++ b/contrib/llvm-project/clang/lib/AST/Decl.cpp @@ -2384,11 +2384,11 @@ EvaluatedStmt *VarDecl::getEvaluatedStmt() const { APValue *VarDecl::evaluateValue() const { SmallVector<PartialDiagnosticAt, 8> Notes; - return evaluateValueImpl(Notes, hasConstantInitialization()); + return evaluateValue(Notes); } -APValue *VarDecl::evaluateValueImpl(SmallVectorImpl<PartialDiagnosticAt> &Notes, - bool IsConstantInitialization) const { +APValue *VarDecl::evaluateValue( + SmallVectorImpl<PartialDiagnosticAt> &Notes) const { EvaluatedStmt *Eval = ensureEvaluatedStmt(); const auto *Init = cast<Expr>(Eval->Value); @@ -2407,16 +2407,8 @@ APValue *VarDecl::evaluateValueImpl(SmallVectorImpl<PartialDiagnosticAt> &Notes, Eval->IsEvaluating = true; - ASTContext &Ctx = getASTContext(); - bool Result = Init->EvaluateAsInitializer(Eval->Evaluated, Ctx, this, Notes, - IsConstantInitialization); - - // In C++11, this isn't a constant initializer if we produced notes. In that - // case, we can't keep the result, because it may only be correct under the - // assumption that the initializer is a constant context. - if (IsConstantInitialization && Ctx.getLangOpts().CPlusPlus11 && - !Notes.empty()) - Result = false; + bool Result = Init->EvaluateAsInitializer(Eval->Evaluated, getASTContext(), + this, Notes); // Ensure the computed APValue is cleaned up later if evaluation succeeded, // or that it's empty (so that there's nothing to clean up) if evaluation @@ -2424,7 +2416,7 @@ APValue *VarDecl::evaluateValueImpl(SmallVectorImpl<PartialDiagnosticAt> &Notes, if (!Result) Eval->Evaluated = APValue(); else if (Eval->Evaluated.needsCleanup()) - Ctx.addDestruction(&Eval->Evaluated); + getASTContext().addDestruction(&Eval->Evaluated); Eval->IsEvaluating = false; Eval->WasEvaluated = true; @@ -2478,14 +2470,7 @@ bool VarDecl::checkForConstantInitialization( assert(!cast<Expr>(Eval->Value)->isValueDependent()); // Evaluate the initializer to check whether it's a constant expression. - Eval->HasConstantInitialization = - evaluateValueImpl(Notes, true) && Notes.empty(); - - // If evaluation as a constant initializer failed, allow re-evaluation as a - // non-constant initializer if we later find we want the value. - if (!Eval->HasConstantInitialization) - Eval->WasEvaluated = false; - + Eval->HasConstantInitialization = evaluateValue(Notes) && Notes.empty(); return Eval->HasConstantInitialization; } diff --git a/contrib/llvm-project/clang/lib/AST/DeclCXX.cpp b/contrib/llvm-project/clang/lib/AST/DeclCXX.cpp index 0368ada0b81c..0375f9b4432e 100644 --- a/contrib/llvm-project/clang/lib/AST/DeclCXX.cpp +++ b/contrib/llvm-project/clang/lib/AST/DeclCXX.cpp @@ -1593,6 +1593,20 @@ Decl *CXXRecordDecl::getLambdaContextDecl() const { return getLambdaData().ContextDecl.get(Source); } +void CXXRecordDecl::setDeviceLambdaManglingNumber(unsigned Num) const { + assert(isLambda() && "Not a lambda closure type!"); + if (Num) + getASTContext().DeviceLambdaManglingNumbers[this] = Num; +} + +unsigned CXXRecordDecl::getDeviceLambdaManglingNumber() const { + assert(isLambda() && "Not a lambda closure type!"); + auto I = getASTContext().DeviceLambdaManglingNumbers.find(this); + if (I != getASTContext().DeviceLambdaManglingNumbers.end()) + return I->second; + return 0; +} + static CanQualType GetConversionType(ASTContext &Context, NamedDecl *Conv) { QualType T = cast<CXXConversionDecl>(Conv->getUnderlyingDecl()->getAsFunction()) diff --git a/contrib/llvm-project/clang/lib/AST/ExprConstant.cpp b/contrib/llvm-project/clang/lib/AST/ExprConstant.cpp index 56181bbe1166..b24025664684 100644 --- a/contrib/llvm-project/clang/lib/AST/ExprConstant.cpp +++ b/contrib/llvm-project/clang/lib/AST/ExprConstant.cpp @@ -3302,9 +3302,12 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E, // Check that we can fold the initializer. In C++, we will have already done // this in the cases where it matters for conformance. - if (!VD->evaluateValue()) { - Info.FFDiag(E, diag::note_constexpr_var_init_non_constant, 1) << VD; + SmallVector<PartialDiagnosticAt, 8> Notes; + if (!VD->evaluateValue(Notes)) { + Info.FFDiag(E, diag::note_constexpr_var_init_non_constant, + Notes.size() + 1) << VD; NoteLValueLocation(Info, Base); + Info.addNotes(Notes); return false; } @@ -3497,8 +3500,8 @@ static bool diagnoseMutableFields(EvalInfo &Info, const Expr *E, AccessKinds AK, static bool lifetimeStartedInEvaluation(EvalInfo &Info, APValue::LValueBase Base, bool MutableSubobject = false) { - // A temporary we created. - if (Base.getCallIndex()) + // A temporary or transient heap allocation we created. + if (Base.getCallIndex() || Base.is<DynamicAllocLValue>()) return true; switch (Info.IsEvaluatingDecl) { @@ -10009,6 +10012,7 @@ bool RecordExprEvaluator::VisitLambdaExpr(const LambdaExpr *E) { auto *CaptureInitIt = E->capture_init_begin(); const LambdaCapture *CaptureIt = ClosureClass->captures_begin(); bool Success = true; + const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(ClosureClass); for (const auto *Field : ClosureClass->fields()) { assert(CaptureInitIt != E->capture_init_end()); // Get the initializer for this field @@ -10019,8 +10023,13 @@ bool RecordExprEvaluator::VisitLambdaExpr(const LambdaExpr *E) { if (!CurFieldInit) return Error(E); + LValue Subobject = This; + + if (!HandleLValueMember(Info, E, Subobject, Field, &Layout)) + return false; + APValue &FieldVal = Result.getStructField(Field->getFieldIndex()); - if (!EvaluateInPlace(FieldVal, Info, This, CurFieldInit)) { + if (!EvaluateInPlace(FieldVal, Info, Subobject, CurFieldInit)) { if (!Info.keepEvaluatingAfterFailure()) return false; Success = false; @@ -14786,11 +14795,14 @@ bool Expr::EvaluateAsLValue(EvalResult &Result, const ASTContext &Ctx, static bool EvaluateDestruction(const ASTContext &Ctx, APValue::LValueBase Base, APValue DestroyedValue, QualType Type, - SourceLocation Loc, Expr::EvalStatus &EStatus) { - EvalInfo Info(Ctx, EStatus, EvalInfo::EM_ConstantExpression); + SourceLocation Loc, Expr::EvalStatus &EStatus, + bool IsConstantDestruction) { + EvalInfo Info(Ctx, EStatus, + IsConstantDestruction ? EvalInfo::EM_ConstantExpression + : EvalInfo::EM_ConstantFold); Info.setEvaluatingDecl(Base, DestroyedValue, EvalInfo::EvaluatingDeclKind::Dtor); - Info.InConstantContext = true; + Info.InConstantContext = IsConstantDestruction; LValue LVal; LVal.set(Base); @@ -14844,7 +14856,8 @@ bool Expr::EvaluateAsConstantExpr(EvalResult &Result, const ASTContext &Ctx, // If this is a class template argument, it's required to have constant // destruction too. if (Kind == ConstantExprKind::ClassTemplateArgument && - (!EvaluateDestruction(Ctx, Base, Result.Val, T, getBeginLoc(), Result) || + (!EvaluateDestruction(Ctx, Base, Result.Val, T, getBeginLoc(), Result, + true) || Result.HasSideEffects)) { // FIXME: Prefix a note to indicate that the problem is lack of constant // destruction. @@ -14856,8 +14869,7 @@ bool Expr::EvaluateAsConstantExpr(EvalResult &Result, const ASTContext &Ctx, bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx, const VarDecl *VD, - SmallVectorImpl<PartialDiagnosticAt> &Notes, - bool IsConstantInitialization) const { + SmallVectorImpl<PartialDiagnosticAt> &Notes) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); @@ -14870,12 +14882,11 @@ bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx, Expr::EvalStatus EStatus; EStatus.Diag = &Notes; - EvalInfo Info(Ctx, EStatus, - (IsConstantInitialization && Ctx.getLangOpts().CPlusPlus11) - ? EvalInfo::EM_ConstantExpression - : EvalInfo::EM_ConstantFold); + EvalInfo Info(Ctx, EStatus, VD->isConstexpr() + ? EvalInfo::EM_ConstantExpression + : EvalInfo::EM_ConstantFold); Info.setEvaluatingDecl(VD, Value); - Info.InConstantContext = IsConstantInitialization; + Info.InConstantContext = true; SourceLocation DeclLoc = VD->getLocation(); QualType DeclTy = VD->getType(); @@ -14910,6 +14921,10 @@ bool VarDecl::evaluateDestruction( Expr::EvalStatus EStatus; EStatus.Diag = &Notes; + // Only treat the destruction as constant destruction if we formally have + // constant initialization (or are usable in a constant expression). + bool IsConstantDestruction = hasConstantInitialization(); + // Make a copy of the value for the destructor to mutate, if we know it. // Otherwise, treat the value as default-initialized; if the destructor works // anyway, then the destruction is constant (and must be essentially empty). @@ -14920,7 +14935,8 @@ bool VarDecl::evaluateDestruction( return false; if (!EvaluateDestruction(getASTContext(), this, std::move(DestroyedValue), - getType(), getLocation(), EStatus) || + getType(), getLocation(), EStatus, + IsConstantDestruction) || EStatus.HasSideEffects) return false; diff --git a/contrib/llvm-project/clang/lib/AST/ItaniumCXXABI.cpp b/contrib/llvm-project/clang/lib/AST/ItaniumCXXABI.cpp index 069add8464ae..be10258a2d77 100644 --- a/contrib/llvm-project/clang/lib/AST/ItaniumCXXABI.cpp +++ b/contrib/llvm-project/clang/lib/AST/ItaniumCXXABI.cpp @@ -258,3 +258,9 @@ public: CXXABI *clang::CreateItaniumCXXABI(ASTContext &Ctx) { return new ItaniumCXXABI(Ctx); } + +std::unique_ptr<MangleNumberingContext> +clang::createItaniumNumberingContext(MangleContext *Mangler) { + return std::make_unique<ItaniumNumberingContext>( + cast<ItaniumMangleContext>(Mangler)); +} diff --git a/contrib/llvm-project/clang/lib/AST/ItaniumMangle.cpp b/contrib/llvm-project/clang/lib/AST/ItaniumMangle.cpp index 6c8d5687c64a..5cad84a96845 100644 --- a/contrib/llvm-project/clang/lib/AST/ItaniumMangle.cpp +++ b/contrib/llvm-project/clang/lib/AST/ItaniumMangle.cpp @@ -125,6 +125,8 @@ class ItaniumMangleContextImpl : public ItaniumMangleContext { llvm::DenseMap<DiscriminatorKeyTy, unsigned> Discriminator; llvm::DenseMap<const NamedDecl*, unsigned> Uniquifier; + bool IsDevCtx = false; + public: explicit ItaniumMangleContextImpl(ASTContext &Context, DiagnosticsEngine &Diags) @@ -137,6 +139,10 @@ public: bool shouldMangleStringLiteral(const StringLiteral *) override { return false; } + + bool isDeviceMangleContext() const override { return IsDevCtx; } + void setDeviceMangleContext(bool IsDev) override { IsDevCtx = IsDev; } + void mangleCXXName(GlobalDecl GD, raw_ostream &) override; void mangleThunk(const CXXMethodDecl *MD, const ThunkInfo &Thunk, raw_ostream &) override; @@ -546,8 +552,8 @@ private: unsigned knownArity); void mangleCastExpression(const Expr *E, StringRef CastEncoding); void mangleInitListElements(const InitListExpr *InitList); - void mangleDeclRefExpr(const NamedDecl *D); - void mangleExpression(const Expr *E, unsigned Arity = UnknownArity); + void mangleExpression(const Expr *E, unsigned Arity = UnknownArity, + bool AsTemplateArg = false); void mangleCXXCtorType(CXXCtorType T, const CXXRecordDecl *InheritedFrom); void mangleCXXDtorType(CXXDtorType T); @@ -558,6 +564,7 @@ private: unsigned NumTemplateArgs); void mangleTemplateArgs(TemplateName TN, const TemplateArgumentList &AL); void mangleTemplateArg(TemplateArgument A, bool NeedExactType); + void mangleTemplateArgExpr(const Expr *E); void mangleValueInTemplateArg(QualType T, const APValue &V, bool TopLevel, bool NeedExactType = false); @@ -726,9 +733,17 @@ void CXXNameMangler::mangleFunctionEncodingBareType(const FunctionDecl *FD) { EnableIfAttr *EIA = dyn_cast<EnableIfAttr>(*I); if (!EIA) continue; - Out << 'X'; - mangleExpression(EIA->getCond()); - Out << 'E'; + if (Context.getASTContext().getLangOpts().getClangABICompat() > + LangOptions::ClangABI::Ver11) { + mangleTemplateArgExpr(EIA->getCond()); + } else { + // Prior to Clang 12, we hardcoded the X/E around enable-if's argument, + // even though <template-arg> should not include an X/E around + // <expr-primary>. + Out << 'X'; + mangleExpression(EIA->getCond()); + Out << 'E'; + } } Out << 'E'; FunctionTypeDepth.pop(Saved); @@ -1837,7 +1852,15 @@ void CXXNameMangler::mangleLambda(const CXXRecordDecl *Lambda) { // (in lexical order) with that same <lambda-sig> and context. // // The AST keeps track of the number for us. - unsigned Number = Lambda->getLambdaManglingNumber(); + // + // In CUDA/HIP, to ensure the consistent lamba numbering between the device- + // and host-side compilations, an extra device mangle context may be created + // if the host-side CXX ABI has different numbering for lambda. In such case, + // if the mangle context is that device-side one, use the device-side lambda + // mangling number for this lambda. + unsigned Number = Context.isDeviceMangleContext() + ? Lambda->getDeviceLambdaManglingNumber() + : Lambda->getLambdaManglingNumber(); assert(Number > 0 && "Lambda should be mangled as an unnamed class"); if (Number > 1) mangleNumber(Number - 2); @@ -3528,8 +3551,8 @@ void CXXNameMangler::mangleType(const DependentSizedMatrixType *T) { Out << "u" << VendorQualifier.size() << VendorQualifier; Out << "I"; - mangleTemplateArg(T->getRowExpr(), false); - mangleTemplateArg(T->getColumnExpr(), false); + mangleTemplateArgExpr(T->getRowExpr()); + mangleTemplateArgExpr(T->getColumnExpr()); mangleType(T->getElementType()); Out << "E"; } @@ -3871,33 +3894,8 @@ void CXXNameMangler::mangleInitListElements(const InitListExpr *InitList) { mangleExpression(InitList->getInit(i)); } -void CXXNameMangler::mangleDeclRefExpr(const NamedDecl *D) { - switch (D->getKind()) { - default: - // <expr-primary> ::= L <mangled-name> E # external name - Out << 'L'; - mangle(D); - Out << 'E'; - break; - - case Decl::ParmVar: - mangleFunctionParam(cast<ParmVarDecl>(D)); - break; - - case Decl::EnumConstant: { - const EnumConstantDecl *ED = cast<EnumConstantDecl>(D); - mangleIntegerLiteral(ED->getType(), ED->getInitVal()); - break; - } - - case Decl::NonTypeTemplateParm: - const NonTypeTemplateParmDecl *PD = cast<NonTypeTemplateParmDecl>(D); - mangleTemplateParameter(PD->getDepth(), PD->getIndex()); - break; - } -} - -void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) { +void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity, + bool AsTemplateArg) { // <expression> ::= <unary operator-name> <expression> // ::= <binary operator-name> <expression> <expression> // ::= <trinary operator-name> <expression> <expression> <expression> @@ -3911,18 +3909,64 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) { // ::= at <type> # alignof (a type) // ::= <template-param> // ::= <function-param> + // ::= fpT # 'this' expression (part of <function-param>) // ::= sr <type> <unqualified-name> # dependent name // ::= sr <type> <unqualified-name> <template-args> # dependent template-id // ::= ds <expression> <expression> # expr.*expr // ::= sZ <template-param> # size of a parameter pack // ::= sZ <function-param> # size of a function parameter pack + // ::= u <source-name> <template-arg>* E # vendor extended expression // ::= <expr-primary> // <expr-primary> ::= L <type> <value number> E # integer literal - // ::= L <type <value float> E # floating literal + // ::= L <type> <value float> E # floating literal + // ::= L <type> <string type> E # string literal + // ::= L <nullptr type> E # nullptr literal "LDnE" + // ::= L <pointer type> 0 E # null pointer template argument + // ::= L <type> <real-part float> _ <imag-part float> E # complex floating point literal (C99); not used by clang // ::= L <mangled-name> E # external name - // ::= fpT # 'this' expression QualType ImplicitlyConvertedToType; + // A top-level expression that's not <expr-primary> needs to be wrapped in + // X...E in a template arg. + bool IsPrimaryExpr = true; + auto NotPrimaryExpr = [&] { + if (AsTemplateArg && IsPrimaryExpr) + Out << 'X'; + IsPrimaryExpr = false; + }; + + auto MangleDeclRefExpr = [&](const NamedDecl *D) { + switch (D->getKind()) { + default: + // <expr-primary> ::= L <mangled-name> E # external name + Out << 'L'; + mangle(D); + Out << 'E'; + break; + + case Decl::ParmVar: + NotPrimaryExpr(); + mangleFunctionParam(cast<ParmVarDecl>(D)); + break; + + case Decl::EnumConstant: { + // <expr-primary> + const EnumConstantDecl *ED = cast<EnumConstantDecl>(D); + mangleIntegerLiteral(ED->getType(), ED->getInitVal()); + break; + } + + case Decl::NonTypeTemplateParm: + NotPrimaryExpr(); + const NonTypeTemplateParmDecl *PD = cast<NonTypeTemplateParmDecl>(D); + mangleTemplateParameter(PD->getDepth(), PD->getIndex()); + break; + } + }; + + // 'goto recurse' is used when handling a simple "unwrapping" node which + // produces no output, where ImplicitlyConvertedToType and AsTemplateArg need + // to be preserved. recurse: switch (E->getStmtClass()) { case Expr::NoStmtClass: @@ -3994,6 +4038,7 @@ recurse: case Expr::SourceLocExprClass: case Expr::BuiltinBitCastExprClass: { + NotPrimaryExpr(); if (!NullOut) { // As bad as this diagnostic is, it's better than crashing. DiagnosticsEngine &Diags = Context.getDiags(); @@ -4001,33 +4046,48 @@ recurse: "cannot yet mangle expression type %0"); Diags.Report(E->getExprLoc(), DiagID) << E->getStmtClassName() << E->getSourceRange(); + return; } break; } case Expr::CXXUuidofExprClass: { + NotPrimaryExpr(); const CXXUuidofExpr *UE = cast<CXXUuidofExpr>(E); - if (UE->isTypeOperand()) { - QualType UuidT = UE->getTypeOperand(Context.getASTContext()); - Out << "u8__uuidoft"; - mangleType(UuidT); + // As of clang 12, uuidof uses the vendor extended expression + // mangling. Previously, it used a special-cased nonstandard extension. + if (Context.getASTContext().getLangOpts().getClangABICompat() > + LangOptions::ClangABI::Ver11) { + Out << "u8__uuidof"; + if (UE->isTypeOperand()) + mangleType(UE->getTypeOperand(Context.getASTContext())); + else + mangleTemplateArgExpr(UE->getExprOperand()); + Out << 'E'; } else { - Expr *UuidExp = UE->getExprOperand(); - Out << "u8__uuidofz"; - mangleExpression(UuidExp, Arity); + if (UE->isTypeOperand()) { + QualType UuidT = UE->getTypeOperand(Context.getASTContext()); + Out << "u8__uuidoft"; + mangleType(UuidT); + } else { + Expr *UuidExp = UE->getExprOperand(); + Out << "u8__uuidofz"; + mangleExpression(UuidExp); + } } break; } // Even gcc-4.5 doesn't mangle this. case Expr::BinaryConditionalOperatorClass: { + NotPrimaryExpr(); DiagnosticsEngine &Diags = Context.getDiags(); unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, "?: operator with omitted middle operand cannot be mangled"); Diags.Report(E->getExprLoc(), DiagID) << E->getStmtClassName() << E->getSourceRange(); - break; + return; } // These are used for internal purposes and cannot be meaningfully mangled. @@ -4035,6 +4095,7 @@ recurse: llvm_unreachable("cannot mangle opaque value; mangling wrong thing?"); case Expr::InitListExprClass: { + NotPrimaryExpr(); Out << "il"; mangleInitListElements(cast<InitListExpr>(E)); Out << "E"; @@ -4042,6 +4103,7 @@ recurse: } case Expr::DesignatedInitExprClass: { + NotPrimaryExpr(); auto *DIE = cast<DesignatedInitExpr>(E); for (const auto &Designator : DIE->designators()) { if (Designator.isFieldDesignator()) { @@ -4063,27 +4125,27 @@ recurse: } case Expr::CXXDefaultArgExprClass: - mangleExpression(cast<CXXDefaultArgExpr>(E)->getExpr(), Arity); - break; + E = cast<CXXDefaultArgExpr>(E)->getExpr(); + goto recurse; case Expr::CXXDefaultInitExprClass: - mangleExpression(cast<CXXDefaultInitExpr>(E)->getExpr(), Arity); - break; + E = cast<CXXDefaultInitExpr>(E)->getExpr(); + goto recurse; case Expr::CXXStdInitializerListExprClass: - mangleExpression(cast<CXXStdInitializerListExpr>(E)->getSubExpr(), Arity); - break; + E = cast<CXXStdInitializerListExpr>(E)->getSubExpr(); + goto recurse; case Expr::SubstNonTypeTemplateParmExprClass: - mangleExpression(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(), - Arity); - break; + E = cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(); + goto recurse; case Expr::UserDefinedLiteralClass: // We follow g++'s approach of mangling a UDL as a call to the literal // operator. case Expr::CXXMemberCallExprClass: // fallthrough case Expr::CallExprClass: { + NotPrimaryExpr(); const CallExpr *CE = cast<CallExpr>(E); // <expression> ::= cp <simple-id> <expression>* E @@ -4114,6 +4176,7 @@ recurse: } case Expr::CXXNewExprClass: { + NotPrimaryExpr(); const CXXNewExpr *New = cast<CXXNewExpr>(E); if (New->isGlobalNew()) Out << "gs"; Out << (New->isArray() ? "na" : "nw"); @@ -4149,6 +4212,7 @@ recurse: } case Expr::CXXPseudoDestructorExprClass: { + NotPrimaryExpr(); const auto *PDE = cast<CXXPseudoDestructorExpr>(E); if (const Expr *Base = PDE->getBase()) mangleMemberExprBase(Base, PDE->isArrow()); @@ -4175,6 +4239,7 @@ recurse: } case Expr::MemberExprClass: { + NotPrimaryExpr(); const MemberExpr *ME = cast<MemberExpr>(E); mangleMemberExpr(ME->getBase(), ME->isArrow(), ME->getQualifier(), nullptr, @@ -4185,6 +4250,7 @@ recurse: } case Expr::UnresolvedMemberExprClass: { + NotPrimaryExpr(); const UnresolvedMemberExpr *ME = cast<UnresolvedMemberExpr>(E); mangleMemberExpr(ME->isImplicitAccess() ? nullptr : ME->getBase(), ME->isArrow(), ME->getQualifier(), nullptr, @@ -4195,6 +4261,7 @@ recurse: } case Expr::CXXDependentScopeMemberExprClass: { + NotPrimaryExpr(); const CXXDependentScopeMemberExpr *ME = cast<CXXDependentScopeMemberExpr>(E); mangleMemberExpr(ME->isImplicitAccess() ? nullptr : ME->getBase(), @@ -4207,6 +4274,7 @@ recurse: } case Expr::UnresolvedLookupExprClass: { + NotPrimaryExpr(); const UnresolvedLookupExpr *ULE = cast<UnresolvedLookupExpr>(E); mangleUnresolvedName(ULE->getQualifier(), ULE->getName(), ULE->getTemplateArgs(), ULE->getNumTemplateArgs(), @@ -4215,6 +4283,7 @@ recurse: } case Expr::CXXUnresolvedConstructExprClass: { + NotPrimaryExpr(); const CXXUnresolvedConstructExpr *CE = cast<CXXUnresolvedConstructExpr>(E); unsigned N = CE->getNumArgs(); @@ -4225,7 +4294,7 @@ recurse: mangleType(CE->getType()); mangleInitListElements(IL); Out << "E"; - return; + break; } Out << "cv"; @@ -4237,14 +4306,17 @@ recurse: } case Expr::CXXConstructExprClass: { + // An implicit cast is silent, thus may contain <expr-primary>. const auto *CE = cast<CXXConstructExpr>(E); if (!CE->isListInitialization() || CE->isStdInitListInitialization()) { assert( CE->getNumArgs() >= 1 && (CE->getNumArgs() == 1 || isa<CXXDefaultArgExpr>(CE->getArg(1))) && "implicit CXXConstructExpr must have one argument"); - return mangleExpression(cast<CXXConstructExpr>(E)->getArg(0)); + E = cast<CXXConstructExpr>(E)->getArg(0); + goto recurse; } + NotPrimaryExpr(); Out << "il"; for (auto *E : CE->arguments()) mangleExpression(E); @@ -4253,6 +4325,7 @@ recurse: } case Expr::CXXTemporaryObjectExprClass: { + NotPrimaryExpr(); const auto *CE = cast<CXXTemporaryObjectExpr>(E); unsigned N = CE->getNumArgs(); bool List = CE->isListInitialization(); @@ -4282,17 +4355,20 @@ recurse: } case Expr::CXXScalarValueInitExprClass: + NotPrimaryExpr(); Out << "cv"; mangleType(E->getType()); Out << "_E"; break; case Expr::CXXNoexceptExprClass: + NotPrimaryExpr(); Out << "nx"; mangleExpression(cast<CXXNoexceptExpr>(E)->getOperand()); break; case Expr::UnaryExprOrTypeTraitExprClass: { + // Non-instantiation-dependent traits are an <expr-primary> integer literal. const UnaryExprOrTypeTraitExpr *SAE = cast<UnaryExprOrTypeTraitExpr>(E); if (!SAE->isInstantiationDependent()) { @@ -4312,13 +4388,41 @@ recurse: break; } + NotPrimaryExpr(); // But otherwise, they are not. + + auto MangleAlignofSizeofArg = [&] { + if (SAE->isArgumentType()) { + Out << 't'; + mangleType(SAE->getArgumentType()); + } else { + Out << 'z'; + mangleExpression(SAE->getArgumentExpr()); + } + }; + switch(SAE->getKind()) { case UETT_SizeOf: Out << 's'; + MangleAlignofSizeofArg(); break; case UETT_PreferredAlignOf: + // As of clang 12, we mangle __alignof__ differently than alignof. (They + // have acted differently since Clang 8, but were previously mangled the + // same.) + if (Context.getASTContext().getLangOpts().getClangABICompat() > + LangOptions::ClangABI::Ver11) { + Out << "u11__alignof__"; + if (SAE->isArgumentType()) + mangleType(SAE->getArgumentType()); + else + mangleTemplateArgExpr(SAE->getArgumentExpr()); + Out << 'E'; + break; + } + LLVM_FALLTHROUGH; case UETT_AlignOf: Out << 'a'; + MangleAlignofSizeofArg(); break; case UETT_VecStep: { DiagnosticsEngine &Diags = Context.getDiags(); @@ -4336,17 +4440,11 @@ recurse: return; } } - if (SAE->isArgumentType()) { - Out << 't'; - mangleType(SAE->getArgumentType()); - } else { - Out << 'z'; - mangleExpression(SAE->getArgumentExpr()); - } break; } case Expr::CXXThrowExprClass: { + NotPrimaryExpr(); const CXXThrowExpr *TE = cast<CXXThrowExpr>(E); // <expression> ::= tw <expression> # throw expression // ::= tr # rethrow @@ -4360,6 +4458,7 @@ recurse: } case Expr::CXXTypeidExprClass: { + NotPrimaryExpr(); const CXXTypeidExpr *TIE = cast<CXXTypeidExpr>(E); // <expression> ::= ti <type> # typeid (type) // ::= te <expression> # typeid (expression) @@ -4374,6 +4473,7 @@ recurse: } case Expr::CXXDeleteExprClass: { + NotPrimaryExpr(); const CXXDeleteExpr *DE = cast<CXXDeleteExpr>(E); // <expression> ::= [gs] dl <expression> # [::] delete expr // ::= [gs] da <expression> # [::] delete [] expr @@ -4384,6 +4484,7 @@ recurse: } case Expr::UnaryOperatorClass: { + NotPrimaryExpr(); const UnaryOperator *UO = cast<UnaryOperator>(E); mangleOperatorName(UnaryOperator::getOverloadedOperator(UO->getOpcode()), /*Arity=*/1); @@ -4392,6 +4493,7 @@ recurse: } case Expr::ArraySubscriptExprClass: { + NotPrimaryExpr(); const ArraySubscriptExpr *AE = cast<ArraySubscriptExpr>(E); // Array subscript is treated as a syntactically weird form of @@ -4403,6 +4505,7 @@ recurse: } case Expr::MatrixSubscriptExprClass: { + NotPrimaryExpr(); const MatrixSubscriptExpr *ME = cast<MatrixSubscriptExpr>(E); Out << "ixix"; mangleExpression(ME->getBase()); @@ -4413,6 +4516,7 @@ recurse: case Expr::CompoundAssignOperatorClass: // fallthrough case Expr::BinaryOperatorClass: { + NotPrimaryExpr(); const BinaryOperator *BO = cast<BinaryOperator>(E); if (BO->getOpcode() == BO_PtrMemD) Out << "ds"; @@ -4425,6 +4529,7 @@ recurse: } case Expr::CXXRewrittenBinaryOperatorClass: { + NotPrimaryExpr(); // The mangled form represents the original syntax. CXXRewrittenBinaryOperator::DecomposedForm Decomposed = cast<CXXRewrittenBinaryOperator>(E)->getDecomposedForm(); @@ -4436,6 +4541,7 @@ recurse: } case Expr::ConditionalOperatorClass: { + NotPrimaryExpr(); const ConditionalOperator *CO = cast<ConditionalOperator>(E); mangleOperatorName(OO_Conditional, /*Arity=*/3); mangleExpression(CO->getCond()); @@ -4451,19 +4557,22 @@ recurse: } case Expr::ObjCBridgedCastExprClass: { + NotPrimaryExpr(); // Mangle ownership casts as a vendor extended operator __bridge, // __bridge_transfer, or __bridge_retain. StringRef Kind = cast<ObjCBridgedCastExpr>(E)->getBridgeKindName(); Out << "v1U" << Kind.size() << Kind; + mangleCastExpression(E, "cv"); + break; } - // Fall through to mangle the cast itself. - LLVM_FALLTHROUGH; case Expr::CStyleCastExprClass: + NotPrimaryExpr(); mangleCastExpression(E, "cv"); break; case Expr::CXXFunctionalCastExprClass: { + NotPrimaryExpr(); auto *Sub = cast<ExplicitCastExpr>(E)->getSubExpr()->IgnoreImplicit(); // FIXME: Add isImplicit to CXXConstructExpr. if (auto *CCE = dyn_cast<CXXConstructExpr>(Sub)) @@ -4483,22 +4592,28 @@ recurse: } case Expr::CXXStaticCastExprClass: + NotPrimaryExpr(); mangleCastExpression(E, "sc"); break; case Expr::CXXDynamicCastExprClass: + NotPrimaryExpr(); mangleCastExpression(E, "dc"); break; case Expr::CXXReinterpretCastExprClass: + NotPrimaryExpr(); mangleCastExpression(E, "rc"); break; case Expr::CXXConstCastExprClass: + NotPrimaryExpr(); mangleCastExpression(E, "cc"); break; case Expr::CXXAddrspaceCastExprClass: + NotPrimaryExpr(); mangleCastExpression(E, "ac"); break; case Expr::CXXOperatorCallExprClass: { + NotPrimaryExpr(); const CXXOperatorCallExpr *CE = cast<CXXOperatorCallExpr>(E); unsigned NumArgs = CE->getNumArgs(); // A CXXOperatorCallExpr for OO_Arrow models only semantics, not syntax @@ -4512,9 +4627,8 @@ recurse: } case Expr::ParenExprClass: - mangleExpression(cast<ParenExpr>(E)->getSubExpr(), Arity); - break; - + E = cast<ParenExpr>(E)->getSubExpr(); + goto recurse; case Expr::ConceptSpecializationExprClass: { // <expr-primary> ::= L <mangled-name> E # external name @@ -4528,10 +4642,12 @@ recurse: } case Expr::DeclRefExprClass: - mangleDeclRefExpr(cast<DeclRefExpr>(E)->getDecl()); + // MangleDeclRefExpr helper handles primary-vs-nonprimary + MangleDeclRefExpr(cast<DeclRefExpr>(E)->getDecl()); break; case Expr::SubstNonTypeTemplateParmPackExprClass: + NotPrimaryExpr(); // FIXME: not clear how to mangle this! // template <unsigned N...> class A { // template <class U...> void foo(U (&x)[N]...); @@ -4540,14 +4656,16 @@ recurse: break; case Expr::FunctionParmPackExprClass: { + NotPrimaryExpr(); // FIXME: not clear how to mangle this! const FunctionParmPackExpr *FPPE = cast<FunctionParmPackExpr>(E); Out << "v110_SUBSTPACK"; - mangleDeclRefExpr(FPPE->getParameterPack()); + MangleDeclRefExpr(FPPE->getParameterPack()); break; } case Expr::DependentScopeDeclRefExprClass: { + NotPrimaryExpr(); const DependentScopeDeclRefExpr *DRE = cast<DependentScopeDeclRefExpr>(E); mangleUnresolvedName(DRE->getQualifier(), DRE->getDeclName(), DRE->getTemplateArgs(), DRE->getNumTemplateArgs(), @@ -4556,24 +4674,27 @@ recurse: } case Expr::CXXBindTemporaryExprClass: - mangleExpression(cast<CXXBindTemporaryExpr>(E)->getSubExpr()); - break; + E = cast<CXXBindTemporaryExpr>(E)->getSubExpr(); + goto recurse; case Expr::ExprWithCleanupsClass: - mangleExpression(cast<ExprWithCleanups>(E)->getSubExpr(), Arity); - break; + E = cast<ExprWithCleanups>(E)->getSubExpr(); + goto recurse; case Expr::FloatingLiteralClass: { + // <expr-primary> const FloatingLiteral *FL = cast<FloatingLiteral>(E); mangleFloatLiteral(FL->getType(), FL->getValue()); break; } case Expr::FixedPointLiteralClass: + // Currently unimplemented -- might be <expr-primary> in future? mangleFixedPointLiteral(); break; case Expr::CharacterLiteralClass: + // <expr-primary> Out << 'L'; mangleType(E->getType()); Out << cast<CharacterLiteral>(E)->getValue(); @@ -4582,18 +4703,21 @@ recurse: // FIXME. __objc_yes/__objc_no are mangled same as true/false case Expr::ObjCBoolLiteralExprClass: + // <expr-primary> Out << "Lb"; Out << (cast<ObjCBoolLiteralExpr>(E)->getValue() ? '1' : '0'); Out << 'E'; break; case Expr::CXXBoolLiteralExprClass: + // <expr-primary> Out << "Lb"; Out << (cast<CXXBoolLiteralExpr>(E)->getValue() ? '1' : '0'); Out << 'E'; break; case Expr::IntegerLiteralClass: { + // <expr-primary> llvm::APSInt Value(cast<IntegerLiteral>(E)->getValue()); if (E->getType()->isSignedIntegerType()) Value.setIsSigned(true); @@ -4602,6 +4726,7 @@ recurse: } case Expr::ImaginaryLiteralClass: { + // <expr-primary> const ImaginaryLiteral *IE = cast<ImaginaryLiteral>(E); // Mangle as if a complex literal. // Proposal from David Vandevoorde, 2010.06.30. @@ -4625,6 +4750,7 @@ recurse: } case Expr::StringLiteralClass: { + // <expr-primary> // Revised proposal from David Vandervoorde, 2010.07.15. Out << 'L'; assert(isa<ConstantArrayType>(E->getType())); @@ -4634,21 +4760,25 @@ recurse: } case Expr::GNUNullExprClass: + // <expr-primary> // Mangle as if an integer literal 0. mangleIntegerLiteral(E->getType(), llvm::APSInt(32)); break; case Expr::CXXNullPtrLiteralExprClass: { + // <expr-primary> Out << "LDnE"; break; } case Expr::PackExpansionExprClass: + NotPrimaryExpr(); Out << "sp"; mangleExpression(cast<PackExpansionExpr>(E)->getPattern()); break; case Expr::SizeOfPackExprClass: { + NotPrimaryExpr(); auto *SPE = cast<SizeOfPackExpr>(E); if (SPE->isPartiallySubstituted()) { Out << "sP"; @@ -4673,12 +4803,12 @@ recurse: break; } - case Expr::MaterializeTemporaryExprClass: { - mangleExpression(cast<MaterializeTemporaryExpr>(E)->getSubExpr()); - break; - } + case Expr::MaterializeTemporaryExprClass: + E = cast<MaterializeTemporaryExpr>(E)->getSubExpr(); + goto recurse; case Expr::CXXFoldExprClass: { + NotPrimaryExpr(); auto *FE = cast<CXXFoldExpr>(E); if (FE->isLeftFold()) Out << (FE->getInit() ? "fL" : "fl"); @@ -4700,27 +4830,34 @@ recurse: } case Expr::CXXThisExprClass: + NotPrimaryExpr(); Out << "fpT"; break; case Expr::CoawaitExprClass: // FIXME: Propose a non-vendor mangling. + NotPrimaryExpr(); Out << "v18co_await"; mangleExpression(cast<CoawaitExpr>(E)->getOperand()); break; case Expr::DependentCoawaitExprClass: // FIXME: Propose a non-vendor mangling. + NotPrimaryExpr(); Out << "v18co_await"; mangleExpression(cast<DependentCoawaitExpr>(E)->getOperand()); break; case Expr::CoyieldExprClass: // FIXME: Propose a non-vendor mangling. + NotPrimaryExpr(); Out << "v18co_yield"; mangleExpression(cast<CoawaitExpr>(E)->getOperand()); break; } + + if (AsTemplateArg && !IsPrimaryExpr) + Out << 'E'; } /// Mangle an expression which refers to a parameter variable. @@ -4970,26 +5107,9 @@ void CXXNameMangler::mangleTemplateArg(TemplateArgument A, bool NeedExactType) { Out << "Dp"; mangleType(A.getAsTemplateOrTemplatePattern()); break; - case TemplateArgument::Expression: { - // It's possible to end up with a DeclRefExpr here in certain - // dependent cases, in which case we should mangle as a - // declaration. - const Expr *E = A.getAsExpr()->IgnoreParenImpCasts(); - if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) { - const ValueDecl *D = DRE->getDecl(); - if (isa<VarDecl>(D) || isa<FunctionDecl>(D)) { - Out << 'L'; - mangle(D); - Out << 'E'; - break; - } - } - - Out << 'X'; - mangleExpression(E); - Out << 'E'; + case TemplateArgument::Expression: + mangleTemplateArgExpr(A.getAsExpr()); break; - } case TemplateArgument::Integral: mangleIntegerLiteral(A.getIntegralType(), A.getAsIntegral()); break; @@ -5044,6 +5164,38 @@ void CXXNameMangler::mangleTemplateArg(TemplateArgument A, bool NeedExactType) { } } +void CXXNameMangler::mangleTemplateArgExpr(const Expr *E) { + ASTContext &Ctx = Context.getASTContext(); + if (Ctx.getLangOpts().getClangABICompat() > LangOptions::ClangABI::Ver11) { + mangleExpression(E, UnknownArity, /*AsTemplateArg=*/true); + return; + } + + // Prior to Clang 12, we didn't omit the X .. E around <expr-primary> + // correctly in cases where the template argument was + // constructed from an expression rather than an already-evaluated + // literal. In such a case, we would then e.g. emit 'XLi0EE' instead of + // 'Li0E'. + // + // We did special-case DeclRefExpr to attempt to DTRT for that one + // expression-kind, but while doing so, unfortunately handled ParmVarDecl + // (subtype of VarDecl) _incorrectly_, and emitted 'L_Z .. E' instead of + // the proper 'Xfp_E'. + E = E->IgnoreParenImpCasts(); + if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) { + const ValueDecl *D = DRE->getDecl(); + if (isa<VarDecl>(D) || isa<FunctionDecl>(D)) { + Out << 'L'; + mangle(D); + Out << 'E'; + return; + } + } + Out << 'X'; + mangleExpression(E); + Out << 'E'; +} + /// Determine whether a given value is equivalent to zero-initialization for /// the purpose of discarding a trailing portion of a 'tl' mangling. /// diff --git a/contrib/llvm-project/clang/lib/AST/MicrosoftCXXABI.cpp b/contrib/llvm-project/clang/lib/AST/MicrosoftCXXABI.cpp index f9f9fe985b6f..166aa3b3bd60 100644 --- a/contrib/llvm-project/clang/lib/AST/MicrosoftCXXABI.cpp +++ b/contrib/llvm-project/clang/lib/AST/MicrosoftCXXABI.cpp @@ -16,6 +16,7 @@ #include "clang/AST/Attr.h" #include "clang/AST/CXXInheritance.h" #include "clang/AST/DeclCXX.h" +#include "clang/AST/Mangle.h" #include "clang/AST/MangleNumberingContext.h" #include "clang/AST/RecordLayout.h" #include "clang/AST/Type.h" @@ -64,6 +65,19 @@ public: } }; +class MSHIPNumberingContext : public MicrosoftNumberingContext { + std::unique_ptr<MangleNumberingContext> DeviceCtx; + +public: + MSHIPNumberingContext(MangleContext *DeviceMangler) { + DeviceCtx = createItaniumNumberingContext(DeviceMangler); + } + + unsigned getDeviceManglingNumber(const CXXMethodDecl *CallOperator) override { + return DeviceCtx->getManglingNumber(CallOperator); + } +}; + class MicrosoftCXXABI : public CXXABI { ASTContext &Context; llvm::SmallDenseMap<CXXRecordDecl *, CXXConstructorDecl *> RecordToCopyCtor; @@ -73,8 +87,20 @@ class MicrosoftCXXABI : public CXXABI { llvm::SmallDenseMap<TagDecl *, TypedefNameDecl *> UnnamedTagDeclToTypedefNameDecl; + // MangleContext for device numbering context, which is based on Itanium C++ + // ABI. + std::unique_ptr<MangleContext> DeviceMangler; + public: - MicrosoftCXXABI(ASTContext &Ctx) : Context(Ctx) { } + MicrosoftCXXABI(ASTContext &Ctx) : Context(Ctx) { + if (Context.getLangOpts().CUDA && Context.getAuxTargetInfo()) { + assert(Context.getTargetInfo().getCXXABI().isMicrosoft() && + Context.getAuxTargetInfo()->getCXXABI().isItaniumFamily() && + "Unexpected combination of C++ ABIs."); + DeviceMangler.reset( + Context.createMangleContext(Context.getAuxTargetInfo())); + } + } MemberPointerInfo getMemberPointerInfo(const MemberPointerType *MPT) const override; @@ -133,6 +159,10 @@ public: std::unique_ptr<MangleNumberingContext> createMangleNumberingContext() const override { + if (Context.getLangOpts().CUDA && Context.getAuxTargetInfo()) { + assert(DeviceMangler && "Missing device mangler"); + return std::make_unique<MSHIPNumberingContext>(DeviceMangler.get()); + } return std::make_unique<MicrosoftNumberingContext>(); } }; @@ -266,4 +296,3 @@ CXXABI::MemberPointerInfo MicrosoftCXXABI::getMemberPointerInfo( CXXABI *clang::CreateMicrosoftCXXABI(ASTContext &Ctx) { return new MicrosoftCXXABI(Ctx); } - diff --git a/contrib/llvm-project/clang/lib/ASTMatchers/ASTMatchFinder.cpp b/contrib/llvm-project/clang/lib/ASTMatchers/ASTMatchFinder.cpp index 8ddd3c87e09d..69957a952d17 100644 --- a/contrib/llvm-project/clang/lib/ASTMatchers/ASTMatchFinder.cpp +++ b/contrib/llvm-project/clang/lib/ASTMatchers/ASTMatchFinder.cpp @@ -243,10 +243,14 @@ public: return true; ScopedIncrement ScopedDepth(&CurrentDepth); if (auto *Init = Node->getInit()) - if (!match(*Init)) + if (!traverse(*Init)) return false; - if (!match(*Node->getLoopVariable()) || !match(*Node->getRangeInit()) || - !match(*Node->getBody())) + if (!match(*Node->getLoopVariable())) + return false; + if (match(*Node->getRangeInit())) + if (!VisitorBase::TraverseStmt(Node->getRangeInit())) + return false; + if (!match(*Node->getBody())) return false; return VisitorBase::TraverseStmt(Node->getBody()); } @@ -291,7 +295,7 @@ public: if (!match(*Node->getBody())) return false; - return true; + return VisitorBase::TraverseStmt(Node->getBody()); } bool shouldVisitTemplateInstantiations() const { return true; } @@ -488,15 +492,21 @@ public: bool dataTraverseNode(Stmt *S, DataRecursionQueue *Queue) { if (auto *RF = dyn_cast<CXXForRangeStmt>(S)) { - for (auto *SubStmt : RF->children()) { - if (SubStmt == RF->getInit() || SubStmt == RF->getLoopVarStmt() || - SubStmt == RF->getRangeInit() || SubStmt == RF->getBody()) { - TraverseStmt(SubStmt, Queue); - } else { - ASTNodeNotSpelledInSourceScope RAII(this, true); - TraverseStmt(SubStmt, Queue); + { + ASTNodeNotAsIsSourceScope RAII(this, true); + TraverseStmt(RF->getInit()); + // Don't traverse under the loop variable + match(*RF->getLoopVariable()); + TraverseStmt(RF->getRangeInit()); + } + { + ASTNodeNotSpelledInSourceScope RAII(this, true); + for (auto *SubStmt : RF->children()) { + if (SubStmt != RF->getBody()) + TraverseStmt(SubStmt); } } + TraverseStmt(RF->getBody()); return true; } else if (auto *RBO = dyn_cast<CXXRewrittenBinaryOperator>(S)) { { @@ -556,9 +566,9 @@ public: if (LE->hasExplicitResultType()) TraverseTypeLoc(Proto.getReturnLoc()); TraverseStmt(LE->getTrailingRequiresClause()); - - TraverseStmt(LE->getBody()); } + + TraverseStmt(LE->getBody()); return true; } return RecursiveASTVisitor<MatchASTVisitor>::dataTraverseNode(S, Queue); @@ -697,6 +707,10 @@ public: bool shouldVisitTemplateInstantiations() const { return true; } bool shouldVisitImplicitCode() const { return true; } + // We visit the lambda body explicitly, so instruct the RAV + // to not visit it on our behalf too. + bool shouldVisitLambdaBody() const { return false; } + bool IsMatchingInASTNodeNotSpelledInSource() const override { return TraversingASTNodeNotSpelledInSource; } @@ -823,6 +837,14 @@ private: if (EnableCheckProfiling) Timer.setBucket(&TimeByBucket[MP.second->getID()]); BoundNodesTreeBuilder Builder; + + { + TraversalKindScope RAII(getASTContext(), MP.first.getTraversalKind()); + if (getASTContext().getParentMapContext().traverseIgnored(DynNode) != + DynNode) + continue; + } + if (MP.first.matches(DynNode, this, &Builder)) { MatchVisitor Visitor(ActiveASTContext, MP.second); Builder.visitMatches(&Visitor); diff --git a/contrib/llvm-project/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/contrib/llvm-project/clang/lib/ASTMatchers/ASTMatchersInternal.cpp index 6c7e14e3499a..705f1cdf3153 100644 --- a/contrib/llvm-project/clang/lib/ASTMatchers/ASTMatchersInternal.cpp +++ b/contrib/llvm-project/clang/lib/ASTMatchers/ASTMatchersInternal.cpp @@ -732,7 +732,7 @@ const internal::VariadicDynCastAllOfMatcher<Decl, TypeAliasDecl> typeAliasDecl; const internal::VariadicDynCastAllOfMatcher<Decl, TypeAliasTemplateDecl> typeAliasTemplateDecl; const internal::VariadicAllOfMatcher<Decl> decl; -const internal::VariadicAllOfMatcher<DecompositionDecl> decompositionDecl; +const internal::VariadicDynCastAllOfMatcher<Decl, DecompositionDecl> decompositionDecl; const internal::VariadicDynCastAllOfMatcher<Decl, LinkageSpecDecl> linkageSpecDecl; const internal::VariadicDynCastAllOfMatcher<Decl, NamedDecl> namedDecl; diff --git a/contrib/llvm-project/clang/lib/Basic/ProfileList.cpp b/contrib/llvm-project/clang/lib/Basic/ProfileList.cpp index 56bc37a79301..2cb05c1c3c07 100644 --- a/contrib/llvm-project/clang/lib/Basic/ProfileList.cpp +++ b/contrib/llvm-project/clang/lib/Basic/ProfileList.cpp @@ -82,6 +82,7 @@ static StringRef getSectionName(CodeGenOptions::ProfileInstrKind Kind) { case CodeGenOptions::ProfileCSIRInstr: return "csllvm"; } + llvm_unreachable("Unhandled CodeGenOptions::ProfileInstrKind enum"); } llvm::Optional<bool> diff --git a/contrib/llvm-project/clang/lib/Basic/Targets/PPC.cpp b/contrib/llvm-project/clang/lib/Basic/Targets/PPC.cpp index cfede6e6e756..ff09c0fa2a23 100644 --- a/contrib/llvm-project/clang/lib/Basic/Targets/PPC.cpp +++ b/contrib/llvm-project/clang/lib/Basic/Targets/PPC.cpp @@ -318,9 +318,6 @@ bool PPCTargetInfo::initFeatureMap( .Case("pwr9", true) .Case("pwr8", true) .Default(false); - Features["float128"] = llvm::StringSwitch<bool>(CPU) - .Case("pwr9", true) - .Default(false); Features["spe"] = llvm::StringSwitch<bool>(CPU) .Case("8548", true) diff --git a/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.cpp b/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.cpp index 0bf02e605740..786201ea340d 100644 --- a/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.cpp +++ b/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.cpp @@ -150,7 +150,7 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts, } if (HasV) { - Builder.defineMacro("__riscv_v", "1000000"); + Builder.defineMacro("__riscv_v", "10000"); Builder.defineMacro("__riscv_vector"); } @@ -191,10 +191,10 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts, Builder.defineMacro("__riscv_zfh", "1000"); if (HasZvamo) - Builder.defineMacro("__riscv_zvamo", "1000000"); + Builder.defineMacro("__riscv_zvamo", "10000"); if (HasZvlsseg) - Builder.defineMacro("__riscv_zvlsseg", "1000000"); + Builder.defineMacro("__riscv_zvlsseg", "10000"); } /// Return true if has this feature, need to sync with handleTargetFeatures. diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp index 113541bd5024..10e3820d9657 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp @@ -13794,12 +13794,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, case X86::BI__builtin_ia32_reduce_fadd_ps512: { Function *F = CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType()); + Builder.getFastMathFlags().setAllowReassoc(true); return Builder.CreateCall(F, {Ops[0], Ops[1]}); } case X86::BI__builtin_ia32_reduce_fmul_pd512: case X86::BI__builtin_ia32_reduce_fmul_ps512: { Function *F = CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType()); + Builder.getFastMathFlags().setAllowReassoc(true); return Builder.CreateCall(F, {Ops[0], Ops[1]}); } case X86::BI__builtin_ia32_reduce_mul_d512: diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp index 33a2d6f4483e..e03631a7243a 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp @@ -184,6 +184,14 @@ CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy)); VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy)); VoidPtrPtrTy = VoidPtrTy->getPointerTo(); + if (CGM.getContext().getAuxTargetInfo()) { + // If the host and device have different C++ ABIs, mark it as the device + // mangle context so that the mangling needs to retrieve the additonal + // device lambda mangling number instead of the regular host one. + DeviceMC->setDeviceMangleContext( + CGM.getContext().getTargetInfo().getCXXABI().isMicrosoft() && + CGM.getContext().getAuxTargetInfo()->getCXXABI().isItaniumFamily()); + } } llvm::FunctionCallee CGNVCUDARuntime::getSetupArgumentFn() const { diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp index 42801372189b..bc7582c67989 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp @@ -1995,9 +1995,14 @@ void CodeGenModule::ConstructAttributeList( if (TargetDecl->hasAttr<ConstAttr>()) { FuncAttrs.addAttribute(llvm::Attribute::ReadNone); FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); + // gcc specifies that 'const' functions have greater restrictions than + // 'pure' functions, so they also cannot have infinite loops. + FuncAttrs.addAttribute(llvm::Attribute::WillReturn); } else if (TargetDecl->hasAttr<PureAttr>()) { FuncAttrs.addAttribute(llvm::Attribute::ReadOnly); FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); + // gcc specifies that 'pure' functions cannot have infinite loops. + FuncAttrs.addAttribute(llvm::Attribute::WillReturn); } else if (TargetDecl->hasAttr<NoAliasAttr>()) { FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly); FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp index 497f9c04c9f8..decb8129e644 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp @@ -1622,8 +1622,8 @@ llvm::Constant *ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &D) { if (CD->isTrivial() && CD->isDefaultConstructor()) return CGM.EmitNullConstant(D.getType()); } + InConstantContext = true; } - InConstantContext = D.hasConstantInitialization(); QualType destType = D.getType(); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 57cc2d60e2af..caa5291ff6fa 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -409,6 +409,7 @@ class InlinedOpenMPRegionRAII { llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields; FieldDecl *LambdaThisCaptureField = nullptr; const CodeGen::CGBlockInfo *BlockInfo = nullptr; + bool NoInheritance = false; public: /// Constructs region for combined constructs. @@ -416,16 +417,19 @@ public: /// a list of functions used for code generation of implicitly inlined /// regions. InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, - OpenMPDirectiveKind Kind, bool HasCancel) - : CGF(CGF) { + OpenMPDirectiveKind Kind, bool HasCancel, + bool NoInheritance = true) + : CGF(CGF), NoInheritance(NoInheritance) { // Start emission for the construct. CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); - std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); - LambdaThisCaptureField = CGF.LambdaThisCaptureField; - CGF.LambdaThisCaptureField = nullptr; - BlockInfo = CGF.BlockInfo; - CGF.BlockInfo = nullptr; + if (NoInheritance) { + std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); + LambdaThisCaptureField = CGF.LambdaThisCaptureField; + CGF.LambdaThisCaptureField = nullptr; + BlockInfo = CGF.BlockInfo; + CGF.BlockInfo = nullptr; + } } ~InlinedOpenMPRegionRAII() { @@ -434,9 +438,11 @@ public: cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); delete CGF.CapturedStmtInfo; CGF.CapturedStmtInfo = OldCSI; - std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); - CGF.LambdaThisCaptureField = LambdaThisCaptureField; - CGF.BlockInfo = BlockInfo; + if (NoInheritance) { + std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); + CGF.LambdaThisCaptureField = LambdaThisCaptureField; + CGF.BlockInfo = BlockInfo; + } } }; @@ -3853,7 +3859,7 @@ static void emitPrivatesInit(CodeGenFunction &CGF, // Processing for implicitly captured variables. InlinedOpenMPRegionRAII Region( CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown, - /*HasCancel=*/false); + /*HasCancel=*/false, /*NoInheritance=*/true); SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); } if (Type->isArrayType()) { @@ -6214,7 +6220,9 @@ void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, bool HasCancel) { if (!CGF.HaveInsertPoint()) return; - InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel); + InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel, + InnerKind != OMPD_critical && + InnerKind != OMPD_master); CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); } @@ -9892,7 +9900,7 @@ void CGOpenMPRuntime::emitTargetNumIterationsCall( llvm::Value *Args[] = {RTLoc, DeviceID, NumIterations}; CGF.EmitRuntimeCall( OMPBuilder.getOrCreateRuntimeFunction( - CGM.getModule(), OMPRTL___kmpc_push_target_tripcount), + CGM.getModule(), OMPRTL___kmpc_push_target_tripcount_mapper), Args); } }; diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h index 8eb7adbc8fcb..95c0b7b4d7c0 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h @@ -507,12 +507,23 @@ public: /// True if the C++ Standard Requires Progress. bool CPlusPlusWithProgress() { + if (CGM.getCodeGenOpts().getFiniteLoops() == + CodeGenOptions::FiniteLoopsKind::Never) + return false; + return getLangOpts().CPlusPlus11 || getLangOpts().CPlusPlus14 || getLangOpts().CPlusPlus17 || getLangOpts().CPlusPlus20; } /// True if the C Standard Requires Progress. bool CWithProgress() { + if (CGM.getCodeGenOpts().getFiniteLoops() == + CodeGenOptions::FiniteLoopsKind::Always) + return true; + if (CGM.getCodeGenOpts().getFiniteLoops() == + CodeGenOptions::FiniteLoopsKind::Never) + return false; + return getLangOpts().C11 || getLangOpts().C17 || getLangOpts().C2x; } diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp index 469a7365c90f..cc5cd1f83f11 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp @@ -63,7 +63,7 @@ isExperimentalExtension(StringRef Ext) { Ext == "zbr" || Ext == "zbs" || Ext == "zbt" || Ext == "zbproposedc") return RISCVExtensionVersion{"0", "93"}; if (Ext == "v" || Ext == "zvamo" || Ext == "zvlsseg") - return RISCVExtensionVersion{"1", "0"}; + return RISCVExtensionVersion{"0", "10"}; if (Ext == "zfh") return RISCVExtensionVersion{"0", "1"}; return None; diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp index 634b1259b927..659989ceb605 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp @@ -4669,20 +4669,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } } - if (Triple.isOSAIX() && Args.hasArg(options::OPT_maltivec)) { - if (Args.getLastArg(options::OPT_mabi_EQ_vec_extabi)) { - CmdArgs.push_back("-mabi=vec-extabi"); - } else { - D.Diag(diag::err_aix_default_altivec_abi); - } - } - if (Arg *A = Args.getLastArg(options::OPT_mabi_EQ_vec_extabi, options::OPT_mabi_EQ_vec_default)) { if (!Triple.isOSAIX()) D.Diag(diag::err_drv_unsupported_opt_for_target) << A->getSpelling() << RawTriple.str(); - if (A->getOption().getID() == options::OPT_mabi_EQ_vec_default) + if (A->getOption().getID() == options::OPT_mabi_EQ_vec_extabi) + CmdArgs.push_back("-mabi=vec-extabi"); + else D.Diag(diag::err_aix_default_altivec_abi); } @@ -5626,6 +5620,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, if (A->getOption().matches(options::OPT_freroll_loops)) CmdArgs.push_back("-freroll-loops"); + Args.AddLastArg(CmdArgs, options::OPT_ffinite_loops, + options::OPT_fno_finite_loops); + Args.AddLastArg(CmdArgs, options::OPT_fwritable_strings); Args.AddLastArg(CmdArgs, options::OPT_funroll_loops, options::OPT_fno_unroll_loops); diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp index 6a95aa5ec628..bcaea71dca94 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -605,6 +605,11 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, CmdArgs.push_back("-plugin-opt=new-pass-manager"); } + // Pass an option to enable pseudo probe emission. + if (Args.hasFlag(options::OPT_fpseudo_probe_for_profiling, + options::OPT_fno_pseudo_probe_for_profiling, false)) + CmdArgs.push_back("-plugin-opt=pseudo-probe-for-profiling"); + // Setup statistics file output. SmallString<128> StatsFile = getStatsFileName(Args, Output, Input, D); if (!StatsFile.empty()) diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp index e17a6bd4bdd2..9663a7390ada 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp @@ -236,15 +236,6 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args) ExtraOpts.push_back("relro"); } - if (Triple.isAndroid() && Triple.isAndroidVersionLT(29)) { - // https://github.com/android/ndk/issues/1196 - // The unwinder used by the crash handler on versions of Android prior to - // API 29 did not correctly handle binaries built with rosegment, which is - // enabled by default for LLD. Android only supports LLD, so it's not an - // issue that this flag is not accepted by other linkers. - ExtraOpts.push_back("--no-rosegment"); - } - // Android ARM/AArch64 use max-page-size=4096 to reduce VMA usage. Note, lld // from 11 onwards default max-page-size to 65536 for both ARM and AArch64. if ((Triple.isARM() || Triple.isAArch64()) && Triple.isAndroid()) { diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/MSVC.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/MSVC.cpp index f4b7a57e0bb7..13943b6c404a 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/MSVC.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/MSVC.cpp @@ -11,6 +11,7 @@ #include "Darwin.h" #include "clang/Basic/CharInfo.h" #include "clang/Basic/Version.h" +#include "clang/Config/config.h" #include "clang/Driver/Compilation.h" #include "clang/Driver/Driver.h" #include "clang/Driver/DriverDiagnostic.h" @@ -520,7 +521,10 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA, // translate 'lld' into 'lld-link', and in the case of the regular msvc // linker, we need to use a special search algorithm. llvm::SmallString<128> linkPath; - StringRef Linker = Args.getLastArgValue(options::OPT_fuse_ld_EQ, "link"); + StringRef Linker + = Args.getLastArgValue(options::OPT_fuse_ld_EQ, CLANG_DEFAULT_LINKER); + if (Linker.empty()) + Linker = "link"; if (Linker.equals_lower("lld")) Linker = "lld-link"; diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/OpenBSD.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/OpenBSD.cpp index f155d74632f9..e162165b2561 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/OpenBSD.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/OpenBSD.cpp @@ -296,6 +296,7 @@ void OpenBSD::AddCXXStdlibLibArgs(const ArgList &Args, CmdArgs.push_back(Profiling ? "-lc++_p" : "-lc++"); CmdArgs.push_back(Profiling ? "-lc++abi_p" : "-lc++abi"); + CmdArgs.push_back(Profiling ? "-lpthread_p" : "-lpthread"); } std::string OpenBSD::getCompilerRT(const ArgList &Args, diff --git a/contrib/llvm-project/clang/lib/Format/UnwrappedLineFormatter.cpp b/contrib/llvm-project/clang/lib/Format/UnwrappedLineFormatter.cpp index d1138bbc9c36..5dd0ccdfa6fd 100644 --- a/contrib/llvm-project/clang/lib/Format/UnwrappedLineFormatter.cpp +++ b/contrib/llvm-project/clang/lib/Format/UnwrappedLineFormatter.cpp @@ -371,7 +371,7 @@ private: if (Previous->is(tok::comment)) Previous = Previous->getPreviousNonComment(); if (Previous) { - if (Previous->is(tok::greater)) + if (Previous->is(tok::greater) && !I[-1]->InPPDirective) return 0; if (Previous->is(tok::identifier)) { const FormatToken *PreviousPrevious = diff --git a/contrib/llvm-project/clang/lib/Frontend/CompilerInvocation.cpp b/contrib/llvm-project/clang/lib/Frontend/CompilerInvocation.cpp index d8be4ea14868..5c5cf46150e2 100644 --- a/contrib/llvm-project/clang/lib/Frontend/CompilerInvocation.cpp +++ b/contrib/llvm-project/clang/lib/Frontend/CompilerInvocation.cpp @@ -1037,7 +1037,6 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, Opts.UnrollLoops = Args.hasFlag(OPT_funroll_loops, OPT_fno_unroll_loops, (Opts.OptimizationLevel > 1)); - Opts.BinutilsVersion = std::string(Args.getLastArgValue(OPT_fbinutils_version_EQ)); @@ -1324,6 +1323,10 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, Opts.EmitVersionIdentMetadata = Args.hasFlag(OPT_Qy, OPT_Qn, true); + if (Args.hasArg(options::OPT_ffinite_loops)) + Opts.FiniteLoops = CodeGenOptions::FiniteLoopsKind::Always; + else if (Args.hasArg(options::OPT_fno_finite_loops)) + Opts.FiniteLoops = CodeGenOptions::FiniteLoopsKind::Never; return Success; } @@ -2470,6 +2473,8 @@ void CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, bool IsTargetSpecified = Opts.OpenMPIsDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ); + Opts.ConvergentFunctions = Opts.ConvergentFunctions || Opts.OpenMPIsDevice; + if (Opts.OpenMP || Opts.OpenMPSimd) { if (int Version = getLastArgIntValue( Args, OPT_fopenmp_version_EQ, diff --git a/contrib/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp b/contrib/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp index d47ad1b74649..c64a912ce919 100644 --- a/contrib/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp +++ b/contrib/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp @@ -565,7 +565,7 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts, Builder.defineMacro("__cpp_aggregate_bases", "201603L"); Builder.defineMacro("__cpp_structured_bindings", "201606L"); Builder.defineMacro("__cpp_nontype_template_args", - LangOpts.CPlusPlus20 ? "201911L" : "201411L"); + "201411L"); // (not latest) Builder.defineMacro("__cpp_fold_expressions", "201603L"); Builder.defineMacro("__cpp_guaranteed_copy_elision", "201606L"); Builder.defineMacro("__cpp_nontype_template_parameter_auto", "201606L"); diff --git a/contrib/llvm-project/clang/lib/Headers/avx512fintrin.h b/contrib/llvm-project/clang/lib/Headers/avx512fintrin.h index 2ee4350b14d4..f226382cbb2c 100644 --- a/contrib/llvm-project/clang/lib/Headers/avx512fintrin.h +++ b/contrib/llvm-project/clang/lib/Headers/avx512fintrin.h @@ -9297,9 +9297,12 @@ _mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A) /* Vector-reduction arithmetic accepts vectors as inputs and produces scalars as * outputs. This class of vector operation forms the basis of many scientific - * computations. In vector-reduction arithmetic, the evaluation off is + * computations. In vector-reduction arithmetic, the evaluation order is * independent of the order of the input elements of V. + * For floating point types, we always assume the elements are reassociable even + * if -fast-math is off. + * Used bisection method. At each step, we partition the vector with previous * step in half, and the operation is performed on its two halves. * This takes log2(n) steps where n is the number of elements in the vector. @@ -9345,8 +9348,11 @@ _mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W) { return __builtin_ia32_reduce_or_q512(__W); } +// -0.0 is used to ignore the start value since it is the neutral value of +// floating point addition. For more information, please refer to +// https://llvm.org/docs/LangRef.html#llvm-vector-reduce-fadd-intrinsic static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_add_pd(__m512d __W) { - return __builtin_ia32_reduce_fadd_pd512(0.0, __W); + return __builtin_ia32_reduce_fadd_pd512(-0.0, __W); } static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_pd(__m512d __W) { @@ -9356,7 +9362,7 @@ static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_pd(__m512d __W) static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_mask_reduce_add_pd(__mmask8 __M, __m512d __W) { __W = _mm512_maskz_mov_pd(__M, __W); - return __builtin_ia32_reduce_fadd_pd512(0.0, __W); + return __builtin_ia32_reduce_fadd_pd512(-0.0, __W); } static __inline__ double __DEFAULT_FN_ATTRS512 @@ -9411,7 +9417,7 @@ _mm512_mask_reduce_or_epi32(__mmask16 __M, __m512i __W) { static __inline__ float __DEFAULT_FN_ATTRS512 _mm512_reduce_add_ps(__m512 __W) { - return __builtin_ia32_reduce_fadd_ps512(0.0f, __W); + return __builtin_ia32_reduce_fadd_ps512(-0.0f, __W); } static __inline__ float __DEFAULT_FN_ATTRS512 @@ -9422,7 +9428,7 @@ _mm512_reduce_mul_ps(__m512 __W) { static __inline__ float __DEFAULT_FN_ATTRS512 _mm512_mask_reduce_add_ps(__mmask16 __M, __m512 __W) { __W = _mm512_maskz_mov_ps(__M, __W); - return __builtin_ia32_reduce_fadd_ps512(0.0f, __W); + return __builtin_ia32_reduce_fadd_ps512(-0.0f, __W); } static __inline__ float __DEFAULT_FN_ATTRS512 diff --git a/contrib/llvm-project/clang/lib/Lex/Preprocessor.cpp b/contrib/llvm-project/clang/lib/Lex/Preprocessor.cpp index 94f1ce91f884..177786d90390 100644 --- a/contrib/llvm-project/clang/lib/Lex/Preprocessor.cpp +++ b/contrib/llvm-project/clang/lib/Lex/Preprocessor.cpp @@ -119,12 +119,8 @@ Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts, // a macro. They get unpoisoned where it is allowed. (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use); - if (getLangOpts().CPlusPlus20) { - (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned(); - SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use); - } else { - Ident__VA_OPT__ = nullptr; - } + (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned(); + SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use); // Initialize the pragma handlers. RegisterBuiltinPragmas(); diff --git a/contrib/llvm-project/clang/lib/Lex/TokenLexer.cpp b/contrib/llvm-project/clang/lib/Lex/TokenLexer.cpp index 97cb2cf0bb8c..da5681aaf478 100644 --- a/contrib/llvm-project/clang/lib/Lex/TokenLexer.cpp +++ b/contrib/llvm-project/clang/lib/Lex/TokenLexer.cpp @@ -148,12 +148,12 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( return false; // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if - // __VA_ARGS__ is empty, but not in strict mode where there are no - // named arguments, where it remains. With GNU extensions, it is removed - // regardless of named arguments. + // __VA_ARGS__ is empty, but not in strict C99 mode where there are no + // named arguments, where it remains. In all other modes, including C99 + // with GNU extensions, it is removed regardless of named arguments. // Microsoft also appears to support this extension, unofficially. - if (!PP.getLangOpts().GNUMode && !PP.getLangOpts().MSVCCompat && - Macro->getNumParams() < 2) + if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode + && Macro->getNumParams() < 2) return false; // Is a comma available to be removed? diff --git a/contrib/llvm-project/clang/lib/Parse/ParseDecl.cpp b/contrib/llvm-project/clang/lib/Parse/ParseDecl.cpp index 571164139630..347d992b1643 100644 --- a/contrib/llvm-project/clang/lib/Parse/ParseDecl.cpp +++ b/contrib/llvm-project/clang/lib/Parse/ParseDecl.cpp @@ -4216,7 +4216,7 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc, } // Parse _Static_assert declaration. - if (Tok.is(tok::kw__Static_assert)) { + if (Tok.isOneOf(tok::kw__Static_assert, tok::kw_static_assert)) { SourceLocation DeclEnd; ParseStaticAssertDeclaration(DeclEnd); continue; @@ -5180,6 +5180,7 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) { case tok::kw_friend: // static_assert-declaration + case tok::kw_static_assert: case tok::kw__Static_assert: // GNU typeof support. diff --git a/contrib/llvm-project/clang/lib/Sema/Sema.cpp b/contrib/llvm-project/clang/lib/Sema/Sema.cpp index 55cb3aee6194..450f9c020f7f 100644 --- a/contrib/llvm-project/clang/lib/Sema/Sema.cpp +++ b/contrib/llvm-project/clang/lib/Sema/Sema.cpp @@ -14,6 +14,7 @@ #include "UsedDeclVisitor.h" #include "clang/AST/ASTContext.h" #include "clang/AST/ASTDiagnostic.h" +#include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclFriend.h" #include "clang/AST/DeclObjC.h" @@ -537,6 +538,13 @@ void Sema::diagnoseZeroToNullptrConversion(CastKind Kind, const Expr* E) { if (E->IgnoreParenImpCasts()->getType()->isNullPtrType()) return; + // Don't diagnose the conversion from a 0 literal to a null pointer argument + // in a synthesized call to operator<=>. + if (!CodeSynthesisContexts.empty() && + CodeSynthesisContexts.back().Kind == + CodeSynthesisContext::RewritingOperatorAsSpaceship) + return; + // If it is a macro from system header, and if the macro name is not "NULL", // do not warn. SourceLocation MaybeMacroLoc = E->getBeginLoc(); @@ -1733,11 +1741,12 @@ Sema::SemaDiagnosticBuilder::~SemaDiagnosticBuilder() { } } -Sema::SemaDiagnosticBuilder Sema::targetDiag(SourceLocation Loc, - unsigned DiagID) { +Sema::SemaDiagnosticBuilder +Sema::targetDiag(SourceLocation Loc, unsigned DiagID, FunctionDecl *FD) { + FD = FD ? FD : getCurFunctionDecl(); if (LangOpts.OpenMP) - return LangOpts.OpenMPIsDevice ? diagIfOpenMPDeviceCode(Loc, DiagID) - : diagIfOpenMPHostCode(Loc, DiagID); + return LangOpts.OpenMPIsDevice ? diagIfOpenMPDeviceCode(Loc, DiagID, FD) + : diagIfOpenMPHostCode(Loc, DiagID, FD); if (getLangOpts().CUDA) return getLangOpts().CUDAIsDevice ? CUDADiagIfDeviceCode(Loc, DiagID) : CUDADiagIfHostCode(Loc, DiagID); @@ -1746,7 +1755,7 @@ Sema::SemaDiagnosticBuilder Sema::targetDiag(SourceLocation Loc, return SYCLDiagIfDeviceCode(Loc, DiagID); return SemaDiagnosticBuilder(SemaDiagnosticBuilder::K_Immediate, Loc, DiagID, - getCurFunctionDecl(), *this); + FD, *this); } Sema::SemaDiagnosticBuilder Sema::Diag(SourceLocation Loc, unsigned DiagID, @@ -1765,15 +1774,14 @@ Sema::SemaDiagnosticBuilder Sema::Diag(SourceLocation Loc, unsigned DiagID, DiagID, getCurFunctionDecl(), *this); } - SemaDiagnosticBuilder DB = - getLangOpts().CUDAIsDevice - ? CUDADiagIfDeviceCode(Loc, DiagID) - : CUDADiagIfHostCode(Loc, DiagID); + SemaDiagnosticBuilder DB = getLangOpts().CUDAIsDevice + ? CUDADiagIfDeviceCode(Loc, DiagID) + : CUDADiagIfHostCode(Loc, DiagID); SetIsLastErrorImmediate(DB.isImmediate()); return DB; } -void Sema::checkDeviceDecl(const ValueDecl *D, SourceLocation Loc) { +void Sema::checkDeviceDecl(ValueDecl *D, SourceLocation Loc) { if (isUnevaluatedContext()) return; @@ -1791,13 +1799,17 @@ void Sema::checkDeviceDecl(const ValueDecl *D, SourceLocation Loc) { return; } + // Try to associate errors with the lexical context, if that is a function, or + // the value declaration otherwise. + FunctionDecl *FD = + isa<FunctionDecl>(C) ? cast<FunctionDecl>(C) : dyn_cast<FunctionDecl>(D); auto CheckType = [&](QualType Ty) { if (Ty->isDependentType()) return; if (Ty->isExtIntType()) { if (!Context.getTargetInfo().hasExtIntType()) { - targetDiag(Loc, diag::err_device_unsupported_type) + targetDiag(Loc, diag::err_device_unsupported_type, FD) << D << false /*show bit size*/ << 0 /*bitsize*/ << Ty << Context.getTargetInfo().getTriple().str(); } @@ -1810,11 +1822,12 @@ void Sema::checkDeviceDecl(const ValueDecl *D, SourceLocation Loc) { !Context.getTargetInfo().hasFloat128Type()) || (Ty->isIntegerType() && Context.getTypeSize(Ty) == 128 && !Context.getTargetInfo().hasInt128Type())) { - targetDiag(Loc, diag::err_device_unsupported_type) + if (targetDiag(Loc, diag::err_device_unsupported_type, FD) << D << true /*show bit size*/ << static_cast<unsigned>(Context.getTypeSize(Ty)) << Ty - << Context.getTargetInfo().getTriple().str(); - targetDiag(D->getLocation(), diag::note_defined_here) << D; + << Context.getTargetInfo().getTriple().str()) + D->setInvalidDecl(); + targetDiag(D->getLocation(), diag::note_defined_here, FD) << D; } }; @@ -1826,6 +1839,8 @@ void Sema::checkDeviceDecl(const ValueDecl *D, SourceLocation Loc) { CheckType(ParamTy); CheckType(FPTy->getReturnType()); } + if (const auto *FNPTy = dyn_cast<FunctionNoProtoType>(Ty)) + CheckType(FNPTy->getReturnType()); } /// Looks through the macro-expansion chain for the given diff --git a/contrib/llvm-project/clang/lib/Sema/SemaCodeComplete.cpp b/contrib/llvm-project/clang/lib/Sema/SemaCodeComplete.cpp index c2785fd60fc2..be04970979b3 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaCodeComplete.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaCodeComplete.cpp @@ -5158,6 +5158,20 @@ private: llvm::DenseMap<const IdentifierInfo *, Member> Results; }; + +// If \p Base is ParenListExpr, assume a chain of comma operators and pick the +// last expr. We expect other ParenListExprs to be resolved to e.g. constructor +// calls before here. (So the ParenListExpr should be nonempty, but check just +// in case) +Expr *unwrapParenList(Expr *Base) { + if (auto *PLE = llvm::dyn_cast_or_null<ParenListExpr>(Base)) { + if (PLE->getNumExprs() == 0) + return nullptr; + Base = PLE->getExpr(PLE->getNumExprs() - 1); + } + return Base; +} + } // namespace void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base, @@ -5165,6 +5179,8 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base, SourceLocation OpLoc, bool IsArrow, bool IsBaseExprStatement, QualType PreferredType) { + Base = unwrapParenList(Base); + OtherOpBase = unwrapParenList(OtherOpBase); if (!Base || !CodeCompleter) return; @@ -5597,12 +5613,13 @@ ProduceSignatureHelp(Sema &SemaRef, Scope *S, QualType Sema::ProduceCallSignatureHelp(Scope *S, Expr *Fn, ArrayRef<Expr *> Args, SourceLocation OpenParLoc) { - if (!CodeCompleter) + Fn = unwrapParenList(Fn); + if (!CodeCompleter || !Fn) return QualType(); // FIXME: Provide support for variadic template functions. // Ignore type-dependent call expressions entirely. - if (!Fn || Fn->isTypeDependent() || anyNullArguments(Args)) + if (Fn->isTypeDependent() || anyNullArguments(Args)) return QualType(); // In presence of dependent args we surface all possible signatures using the // non-dependent args in the prefix. Afterwards we do a post filtering to make diff --git a/contrib/llvm-project/clang/lib/Sema/SemaDecl.cpp b/contrib/llvm-project/clang/lib/Sema/SemaDecl.cpp index 3ee0c43097d7..1f7ab49ccdd7 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaDecl.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaDecl.cpp @@ -9420,6 +9420,9 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, } } + if (LangOpts.SYCLIsDevice || (LangOpts.OpenMP && LangOpts.OpenMPIsDevice)) + checkDeviceDecl(NewFD, D.getBeginLoc()); + if (!getLangOpts().CPlusPlus) { // Perform semantic checking on the function declaration. if (!NewFD->isInvalidDecl() && NewFD->isMain()) @@ -18329,42 +18332,51 @@ Sema::FunctionEmissionStatus Sema::getEmissionStatus(FunctionDecl *FD, if (FD->isDependentContext()) return FunctionEmissionStatus::TemplateDiscarded; - FunctionEmissionStatus OMPES = FunctionEmissionStatus::Unknown; + // Check whether this function is an externally visible definition. + auto IsEmittedForExternalSymbol = [this, FD]() { + // We have to check the GVA linkage of the function's *definition* -- if we + // only have a declaration, we don't know whether or not the function will + // be emitted, because (say) the definition could include "inline". + FunctionDecl *Def = FD->getDefinition(); + + return Def && !isDiscardableGVALinkage( + getASTContext().GetGVALinkageForFunction(Def)); + }; + if (LangOpts.OpenMPIsDevice) { + // In OpenMP device mode we will not emit host only functions, or functions + // we don't need due to their linkage. Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy = OMPDeclareTargetDeclAttr::getDeviceType(FD->getCanonicalDecl()); - if (DevTy.hasValue()) { + // DevTy may be changed later by + // #pragma omp declare target to(*) device_type(*). + // Therefore DevTyhaving no value does not imply host. The emission status + // will be checked again at the end of compilation unit with Final = true. + if (DevTy.hasValue()) if (*DevTy == OMPDeclareTargetDeclAttr::DT_Host) - OMPES = FunctionEmissionStatus::OMPDiscarded; - else if (*DevTy == OMPDeclareTargetDeclAttr::DT_NoHost || - *DevTy == OMPDeclareTargetDeclAttr::DT_Any) { - OMPES = FunctionEmissionStatus::Emitted; - } - } - } else if (LangOpts.OpenMP) { - // In OpenMP 4.5 all the functions are host functions. - if (LangOpts.OpenMP <= 45) { - OMPES = FunctionEmissionStatus::Emitted; - } else { - Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy = - OMPDeclareTargetDeclAttr::getDeviceType(FD->getCanonicalDecl()); - // In OpenMP 5.0 or above, DevTy may be changed later by - // #pragma omp declare target to(*) device_type(*). Therefore DevTy - // having no value does not imply host. The emission status will be - // checked again at the end of compilation unit. - if (DevTy.hasValue()) { - if (*DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) { - OMPES = FunctionEmissionStatus::OMPDiscarded; - } else if (*DevTy == OMPDeclareTargetDeclAttr::DT_Host || - *DevTy == OMPDeclareTargetDeclAttr::DT_Any) - OMPES = FunctionEmissionStatus::Emitted; - } else if (Final) - OMPES = FunctionEmissionStatus::Emitted; - } - } - if (OMPES == FunctionEmissionStatus::OMPDiscarded || - (OMPES == FunctionEmissionStatus::Emitted && !LangOpts.CUDA)) - return OMPES; + return FunctionEmissionStatus::OMPDiscarded; + // If we have an explicit value for the device type, or we are in a target + // declare context, we need to emit all extern and used symbols. + if (isInOpenMPDeclareTargetContext() || DevTy.hasValue()) + if (IsEmittedForExternalSymbol()) + return FunctionEmissionStatus::Emitted; + // Device mode only emits what it must, if it wasn't tagged yet and needed, + // we'll omit it. + if (Final) + return FunctionEmissionStatus::OMPDiscarded; + } else if (LangOpts.OpenMP > 45) { + // In OpenMP host compilation prior to 5.0 everything was an emitted host + // function. In 5.0, no_host was introduced which might cause a function to + // be ommitted. + Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy = + OMPDeclareTargetDeclAttr::getDeviceType(FD->getCanonicalDecl()); + if (DevTy.hasValue()) + if (*DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) + return FunctionEmissionStatus::OMPDiscarded; + } + + if (Final && LangOpts.OpenMP && !LangOpts.CUDA) + return FunctionEmissionStatus::Emitted; if (LangOpts.CUDA) { // When compiling for device, host functions are never emitted. Similarly, @@ -18378,17 +18390,7 @@ Sema::FunctionEmissionStatus Sema::getEmissionStatus(FunctionDecl *FD, (T == Sema::CFT_Device || T == Sema::CFT_Global)) return FunctionEmissionStatus::CUDADiscarded; - // Check whether this function is externally visible -- if so, it's - // known-emitted. - // - // We have to check the GVA linkage of the function's *definition* -- if we - // only have a declaration, we don't know whether or not the function will - // be emitted, because (say) the definition could include "inline". - FunctionDecl *Def = FD->getDefinition(); - - if (Def && - !isDiscardableGVALinkage(getASTContext().GetGVALinkageForFunction(Def)) - && (!LangOpts.OpenMP || OMPES == FunctionEmissionStatus::Emitted)) + if (IsEmittedForExternalSymbol()) return FunctionEmissionStatus::Emitted; } diff --git a/contrib/llvm-project/clang/lib/Sema/SemaExpr.cpp b/contrib/llvm-project/clang/lib/Sema/SemaExpr.cpp index 45616dadcbee..ae8508d6c601 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaExpr.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaExpr.cpp @@ -373,7 +373,7 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, ArrayRef<SourceLocation> Locs, } if (LangOpts.SYCLIsDevice || (LangOpts.OpenMP && LangOpts.OpenMPIsDevice)) { - if (const auto *VD = dyn_cast<ValueDecl>(D)) + if (auto *VD = dyn_cast<ValueDecl>(D)) checkDeviceDecl(VD, Loc); if (!Context.getTargetInfo().isTLSSupported()) diff --git a/contrib/llvm-project/clang/lib/Sema/SemaLambda.cpp b/contrib/llvm-project/clang/lib/Sema/SemaLambda.cpp index af61c82c2002..c1c6a4bf5c68 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaLambda.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaLambda.cpp @@ -432,15 +432,16 @@ CXXMethodDecl *Sema::startLambdaDefinition(CXXRecordDecl *Class, void Sema::handleLambdaNumbering( CXXRecordDecl *Class, CXXMethodDecl *Method, - Optional<std::tuple<unsigned, bool, Decl *>> Mangling) { + Optional<std::tuple<bool, unsigned, unsigned, Decl *>> Mangling) { if (Mangling) { - unsigned ManglingNumber; bool HasKnownInternalLinkage; + unsigned ManglingNumber, DeviceManglingNumber; Decl *ManglingContextDecl; - std::tie(ManglingNumber, HasKnownInternalLinkage, ManglingContextDecl) = - Mangling.getValue(); + std::tie(HasKnownInternalLinkage, ManglingNumber, DeviceManglingNumber, + ManglingContextDecl) = Mangling.getValue(); Class->setLambdaMangling(ManglingNumber, ManglingContextDecl, HasKnownInternalLinkage); + Class->setDeviceLambdaManglingNumber(DeviceManglingNumber); return; } @@ -476,6 +477,7 @@ void Sema::handleLambdaNumbering( unsigned ManglingNumber = MCtx->getManglingNumber(Method); Class->setLambdaMangling(ManglingNumber, ManglingContextDecl, HasKnownInternalLinkage); + Class->setDeviceLambdaManglingNumber(MCtx->getDeviceManglingNumber(Method)); } } diff --git a/contrib/llvm-project/clang/lib/Sema/SemaOpenMP.cpp b/contrib/llvm-project/clang/lib/Sema/SemaOpenMP.cpp index 78707484f588..4063c185388d 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaOpenMP.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaOpenMP.cpp @@ -1884,8 +1884,7 @@ void Sema::popOpenMPFunctionRegion(const FunctionScopeInfo *OldFSI) { static bool isOpenMPDeviceDelayedContext(Sema &S) { assert(S.LangOpts.OpenMP && S.LangOpts.OpenMPIsDevice && "Expected OpenMP device compilation."); - return !S.isInOpenMPTargetExecutionDirective() && - !S.isInOpenMPDeclareTargetContext(); + return !S.isInOpenMPTargetExecutionDirective(); } namespace { @@ -1898,11 +1897,11 @@ enum class FunctionEmissionStatus { } // anonymous namespace Sema::SemaDiagnosticBuilder Sema::diagIfOpenMPDeviceCode(SourceLocation Loc, - unsigned DiagID) { + unsigned DiagID, + FunctionDecl *FD) { assert(LangOpts.OpenMP && LangOpts.OpenMPIsDevice && "Expected OpenMP device compilation."); - FunctionDecl *FD = getCurFunctionDecl(); SemaDiagnosticBuilder::Kind Kind = SemaDiagnosticBuilder::K_Nop; if (FD) { FunctionEmissionStatus FES = getEmissionStatus(FD); @@ -1911,6 +1910,13 @@ Sema::SemaDiagnosticBuilder Sema::diagIfOpenMPDeviceCode(SourceLocation Loc, Kind = SemaDiagnosticBuilder::K_Immediate; break; case FunctionEmissionStatus::Unknown: + // TODO: We should always delay diagnostics here in case a target + // region is in a function we do not emit. However, as the + // current diagnostics are associated with the function containing + // the target region and we do not emit that one, we would miss out + // on diagnostics for the target region itself. We need to anchor + // the diagnostics with the new generated function *or* ensure we + // emit diagnostics associated with the surrounding function. Kind = isOpenMPDeviceDelayedContext(*this) ? SemaDiagnosticBuilder::K_Deferred : SemaDiagnosticBuilder::K_Immediate; @@ -1925,14 +1931,15 @@ Sema::SemaDiagnosticBuilder Sema::diagIfOpenMPDeviceCode(SourceLocation Loc, } } - return SemaDiagnosticBuilder(Kind, Loc, DiagID, getCurFunctionDecl(), *this); + return SemaDiagnosticBuilder(Kind, Loc, DiagID, FD, *this); } Sema::SemaDiagnosticBuilder Sema::diagIfOpenMPHostCode(SourceLocation Loc, - unsigned DiagID) { + unsigned DiagID, + FunctionDecl *FD) { assert(LangOpts.OpenMP && !LangOpts.OpenMPIsDevice && "Expected OpenMP host compilation."); - FunctionEmissionStatus FES = getEmissionStatus(getCurFunctionDecl()); + FunctionEmissionStatus FES = getEmissionStatus(FD); SemaDiagnosticBuilder::Kind Kind = SemaDiagnosticBuilder::K_Nop; switch (FES) { case FunctionEmissionStatus::Emitted: @@ -1948,7 +1955,7 @@ Sema::SemaDiagnosticBuilder Sema::diagIfOpenMPHostCode(SourceLocation Loc, break; } - return SemaDiagnosticBuilder(Kind, Loc, DiagID, getCurFunctionDecl(), *this); + return SemaDiagnosticBuilder(Kind, Loc, DiagID, FD, *this); } static OpenMPDefaultmapClauseKind diff --git a/contrib/llvm-project/clang/lib/Sema/TreeTransform.h b/contrib/llvm-project/clang/lib/Sema/TreeTransform.h index 0a596e50658b..3c68f9458e58 100644 --- a/contrib/llvm-project/clang/lib/Sema/TreeTransform.h +++ b/contrib/llvm-project/clang/lib/Sema/TreeTransform.h @@ -12504,10 +12504,11 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) { E->getCaptureDefault()); getDerived().transformedLocalDecl(OldClass, {Class}); - Optional<std::tuple<unsigned, bool, Decl *>> Mangling; + Optional<std::tuple<bool, unsigned, unsigned, Decl *>> Mangling; if (getDerived().ReplacingOriginal()) - Mangling = std::make_tuple(OldClass->getLambdaManglingNumber(), - OldClass->hasKnownLambdaInternalLinkage(), + Mangling = std::make_tuple(OldClass->hasKnownLambdaInternalLinkage(), + OldClass->getLambdaManglingNumber(), + OldClass->getDeviceLambdaManglingNumber(), OldClass->getLambdaContextDecl()); // Build the call operator. diff --git a/contrib/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp b/contrib/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp index 6bfb9bd783b5..18ab4666a7d8 100644 --- a/contrib/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/contrib/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp @@ -1748,6 +1748,7 @@ void ASTDeclReader::ReadCXXDefinitionData( Lambda.NumExplicitCaptures = Record.readInt(); Lambda.HasKnownInternalLinkage = Record.readInt(); Lambda.ManglingNumber = Record.readInt(); + D->setDeviceLambdaManglingNumber(Record.readInt()); Lambda.ContextDecl = readDeclID(); Lambda.Captures = (Capture *)Reader.getContext().Allocate( sizeof(Capture) * Lambda.NumCaptures); diff --git a/contrib/llvm-project/clang/lib/Serialization/ASTWriter.cpp b/contrib/llvm-project/clang/lib/Serialization/ASTWriter.cpp index 6bfa7b0e7d6d..40900af6f9e0 100644 --- a/contrib/llvm-project/clang/lib/Serialization/ASTWriter.cpp +++ b/contrib/llvm-project/clang/lib/Serialization/ASTWriter.cpp @@ -5667,6 +5667,7 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) { Record->push_back(Lambda.NumExplicitCaptures); Record->push_back(Lambda.HasKnownInternalLinkage); Record->push_back(Lambda.ManglingNumber); + Record->push_back(D->getDeviceLambdaManglingNumber()); AddDeclRef(D->getLambdaContextDecl()); AddTypeSourceInfo(Lambda.MethodTyInfo); for (unsigned I = 0, N = Lambda.NumCaptures; I != N; ++I) { diff --git a/contrib/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/contrib/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h index 7f7b38d4215b..068fc9829e57 100644 --- a/contrib/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/contrib/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -226,7 +226,7 @@ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETPWENT \ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) -#define SANITIZER_INTERCEPT_FGETGRENT_R (SI_FREEBSD || SI_GLIBC || SI_SOLARIS) +#define SANITIZER_INTERCEPT_FGETGRENT_R (SI_GLIBC || SI_SOLARIS) #define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_GETPWENT_R \ (SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS) diff --git a/contrib/llvm-project/libcxx/include/__locale b/contrib/llvm-project/libcxx/include/__locale index a2da7d78049f..77e5faab2676 100644 --- a/contrib/llvm-project/libcxx/include/__locale +++ b/contrib/llvm-project/libcxx/include/__locale @@ -21,30 +21,30 @@ #include <locale.h> #if defined(_LIBCPP_MSVCRT_LIKE) # include <cstring> -# include <support/win32/locale_win32.h> +# include <__support/win32/locale_win32.h> #elif defined(__NuttX__) -# include <support/nuttx/xlocale.h> +# include <__support/nuttx/xlocale.h> #elif defined(_AIX) || defined(__MVS__) -# include <support/ibm/xlocale.h> +# include <__support/ibm/xlocale.h> #elif defined(__ANDROID__) -# include <support/android/locale_bionic.h> +# include <__support/android/locale_bionic.h> #elif defined(__sun__) # include <xlocale.h> -# include <support/solaris/xlocale.h> +# include <__support/solaris/xlocale.h> #elif defined(_NEWLIB_VERSION) -# include <support/newlib/xlocale.h> +# include <__support/newlib/xlocale.h> #elif defined(__OpenBSD__) -# include <support/openbsd/xlocale.h> +# include <__support/openbsd/xlocale.h> #elif (defined(__APPLE__) || defined(__FreeBSD__) \ || defined(__EMSCRIPTEN__) || defined(__IBMCPP__)) # include <xlocale.h> #elif defined(__Fuchsia__) -# include <support/fuchsia/xlocale.h> +# include <__support/fuchsia/xlocale.h> #elif defined(__wasi__) // WASI libc uses musl's locales support. -# include <support/musl/xlocale.h> +# include <__support/musl/xlocale.h> #elif defined(_LIBCPP_HAS_MUSL_LIBC) -# include <support/musl/xlocale.h> +# include <__support/musl/xlocale.h> #endif #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) diff --git a/contrib/llvm-project/libcxx/include/__threading_support b/contrib/llvm-project/libcxx/include/__threading_support index 473c9c3bbe49..de572f3ff84d 100644 --- a/contrib/llvm-project/libcxx/include/__threading_support +++ b/contrib/llvm-project/libcxx/include/__threading_support @@ -17,7 +17,7 @@ #include <errno.h> #ifdef __MVS__ -# include <support/ibm/nanosleep.h> +# include <__support/ibm/nanosleep.h> #endif #ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER diff --git a/contrib/llvm-project/libcxx/include/bit b/contrib/llvm-project/libcxx/include/bit index fe360179c5ca..f8c37c3d6bbf 100644 --- a/contrib/llvm-project/libcxx/include/bit +++ b/contrib/llvm-project/libcxx/include/bit @@ -62,7 +62,7 @@ namespace std { #include <__debug> #if defined(__IBMCPP__) -#include "support/ibm/support.h" +#include "__support/ibm/support.h" #endif #if defined(_LIBCPP_COMPILER_MSVC) #include <intrin.h> diff --git a/contrib/llvm-project/libcxx/include/limits b/contrib/llvm-project/libcxx/include/limits index 6d5d1e1aca75..8f97cd10a8b1 100644 --- a/contrib/llvm-project/libcxx/include/limits +++ b/contrib/llvm-project/libcxx/include/limits @@ -105,11 +105,11 @@ template<> class numeric_limits<cv long double>; #include <type_traits> #if defined(_LIBCPP_COMPILER_MSVC) -#include "support/win32/limits_msvc_win32.h" +#include "__support/win32/limits_msvc_win32.h" #endif // _LIBCPP_MSVCRT #if defined(__IBMCPP__) -#include "support/ibm/limits.h" +#include "__support/ibm/limits.h" #endif // __IBMCPP__ #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) diff --git a/contrib/llvm-project/libcxx/include/memory b/contrib/llvm-project/libcxx/include/memory index a00916c8c03f..39d0f5bee6a5 100644 --- a/contrib/llvm-project/libcxx/include/memory +++ b/contrib/llvm-project/libcxx/include/memory @@ -2647,7 +2647,7 @@ private: _Alloc *__alloc = reinterpret_cast<_Alloc*>(__first); return __alloc; } - _Tp* __get_elem() _NOEXCEPT { + _LIBCPP_NO_CFI _Tp* __get_elem() _NOEXCEPT { _CompressedPair *__as_pair = reinterpret_cast<_CompressedPair*>(__blob_); typename _CompressedPair::_Base2* __second = _CompressedPair::__get_second_base(__as_pair); _Tp *__elem = reinterpret_cast<_Tp*>(__second); diff --git a/contrib/llvm-project/libcxx/src/atomic.cpp b/contrib/llvm-project/libcxx/src/atomic.cpp index 6b73ed771cd1..9ae1fb5199bf 100644 --- a/contrib/llvm-project/libcxx/src/atomic.cpp +++ b/contrib/llvm-project/libcxx/src/atomic.cpp @@ -19,6 +19,12 @@ #include <linux/futex.h> #include <sys/syscall.h> +// libc++ uses SYS_futex as a universal syscall name. However, on 32 bit architectures +// with a 64 bit time_t, we need to specify SYS_futex_time64. +#if !defined(SYS_futex) && defined(SYS_futex_time64) +# define SYS_futex SYS_futex_time64 +#endif + #else // <- Add other operating systems here // Baseline needs no new headers diff --git a/contrib/llvm-project/libcxx/src/locale.cpp b/contrib/llvm-project/libcxx/src/locale.cpp index f109389f68f3..a0209d0ce8cf 100644 --- a/contrib/llvm-project/libcxx/src/locale.cpp +++ b/contrib/llvm-project/libcxx/src/locale.cpp @@ -29,7 +29,7 @@ #include "cwctype" #include "__sso_allocator" #if defined(_LIBCPP_MSVCRT) || defined(__MINGW32__) -#include "support/win32/locale_win32.h" +#include "__support/win32/locale_win32.h" #elif !defined(__BIONIC__) && !defined(__NuttX__) #include <langinfo.h> #endif diff --git a/contrib/llvm-project/lld/ELF/InputSection.cpp b/contrib/llvm-project/lld/ELF/InputSection.cpp index f40bb258b9af..6f16fc7abc48 100644 --- a/contrib/llvm-project/lld/ELF/InputSection.cpp +++ b/contrib/llvm-project/lld/ELF/InputSection.cpp @@ -901,7 +901,10 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) { continue; } - if (expr != R_ABS && expr != R_DTPREL && expr != R_RISCV_ADD) { + // R_ABS/R_DTPREL and some other relocations can be used from non-SHF_ALLOC + // sections. + if (expr != R_ABS && expr != R_DTPREL && expr != R_GOTPLTREL && + expr != R_RISCV_ADD) { std::string msg = getLocation<ELFT>(offset) + ": has non-ABS relocation " + toString(type) + " against symbol '" + toString(sym) + "'"; diff --git a/contrib/llvm-project/lld/docs/ReleaseNotes.rst b/contrib/llvm-project/lld/docs/ReleaseNotes.rst index e0b17ca3e030..24ed23bb2b7d 100644 --- a/contrib/llvm-project/lld/docs/ReleaseNotes.rst +++ b/contrib/llvm-project/lld/docs/ReleaseNotes.rst @@ -24,28 +24,124 @@ Non-comprehensive list of changes in this release ELF Improvements ---------------- -* ``--error-handling-script`` is added to allow for user-defined handlers upon +* ``--dependency-file`` has been added. (Similar to ``cc -M -MF``.) + (`D82437 <https://reviews.llvm.org/D82437>`_) +* ``--error-handling-script`` has been added to allow for user-defined handlers upon missing libraries. (`D87758 <https://reviews.llvm.org/D87758>`_) +* ``--exclude-libs`` can now localize defined version symbols and bitcode referenced libcall symbols. + (`D94280 <https://reviews.llvm.org/D94280>`_) +* ``--gdb-index`` now works with DWARF v5 and ``--icf={safe,all}``. + (`D85579 <https://reviews.llvm.org/D85579>`_) + (`D89751 <https://reviews.llvm.org/D89751>`_) +* ``--gdb-index --emit-relocs`` can now be used together. + (`D94354 <https://reviews.llvm.org/D94354>`_) +* ``--icf={safe,all}`` conservatively no longer fold text sections with LSDA. + Previously ICF on ``-fexceptions`` code could be unsafe. + (`D84610 <https://reviews.llvm.org/D84610>`_) +* ``--icf={safe,all}`` can now fold two sections with relocations referencing aliased symbols. + (`D88830 <https://reviews.llvm.org/D88830>`_) +* ``--lto-pseudo-probe-for-profiling`` has been added. + (`D95056 <https://reviews.llvm.org/D95056>`_) +* ``--no-lto-whole-program-visibility`` has been added. + (`D92060 <https://reviews.llvm.org/D92060>`_) +* ``--oformat-binary`` has been fixed to respect LMA. + (`D85086 <https://reviews.llvm.org/D85086>`_) +* ``--reproduce`` includes ``--lto-sample-profile``, ``--just-symbols``, ``--call-graph-ordering-file``, ``--retain-symbols-file`` files. +* ``-r --gc-sections`` is now supported. + (`D84131 <https://reviews.llvm.org/D84131>`_) +* A ``-u`` specified symbol will no longer change the binding to ``STB_WEAK``. + (`D88945 <https://reviews.llvm.org/D88945>`_) +* ``--wrap`` support has been improved. + + If ``foo`` is not referenced, there is no longer an undefined symbol ``__wrap_foo``. + + If ``__real_foo`` is not referenced, there is no longer an undefined symbol ``foo``. +* ``SHF_LINK_ORDER`` sections can now have zero ``sh_link`` values. +* ``SHF_LINK_ORDER`` and non-``SHF_LINK_ORDER`` sections can now be mixed within an input section description. + (`D84001 <https://reviews.llvm.org/D84001>`_) +* ``LOG2CEIL`` is now supported in linker scripts. + (`D84054 <https://reviews.llvm.org/D84054>`_) +* ``DEFINED`` has been fixed to check whether the symbol is defined. + (`D83758 <https://reviews.llvm.org/D83758>`_) +* An input section description may now have multiple ``SORT_*``. + The matched sections are ordered by radix sort with the keys being ``(SORT*, --sort-section, input order)``. + (`D91127 <https://reviews.llvm.org/D91127>`_) +* Users can now provide a GNU style linker script to convert ``.ctors`` into ``.init_array``. + (`D91187 <https://reviews.llvm.org/D91187>`_) +* An empty output section can now be discarded even if it is assigned to a program header. + (`D92301 <https://reviews.llvm.org/D92301>`_) +* Non-``SHF_ALLOC`` sections now have larger file offsets than ``SHF_ALLOC`` sections. + (`D85867 <https://reviews.llvm.org/D85867>`_) +* Some symbol versioning improvements. + + Defined ``foo@@v1`` now resolve undefined ``foo@v1`` (`D92259 <https://reviews.llvm.org/D92259>`_) + + Undefined ``foo@v1`` now gets an error (`D92260 <https://reviews.llvm.org/D92260>`_) +* The AArch64 port now has support for ``STO_AARCH64_VARIANT_PCS`` and ``DT_AARCH64_VARIANT_PCS``. + (`D93045 <https://reviews.llvm.org/D93045>`_) +* The AArch64 port now has support for ``R_AARCH64_LD64_GOTPAGE_LO15``. +* The PowerPC64 port now detects missing R_PPC64_TLSGD/R_PPC64_TLSLD and disables TLS relaxation. + This allows linking with object files produced by very old IBM XL compilers. + (`D92959 <https://reviews.llvm.org/D92959>`_) +* Many PowerPC PC-relative relocations are now supported. +* ``R_PPC_ADDR24`` and ``R_PPC64_ADDR16_HIGH`` are now supported. +* powerpcle is now supported. Tested with FreeBSD loader and freestanding. + (`D93917 <https://reviews.llvm.org/D93917>`_) +* RISC-V: the first ``SHT_RISCV_ATTRIBUTES`` section is now retained. + (`D86309 <https://reviews.llvm.org/D86309>`_) +* LTO pipeline now defaults to the new PM if the CMake variable ``ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER`` is on. + (`D92885 <https://reviews.llvm.org/D92885>`_) Breaking changes ---------------- -* ... +* A COMMON symbol can now cause the fetch of an archive providing a ``STB_GLOBAL`` definition. + This behavior follows GNU ld newer than December 1999. + If you see ``duplicate symbol`` errors with the new behavior, check out `PR49226 <https://bugs.llvm.org//show_bug.cgi?id=49226>`_. + (`D86142 <https://reviews.llvm.org/D86142>`_) COFF Improvements ----------------- -* ... +* Error out clearly if creating a DLL with too many exported symbols. + (`D86701 <https://reviews.llvm.org/D86701>`_) MinGW Improvements ------------------ -* ... +* Enabled dynamicbase by default. (`D86654 <https://reviews.llvm.org/D86654>`_) -MachO Improvements +* Tolerate mismatches between COMDAT section sizes with different amount of + padding (produced by binutils) by inspecting the aux section definition. + (`D86659 <https://reviews.llvm.org/D86659>`_) + +* Support setting the subsystem version via the subsystem argument. + (`D88804 <https://reviews.llvm.org/D88804>`_) + +* Implemented the GNU -wrap option. + (`D89004 <https://reviews.llvm.org/D89004>`_, + `D91689 <https://reviews.llvm.org/D91689>`_) + +* Handle the ``--demangle`` and ``--no-demangle`` options. + (`D93950 <https://reviews.llvm.org/D93950>`_) + + +Mach-O Improvements ------------------ -* Item 1. +We've gotten the new implementation of LLD for Mach-O to the point where it is +able to link large x86_64 programs, and we'd love to get some alpha testing on +it. The new Darwin back-end can be invoked as follows: + +.. code-block:: + clang -fuse-ld=lld.darwinnew /path/to/file.c + +To reach this point, we implemented numerous features, and it's easier to list +the major features we *haven't* yet completed: + +* LTO support +* Stack unwinding for exceptions +* Support for arm64, arm, and i386 architectures + +If you stumble upon an issue and it doesn't fall into one of these categories, +please file a bug report! + WebAssembly Improvements ------------------------ diff --git a/contrib/llvm-project/lldb/source/Host/common/NativeProcessProtocol.cpp b/contrib/llvm-project/lldb/source/Host/common/NativeProcessProtocol.cpp index 493e14cb904b..070fda664678 100644 --- a/contrib/llvm-project/lldb/source/Host/common/NativeProcessProtocol.cpp +++ b/contrib/llvm-project/lldb/source/Host/common/NativeProcessProtocol.cpp @@ -522,7 +522,8 @@ NativeProcessProtocol::GetSoftwareBreakpointTrapOpcode(size_t size_hint) { static const uint8_t g_mips64_opcode[] = {0x00, 0x00, 0x00, 0x0d}; static const uint8_t g_mips64el_opcode[] = {0x0d, 0x00, 0x00, 0x00}; static const uint8_t g_s390x_opcode[] = {0x00, 0x01}; - static const uint8_t g_ppc64le_opcode[] = {0x08, 0x00, 0xe0, 0x7f}; // trap + static const uint8_t g_ppc_opcode[] = {0x7f, 0xe0, 0x00, 0x08}; // trap + static const uint8_t g_ppcle_opcode[] = {0x08, 0x00, 0xe0, 0x7f}; // trap switch (GetArchitecture().GetMachine()) { case llvm::Triple::aarch64: @@ -544,8 +545,12 @@ NativeProcessProtocol::GetSoftwareBreakpointTrapOpcode(size_t size_hint) { case llvm::Triple::systemz: return llvm::makeArrayRef(g_s390x_opcode); + case llvm::Triple::ppc: + case llvm::Triple::ppc64: + return llvm::makeArrayRef(g_ppc_opcode); + case llvm::Triple::ppc64le: - return llvm::makeArrayRef(g_ppc64le_opcode); + return llvm::makeArrayRef(g_ppcle_opcode); default: return llvm::createStringError(llvm::inconvertibleErrorCode(), @@ -568,6 +573,8 @@ size_t NativeProcessProtocol::GetSoftwareBreakpointPCOffset() { case llvm::Triple::mips64el: case llvm::Triple::mips: case llvm::Triple::mipsel: + case llvm::Triple::ppc: + case llvm::Triple::ppc64: case llvm::Triple::ppc64le: // On these architectures the PC doesn't get updated for breakpoint hits. return 0; diff --git a/contrib/llvm-project/lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.cpp b/contrib/llvm-project/lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.cpp index f4d44eb7e745..6b39a83fd668 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.cpp +++ b/contrib/llvm-project/lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.cpp @@ -214,55 +214,9 @@ void PlatformFreeBSD::GetStatus(Stream &strm) { #endif } -size_t -PlatformFreeBSD::GetSoftwareBreakpointTrapOpcode(Target &target, - BreakpointSite *bp_site) { - switch (target.GetArchitecture().GetMachine()) { - case llvm::Triple::arm: { - lldb::BreakpointLocationSP bp_loc_sp(bp_site->GetOwnerAtIndex(0)); - AddressClass addr_class = AddressClass::eUnknown; - - if (bp_loc_sp) { - addr_class = bp_loc_sp->GetAddress().GetAddressClass(); - if (addr_class == AddressClass::eUnknown && - (bp_loc_sp->GetAddress().GetFileAddress() & 1)) - addr_class = AddressClass::eCodeAlternateISA; - } - - if (addr_class == AddressClass::eCodeAlternateISA) { - // TODO: Enable when FreeBSD supports thumb breakpoints. - // FreeBSD kernel as of 10.x, does not support thumb breakpoints - return 0; - } - - static const uint8_t g_arm_breakpoint_opcode[] = {0xFE, 0xDE, 0xFF, 0xE7}; - size_t trap_opcode_size = sizeof(g_arm_breakpoint_opcode); - assert(bp_site); - if (bp_site->SetTrapOpcode(g_arm_breakpoint_opcode, trap_opcode_size)) - return trap_opcode_size; - } - LLVM_FALLTHROUGH; - default: - return Platform::GetSoftwareBreakpointTrapOpcode(target, bp_site); - } -} - bool PlatformFreeBSD::CanDebugProcess() { if (IsHost()) { - llvm::Triple host_triple{llvm::sys::getProcessTriple()}; - bool use_legacy_plugin; - - switch (host_triple.getArch()) { - case llvm::Triple::x86: - case llvm::Triple::x86_64: - // FreeBSDRemote plugin supports x86 only at the moment - use_legacy_plugin = !!getenv("FREEBSD_LEGACY_PLUGIN"); - break; - default: - use_legacy_plugin = true; - } - - return !use_legacy_plugin; + return true; } else { // If we're connected, we can debug. return IsConnected(); diff --git a/contrib/llvm-project/lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.h b/contrib/llvm-project/lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.h index c198ea18638d..4fd10fb1be73 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.h +++ b/contrib/llvm-project/lldb/source/Plugins/Platform/FreeBSD/PlatformFreeBSD.h @@ -44,9 +44,6 @@ public: bool CanDebugProcess() override; - size_t GetSoftwareBreakpointTrapOpcode(Target &target, - BreakpointSite *bp_site) override; - void CalculateTrapHandlerSymbolNames() override; MmapArgList GetMmapArgumentList(const ArchSpec &arch, lldb::addr_t addr, diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.cpp deleted file mode 100644 index de9036fbfe56..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.cpp +++ /dev/null @@ -1,615 +0,0 @@ -//===-- FreeBSDThread.cpp -------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include <errno.h> -#include <pthread.h> -#include <pthread_np.h> -#include <stdlib.h> -#include <sys/sysctl.h> -#include <sys/types.h> -#include <sys/user.h> - -#include "FreeBSDThread.h" -#include "POSIXStopInfo.h" -#include "Plugins/Process/POSIX/ProcessPOSIXLog.h" -#include "Plugins/Process/Utility/RegisterContextFreeBSD_i386.h" -#include "Plugins/Process/Utility/RegisterContextFreeBSD_mips64.h" -#include "Plugins/Process/Utility/RegisterContextFreeBSD_powerpc.h" -#include "Plugins/Process/Utility/RegisterContextFreeBSD_x86_64.h" -#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm.h" -#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h" -#include "ProcessFreeBSD.h" -#include "ProcessMonitor.h" -#include "RegisterContextPOSIXProcessMonitor_arm.h" -#include "RegisterContextPOSIXProcessMonitor_arm64.h" -#include "RegisterContextPOSIXProcessMonitor_mips64.h" -#include "RegisterContextPOSIXProcessMonitor_powerpc.h" -#include "RegisterContextPOSIXProcessMonitor_x86.h" -#include "lldb/Breakpoint/BreakpointLocation.h" -#include "lldb/Breakpoint/Watchpoint.h" -#include "lldb/Core/Debugger.h" -#include "lldb/Host/Host.h" -#include "lldb/Host/HostInfo.h" -#include "lldb/Host/HostNativeThread.h" -#include "lldb/Target/Process.h" -#include "lldb/Target/StopInfo.h" -#include "lldb/Target/Target.h" -#include "lldb/Target/ThreadSpec.h" -#include "lldb/Target/UnixSignals.h" -#include "lldb/Target/Unwind.h" -#include "lldb/Utility/State.h" -#include "llvm/ADT/SmallString.h" - -using namespace lldb; -using namespace lldb_private; - -FreeBSDThread::FreeBSDThread(Process &process, lldb::tid_t tid) - : Thread(process, tid), m_frame_up(), m_breakpoint(), - m_thread_name_valid(false), m_thread_name(), m_posix_thread(nullptr) { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_THREAD)); - LLDB_LOGV(log, "tid = {0}", tid); - - // Set the current watchpoints for this thread. - Target &target = GetProcess()->GetTarget(); - const WatchpointList &wp_list = target.GetWatchpointList(); - size_t wp_size = wp_list.GetSize(); - - for (uint32_t wp_idx = 0; wp_idx < wp_size; wp_idx++) { - lldb::WatchpointSP wp = wp_list.GetByIndex(wp_idx); - if (wp.get() && wp->IsEnabled()) { - // This watchpoint as been enabled; obviously this "new" thread has been - // created since that watchpoint was enabled. Since the - // POSIXBreakpointProtocol has yet to be initialized, its - // m_watchpoints_initialized member will be FALSE. Attempting to read - // the debug status register to determine if a watchpoint has been hit - // would result in the zeroing of that register. Since the active debug - // registers would have been cloned when this thread was created, simply - // force the m_watchpoints_initized member to TRUE and avoid resetting - // dr6 and dr7. - GetPOSIXBreakpointProtocol()->ForceWatchpointsInitialized(); - } - } -} - -FreeBSDThread::~FreeBSDThread() { DestroyThread(); } - -ProcessMonitor &FreeBSDThread::GetMonitor() { - ProcessSP base = GetProcess(); - ProcessFreeBSD &process = static_cast<ProcessFreeBSD &>(*base); - return process.GetMonitor(); -} - -void FreeBSDThread::RefreshStateAfterStop() { - // Invalidate all registers in our register context. We don't set "force" to - // true because the stop reply packet might have had some register values - // that were expedited and these will already be copied into the register - // context by the time this function gets called. The KDPRegisterContext - // class has been made smart enough to detect when it needs to invalidate - // which registers are valid by putting hooks in the register read and - // register supply functions where they check the process stop ID and do the - // right thing. if (StateIsStoppedState(GetState()) - { - const bool force = false; - GetRegisterContext()->InvalidateIfNeeded(force); - } -} - -const char *FreeBSDThread::GetInfo() { return nullptr; } - -void FreeBSDThread::SetName(const char *name) { - m_thread_name_valid = (name && name[0]); - if (m_thread_name_valid) - m_thread_name.assign(name); - else - m_thread_name.clear(); -} - -const char *FreeBSDThread::GetName() { - if (!m_thread_name_valid) { - m_thread_name.clear(); - int pid = GetProcess()->GetID(); - - struct kinfo_proc *kp = nullptr, *nkp; - size_t len = 0; - int error; - int ctl[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PID | KERN_PROC_INC_THREAD, - pid}; - - while (1) { - error = sysctl(ctl, 4, kp, &len, nullptr, 0); - if (kp == nullptr || (error != 0 && errno == ENOMEM)) { - // Add extra space in case threads are added before next call. - len += sizeof(*kp) + len / 10; - nkp = (struct kinfo_proc *)realloc(kp, len); - if (nkp == nullptr) { - free(kp); - return nullptr; - } - kp = nkp; - continue; - } - if (error != 0) - len = 0; - break; - } - - for (size_t i = 0; i < len / sizeof(*kp); i++) { - if (kp[i].ki_tid == (lwpid_t)GetID()) { - m_thread_name.append(kp[i].ki_tdname, - kp[i].ki_tdname + strlen(kp[i].ki_tdname)); - break; - } - } - free(kp); - m_thread_name_valid = true; - } - - if (m_thread_name.empty()) - return nullptr; - return m_thread_name.c_str(); -} - -lldb::RegisterContextSP FreeBSDThread::GetRegisterContext() { - if (!m_reg_context_sp) { - m_posix_thread = nullptr; - - RegisterInfoInterface *reg_interface = nullptr; - const ArchSpec &target_arch = GetProcess()->GetTarget().GetArchitecture(); - - switch (target_arch.GetMachine()) { - case llvm::Triple::aarch64: - case llvm::Triple::arm: - break; - case llvm::Triple::ppc: -#ifndef __powerpc64__ - reg_interface = new RegisterContextFreeBSD_powerpc32(target_arch); - break; -#endif - case llvm::Triple::ppc64: - reg_interface = new RegisterContextFreeBSD_powerpc64(target_arch); - break; - case llvm::Triple::mips64: - reg_interface = new RegisterContextFreeBSD_mips64(target_arch); - break; - case llvm::Triple::x86: - reg_interface = new RegisterContextFreeBSD_i386(target_arch); - break; - case llvm::Triple::x86_64: - reg_interface = new RegisterContextFreeBSD_x86_64(target_arch); - break; - default: - llvm_unreachable("CPU not supported"); - } - - switch (target_arch.GetMachine()) { - case llvm::Triple::aarch64: { - RegisterContextPOSIXProcessMonitor_arm64 *reg_ctx = - new RegisterContextPOSIXProcessMonitor_arm64( - *this, std::make_unique<RegisterInfoPOSIX_arm64>(target_arch)); - m_posix_thread = reg_ctx; - m_reg_context_sp.reset(reg_ctx); - break; - } - case llvm::Triple::arm: { - RegisterContextPOSIXProcessMonitor_arm *reg_ctx = - new RegisterContextPOSIXProcessMonitor_arm( - *this, std::make_unique<RegisterInfoPOSIX_arm>(target_arch)); - m_posix_thread = reg_ctx; - m_reg_context_sp.reset(reg_ctx); - break; - } - case llvm::Triple::mips64: { - RegisterContextPOSIXProcessMonitor_mips64 *reg_ctx = - new RegisterContextPOSIXProcessMonitor_mips64(*this, 0, - reg_interface); - m_posix_thread = reg_ctx; - m_reg_context_sp.reset(reg_ctx); - break; - } - case llvm::Triple::ppc: - case llvm::Triple::ppc64: { - RegisterContextPOSIXProcessMonitor_powerpc *reg_ctx = - new RegisterContextPOSIXProcessMonitor_powerpc(*this, 0, - reg_interface); - m_posix_thread = reg_ctx; - m_reg_context_sp.reset(reg_ctx); - break; - } - case llvm::Triple::x86: - case llvm::Triple::x86_64: { - RegisterContextPOSIXProcessMonitor_x86_64 *reg_ctx = - new RegisterContextPOSIXProcessMonitor_x86_64(*this, 0, - reg_interface); - m_posix_thread = reg_ctx; - m_reg_context_sp.reset(reg_ctx); - break; - } - default: - break; - } - } - return m_reg_context_sp; -} - -lldb::RegisterContextSP -FreeBSDThread::CreateRegisterContextForFrame(lldb_private::StackFrame *frame) { - lldb::RegisterContextSP reg_ctx_sp; - uint32_t concrete_frame_idx = 0; - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_THREAD)); - LLDB_LOGV(log, "called"); - - if (frame) - concrete_frame_idx = frame->GetConcreteFrameIndex(); - - if (concrete_frame_idx == 0) - reg_ctx_sp = GetRegisterContext(); - else { - reg_ctx_sp = GetUnwinder().CreateRegisterContextForFrame(frame); - } - - return reg_ctx_sp; -} - -lldb::addr_t FreeBSDThread::GetThreadPointer() { - ProcessMonitor &monitor = GetMonitor(); - addr_t addr; - if (monitor.ReadThreadPointer(GetID(), addr)) - return addr; - else - return LLDB_INVALID_ADDRESS; -} - -bool FreeBSDThread::CalculateStopInfo() { - SetStopInfo(m_stop_info_sp); - return true; -} - -void FreeBSDThread::DidStop() { - // Don't set the thread state to stopped unless we really stopped. -} - -void FreeBSDThread::WillResume(lldb::StateType resume_state) { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_THREAD)); - LLDB_LOGF(log, "tid %" PRIu64 " resume_state = %s", GetID(), - lldb_private::StateAsCString(resume_state)); - ProcessSP process_sp(GetProcess()); - ProcessFreeBSD *process = static_cast<ProcessFreeBSD *>(process_sp.get()); - int signo = GetResumeSignal(); - bool signo_valid = process->GetUnixSignals()->SignalIsValid(signo); - - switch (resume_state) { - case eStateSuspended: - case eStateStopped: - process->m_suspend_tids.push_back(GetID()); - break; - case eStateRunning: - process->m_run_tids.push_back(GetID()); - if (signo_valid) - process->m_resume_signo = signo; - break; - case eStateStepping: - process->m_step_tids.push_back(GetID()); - if (signo_valid) - process->m_resume_signo = signo; - break; - default: - break; - } -} - -bool FreeBSDThread::Resume() { - lldb::StateType resume_state = GetResumeState(); - ProcessMonitor &monitor = GetMonitor(); - bool status; - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_THREAD)); - LLDB_LOGF(log, "FreeBSDThread::%s (), resume_state = %s", __FUNCTION__, - StateAsCString(resume_state)); - - switch (resume_state) { - default: - assert(false && "Unexpected state for resume!"); - status = false; - break; - - case lldb::eStateRunning: - SetState(resume_state); - status = monitor.Resume(GetID(), GetResumeSignal()); - break; - - case lldb::eStateStepping: - SetState(resume_state); - status = monitor.SingleStep(GetID(), GetResumeSignal()); - break; - case lldb::eStateStopped: - case lldb::eStateSuspended: - status = true; - break; - } - - return status; -} - -void FreeBSDThread::Notify(const ProcessMessage &message) { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_THREAD)); - LLDB_LOGF(log, "FreeBSDThread::%s () message kind = '%s' for tid %" PRIu64, - __FUNCTION__, message.PrintKind(), GetID()); - - switch (message.GetKind()) { - default: - assert(false && "Unexpected message kind!"); - break; - - case ProcessMessage::eExitMessage: - // Nothing to be done. - break; - - case ProcessMessage::eLimboMessage: - LimboNotify(message); - break; - - case ProcessMessage::eCrashMessage: - case ProcessMessage::eSignalMessage: - SignalNotify(message); - break; - - case ProcessMessage::eSignalDeliveredMessage: - SignalDeliveredNotify(message); - break; - - case ProcessMessage::eTraceMessage: - TraceNotify(message); - break; - - case ProcessMessage::eBreakpointMessage: - BreakNotify(message); - break; - - case ProcessMessage::eWatchpointMessage: - WatchNotify(message); - break; - - case ProcessMessage::eExecMessage: - ExecNotify(message); - break; - } -} - -bool FreeBSDThread::EnableHardwareWatchpoint(Watchpoint *wp) { - bool wp_set = false; - if (wp) { - addr_t wp_addr = wp->GetLoadAddress(); - size_t wp_size = wp->GetByteSize(); - bool wp_read = wp->WatchpointRead(); - bool wp_write = wp->WatchpointWrite(); - uint32_t wp_hw_index = wp->GetHardwareIndex(); - POSIXBreakpointProtocol *reg_ctx = GetPOSIXBreakpointProtocol(); - if (reg_ctx) - wp_set = reg_ctx->SetHardwareWatchpointWithIndex( - wp_addr, wp_size, wp_read, wp_write, wp_hw_index); - } - return wp_set; -} - -bool FreeBSDThread::DisableHardwareWatchpoint(Watchpoint *wp) { - bool result = false; - if (wp) { - lldb::RegisterContextSP reg_ctx_sp = GetRegisterContext(); - if (reg_ctx_sp.get()) - result = reg_ctx_sp->ClearHardwareWatchpoint(wp->GetHardwareIndex()); - } - return result; -} - -uint32_t FreeBSDThread::NumSupportedHardwareWatchpoints() { - lldb::RegisterContextSP reg_ctx_sp = GetRegisterContext(); - if (reg_ctx_sp.get()) - return reg_ctx_sp->NumSupportedHardwareWatchpoints(); - return 0; -} - -uint32_t FreeBSDThread::FindVacantWatchpointIndex() { - uint32_t hw_index = LLDB_INVALID_INDEX32; - uint32_t num_hw_wps = NumSupportedHardwareWatchpoints(); - uint32_t wp_idx; - POSIXBreakpointProtocol *reg_ctx = GetPOSIXBreakpointProtocol(); - if (reg_ctx) { - for (wp_idx = 0; wp_idx < num_hw_wps; wp_idx++) { - if (reg_ctx->IsWatchpointVacant(wp_idx)) { - hw_index = wp_idx; - break; - } - } - } - return hw_index; -} - -void FreeBSDThread::BreakNotify(const ProcessMessage &message) { - bool status; - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_THREAD)); - - assert(GetRegisterContext()); - status = GetPOSIXBreakpointProtocol()->UpdateAfterBreakpoint(); - assert(status && "Breakpoint update failed!"); - - // With our register state restored, resolve the breakpoint object - // corresponding to our current PC. - assert(GetRegisterContext()); - lldb::addr_t pc = GetRegisterContext()->GetPC(); - LLDB_LOGF(log, "FreeBSDThread::%s () PC=0x%8.8" PRIx64, __FUNCTION__, pc); - lldb::BreakpointSiteSP bp_site( - GetProcess()->GetBreakpointSiteList().FindByAddress(pc)); - - // If the breakpoint is for this thread, then we'll report the hit, but if it - // is for another thread, we create a stop reason with should_stop=false. If - // there is no breakpoint location, then report an invalid stop reason. We - // don't need to worry about stepping over the breakpoint here, that will be - // taken care of when the thread resumes and notices that there's a - // breakpoint under the pc. - if (bp_site) { - lldb::break_id_t bp_id = bp_site->GetID(); - // If we have an operating system plug-in, we might have set a thread - // specific breakpoint using the operating system thread ID, so we can't - // make any assumptions about the thread ID so we must always report the - // breakpoint regardless of the thread. - if (bp_site->ValidForThisThread(this) || - GetProcess()->GetOperatingSystem() != nullptr) - SetStopInfo(StopInfo::CreateStopReasonWithBreakpointSiteID(*this, bp_id)); - else { - const bool should_stop = false; - SetStopInfo(StopInfo::CreateStopReasonWithBreakpointSiteID(*this, bp_id, - should_stop)); - } - } else - SetStopInfo(StopInfoSP()); -} - -void FreeBSDThread::WatchNotify(const ProcessMessage &message) { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_THREAD)); - - lldb::addr_t halt_addr = message.GetHWAddress(); - LLDB_LOGF(log, - "FreeBSDThread::%s () Hardware Watchpoint Address = 0x%8.8" PRIx64, - __FUNCTION__, halt_addr); - - POSIXBreakpointProtocol *reg_ctx = GetPOSIXBreakpointProtocol(); - if (reg_ctx) { - uint32_t num_hw_wps = reg_ctx->NumSupportedHardwareWatchpoints(); - uint32_t wp_idx; - for (wp_idx = 0; wp_idx < num_hw_wps; wp_idx++) { - if (reg_ctx->IsWatchpointHit(wp_idx)) { - // Clear the watchpoint hit here - reg_ctx->ClearWatchpointHits(); - break; - } - } - - if (wp_idx == num_hw_wps) - return; - - Target &target = GetProcess()->GetTarget(); - lldb::addr_t wp_monitor_addr = reg_ctx->GetWatchpointAddress(wp_idx); - const WatchpointList &wp_list = target.GetWatchpointList(); - lldb::WatchpointSP wp_sp = wp_list.FindByAddress(wp_monitor_addr); - - assert(wp_sp.get() && "No watchpoint found"); - SetStopInfo( - StopInfo::CreateStopReasonWithWatchpointID(*this, wp_sp->GetID())); - } -} - -void FreeBSDThread::TraceNotify(const ProcessMessage &message) { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_THREAD)); - - // Try to resolve the breakpoint object corresponding to the current PC. - assert(GetRegisterContext()); - lldb::addr_t pc = GetRegisterContext()->GetPC(); - LLDB_LOGF(log, "FreeBSDThread::%s () PC=0x%8.8" PRIx64, __FUNCTION__, pc); - lldb::BreakpointSiteSP bp_site( - GetProcess()->GetBreakpointSiteList().FindByAddress(pc)); - - // If the current pc is a breakpoint site then set the StopInfo to - // Breakpoint. Otherwise, set the StopInfo to Watchpoint or Trace. If we have - // an operating system plug-in, we might have set a thread specific - // breakpoint using the operating system thread ID, so we can't make any - // assumptions about the thread ID so we must always report the breakpoint - // regardless of the thread. - if (bp_site && (bp_site->ValidForThisThread(this) || - GetProcess()->GetOperatingSystem() != nullptr)) - SetStopInfo(StopInfo::CreateStopReasonWithBreakpointSiteID( - *this, bp_site->GetID())); - else { - POSIXBreakpointProtocol *reg_ctx = GetPOSIXBreakpointProtocol(); - if (reg_ctx) { - uint32_t num_hw_wps = reg_ctx->NumSupportedHardwareWatchpoints(); - uint32_t wp_idx; - for (wp_idx = 0; wp_idx < num_hw_wps; wp_idx++) { - if (reg_ctx->IsWatchpointHit(wp_idx)) { - WatchNotify(message); - return; - } - } - } - SetStopInfo(StopInfo::CreateStopReasonToTrace(*this)); - } -} - -void FreeBSDThread::LimboNotify(const ProcessMessage &message) { - SetStopInfo(lldb::StopInfoSP(new POSIXLimboStopInfo(*this))); -} - -void FreeBSDThread::SignalNotify(const ProcessMessage &message) { - int signo = message.GetSignal(); - if (message.GetKind() == ProcessMessage::eCrashMessage) { - std::string stop_description = GetCrashReasonString( - message.GetCrashReason(), message.GetFaultAddress()); - SetStopInfo(StopInfo::CreateStopReasonWithSignal( - *this, signo, stop_description.c_str())); - } else { - SetStopInfo(StopInfo::CreateStopReasonWithSignal(*this, signo)); - } -} - -void FreeBSDThread::SignalDeliveredNotify(const ProcessMessage &message) { - int signo = message.GetSignal(); - SetStopInfo(StopInfo::CreateStopReasonWithSignal(*this, signo)); -} - -unsigned FreeBSDThread::GetRegisterIndexFromOffset(unsigned offset) { - unsigned reg = LLDB_INVALID_REGNUM; - ArchSpec arch = HostInfo::GetArchitecture(); - - switch (arch.GetMachine()) { - default: - llvm_unreachable("CPU type not supported!"); - break; - - case llvm::Triple::aarch64: - case llvm::Triple::arm: - case llvm::Triple::mips64: - case llvm::Triple::ppc: - case llvm::Triple::ppc64: - case llvm::Triple::x86: - case llvm::Triple::x86_64: { - POSIXBreakpointProtocol *reg_ctx = GetPOSIXBreakpointProtocol(); - reg = reg_ctx->GetRegisterIndexFromOffset(offset); - } break; - } - return reg; -} - -void FreeBSDThread::ExecNotify(const ProcessMessage &message) { - SetStopInfo(StopInfo::CreateStopReasonWithExec(*this)); -} - -const char *FreeBSDThread::GetRegisterName(unsigned reg) { - const char *name = nullptr; - ArchSpec arch = HostInfo::GetArchitecture(); - - switch (arch.GetMachine()) { - default: - assert(false && "CPU type not supported!"); - break; - - case llvm::Triple::aarch64: - case llvm::Triple::arm: - case llvm::Triple::mips64: - case llvm::Triple::ppc: - case llvm::Triple::ppc64: - case llvm::Triple::x86: - case llvm::Triple::x86_64: - name = GetRegisterContext()->GetRegisterName(reg); - break; - } - return name; -} - -const char *FreeBSDThread::GetRegisterNameFromOffset(unsigned offset) { - return GetRegisterName(GetRegisterIndexFromOffset(offset)); -} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.h deleted file mode 100644 index 774ffb511bc6..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.h +++ /dev/null @@ -1,111 +0,0 @@ -//===-- FreeBSDThread.h -----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_FreeBSDThread_H_ -#define liblldb_FreeBSDThread_H_ - -#include <memory> -#include <string> - -#include "RegisterContextPOSIX.h" -#include "lldb/Target/Thread.h" - -class ProcessMessage; -class ProcessMonitor; -class POSIXBreakpointProtocol; - -// @class FreeBSDThread -// Abstraction of a FreeBSD thread. -class FreeBSDThread : public lldb_private::Thread { -public: - // Constructors and destructors - FreeBSDThread(lldb_private::Process &process, lldb::tid_t tid); - - virtual ~FreeBSDThread(); - - // POSIXThread - void RefreshStateAfterStop() override; - - // This notifies the thread when a private stop occurs. - void DidStop() override; - - const char *GetInfo() override; - - void SetName(const char *name) override; - - const char *GetName() override; - - lldb::RegisterContextSP GetRegisterContext() override; - - lldb::RegisterContextSP - CreateRegisterContextForFrame(lldb_private::StackFrame *frame) override; - - lldb::addr_t GetThreadPointer() override; - - // These functions provide a mapping from the register offset - // back to the register index or name for use in debugging or log - // output. - - unsigned GetRegisterIndexFromOffset(unsigned offset); - - const char *GetRegisterName(unsigned reg); - - const char *GetRegisterNameFromOffset(unsigned offset); - - // These methods form a specialized interface to POSIX threads. - // - bool Resume(); - - void Notify(const ProcessMessage &message); - - // These methods provide an interface to watchpoints - // - bool EnableHardwareWatchpoint(lldb_private::Watchpoint *wp); - - bool DisableHardwareWatchpoint(lldb_private::Watchpoint *wp); - - uint32_t NumSupportedHardwareWatchpoints(); - - uint32_t FindVacantWatchpointIndex(); - -protected: - POSIXBreakpointProtocol *GetPOSIXBreakpointProtocol() { - if (!m_reg_context_sp) - m_reg_context_sp = GetRegisterContext(); - return m_posix_thread; - } - - std::unique_ptr<lldb_private::StackFrame> m_frame_up; - - lldb::BreakpointSiteSP m_breakpoint; - - bool m_thread_name_valid; - std::string m_thread_name; - POSIXBreakpointProtocol *m_posix_thread; - - ProcessMonitor &GetMonitor(); - - bool CalculateStopInfo() override; - - void BreakNotify(const ProcessMessage &message); - void WatchNotify(const ProcessMessage &message); - virtual void TraceNotify(const ProcessMessage &message); - void LimboNotify(const ProcessMessage &message); - void SignalNotify(const ProcessMessage &message); - void SignalDeliveredNotify(const ProcessMessage &message); - void CrashNotify(const ProcessMessage &message); - void ExitNotify(const ProcessMessage &message); - void ExecNotify(const ProcessMessage &message); - - // FreeBSDThread internal API. - - // POSIXThread override - virtual void WillResume(lldb::StateType resume_state) override; -}; - -#endif // #ifndef liblldb_FreeBSDThread_H_ diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp index 163093c2ab1f..5961ff4439db 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.cpp +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp @@ -213,8 +213,9 @@ void NativeProcessFreeBSD::MonitorSIGTRAP(lldb::pid_t pid) { llvm::Error error = t.CopyWatchpointsFrom( static_cast<NativeThreadFreeBSD &>(*GetCurrentThread())); if (error) { - LLDB_LOG(log, "failed to copy watchpoints to new thread {0}: {1}", - info.pl_lwpid, llvm::toString(std::move(error))); + LLDB_LOG_ERROR(log, std::move(error), + "failed to copy watchpoints to new thread {1}: {0}", + info.pl_lwpid); SetState(StateType::eStateInvalid); return; } @@ -264,19 +265,35 @@ void NativeProcessFreeBSD::MonitorSIGTRAP(lldb::pid_t pid) { switch (info.pl_siginfo.si_code) { case TRAP_BRKPT: + LLDB_LOG(log, "SIGTRAP/TRAP_BRKPT: si_addr: {0}", + info.pl_siginfo.si_addr); + if (thread) { - thread->SetStoppedByBreakpoint(); + auto thread_info = + m_threads_stepping_with_breakpoint.find(thread->GetID()); + if (thread_info != m_threads_stepping_with_breakpoint.end()) { + thread->SetStoppedByTrace(); + Status brkpt_error = RemoveBreakpoint(thread_info->second); + if (brkpt_error.Fail()) + LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}", + thread_info->first, brkpt_error); + m_threads_stepping_with_breakpoint.erase(thread_info); + } else + thread->SetStoppedByBreakpoint(); FixupBreakpointPCAsNeeded(*thread); } SetState(StateType::eStateStopped, true); return; case TRAP_TRACE: + LLDB_LOG(log, "SIGTRAP/TRAP_TRACE: si_addr: {0}", + info.pl_siginfo.si_addr); + if (thread) { auto ®ctx = static_cast<NativeRegisterContextFreeBSD &>( thread->GetRegisterContext()); uint32_t wp_index = LLDB_INVALID_INDEX32; - Status error = - regctx.GetWatchpointHitIndex(wp_index, LLDB_INVALID_ADDRESS); + Status error = regctx.GetWatchpointHitIndex( + wp_index, reinterpret_cast<uintptr_t>(info.pl_siginfo.si_addr)); if (error.Fail()) LLDB_LOG(log, "received error while checking for watchpoint hits, pid = " @@ -354,6 +371,27 @@ Status NativeProcessFreeBSD::PtraceWrapper(int req, lldb::pid_t pid, void *addr, return error; } +llvm::Expected<llvm::ArrayRef<uint8_t>> +NativeProcessFreeBSD::GetSoftwareBreakpointTrapOpcode(size_t size_hint) { + static const uint8_t g_arm_opcode[] = {0xfe, 0xde, 0xff, 0xe7}; + static const uint8_t g_thumb_opcode[] = {0x01, 0xde}; + + switch (GetArchitecture().GetMachine()) { + case llvm::Triple::arm: + switch (size_hint) { + case 2: + return llvm::makeArrayRef(g_thumb_opcode); + case 4: + return llvm::makeArrayRef(g_arm_opcode); + default: + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Unrecognised trap opcode size hint!"); + } + default: + return NativeProcessProtocol::GetSoftwareBreakpointTrapOpcode(size_hint); + } +} + Status NativeProcessFreeBSD::Resume(const ResumeActionList &resume_actions) { Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); LLDB_LOG(log, "pid {0}", GetID()); @@ -623,9 +661,8 @@ size_t NativeProcessFreeBSD::UpdateThreads() { return m_threads.size(); } Status NativeProcessFreeBSD::SetBreakpoint(lldb::addr_t addr, uint32_t size, bool hardware) { if (hardware) - return Status("NativeProcessFreeBSD does not support hardware breakpoints"); - else - return SetSoftwareBreakpoint(addr, size); + return SetHardwareBreakpoint(addr, size); + return SetSoftwareBreakpoint(addr, size); } Status NativeProcessFreeBSD::GetLoadedModuleFileSpec(const char *module_path, @@ -878,3 +915,7 @@ Status NativeProcessFreeBSD::ReinitializeThreads() { return error; } + +bool NativeProcessFreeBSD::SupportHardwareSingleStepping() const { + return !m_arch.IsMIPS(); +} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.h index 3c7a9400f9c4..ceffc370ca33 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.h +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.h @@ -10,6 +10,8 @@ #define liblldb_NativeProcessFreeBSD_H_ #include "Plugins/Process/POSIX/NativeProcessELF.h" +#include "Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h" + #include "lldb/Target/MemoryRegionInfo.h" #include "lldb/Utility/ArchSpec.h" #include "lldb/Utility/FileSpec.h" @@ -25,7 +27,8 @@ namespace process_freebsd { /// for debugging. /// /// Changes in the inferior process state are broadcasted. -class NativeProcessFreeBSD : public NativeProcessELF { +class NativeProcessFreeBSD : public NativeProcessELF, + private NativeProcessSoftwareSingleStep { public: class Factory : public NativeProcessProtocol::Factory { public: @@ -84,6 +87,12 @@ public: static Status PtraceWrapper(int req, lldb::pid_t pid, void *addr = nullptr, int data = 0, int *result = nullptr); + bool SupportHardwareSingleStepping() const; + +protected: + llvm::Expected<llvm::ArrayRef<uint8_t>> + GetSoftwareBreakpointTrapOpcode(size_t size_hint) override; + private: MainLoop::SignalHandleUP m_sigchld_handle; ArchSpec m_arch; diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.cpp index ac3cc4fe788a..3d744f773a26 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD.cpp +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.cpp @@ -8,7 +8,7 @@ #include "NativeRegisterContextFreeBSD.h" -#include "Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.h" +#include "Plugins/Process/FreeBSD/NativeProcessFreeBSD.h" #include "lldb/Host/common/NativeProcessProtocol.h" diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h index 0000484beac9..0000484beac9 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD.h +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm.cpp new file mode 100644 index 000000000000..c4ee3773eaeb --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm.cpp @@ -0,0 +1,202 @@ +//===-- NativeRegisterContextFreeBSD_arm.cpp ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__arm__) + +#include "NativeRegisterContextFreeBSD_arm.h" + +#include "lldb/Utility/DataBufferHeap.h" +#include "lldb/Utility/RegisterValue.h" +#include "lldb/Utility/Status.h" + +#include "Plugins/Process/FreeBSD/NativeProcessFreeBSD.h" +#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm.h" + +// clang-format off +#include <sys/param.h> +#include <sys/ptrace.h> +#include <sys/types.h> +// clang-format on + +using namespace lldb; +using namespace lldb_private; +using namespace lldb_private::process_freebsd; + +NativeRegisterContextFreeBSD * +NativeRegisterContextFreeBSD::CreateHostNativeRegisterContextFreeBSD( + const ArchSpec &target_arch, NativeThreadProtocol &native_thread) { + return new NativeRegisterContextFreeBSD_arm(target_arch, native_thread); +} + +NativeRegisterContextFreeBSD_arm::NativeRegisterContextFreeBSD_arm( + const ArchSpec &target_arch, NativeThreadProtocol &native_thread) + : NativeRegisterContextRegisterInfo( + native_thread, new RegisterInfoPOSIX_arm(target_arch)) {} + +RegisterInfoPOSIX_arm & +NativeRegisterContextFreeBSD_arm::GetRegisterInfo() const { + return static_cast<RegisterInfoPOSIX_arm &>(*m_register_info_interface_up); +} + +uint32_t NativeRegisterContextFreeBSD_arm::GetRegisterSetCount() const { + return GetRegisterInfo().GetRegisterSetCount(); +} + +const RegisterSet * +NativeRegisterContextFreeBSD_arm::GetRegisterSet(uint32_t set_index) const { + return GetRegisterInfo().GetRegisterSet(set_index); +} + +uint32_t NativeRegisterContextFreeBSD_arm::GetUserRegisterCount() const { + uint32_t count = 0; + for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index) + count += GetRegisterSet(set_index)->num_registers; + return count; +} + +Status NativeRegisterContextFreeBSD_arm::ReadRegisterSet(uint32_t set) { + switch (set) { + case RegisterInfoPOSIX_arm::GPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_GETREGS, m_thread.GetID(), + m_reg_data.data()); + case RegisterInfoPOSIX_arm::FPRegSet: + return NativeProcessFreeBSD::PtraceWrapper( + PT_GETVFPREGS, m_thread.GetID(), + m_reg_data.data() + sizeof(RegisterInfoPOSIX_arm::GPR)); + } + llvm_unreachable("NativeRegisterContextFreeBSD_arm::ReadRegisterSet"); +} + +Status NativeRegisterContextFreeBSD_arm::WriteRegisterSet(uint32_t set) { + switch (set) { + case RegisterInfoPOSIX_arm::GPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_SETREGS, m_thread.GetID(), + m_reg_data.data()); + case RegisterInfoPOSIX_arm::FPRegSet: + return NativeProcessFreeBSD::PtraceWrapper( + PT_SETVFPREGS, m_thread.GetID(), + m_reg_data.data() + sizeof(RegisterInfoPOSIX_arm::GPR)); + } + llvm_unreachable("NativeRegisterContextFreeBSD_arm::WriteRegisterSet"); +} + +Status +NativeRegisterContextFreeBSD_arm::ReadRegister(const RegisterInfo *reg_info, + RegisterValue ®_value) { + Status error; + + if (!reg_info) { + error.SetErrorString("reg_info NULL"); + return error; + } + + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + + if (reg == LLDB_INVALID_REGNUM) + return Status("no lldb regnum for %s", reg_info && reg_info->name + ? reg_info->name + : "<unknown register>"); + + uint32_t set = GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg); + error = ReadRegisterSet(set); + if (error.Fail()) + return error; + + assert(reg_info->byte_offset + reg_info->byte_size <= m_reg_data.size()); + reg_value.SetBytes(m_reg_data.data() + reg_info->byte_offset, + reg_info->byte_size, endian::InlHostByteOrder()); + return error; +} + +Status NativeRegisterContextFreeBSD_arm::WriteRegister( + const RegisterInfo *reg_info, const RegisterValue ®_value) { + Status error; + + if (!reg_info) + return Status("reg_info NULL"); + + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + + if (reg == LLDB_INVALID_REGNUM) + return Status("no lldb regnum for %s", reg_info && reg_info->name + ? reg_info->name + : "<unknown register>"); + + uint32_t set = GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg); + error = ReadRegisterSet(set); + if (error.Fail()) + return error; + + assert(reg_info->byte_offset + reg_info->byte_size <= m_reg_data.size()); + ::memcpy(m_reg_data.data() + reg_info->byte_offset, reg_value.GetBytes(), + reg_info->byte_size); + + return WriteRegisterSet(set); +} + +Status NativeRegisterContextFreeBSD_arm::ReadAllRegisterValues( + lldb::DataBufferSP &data_sp) { + Status error; + + error = ReadRegisterSet(RegisterInfoPOSIX_arm::GPRegSet); + if (error.Fail()) + return error; + + error = ReadRegisterSet(RegisterInfoPOSIX_arm::FPRegSet); + if (error.Fail()) + return error; + + data_sp.reset(new DataBufferHeap(m_reg_data.size(), 0)); + uint8_t *dst = data_sp->GetBytes(); + ::memcpy(dst, m_reg_data.data(), m_reg_data.size()); + + return error; +} + +Status NativeRegisterContextFreeBSD_arm::WriteAllRegisterValues( + const lldb::DataBufferSP &data_sp) { + Status error; + + if (!data_sp) { + error.SetErrorStringWithFormat( + "NativeRegisterContextFreeBSD_arm::%s invalid data_sp provided", + __FUNCTION__); + return error; + } + + if (data_sp->GetByteSize() != m_reg_data.size()) { + error.SetErrorStringWithFormat( + "NativeRegisterContextFreeBSD_arm::%s data_sp contained mismatched " + "data size, expected %" PRIu64 ", actual %" PRIu64, + __FUNCTION__, m_reg_data.size(), data_sp->GetByteSize()); + return error; + } + + uint8_t *src = data_sp->GetBytes(); + if (src == nullptr) { + error.SetErrorStringWithFormat("NativeRegisterContextFreeBSD_arm::%s " + "DataBuffer::GetBytes() returned a null " + "pointer", + __FUNCTION__); + return error; + } + ::memcpy(m_reg_data.data(), src, m_reg_data.size()); + + error = WriteRegisterSet(RegisterInfoPOSIX_arm::GPRegSet); + if (error.Fail()) + return error; + + return WriteRegisterSet(RegisterInfoPOSIX_arm::FPRegSet); +} + +llvm::Error NativeRegisterContextFreeBSD_arm::CopyHardwareWatchpointsFrom( + NativeRegisterContextFreeBSD &source) { + return llvm::Error::success(); +} + +#endif // defined (__arm__) diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm.h new file mode 100644 index 000000000000..4be75b958fc1 --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm.h @@ -0,0 +1,68 @@ +//===-- NativeRegisterContextFreeBSD_arm.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__arm__) + +#ifndef lldb_NativeRegisterContextFreeBSD_arm_h +#define lldb_NativeRegisterContextFreeBSD_arm_h + +// clang-format off +#include <sys/types.h> +#include <machine/reg.h> +#include <machine/vfp.h> +// clang-format on + +#include "Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h" +#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm.h" + +#include <array> + +namespace lldb_private { +namespace process_freebsd { + +class NativeProcessFreeBSD; + +class NativeRegisterContextFreeBSD_arm : public NativeRegisterContextFreeBSD { +public: + NativeRegisterContextFreeBSD_arm(const ArchSpec &target_arch, + NativeThreadProtocol &native_thread); + + uint32_t GetRegisterSetCount() const override; + + uint32_t GetUserRegisterCount() const override; + + const RegisterSet *GetRegisterSet(uint32_t set_index) const override; + + Status ReadRegister(const RegisterInfo *reg_info, + RegisterValue ®_value) override; + + Status WriteRegister(const RegisterInfo *reg_info, + const RegisterValue ®_value) override; + + Status ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; + + Status WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; + + llvm::Error + CopyHardwareWatchpointsFrom(NativeRegisterContextFreeBSD &source) override; + +private: + std::array<uint8_t, sizeof(reg) + sizeof(vfp_state)> m_reg_data; + + Status ReadRegisterSet(uint32_t set); + Status WriteRegisterSet(uint32_t set); + + RegisterInfoPOSIX_arm &GetRegisterInfo() const; +}; + +} // namespace process_freebsd +} // namespace lldb_private + +#endif // #ifndef lldb_NativeRegisterContextFreeBSD_arm_h + +#endif // defined (__arm__) diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.cpp new file mode 100644 index 000000000000..e98e0a8a0caa --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.cpp @@ -0,0 +1,288 @@ +//===-- NativeRegisterContextFreeBSD_arm64.cpp ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__aarch64__) + +#include "NativeRegisterContextFreeBSD_arm64.h" + +#include "lldb/Utility/DataBufferHeap.h" +#include "lldb/Utility/RegisterValue.h" +#include "lldb/Utility/Status.h" + +#include "Plugins/Process/FreeBSD/NativeProcessFreeBSD.h" +#include "Plugins/Process/POSIX/ProcessPOSIXLog.h" +#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h" + +// clang-format off +#include <sys/param.h> +#include <sys/ptrace.h> +#include <sys/types.h> +// clang-format on + +using namespace lldb; +using namespace lldb_private; +using namespace lldb_private::process_freebsd; + +NativeRegisterContextFreeBSD * +NativeRegisterContextFreeBSD::CreateHostNativeRegisterContextFreeBSD( + const ArchSpec &target_arch, NativeThreadProtocol &native_thread) { + return new NativeRegisterContextFreeBSD_arm64(target_arch, native_thread); +} + +NativeRegisterContextFreeBSD_arm64::NativeRegisterContextFreeBSD_arm64( + const ArchSpec &target_arch, NativeThreadProtocol &native_thread) + : NativeRegisterContextRegisterInfo( + native_thread, new RegisterInfoPOSIX_arm64(target_arch)) +#ifdef LLDB_HAS_FREEBSD_WATCHPOINT + , + m_read_dbreg(false) +#endif +{ + GetRegisterInfo().ConfigureVectorRegisterInfos( + RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64); + ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs)); + ::memset(&m_hbp_regs, 0, sizeof(m_hbp_regs)); +} + +RegisterInfoPOSIX_arm64 & +NativeRegisterContextFreeBSD_arm64::GetRegisterInfo() const { + return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up); +} + +uint32_t NativeRegisterContextFreeBSD_arm64::GetRegisterSetCount() const { + return GetRegisterInfo().GetRegisterSetCount(); +} + +const RegisterSet * +NativeRegisterContextFreeBSD_arm64::GetRegisterSet(uint32_t set_index) const { + return GetRegisterInfo().GetRegisterSet(set_index); +} + +uint32_t NativeRegisterContextFreeBSD_arm64::GetUserRegisterCount() const { + uint32_t count = 0; + for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index) + count += GetRegisterSet(set_index)->num_registers; + return count; +} + +Status NativeRegisterContextFreeBSD_arm64::ReadRegisterSet(uint32_t set) { + switch (set) { + case RegisterInfoPOSIX_arm64::GPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_GETREGS, m_thread.GetID(), + m_reg_data.data()); + case RegisterInfoPOSIX_arm64::FPRegSet: + return NativeProcessFreeBSD::PtraceWrapper( + PT_GETFPREGS, m_thread.GetID(), + m_reg_data.data() + sizeof(RegisterInfoPOSIX_arm64::GPR)); + case RegisterInfoPOSIX_arm64::SVERegSet: + return Status("not supported"); + } + llvm_unreachable("NativeRegisterContextFreeBSD_arm64::ReadRegisterSet"); +} + +Status NativeRegisterContextFreeBSD_arm64::WriteRegisterSet(uint32_t set) { + switch (set) { + case RegisterInfoPOSIX_arm64::GPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_SETREGS, m_thread.GetID(), + m_reg_data.data()); + case RegisterInfoPOSIX_arm64::FPRegSet: + return NativeProcessFreeBSD::PtraceWrapper( + PT_SETFPREGS, m_thread.GetID(), + m_reg_data.data() + sizeof(RegisterInfoPOSIX_arm64::GPR)); + case RegisterInfoPOSIX_arm64::SVERegSet: + return Status("not supported"); + } + llvm_unreachable("NativeRegisterContextFreeBSD_arm64::WriteRegisterSet"); +} + +Status +NativeRegisterContextFreeBSD_arm64::ReadRegister(const RegisterInfo *reg_info, + RegisterValue ®_value) { + Status error; + + if (!reg_info) { + error.SetErrorString("reg_info NULL"); + return error; + } + + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + + if (reg == LLDB_INVALID_REGNUM) + return Status("no lldb regnum for %s", reg_info && reg_info->name + ? reg_info->name + : "<unknown register>"); + + uint32_t set = GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg); + error = ReadRegisterSet(set); + if (error.Fail()) + return error; + + assert(reg_info->byte_offset + reg_info->byte_size <= m_reg_data.size()); + reg_value.SetBytes(m_reg_data.data() + reg_info->byte_offset, + reg_info->byte_size, endian::InlHostByteOrder()); + return error; +} + +Status NativeRegisterContextFreeBSD_arm64::WriteRegister( + const RegisterInfo *reg_info, const RegisterValue ®_value) { + Status error; + + if (!reg_info) + return Status("reg_info NULL"); + + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + + if (reg == LLDB_INVALID_REGNUM) + return Status("no lldb regnum for %s", reg_info && reg_info->name + ? reg_info->name + : "<unknown register>"); + + uint32_t set = GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg); + error = ReadRegisterSet(set); + if (error.Fail()) + return error; + + assert(reg_info->byte_offset + reg_info->byte_size <= m_reg_data.size()); + ::memcpy(m_reg_data.data() + reg_info->byte_offset, reg_value.GetBytes(), + reg_info->byte_size); + + return WriteRegisterSet(set); +} + +Status NativeRegisterContextFreeBSD_arm64::ReadAllRegisterValues( + lldb::DataBufferSP &data_sp) { + Status error; + + error = ReadRegisterSet(RegisterInfoPOSIX_arm64::GPRegSet); + if (error.Fail()) + return error; + + error = ReadRegisterSet(RegisterInfoPOSIX_arm64::FPRegSet); + if (error.Fail()) + return error; + + data_sp.reset(new DataBufferHeap(m_reg_data.size(), 0)); + uint8_t *dst = data_sp->GetBytes(); + ::memcpy(dst, m_reg_data.data(), m_reg_data.size()); + + return error; +} + +Status NativeRegisterContextFreeBSD_arm64::WriteAllRegisterValues( + const lldb::DataBufferSP &data_sp) { + Status error; + + if (!data_sp) { + error.SetErrorStringWithFormat( + "NativeRegisterContextFreeBSD_arm64::%s invalid data_sp provided", + __FUNCTION__); + return error; + } + + if (data_sp->GetByteSize() != m_reg_data.size()) { + error.SetErrorStringWithFormat( + "NativeRegisterContextFreeBSD_arm64::%s data_sp contained mismatched " + "data size, expected %" PRIu64 ", actual %" PRIu64, + __FUNCTION__, m_reg_data.size(), data_sp->GetByteSize()); + return error; + } + + uint8_t *src = data_sp->GetBytes(); + if (src == nullptr) { + error.SetErrorStringWithFormat("NativeRegisterContextFreeBSD_arm64::%s " + "DataBuffer::GetBytes() returned a null " + "pointer", + __FUNCTION__); + return error; + } + ::memcpy(m_reg_data.data(), src, m_reg_data.size()); + + error = WriteRegisterSet(RegisterInfoPOSIX_arm64::GPRegSet); + if (error.Fail()) + return error; + + return WriteRegisterSet(RegisterInfoPOSIX_arm64::FPRegSet); +} + +llvm::Error NativeRegisterContextFreeBSD_arm64::CopyHardwareWatchpointsFrom( + NativeRegisterContextFreeBSD &source) { +#ifdef LLDB_HAS_FREEBSD_WATCHPOINT + auto &r_source = static_cast<NativeRegisterContextFreeBSD_arm64 &>(source); + llvm::Error error = r_source.ReadHardwareDebugInfo(); + if (error) + return error; + + m_dbreg = r_source.m_dbreg; + m_hbp_regs = r_source.m_hbp_regs; + m_hwp_regs = r_source.m_hwp_regs; + m_max_hbp_supported = r_source.m_max_hbp_supported; + m_max_hwp_supported = r_source.m_max_hwp_supported; + m_read_dbreg = true; + + // on FreeBSD this writes both breakpoints and watchpoints + return WriteHardwareDebugRegs(eDREGTypeWATCH); +#else + return llvm::Error::success(); +#endif +} + +llvm::Error NativeRegisterContextFreeBSD_arm64::ReadHardwareDebugInfo() { +#ifdef LLDB_HAS_FREEBSD_WATCHPOINT + Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_REGISTERS)); + + // we're fully stateful, so no need to reread control registers ever + if (m_read_dbreg) + return llvm::Error::success(); + + Status res = NativeProcessFreeBSD::PtraceWrapper(PT_GETDBREGS, + m_thread.GetID(), &m_dbreg); + if (res.Fail()) + return res.ToError(); + + LLDB_LOG(log, "m_dbreg read: debug_ver={0}, nbkpts={1}, nwtpts={2}", + m_dbreg.db_debug_ver, m_dbreg.db_nbkpts, m_dbreg.db_nwtpts); + m_max_hbp_supported = m_dbreg.db_nbkpts; + m_max_hwp_supported = m_dbreg.db_nwtpts; + assert(m_max_hbp_supported <= m_hbp_regs.size()); + assert(m_max_hwp_supported <= m_hwp_regs.size()); + + m_read_dbreg = true; + return llvm::Error::success(); +#else + return llvm::createStringError( + llvm::inconvertibleErrorCode(), + "Hardware breakpoints/watchpoints require FreeBSD 14.0"); +#endif +} + +llvm::Error +NativeRegisterContextFreeBSD_arm64::WriteHardwareDebugRegs(DREGType) { +#ifdef LLDB_HAS_FREEBSD_WATCHPOINT + assert(m_read_dbreg && "dbregs must be read before writing them back"); + + // copy data from m_*_regs to m_dbreg before writing it back + for (uint32_t i = 0; i < m_max_hbp_supported; i++) { + m_dbreg.db_breakregs[i].dbr_addr = m_hbp_regs[i].address; + m_dbreg.db_breakregs[i].dbr_ctrl = m_hbp_regs[i].control; + } + for (uint32_t i = 0; i < m_max_hwp_supported; i++) { + m_dbreg.db_watchregs[i].dbw_addr = m_hwp_regs[i].address; + m_dbreg.db_watchregs[i].dbw_ctrl = m_hwp_regs[i].control; + } + + return NativeProcessFreeBSD::PtraceWrapper(PT_SETDBREGS, m_thread.GetID(), + &m_dbreg) + .ToError(); +#else + return llvm::createStringError( + llvm::inconvertibleErrorCode(), + "Hardware breakpoints/watchpoints require FreeBSD 14.0"); +#endif +} + +#endif // defined (__aarch64__) diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.h new file mode 100644 index 000000000000..a230f8fed48a --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.h @@ -0,0 +1,86 @@ +//===-- NativeRegisterContextFreeBSD_arm64.h --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__aarch64__) + +#ifndef lldb_NativeRegisterContextFreeBSD_arm64_h +#define lldb_NativeRegisterContextFreeBSD_arm64_h + +// clang-format off +#include <sys/types.h> +#include <sys/param.h> +#include <machine/reg.h> +// clang-format on + +#include "Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h" +#include "Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.h" +#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h" + +#include <array> + +#if __FreeBSD_version >= 1300139 +# define LLDB_HAS_FREEBSD_WATCHPOINT 1 +#endif + +namespace lldb_private { +namespace process_freebsd { + +class NativeProcessFreeBSD; + +class NativeRegisterContextFreeBSD_arm64 + : public NativeRegisterContextFreeBSD, + public NativeRegisterContextDBReg_arm64 { +public: + NativeRegisterContextFreeBSD_arm64(const ArchSpec &target_arch, + NativeThreadProtocol &native_thread); + + uint32_t GetRegisterSetCount() const override; + + uint32_t GetUserRegisterCount() const override; + + const RegisterSet *GetRegisterSet(uint32_t set_index) const override; + + Status ReadRegister(const RegisterInfo *reg_info, + RegisterValue ®_value) override; + + Status WriteRegister(const RegisterInfo *reg_info, + const RegisterValue ®_value) override; + + Status ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; + + Status WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; + + llvm::Error + CopyHardwareWatchpointsFrom(NativeRegisterContextFreeBSD &source) override; + +private: + // Due to alignment, FreeBSD reg/fpreg are a few bytes larger than + // LLDB's GPR/FPU structs. However, all fields have matching offsets + // and sizes, so we do not have to worry about these (and we have + // a unittest to assert that). + std::array<uint8_t, sizeof(reg) + sizeof(fpreg)> m_reg_data; +#ifdef LLDB_HAS_FREEBSD_WATCHPOINT + dbreg m_dbreg; + bool m_read_dbreg; +#endif + + Status ReadRegisterSet(uint32_t set); + Status WriteRegisterSet(uint32_t set); + + llvm::Error ReadHardwareDebugInfo() override; + llvm::Error WriteHardwareDebugRegs(DREGType hwbType) override; + + RegisterInfoPOSIX_arm64 &GetRegisterInfo() const; +}; + +} // namespace process_freebsd +} // namespace lldb_private + +#endif // #ifndef lldb_NativeRegisterContextFreeBSD_arm64_h + +#endif // defined (__aarch64__) diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_mips64.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_mips64.cpp new file mode 100644 index 000000000000..8e722c09314c --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_mips64.cpp @@ -0,0 +1,186 @@ +//===-- NativeRegisterContextFreeBSD_mips64.cpp ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__mips64__) + +#include "NativeRegisterContextFreeBSD_mips64.h" + +#include "lldb/Utility/DataBufferHeap.h" +#include "lldb/Utility/RegisterValue.h" +#include "lldb/Utility/Status.h" + +#include "Plugins/Process/FreeBSD/NativeProcessFreeBSD.h" + +// clang-format off +#include <sys/param.h> +#include <sys/ptrace.h> +#include <sys/types.h> +// clang-format on + +using namespace lldb; +using namespace lldb_private; +using namespace lldb_private::process_freebsd; + +NativeRegisterContextFreeBSD * +NativeRegisterContextFreeBSD::CreateHostNativeRegisterContextFreeBSD( + const ArchSpec &target_arch, NativeThreadProtocol &native_thread) { + return new NativeRegisterContextFreeBSD_mips64(target_arch, native_thread); +} + +NativeRegisterContextFreeBSD_mips64::NativeRegisterContextFreeBSD_mips64( + const ArchSpec &target_arch, NativeThreadProtocol &native_thread) + : NativeRegisterContextRegisterInfo( + native_thread, new RegisterContextFreeBSD_mips64(target_arch)) {} + +RegisterContextFreeBSD_mips64 & +NativeRegisterContextFreeBSD_mips64::GetRegisterInfo() const { + return static_cast<RegisterContextFreeBSD_mips64 &>( + *m_register_info_interface_up); +} + +uint32_t NativeRegisterContextFreeBSD_mips64::GetRegisterSetCount() const { + return GetRegisterInfo().GetRegisterSetCount(); +} + +const RegisterSet * +NativeRegisterContextFreeBSD_mips64::GetRegisterSet(uint32_t set_index) const { + return GetRegisterInfo().GetRegisterSet(set_index); +} + +uint32_t NativeRegisterContextFreeBSD_mips64::GetUserRegisterCount() const { + uint32_t count = 0; + for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index) + count += GetRegisterSet(set_index)->num_registers; + return count; +} + +Status NativeRegisterContextFreeBSD_mips64::ReadRegisterSet(RegSetKind set) { + switch (set) { + case GPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_GETREGS, m_thread.GetID(), + m_reg_data.data()); + } + llvm_unreachable("NativeRegisterContextFreeBSD_mips64::ReadRegisterSet"); +} + +Status NativeRegisterContextFreeBSD_mips64::WriteRegisterSet(RegSetKind set) { + switch (set) { + case GPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_SETREGS, m_thread.GetID(), + m_reg_data.data()); + } + llvm_unreachable("NativeRegisterContextFreeBSD_mips64::WriteRegisterSet"); +} + +Status +NativeRegisterContextFreeBSD_mips64::ReadRegister(const RegisterInfo *reg_info, + RegisterValue ®_value) { + Status error; + + if (!reg_info) { + error.SetErrorString("reg_info NULL"); + return error; + } + + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + + if (reg == LLDB_INVALID_REGNUM) + return Status("no lldb regnum for %s", reg_info && reg_info->name + ? reg_info->name + : "<unknown register>"); + + RegSetKind set = GPRegSet; + error = ReadRegisterSet(set); + if (error.Fail()) + return error; + + assert(reg_info->byte_offset + reg_info->byte_size <= m_reg_data.size()); + reg_value.SetBytes(m_reg_data.data() + reg_info->byte_offset, + reg_info->byte_size, endian::InlHostByteOrder()); + return error; +} + +Status NativeRegisterContextFreeBSD_mips64::WriteRegister( + const RegisterInfo *reg_info, const RegisterValue ®_value) { + Status error; + + if (!reg_info) + return Status("reg_info NULL"); + + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + + if (reg == LLDB_INVALID_REGNUM) + return Status("no lldb regnum for %s", reg_info && reg_info->name + ? reg_info->name + : "<unknown register>"); + + RegSetKind set = GPRegSet; + error = ReadRegisterSet(set); + if (error.Fail()) + return error; + + assert(reg_info->byte_offset + reg_info->byte_size <= m_reg_data.size()); + ::memcpy(m_reg_data.data() + reg_info->byte_offset, reg_value.GetBytes(), + reg_info->byte_size); + + return WriteRegisterSet(set); +} + +Status NativeRegisterContextFreeBSD_mips64::ReadAllRegisterValues( + lldb::DataBufferSP &data_sp) { + Status error; + + error = ReadRegisterSet(GPRegSet); + if (error.Fail()) + return error; + + data_sp.reset(new DataBufferHeap(m_reg_data.size(), 0)); + uint8_t *dst = data_sp->GetBytes(); + ::memcpy(dst, m_reg_data.data(), m_reg_data.size()); + + return error; +} + +Status NativeRegisterContextFreeBSD_mips64::WriteAllRegisterValues( + const lldb::DataBufferSP &data_sp) { + Status error; + + if (!data_sp) { + error.SetErrorStringWithFormat( + "NativeRegisterContextFreeBSD_mips64::%s invalid data_sp provided", + __FUNCTION__); + return error; + } + + if (data_sp->GetByteSize() != m_reg_data.size()) { + error.SetErrorStringWithFormat( + "NativeRegisterContextFreeBSD_mips64::%s data_sp contained mismatched " + "data size, expected %" PRIu64 ", actual %" PRIu64, + __FUNCTION__, m_reg_data.size(), data_sp->GetByteSize()); + return error; + } + + uint8_t *src = data_sp->GetBytes(); + if (src == nullptr) { + error.SetErrorStringWithFormat("NativeRegisterContextFreeBSD_mips64::%s " + "DataBuffer::GetBytes() returned a null " + "pointer", + __FUNCTION__); + return error; + } + ::memcpy(m_reg_data.data(), src, m_reg_data.size()); + + return WriteRegisterSet(GPRegSet); +} + +llvm::Error NativeRegisterContextFreeBSD_mips64::CopyHardwareWatchpointsFrom( + NativeRegisterContextFreeBSD &source) { + return llvm::Error::success(); +} + +#endif // defined (__mips64__) diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_mips64.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_mips64.h new file mode 100644 index 000000000000..6a3eb86a9231 --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_mips64.h @@ -0,0 +1,71 @@ +//===-- NativeRegisterContextFreeBSD_mips64.h -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__mips64__) + +#ifndef lldb_NativeRegisterContextFreeBSD_mips64_h +#define lldb_NativeRegisterContextFreeBSD_mips64_h + +// clang-format off +#include <sys/types.h> +#include <machine/reg.h> +// clang-format on + +#include "Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h" +#include "Plugins/Process/Utility/RegisterContextFreeBSD_mips64.h" + +#include <array> + +namespace lldb_private { +namespace process_freebsd { + +class NativeProcessFreeBSD; + +class NativeRegisterContextFreeBSD_mips64 + : public NativeRegisterContextFreeBSD { +public: + NativeRegisterContextFreeBSD_mips64(const ArchSpec &target_arch, + NativeThreadProtocol &native_thread); + + uint32_t GetRegisterSetCount() const override; + + uint32_t GetUserRegisterCount() const override; + + const RegisterSet *GetRegisterSet(uint32_t set_index) const override; + + Status ReadRegister(const RegisterInfo *reg_info, + RegisterValue ®_value) override; + + Status WriteRegister(const RegisterInfo *reg_info, + const RegisterValue ®_value) override; + + Status ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; + + Status WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; + + llvm::Error + CopyHardwareWatchpointsFrom(NativeRegisterContextFreeBSD &source) override; + +private: + enum RegSetKind { + GPRegSet, + }; + std::array<uint8_t, sizeof(reg)> m_reg_data; + + Status ReadRegisterSet(RegSetKind set); + Status WriteRegisterSet(RegSetKind set); + + RegisterContextFreeBSD_mips64 &GetRegisterInfo() const; +}; + +} // namespace process_freebsd +} // namespace lldb_private + +#endif // #ifndef lldb_NativeRegisterContextFreeBSD_mips64_h + +#endif // defined (__mips64__) diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_powerpc.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_powerpc.cpp new file mode 100644 index 000000000000..5b5d44a308b1 --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_powerpc.cpp @@ -0,0 +1,289 @@ +//===-- NativeRegisterContextFreeBSD_powerpc.cpp --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__powerpc__) + +#include "NativeRegisterContextFreeBSD_powerpc.h" + +#include "lldb/Host/HostInfo.h" +#include "lldb/Utility/DataBufferHeap.h" +#include "lldb/Utility/RegisterValue.h" +#include "lldb/Utility/Status.h" + +#include "Plugins/Process/FreeBSD/NativeProcessFreeBSD.h" +// for register enum definitions +#include "Plugins/Process/Utility/RegisterContextPOSIX_powerpc.h" + +// clang-format off +#include <sys/param.h> +#include <sys/ptrace.h> +#include <sys/types.h> +// clang-format on + +using namespace lldb; +using namespace lldb_private; +using namespace lldb_private::process_freebsd; + +static const uint32_t g_gpr_regnums[] = { + gpr_r0_powerpc, gpr_r1_powerpc, gpr_r2_powerpc, gpr_r3_powerpc, + gpr_r4_powerpc, gpr_r5_powerpc, gpr_r6_powerpc, gpr_r7_powerpc, + gpr_r8_powerpc, gpr_r9_powerpc, gpr_r10_powerpc, gpr_r11_powerpc, + gpr_r12_powerpc, gpr_r13_powerpc, gpr_r14_powerpc, gpr_r15_powerpc, + gpr_r16_powerpc, gpr_r17_powerpc, gpr_r18_powerpc, gpr_r19_powerpc, + gpr_r20_powerpc, gpr_r21_powerpc, gpr_r22_powerpc, gpr_r23_powerpc, + gpr_r24_powerpc, gpr_r25_powerpc, gpr_r26_powerpc, gpr_r27_powerpc, + gpr_r28_powerpc, gpr_r29_powerpc, gpr_r30_powerpc, gpr_r31_powerpc, + gpr_lr_powerpc, gpr_cr_powerpc, gpr_xer_powerpc, gpr_ctr_powerpc, + gpr_pc_powerpc, +}; + +static const uint32_t g_fpr_regnums[] = { + fpr_f0_powerpc, fpr_f1_powerpc, fpr_f2_powerpc, fpr_f3_powerpc, + fpr_f4_powerpc, fpr_f5_powerpc, fpr_f6_powerpc, fpr_f7_powerpc, + fpr_f8_powerpc, fpr_f9_powerpc, fpr_f10_powerpc, fpr_f11_powerpc, + fpr_f12_powerpc, fpr_f13_powerpc, fpr_f14_powerpc, fpr_f15_powerpc, + fpr_f16_powerpc, fpr_f17_powerpc, fpr_f18_powerpc, fpr_f19_powerpc, + fpr_f20_powerpc, fpr_f21_powerpc, fpr_f22_powerpc, fpr_f23_powerpc, + fpr_f24_powerpc, fpr_f25_powerpc, fpr_f26_powerpc, fpr_f27_powerpc, + fpr_f28_powerpc, fpr_f29_powerpc, fpr_f30_powerpc, fpr_f31_powerpc, + fpr_fpscr_powerpc, +}; + +// Number of register sets provided by this context. +enum { k_num_register_sets = 2 }; + +static const RegisterSet g_reg_sets_powerpc[k_num_register_sets] = { + {"General Purpose Registers", "gpr", k_num_gpr_registers_powerpc, + g_gpr_regnums}, + {"Floating Point Registers", "fpr", k_num_fpr_registers_powerpc, + g_fpr_regnums}, +}; + +NativeRegisterContextFreeBSD * +NativeRegisterContextFreeBSD::CreateHostNativeRegisterContextFreeBSD( + const ArchSpec &target_arch, NativeThreadProtocol &native_thread) { + return new NativeRegisterContextFreeBSD_powerpc(target_arch, native_thread); +} + +static RegisterInfoInterface * +CreateRegisterInfoInterface(const ArchSpec &target_arch) { + if (HostInfo::GetArchitecture().GetAddressByteSize() == 4) { + return new RegisterContextFreeBSD_powerpc32(target_arch); + } else { + assert((HostInfo::GetArchitecture().GetAddressByteSize() == 8) && + "Register setting path assumes this is a 64-bit host"); + return new RegisterContextFreeBSD_powerpc64(target_arch); + } +} + +NativeRegisterContextFreeBSD_powerpc::NativeRegisterContextFreeBSD_powerpc( + const ArchSpec &target_arch, NativeThreadProtocol &native_thread) + : NativeRegisterContextRegisterInfo( + native_thread, CreateRegisterInfoInterface(target_arch)) {} + +RegisterContextFreeBSD_powerpc & +NativeRegisterContextFreeBSD_powerpc::GetRegisterInfo() const { + return static_cast<RegisterContextFreeBSD_powerpc &>( + *m_register_info_interface_up); +} + +uint32_t NativeRegisterContextFreeBSD_powerpc::GetRegisterSetCount() const { + return k_num_register_sets; +} + +const RegisterSet * +NativeRegisterContextFreeBSD_powerpc::GetRegisterSet(uint32_t set_index) const { + switch (GetRegisterInfoInterface().GetTargetArchitecture().GetMachine()) { + case llvm::Triple::ppc: + return &g_reg_sets_powerpc[set_index]; + default: + llvm_unreachable("Unhandled target architecture."); + } +} + +llvm::Optional<NativeRegisterContextFreeBSD_powerpc::RegSetKind> +NativeRegisterContextFreeBSD_powerpc::GetSetForNativeRegNum( + uint32_t reg_num) const { + switch (GetRegisterInfoInterface().GetTargetArchitecture().GetMachine()) { + case llvm::Triple::ppc: + if (reg_num >= k_first_gpr_powerpc && reg_num <= k_last_gpr_powerpc) + return GPRegSet; + if (reg_num >= k_first_fpr && reg_num <= k_last_fpr) + return FPRegSet; + break; + default: + llvm_unreachable("Unhandled target architecture."); + } + + llvm_unreachable("Register does not belong to any register set"); +} + +uint32_t NativeRegisterContextFreeBSD_powerpc::GetUserRegisterCount() const { + uint32_t count = 0; + for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index) + count += GetRegisterSet(set_index)->num_registers; + return count; +} + +Status NativeRegisterContextFreeBSD_powerpc::ReadRegisterSet(RegSetKind set) { + switch (set) { + case GPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_GETREGS, m_thread.GetID(), + m_reg_data.data()); + case FPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_GETFPREGS, m_thread.GetID(), + m_reg_data.data() + sizeof(reg)); + } + llvm_unreachable("NativeRegisterContextFreeBSD_powerpc::ReadRegisterSet"); +} + +Status NativeRegisterContextFreeBSD_powerpc::WriteRegisterSet(RegSetKind set) { + switch (set) { + case GPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_SETREGS, m_thread.GetID(), + m_reg_data.data()); + case FPRegSet: + return NativeProcessFreeBSD::PtraceWrapper(PT_SETFPREGS, m_thread.GetID(), + m_reg_data.data() + sizeof(reg)); + } + llvm_unreachable("NativeRegisterContextFreeBSD_powerpc::WriteRegisterSet"); +} + +Status +NativeRegisterContextFreeBSD_powerpc::ReadRegister(const RegisterInfo *reg_info, + RegisterValue ®_value) { + Status error; + + if (!reg_info) { + error.SetErrorString("reg_info NULL"); + return error; + } + + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + + if (reg == LLDB_INVALID_REGNUM) + return Status("no lldb regnum for %s", reg_info && reg_info->name + ? reg_info->name + : "<unknown register>"); + + llvm::Optional<RegSetKind> opt_set = GetSetForNativeRegNum(reg); + if (!opt_set) { + // This is likely an internal register for lldb use only and should not be + // directly queried. + error.SetErrorStringWithFormat("register \"%s\" is in unrecognized set", + reg_info->name); + return error; + } + + RegSetKind set = opt_set.getValue(); + error = ReadRegisterSet(set); + if (error.Fail()) + return error; + + assert(reg_info->byte_offset + reg_info->byte_size <= m_reg_data.size()); + reg_value.SetBytes(m_reg_data.data() + reg_info->byte_offset, + reg_info->byte_size, endian::InlHostByteOrder()); + return error; +} + +Status NativeRegisterContextFreeBSD_powerpc::WriteRegister( + const RegisterInfo *reg_info, const RegisterValue ®_value) { + Status error; + + if (!reg_info) + return Status("reg_info NULL"); + + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + + if (reg == LLDB_INVALID_REGNUM) + return Status("no lldb regnum for %s", reg_info && reg_info->name + ? reg_info->name + : "<unknown register>"); + + llvm::Optional<RegSetKind> opt_set = GetSetForNativeRegNum(reg); + if (!opt_set) { + // This is likely an internal register for lldb use only and should not be + // directly queried. + error.SetErrorStringWithFormat("register \"%s\" is in unrecognized set", + reg_info->name); + return error; + } + + RegSetKind set = opt_set.getValue(); + error = ReadRegisterSet(set); + if (error.Fail()) + return error; + + assert(reg_info->byte_offset + reg_info->byte_size <= m_reg_data.size()); + ::memcpy(m_reg_data.data() + reg_info->byte_offset, reg_value.GetBytes(), + reg_info->byte_size); + + return WriteRegisterSet(set); +} + +Status NativeRegisterContextFreeBSD_powerpc::ReadAllRegisterValues( + lldb::DataBufferSP &data_sp) { + Status error; + + error = ReadRegisterSet(GPRegSet); + if (error.Fail()) + return error; + + error = ReadRegisterSet(FPRegSet); + if (error.Fail()) + return error; + + data_sp.reset(new DataBufferHeap(m_reg_data.size(), 0)); + uint8_t *dst = data_sp->GetBytes(); + ::memcpy(dst, m_reg_data.data(), m_reg_data.size()); + + return error; +} + +Status NativeRegisterContextFreeBSD_powerpc::WriteAllRegisterValues( + const lldb::DataBufferSP &data_sp) { + Status error; + + if (!data_sp) { + error.SetErrorStringWithFormat( + "NativeRegisterContextFreeBSD_powerpc::%s invalid data_sp provided", + __FUNCTION__); + return error; + } + + if (data_sp->GetByteSize() != m_reg_data.size()) { + error.SetErrorStringWithFormat( + "NativeRegisterContextFreeBSD_powerpc::%s data_sp contained mismatched " + "data size, expected %zu, actual %" PRIu64, + __FUNCTION__, m_reg_data.size(), data_sp->GetByteSize()); + return error; + } + + uint8_t *src = data_sp->GetBytes(); + if (src == nullptr) { + error.SetErrorStringWithFormat("NativeRegisterContextFreeBSD_powerpc::%s " + "DataBuffer::GetBytes() returned a null " + "pointer", + __FUNCTION__); + return error; + } + ::memcpy(m_reg_data.data(), src, m_reg_data.size()); + + error = WriteRegisterSet(GPRegSet); + if (error.Fail()) + return error; + + return WriteRegisterSet(FPRegSet); +} + +llvm::Error NativeRegisterContextFreeBSD_powerpc::CopyHardwareWatchpointsFrom( + NativeRegisterContextFreeBSD &source) { + return llvm::Error::success(); +} + +#endif // defined (__powerpc__) diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_powerpc.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_powerpc.h new file mode 100644 index 000000000000..884c25988ce1 --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_powerpc.h @@ -0,0 +1,74 @@ +//===-- NativeRegisterContextFreeBSD_powerpc.h ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__powerpc__) + +#ifndef lldb_NativeRegisterContextFreeBSD_powerpc_h +#define lldb_NativeRegisterContextFreeBSD_powerpc_h + +// clang-format off +#include <sys/types.h> +#include <machine/reg.h> +// clang-format on + +#include "Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h" +#include "Plugins/Process/Utility/RegisterContextFreeBSD_powerpc.h" + +#include <array> + +namespace lldb_private { +namespace process_freebsd { + +class NativeProcessFreeBSD; + +class NativeRegisterContextFreeBSD_powerpc + : public NativeRegisterContextFreeBSD { +public: + NativeRegisterContextFreeBSD_powerpc(const ArchSpec &target_arch, + NativeThreadProtocol &native_thread); + + uint32_t GetRegisterSetCount() const override; + + uint32_t GetUserRegisterCount() const override; + + const RegisterSet *GetRegisterSet(uint32_t set_index) const override; + + Status ReadRegister(const RegisterInfo *reg_info, + RegisterValue ®_value) override; + + Status WriteRegister(const RegisterInfo *reg_info, + const RegisterValue ®_value) override; + + Status ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; + + Status WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; + + llvm::Error + CopyHardwareWatchpointsFrom(NativeRegisterContextFreeBSD &source) override; + +private: + enum RegSetKind { + GPRegSet, + FPRegSet, + }; + std::array<uint8_t, sizeof(reg) + sizeof(fpreg)> m_reg_data; + + llvm::Optional<RegSetKind> GetSetForNativeRegNum(uint32_t reg_num) const; + + Status ReadRegisterSet(RegSetKind set); + Status WriteRegisterSet(RegSetKind set); + + RegisterContextFreeBSD_powerpc &GetRegisterInfo() const; +}; + +} // namespace process_freebsd +} // namespace lldb_private + +#endif // #ifndef lldb_NativeRegisterContextFreeBSD_powerpc_h + +#endif // defined (__powerpc__) diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD_x86_64.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_x86_64.cpp index d5052e7d1b3a..d5052e7d1b3a 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD_x86_64.cpp +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_x86_64.cpp diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD_x86_64.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_x86_64.h index 673cffd6e849..efd0f91f77b9 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD_x86_64.h +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_x86_64.h @@ -20,9 +20,9 @@ #include <array> -#include "Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD.h" +#include "Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h" #include "Plugins/Process/Utility/RegisterContext_x86.h" -#include "Plugins/Process/Utility/NativeRegisterContextWatchpoint_x86.h" +#include "Plugins/Process/Utility/NativeRegisterContextDBReg_x86.h" #include "Plugins/Process/Utility/lldb-x86-register-enums.h" #define LLDB_INVALID_XSAVE_OFFSET UINT32_MAX @@ -34,7 +34,7 @@ class NativeProcessFreeBSD; class NativeRegisterContextFreeBSD_x86_64 : public NativeRegisterContextFreeBSD, - public NativeRegisterContextWatchpoint_x86 { + public NativeRegisterContextDBReg_x86 { public: NativeRegisterContextFreeBSD_x86_64(const ArchSpec &target_arch, NativeThreadProtocol &native_thread); diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeThreadFreeBSD.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeThreadFreeBSD.cpp index 43494871be07..63be12fc7b2b 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeThreadFreeBSD.cpp +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeThreadFreeBSD.cpp @@ -46,6 +46,11 @@ Status NativeThreadFreeBSD::Resume() { if (!ret.Success()) return ret; ret = NativeProcessFreeBSD::PtraceWrapper(PT_CLEARSTEP, GetID()); + // we can get EINVAL if the architecture in question does not support + // hardware single-stepping -- that's fine, we have nothing to clear + // then + if (ret.GetError() == EINVAL) + ret.Clear(); if (ret.Success()) SetRunning(); return ret; diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeThreadFreeBSD.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeThreadFreeBSD.h index 4e997b3fb4bb..249d2486b4f7 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSDRemote/NativeThreadFreeBSD.h +++ b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/NativeThreadFreeBSD.h @@ -11,7 +11,7 @@ #include "lldb/Host/common/NativeThreadProtocol.h" -#include "Plugins/Process/FreeBSDRemote/NativeRegisterContextFreeBSD.h" +#include "Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD.h" #include <csignal> #include <map> diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/POSIXStopInfo.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/POSIXStopInfo.cpp deleted file mode 100644 index 4e6f3afda0ab..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/POSIXStopInfo.cpp +++ /dev/null @@ -1,44 +0,0 @@ -//===-- POSIXStopInfo.cpp -------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "POSIXStopInfo.h" - -using namespace lldb; -using namespace lldb_private; - -//===----------------------------------------------------------------------===// -// POSIXLimboStopInfo - -POSIXLimboStopInfo::~POSIXLimboStopInfo() {} - -lldb::StopReason POSIXLimboStopInfo::GetStopReason() const { - return lldb::eStopReasonThreadExiting; -} - -const char *POSIXLimboStopInfo::GetDescription() { return "thread exiting"; } - -bool POSIXLimboStopInfo::ShouldStop(Event *event_ptr) { return false; } - -bool POSIXLimboStopInfo::ShouldNotify(Event *event_ptr) { return false; } - -//===----------------------------------------------------------------------===// -// POSIXNewThreadStopInfo - -POSIXNewThreadStopInfo::~POSIXNewThreadStopInfo() {} - -lldb::StopReason POSIXNewThreadStopInfo::GetStopReason() const { - return lldb::eStopReasonNone; -} - -const char *POSIXNewThreadStopInfo::GetDescription() { - return "thread spawned"; -} - -bool POSIXNewThreadStopInfo::ShouldStop(Event *event_ptr) { return false; } - -bool POSIXNewThreadStopInfo::ShouldNotify(Event *event_ptr) { return false; } diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/POSIXStopInfo.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/POSIXStopInfo.h deleted file mode 100644 index 5a022c485b68..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/POSIXStopInfo.h +++ /dev/null @@ -1,66 +0,0 @@ -//===-- POSIXStopInfo.h -----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_POSIXStopInfo_H_ -#define liblldb_POSIXStopInfo_H_ - -#include "FreeBSDThread.h" -#include "Plugins/Process/POSIX/CrashReason.h" -#include "lldb/Target/StopInfo.h" -#include <string> - -//===----------------------------------------------------------------------===// -/// \class POSIXStopInfo -/// Simple base class for all POSIX-specific StopInfo objects. -/// -class POSIXStopInfo : public lldb_private::StopInfo { -public: - POSIXStopInfo(lldb_private::Thread &thread, uint32_t status) - : StopInfo(thread, status) {} -}; - -//===----------------------------------------------------------------------===// -/// \class POSIXLimboStopInfo -/// Represents the stop state of a process ready to exit. -/// -class POSIXLimboStopInfo : public POSIXStopInfo { -public: - POSIXLimboStopInfo(FreeBSDThread &thread) : POSIXStopInfo(thread, 0) {} - - ~POSIXLimboStopInfo(); - - lldb::StopReason GetStopReason() const override; - - const char *GetDescription() override; - - bool ShouldStop(lldb_private::Event *event_ptr) override; - - bool ShouldNotify(lldb_private::Event *event_ptr) override; -}; - -//===----------------------------------------------------------------------===// -/// \class POSIXNewThreadStopInfo -/// Represents the stop state of process when a new thread is spawned. -/// - -class POSIXNewThreadStopInfo : public POSIXStopInfo { -public: - POSIXNewThreadStopInfo(FreeBSDThread &thread) : POSIXStopInfo(thread, 0) {} - - ~POSIXNewThreadStopInfo(); - - lldb::StopReason GetStopReason() const override; - - const char *GetDescription() override; - - bool ShouldStop(lldb_private::Event *event_ptr) override; - - bool ShouldNotify(lldb_private::Event *event_ptr) override; -}; - -#endif diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp deleted file mode 100644 index a1fe45b84ca2..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp +++ /dev/null @@ -1,1080 +0,0 @@ -//===-- ProcessFreeBSD.cpp ------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include <errno.h> -#include <pthread.h> -#include <pthread_np.h> -#include <stdlib.h> -#include <sys/sysctl.h> -#include <sys/types.h> -#include <sys/user.h> -#include <machine/elf.h> - -#include <mutex> -#include <unordered_map> - -#include "lldb/Core/PluginManager.h" -#include "lldb/Host/FileSystem.h" -#include "lldb/Host/Host.h" -#include "lldb/Symbol/ObjectFile.h" -#include "lldb/Target/DynamicLoader.h" -#include "lldb/Target/Target.h" -#include "lldb/Utility/RegisterValue.h" -#include "lldb/Utility/State.h" - -#include "FreeBSDThread.h" -#include "Plugins/Process/POSIX/ProcessPOSIXLog.h" -#include "Plugins/Process/Utility/FreeBSDSignals.h" -#include "Plugins/Process/Utility/InferiorCallPOSIX.h" -#include "ProcessFreeBSD.h" -#include "ProcessMonitor.h" - -#include "lldb/Breakpoint/BreakpointLocation.h" -#include "lldb/Breakpoint/Watchpoint.h" -#include "lldb/Core/Module.h" -#include "lldb/Core/ModuleSpec.h" -#include "lldb/Core/PluginManager.h" -#include "lldb/Host/Host.h" -#include "lldb/Symbol/ObjectFile.h" -#include "lldb/Target/DynamicLoader.h" -#include "lldb/Target/Platform.h" -#include "lldb/Target/Target.h" -#include "lldb/Utility/DataBufferHeap.h" -#include "lldb/Utility/FileSpec.h" -#include "lldb/Utility/State.h" - -#include "lldb/Host/posix/Fcntl.h" - -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Threading.h" - -using namespace lldb; -using namespace lldb_private; - -LLDB_PLUGIN_DEFINE(ProcessFreeBSD) - -namespace { -UnixSignalsSP &GetFreeBSDSignals() { - static UnixSignalsSP s_freebsd_signals_sp(new FreeBSDSignals()); - return s_freebsd_signals_sp; -} -} - -// Static functions. - -lldb::ProcessSP -ProcessFreeBSD::CreateInstance(lldb::TargetSP target_sp, - lldb::ListenerSP listener_sp, - const FileSpec *crash_file_path, - bool can_connect) { - lldb::ProcessSP process_sp; - if (crash_file_path == NULL && !can_connect) - process_sp.reset( - new ProcessFreeBSD(target_sp, listener_sp, GetFreeBSDSignals())); - return process_sp; -} - -void ProcessFreeBSD::Initialize() { - static llvm::once_flag g_once_flag; - - llvm::call_once(g_once_flag, []() { - PluginManager::RegisterPlugin(GetPluginNameStatic(), - GetPluginDescriptionStatic(), CreateInstance); - }); -} - -lldb_private::ConstString ProcessFreeBSD::GetPluginNameStatic() { - static ConstString g_name("freebsd"); - return g_name; -} - -const char *ProcessFreeBSD::GetPluginDescriptionStatic() { - return "Process plugin for FreeBSD"; -} - -// ProcessInterface protocol. - -lldb_private::ConstString ProcessFreeBSD::GetPluginName() { - return GetPluginNameStatic(); -} - -uint32_t ProcessFreeBSD::GetPluginVersion() { return 1; } - -void ProcessFreeBSD::Terminate() {} - -Status ProcessFreeBSD::DoDetach(bool keep_stopped) { - Status error; - if (keep_stopped) { - error.SetErrorString("Detaching with keep_stopped true is not currently " - "supported on FreeBSD."); - return error; - } - - error = m_monitor->Detach(GetID()); - - if (error.Success()) - SetPrivateState(eStateDetached); - - return error; -} - -Status ProcessFreeBSD::DoResume() { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - - SetPrivateState(eStateRunning); - - std::lock_guard<std::recursive_mutex> guard(m_thread_list.GetMutex()); - bool do_step = false; - bool software_single_step = !SupportHardwareSingleStepping(); - - for (tid_collection::const_iterator t_pos = m_run_tids.begin(), - t_end = m_run_tids.end(); - t_pos != t_end; ++t_pos) { - m_monitor->ThreadSuspend(*t_pos, false); - } - for (tid_collection::const_iterator t_pos = m_step_tids.begin(), - t_end = m_step_tids.end(); - t_pos != t_end; ++t_pos) { - m_monitor->ThreadSuspend(*t_pos, false); - do_step = true; - if (software_single_step) { - Status error = SetupSoftwareSingleStepping(*t_pos); - if (error.Fail()) - return error; - } - } - for (tid_collection::const_iterator t_pos = m_suspend_tids.begin(), - t_end = m_suspend_tids.end(); - t_pos != t_end; ++t_pos) { - m_monitor->ThreadSuspend(*t_pos, true); - // XXX Cannot PT_CONTINUE properly with suspended threads. - do_step = true; - } - - LLDB_LOGF(log, "process %" PRIu64 " resuming (%s)", GetID(), - do_step ? "step" : "continue"); - if (do_step && !software_single_step) - m_monitor->SingleStep(GetID(), m_resume_signo); - else - m_monitor->Resume(GetID(), m_resume_signo); - - return Status(); -} - -bool ProcessFreeBSD::DoUpdateThreadList(ThreadList &old_thread_list, - ThreadList &new_thread_list) { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - LLDB_LOGF(log, "ProcessFreeBSD::%s (pid = %" PRIu64 ")", __FUNCTION__, - GetID()); - - std::vector<lldb::pid_t> tds; - if (!GetMonitor().GetCurrentThreadIDs(tds)) { - return false; - } - - ThreadList old_thread_list_copy(old_thread_list); - for (size_t i = 0; i < tds.size(); ++i) { - tid_t tid = tds[i]; - ThreadSP thread_sp(old_thread_list_copy.RemoveThreadByID(tid, false)); - if (!thread_sp) { - thread_sp.reset(new FreeBSDThread(*this, tid)); - LLDB_LOGF(log, "ProcessFreeBSD::%s new tid = %" PRIu64, __FUNCTION__, - tid); - } else { - LLDB_LOGF(log, "ProcessFreeBSD::%s existing tid = %" PRIu64, __FUNCTION__, - tid); - } - new_thread_list.AddThread(thread_sp); - } - for (size_t i = 0; i < old_thread_list_copy.GetSize(false); ++i) { - ThreadSP old_thread_sp(old_thread_list_copy.GetThreadAtIndex(i, false)); - if (old_thread_sp) { - LLDB_LOGF(log, "ProcessFreeBSD::%s remove tid", __FUNCTION__); - } - } - - return true; -} - -Status ProcessFreeBSD::WillResume() { - m_resume_signo = 0; - m_suspend_tids.clear(); - m_run_tids.clear(); - m_step_tids.clear(); - return Process::WillResume(); -} - -void ProcessFreeBSD::SendMessage(const ProcessMessage &message) { - std::lock_guard<std::recursive_mutex> guard(m_message_mutex); - - switch (message.GetKind()) { - case ProcessMessage::eInvalidMessage: - return; - - case ProcessMessage::eAttachMessage: - SetPrivateState(eStateStopped); - return; - - case ProcessMessage::eLimboMessage: - case ProcessMessage::eExitMessage: - SetExitStatus(message.GetExitStatus(), NULL); - break; - - case ProcessMessage::eSignalMessage: - case ProcessMessage::eSignalDeliveredMessage: - case ProcessMessage::eBreakpointMessage: - case ProcessMessage::eTraceMessage: - case ProcessMessage::eWatchpointMessage: - case ProcessMessage::eCrashMessage: - SetPrivateState(eStateStopped); - break; - - case ProcessMessage::eNewThreadMessage: - llvm_unreachable("eNewThreadMessage unexpected on FreeBSD"); - break; - - case ProcessMessage::eExecMessage: - SetPrivateState(eStateStopped); - break; - } - - m_message_queue.push(message); -} - -// Constructors and destructors. - -ProcessFreeBSD::ProcessFreeBSD(lldb::TargetSP target_sp, - lldb::ListenerSP listener_sp, - UnixSignalsSP &unix_signals_sp) - : Process(target_sp, listener_sp, unix_signals_sp), - m_byte_order(endian::InlHostByteOrder()), m_monitor(NULL), m_module(NULL), - m_message_mutex(), m_exit_now(false), m_seen_initial_stop(), - m_resume_signo(0) { - // FIXME: Putting this code in the ctor and saving the byte order in a - // member variable is a hack to avoid const qual issues in GetByteOrder. - lldb::ModuleSP module = GetTarget().GetExecutableModule(); - if (module && module->GetObjectFile()) - m_byte_order = module->GetObjectFile()->GetByteOrder(); -} - -ProcessFreeBSD::~ProcessFreeBSD() { delete m_monitor; } - -// Process protocol. -void ProcessFreeBSD::Finalize() { - Process::Finalize(); - - if (m_monitor) - m_monitor->StopMonitor(); -} - -bool ProcessFreeBSD::CanDebug(lldb::TargetSP target_sp, - bool plugin_specified_by_name) { - // For now we are just making sure the file exists for a given module - ModuleSP exe_module_sp(target_sp->GetExecutableModule()); - if (exe_module_sp.get()) - return FileSystem::Instance().Exists(exe_module_sp->GetFileSpec()); - // If there is no executable module, we return true since we might be - // preparing to attach. - return true; -} - -Status -ProcessFreeBSD::DoAttachToProcessWithID(lldb::pid_t pid, - const ProcessAttachInfo &attach_info) { - Status error; - assert(m_monitor == NULL); - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - LLDB_LOGV(log, "pid = {0}", GetID()); - - m_monitor = new ProcessMonitor(this, pid, error); - - if (!error.Success()) - return error; - - PlatformSP platform_sp(GetTarget().GetPlatform()); - assert(platform_sp.get()); - if (!platform_sp) - return error; // FIXME: Detatch? - - // Find out what we can about this process - ProcessInstanceInfo process_info; - platform_sp->GetProcessInfo(pid, process_info); - - // Resolve the executable module - ModuleSP exe_module_sp; - FileSpecList executable_search_paths( - Target::GetDefaultExecutableSearchPaths()); - ModuleSpec exe_module_spec(process_info.GetExecutableFile(), - GetTarget().GetArchitecture()); - error = platform_sp->ResolveExecutable( - exe_module_spec, exe_module_sp, - executable_search_paths.GetSize() ? &executable_search_paths : NULL); - if (!error.Success()) - return error; - - // Fix the target architecture if necessary - const ArchSpec &module_arch = exe_module_sp->GetArchitecture(); - if (module_arch.IsValid() && - !GetTarget().GetArchitecture().IsExactMatch(module_arch)) - GetTarget().SetArchitecture(module_arch); - - // Initialize the target module list - GetTarget().SetExecutableModule(exe_module_sp, eLoadDependentsYes); - - SetSTDIOFileDescriptor(m_monitor->GetTerminalFD()); - - SetID(pid); - - return error; -} - -Status ProcessFreeBSD::WillLaunch(Module *module) { - Status error; - return error; -} - -FileSpec -ProcessFreeBSD::GetFileSpec(const lldb_private::FileAction *file_action, - const FileSpec &default_file_spec, - const FileSpec &dbg_pts_file_spec) { - FileSpec file_spec{}; - - if (file_action && file_action->GetAction() == FileAction::eFileActionOpen) { - file_spec = file_action->GetFileSpec(); - // By default the stdio paths passed in will be pseudo-terminal (/dev/pts). - // If so, convert to using a different default path instead to redirect I/O - // to the debugger console. This should also handle user overrides to - // /dev/null or a different file. - if (!file_spec || file_spec == dbg_pts_file_spec) - file_spec = default_file_spec; - } - return file_spec; -} - -Status ProcessFreeBSD::DoLaunch(Module *module, - ProcessLaunchInfo &launch_info) { - Status error; - assert(m_monitor == NULL); - - FileSpec working_dir = launch_info.GetWorkingDirectory(); - if (working_dir) { - FileSystem::Instance().Resolve(working_dir); - if (!FileSystem::Instance().IsDirectory(working_dir.GetPath())) { - error.SetErrorStringWithFormat("No such file or directory: %s", - working_dir.GetCString()); - return error; - } - } - - SetPrivateState(eStateLaunching); - - const lldb_private::FileAction *file_action; - - // Default of empty will mean to use existing open file descriptors - FileSpec stdin_file_spec{}; - FileSpec stdout_file_spec{}; - FileSpec stderr_file_spec{}; - - const FileSpec dbg_pts_file_spec{launch_info.GetPTY().GetSecondaryName()}; - - file_action = launch_info.GetFileActionForFD(STDIN_FILENO); - stdin_file_spec = - GetFileSpec(file_action, stdin_file_spec, dbg_pts_file_spec); - - file_action = launch_info.GetFileActionForFD(STDOUT_FILENO); - stdout_file_spec = - GetFileSpec(file_action, stdout_file_spec, dbg_pts_file_spec); - - file_action = launch_info.GetFileActionForFD(STDERR_FILENO); - stderr_file_spec = - GetFileSpec(file_action, stderr_file_spec, dbg_pts_file_spec); - - m_monitor = new ProcessMonitor( - this, module, launch_info.GetArguments().GetConstArgumentVector(), - launch_info.GetEnvironment(), stdin_file_spec, stdout_file_spec, - stderr_file_spec, working_dir, launch_info, error); - - m_module = module; - - if (!error.Success()) - return error; - - int terminal = m_monitor->GetTerminalFD(); - if (terminal >= 0) { -// The reader thread will close the file descriptor when done, so we pass it a -// copy. -#ifdef F_DUPFD_CLOEXEC - int stdio = fcntl(terminal, F_DUPFD_CLOEXEC, 0); - if (stdio == -1) { - error.SetErrorToErrno(); - return error; - } -#else - // Special case when F_DUPFD_CLOEXEC does not exist (Debian kFreeBSD) - int stdio = fcntl(terminal, F_DUPFD, 0); - if (stdio == -1) { - error.SetErrorToErrno(); - return error; - } - stdio = fcntl(terminal, F_SETFD, FD_CLOEXEC); - if (stdio == -1) { - error.SetErrorToErrno(); - return error; - } -#endif - SetSTDIOFileDescriptor(stdio); - } - - SetID(m_monitor->GetPID()); - return error; -} - -void ProcessFreeBSD::DidLaunch() {} - -addr_t ProcessFreeBSD::GetImageInfoAddress() { - Target *target = &GetTarget(); - ObjectFile *obj_file = target->GetExecutableModule()->GetObjectFile(); - Address addr = obj_file->GetImageInfoAddress(target); - - if (addr.IsValid()) - return addr.GetLoadAddress(target); - return LLDB_INVALID_ADDRESS; -} - -Status ProcessFreeBSD::DoHalt(bool &caused_stop) { - Status error; - - if (IsStopped()) { - caused_stop = false; - } else if (kill(GetID(), SIGSTOP)) { - caused_stop = false; - error.SetErrorToErrno(); - } else { - caused_stop = true; - } - return error; -} - -Status ProcessFreeBSD::DoSignal(int signal) { - Status error; - - if (kill(GetID(), signal)) - error.SetErrorToErrno(); - - return error; -} - -Status ProcessFreeBSD::DoDestroy() { - Status error; - - if (!HasExited()) { - assert(m_monitor); - m_exit_now = true; - if (GetID() == LLDB_INVALID_PROCESS_ID) { - error.SetErrorString("invalid process id"); - return error; - } - if (!m_monitor->Kill()) { - error.SetErrorToErrno(); - return error; - } - - SetPrivateState(eStateExited); - } - - return error; -} - -void ProcessFreeBSD::DoDidExec() { - Target *target = &GetTarget(); - if (target) { - PlatformSP platform_sp(target->GetPlatform()); - assert(platform_sp.get()); - if (platform_sp) { - ProcessInstanceInfo process_info; - platform_sp->GetProcessInfo(GetID(), process_info); - ModuleSP exe_module_sp; - ModuleSpec exe_module_spec(process_info.GetExecutableFile(), - target->GetArchitecture()); - FileSpecList executable_search_paths( - Target::GetDefaultExecutableSearchPaths()); - Status error = platform_sp->ResolveExecutable( - exe_module_spec, exe_module_sp, - executable_search_paths.GetSize() ? &executable_search_paths : NULL); - if (!error.Success()) - return; - target->SetExecutableModule(exe_module_sp, eLoadDependentsYes); - } - } -} - -bool ProcessFreeBSD::AddThreadForInitialStopIfNeeded(lldb::tid_t stop_tid) { - bool added_to_set = false; - ThreadStopSet::iterator it = m_seen_initial_stop.find(stop_tid); - if (it == m_seen_initial_stop.end()) { - m_seen_initial_stop.insert(stop_tid); - added_to_set = true; - } - return added_to_set; -} - -bool ProcessFreeBSD::WaitingForInitialStop(lldb::tid_t stop_tid) { - return (m_seen_initial_stop.find(stop_tid) == m_seen_initial_stop.end()); -} - -FreeBSDThread * -ProcessFreeBSD::CreateNewFreeBSDThread(lldb_private::Process &process, - lldb::tid_t tid) { - return new FreeBSDThread(process, tid); -} - -void ProcessFreeBSD::RefreshStateAfterStop() { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - LLDB_LOGV(log, "message_queue size = {0}", m_message_queue.size()); - - std::lock_guard<std::recursive_mutex> guard(m_message_mutex); - - // This method used to only handle one message. Changing it to loop allows - // it to handle the case where we hit a breakpoint while handling a different - // breakpoint. - while (!m_message_queue.empty()) { - ProcessMessage &message = m_message_queue.front(); - - // Resolve the thread this message corresponds to and pass it along. - lldb::tid_t tid = message.GetTID(); - LLDB_LOGV(log, " message_queue size = {0}, pid = {1}", - m_message_queue.size(), tid); - - m_thread_list.RefreshStateAfterStop(); - - FreeBSDThread *thread = static_cast<FreeBSDThread *>( - GetThreadList().FindThreadByID(tid, false).get()); - if (thread) - thread->Notify(message); - - if (message.GetKind() == ProcessMessage::eExitMessage) { - // FIXME: We should tell the user about this, but the limbo message is - // probably better for that. - LLDB_LOG(log, "removing thread, tid = {0}", tid); - std::lock_guard<std::recursive_mutex> guard(m_thread_list.GetMutex()); - - ThreadSP thread_sp = m_thread_list.RemoveThreadByID(tid, false); - thread_sp.reset(); - m_seen_initial_stop.erase(tid); - } - - m_message_queue.pop(); - } -} - -bool ProcessFreeBSD::IsAlive() { - StateType state = GetPrivateState(); - return state != eStateDetached && state != eStateExited && - state != eStateInvalid && state != eStateUnloaded; -} - -size_t ProcessFreeBSD::DoReadMemory(addr_t vm_addr, void *buf, size_t size, - Status &error) { - assert(m_monitor); - return m_monitor->ReadMemory(vm_addr, buf, size, error); -} - -size_t ProcessFreeBSD::DoWriteMemory(addr_t vm_addr, const void *buf, - size_t size, Status &error) { - assert(m_monitor); - return m_monitor->WriteMemory(vm_addr, buf, size, error); -} - -addr_t ProcessFreeBSD::DoAllocateMemory(size_t size, uint32_t permissions, - Status &error) { - addr_t allocated_addr = LLDB_INVALID_ADDRESS; - - unsigned prot = 0; - if (permissions & lldb::ePermissionsReadable) - prot |= eMmapProtRead; - if (permissions & lldb::ePermissionsWritable) - prot |= eMmapProtWrite; - if (permissions & lldb::ePermissionsExecutable) - prot |= eMmapProtExec; - - if (InferiorCallMmap(this, allocated_addr, 0, size, prot, - eMmapFlagsAnon | eMmapFlagsPrivate, -1, 0)) { - m_addr_to_mmap_size[allocated_addr] = size; - error.Clear(); - } else { - allocated_addr = LLDB_INVALID_ADDRESS; - error.SetErrorStringWithFormat( - "unable to allocate %zu bytes of memory with permissions %s", size, - GetPermissionsAsCString(permissions)); - } - - return allocated_addr; -} - -Status ProcessFreeBSD::DoDeallocateMemory(lldb::addr_t addr) { - Status error; - MMapMap::iterator pos = m_addr_to_mmap_size.find(addr); - if (pos != m_addr_to_mmap_size.end() && - InferiorCallMunmap(this, addr, pos->second)) - m_addr_to_mmap_size.erase(pos); - else - error.SetErrorStringWithFormat("unable to deallocate memory at 0x%" PRIx64, - addr); - - return error; -} - -size_t -ProcessFreeBSD::GetSoftwareBreakpointTrapOpcode(BreakpointSite *bp_site) { - static const uint8_t g_aarch64_opcode[] = {0x00, 0x00, 0x20, 0xD4}; - static const uint8_t g_i386_opcode[] = {0xCC}; - - ArchSpec arch = GetTarget().GetArchitecture(); - const uint8_t *opcode = NULL; - size_t opcode_size = 0; - - switch (arch.GetMachine()) { - default: - assert(false && "CPU type not supported!"); - break; - - case llvm::Triple::arm: { - // The ARM reference recommends the use of 0xe7fddefe and 0xdefe but the - // linux kernel does otherwise. - static const uint8_t g_arm_breakpoint_opcode[] = {0xf0, 0x01, 0xf0, 0xe7}; - static const uint8_t g_thumb_breakpoint_opcode[] = {0x01, 0xde}; - - lldb::BreakpointLocationSP bp_loc_sp(bp_site->GetOwnerAtIndex(0)); - AddressClass addr_class = AddressClass::eUnknown; - - if (bp_loc_sp) - addr_class = bp_loc_sp->GetAddress().GetAddressClass(); - - if (addr_class == AddressClass::eCodeAlternateISA || - (addr_class == AddressClass::eUnknown && - bp_loc_sp->GetAddress().GetOffset() & 1)) { - opcode = g_thumb_breakpoint_opcode; - opcode_size = sizeof(g_thumb_breakpoint_opcode); - } else { - opcode = g_arm_breakpoint_opcode; - opcode_size = sizeof(g_arm_breakpoint_opcode); - } - } break; - case llvm::Triple::aarch64: - opcode = g_aarch64_opcode; - opcode_size = sizeof(g_aarch64_opcode); - break; - - case llvm::Triple::x86: - case llvm::Triple::x86_64: - opcode = g_i386_opcode; - opcode_size = sizeof(g_i386_opcode); - break; - } - - bp_site->SetTrapOpcode(opcode, opcode_size); - return opcode_size; -} - -Status ProcessFreeBSD::EnableBreakpointSite(BreakpointSite *bp_site) { - if (bp_site->HardwareRequired()) - return Status("Hardware breakpoints are not supported."); - - return EnableSoftwareBreakpoint(bp_site); -} - -Status ProcessFreeBSD::DisableBreakpointSite(BreakpointSite *bp_site) { - return DisableSoftwareBreakpoint(bp_site); -} - -Status ProcessFreeBSD::EnableWatchpoint(Watchpoint *wp, bool notify) { - Status error; - if (wp) { - user_id_t watchID = wp->GetID(); - addr_t addr = wp->GetLoadAddress(); - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_WATCHPOINTS)); - LLDB_LOGF(log, "ProcessFreeBSD::EnableWatchpoint(watchID = %" PRIu64 ")", - watchID); - if (wp->IsEnabled()) { - LLDB_LOGF(log, - "ProcessFreeBSD::EnableWatchpoint(watchID = %" PRIu64 - ") addr = 0x%8.8" PRIx64 ": watchpoint already enabled.", - watchID, (uint64_t)addr); - return error; - } - - // Try to find a vacant watchpoint slot in the inferiors' main thread - uint32_t wp_hw_index = LLDB_INVALID_INDEX32; - std::lock_guard<std::recursive_mutex> guard(m_thread_list.GetMutex()); - FreeBSDThread *thread = static_cast<FreeBSDThread *>( - m_thread_list.GetThreadAtIndex(0, false).get()); - - if (thread) - wp_hw_index = thread->FindVacantWatchpointIndex(); - - if (wp_hw_index == LLDB_INVALID_INDEX32) { - error.SetErrorString("Setting hardware watchpoint failed."); - } else { - wp->SetHardwareIndex(wp_hw_index); - bool wp_enabled = true; - uint32_t thread_count = m_thread_list.GetSize(false); - for (uint32_t i = 0; i < thread_count; ++i) { - thread = static_cast<FreeBSDThread *>( - m_thread_list.GetThreadAtIndex(i, false).get()); - if (thread) - wp_enabled &= thread->EnableHardwareWatchpoint(wp); - else - wp_enabled = false; - } - if (wp_enabled) { - wp->SetEnabled(true, notify); - return error; - } else { - // Watchpoint enabling failed on at least one of the threads so roll - // back all of them - DisableWatchpoint(wp, false); - error.SetErrorString("Setting hardware watchpoint failed"); - } - } - } else - error.SetErrorString("Watchpoint argument was NULL."); - return error; -} - -Status ProcessFreeBSD::DisableWatchpoint(Watchpoint *wp, bool notify) { - Status error; - if (wp) { - user_id_t watchID = wp->GetID(); - addr_t addr = wp->GetLoadAddress(); - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_WATCHPOINTS)); - LLDB_LOGF(log, "ProcessFreeBSD::DisableWatchpoint(watchID = %" PRIu64 ")", - watchID); - if (!wp->IsEnabled()) { - LLDB_LOGF(log, - "ProcessFreeBSD::DisableWatchpoint(watchID = %" PRIu64 - ") addr = 0x%8.8" PRIx64 ": watchpoint already disabled.", - watchID, (uint64_t)addr); - // This is needed (for now) to keep watchpoints disabled correctly - wp->SetEnabled(false, notify); - return error; - } - - if (wp->IsHardware()) { - bool wp_disabled = true; - std::lock_guard<std::recursive_mutex> guard(m_thread_list.GetMutex()); - uint32_t thread_count = m_thread_list.GetSize(false); - for (uint32_t i = 0; i < thread_count; ++i) { - FreeBSDThread *thread = static_cast<FreeBSDThread *>( - m_thread_list.GetThreadAtIndex(i, false).get()); - if (thread) - wp_disabled &= thread->DisableHardwareWatchpoint(wp); - else - wp_disabled = false; - } - if (wp_disabled) { - wp->SetHardwareIndex(LLDB_INVALID_INDEX32); - wp->SetEnabled(false, notify); - return error; - } else - error.SetErrorString("Disabling hardware watchpoint failed"); - } - } else - error.SetErrorString("Watchpoint argument was NULL."); - return error; -} - -Status ProcessFreeBSD::GetWatchpointSupportInfo(uint32_t &num) { - Status error; - std::lock_guard<std::recursive_mutex> guard(m_thread_list.GetMutex()); - FreeBSDThread *thread = static_cast<FreeBSDThread *>( - m_thread_list.GetThreadAtIndex(0, false).get()); - if (thread) - num = thread->NumSupportedHardwareWatchpoints(); - else - error.SetErrorString("Process does not exist."); - return error; -} - -Status ProcessFreeBSD::GetWatchpointSupportInfo(uint32_t &num, bool &after) { - Status error = GetWatchpointSupportInfo(num); - // Watchpoints trigger and halt the inferior after the corresponding - // instruction has been executed. - after = true; - return error; -} - -uint32_t ProcessFreeBSD::UpdateThreadListIfNeeded() { - std::lock_guard<std::recursive_mutex> guard(m_thread_list.GetMutex()); - // Do not allow recursive updates. - return m_thread_list.GetSize(false); -} - -ByteOrder ProcessFreeBSD::GetByteOrder() const { - // FIXME: We should be able to extract this value directly. See comment in - // ProcessFreeBSD(). - return m_byte_order; -} - -size_t ProcessFreeBSD::PutSTDIN(const char *buf, size_t len, Status &error) { - ssize_t status; - if ((status = write(m_monitor->GetTerminalFD(), buf, len)) < 0) { - error.SetErrorToErrno(); - return 0; - } - return status; -} - -// Utility functions. - -bool ProcessFreeBSD::HasExited() { - switch (GetPrivateState()) { - default: - break; - - case eStateDetached: - case eStateExited: - return true; - } - - return false; -} - -bool ProcessFreeBSD::IsStopped() { - switch (GetPrivateState()) { - default: - break; - - case eStateStopped: - case eStateCrashed: - case eStateSuspended: - return true; - } - - return false; -} - -bool ProcessFreeBSD::IsAThreadRunning() { - bool is_running = false; - std::lock_guard<std::recursive_mutex> guard(m_thread_list.GetMutex()); - uint32_t thread_count = m_thread_list.GetSize(false); - for (uint32_t i = 0; i < thread_count; ++i) { - FreeBSDThread *thread = static_cast<FreeBSDThread *>( - m_thread_list.GetThreadAtIndex(i, false).get()); - StateType thread_state = thread->GetState(); - if (thread_state == eStateRunning || thread_state == eStateStepping) { - is_running = true; - break; - } - } - return is_running; -} - -lldb_private::DataExtractor ProcessFreeBSD::GetAuxvData() { - // If we're the local platform, we can ask the host for auxv data. - PlatformSP platform_sp = GetTarget().GetPlatform(); - assert(platform_sp && platform_sp->IsHost()); - - int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_AUXV, (int)m_process->GetID()}; - size_t auxv_size = AT_COUNT * sizeof(Elf_Auxinfo); - DataBufferSP buf_sp(new DataBufferHeap(auxv_size, 0)); - - if (::sysctl(mib, 4, buf_sp->GetBytes(), &auxv_size, NULL, 0) != 0) { - perror("sysctl failed on auxv"); - buf_sp.reset(); - } - - return DataExtractor(buf_sp, GetByteOrder(), GetAddressByteSize()); -} - -struct EmulatorBaton { - ProcessFreeBSD *m_process; - RegisterContext *m_reg_context; - - // eRegisterKindDWARF -> RegisterValue - std::unordered_map<uint32_t, RegisterValue> m_register_values; - - EmulatorBaton(ProcessFreeBSD *process, RegisterContext *reg_context) - : m_process(process), m_reg_context(reg_context) {} -}; - -static size_t ReadMemoryCallback(EmulateInstruction *instruction, void *baton, - const EmulateInstruction::Context &context, - lldb::addr_t addr, void *dst, size_t length) { - EmulatorBaton *emulator_baton = static_cast<EmulatorBaton *>(baton); - - Status error; - size_t bytes_read = - emulator_baton->m_process->DoReadMemory(addr, dst, length, error); - if (!error.Success()) - bytes_read = 0; - return bytes_read; -} - -static bool ReadRegisterCallback(EmulateInstruction *instruction, void *baton, - const RegisterInfo *reg_info, - RegisterValue ®_value) { - EmulatorBaton *emulator_baton = static_cast<EmulatorBaton *>(baton); - - auto it = emulator_baton->m_register_values.find( - reg_info->kinds[eRegisterKindDWARF]); - if (it != emulator_baton->m_register_values.end()) { - reg_value = it->second; - return true; - } - - // The emulator only fills in the dwarf register numbers (and in some cases - // the generic register numbers). Get the full register info from the - // register context based on the dwarf register numbers. - const RegisterInfo *full_reg_info = - emulator_baton->m_reg_context->GetRegisterInfo( - eRegisterKindDWARF, reg_info->kinds[eRegisterKindDWARF]); - - bool error = - emulator_baton->m_reg_context->ReadRegister(full_reg_info, reg_value); - return error; -} - -static bool WriteRegisterCallback(EmulateInstruction *instruction, void *baton, - const EmulateInstruction::Context &context, - const RegisterInfo *reg_info, - const RegisterValue ®_value) { - EmulatorBaton *emulator_baton = static_cast<EmulatorBaton *>(baton); - emulator_baton->m_register_values[reg_info->kinds[eRegisterKindDWARF]] = - reg_value; - return true; -} - -static size_t WriteMemoryCallback(EmulateInstruction *instruction, void *baton, - const EmulateInstruction::Context &context, - lldb::addr_t addr, const void *dst, - size_t length) { - return length; -} - -bool ProcessFreeBSD::SingleStepBreakpointHit( - void *baton, lldb_private::StoppointCallbackContext *context, - lldb::user_id_t break_id, lldb::user_id_t break_loc_id) { - return false; -} - -Status ProcessFreeBSD::SetSoftwareSingleStepBreakpoint(lldb::tid_t tid, - lldb::addr_t addr) { - Status error; - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - if (log) { - LLDB_LOGF(log, "ProcessFreeBSD::%s addr = 0x%" PRIx64, __FUNCTION__, addr); - LLDB_LOGF(log, "SoftwareBreakpoint::%s addr = 0x%" PRIx64, __FUNCTION__, - addr); - } - - // Validate the address. - if (addr == LLDB_INVALID_ADDRESS) - return Status("ProcessFreeBSD::%s invalid load address specified.", - __FUNCTION__); - - Breakpoint *const sw_step_break = - m_process->GetTarget().CreateBreakpoint(addr, true, false).get(); - sw_step_break->SetCallback(SingleStepBreakpointHit, this, true); - sw_step_break->SetBreakpointKind("software-single-step"); - - LLDB_LOGF(log, "ProcessFreeBSD::%s addr = 0x%" PRIx64 " -- SUCCESS", - __FUNCTION__, addr); - - m_threads_stepping_with_breakpoint.insert({tid, sw_step_break->GetID()}); - return Status(); -} - -bool ProcessFreeBSD::IsSoftwareStepBreakpoint(lldb::tid_t tid) { - ThreadSP thread = GetThreadList().FindThreadByID(tid); - if (!thread) - return false; - - assert(thread->GetRegisterContext()); - lldb::addr_t stop_pc = thread->GetRegisterContext()->GetPC(); - - const auto &iter = m_threads_stepping_with_breakpoint.find(tid); - if (iter == m_threads_stepping_with_breakpoint.end()) - return false; - - lldb::break_id_t bp_id = iter->second; - BreakpointSP bp = GetTarget().GetBreakpointByID(bp_id); - if (!bp) - return false; - - BreakpointLocationSP bp_loc = bp->FindLocationByAddress(stop_pc); - if (!bp_loc) - return false; - - GetTarget().RemoveBreakpointByID(bp_id); - m_threads_stepping_with_breakpoint.erase(tid); - return true; -} - -bool ProcessFreeBSD::SupportHardwareSingleStepping() const { - lldb_private::ArchSpec arch = GetTarget().GetArchitecture(); - if (arch.GetMachine() == llvm::Triple::arm || arch.IsMIPS()) - return false; - return true; -} - -Status ProcessFreeBSD::SetupSoftwareSingleStepping(lldb::tid_t tid) { - std::unique_ptr<EmulateInstruction> emulator_up( - EmulateInstruction::FindPlugin(GetTarget().GetArchitecture(), - eInstructionTypePCModifying, nullptr)); - - if (emulator_up == nullptr) - return Status("Instruction emulator not found!"); - - FreeBSDThread *thread = static_cast<FreeBSDThread *>( - m_thread_list.FindThreadByID(tid, false).get()); - if (thread == NULL) - return Status("Thread not found not found!"); - - lldb::RegisterContextSP register_context_sp = thread->GetRegisterContext(); - - EmulatorBaton baton(this, register_context_sp.get()); - emulator_up->SetBaton(&baton); - emulator_up->SetReadMemCallback(&ReadMemoryCallback); - emulator_up->SetReadRegCallback(&ReadRegisterCallback); - emulator_up->SetWriteMemCallback(&WriteMemoryCallback); - emulator_up->SetWriteRegCallback(&WriteRegisterCallback); - - if (!emulator_up->ReadInstruction()) - return Status("Read instruction failed!"); - - bool emulation_result = - emulator_up->EvaluateInstruction(eEmulateInstructionOptionAutoAdvancePC); - const RegisterInfo *reg_info_pc = register_context_sp->GetRegisterInfo( - eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC); - auto pc_it = - baton.m_register_values.find(reg_info_pc->kinds[eRegisterKindDWARF]); - - lldb::addr_t next_pc; - if (emulation_result) { - assert(pc_it != baton.m_register_values.end() && - "Emulation was successful but PC wasn't updated"); - next_pc = pc_it->second.GetAsUInt64(); - } else if (pc_it == baton.m_register_values.end()) { - // Emulate instruction failed and it haven't changed PC. Advance PC with - // the size of the current opcode because the emulation of all - // PC modifying instruction should be successful. The failure most - // likely caused by a not supported instruction which don't modify PC. - next_pc = - register_context_sp->GetPC() + emulator_up->GetOpcode().GetByteSize(); - } else { - // The instruction emulation failed after it modified the PC. It is an - // unknown error where we can't continue because the next instruction is - // modifying the PC but we don't know how. - return Status("Instruction emulation failed unexpectedly"); - } - - SetSoftwareSingleStepBreakpoint(tid, next_pc); - return Status(); -} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.h deleted file mode 100644 index b60bcd279021..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.h +++ /dev/null @@ -1,221 +0,0 @@ -//===-- ProcessFreeBSD.h ------------------------------------------*- C++ -//-*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_ProcessFreeBSD_H_ -#define liblldb_ProcessFreeBSD_H_ - -#include "Plugins/Process/POSIX/ProcessMessage.h" -#include "lldb/Target/Process.h" -#include "lldb/Target/ThreadList.h" -#include <mutex> -#include <queue> -#include <set> - -class ProcessMonitor; -class FreeBSDThread; - -class ProcessFreeBSD : public lldb_private::Process { - -public: - // Static functions. - static lldb::ProcessSP - CreateInstance(lldb::TargetSP target_sp, lldb::ListenerSP listener_sp, - const lldb_private::FileSpec *crash_file_path, - bool can_connect); - - static void Initialize(); - - static void Terminate(); - - static lldb_private::ConstString GetPluginNameStatic(); - - static const char *GetPluginDescriptionStatic(); - - // Constructors and destructors - ProcessFreeBSD(lldb::TargetSP target_sp, lldb::ListenerSP listener_sp, - lldb::UnixSignalsSP &unix_signals_sp); - - ~ProcessFreeBSD(); - - virtual lldb_private::Status WillResume() override; - - // PluginInterface protocol - virtual lldb_private::ConstString GetPluginName() override; - - virtual uint32_t GetPluginVersion() override; - -public: - // Process protocol. - void Finalize() override; - - bool CanDebug(lldb::TargetSP target_sp, - bool plugin_specified_by_name) override; - - lldb_private::Status WillLaunch(lldb_private::Module *module) override; - - lldb_private::Status DoAttachToProcessWithID( - lldb::pid_t pid, - const lldb_private::ProcessAttachInfo &attach_info) override; - - lldb_private::Status - DoLaunch(lldb_private::Module *exe_module, - lldb_private::ProcessLaunchInfo &launch_info) override; - - void DidLaunch() override; - - lldb_private::Status DoResume() override; - - lldb_private::Status DoHalt(bool &caused_stop) override; - - lldb_private::Status DoDetach(bool keep_stopped) override; - - lldb_private::Status DoSignal(int signal) override; - - lldb_private::Status DoDestroy() override; - - void DoDidExec() override; - - void RefreshStateAfterStop() override; - - bool IsAlive() override; - - size_t DoReadMemory(lldb::addr_t vm_addr, void *buf, size_t size, - lldb_private::Status &error) override; - - size_t DoWriteMemory(lldb::addr_t vm_addr, const void *buf, size_t size, - lldb_private::Status &error) override; - - lldb::addr_t DoAllocateMemory(size_t size, uint32_t permissions, - lldb_private::Status &error) override; - - lldb_private::Status DoDeallocateMemory(lldb::addr_t ptr) override; - - virtual size_t - GetSoftwareBreakpointTrapOpcode(lldb_private::BreakpointSite *bp_site); - - lldb_private::Status - EnableBreakpointSite(lldb_private::BreakpointSite *bp_site) override; - - lldb_private::Status - DisableBreakpointSite(lldb_private::BreakpointSite *bp_site) override; - - lldb_private::Status EnableWatchpoint(lldb_private::Watchpoint *wp, - bool notify = true) override; - - lldb_private::Status DisableWatchpoint(lldb_private::Watchpoint *wp, - bool notify = true) override; - - lldb_private::Status GetWatchpointSupportInfo(uint32_t &num) override; - - lldb_private::Status GetWatchpointSupportInfo(uint32_t &num, - bool &after) override; - - virtual uint32_t UpdateThreadListIfNeeded(); - - bool DoUpdateThreadList(lldb_private::ThreadList &old_thread_list, - lldb_private::ThreadList &new_thread_list) override; - - virtual lldb::ByteOrder GetByteOrder() const; - - lldb::addr_t GetImageInfoAddress() override; - - size_t PutSTDIN(const char *buf, size_t len, - lldb_private::Status &error) override; - - lldb_private::DataExtractor GetAuxvData() override; - - // ProcessFreeBSD internal API. - - /// Registers the given message with this process. - virtual void SendMessage(const ProcessMessage &message); - - ProcessMonitor &GetMonitor() { - assert(m_monitor); - return *m_monitor; - } - - lldb_private::FileSpec - GetFileSpec(const lldb_private::FileAction *file_action, - const lldb_private::FileSpec &default_file_spec, - const lldb_private::FileSpec &dbg_pts_file_spec); - - /// Adds the thread to the list of threads for which we have received the - /// initial stopping signal. - /// The \p stop_tid parameter indicates the thread which the stop happened - /// for. - bool AddThreadForInitialStopIfNeeded(lldb::tid_t stop_tid); - - bool WaitingForInitialStop(lldb::tid_t stop_tid); - - virtual FreeBSDThread *CreateNewFreeBSDThread(lldb_private::Process &process, - lldb::tid_t tid); - - static bool SingleStepBreakpointHit( - void *baton, lldb_private::StoppointCallbackContext *context, - lldb::user_id_t break_id, lldb::user_id_t break_loc_id); - - lldb_private::Status SetupSoftwareSingleStepping(lldb::tid_t tid); - - lldb_private::Status SetSoftwareSingleStepBreakpoint(lldb::tid_t tid, - lldb::addr_t addr); - - bool IsSoftwareStepBreakpoint(lldb::tid_t tid); - - bool SupportHardwareSingleStepping() const; - - typedef std::vector<lldb::tid_t> tid_collection; - tid_collection &GetStepTids() { return m_step_tids; } - -protected: - static const size_t MAX_TRAP_OPCODE_SIZE = 8; - - /// Target byte order. - lldb::ByteOrder m_byte_order; - - /// Process monitor; - ProcessMonitor *m_monitor; - - /// The module we are executing. - lldb_private::Module *m_module; - - /// Message queue notifying this instance of inferior process state changes. - std::recursive_mutex m_message_mutex; - std::queue<ProcessMessage> m_message_queue; - - /// Drive any exit events to completion. - bool m_exit_now; - - /// Returns true if the process has exited. - bool HasExited(); - - /// Returns true if the process is stopped. - bool IsStopped(); - - /// Returns true if at least one running is currently running - bool IsAThreadRunning(); - - typedef std::map<lldb::addr_t, lldb::addr_t> MMapMap; - MMapMap m_addr_to_mmap_size; - - typedef std::set<lldb::tid_t> ThreadStopSet; - /// Every thread begins with a stop signal. This keeps track - /// of the threads for which we have received the stop signal. - ThreadStopSet m_seen_initial_stop; - - friend class FreeBSDThread; - - tid_collection m_suspend_tids; - tid_collection m_run_tids; - tid_collection m_step_tids; - std::map<lldb::tid_t, lldb::break_id_t> m_threads_stepping_with_breakpoint; - - int m_resume_signo; -}; - -#endif // liblldb_ProcessFreeBSD_H_ diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp deleted file mode 100644 index 0738cced44ab..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp +++ /dev/null @@ -1,1424 +0,0 @@ -//===-- ProcessMonitor.cpp ------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include <errno.h> -#include <poll.h> -#include <signal.h> -#include <stdint.h> -#include <string.h> -#include <sys/ptrace.h> -#include <sys/socket.h> -#include <sys/types.h> -#include <sys/wait.h> -#include <unistd.h> - -#include "lldb/Host/Host.h" -#include "lldb/Host/PseudoTerminal.h" -#include "lldb/Host/ThreadLauncher.h" -#include "lldb/Target/RegisterContext.h" -#include "lldb/Target/Thread.h" -#include "lldb/Target/UnixSignals.h" -#include "lldb/Utility/RegisterValue.h" -#include "lldb/Utility/Scalar.h" -#include "lldb/Utility/Status.h" -#include "llvm/Support/Errno.h" - -#include "FreeBSDThread.h" -#include "Plugins/Process/POSIX/CrashReason.h" -#include "Plugins/Process/POSIX/ProcessPOSIXLog.h" -#include "ProcessFreeBSD.h" -#include "ProcessMonitor.h" - -using namespace lldb; -using namespace lldb_private; - -// Wrapper for ptrace to catch errors and log calls. - -const char *Get_PT_IO_OP(int op) { - switch (op) { - case PIOD_READ_D: - return "READ_D"; - case PIOD_WRITE_D: - return "WRITE_D"; - case PIOD_READ_I: - return "READ_I"; - case PIOD_WRITE_I: - return "WRITE_I"; - default: - return "Unknown op"; - } -} - -// Wrapper for ptrace to catch errors and log calls. Note that ptrace sets -// errno on error because -1 is reserved as a valid result. -extern long PtraceWrapper(int req, lldb::pid_t pid, void *addr, int data, - const char *reqName, const char *file, int line) { - long int result; - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PTRACE)); - - if (log) { - LLDB_LOGF(log, - "ptrace(%s, %" PRIu64 ", %p, %x) called from file %s line %d", - reqName, pid, addr, data, file, line); - if (req == PT_IO) { - struct ptrace_io_desc *pi = (struct ptrace_io_desc *)addr; - - LLDB_LOGF(log, "PT_IO: op=%s offs=%zx size=%zu", - Get_PT_IO_OP(pi->piod_op), (size_t)pi->piod_offs, pi->piod_len); - } - } - - // PtraceDisplayBytes(req, data); - - errno = 0; - result = ptrace(req, pid, (caddr_t)addr, data); - - // PtraceDisplayBytes(req, data); - - if (log && errno != 0) { - const char *str; - switch (errno) { - case ESRCH: - str = "ESRCH"; - break; - case EINVAL: - str = "EINVAL"; - break; - case EBUSY: - str = "EBUSY"; - break; - case EPERM: - str = "EPERM"; - break; - default: - str = "<unknown>"; - } - LLDB_LOGF(log, "ptrace() failed; errno=%d (%s)", errno, str); - } - - if (log) { -#ifdef __amd64__ - if (req == PT_GETREGS) { - struct reg *r = (struct reg *)addr; - - LLDB_LOGF(log, "PT_GETREGS: rip=0x%lx rsp=0x%lx rbp=0x%lx rax=0x%lx", - r->r_rip, r->r_rsp, r->r_rbp, r->r_rax); - } - if (req == PT_GETDBREGS || req == PT_SETDBREGS) { - struct dbreg *r = (struct dbreg *)addr; - char setget = (req == PT_GETDBREGS) ? 'G' : 'S'; - - for (int i = 0; i <= 7; i++) - LLDB_LOGF(log, "PT_%cETDBREGS: dr[%d]=0x%lx", setget, i, r->dr[i]); - } -#endif - } - - return result; -} - -// Wrapper for ptrace when logging is not required. Sets errno to 0 prior to -// calling ptrace. -extern long PtraceWrapper(int req, lldb::pid_t pid, void *addr, int data) { - long result = 0; - errno = 0; - result = ptrace(req, pid, (caddr_t)addr, data); - return result; -} - -#define PTRACE(req, pid, addr, data) \ - PtraceWrapper((req), (pid), (addr), (data), #req, __FILE__, __LINE__) - -// Static implementations of ProcessMonitor::ReadMemory and -// ProcessMonitor::WriteMemory. This enables mutual recursion between these -// functions without needed to go thru the thread funnel. - -static size_t DoReadMemory(lldb::pid_t pid, lldb::addr_t vm_addr, void *buf, - size_t size, Status &error) { - struct ptrace_io_desc pi_desc; - - pi_desc.piod_op = PIOD_READ_D; - pi_desc.piod_offs = (void *)vm_addr; - pi_desc.piod_addr = buf; - pi_desc.piod_len = size; - - if (PTRACE(PT_IO, pid, (caddr_t)&pi_desc, 0) < 0) { - error.SetErrorToErrno(); - return 0; - } - return pi_desc.piod_len; -} - -static size_t DoWriteMemory(lldb::pid_t pid, lldb::addr_t vm_addr, - const void *buf, size_t size, Status &error) { - struct ptrace_io_desc pi_desc; - - pi_desc.piod_op = PIOD_WRITE_D; - pi_desc.piod_offs = (void *)vm_addr; - pi_desc.piod_addr = const_cast<void *>(buf); - pi_desc.piod_len = size; - - if (PTRACE(PT_IO, pid, (caddr_t)&pi_desc, 0) < 0) { - error.SetErrorToErrno(); - return 0; - } - return pi_desc.piod_len; -} - -// Simple helper function to ensure flags are enabled on the given file -// descriptor. -static bool EnsureFDFlags(int fd, int flags, Status &error) { - int status; - - if ((status = fcntl(fd, F_GETFL)) == -1) { - error.SetErrorToErrno(); - return false; - } - - if (fcntl(fd, F_SETFL, status | flags) == -1) { - error.SetErrorToErrno(); - return false; - } - - return true; -} - -/// \class Operation -/// Represents a ProcessMonitor operation. -/// -/// Under FreeBSD, it is not possible to ptrace() from any other thread but -/// the one that spawned or attached to the process from the start. -/// Therefore, when a ProcessMonitor is asked to deliver or change the state -/// of an inferior process the operation must be "funneled" to a specific -/// thread to perform the task. The Operation class provides an abstract base -/// for all services the ProcessMonitor must perform via the single virtual -/// function Execute, thus encapsulating the code that needs to run in the -/// privileged context. -class Operation { -public: - virtual ~Operation() {} - virtual void Execute(ProcessMonitor *monitor) = 0; -}; - -/// \class ReadOperation -/// Implements ProcessMonitor::ReadMemory. -class ReadOperation : public Operation { -public: - ReadOperation(lldb::addr_t addr, void *buff, size_t size, Status &error, - size_t &result) - : m_addr(addr), m_buff(buff), m_size(size), m_error(error), - m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::addr_t m_addr; - void *m_buff; - size_t m_size; - Status &m_error; - size_t &m_result; -}; - -void ReadOperation::Execute(ProcessMonitor *monitor) { - lldb::pid_t pid = monitor->GetPID(); - - m_result = DoReadMemory(pid, m_addr, m_buff, m_size, m_error); -} - -/// \class WriteOperation -/// Implements ProcessMonitor::WriteMemory. -class WriteOperation : public Operation { -public: - WriteOperation(lldb::addr_t addr, const void *buff, size_t size, - Status &error, size_t &result) - : m_addr(addr), m_buff(buff), m_size(size), m_error(error), - m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::addr_t m_addr; - const void *m_buff; - size_t m_size; - Status &m_error; - size_t &m_result; -}; - -void WriteOperation::Execute(ProcessMonitor *monitor) { - lldb::pid_t pid = monitor->GetPID(); - - m_result = DoWriteMemory(pid, m_addr, m_buff, m_size, m_error); -} - -/// \class ReadRegOperation -/// Implements ProcessMonitor::ReadRegisterValue. -class ReadRegOperation : public Operation { -public: - ReadRegOperation(lldb::tid_t tid, unsigned offset, unsigned size, - RegisterValue &value, bool &result) - : m_tid(tid), m_offset(offset), m_size(size), m_value(value), - m_result(result) {} - - void Execute(ProcessMonitor *monitor); - -private: - lldb::tid_t m_tid; - unsigned m_offset; - unsigned m_size; - RegisterValue &m_value; - bool &m_result; -}; - -void ReadRegOperation::Execute(ProcessMonitor *monitor) { - struct reg regs; - int rc; - - if ((rc = PTRACE(PT_GETREGS, m_tid, (caddr_t)®s, 0)) < 0) { - m_result = false; - } else { - // 'struct reg' contains only 32- or 64-bit register values. Punt on - // others. Also, not all entries may be uintptr_t sized, such as 32-bit - // processes on powerpc64 (probably the same for i386 on amd64) - if (m_size == sizeof(uint32_t)) - m_value = *(uint32_t *)(((caddr_t)®s) + m_offset); - else if (m_size == sizeof(uint64_t)) - m_value = *(uint64_t *)(((caddr_t)®s) + m_offset); - else - memcpy((void *)&m_value, (((caddr_t)®s) + m_offset), m_size); - m_result = true; - } -} - -/// \class WriteRegOperation -/// Implements ProcessMonitor::WriteRegisterValue. -class WriteRegOperation : public Operation { -public: - WriteRegOperation(lldb::tid_t tid, unsigned offset, - const RegisterValue &value, bool &result) - : m_tid(tid), m_offset(offset), m_value(value), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - unsigned m_offset; - const RegisterValue &m_value; - bool &m_result; -}; - -void WriteRegOperation::Execute(ProcessMonitor *monitor) { - struct reg regs; - - if (PTRACE(PT_GETREGS, m_tid, (caddr_t)®s, 0) < 0) { - m_result = false; - return; - } - *(uintptr_t *)(((caddr_t)®s) + m_offset) = - (uintptr_t)m_value.GetAsUInt64(); - if (PTRACE(PT_SETREGS, m_tid, (caddr_t)®s, 0) < 0) - m_result = false; - else - m_result = true; -} - -/// \class ReadDebugRegOperation -/// Implements ProcessMonitor::ReadDebugRegisterValue. -class ReadDebugRegOperation : public Operation { -public: - ReadDebugRegOperation(lldb::tid_t tid, unsigned offset, unsigned size, - RegisterValue &value, bool &result) - : m_tid(tid), m_offset(offset), m_size(size), m_value(value), - m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - unsigned m_offset; - unsigned m_size; - RegisterValue &m_value; - bool &m_result; -}; - -void ReadDebugRegOperation::Execute(ProcessMonitor *monitor) { - struct dbreg regs; - int rc; - - if ((rc = PTRACE(PT_GETDBREGS, m_tid, (caddr_t)®s, 0)) < 0) { - m_result = false; - } else { - if (m_size == sizeof(uintptr_t)) - m_value = *(uintptr_t *)(((caddr_t)®s) + m_offset); - else - memcpy((void *)&m_value, (((caddr_t)®s) + m_offset), m_size); - m_result = true; - } -} - -/// \class WriteDebugRegOperation -/// Implements ProcessMonitor::WriteDebugRegisterValue. -class WriteDebugRegOperation : public Operation { -public: - WriteDebugRegOperation(lldb::tid_t tid, unsigned offset, - const RegisterValue &value, bool &result) - : m_tid(tid), m_offset(offset), m_value(value), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - unsigned m_offset; - const RegisterValue &m_value; - bool &m_result; -}; - -void WriteDebugRegOperation::Execute(ProcessMonitor *monitor) { - struct dbreg regs; - - if (PTRACE(PT_GETDBREGS, m_tid, (caddr_t)®s, 0) < 0) { - m_result = false; - return; - } - *(uintptr_t *)(((caddr_t)®s) + m_offset) = - (uintptr_t)m_value.GetAsUInt64(); - if (PTRACE(PT_SETDBREGS, m_tid, (caddr_t)®s, 0) < 0) - m_result = false; - else - m_result = true; -} - -/// \class ReadGPROperation -/// Implements ProcessMonitor::ReadGPR. -class ReadGPROperation : public Operation { -public: - ReadGPROperation(lldb::tid_t tid, void *buf, bool &result) - : m_tid(tid), m_buf(buf), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - void *m_buf; - bool &m_result; -}; - -void ReadGPROperation::Execute(ProcessMonitor *monitor) { - int rc; - - errno = 0; - rc = PTRACE(PT_GETREGS, m_tid, (caddr_t)m_buf, 0); - if (errno != 0) - m_result = false; - else - m_result = true; -} - -/// \class ReadFPROperation -/// Implements ProcessMonitor::ReadFPR. -class ReadFPROperation : public Operation { -public: - ReadFPROperation(lldb::tid_t tid, void *buf, bool &result) - : m_tid(tid), m_buf(buf), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - void *m_buf; - bool &m_result; -}; - -void ReadFPROperation::Execute(ProcessMonitor *monitor) { - if (PTRACE(PT_GETFPREGS, m_tid, (caddr_t)m_buf, 0) < 0) - m_result = false; - else - m_result = true; -} - -/// \class WriteGPROperation -/// Implements ProcessMonitor::WriteGPR. -class WriteGPROperation : public Operation { -public: - WriteGPROperation(lldb::tid_t tid, void *buf, bool &result) - : m_tid(tid), m_buf(buf), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - void *m_buf; - bool &m_result; -}; - -void WriteGPROperation::Execute(ProcessMonitor *monitor) { - if (PTRACE(PT_SETREGS, m_tid, (caddr_t)m_buf, 0) < 0) - m_result = false; - else - m_result = true; -} - -/// \class WriteFPROperation -/// Implements ProcessMonitor::WriteFPR. -class WriteFPROperation : public Operation { -public: - WriteFPROperation(lldb::tid_t tid, void *buf, bool &result) - : m_tid(tid), m_buf(buf), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - void *m_buf; - bool &m_result; -}; - -void WriteFPROperation::Execute(ProcessMonitor *monitor) { - if (PTRACE(PT_SETFPREGS, m_tid, (caddr_t)m_buf, 0) < 0) - m_result = false; - else - m_result = true; -} - -/// \class ResumeOperation -/// Implements ProcessMonitor::Resume. -class ResumeOperation : public Operation { -public: - ResumeOperation(uint32_t signo, bool &result) - : m_signo(signo), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - uint32_t m_signo; - bool &m_result; -}; - -void ResumeOperation::Execute(ProcessMonitor *monitor) { - lldb::pid_t pid = monitor->GetPID(); - int data = 0; - - if (m_signo != LLDB_INVALID_SIGNAL_NUMBER) - data = m_signo; - - if (PTRACE(PT_CONTINUE, pid, (caddr_t)1, data)) { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - LLDB_LOG(log, "ResumeOperation ({0}) failed: {1}", pid, - llvm::sys::StrError(errno)); - m_result = false; - } else - m_result = true; -} - -/// \class SingleStepOperation -/// Implements ProcessMonitor::SingleStep. -class SingleStepOperation : public Operation { -public: - SingleStepOperation(uint32_t signo, bool &result) - : m_signo(signo), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - uint32_t m_signo; - bool &m_result; -}; - -void SingleStepOperation::Execute(ProcessMonitor *monitor) { - lldb::pid_t pid = monitor->GetPID(); - int data = 0; - - if (m_signo != LLDB_INVALID_SIGNAL_NUMBER) - data = m_signo; - - if (PTRACE(PT_STEP, pid, NULL, data)) - m_result = false; - else - m_result = true; -} - -/// \class LwpInfoOperation -/// Implements ProcessMonitor::GetLwpInfo. -class LwpInfoOperation : public Operation { -public: - LwpInfoOperation(lldb::tid_t tid, void *info, bool &result, int &ptrace_err) - : m_tid(tid), m_info(info), m_result(result), m_err(ptrace_err) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - void *m_info; - bool &m_result; - int &m_err; -}; - -void LwpInfoOperation::Execute(ProcessMonitor *monitor) { - struct ptrace_lwpinfo plwp; - - if (PTRACE(PT_LWPINFO, m_tid, (caddr_t)&plwp, sizeof(plwp))) { - m_result = false; - m_err = errno; - } else { - memcpy(m_info, &plwp, sizeof(plwp)); - m_result = true; - } -} - -/// \class ThreadSuspendOperation -/// Implements ProcessMonitor::ThreadSuspend. -class ThreadSuspendOperation : public Operation { -public: - ThreadSuspendOperation(lldb::tid_t tid, bool suspend, bool &result) - : m_tid(tid), m_suspend(suspend), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - bool m_suspend; - bool &m_result; -}; - -void ThreadSuspendOperation::Execute(ProcessMonitor *monitor) { - m_result = !PTRACE(m_suspend ? PT_SUSPEND : PT_RESUME, m_tid, NULL, 0); -} - -/// \class EventMessageOperation -/// Implements ProcessMonitor::GetEventMessage. -class EventMessageOperation : public Operation { -public: - EventMessageOperation(lldb::tid_t tid, unsigned long *message, bool &result) - : m_tid(tid), m_message(message), m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - lldb::tid_t m_tid; - unsigned long *m_message; - bool &m_result; -}; - -void EventMessageOperation::Execute(ProcessMonitor *monitor) { - struct ptrace_lwpinfo plwp; - - if (PTRACE(PT_LWPINFO, m_tid, (caddr_t)&plwp, sizeof(plwp))) - m_result = false; - else { - if (plwp.pl_flags & PL_FLAG_FORKED) { - *m_message = plwp.pl_child_pid; - m_result = true; - } else - m_result = false; - } -} - -/// \class KillOperation -/// Implements ProcessMonitor::Kill. -class KillOperation : public Operation { -public: - KillOperation(bool &result) : m_result(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - bool &m_result; -}; - -void KillOperation::Execute(ProcessMonitor *monitor) { - lldb::pid_t pid = monitor->GetPID(); - - if (PTRACE(PT_KILL, pid, NULL, 0)) - m_result = false; - else - m_result = true; -} - -/// \class DetachOperation -/// Implements ProcessMonitor::Detach. -class DetachOperation : public Operation { -public: - DetachOperation(Status &result) : m_error(result) {} - - void Execute(ProcessMonitor *monitor) override; - -private: - Status &m_error; -}; - -void DetachOperation::Execute(ProcessMonitor *monitor) { - lldb::pid_t pid = monitor->GetPID(); - - if (PTRACE(PT_DETACH, pid, NULL, 0) < 0) - m_error.SetErrorToErrno(); -} - -ProcessMonitor::OperationArgs::OperationArgs(ProcessMonitor *monitor) - : m_monitor(monitor) { - sem_init(&m_semaphore, 0, 0); -} - -ProcessMonitor::OperationArgs::~OperationArgs() { sem_destroy(&m_semaphore); } - -ProcessMonitor::LaunchArgs::LaunchArgs(ProcessMonitor *monitor, - lldb_private::Module *module, - char const **argv, Environment env, - const FileSpec &stdin_file_spec, - const FileSpec &stdout_file_spec, - const FileSpec &stderr_file_spec, - const FileSpec &working_dir) - : OperationArgs(monitor), m_module(module), m_argv(argv), - m_env(std::move(env)), m_stdin_file_spec(stdin_file_spec), - m_stdout_file_spec(stdout_file_spec), - m_stderr_file_spec(stderr_file_spec), m_working_dir(working_dir) {} - -ProcessMonitor::LaunchArgs::~LaunchArgs() {} - -ProcessMonitor::AttachArgs::AttachArgs(ProcessMonitor *monitor, lldb::pid_t pid) - : OperationArgs(monitor), m_pid(pid) {} - -ProcessMonitor::AttachArgs::~AttachArgs() {} - -/// The basic design of the ProcessMonitor is built around two threads. -/// -/// One thread (@see SignalThread) simply blocks on a call to waitpid() -/// looking for changes in the debugee state. When a change is detected a -/// ProcessMessage is sent to the associated ProcessFreeBSD instance. This -/// thread "drives" state changes in the debugger. -/// -/// The second thread (@see OperationThread) is responsible for two things 1) -/// launching or attaching to the inferior process, and then 2) servicing -/// operations such as register reads/writes, stepping, etc. See the comments -/// on the Operation class for more info as to why this is needed. -ProcessMonitor::ProcessMonitor( - ProcessFreeBSD *process, Module *module, const char *argv[], - Environment env, const FileSpec &stdin_file_spec, - const FileSpec &stdout_file_spec, const FileSpec &stderr_file_spec, - const FileSpec &working_dir, - const lldb_private::ProcessLaunchInfo & /* launch_info */, - lldb_private::Status &error) - : m_process(static_cast<ProcessFreeBSD *>(process)), - m_operation_thread(), m_monitor_thread(), m_pid(LLDB_INVALID_PROCESS_ID), m_terminal_fd(-1), m_operation(0) { - using namespace std::placeholders; - - std::unique_ptr<LaunchArgs> args( - new LaunchArgs(this, module, argv, std::move(env), stdin_file_spec, - stdout_file_spec, stderr_file_spec, working_dir)); - - sem_init(&m_operation_pending, 0, 0); - sem_init(&m_operation_done, 0, 0); - - StartLaunchOpThread(args.get(), error); - if (!error.Success()) - return; - - if (llvm::sys::RetryAfterSignal(-1, sem_wait, &args->m_semaphore) == -1) { - error.SetErrorToErrno(); - return; - } - - // Check that the launch was a success. - if (!args->m_error.Success()) { - StopOpThread(); - error = args->m_error; - return; - } - - // Finally, start monitoring the child process for change in state. - llvm::Expected<lldb_private::HostThread> monitor_thread = - Host::StartMonitoringChildProcess( - std::bind(&ProcessMonitor::MonitorCallback, this, _1, _2, _3, _4), - GetPID(), true); - if (!monitor_thread || !monitor_thread->IsJoinable()) { - error.SetErrorToGenericError(); - error.SetErrorString("Process launch failed."); - return; - } - m_monitor_thread = *monitor_thread; -} - -ProcessMonitor::ProcessMonitor(ProcessFreeBSD *process, lldb::pid_t pid, - lldb_private::Status &error) - : m_process(static_cast<ProcessFreeBSD *>(process)), - m_operation_thread(), m_monitor_thread(), m_pid(pid), m_terminal_fd(-1), m_operation(0) { - using namespace std::placeholders; - - sem_init(&m_operation_pending, 0, 0); - sem_init(&m_operation_done, 0, 0); - - std::unique_ptr<AttachArgs> args(new AttachArgs(this, pid)); - - StartAttachOpThread(args.get(), error); - if (!error.Success()) - return; - - if (llvm::sys::RetryAfterSignal(-1, sem_wait, &args->m_semaphore) == -1) { - error.SetErrorToErrno(); - return; - } - - // Check that the attach was a success. - if (!args->m_error.Success()) { - StopOpThread(); - error = args->m_error; - return; - } - - // Finally, start monitoring the child process for change in state. - llvm::Expected<lldb_private::HostThread> monitor_thread = - Host::StartMonitoringChildProcess( - std::bind(&ProcessMonitor::MonitorCallback, this, _1, _2, _3, _4), - GetPID(), true); - if (!monitor_thread || !monitor_thread->IsJoinable()) { - error.SetErrorToGenericError(); - error.SetErrorString("Process attach failed."); - return; - } - m_monitor_thread = *monitor_thread; -} - -ProcessMonitor::~ProcessMonitor() { StopMonitor(); } - -// Thread setup and tear down. -void ProcessMonitor::StartLaunchOpThread(LaunchArgs *args, Status &error) { - static const char *g_thread_name = "freebsd.op"; - - if (m_operation_thread && m_operation_thread->IsJoinable()) - return; - - llvm::Expected<lldb_private::HostThread> operation_thread = - ThreadLauncher::LaunchThread(g_thread_name, LaunchOpThread, args); - if (operation_thread) - m_operation_thread = *operation_thread; - else - error = operation_thread.takeError(); -} - -void *ProcessMonitor::LaunchOpThread(void *arg) { - LaunchArgs *args = static_cast<LaunchArgs *>(arg); - - if (!Launch(args)) { - sem_post(&args->m_semaphore); - return NULL; - } - - ServeOperation(args); - return NULL; -} - -bool ProcessMonitor::Launch(LaunchArgs *args) { - ProcessMonitor *monitor = args->m_monitor; - ProcessFreeBSD &process = monitor->GetProcess(); - const char **argv = args->m_argv; - const FileSpec &stdin_file_spec = args->m_stdin_file_spec; - const FileSpec &stdout_file_spec = args->m_stdout_file_spec; - const FileSpec &stderr_file_spec = args->m_stderr_file_spec; - const FileSpec &working_dir = args->m_working_dir; - - PseudoTerminal terminal; - - // Propagate the environment if one is not supplied. - Environment::Envp envp = - (args->m_env.empty() ? Host::GetEnvironment() : args->m_env).getEnvp(); - - llvm::Expected<lldb::pid_t> pid = terminal.Fork(); - if (!pid) { - args->m_error = pid.takeError(); - goto FINISH; - } - - // Recognized child exit status codes. - enum { - ePtraceFailed = 1, - eDupStdinFailed, - eDupStdoutFailed, - eDupStderrFailed, - eChdirFailed, - eExecFailed, - eSetGidFailed - }; - - // Child process. - if (*pid == 0) { - // Trace this process. - if (PTRACE(PT_TRACE_ME, 0, NULL, 0) < 0) - exit(ePtraceFailed); - - // terminal has already dupped the tty descriptors to stdin/out/err. This - // closes original fd from which they were copied (and avoids leaking - // descriptors to the debugged process. - terminal.CloseSecondaryFileDescriptor(); - - // Do not inherit setgid powers. - if (setgid(getgid()) != 0) - exit(eSetGidFailed); - - // Let us have our own process group. - setpgid(0, 0); - - // Dup file descriptors if needed. - // - // FIXME: If two or more of the paths are the same we needlessly open - // the same file multiple times. - if (stdin_file_spec) - if (!DupDescriptor(stdin_file_spec, STDIN_FILENO, O_RDONLY)) - exit(eDupStdinFailed); - - if (stdout_file_spec) - if (!DupDescriptor(stdout_file_spec, STDOUT_FILENO, O_WRONLY | O_CREAT)) - exit(eDupStdoutFailed); - - if (stderr_file_spec) - if (!DupDescriptor(stderr_file_spec, STDERR_FILENO, O_WRONLY | O_CREAT)) - exit(eDupStderrFailed); - - // Change working directory - if (working_dir && 0 != ::chdir(working_dir.GetCString())) - exit(eChdirFailed); - - // Execute. We should never return. - execve(argv[0], const_cast<char *const *>(argv), envp); - exit(eExecFailed); - } - - // Wait for the child process to to trap on its call to execve. - ::pid_t wpid; - int status; - if ((wpid = waitpid(*pid, &status, 0)) < 0) { - args->m_error.SetErrorToErrno(); - goto FINISH; - } else if (WIFEXITED(status)) { - // open, dup or execve likely failed for some reason. - args->m_error.SetErrorToGenericError(); - switch (WEXITSTATUS(status)) { - case ePtraceFailed: - args->m_error.SetErrorString("Child ptrace failed."); - break; - case eDupStdinFailed: - args->m_error.SetErrorString("Child open stdin failed."); - break; - case eDupStdoutFailed: - args->m_error.SetErrorString("Child open stdout failed."); - break; - case eDupStderrFailed: - args->m_error.SetErrorString("Child open stderr failed."); - break; - case eChdirFailed: - args->m_error.SetErrorString("Child failed to set working directory."); - break; - case eExecFailed: - args->m_error.SetErrorString("Child exec failed."); - break; - case eSetGidFailed: - args->m_error.SetErrorString("Child setgid failed."); - break; - default: - args->m_error.SetErrorString("Child returned unknown exit status."); - break; - } - goto FINISH; - } - assert(WIFSTOPPED(status) && wpid == (::pid_t)*pid && - "Could not sync with inferior process."); - -#ifdef notyet - // Have the child raise an event on exit. This is used to keep the child in - // limbo until it is destroyed. - if (PTRACE(PTRACE_SETOPTIONS, *pid, NULL, PTRACE_O_TRACEEXIT) < 0) { - args->m_error.SetErrorToErrno(); - goto FINISH; - } -#endif - // Release the master terminal descriptor and pass it off to the - // ProcessMonitor instance. Similarly stash the inferior pid. - monitor->m_terminal_fd = terminal.ReleasePrimaryFileDescriptor(); - monitor->m_pid = *pid; - - // Set the terminal fd to be in non blocking mode (it simplifies the - // implementation of ProcessFreeBSD::GetSTDOUT to have a non-blocking - // descriptor to read from). - if (!EnsureFDFlags(monitor->m_terminal_fd, O_NONBLOCK, args->m_error)) - goto FINISH; - - process.SendMessage(ProcessMessage::Attach(*pid)); - -FINISH: - return args->m_error.Success(); -} - -void ProcessMonitor::StartAttachOpThread(AttachArgs *args, - lldb_private::Status &error) { - static const char *g_thread_name = "freebsd.op"; - - if (m_operation_thread && m_operation_thread->IsJoinable()) - return; - - llvm::Expected<lldb_private::HostThread> operation_thread = - ThreadLauncher::LaunchThread(g_thread_name, AttachOpThread, args); - if (operation_thread) - m_operation_thread = *operation_thread; - else - error = operation_thread.takeError(); -} - -void *ProcessMonitor::AttachOpThread(void *arg) { - AttachArgs *args = static_cast<AttachArgs *>(arg); - - Attach(args); - - ServeOperation(args); - return NULL; -} - -void ProcessMonitor::Attach(AttachArgs *args) { - lldb::pid_t pid = args->m_pid; - - ProcessMonitor *monitor = args->m_monitor; - ProcessFreeBSD &process = monitor->GetProcess(); - - if (pid <= 1) { - args->m_error.SetErrorToGenericError(); - args->m_error.SetErrorString("Attaching to process 1 is not allowed."); - return; - } - - // Attach to the requested process. - if (PTRACE(PT_ATTACH, pid, NULL, 0) < 0) { - args->m_error.SetErrorToErrno(); - return; - } - - int status; - if ((status = waitpid(pid, NULL, 0)) < 0) { - args->m_error.SetErrorToErrno(); - return; - } - - process.SendMessage(ProcessMessage::Attach(pid)); -} - -size_t -ProcessMonitor::GetCurrentThreadIDs(std::vector<lldb::tid_t> &thread_ids) { - lwpid_t *tids; - int tdcnt; - - thread_ids.clear(); - - tdcnt = PTRACE(PT_GETNUMLWPS, m_pid, NULL, 0); - if (tdcnt <= 0) - return 0; - tids = (lwpid_t *)malloc(tdcnt * sizeof(*tids)); - if (tids == NULL) - return 0; - if (PTRACE(PT_GETLWPLIST, m_pid, (void *)tids, tdcnt) < 0) { - free(tids); - return 0; - } - thread_ids = std::vector<lldb::tid_t>(tids, tids + tdcnt); - free(tids); - return thread_ids.size(); -} - -bool ProcessMonitor::MonitorCallback(ProcessMonitor *monitor, lldb::pid_t pid, - bool exited, int signal, int status) { - ProcessMessage message; - ProcessFreeBSD *process = monitor->m_process; - assert(process); - bool stop_monitoring; - struct ptrace_lwpinfo plwp; - int ptrace_err; - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - - if (exited) { - LLDB_LOGF(log, "ProcessMonitor::%s() got exit signal, tid = %" PRIu64, - __FUNCTION__, pid); - message = ProcessMessage::Exit(pid, status); - process->SendMessage(message); - return pid == process->GetID(); - } - - if (!monitor->GetLwpInfo(pid, &plwp, ptrace_err)) - stop_monitoring = true; // pid is gone. Bail. - else { - switch (plwp.pl_siginfo.si_signo) { - case SIGTRAP: - message = MonitorSIGTRAP(monitor, &plwp.pl_siginfo, plwp.pl_lwpid); - break; - - default: - message = MonitorSignal(monitor, &plwp.pl_siginfo, plwp.pl_lwpid); - break; - } - - process->SendMessage(message); - stop_monitoring = message.GetKind() == ProcessMessage::eExitMessage; - } - - return stop_monitoring; -} - -ProcessMessage ProcessMonitor::MonitorSIGTRAP(ProcessMonitor *monitor, - const siginfo_t *info, - lldb::tid_t tid) { - ProcessMessage message; - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - - assert(monitor); - assert(info && info->si_signo == SIGTRAP && "Unexpected child signal!"); - - switch (info->si_code) { - default: - assert(false && "Unexpected SIGTRAP code!"); - break; - - case (SIGTRAP /* | (PTRACE_EVENT_EXIT << 8) */): { - // The inferior process is about to exit. Maintain the process in a state - // of "limbo" until we are explicitly commanded to detach, destroy, resume, - // etc. - unsigned long data = 0; - if (!monitor->GetEventMessage(tid, &data)) - data = -1; - LLDB_LOGF(log, - "ProcessMonitor::%s() received exit? event, data = %lx, tid " - "= %" PRIu64, - __FUNCTION__, data, tid); - message = ProcessMessage::Limbo(tid, (data >> 8)); - break; - } - - case 0: - case TRAP_TRACE: -#ifdef TRAP_CAP - // Map TRAP_CAP to a trace trap in the absense of a more specific handler. - case TRAP_CAP: -#endif - LLDB_LOGF(log, - "ProcessMonitor::%s() received trace event, tid = %" PRIu64 - " : si_code = %d", - __FUNCTION__, tid, info->si_code); - message = ProcessMessage::Trace(tid); - break; - - case SI_KERNEL: - case TRAP_BRKPT: - if (monitor->m_process->IsSoftwareStepBreakpoint(tid)) { - LLDB_LOGF(log, - "ProcessMonitor::%s() received sw single step breakpoint " - "event, tid = %" PRIu64, - __FUNCTION__, tid); - message = ProcessMessage::Trace(tid); - } else { - LLDB_LOGF( - log, "ProcessMonitor::%s() received breakpoint event, tid = %" PRIu64, - __FUNCTION__, tid); - message = ProcessMessage::Break(tid); - } - break; - } - - return message; -} - -ProcessMessage ProcessMonitor::MonitorSignal(ProcessMonitor *monitor, - const siginfo_t *info, - lldb::tid_t tid) { - ProcessMessage message; - int signo = info->si_signo; - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - - // POSIX says that process behaviour is undefined after it ignores a SIGFPE, - // SIGILL, SIGSEGV, or SIGBUS *unless* that signal was generated by a kill(2) - // or raise(3). Similarly for tgkill(2) on FreeBSD. - // - // IOW, user generated signals never generate what we consider to be a - // "crash". - // - // Similarly, ACK signals generated by this monitor. - if (info->si_code == SI_USER) { - LLDB_LOGF(log, - "ProcessMonitor::%s() received signal %s with code %s, pid = %d", - __FUNCTION__, - monitor->m_process->GetUnixSignals()->GetSignalAsCString(signo), - "SI_USER", info->si_pid); - if (info->si_pid == getpid()) - return ProcessMessage::SignalDelivered(tid, signo); - else - return ProcessMessage::Signal(tid, signo); - } - - LLDB_LOGF(log, "ProcessMonitor::%s() received signal %s", __FUNCTION__, - monitor->m_process->GetUnixSignals()->GetSignalAsCString(signo)); - - switch (signo) { - case SIGSEGV: - case SIGILL: - case SIGFPE: - case SIGBUS: - lldb::addr_t fault_addr = reinterpret_cast<lldb::addr_t>(info->si_addr); - const auto reason = GetCrashReason(*info); - if (reason != CrashReason::eInvalidCrashReason) { - return ProcessMessage::Crash(tid, reason, signo, fault_addr); - } // else; Use atleast si_signo info for other si_code - } - - // Everything else is "normal" and does not require any special action on our - // part. - return ProcessMessage::Signal(tid, signo); -} - -void ProcessMonitor::ServeOperation(OperationArgs *args) { - ProcessMonitor *monitor = args->m_monitor; - - // We are finised with the arguments and are ready to go. Sync with the - // parent thread and start serving operations on the inferior. - sem_post(&args->m_semaphore); - - for (;;) { - // wait for next pending operation - sem_wait(&monitor->m_operation_pending); - - monitor->m_operation->Execute(monitor); - - // notify calling thread that operation is complete - sem_post(&monitor->m_operation_done); - } -} - -void ProcessMonitor::DoOperation(Operation *op) { - std::lock_guard<std::mutex> guard(m_operation_mutex); - - m_operation = op; - - // notify operation thread that an operation is ready to be processed - sem_post(&m_operation_pending); - - // wait for operation to complete - sem_wait(&m_operation_done); -} - -size_t ProcessMonitor::ReadMemory(lldb::addr_t vm_addr, void *buf, size_t size, - Status &error) { - size_t result; - ReadOperation op(vm_addr, buf, size, error, result); - DoOperation(&op); - return result; -} - -size_t ProcessMonitor::WriteMemory(lldb::addr_t vm_addr, const void *buf, - size_t size, lldb_private::Status &error) { - size_t result; - WriteOperation op(vm_addr, buf, size, error, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::ReadRegisterValue(lldb::tid_t tid, unsigned offset, - const char *reg_name, unsigned size, - RegisterValue &value) { - bool result; - ReadRegOperation op(tid, offset, size, value, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::WriteRegisterValue(lldb::tid_t tid, unsigned offset, - const char *reg_name, - const RegisterValue &value) { - bool result; - WriteRegOperation op(tid, offset, value, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::ReadDebugRegisterValue( - lldb::tid_t tid, unsigned offset, const char *reg_name, unsigned size, - lldb_private::RegisterValue &value) { - bool result; - ReadDebugRegOperation op(tid, offset, size, value, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::WriteDebugRegisterValue( - lldb::tid_t tid, unsigned offset, const char *reg_name, - const lldb_private::RegisterValue &value) { - bool result; - WriteDebugRegOperation op(tid, offset, value, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::ReadGPR(lldb::tid_t tid, void *buf, size_t buf_size) { - bool result; - ReadGPROperation op(tid, buf, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::ReadFPR(lldb::tid_t tid, void *buf, size_t buf_size) { - bool result; - ReadFPROperation op(tid, buf, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::ReadRegisterSet(lldb::tid_t tid, void *buf, - size_t buf_size, unsigned int regset) { - return false; -} - -bool ProcessMonitor::WriteGPR(lldb::tid_t tid, void *buf, size_t buf_size) { - bool result; - WriteGPROperation op(tid, buf, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::WriteFPR(lldb::tid_t tid, void *buf, size_t buf_size) { - bool result; - WriteFPROperation op(tid, buf, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::WriteRegisterSet(lldb::tid_t tid, void *buf, - size_t buf_size, unsigned int regset) { - return false; -} - -bool ProcessMonitor::ReadThreadPointer(lldb::tid_t tid, lldb::addr_t &value) { - return false; -} - -bool ProcessMonitor::Resume(lldb::tid_t unused, uint32_t signo) { - bool result; - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - - if (log) { - const char *signame = - m_process->GetUnixSignals()->GetSignalAsCString(signo); - if (signame == nullptr) - signame = "<none>"; - LLDB_LOGF(log, - "ProcessMonitor::%s() resuming pid %" PRIu64 " with signal %s", - __FUNCTION__, GetPID(), signame); - } - ResumeOperation op(signo, result); - DoOperation(&op); - LLDB_LOGF(log, "ProcessMonitor::%s() resuming result = %s", __FUNCTION__, - result ? "true" : "false"); - return result; -} - -bool ProcessMonitor::SingleStep(lldb::tid_t unused, uint32_t signo) { - bool result; - SingleStepOperation op(signo, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::Kill() { - bool result; - KillOperation op(result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::GetLwpInfo(lldb::tid_t tid, void *lwpinfo, - int &ptrace_err) { - bool result; - LwpInfoOperation op(tid, lwpinfo, result, ptrace_err); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::ThreadSuspend(lldb::tid_t tid, bool suspend) { - bool result; - ThreadSuspendOperation op(tid, suspend, result); - DoOperation(&op); - return result; -} - -bool ProcessMonitor::GetEventMessage(lldb::tid_t tid, unsigned long *message) { - bool result; - EventMessageOperation op(tid, message, result); - DoOperation(&op); - return result; -} - -lldb_private::Status ProcessMonitor::Detach(lldb::tid_t tid) { - lldb_private::Status error; - if (tid != LLDB_INVALID_THREAD_ID) { - DetachOperation op(error); - DoOperation(&op); - } - return error; -} - -bool ProcessMonitor::DupDescriptor(const FileSpec &file_spec, int fd, - int flags) { - int target_fd = llvm::sys::RetryAfterSignal(-1, open, - file_spec.GetCString(), flags, 0666); - - if (target_fd == -1) - return false; - - if (dup2(target_fd, fd) == -1) - return false; - - return (close(target_fd) == -1) ? false : true; -} - -void ProcessMonitor::StopMonitoringChildProcess() { - if (m_monitor_thread && m_monitor_thread->IsJoinable()) { - m_monitor_thread->Cancel(); - m_monitor_thread->Join(nullptr); - m_monitor_thread->Reset(); - } -} - -void ProcessMonitor::StopMonitor() { - StopMonitoringChildProcess(); - StopOpThread(); - sem_destroy(&m_operation_pending); - sem_destroy(&m_operation_done); - if (m_terminal_fd >= 0) { - close(m_terminal_fd); - m_terminal_fd = -1; - } -} - -// FIXME: On Linux, when a new thread is created, we receive to notifications, -// (1) a SIGTRAP|PTRACE_EVENT_CLONE from the main process thread with the child -// thread id as additional information, and (2) a SIGSTOP|SI_USER from the new -// child thread indicating that it has is stopped because we attached. We have -// no guarantee of the order in which these arrive, but we need both before we -// are ready to proceed. We currently keep a list of threads which have sent -// the initial SIGSTOP|SI_USER event. Then when we receive the -// SIGTRAP|PTRACE_EVENT_CLONE notification, if the initial stop has not -// occurred we call ProcessMonitor::WaitForInitialTIDStop() to wait for it. -// -// Right now, the above logic is in ProcessPOSIX, so we need a definition of -// this function in the FreeBSD ProcessMonitor implementation even if it isn't -// logically needed. -// -// We really should figure out what actually happens on FreeBSD and move the -// Linux-specific logic out of ProcessPOSIX as needed. - -bool ProcessMonitor::WaitForInitialTIDStop(lldb::tid_t tid) { return true; } - -void ProcessMonitor::StopOpThread() { - if (m_operation_thread && m_operation_thread->IsJoinable()) { - m_operation_thread->Cancel(); - m_operation_thread->Join(nullptr); - m_operation_thread->Reset(); - } -} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.h deleted file mode 100644 index c5edfc0be95a..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.h +++ /dev/null @@ -1,279 +0,0 @@ -//===-- ProcessMonitor.h -------------------------------------- -*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_ProcessMonitor_H_ -#define liblldb_ProcessMonitor_H_ - -#include <semaphore.h> -#include <signal.h> - -#include <mutex> - -#include "lldb/Host/HostThread.h" -#include "lldb/Utility/FileSpec.h" -#include "lldb/lldb-types.h" - -namespace lldb_private { -class Status; -class Module; -class Scalar; -} // End lldb_private namespace. - -class ProcessFreeBSD; -class Operation; - -/// \class ProcessMonitor -/// Manages communication with the inferior (debugee) process. -/// -/// Upon construction, this class prepares and launches an inferior process -/// for debugging. -/// -/// Changes in the inferior process state are propagated to the associated -/// ProcessFreeBSD instance by calling ProcessFreeBSD::SendMessage with the -/// appropriate ProcessMessage events. -/// -/// A purposely minimal set of operations are provided to interrogate and change -/// the inferior process state. -class ProcessMonitor { -public: - /// Launches an inferior process ready for debugging. Forms the - /// implementation of Process::DoLaunch. - ProcessMonitor(ProcessFreeBSD *process, lldb_private::Module *module, - char const *argv[], lldb_private::Environment env, - const lldb_private::FileSpec &stdin_file_spec, - const lldb_private::FileSpec &stdout_file_spec, - const lldb_private::FileSpec &stderr_file_spec, - const lldb_private::FileSpec &working_dir, - const lldb_private::ProcessLaunchInfo &launch_info, - lldb_private::Status &error); - - ProcessMonitor(ProcessFreeBSD *process, lldb::pid_t pid, - lldb_private::Status &error); - - ~ProcessMonitor(); - - /// Provides the process number of debugee. - lldb::pid_t GetPID() const { return m_pid; } - - /// Returns the process associated with this ProcessMonitor. - ProcessFreeBSD &GetProcess() { return *m_process; } - - /// Returns a file descriptor to the controlling terminal of the inferior - /// process. - /// - /// Reads from this file descriptor yield both the standard output and - /// standard error of this debugee. Even if stderr and stdout were - /// redirected on launch it may still happen that data is available on this - /// descriptor (if the inferior process opens /dev/tty, for example). This - /// descriptor is closed after a call to StopMonitor(). - /// - /// If this monitor was attached to an existing process this method returns - /// -1. - int GetTerminalFD() const { return m_terminal_fd; } - - /// Reads \p size bytes from address @vm_adder in the inferior process - /// address space. - /// - /// This method is provided to implement Process::DoReadMemory. - size_t ReadMemory(lldb::addr_t vm_addr, void *buf, size_t size, - lldb_private::Status &error); - - /// Writes \p size bytes from address \p vm_adder in the inferior process - /// address space. - /// - /// This method is provided to implement Process::DoWriteMemory. - size_t WriteMemory(lldb::addr_t vm_addr, const void *buf, size_t size, - lldb_private::Status &error); - - /// Reads the contents from the register identified by the given - /// (architecture dependent) offset. - /// - /// This method is provided for use by RegisterContextFreeBSD derivatives. - bool ReadRegisterValue(lldb::tid_t tid, unsigned offset, const char *reg_name, - unsigned size, lldb_private::RegisterValue &value); - - /// Writes the given value to the register identified by the given - /// (architecture dependent) offset. - /// - /// This method is provided for use by RegisterContextFreeBSD derivatives. - bool WriteRegisterValue(lldb::tid_t tid, unsigned offset, - const char *reg_name, - const lldb_private::RegisterValue &value); - - /// Reads the contents from the debug register identified by the given - /// (architecture dependent) offset. - /// - /// This method is provided for use by RegisterContextFreeBSD derivatives. - bool ReadDebugRegisterValue(lldb::tid_t tid, unsigned offset, - const char *reg_name, unsigned size, - lldb_private::RegisterValue &value); - - /// Writes the given value to the debug register identified by the given - /// (architecture dependent) offset. - /// - /// This method is provided for use by RegisterContextFreeBSD derivatives. - bool WriteDebugRegisterValue(lldb::tid_t tid, unsigned offset, - const char *reg_name, - const lldb_private::RegisterValue &value); - /// Reads all general purpose registers into the specified buffer. - bool ReadGPR(lldb::tid_t tid, void *buf, size_t buf_size); - - /// Reads all floating point registers into the specified buffer. - bool ReadFPR(lldb::tid_t tid, void *buf, size_t buf_size); - - /// Reads the specified register set into the specified buffer. - /// - /// This method is provided for use by RegisterContextFreeBSD derivatives. - bool ReadRegisterSet(lldb::tid_t tid, void *buf, size_t buf_size, - unsigned int regset); - - /// Writes all general purpose registers into the specified buffer. - bool WriteGPR(lldb::tid_t tid, void *buf, size_t buf_size); - - /// Writes all floating point registers into the specified buffer. - bool WriteFPR(lldb::tid_t tid, void *buf, size_t buf_size); - - /// Writes the specified register set into the specified buffer. - /// - /// This method is provided for use by RegisterContextFreeBSD derivatives. - bool WriteRegisterSet(lldb::tid_t tid, void *buf, size_t buf_size, - unsigned int regset); - - /// Reads the value of the thread-specific pointer for a given thread ID. - bool ReadThreadPointer(lldb::tid_t tid, lldb::addr_t &value); - - /// Returns current thread IDs in process - size_t GetCurrentThreadIDs(std::vector<lldb::tid_t> &thread_ids); - - /// Writes a ptrace_lwpinfo structure corresponding to the given thread ID - /// to the memory region pointed to by \p lwpinfo. - bool GetLwpInfo(lldb::tid_t tid, void *lwpinfo, int &error_no); - - /// Suspends or unsuspends a thread prior to process resume or step. - bool ThreadSuspend(lldb::tid_t tid, bool suspend); - - /// Writes the raw event message code (vis-a-vis PTRACE_GETEVENTMSG) - /// corresponding to the given thread IDto the memory pointed to by @p - /// message. - bool GetEventMessage(lldb::tid_t tid, unsigned long *message); - - /// Resumes the process. If \p signo is anything but - /// LLDB_INVALID_SIGNAL_NUMBER, deliver that signal to the process. - bool Resume(lldb::tid_t unused, uint32_t signo); - - /// Single steps the process. If \p signo is anything but - /// LLDB_INVALID_SIGNAL_NUMBER, deliver that signal to the process. - bool SingleStep(lldb::tid_t unused, uint32_t signo); - - /// Terminate the traced process. - bool Kill(); - - lldb_private::Status Detach(lldb::tid_t tid); - - void StopMonitor(); - - // Waits for the initial stop message from a new thread. - bool WaitForInitialTIDStop(lldb::tid_t tid); - -private: - ProcessFreeBSD *m_process; - - llvm::Optional<lldb_private::HostThread> m_operation_thread; - llvm::Optional<lldb_private::HostThread> m_monitor_thread; - lldb::pid_t m_pid; - - int m_terminal_fd; - - // current operation which must be executed on the privileged thread - Operation *m_operation; - std::mutex m_operation_mutex; - - // semaphores notified when Operation is ready to be processed and when - // the operation is complete. - sem_t m_operation_pending; - sem_t m_operation_done; - - struct OperationArgs { - OperationArgs(ProcessMonitor *monitor); - - ~OperationArgs(); - - ProcessMonitor *m_monitor; // The monitor performing the attach. - sem_t m_semaphore; // Posted to once operation complete. - lldb_private::Status m_error; // Set if process operation failed. - }; - - /// \class LauchArgs - /// - /// Simple structure to pass data to the thread responsible for launching a - /// child process. - struct LaunchArgs : OperationArgs { - LaunchArgs(ProcessMonitor *monitor, lldb_private::Module *module, - char const **argv, lldb_private::Environment env, - const lldb_private::FileSpec &stdin_file_spec, - const lldb_private::FileSpec &stdout_file_spec, - const lldb_private::FileSpec &stderr_file_spec, - const lldb_private::FileSpec &working_dir); - - ~LaunchArgs(); - - lldb_private::Module *m_module; // The executable image to launch. - char const **m_argv; // Process arguments. - lldb_private::Environment m_env; // Process environment. - const lldb_private::FileSpec m_stdin_file_spec; // Redirect stdin or empty. - const lldb_private::FileSpec - m_stdout_file_spec; // Redirect stdout or empty. - const lldb_private::FileSpec - m_stderr_file_spec; // Redirect stderr or empty. - const lldb_private::FileSpec m_working_dir; // Working directory or empty. - }; - - void StartLaunchOpThread(LaunchArgs *args, lldb_private::Status &error); - - static void *LaunchOpThread(void *arg); - - static bool Launch(LaunchArgs *args); - - struct AttachArgs : OperationArgs { - AttachArgs(ProcessMonitor *monitor, lldb::pid_t pid); - - ~AttachArgs(); - - lldb::pid_t m_pid; // pid of the process to be attached. - }; - - void StartAttachOpThread(AttachArgs *args, lldb_private::Status &error); - - static void *AttachOpThread(void *args); - - static void Attach(AttachArgs *args); - - static void ServeOperation(OperationArgs *args); - - static bool DupDescriptor(const lldb_private::FileSpec &file_spec, int fd, - int flags); - - static bool MonitorCallback(ProcessMonitor *monitor, lldb::pid_t pid, - bool exited, int signal, int status); - - static ProcessMessage MonitorSIGTRAP(ProcessMonitor *monitor, - const siginfo_t *info, lldb::pid_t pid); - - static ProcessMessage MonitorSignal(ProcessMonitor *monitor, - const siginfo_t *info, lldb::pid_t pid); - - void DoOperation(Operation *op); - - /// Stops the child monitor thread. - void StopMonitoringChildProcess(); - - /// Stops the operation thread used to attach/launch a process. - void StopOpThread(); -}; - -#endif // #ifndef liblldb_ProcessMonitor_H_ diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIX.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIX.h deleted file mode 100644 index cf52a065232c..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIX.h +++ /dev/null @@ -1,63 +0,0 @@ -//===-- RegisterContextPOSIX.h --------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_RegisterContextPOSIX_H_ -#define liblldb_RegisterContextPOSIX_H_ - -#include "Plugins/Process/Utility/RegisterInfoInterface.h" -#include "lldb/Target/RegisterContext.h" -#include "lldb/Utility/ArchSpec.h" - -/// \class POSIXBreakpointProtocol -/// -/// Extends RegisterClass with a few virtual operations useful on POSIX. -class POSIXBreakpointProtocol { -public: - POSIXBreakpointProtocol() { m_watchpoints_initialized = false; } - virtual ~POSIXBreakpointProtocol() {} - - /// Updates the register state of the associated thread after hitting a - /// breakpoint (if that make sense for the architecture). Default - /// implementation simply returns true for architectures which do not - /// require any update. - /// - /// \return - /// True if the operation succeeded and false otherwise. - virtual bool UpdateAfterBreakpoint() = 0; - - /// Determines the index in lldb's register file given a kernel byte offset. - virtual unsigned GetRegisterIndexFromOffset(unsigned offset) = 0; - - // Checks to see if a watchpoint specified by hw_index caused the inferior - // to stop. - virtual bool IsWatchpointHit(uint32_t hw_index) = 0; - - // Resets any watchpoints that have been hit. - virtual bool ClearWatchpointHits() = 0; - - // Returns the watchpoint address associated with a watchpoint hardware - // index. - virtual lldb::addr_t GetWatchpointAddress(uint32_t hw_index) = 0; - - virtual bool IsWatchpointVacant(uint32_t hw_index) = 0; - - virtual bool SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size, - bool read, bool write, - uint32_t hw_index) = 0; - - // From lldb_private::RegisterContext - virtual uint32_t NumSupportedHardwareWatchpoints() = 0; - - // Force m_watchpoints_initialized to TRUE - void ForceWatchpointsInitialized() { m_watchpoints_initialized = true; } - -protected: - bool m_watchpoints_initialized; -}; - -#endif // #ifndef liblldb_RegisterContextPOSIX_H_ diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm.cpp deleted file mode 100644 index afb92e848466..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm.cpp +++ /dev/null @@ -1,260 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_arm.cpp ------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===---------------------------------------------------------------------===// - -#include "lldb/Target/Thread.h" -#include "lldb/Utility/DataBufferHeap.h" -#include "lldb/Utility/RegisterValue.h" - -#include "ProcessFreeBSD.h" -#include "ProcessMonitor.h" -#include "RegisterContextPOSIXProcessMonitor_arm.h" -#include "Plugins/Process/Utility/RegisterContextPOSIX_arm.h" -#include "Plugins/Process/Utility/lldb-arm-register-enums.h" - -using namespace lldb_private; -using namespace lldb; - -#define REG_CONTEXT_SIZE (GetGPRSize()) - -RegisterContextPOSIXProcessMonitor_arm::RegisterContextPOSIXProcessMonitor_arm( - lldb_private::Thread &thread, - std::unique_ptr<RegisterInfoPOSIX_arm> register_info) - : RegisterContextPOSIX_arm(thread, std::move(register_info)) {} - -ProcessMonitor &RegisterContextPOSIXProcessMonitor_arm::GetMonitor() { - ProcessSP base = CalculateProcess(); - ProcessFreeBSD *process = static_cast<ProcessFreeBSD *>(base.get()); - return process->GetMonitor(); -} - -bool RegisterContextPOSIXProcessMonitor_arm::ReadGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadGPR(m_thread.GetID(), &m_gpr_arm, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_arm::ReadFPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadFPR(m_thread.GetID(), &m_fpr, sizeof(m_fpr)); -} - -bool RegisterContextPOSIXProcessMonitor_arm::WriteGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteGPR(m_thread.GetID(), &m_gpr_arm, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_arm::WriteFPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteFPR(m_thread.GetID(), &m_fpr, sizeof(m_fpr)); -} - -bool RegisterContextPOSIXProcessMonitor_arm::ReadRegister( - const unsigned reg, RegisterValue &value) { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadRegisterValue(m_thread.GetID(), GetRegisterOffset(reg), - GetRegisterName(reg), GetRegisterSize(reg), - value); -} - -bool RegisterContextPOSIXProcessMonitor_arm::WriteRegister( - const unsigned reg, const RegisterValue &value) { - unsigned reg_to_write = reg; - RegisterValue value_to_write = value; - - // Check if this is a subregister of a full register. - const RegisterInfo *reg_info = GetRegisterInfoAtIndex(reg); - if (reg_info->invalidate_regs && - (reg_info->invalidate_regs[0] != LLDB_INVALID_REGNUM)) { - RegisterValue full_value; - uint32_t full_reg = reg_info->invalidate_regs[0]; - const RegisterInfo *full_reg_info = GetRegisterInfoAtIndex(full_reg); - - // Read the full register. - if (ReadRegister(full_reg_info, full_value)) { - Status error; - ByteOrder byte_order = GetByteOrder(); - uint8_t dst[RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the full register. - const uint32_t dest_size = full_value.GetAsMemoryData( - full_reg_info, dst, sizeof(dst), byte_order, error); - if (error.Success() && dest_size) { - uint8_t src[RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the source data. - const uint32_t src_size = value.GetAsMemoryData( - reg_info, src, sizeof(src), byte_order, error); - if (error.Success() && src_size && (src_size < dest_size)) { - // Copy the src bytes to the destination. - memcpy(dst + (reg_info->byte_offset & 0x1), src, src_size); - // Set this full register as the value to write. - value_to_write.SetBytes(dst, full_value.GetByteSize(), byte_order); - value_to_write.SetType(full_reg_info); - reg_to_write = full_reg; - } - } - } - } - - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteRegisterValue( - m_thread.GetID(), GetRegisterOffset(reg_to_write), - GetRegisterName(reg_to_write), value_to_write); -} - -bool RegisterContextPOSIXProcessMonitor_arm::ReadRegister( - const RegisterInfo *reg_info, RegisterValue &value) { - if (!reg_info) - return false; - - const uint32_t reg = reg_info->kinds[eRegisterKindLLDB]; - - if (IsFPR(reg)) { - if (!ReadFPR()) - return false; - } else { - return ReadRegister(reg, value); - } - - // Get pointer to m_fpr variable and set the data from it. - assert(reg_info->byte_offset < sizeof m_fpr); - uint8_t *src = (uint8_t *)&m_fpr + reg_info->byte_offset; - switch (reg_info->byte_size) { - case 2: - value.SetUInt16(*(uint16_t *)src); - return true; - case 4: - value.SetUInt32(*(uint32_t *)src); - return true; - case 8: - value.SetUInt64(*(uint64_t *)src); - return true; - default: - assert(false && "Unhandled data size."); - return false; - } -} - -bool RegisterContextPOSIXProcessMonitor_arm::WriteRegister( - const RegisterInfo *reg_info, const RegisterValue &value) { - const uint32_t reg = reg_info->kinds[eRegisterKindLLDB]; - - if (IsGPR(reg)) { - return WriteRegister(reg, value); - } else if (IsFPR(reg)) { - return WriteFPR(); - } - - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm::ReadAllRegisterValues( - DataBufferSP &data_sp) { - bool success = false; - data_sp.reset(new DataBufferHeap(REG_CONTEXT_SIZE, 0)); - if (ReadGPR() && ReadFPR()) { - uint8_t *dst = data_sp->GetBytes(); - success = dst != 0; - - if (success) { - ::memcpy(dst, &m_gpr_arm, GetGPRSize()); - dst += GetGPRSize(); - ::memcpy(dst, &m_fpr, sizeof(m_fpr)); - } - } - return success; -} - -bool RegisterContextPOSIXProcessMonitor_arm::WriteAllRegisterValues( - const DataBufferSP &data_sp) { - bool success = false; - if (data_sp && data_sp->GetByteSize() == REG_CONTEXT_SIZE) { - uint8_t *src = data_sp->GetBytes(); - if (src) { - ::memcpy(&m_gpr_arm, src, GetGPRSize()); - - if (WriteGPR()) { - src += GetGPRSize(); - ::memcpy(&m_fpr, src, sizeof(m_fpr)); - - success = WriteFPR(); - } - } - } - return success; -} - -uint32_t RegisterContextPOSIXProcessMonitor_arm::SetHardwareWatchpoint( - addr_t addr, size_t size, bool read, bool write) { - const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints(); - uint32_t hw_index; - - for (hw_index = 0; hw_index < num_hw_watchpoints; ++hw_index) { - if (IsWatchpointVacant(hw_index)) - return SetHardwareWatchpointWithIndex(addr, size, read, write, hw_index); - } - - return LLDB_INVALID_INDEX32; -} - -bool RegisterContextPOSIXProcessMonitor_arm::ClearHardwareWatchpoint( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm::HardwareSingleStep(bool enable) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm::UpdateAfterBreakpoint() { - lldb::addr_t pc; - - if ((pc = GetPC()) == LLDB_INVALID_ADDRESS) - return false; - - return true; -} - -unsigned RegisterContextPOSIXProcessMonitor_arm::GetRegisterIndexFromOffset( - unsigned offset) { - unsigned reg; - for (reg = 0; reg < k_num_registers_arm; reg++) { - if (GetRegisterInfo()[reg].byte_offset == offset) - break; - } - assert(reg < k_num_registers_arm && "Invalid register offset."); - return reg; -} - -bool RegisterContextPOSIXProcessMonitor_arm::IsWatchpointHit( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm::ClearWatchpointHits() { - return false; -} - -addr_t RegisterContextPOSIXProcessMonitor_arm::GetWatchpointAddress( - uint32_t hw_index) { - return LLDB_INVALID_ADDRESS; -} - -bool RegisterContextPOSIXProcessMonitor_arm::IsWatchpointVacant( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm::SetHardwareWatchpointWithIndex( - addr_t addr, size_t size, bool read, bool write, uint32_t hw_index) { - return false; -} - -uint32_t -RegisterContextPOSIXProcessMonitor_arm::NumSupportedHardwareWatchpoints() { - return 0; -} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm.h deleted file mode 100644 index bb455841dff1..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm.h +++ /dev/null @@ -1,80 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_arm.h --------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_RegisterContextPOSIXProcessMonitor_arm_H_ -#define liblldb_RegisterContextPOSIXProcessMonitor_arm_H_ - -#include "Plugins/Process/Utility/RegisterContextPOSIX_arm.h" -#include "RegisterContextPOSIX.h" - -class RegisterContextPOSIXProcessMonitor_arm : public RegisterContextPOSIX_arm, - public POSIXBreakpointProtocol { -public: - RegisterContextPOSIXProcessMonitor_arm( - lldb_private::Thread &thread, - std::unique_ptr<RegisterInfoPOSIX_arm> register_info); - -protected: - bool ReadGPR() override; - - bool ReadFPR() override; - - bool WriteGPR() override; - - bool WriteFPR() override; - - // lldb_private::RegisterContext - bool ReadRegister(const unsigned reg, lldb_private::RegisterValue &value); - - bool WriteRegister(const unsigned reg, - const lldb_private::RegisterValue &value); - - bool ReadRegister(const lldb_private::RegisterInfo *reg_info, - lldb_private::RegisterValue &value) override; - - bool WriteRegister(const lldb_private::RegisterInfo *reg_info, - const lldb_private::RegisterValue &value) override; - - bool ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; - - bool WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; - - uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size, bool read, - bool write) override; - - bool ClearHardwareWatchpoint(uint32_t hw_index) override; - - bool HardwareSingleStep(bool enable) override; - - // POSIXBreakpointProtocol - bool UpdateAfterBreakpoint() override; - - unsigned GetRegisterIndexFromOffset(unsigned offset) override; - - bool IsWatchpointHit(uint32_t hw_index) override; - - bool ClearWatchpointHits() override; - - lldb::addr_t GetWatchpointAddress(uint32_t hw_index) override; - - bool IsWatchpointVacant(uint32_t hw_index) override; - - bool SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size, bool read, - bool write, uint32_t hw_index) override; - - uint32_t NumSupportedHardwareWatchpoints() override; - -private: - RegisterInfoPOSIX_arm::GPR m_gpr_arm; - - RegisterInfoPOSIX_arm::FPU m_fpr; - - ProcessMonitor &GetMonitor(); -}; - -#endif diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp deleted file mode 100644 index 39ae0b9b9e7f..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp +++ /dev/null @@ -1,267 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_arm64.cpp ----------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===---------------------------------------------------------------------===// - -#include "lldb/Target/Thread.h" -#include "lldb/Utility/DataBufferHeap.h" -#include "lldb/Utility/RegisterValue.h" - -#include "Plugins/Process/Utility/RegisterContextPOSIX_arm64.h" -#include "ProcessFreeBSD.h" -#include "ProcessMonitor.h" -#include "RegisterContextPOSIXProcessMonitor_arm64.h" - -#define REG_CONTEXT_SIZE (GetGPRSize()) - -using namespace lldb; -using namespace lldb_private; - -RegisterContextPOSIXProcessMonitor_arm64:: - RegisterContextPOSIXProcessMonitor_arm64( - lldb_private::Thread &thread, - std::unique_ptr<RegisterInfoPOSIX_arm64> register_info) - : RegisterContextPOSIX_arm64(thread, std::move(register_info)) { - ::memset(&m_gpr_arm64, 0, sizeof m_gpr_arm64); - ::memset(&m_fpr, 0, sizeof m_fpr); -} - -ProcessMonitor &RegisterContextPOSIXProcessMonitor_arm64::GetMonitor() { - lldb::ProcessSP base = CalculateProcess(); - ProcessFreeBSD *process = static_cast<ProcessFreeBSD *>(base.get()); - return process->GetMonitor(); -} - -bool RegisterContextPOSIXProcessMonitor_arm64::ReadGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadGPR(m_thread.GetID(), &m_gpr_arm64, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_arm64::ReadFPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadFPR(m_thread.GetID(), &m_fpr, sizeof m_fpr); -} - -bool RegisterContextPOSIXProcessMonitor_arm64::WriteGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteGPR(m_thread.GetID(), &m_gpr_arm64, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_arm64::WriteFPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteFPR(m_thread.GetID(), &m_fpr, sizeof m_fpr); -} - -bool RegisterContextPOSIXProcessMonitor_arm64::ReadRegister( - const unsigned reg, lldb_private::RegisterValue &value) { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadRegisterValue(m_thread.GetID(), GetRegisterOffset(reg), - GetRegisterName(reg), GetRegisterSize(reg), - value); -} - -bool RegisterContextPOSIXProcessMonitor_arm64::WriteRegister( - const unsigned reg, const lldb_private::RegisterValue &value) { - unsigned reg_to_write = reg; - lldb_private::RegisterValue value_to_write = value; - - // Check if this is a subregister of a full register. - const lldb_private::RegisterInfo *reg_info = GetRegisterInfoAtIndex(reg); - if (reg_info->invalidate_regs && - (reg_info->invalidate_regs[0] != LLDB_INVALID_REGNUM)) { - lldb_private::RegisterValue full_value; - uint32_t full_reg = reg_info->invalidate_regs[0]; - const lldb_private::RegisterInfo *full_reg_info = - GetRegisterInfoAtIndex(full_reg); - - // Read the full register. - if (ReadRegister(full_reg_info, full_value)) { - lldb_private::Status error; - lldb::ByteOrder byte_order = GetByteOrder(); - uint8_t dst[lldb_private::RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the full register. - const uint32_t dest_size = full_value.GetAsMemoryData( - full_reg_info, dst, sizeof(dst), byte_order, error); - if (error.Success() && dest_size) { - uint8_t src[lldb_private::RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the source data. - const uint32_t src_size = value.GetAsMemoryData( - reg_info, src, sizeof(src), byte_order, error); - if (error.Success() && src_size && (src_size < dest_size)) { - // Copy the src bytes to the destination. - ::memcpy(dst + (reg_info->byte_offset & 0x1), src, src_size); - // Set this full register as the value to write. - value_to_write.SetBytes(dst, full_value.GetByteSize(), byte_order); - value_to_write.SetType(full_reg_info); - reg_to_write = full_reg; - } - } - } - } - - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteRegisterValue( - m_thread.GetID(), GetRegisterOffset(reg_to_write), - GetRegisterName(reg_to_write), value_to_write); -} - -bool RegisterContextPOSIXProcessMonitor_arm64::ReadRegister( - const lldb_private::RegisterInfo *reg_info, - lldb_private::RegisterValue &value) { - if (!reg_info) - return false; - - const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; - - if (IsFPR(reg)) { - if (!ReadFPR()) - return false; - } else { - uint32_t full_reg = reg; - bool is_subreg = reg_info->invalidate_regs && - (reg_info->invalidate_regs[0] != LLDB_INVALID_REGNUM); - - if (is_subreg) { - // Read the full aligned 64-bit register. - full_reg = reg_info->invalidate_regs[0]; - } - return ReadRegister(full_reg, value); - } - - // Get pointer to m_fpr variable and set the data from it. - assert(reg_info->byte_offset < sizeof m_fpr); - uint8_t *src = (uint8_t *)&m_fpr + reg_info->byte_offset; - switch (reg_info->byte_size) { - case 2: - value.SetUInt16(*(uint16_t *)src); - return true; - case 4: - value.SetUInt32(*(uint32_t *)src); - return true; - case 8: - value.SetUInt64(*(uint64_t *)src); - return true; - default: - assert(false && "Unhandled data size."); - return false; - } -} - -bool RegisterContextPOSIXProcessMonitor_arm64::WriteRegister( - const lldb_private::RegisterInfo *reg_info, - const lldb_private::RegisterValue &value) { - const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; - - if (IsGPR(reg)) - return WriteRegister(reg, value); - - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::ReadAllRegisterValues( - lldb::DataBufferSP &data_sp) { - bool success = false; - data_sp.reset(new lldb_private::DataBufferHeap(REG_CONTEXT_SIZE, 0)); - if (ReadGPR() && ReadFPR()) { - uint8_t *dst = data_sp->GetBytes(); - success = dst != 0; - - if (success) { - ::memcpy(dst, &m_gpr_arm64, GetGPRSize()); - dst += GetGPRSize(); - ::memcpy(dst, &m_fpr, sizeof m_fpr); - } - } - return success; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::WriteAllRegisterValues( - const lldb::DataBufferSP &data_sp) { - bool success = false; - if (data_sp && data_sp->GetByteSize() == REG_CONTEXT_SIZE) { - uint8_t *src = data_sp->GetBytes(); - if (src) { - ::memcpy(&m_gpr_arm64, src, GetGPRSize()); - if (WriteGPR()) { - src += GetGPRSize(); - ::memcpy(&m_fpr, src, sizeof m_fpr); - success = WriteFPR(); - } - } - } - return success; -} - -uint32_t RegisterContextPOSIXProcessMonitor_arm64::SetHardwareWatchpoint( - lldb::addr_t addr, size_t size, bool read, bool write) { - const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints(); - uint32_t hw_index; - - for (hw_index = 0; hw_index < num_hw_watchpoints; ++hw_index) { - if (IsWatchpointVacant(hw_index)) - return SetHardwareWatchpointWithIndex(addr, size, read, write, hw_index); - } - - return LLDB_INVALID_INDEX32; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::ClearHardwareWatchpoint( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::HardwareSingleStep(bool enable) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::UpdateAfterBreakpoint() { - if (GetPC() == LLDB_INVALID_ADDRESS) - return false; - - return true; -} - -unsigned RegisterContextPOSIXProcessMonitor_arm64::GetRegisterIndexFromOffset( - unsigned offset) { - unsigned reg; - for (reg = 0; reg < GetRegisterCount(); reg++) { - if (GetRegisterInfo()[reg].byte_offset == offset) - break; - } - assert(reg < GetRegisterCount() && "Invalid register offset."); - return reg; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::IsWatchpointHit( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::ClearWatchpointHits() { - return false; -} - -lldb::addr_t RegisterContextPOSIXProcessMonitor_arm64::GetWatchpointAddress( - uint32_t hw_index) { - return LLDB_INVALID_ADDRESS; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::IsWatchpointVacant( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_arm64::SetHardwareWatchpointWithIndex( - lldb::addr_t addr, size_t size, bool read, bool write, uint32_t hw_index) { - return false; -} - -uint32_t -RegisterContextPOSIXProcessMonitor_arm64::NumSupportedHardwareWatchpoints() { - return 0; -} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h deleted file mode 100644 index dcae1d46de9b..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h +++ /dev/null @@ -1,82 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_arm64.h --------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_RegisterContextPOSIXProcessMonitor_arm64_H_ -#define liblldb_RegisterContextPOSIXProcessMonitor_arm64_H_ - -#include "Plugins/Process/Utility/RegisterContextPOSIX_arm64.h" -#include "RegisterContextPOSIX.h" - -class RegisterContextPOSIXProcessMonitor_arm64 - : public RegisterContextPOSIX_arm64, - public POSIXBreakpointProtocol { -public: - RegisterContextPOSIXProcessMonitor_arm64( - lldb_private::Thread &thread, - std::unique_ptr<RegisterInfoPOSIX_arm64> register_info); - -protected: - bool ReadGPR() override; - - bool ReadFPR() override; - - bool WriteGPR() override; - - bool WriteFPR() override; - - // lldb_private::RegisterContext - bool ReadRegister(const unsigned reg, lldb_private::RegisterValue &value); - - bool WriteRegister(const unsigned reg, - const lldb_private::RegisterValue &value); - - bool ReadRegister(const lldb_private::RegisterInfo *reg_info, - lldb_private::RegisterValue &value) override; - - bool WriteRegister(const lldb_private::RegisterInfo *reg_info, - const lldb_private::RegisterValue &value) override; - - bool ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; - - bool WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; - - uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size, bool read, - bool write) override; - - bool ClearHardwareWatchpoint(uint32_t hw_index) override; - - bool HardwareSingleStep(bool enable) override; - - // POSIXBreakpointProtocol - bool UpdateAfterBreakpoint() override; - - unsigned GetRegisterIndexFromOffset(unsigned offset) override; - - bool IsWatchpointHit(uint32_t hw_index) override; - - bool ClearWatchpointHits() override; - - lldb::addr_t GetWatchpointAddress(uint32_t hw_index) override; - - bool IsWatchpointVacant(uint32_t hw_index) override; - - bool SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size, bool read, - bool write, uint32_t hw_index) override; - - uint32_t NumSupportedHardwareWatchpoints() override; - -private: - RegisterInfoPOSIX_arm64::GPR m_gpr_arm64; // 64-bit general purpose registers. - - RegisterInfoPOSIX_arm64::FPU - m_fpr; // floating-point registers including extended register sets. - - ProcessMonitor &GetMonitor(); -}; - -#endif diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_mips64.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_mips64.cpp deleted file mode 100644 index 23c76f234c8e..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_mips64.cpp +++ /dev/null @@ -1,262 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_mips64.cpp ---------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "lldb/Target/Thread.h" -#include "lldb/Utility/DataBufferHeap.h" -#include "lldb/Utility/RegisterValue.h" - -#include "Plugins/Process/Utility/RegisterContextPOSIX_mips64.h" -#include "ProcessFreeBSD.h" -#include "ProcessMonitor.h" -#include "RegisterContextPOSIXProcessMonitor_mips64.h" - -using namespace lldb_private; -using namespace lldb; - -#define REG_CONTEXT_SIZE (GetGPRSize()) - -RegisterContextPOSIXProcessMonitor_mips64:: - RegisterContextPOSIXProcessMonitor_mips64( - Thread &thread, uint32_t concrete_frame_idx, - lldb_private::RegisterInfoInterface *register_info) - : RegisterContextPOSIX_mips64(thread, concrete_frame_idx, register_info) {} - -ProcessMonitor &RegisterContextPOSIXProcessMonitor_mips64::GetMonitor() { - ProcessSP base = CalculateProcess(); - ProcessFreeBSD *process = static_cast<ProcessFreeBSD *>(base.get()); - return process->GetMonitor(); -} - -bool RegisterContextPOSIXProcessMonitor_mips64::ReadGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadGPR(m_thread.GetID(), &m_gpr_mips64, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_mips64::ReadFPR() { - // XXX not yet implemented - return false; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::WriteGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteGPR(m_thread.GetID(), &m_gpr_mips64, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_mips64::WriteFPR() { - // XXX not yet implemented - return false; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::ReadRegister( - const unsigned reg, RegisterValue &value) { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadRegisterValue(m_thread.GetID(), GetRegisterOffset(reg), - GetRegisterName(reg), GetRegisterSize(reg), - value); -} - -bool RegisterContextPOSIXProcessMonitor_mips64::WriteRegister( - const unsigned reg, const RegisterValue &value) { - unsigned reg_to_write = reg; - RegisterValue value_to_write = value; - - // Check if this is a subregister of a full register. - const RegisterInfo *reg_info = GetRegisterInfoAtIndex(reg); - if (reg_info->invalidate_regs && - (reg_info->invalidate_regs[0] != LLDB_INVALID_REGNUM)) { - RegisterValue full_value; - uint32_t full_reg = reg_info->invalidate_regs[0]; - const RegisterInfo *full_reg_info = GetRegisterInfoAtIndex(full_reg); - - // Read the full register. - if (ReadRegister(full_reg_info, full_value)) { - Status error; - ByteOrder byte_order = GetByteOrder(); - uint8_t dst[RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the full register. - const uint32_t dest_size = full_value.GetAsMemoryData( - full_reg_info, dst, sizeof(dst), byte_order, error); - if (error.Success() && dest_size) { - uint8_t src[RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the source data. - const uint32_t src_size = value.GetAsMemoryData( - reg_info, src, sizeof(src), byte_order, error); - if (error.Success() && src_size && (src_size < dest_size)) { - // Copy the src bytes to the destination. - memcpy(dst + (reg_info->byte_offset & 0x1), src, src_size); - // Set this full register as the value to write. - value_to_write.SetBytes(dst, full_value.GetByteSize(), byte_order); - value_to_write.SetType(full_reg_info); - reg_to_write = full_reg; - } - } - } - } - - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteRegisterValue( - m_thread.GetID(), GetRegisterOffset(reg_to_write), - GetRegisterName(reg_to_write), value_to_write); -} - -bool RegisterContextPOSIXProcessMonitor_mips64::ReadRegister( - const RegisterInfo *reg_info, RegisterValue &value) { - if (!reg_info) - return false; - - const uint32_t reg = reg_info->kinds[eRegisterKindLLDB]; - - if (IsFPR(reg)) { - if (!ReadFPR()) - return false; - } else { - uint32_t full_reg = reg; - bool is_subreg = reg_info->invalidate_regs && - (reg_info->invalidate_regs[0] != LLDB_INVALID_REGNUM); - - if (is_subreg) { - // Read the full aligned 64-bit register. - full_reg = reg_info->invalidate_regs[0]; - } - - bool success = ReadRegister(full_reg, value); - - if (success) { - // If our read was not aligned (for ah,bh,ch,dh), shift our returned - // value one byte to the right. - if (is_subreg && (reg_info->byte_offset & 0x1)) - value.SetUInt64(value.GetAsUInt64() >> 8); - - // If our return byte size was greater than the return value reg size, - // then use the type specified by reg_info rather than the uint64_t - // default - if (value.GetByteSize() > reg_info->byte_size) - value.SetType(reg_info); - } - return success; - } - - return false; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::WriteRegister( - const RegisterInfo *reg_info, const RegisterValue &value) { - const uint32_t reg = reg_info->kinds[eRegisterKindLLDB]; - - if (IsGPR(reg)) - return WriteRegister(reg, value); - - return false; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::ReadAllRegisterValues( - DataBufferSP &data_sp) { - bool success = false; - data_sp.reset(new DataBufferHeap(REG_CONTEXT_SIZE, 0)); - if (ReadGPR() && ReadFPR()) { - uint8_t *dst = data_sp->GetBytes(); - success = dst != 0; - - if (success) { - ::memcpy(dst, &m_gpr_mips64, GetGPRSize()); - } - } - return success; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::WriteAllRegisterValues( - const DataBufferSP &data_sp) { - bool success = false; - if (data_sp && data_sp->GetByteSize() == REG_CONTEXT_SIZE) { - uint8_t *src = data_sp->GetBytes(); - if (src) { - ::memcpy(&m_gpr_mips64, src, GetGPRSize()); - - if (WriteGPR()) { - src += GetGPRSize(); - } - } - } - return success; -} - -uint32_t RegisterContextPOSIXProcessMonitor_mips64::SetHardwareWatchpoint( - addr_t addr, size_t size, bool read, bool write) { - const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints(); - uint32_t hw_index; - - for (hw_index = 0; hw_index < num_hw_watchpoints; ++hw_index) { - if (IsWatchpointVacant(hw_index)) - return SetHardwareWatchpointWithIndex(addr, size, read, write, hw_index); - } - - return LLDB_INVALID_INDEX32; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::ClearHardwareWatchpoint( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::HardwareSingleStep( - bool enable) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::UpdateAfterBreakpoint() { - // PC points one byte past the int3 responsible for the breakpoint. - lldb::addr_t pc; - - if ((pc = GetPC()) == LLDB_INVALID_ADDRESS) - return false; - - SetPC(pc - 1); - return true; -} - -unsigned RegisterContextPOSIXProcessMonitor_mips64::GetRegisterIndexFromOffset( - unsigned offset) { - unsigned reg; - for (reg = 0; reg < k_num_registers_mips64; reg++) { - if (GetRegisterInfo()[reg].byte_offset == offset) - break; - } - assert(reg < k_num_registers_mips64 && "Invalid register offset."); - return reg; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::IsWatchpointHit( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::ClearWatchpointHits() { - return false; -} - -addr_t RegisterContextPOSIXProcessMonitor_mips64::GetWatchpointAddress( - uint32_t hw_index) { - return LLDB_INVALID_ADDRESS; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::IsWatchpointVacant( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_mips64::SetHardwareWatchpointWithIndex( - addr_t addr, size_t size, bool read, bool write, uint32_t hw_index) { - return false; -} - -uint32_t -RegisterContextPOSIXProcessMonitor_mips64::NumSupportedHardwareWatchpoints() { - return 0; -} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_mips64.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_mips64.h deleted file mode 100644 index be404cc08c34..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_mips64.h +++ /dev/null @@ -1,82 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_mips64.h -------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_RegisterContextPOSIXProcessMonitor_mips64_H_ -#define liblldb_RegisterContextPOSIXProcessMonitor_mips64_H_ - -#include "Plugins/Process/Utility/RegisterContextPOSIX_mips64.h" -#include "Plugins/Process/Utility/lldb-mips-freebsd-register-enums.h" -#include "RegisterContextPOSIX.h" - -class ProcessMonitor; - -class RegisterContextPOSIXProcessMonitor_mips64 - : public RegisterContextPOSIX_mips64, - public POSIXBreakpointProtocol { -public: - RegisterContextPOSIXProcessMonitor_mips64( - lldb_private::Thread &thread, uint32_t concrete_frame_idx, - lldb_private::RegisterInfoInterface *register_info); - -protected: - bool ReadGPR() override; - - bool ReadFPR() override; - - bool WriteGPR() override; - - bool WriteFPR() override; - - // lldb_private::RegisterContext - bool ReadRegister(const unsigned reg, lldb_private::RegisterValue &value); - - bool WriteRegister(const unsigned reg, - const lldb_private::RegisterValue &value); - - bool ReadRegister(const lldb_private::RegisterInfo *reg_info, - lldb_private::RegisterValue &value) override; - - bool WriteRegister(const lldb_private::RegisterInfo *reg_info, - const lldb_private::RegisterValue &value) override; - - bool ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; - - bool WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; - - uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size, bool read, - bool write) override; - - bool ClearHardwareWatchpoint(uint32_t hw_index) override; - - bool HardwareSingleStep(bool enable) override; - - // POSIXBreakpointProtocol - bool UpdateAfterBreakpoint() override; - - unsigned GetRegisterIndexFromOffset(unsigned offset) override; - - bool IsWatchpointHit(uint32_t hw_index) override; - - bool ClearWatchpointHits() override; - - lldb::addr_t GetWatchpointAddress(uint32_t hw_index) override; - - bool IsWatchpointVacant(uint32_t hw_index) override; - - bool SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size, bool read, - bool write, uint32_t hw_index) override; - - uint32_t NumSupportedHardwareWatchpoints() override; - -private: - uint64_t - m_gpr_mips64[lldb_private::k_num_gpr_registers_mips64]; // general purpose registers. - ProcessMonitor &GetMonitor(); -}; - -#endif diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_powerpc.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_powerpc.cpp deleted file mode 100644 index f8342775a81b..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_powerpc.cpp +++ /dev/null @@ -1,274 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_powerpc.cpp --------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "lldb/Target/Thread.h" -#include "lldb/Utility/DataBufferHeap.h" -#include "lldb/Utility/RegisterValue.h" - -#include "ProcessFreeBSD.h" -#include "ProcessMonitor.h" -#include "RegisterContextPOSIXProcessMonitor_powerpc.h" -#include "Plugins/Process/Utility/RegisterContextPOSIX_powerpc.h" - -using namespace lldb_private; -using namespace lldb; - -#define REG_CONTEXT_SIZE (GetGPRSize()) - -RegisterContextPOSIXProcessMonitor_powerpc:: - RegisterContextPOSIXProcessMonitor_powerpc( - Thread &thread, uint32_t concrete_frame_idx, - lldb_private::RegisterInfoInterface *register_info) - : RegisterContextPOSIX_powerpc(thread, concrete_frame_idx, register_info) {} - -ProcessMonitor &RegisterContextPOSIXProcessMonitor_powerpc::GetMonitor() { - ProcessSP base = CalculateProcess(); - ProcessFreeBSD *process = static_cast<ProcessFreeBSD *>(base.get()); - return process->GetMonitor(); -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::ReadGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadGPR(m_thread.GetID(), &m_gpr_powerpc, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::ReadFPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadFPR(m_thread.GetID(), &m_fpr_powerpc, - sizeof(m_fpr_powerpc)); -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::ReadVMX() { - // XXX: Need a way to read/write process VMX registers with ptrace. - return false; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::WriteGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteGPR(m_thread.GetID(), &m_gpr_powerpc, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::WriteFPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteFPR(m_thread.GetID(), &m_fpr_powerpc, - sizeof(m_fpr_powerpc)); -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::WriteVMX() { - // XXX: Need a way to read/write process VMX registers with ptrace. - return false; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::ReadRegister( - const unsigned reg, RegisterValue &value) { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadRegisterValue(m_thread.GetID(), GetRegisterOffset(reg), - GetRegisterName(reg), GetRegisterSize(reg), - value); -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::WriteRegister( - const unsigned reg, const RegisterValue &value) { - unsigned reg_to_write = reg; - RegisterValue value_to_write = value; - - // Check if this is a subregister of a full register. - const RegisterInfo *reg_info = GetRegisterInfoAtIndex(reg); - if (reg_info->invalidate_regs && - (reg_info->invalidate_regs[0] != LLDB_INVALID_REGNUM)) { - RegisterValue full_value; - uint32_t full_reg = reg_info->invalidate_regs[0]; - const RegisterInfo *full_reg_info = GetRegisterInfoAtIndex(full_reg); - - // Read the full register. - if (ReadRegister(full_reg_info, full_value)) { - Status error; - ByteOrder byte_order = GetByteOrder(); - uint8_t dst[RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the full register. - const uint32_t dest_size = full_value.GetAsMemoryData( - full_reg_info, dst, sizeof(dst), byte_order, error); - if (error.Success() && dest_size) { - uint8_t src[RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the source data. - const uint32_t src_size = value.GetAsMemoryData( - reg_info, src, sizeof(src), byte_order, error); - if (error.Success() && src_size && (src_size < dest_size)) { - // Copy the src bytes to the destination. - memcpy(dst + (reg_info->byte_offset & 0x1), src, src_size); - // Set this full register as the value to write. - value_to_write.SetBytes(dst, full_value.GetByteSize(), byte_order); - value_to_write.SetType(full_reg_info); - reg_to_write = full_reg; - } - } - } - } - - ProcessMonitor &monitor = GetMonitor(); - // Account for the fact that 32-bit targets on powerpc64 really use 64-bit - // registers in ptrace, but expose here 32-bit registers with a higher - // offset. - uint64_t offset = GetRegisterOffset(reg_to_write); - offset &= ~(sizeof(uintptr_t) - 1); - return monitor.WriteRegisterValue( - m_thread.GetID(), offset, GetRegisterName(reg_to_write), value_to_write); -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::ReadRegister( - const RegisterInfo *reg_info, RegisterValue &value) { - if (!reg_info) - return false; - - const uint32_t reg = reg_info->kinds[eRegisterKindLLDB]; - - if (IsFPR(reg)) { - if (!ReadFPR()) - return false; - uint8_t *src = (uint8_t *)&m_fpr_powerpc + reg_info->byte_offset; - value.SetUInt64(*(uint64_t *)src); - } else if (IsGPR(reg)) { - bool success = ReadRegister(reg, value); - - if (success) { - // If our return byte size was greater than the return value reg size, - // then use the type specified by reg_info rather than the uint64_t - // default - if (value.GetByteSize() > reg_info->byte_size) - value.SetType(reg_info); - } - return success; - } - - return false; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::WriteRegister( - const RegisterInfo *reg_info, const RegisterValue &value) { - const uint32_t reg = reg_info->kinds[eRegisterKindLLDB]; - - if (IsGPR(reg)) { - return WriteRegister(reg, value); - } else if (IsFPR(reg)) { - assert(reg_info->byte_offset < sizeof(m_fpr_powerpc)); - uint8_t *dst = (uint8_t *)&m_fpr_powerpc + reg_info->byte_offset; - *(uint64_t *)dst = value.GetAsUInt64(); - return WriteFPR(); - } - - return false; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::ReadAllRegisterValues( - DataBufferSP &data_sp) { - bool success = false; - data_sp.reset(new DataBufferHeap(REG_CONTEXT_SIZE, 0)); - if (ReadGPR() && ReadFPR()) { - uint8_t *dst = data_sp->GetBytes(); - success = dst != 0; - - if (success) { - ::memcpy(dst, &m_gpr_powerpc, GetGPRSize()); - dst += GetGPRSize(); - } - } - return success; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::WriteAllRegisterValues( - const DataBufferSP &data_sp) { - bool success = false; - if (data_sp && data_sp->GetByteSize() == REG_CONTEXT_SIZE) { - uint8_t *src = data_sp->GetBytes(); - if (src) { - ::memcpy(&m_gpr_powerpc, src, GetGPRSize()); - - if (WriteGPR()) { - src += GetGPRSize(); - ::memcpy(&m_fpr_powerpc, src, sizeof(m_fpr_powerpc)); - - success = WriteFPR(); - } - } - } - return success; -} - -uint32_t RegisterContextPOSIXProcessMonitor_powerpc::SetHardwareWatchpoint( - addr_t addr, size_t size, bool read, bool write) { - const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints(); - uint32_t hw_index; - - for (hw_index = 0; hw_index < num_hw_watchpoints; ++hw_index) { - if (IsWatchpointVacant(hw_index)) - return SetHardwareWatchpointWithIndex(addr, size, read, write, hw_index); - } - - return LLDB_INVALID_INDEX32; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::ClearHardwareWatchpoint( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::HardwareSingleStep( - bool enable) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::UpdateAfterBreakpoint() { - lldb::addr_t pc; - - if ((pc = GetPC()) == LLDB_INVALID_ADDRESS) - return false; - - return true; -} - -unsigned RegisterContextPOSIXProcessMonitor_powerpc::GetRegisterIndexFromOffset( - unsigned offset) { - unsigned reg; - for (reg = 0; reg < k_num_registers_powerpc; reg++) { - if (GetRegisterInfo()[reg].byte_offset == offset) - break; - } - assert(reg < k_num_registers_powerpc && "Invalid register offset."); - return reg; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::IsWatchpointHit( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::ClearWatchpointHits() { - return false; -} - -addr_t RegisterContextPOSIXProcessMonitor_powerpc::GetWatchpointAddress( - uint32_t hw_index) { - return LLDB_INVALID_ADDRESS; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::IsWatchpointVacant( - uint32_t hw_index) { - return false; -} - -bool RegisterContextPOSIXProcessMonitor_powerpc::SetHardwareWatchpointWithIndex( - addr_t addr, size_t size, bool read, bool write, uint32_t hw_index) { - return false; -} - -uint32_t -RegisterContextPOSIXProcessMonitor_powerpc::NumSupportedHardwareWatchpoints() { - return 0; -} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_powerpc.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_powerpc.h deleted file mode 100644 index 328db4479ce6..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_powerpc.h +++ /dev/null @@ -1,84 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_powerpc.h -------------*- C++ -//-*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_RegisterContextPOSIXProcessMonitor_powerpc_H_ -#define liblldb_RegisterContextPOSIXProcessMonitor_powerpc_H_ - -#include "Plugins/Process/Utility/RegisterContextPOSIX_powerpc.h" -#include "RegisterContextPOSIX.h" - -class RegisterContextPOSIXProcessMonitor_powerpc - : public RegisterContextPOSIX_powerpc, - public POSIXBreakpointProtocol { -public: - RegisterContextPOSIXProcessMonitor_powerpc( - lldb_private::Thread &thread, uint32_t concrete_frame_idx, - lldb_private::RegisterInfoInterface *register_info); - -protected: - bool IsVMX(); - - bool ReadGPR() override; - - bool ReadFPR() override; - - bool ReadVMX() override; - - bool WriteGPR() override; - - bool WriteFPR() override; - - bool WriteVMX() override; - - // lldb_private::RegisterContext - bool ReadRegister(const unsigned reg, lldb_private::RegisterValue &value); - - bool WriteRegister(const unsigned reg, - const lldb_private::RegisterValue &value); - - bool ReadRegister(const lldb_private::RegisterInfo *reg_info, - lldb_private::RegisterValue &value) override; - - bool WriteRegister(const lldb_private::RegisterInfo *reg_info, - const lldb_private::RegisterValue &value) override; - - bool ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; - - bool WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; - - uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size, bool read, - bool write) override; - - bool ClearHardwareWatchpoint(uint32_t hw_index) override; - - bool HardwareSingleStep(bool enable) override; - - // POSIXBreakpointProtocol - bool UpdateAfterBreakpoint() override; - - unsigned GetRegisterIndexFromOffset(unsigned offset) override; - - bool IsWatchpointHit(uint32_t hw_index) override; - - bool ClearWatchpointHits() override; - - lldb::addr_t GetWatchpointAddress(uint32_t hw_index) override; - - bool IsWatchpointVacant(uint32_t hw_index) override; - - bool SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size, bool read, - bool write, uint32_t hw_index) override; - - uint32_t NumSupportedHardwareWatchpoints() override; - -private: - ProcessMonitor &GetMonitor(); -}; - -#endif diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_x86.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_x86.cpp deleted file mode 100644 index b1739e1e3bd8..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_x86.cpp +++ /dev/null @@ -1,613 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_x86.cpp ------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "lldb/Target/Thread.h" -#include "lldb/Utility/DataBufferHeap.h" -#include "lldb/Utility/RegisterValue.h" - -#include "Plugins/Process/FreeBSD/ProcessFreeBSD.h" -#include "Plugins/Process/FreeBSD/ProcessMonitor.h" -#include "RegisterContextPOSIXProcessMonitor_x86.h" - -using namespace lldb_private; -using namespace lldb; - -// Support ptrace extensions even when compiled without required kernel support -#ifndef NT_X86_XSTATE -#define NT_X86_XSTATE 0x202 -#endif - -#define REG_CONTEXT_SIZE (GetGPRSize() + sizeof(FPR)) - -static uint32_t size_and_rw_bits(size_t size, bool read, bool write) { - uint32_t rw; - - if (read) - rw = 0x3; // READ or READ/WRITE - else if (write) - rw = 0x1; // WRITE - else - assert(0 && "read and write cannot both be false"); - - switch (size) { - case 1: - return rw; - case 2: - return (0x1 << 2) | rw; - case 4: - return (0x3 << 2) | rw; - case 8: - return (0x2 << 2) | rw; - default: - assert(0 && "invalid size, must be one of 1, 2, 4, or 8"); - return 0; // Unreachable. Just to silence compiler. - } -} - -RegisterContextPOSIXProcessMonitor_x86_64:: - RegisterContextPOSIXProcessMonitor_x86_64( - Thread &thread, uint32_t concrete_frame_idx, - lldb_private::RegisterInfoInterface *register_info) - : RegisterContextPOSIX_x86(thread, concrete_frame_idx, register_info) { - // Store byte offset of fctrl (i.e. first register of FPR) wrt 'UserArea' - const RegisterInfo *reg_info_fctrl = GetRegisterInfoByName("fctrl"); - m_fctrl_offset_in_userarea = reg_info_fctrl->byte_offset; - - m_iovec.iov_base = &m_fpr.xsave; - m_iovec.iov_len = sizeof(m_fpr.xsave); -} - -ProcessMonitor &RegisterContextPOSIXProcessMonitor_x86_64::GetMonitor() { - ProcessSP base = CalculateProcess(); - ProcessFreeBSD *process = static_cast<ProcessFreeBSD *>(base.get()); - return process->GetMonitor(); -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::ReadGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.ReadGPR(m_thread.GetID(), &m_gpr_x86_64, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::ReadFPR() { - ProcessMonitor &monitor = GetMonitor(); - if (GetFPRType() == eFXSAVE) - return monitor.ReadFPR(m_thread.GetID(), &m_fpr.fxsave, - sizeof(m_fpr.fxsave)); - - if (GetFPRType() == eXSAVE) - return monitor.ReadRegisterSet(m_thread.GetID(), &m_iovec, - sizeof(m_fpr.xsave), NT_X86_XSTATE); - return false; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::WriteGPR() { - ProcessMonitor &monitor = GetMonitor(); - return monitor.WriteGPR(m_thread.GetID(), &m_gpr_x86_64, GetGPRSize()); -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::WriteFPR() { - ProcessMonitor &monitor = GetMonitor(); - if (GetFPRType() == eFXSAVE) - return monitor.WriteFPR(m_thread.GetID(), &m_fpr.fxsave, - sizeof(m_fpr.fxsave)); - - if (GetFPRType() == eXSAVE) - return monitor.WriteRegisterSet(m_thread.GetID(), &m_iovec, - sizeof(m_fpr.xsave), NT_X86_XSTATE); - return false; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::ReadRegister( - const unsigned reg, RegisterValue &value) { - ProcessMonitor &monitor = GetMonitor(); - -#if defined(__FreeBSD__) - if (reg >= m_reg_info.first_dr) - return monitor.ReadDebugRegisterValue( - m_thread.GetID(), GetRegisterOffset(reg), GetRegisterName(reg), - GetRegisterSize(reg), value); -#endif - return monitor.ReadRegisterValue(m_thread.GetID(), GetRegisterOffset(reg), - GetRegisterName(reg), GetRegisterSize(reg), - value); -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::WriteRegister( - const unsigned reg, const RegisterValue &value) { - unsigned reg_to_write = reg; - RegisterValue value_to_write = value; - - // Check if this is a subregister of a full register. - const RegisterInfo *reg_info = GetRegisterInfoAtIndex(reg); - if (reg_info->invalidate_regs && - (reg_info->invalidate_regs[0] != LLDB_INVALID_REGNUM)) { - RegisterValue full_value; - uint32_t full_reg = reg_info->invalidate_regs[0]; - const RegisterInfo *full_reg_info = GetRegisterInfoAtIndex(full_reg); - - // Read the full register. - if (ReadRegister(full_reg_info, full_value)) { - Status error; - ByteOrder byte_order = GetByteOrder(); - uint8_t dst[RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the full register. - const uint32_t dest_size = full_value.GetAsMemoryData( - full_reg_info, dst, sizeof(dst), byte_order, error); - if (error.Success() && dest_size) { - uint8_t src[RegisterValue::kMaxRegisterByteSize]; - - // Get the bytes for the source data. - const uint32_t src_size = value.GetAsMemoryData( - reg_info, src, sizeof(src), byte_order, error); - if (error.Success() && src_size && (src_size < dest_size)) { - // Copy the src bytes to the destination. - memcpy(dst + (reg_info->byte_offset & 0x1), src, src_size); - // Set this full register as the value to write. - value_to_write.SetBytes(dst, full_value.GetByteSize(), byte_order); - value_to_write.SetType(full_reg_info); - reg_to_write = full_reg; - } - } - } - } - - ProcessMonitor &monitor = GetMonitor(); -#if defined(__FreeBSD__) - if (reg >= m_reg_info.first_dr) - return monitor.WriteDebugRegisterValue( - m_thread.GetID(), GetRegisterOffset(reg_to_write), - GetRegisterName(reg_to_write), value_to_write); -#endif - return monitor.WriteRegisterValue( - m_thread.GetID(), GetRegisterOffset(reg_to_write), - GetRegisterName(reg_to_write), value_to_write); -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::ReadRegister( - const RegisterInfo *reg_info, RegisterValue &value) { - if (!reg_info) - return false; - - const uint32_t reg = reg_info->kinds[eRegisterKindLLDB]; - - if (IsFPR(reg, GetFPRType())) { - if (!ReadFPR()) - return false; - } else { - uint32_t full_reg = reg; - bool is_subreg = reg_info->invalidate_regs && - (reg_info->invalidate_regs[0] != LLDB_INVALID_REGNUM); - - if (is_subreg) { - // Read the full aligned 64-bit register. - full_reg = reg_info->invalidate_regs[0]; - } - - bool success = ReadRegister(full_reg, value); - - if (success) { - // If our read was not aligned (for ah,bh,ch,dh), shift our returned - // value one byte to the right. - if (is_subreg && (reg_info->byte_offset & 0x1)) - value.SetUInt64(value.GetAsUInt64() >> 8); - - // If our return byte size was greater than the return value reg size, - // then use the type specified by reg_info rather than the uint64_t - // default - if (value.GetByteSize() > reg_info->byte_size) - value.SetType(reg_info); - } - return success; - } - - if (reg_info->encoding == eEncodingVector) { - ByteOrder byte_order = GetByteOrder(); - - if (byte_order != ByteOrder::eByteOrderInvalid) { - if (reg >= m_reg_info.first_st && reg <= m_reg_info.last_st) - value.SetBytes(m_fpr.fxsave.stmm[reg - m_reg_info.first_st].bytes, - reg_info->byte_size, byte_order); - if (reg >= m_reg_info.first_mm && reg <= m_reg_info.last_mm) - value.SetBytes(m_fpr.fxsave.stmm[reg - m_reg_info.first_mm].bytes, - reg_info->byte_size, byte_order); - if (reg >= m_reg_info.first_xmm && reg <= m_reg_info.last_xmm) - value.SetBytes(m_fpr.fxsave.xmm[reg - m_reg_info.first_xmm].bytes, - reg_info->byte_size, byte_order); - if (reg >= m_reg_info.first_ymm && reg <= m_reg_info.last_ymm) { - // Concatenate ymm using the register halves in xmm.bytes and - // ymmh.bytes - if (GetFPRType() == eXSAVE && CopyXSTATEtoYMM(reg, byte_order)) - value.SetBytes(m_ymm_set.ymm[reg - m_reg_info.first_ymm].bytes, - reg_info->byte_size, byte_order); - else - return false; - } - return value.GetType() == RegisterValue::eTypeBytes; - } - return false; - } - - // Get pointer to m_fpr.fxsave variable and set the data from it. Byte - // offsets of all registers are calculated wrt 'UserArea' structure. However, - // ReadFPR() reads fpu registers {using ptrace(PT_GETFPREGS,..)} and stores - // them in 'm_fpr' (of type FPR structure). To extract values of fpu - // registers, m_fpr should be read at byte offsets calculated wrt to FPR - // structure. - - // Since, FPR structure is also one of the member of UserArea structure. - // byte_offset(fpu wrt FPR) = byte_offset(fpu wrt UserArea) - - // byte_offset(fctrl wrt UserArea) - assert((reg_info->byte_offset - m_fctrl_offset_in_userarea) < sizeof(m_fpr)); - uint8_t *src = - (uint8_t *)&m_fpr + reg_info->byte_offset - m_fctrl_offset_in_userarea; - switch (reg_info->byte_size) { - case 1: - value.SetUInt8(*(uint8_t *)src); - return true; - case 2: - value.SetUInt16(*(uint16_t *)src); - return true; - case 4: - value.SetUInt32(*(uint32_t *)src); - return true; - case 8: - value.SetUInt64(*(uint64_t *)src); - return true; - default: - assert(false && "Unhandled data size."); - return false; - } -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::WriteRegister( - const RegisterInfo *reg_info, const RegisterValue &value) { - const uint32_t reg = reg_info->kinds[eRegisterKindLLDB]; - - if (IsGPR(reg)) - return WriteRegister(reg, value); - - if (IsFPR(reg, GetFPRType())) { - if (reg_info->encoding == eEncodingVector) { - if (reg >= m_reg_info.first_st && reg <= m_reg_info.last_st) - ::memcpy(m_fpr.fxsave.stmm[reg - m_reg_info.first_st].bytes, - value.GetBytes(), value.GetByteSize()); - - if (reg >= m_reg_info.first_mm && reg <= m_reg_info.last_mm) - ::memcpy(m_fpr.fxsave.stmm[reg - m_reg_info.first_mm].bytes, - value.GetBytes(), value.GetByteSize()); - - if (reg >= m_reg_info.first_xmm && reg <= m_reg_info.last_xmm) - ::memcpy(m_fpr.fxsave.xmm[reg - m_reg_info.first_xmm].bytes, - value.GetBytes(), value.GetByteSize()); - - if (reg >= m_reg_info.first_ymm && reg <= m_reg_info.last_ymm) { - if (GetFPRType() != eXSAVE) - return false; // the target processor does not support AVX - - // Store ymm register content, and split into the register halves in - // xmm.bytes and ymmh.bytes - ::memcpy(m_ymm_set.ymm[reg - m_reg_info.first_ymm].bytes, - value.GetBytes(), value.GetByteSize()); - if (false == CopyYMMtoXSTATE(reg, GetByteOrder())) - return false; - } - } else { - // Get pointer to m_fpr.fxsave variable and set the data to it. Byte - // offsets of all registers are calculated wrt 'UserArea' structure. - // However, WriteFPR() takes m_fpr (of type FPR structure) and writes - // only fpu registers using ptrace(PT_SETFPREGS,..) API. Hence fpu - // registers should be written in m_fpr at byte offsets calculated wrt - // FPR structure. - - // Since, FPR structure is also one of the member of UserArea structure. - // byte_offset(fpu wrt FPR) = byte_offset(fpu wrt UserArea) - - // byte_offset(fctrl wrt UserArea) - assert((reg_info->byte_offset - m_fctrl_offset_in_userarea) < - sizeof(m_fpr)); - uint8_t *dst = (uint8_t *)&m_fpr + reg_info->byte_offset - - m_fctrl_offset_in_userarea; - switch (reg_info->byte_size) { - case 1: - *(uint8_t *)dst = value.GetAsUInt8(); - break; - case 2: - *(uint16_t *)dst = value.GetAsUInt16(); - break; - case 4: - *(uint32_t *)dst = value.GetAsUInt32(); - break; - case 8: - *(uint64_t *)dst = value.GetAsUInt64(); - break; - default: - assert(false && "Unhandled data size."); - return false; - } - } - - if (WriteFPR()) { - if (IsAVX(reg)) - return CopyYMMtoXSTATE(reg, GetByteOrder()); - return true; - } - } - return false; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::ReadAllRegisterValues( - DataBufferSP &data_sp) { - bool success = false; - data_sp.reset(new DataBufferHeap(REG_CONTEXT_SIZE, 0)); - if (ReadGPR() && ReadFPR()) { - uint8_t *dst = data_sp->GetBytes(); - success = dst != 0; - - if (success) { - ::memcpy(dst, &m_gpr_x86_64, GetGPRSize()); - dst += GetGPRSize(); - if (GetFPRType() == eFXSAVE) - ::memcpy(dst, &m_fpr.fxsave, sizeof(m_fpr.fxsave)); - } - - if (GetFPRType() == eXSAVE) { - ByteOrder byte_order = GetByteOrder(); - - // Assemble the YMM register content from the register halves. - for (uint32_t reg = m_reg_info.first_ymm; - success && reg <= m_reg_info.last_ymm; ++reg) - success = CopyXSTATEtoYMM(reg, byte_order); - - if (success) { - // Copy the extended register state including the assembled ymm - // registers. - ::memcpy(dst, &m_fpr, sizeof(m_fpr)); - } - } - } - return success; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::WriteAllRegisterValues( - const DataBufferSP &data_sp) { - bool success = false; - if (data_sp && data_sp->GetByteSize() == REG_CONTEXT_SIZE) { - uint8_t *src = data_sp->GetBytes(); - if (src) { - ::memcpy(&m_gpr_x86_64, src, GetGPRSize()); - - if (WriteGPR()) { - src += GetGPRSize(); - if (GetFPRType() == eFXSAVE) - ::memcpy(&m_fpr.fxsave, src, sizeof(m_fpr.fxsave)); - if (GetFPRType() == eXSAVE) - ::memcpy(&m_fpr.xsave, src, sizeof(m_fpr.xsave)); - - success = WriteFPR(); - if (success) { - if (GetFPRType() == eXSAVE) { - ByteOrder byte_order = GetByteOrder(); - - // Parse the YMM register content from the register halves. - for (uint32_t reg = m_reg_info.first_ymm; - success && reg <= m_reg_info.last_ymm; ++reg) - success = CopyYMMtoXSTATE(reg, byte_order); - } - } - } - } - } - return success; -} - -uint32_t RegisterContextPOSIXProcessMonitor_x86_64::SetHardwareWatchpoint( - addr_t addr, size_t size, bool read, bool write) { - const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints(); - uint32_t hw_index; - - for (hw_index = 0; hw_index < num_hw_watchpoints; ++hw_index) { - if (IsWatchpointVacant(hw_index)) - return SetHardwareWatchpointWithIndex(addr, size, read, write, hw_index); - } - - return LLDB_INVALID_INDEX32; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::ClearHardwareWatchpoint( - uint32_t hw_index) { - if (hw_index < NumSupportedHardwareWatchpoints()) { - RegisterValue current_dr7_bits; - - if (ReadRegister(m_reg_info.first_dr + 7, current_dr7_bits)) { - uint64_t new_dr7_bits = - current_dr7_bits.GetAsUInt64() & ~(3 << (2 * hw_index)); - - if (WriteRegister(m_reg_info.first_dr + 7, RegisterValue(new_dr7_bits))) - return true; - } - } - - return false; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::HardwareSingleStep( - bool enable) { - enum { TRACE_BIT = 0x100 }; - uint64_t rflags; - - if ((rflags = ReadRegisterAsUnsigned(m_reg_info.gpr_flags, -1UL)) == -1UL) - return false; - - if (enable) { - if (rflags & TRACE_BIT) - return true; - - rflags |= TRACE_BIT; - } else { - if (!(rflags & TRACE_BIT)) - return false; - - rflags &= ~TRACE_BIT; - } - - return WriteRegisterFromUnsigned(m_reg_info.gpr_flags, rflags); -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::UpdateAfterBreakpoint() { - // PC points one byte past the int3 responsible for the breakpoint. - lldb::addr_t pc; - - if ((pc = GetPC()) == LLDB_INVALID_ADDRESS) - return false; - - SetPC(pc - 1); - return true; -} - -unsigned RegisterContextPOSIXProcessMonitor_x86_64::GetRegisterIndexFromOffset( - unsigned offset) { - unsigned reg; - for (reg = 0; reg < m_reg_info.num_registers; reg++) { - if (GetRegisterInfo()[reg].byte_offset == offset) - break; - } - assert(reg < m_reg_info.num_registers && "Invalid register offset."); - return reg; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::IsWatchpointHit( - uint32_t hw_index) { - bool is_hit = false; - - if (m_watchpoints_initialized == false) { - // Reset the debug status and debug control registers - RegisterValue zero_bits = RegisterValue(uint64_t(0)); - if (!WriteRegister(m_reg_info.first_dr + 6, zero_bits) || - !WriteRegister(m_reg_info.first_dr + 7, zero_bits)) - assert(false && "Could not initialize watchpoint registers"); - m_watchpoints_initialized = true; - } - - if (hw_index < NumSupportedHardwareWatchpoints()) { - RegisterValue value; - - if (ReadRegister(m_reg_info.first_dr + 6, value)) { - uint64_t val = value.GetAsUInt64(); - is_hit = val & (1 << hw_index); - } - } - - return is_hit; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::ClearWatchpointHits() { - return WriteRegister(m_reg_info.first_dr + 6, RegisterValue((uint64_t)0)); -} - -addr_t RegisterContextPOSIXProcessMonitor_x86_64::GetWatchpointAddress( - uint32_t hw_index) { - addr_t wp_monitor_addr = LLDB_INVALID_ADDRESS; - - if (hw_index < NumSupportedHardwareWatchpoints()) { - if (!IsWatchpointVacant(hw_index)) { - RegisterValue value; - - if (ReadRegister(m_reg_info.first_dr + hw_index, value)) - wp_monitor_addr = value.GetAsUInt64(); - } - } - - return wp_monitor_addr; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::IsWatchpointVacant( - uint32_t hw_index) { - bool is_vacant = false; - RegisterValue value; - - assert(hw_index < NumSupportedHardwareWatchpoints()); - - if (m_watchpoints_initialized == false) { - // Reset the debug status and debug control registers - RegisterValue zero_bits = RegisterValue(uint64_t(0)); - if (!WriteRegister(m_reg_info.first_dr + 6, zero_bits) || - !WriteRegister(m_reg_info.first_dr + 7, zero_bits)) - assert(false && "Could not initialize watchpoint registers"); - m_watchpoints_initialized = true; - } - - if (ReadRegister(m_reg_info.first_dr + 7, value)) { - uint64_t val = value.GetAsUInt64(); - is_vacant = (val & (3 << 2 * hw_index)) == 0; - } - - return is_vacant; -} - -bool RegisterContextPOSIXProcessMonitor_x86_64::SetHardwareWatchpointWithIndex( - addr_t addr, size_t size, bool read, bool write, uint32_t hw_index) { - const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints(); - - if (num_hw_watchpoints == 0 || hw_index >= num_hw_watchpoints) - return false; - - if (!(size == 1 || size == 2 || size == 4 || size == 8)) - return false; - - if (read == false && write == false) - return false; - - if (!IsWatchpointVacant(hw_index)) - return false; - - // Set both dr7 (debug control register) and dri (debug address register). - - // dr7{7-0} encodes the local/global enable bits: - // global enable --. .-- local enable - // | | - // v v - // dr0 -> bits{1-0} - // dr1 -> bits{3-2} - // dr2 -> bits{5-4} - // dr3 -> bits{7-6} - // - // dr7{31-16} encodes the rw/len bits: - // b_x+3, b_x+2, b_x+1, b_x - // where bits{x+1, x} => rw - // 0b00: execute, 0b01: write, 0b11: read-or-write, - // 0b10: io read-or-write (unused) - // and bits{x+3, x+2} => len - // 0b00: 1-byte, 0b01: 2-byte, 0b11: 4-byte, 0b10: 8-byte - // - // dr0 -> bits{19-16} - // dr1 -> bits{23-20} - // dr2 -> bits{27-24} - // dr3 -> bits{31-28} - if (hw_index < num_hw_watchpoints) { - RegisterValue current_dr7_bits; - - if (ReadRegister(m_reg_info.first_dr + 7, current_dr7_bits)) { - uint64_t new_dr7_bits = - current_dr7_bits.GetAsUInt64() | - (1 << (2 * hw_index) | - size_and_rw_bits(size, read, write) << (16 + 4 * hw_index)); - - if (WriteRegister(m_reg_info.first_dr + hw_index, RegisterValue(addr)) && - WriteRegister(m_reg_info.first_dr + 7, RegisterValue(new_dr7_bits))) - return true; - } - } - - return false; -} - -uint32_t -RegisterContextPOSIXProcessMonitor_x86_64::NumSupportedHardwareWatchpoints() { - // Available debug address registers: dr0, dr1, dr2, dr3 - return 4; -} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_x86.h b/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_x86.h deleted file mode 100644 index 1afb366eeba3..000000000000 --- a/contrib/llvm-project/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_x86.h +++ /dev/null @@ -1,81 +0,0 @@ -//===-- RegisterContextPOSIXProcessMonitor_x86.h ----------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef liblldb_RegisterContextPOSIXProcessMonitor_x86_H_ -#define liblldb_RegisterContextPOSIXProcessMonitor_x86_H_ - -#include "Plugins/Process/Utility/RegisterContextPOSIX_x86.h" -#include "RegisterContextPOSIX.h" -#include <sys/uio.h> - -class RegisterContextPOSIXProcessMonitor_x86_64 - : public RegisterContextPOSIX_x86, - public POSIXBreakpointProtocol { -public: - RegisterContextPOSIXProcessMonitor_x86_64( - lldb_private::Thread &thread, uint32_t concrete_frame_idx, - lldb_private::RegisterInfoInterface *register_info); - -protected: - bool ReadGPR() override; - - bool ReadFPR() override; - - bool WriteGPR() override; - - bool WriteFPR() override; - - // lldb_private::RegisterContext - bool ReadRegister(const unsigned reg, lldb_private::RegisterValue &value); - - bool WriteRegister(const unsigned reg, - const lldb_private::RegisterValue &value); - - bool ReadRegister(const lldb_private::RegisterInfo *reg_info, - lldb_private::RegisterValue &value) override; - - bool WriteRegister(const lldb_private::RegisterInfo *reg_info, - const lldb_private::RegisterValue &value) override; - - bool ReadAllRegisterValues(lldb::DataBufferSP &data_sp) override; - - bool WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override; - - uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size, bool read, - bool write) override; - - bool ClearHardwareWatchpoint(uint32_t hw_index) override; - - bool HardwareSingleStep(bool enable) override; - - // POSIXBreakpointProtocol - bool UpdateAfterBreakpoint() override; - - unsigned GetRegisterIndexFromOffset(unsigned offset) override; - - bool IsWatchpointHit(uint32_t hw_index) override; - - bool ClearWatchpointHits() override; - - lldb::addr_t GetWatchpointAddress(uint32_t hw_index) override; - - bool IsWatchpointVacant(uint32_t hw_index) override; - - bool SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size, bool read, - bool write, uint32_t hw_index) override; - - uint32_t NumSupportedHardwareWatchpoints() override; - -private: - ProcessMonitor &GetMonitor(); - uint32_t - m_fctrl_offset_in_userarea; // Offset of 'fctrl' in 'UserArea' Structure - struct iovec m_iovec; -}; - -#endif diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/NetBSD/NativeRegisterContextNetBSD_x86_64.h b/contrib/llvm-project/lldb/source/Plugins/Process/NetBSD/NativeRegisterContextNetBSD_x86_64.h index d20fd67cdc5d..31005952dd75 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/NetBSD/NativeRegisterContextNetBSD_x86_64.h +++ b/contrib/llvm-project/lldb/source/Plugins/Process/NetBSD/NativeRegisterContextNetBSD_x86_64.h @@ -22,7 +22,7 @@ #include "Plugins/Process/NetBSD/NativeRegisterContextNetBSD.h" #include "Plugins/Process/Utility/RegisterContext_x86.h" -#include "Plugins/Process/Utility/NativeRegisterContextWatchpoint_x86.h" +#include "Plugins/Process/Utility/NativeRegisterContextDBReg_x86.h" #include "Plugins/Process/Utility/lldb-x86-register-enums.h" namespace lldb_private { @@ -32,7 +32,7 @@ class NativeProcessNetBSD; class NativeRegisterContextNetBSD_x86_64 : public NativeRegisterContextNetBSD, - public NativeRegisterContextWatchpoint_x86 { + public NativeRegisterContextDBReg_x86 { public: NativeRegisterContextNetBSD_x86_64(const ArchSpec &target_arch, NativeThreadProtocol &native_thread); diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp new file mode 100644 index 000000000000..ee5295bf6565 --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp @@ -0,0 +1,182 @@ +//===-- NativeProcessSoftwareSingleStep.cpp -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NativeProcessSoftwareSingleStep.h" + +#include "lldb/Core/EmulateInstruction.h" +#include "lldb/Host/common/NativeRegisterContext.h" +#include "lldb/Utility/RegisterValue.h" + +#include <unordered_map> + +using namespace lldb; +using namespace lldb_private; + +namespace { + +struct EmulatorBaton { + NativeProcessProtocol &m_process; + NativeRegisterContext &m_reg_context; + + // eRegisterKindDWARF -> RegsiterValue + std::unordered_map<uint32_t, RegisterValue> m_register_values; + + EmulatorBaton(NativeProcessProtocol &process, + NativeRegisterContext ®_context) + : m_process(process), m_reg_context(reg_context) {} +}; + +} // anonymous namespace + +static size_t ReadMemoryCallback(EmulateInstruction *instruction, void *baton, + const EmulateInstruction::Context &context, + lldb::addr_t addr, void *dst, size_t length) { + EmulatorBaton *emulator_baton = static_cast<EmulatorBaton *>(baton); + + size_t bytes_read; + emulator_baton->m_process.ReadMemory(addr, dst, length, bytes_read); + return bytes_read; +} + +static bool ReadRegisterCallback(EmulateInstruction *instruction, void *baton, + const RegisterInfo *reg_info, + RegisterValue ®_value) { + EmulatorBaton *emulator_baton = static_cast<EmulatorBaton *>(baton); + + auto it = emulator_baton->m_register_values.find( + reg_info->kinds[eRegisterKindDWARF]); + if (it != emulator_baton->m_register_values.end()) { + reg_value = it->second; + return true; + } + + // The emulator only fill in the dwarf regsiter numbers (and in some case the + // generic register numbers). Get the full register info from the register + // context based on the dwarf register numbers. + const RegisterInfo *full_reg_info = + emulator_baton->m_reg_context.GetRegisterInfo( + eRegisterKindDWARF, reg_info->kinds[eRegisterKindDWARF]); + + Status error = + emulator_baton->m_reg_context.ReadRegister(full_reg_info, reg_value); + if (error.Success()) + return true; + + return false; +} + +static bool WriteRegisterCallback(EmulateInstruction *instruction, void *baton, + const EmulateInstruction::Context &context, + const RegisterInfo *reg_info, + const RegisterValue ®_value) { + EmulatorBaton *emulator_baton = static_cast<EmulatorBaton *>(baton); + emulator_baton->m_register_values[reg_info->kinds[eRegisterKindDWARF]] = + reg_value; + return true; +} + +static size_t WriteMemoryCallback(EmulateInstruction *instruction, void *baton, + const EmulateInstruction::Context &context, + lldb::addr_t addr, const void *dst, + size_t length) { + return length; +} + +static lldb::addr_t ReadFlags(NativeRegisterContext ®siter_context) { + const RegisterInfo *flags_info = regsiter_context.GetRegisterInfo( + eRegisterKindGeneric, LLDB_REGNUM_GENERIC_FLAGS); + return regsiter_context.ReadRegisterAsUnsigned(flags_info, + LLDB_INVALID_ADDRESS); +} + +Status NativeProcessSoftwareSingleStep::SetupSoftwareSingleStepping( + NativeThreadProtocol &thread) { + Status error; + NativeProcessProtocol &process = thread.GetProcess(); + NativeRegisterContext ®ister_context = thread.GetRegisterContext(); + const ArchSpec &arch = process.GetArchitecture(); + + std::unique_ptr<EmulateInstruction> emulator_up( + EmulateInstruction::FindPlugin(arch, eInstructionTypePCModifying, + nullptr)); + + if (emulator_up == nullptr) + return Status("Instruction emulator not found!"); + + EmulatorBaton baton(process, register_context); + emulator_up->SetBaton(&baton); + emulator_up->SetReadMemCallback(&ReadMemoryCallback); + emulator_up->SetReadRegCallback(&ReadRegisterCallback); + emulator_up->SetWriteMemCallback(&WriteMemoryCallback); + emulator_up->SetWriteRegCallback(&WriteRegisterCallback); + + if (!emulator_up->ReadInstruction()) + return Status("Read instruction failed!"); + + bool emulation_result = + emulator_up->EvaluateInstruction(eEmulateInstructionOptionAutoAdvancePC); + + const RegisterInfo *reg_info_pc = register_context.GetRegisterInfo( + eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC); + const RegisterInfo *reg_info_flags = register_context.GetRegisterInfo( + eRegisterKindGeneric, LLDB_REGNUM_GENERIC_FLAGS); + + auto pc_it = + baton.m_register_values.find(reg_info_pc->kinds[eRegisterKindDWARF]); + auto flags_it = + baton.m_register_values.find(reg_info_flags->kinds[eRegisterKindDWARF]); + + lldb::addr_t next_pc; + lldb::addr_t next_flags; + if (emulation_result) { + assert(pc_it != baton.m_register_values.end() && + "Emulation was successfull but PC wasn't updated"); + next_pc = pc_it->second.GetAsUInt64(); + + if (flags_it != baton.m_register_values.end()) + next_flags = flags_it->second.GetAsUInt64(); + else + next_flags = ReadFlags(register_context); + } else if (pc_it == baton.m_register_values.end()) { + // Emulate instruction failed and it haven't changed PC. Advance PC with + // the size of the current opcode because the emulation of all + // PC modifying instruction should be successful. The failure most + // likely caused by a not supported instruction which don't modify PC. + next_pc = register_context.GetPC() + emulator_up->GetOpcode().GetByteSize(); + next_flags = ReadFlags(register_context); + } else { + // The instruction emulation failed after it modified the PC. It is an + // unknown error where we can't continue because the next instruction is + // modifying the PC but we don't know how. + return Status("Instruction emulation failed unexpectedly."); + } + + int size_hint = 0; + if (arch.GetMachine() == llvm::Triple::arm) { + if (next_flags & 0x20) { + // Thumb mode + size_hint = 2; + } else { + // Arm mode + size_hint = 4; + } + } else if (arch.IsMIPS() || arch.GetTriple().isPPC64()) + size_hint = 4; + error = process.SetBreakpoint(next_pc, size_hint, /*hardware=*/false); + + // If setting the breakpoint fails because next_pc is out of the address + // space, ignore it and let the debugee segfault. + if (error.GetError() == EIO || error.GetError() == EFAULT) { + return Status(); + } else if (error.Fail()) + return error; + + m_threads_stepping_with_breakpoint.insert({thread.GetID(), next_pc}); + + return Status(); +} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h new file mode 100644 index 000000000000..f9435b7a84ba --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h @@ -0,0 +1,31 @@ +//===-- NativeProcessSoftwareSingleStep.h -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef lldb_NativeProcessSoftwareSingleStep_h +#define lldb_NativeProcessSoftwareSingleStep_h + +#include "lldb/Host/common/NativeProcessProtocol.h" +#include "lldb/Host/common/NativeThreadProtocol.h" + +#include <map> + +namespace lldb_private { + +class NativeProcessSoftwareSingleStep { +public: + Status SetupSoftwareSingleStepping(NativeThreadProtocol &thread); + +protected: + // List of thread ids stepping with a breakpoint with the address of + // the relevan breakpoint + std::map<lldb::tid_t, lldb::addr_t> m_threads_stepping_with_breakpoint; +}; + +} // namespace lldb_private + +#endif // #ifndef lldb_NativeProcessSoftwareSingleStep_h diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.cpp new file mode 100644 index 000000000000..5c05baf71764 --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.cpp @@ -0,0 +1,466 @@ +//===-- NativeRegisterContextDBReg_arm64.cpp ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NativeRegisterContextDBReg_arm64.h" + +#include "lldb/Utility/Log.h" +#include "lldb/Utility/RegisterValue.h" + +using namespace lldb_private; + +// E (bit 0), used to enable breakpoint/watchpoint +constexpr uint32_t g_enable_bit = 1; +// PAC (bits 2:1): 0b10 +constexpr uint32_t g_pac_bits = (2 << 1); + +// Returns appropriate control register bits for the specified size +static constexpr inline uint64_t GetSizeBits(int size) { + // BAS (bits 12:5) hold a bit-mask of addresses to watch + // e.g. 0b00000001 means 1 byte at address + // 0b00000011 means 2 bytes (addr..addr+1) + // ... + // 0b11111111 means 8 bytes (addr..addr+7) + return ((1 << size) - 1) << 5; +} + +uint32_t NativeRegisterContextDBReg_arm64::NumSupportedHardwareBreakpoints() { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_BREAKPOINTS)); + llvm::Error error = ReadHardwareDebugInfo(); + if (error) { + LLDB_LOG_ERROR(log, std::move(error), + "failed to read debug registers: {0}"); + return 0; + } + + return m_max_hbp_supported; +} + +uint32_t +NativeRegisterContextDBReg_arm64::SetHardwareBreakpoint(lldb::addr_t addr, + size_t size) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_BREAKPOINTS)); + LLDB_LOG(log, "addr: {0:x}, size: {1:x}", addr, size); + + // Read hardware breakpoint and watchpoint information. + llvm::Error error = ReadHardwareDebugInfo(); + if (error) { + LLDB_LOG_ERROR( + log, std::move(error), + "unable to set breakpoint: failed to read debug registers: {0}"); + return LLDB_INVALID_INDEX32; + } + + uint32_t control_value = 0, bp_index = 0; + + // Check if size has a valid hardware breakpoint length. + if (size != 4) + return LLDB_INVALID_INDEX32; // Invalid size for a AArch64 hardware + // breakpoint + + // Check 4-byte alignment for hardware breakpoint target address. + if (addr & 0x03) + return LLDB_INVALID_INDEX32; // Invalid address, should be 4-byte aligned. + + // Setup control value + control_value = g_enable_bit | g_pac_bits | GetSizeBits(size); + + // Iterate over stored breakpoints and find a free bp_index + bp_index = LLDB_INVALID_INDEX32; + for (uint32_t i = 0; i < m_max_hbp_supported; i++) { + if (!BreakpointIsEnabled(i)) + bp_index = i; // Mark last free slot + else if (m_hbp_regs[i].address == addr) + return LLDB_INVALID_INDEX32; // We do not support duplicate breakpoints. + } + + if (bp_index == LLDB_INVALID_INDEX32) + return LLDB_INVALID_INDEX32; + + // Update breakpoint in local cache + m_hbp_regs[bp_index].real_addr = addr; + m_hbp_regs[bp_index].address = addr; + m_hbp_regs[bp_index].control = control_value; + + // PTRACE call to set corresponding hardware breakpoint register. + error = WriteHardwareDebugRegs(eDREGTypeBREAK); + + if (error) { + m_hbp_regs[bp_index].address = 0; + m_hbp_regs[bp_index].control &= ~1; + + LLDB_LOG_ERROR( + log, std::move(error), + "unable to set breakpoint: failed to write debug registers: {0}"); + return LLDB_INVALID_INDEX32; + } + + return bp_index; +} + +bool NativeRegisterContextDBReg_arm64::ClearHardwareBreakpoint( + uint32_t hw_idx) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_BREAKPOINTS)); + LLDB_LOG(log, "hw_idx: {0}", hw_idx); + + // Read hardware breakpoint and watchpoint information. + llvm::Error error = ReadHardwareDebugInfo(); + if (error) { + LLDB_LOG_ERROR( + log, std::move(error), + "unable to clear breakpoint: failed to read debug registers: {0}"); + return false; + } + + if (hw_idx >= m_max_hbp_supported) + return false; + + // Create a backup we can revert to in case of failure. + lldb::addr_t tempAddr = m_hbp_regs[hw_idx].address; + uint32_t tempControl = m_hbp_regs[hw_idx].control; + + m_hbp_regs[hw_idx].control &= ~g_enable_bit; + m_hbp_regs[hw_idx].address = 0; + + // PTRACE call to clear corresponding hardware breakpoint register. + error = WriteHardwareDebugRegs(eDREGTypeBREAK); + + if (error) { + m_hbp_regs[hw_idx].control = tempControl; + m_hbp_regs[hw_idx].address = tempAddr; + + LLDB_LOG_ERROR( + log, std::move(error), + "unable to clear breakpoint: failed to write debug registers: {0}"); + return false; + } + + return true; +} + +Status NativeRegisterContextDBReg_arm64::GetHardwareBreakHitIndex( + uint32_t &bp_index, lldb::addr_t trap_addr) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_BREAKPOINTS)); + + LLDB_LOGF(log, "NativeRegisterContextDBReg_arm64::%s()", __FUNCTION__); + + lldb::addr_t break_addr; + + for (bp_index = 0; bp_index < m_max_hbp_supported; ++bp_index) { + break_addr = m_hbp_regs[bp_index].address; + + if (BreakpointIsEnabled(bp_index) && trap_addr == break_addr) { + m_hbp_regs[bp_index].hit_addr = trap_addr; + return Status(); + } + } + + bp_index = LLDB_INVALID_INDEX32; + return Status(); +} + +Status NativeRegisterContextDBReg_arm64::ClearAllHardwareBreakpoints() { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_BREAKPOINTS)); + + LLDB_LOGF(log, "NativeRegisterContextDBReg_arm64::%s()", __FUNCTION__); + + // Read hardware breakpoint and watchpoint information. + llvm::Error error = ReadHardwareDebugInfo(); + if (error) + return Status(std::move(error)); + + for (uint32_t i = 0; i < m_max_hbp_supported; i++) { + if (BreakpointIsEnabled(i)) { + // Create a backup we can revert to in case of failure. + lldb::addr_t tempAddr = m_hbp_regs[i].address; + uint32_t tempControl = m_hbp_regs[i].control; + + // Clear watchpoints in local cache + m_hbp_regs[i].control &= ~g_enable_bit; + m_hbp_regs[i].address = 0; + + // Ptrace call to update hardware debug registers + error = WriteHardwareDebugRegs(eDREGTypeBREAK); + + if (error) { + m_hbp_regs[i].control = tempControl; + m_hbp_regs[i].address = tempAddr; + + return Status(std::move(error)); + } + } + } + + return Status(); +} + +bool NativeRegisterContextDBReg_arm64::BreakpointIsEnabled(uint32_t bp_index) { + if ((m_hbp_regs[bp_index].control & g_enable_bit) != 0) + return true; + else + return false; +} + +uint32_t NativeRegisterContextDBReg_arm64::NumSupportedHardwareWatchpoints() { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); + llvm::Error error = ReadHardwareDebugInfo(); + if (error) { + LLDB_LOG_ERROR(log, std::move(error), + "failed to read debug registers: {0}"); + return 0; + } + + return m_max_hwp_supported; +} + +uint32_t NativeRegisterContextDBReg_arm64::SetHardwareWatchpoint( + lldb::addr_t addr, size_t size, uint32_t watch_flags) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); + LLDB_LOG(log, "addr: {0:x}, size: {1:x} watch_flags: {2:x}", addr, size, + watch_flags); + + // Read hardware breakpoint and watchpoint information. + llvm::Error error = ReadHardwareDebugInfo(); + if (error) { + LLDB_LOG_ERROR( + log, std::move(error), + "unable to set watchpoint: failed to read debug registers: {0}"); + return LLDB_INVALID_INDEX32; + } + + uint32_t control_value = 0, wp_index = 0; + lldb::addr_t real_addr = addr; + + // Check if we are setting watchpoint other than read/write/access Also + // update watchpoint flag to match AArch64 write-read bit configuration. + switch (watch_flags) { + case 1: + watch_flags = 2; + break; + case 2: + watch_flags = 1; + break; + case 3: + break; + default: + return LLDB_INVALID_INDEX32; + } + + // Check if size has a valid hardware watchpoint length. + if (size != 1 && size != 2 && size != 4 && size != 8) + return LLDB_INVALID_INDEX32; + + // Check 8-byte alignment for hardware watchpoint target address. Below is a + // hack to recalculate address and size in order to make sure we can watch + // non 8-byte aligned addresses as well. + if (addr & 0x07) { + uint8_t watch_mask = (addr & 0x07) + size; + + if (watch_mask > 0x08) + return LLDB_INVALID_INDEX32; + else if (watch_mask <= 0x02) + size = 2; + else if (watch_mask <= 0x04) + size = 4; + else + size = 8; + + addr = addr & (~0x07); + } + + // Setup control value + control_value = g_enable_bit | g_pac_bits | GetSizeBits(size); + control_value |= watch_flags << 3; + + // Iterate over stored watchpoints and find a free wp_index + wp_index = LLDB_INVALID_INDEX32; + for (uint32_t i = 0; i < m_max_hwp_supported; i++) { + if (!WatchpointIsEnabled(i)) + wp_index = i; // Mark last free slot + else if (m_hwp_regs[i].address == addr) { + return LLDB_INVALID_INDEX32; // We do not support duplicate watchpoints. + } + } + + if (wp_index == LLDB_INVALID_INDEX32) + return LLDB_INVALID_INDEX32; + + // Update watchpoint in local cache + m_hwp_regs[wp_index].real_addr = real_addr; + m_hwp_regs[wp_index].address = addr; + m_hwp_regs[wp_index].control = control_value; + + // PTRACE call to set corresponding watchpoint register. + error = WriteHardwareDebugRegs(eDREGTypeWATCH); + + if (error) { + m_hwp_regs[wp_index].address = 0; + m_hwp_regs[wp_index].control &= ~g_enable_bit; + + LLDB_LOG_ERROR( + log, std::move(error), + "unable to set watchpoint: failed to write debug registers: {0}"); + return LLDB_INVALID_INDEX32; + } + + return wp_index; +} + +bool NativeRegisterContextDBReg_arm64::ClearHardwareWatchpoint( + uint32_t wp_index) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); + LLDB_LOG(log, "wp_index: {0}", wp_index); + + // Read hardware breakpoint and watchpoint information. + llvm::Error error = ReadHardwareDebugInfo(); + if (error) { + LLDB_LOG_ERROR( + log, std::move(error), + "unable to clear watchpoint: failed to read debug registers: {0}"); + return false; + } + + if (wp_index >= m_max_hwp_supported) + return false; + + // Create a backup we can revert to in case of failure. + lldb::addr_t tempAddr = m_hwp_regs[wp_index].address; + uint32_t tempControl = m_hwp_regs[wp_index].control; + + // Update watchpoint in local cache + m_hwp_regs[wp_index].control &= ~g_enable_bit; + m_hwp_regs[wp_index].address = 0; + + // Ptrace call to update hardware debug registers + error = WriteHardwareDebugRegs(eDREGTypeWATCH); + + if (error) { + m_hwp_regs[wp_index].control = tempControl; + m_hwp_regs[wp_index].address = tempAddr; + + LLDB_LOG_ERROR( + log, std::move(error), + "unable to clear watchpoint: failed to write debug registers: {0}"); + return false; + } + + return true; +} + +Status NativeRegisterContextDBReg_arm64::ClearAllHardwareWatchpoints() { + // Read hardware breakpoint and watchpoint information. + llvm::Error error = ReadHardwareDebugInfo(); + if (error) + return Status(std::move(error)); + + for (uint32_t i = 0; i < m_max_hwp_supported; i++) { + if (WatchpointIsEnabled(i)) { + // Create a backup we can revert to in case of failure. + lldb::addr_t tempAddr = m_hwp_regs[i].address; + uint32_t tempControl = m_hwp_regs[i].control; + + // Clear watchpoints in local cache + m_hwp_regs[i].control &= ~g_enable_bit; + m_hwp_regs[i].address = 0; + + // Ptrace call to update hardware debug registers + error = WriteHardwareDebugRegs(eDREGTypeWATCH); + + if (error) { + m_hwp_regs[i].control = tempControl; + m_hwp_regs[i].address = tempAddr; + + return Status(std::move(error)); + } + } + } + + return Status(); +} + +uint32_t +NativeRegisterContextDBReg_arm64::GetWatchpointSize(uint32_t wp_index) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); + LLDB_LOG(log, "wp_index: {0}", wp_index); + + switch ((m_hwp_regs[wp_index].control >> 5) & 0xff) { + case 0x01: + return 1; + case 0x03: + return 2; + case 0x0f: + return 4; + case 0xff: + return 8; + default: + return 0; + } +} + +bool NativeRegisterContextDBReg_arm64::WatchpointIsEnabled(uint32_t wp_index) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); + LLDB_LOG(log, "wp_index: {0}", wp_index); + + if ((m_hwp_regs[wp_index].control & g_enable_bit) != 0) + return true; + else + return false; +} + +Status NativeRegisterContextDBReg_arm64::GetWatchpointHitIndex( + uint32_t &wp_index, lldb::addr_t trap_addr) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); + LLDB_LOG(log, "wp_index: {0}, trap_addr: {1:x}", wp_index, trap_addr); + + // Read hardware breakpoint and watchpoint information. + llvm::Error error = ReadHardwareDebugInfo(); + if (error) + return Status(std::move(error)); + + uint32_t watch_size; + lldb::addr_t watch_addr; + + for (wp_index = 0; wp_index < m_max_hwp_supported; ++wp_index) { + watch_size = GetWatchpointSize(wp_index); + watch_addr = m_hwp_regs[wp_index].address; + + if (WatchpointIsEnabled(wp_index) && trap_addr >= watch_addr && + trap_addr < watch_addr + watch_size) { + m_hwp_regs[wp_index].hit_addr = trap_addr; + return Status(); + } + } + + wp_index = LLDB_INVALID_INDEX32; + return Status(); +} + +lldb::addr_t +NativeRegisterContextDBReg_arm64::GetWatchpointAddress(uint32_t wp_index) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); + LLDB_LOG(log, "wp_index: {0}", wp_index); + + if (wp_index >= m_max_hwp_supported) + return LLDB_INVALID_ADDRESS; + + if (WatchpointIsEnabled(wp_index)) + return m_hwp_regs[wp_index].real_addr; + return LLDB_INVALID_ADDRESS; +} + +lldb::addr_t +NativeRegisterContextDBReg_arm64::GetWatchpointHitAddress(uint32_t wp_index) { + Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); + LLDB_LOG(log, "wp_index: {0}", wp_index); + + if (wp_index >= m_max_hwp_supported) + return LLDB_INVALID_ADDRESS; + + if (WatchpointIsEnabled(wp_index)) + return m_hwp_regs[wp_index].hit_addr; + return LLDB_INVALID_ADDRESS; +} diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.h b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.h new file mode 100644 index 000000000000..12ef5571f64c --- /dev/null +++ b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_arm64.h @@ -0,0 +1,79 @@ +//===-- NativeRegisterContextDBReg_arm64.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef lldb_NativeRegisterContextDBReg_arm64_h +#define lldb_NativeRegisterContextDBReg_arm64_h + +#include "Plugins/Process/Utility/NativeRegisterContextRegisterInfo.h" + +#include <array> + +namespace lldb_private { + +class NativeRegisterContextDBReg_arm64 + : public virtual NativeRegisterContextRegisterInfo { +public: + uint32_t NumSupportedHardwareBreakpoints() override; + + uint32_t SetHardwareBreakpoint(lldb::addr_t addr, size_t size) override; + + bool ClearHardwareBreakpoint(uint32_t hw_idx) override; + + Status ClearAllHardwareBreakpoints() override; + + Status GetHardwareBreakHitIndex(uint32_t &bp_index, + lldb::addr_t trap_addr) override; + + bool BreakpointIsEnabled(uint32_t bp_index); + + uint32_t NumSupportedHardwareWatchpoints() override; + + uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size, + uint32_t watch_flags) override; + + bool ClearHardwareWatchpoint(uint32_t hw_index) override; + + Status ClearAllHardwareWatchpoints() override; + + Status GetWatchpointHitIndex(uint32_t &wp_index, + lldb::addr_t trap_addr) override; + + lldb::addr_t GetWatchpointHitAddress(uint32_t wp_index) override; + + lldb::addr_t GetWatchpointAddress(uint32_t wp_index) override; + + uint32_t GetWatchpointSize(uint32_t wp_index); + + bool WatchpointIsEnabled(uint32_t wp_index); + + // Debug register type select + enum DREGType { eDREGTypeWATCH = 0, eDREGTypeBREAK }; + +protected: + // Debug register info for hardware breakpoints and watchpoints management. + struct DREG { + lldb::addr_t address; // Breakpoint/watchpoint address value. + lldb::addr_t hit_addr; // Address at which last watchpoint trigger exception + // occurred. + lldb::addr_t real_addr; // Address value that should cause target to stop. + uint32_t control; // Breakpoint/watchpoint control value. + }; + + std::array<struct DREG, 16> m_hbp_regs; // hardware breakpoints + std::array<struct DREG, 16> m_hwp_regs; // hardware watchpoints + + uint32_t m_max_hbp_supported; + uint32_t m_max_hwp_supported; + + virtual llvm::Error ReadHardwareDebugInfo() = 0; + virtual llvm::Error WriteHardwareDebugRegs(DREGType hwbType) = 0; +}; + +} // namespace lldb_private + +#endif // #ifndef lldb_NativeRegisterContextDBReg_arm64_h diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextWatchpoint_x86.cpp b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_x86.cpp index 0f57f2ed3f35..56c1757ee89d 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextWatchpoint_x86.cpp +++ b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_x86.cpp @@ -1,4 +1,4 @@ -//===-- NativeRegisterContextWatchpoint_x86.cpp ---------------------------===// +//===-- NativeRegisterContextDBReg_x86.cpp --------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "NativeRegisterContextWatchpoint_x86.h" +#include "NativeRegisterContextDBReg_x86.h" #include "lldb/Utility/Log.h" #include "lldb/Utility/RegisterValue.h" @@ -80,7 +80,7 @@ static inline uint64_t GetWatchControlBitmask(uint32_t wp_index) { // Bit mask for control bits regarding all watchpoints. static constexpr uint64_t watchpoint_all_control_bit_mask = 0xFFFF00FF; -const RegisterInfo *NativeRegisterContextWatchpoint_x86::GetDR(int num) const { +const RegisterInfo *NativeRegisterContextDBReg_x86::GetDR(int num) const { assert(num >= 0 && num <= 7); switch (GetRegisterInfoInterface().GetTargetArchitecture().GetMachine()) { case llvm::Triple::x86: @@ -92,8 +92,8 @@ const RegisterInfo *NativeRegisterContextWatchpoint_x86::GetDR(int num) const { } } -Status NativeRegisterContextWatchpoint_x86::IsWatchpointHit(uint32_t wp_index, - bool &is_hit) { +Status NativeRegisterContextDBReg_x86::IsWatchpointHit(uint32_t wp_index, + bool &is_hit) { if (wp_index >= NumSupportedHardwareWatchpoints()) return Status("Watchpoint index out of range"); @@ -107,8 +107,9 @@ Status NativeRegisterContextWatchpoint_x86::IsWatchpointHit(uint32_t wp_index, return error; } -Status NativeRegisterContextWatchpoint_x86::GetWatchpointHitIndex( - uint32_t &wp_index, lldb::addr_t trap_addr) { +Status +NativeRegisterContextDBReg_x86::GetWatchpointHitIndex(uint32_t &wp_index, + lldb::addr_t trap_addr) { uint32_t num_hw_wps = NumSupportedHardwareWatchpoints(); for (wp_index = 0; wp_index < num_hw_wps; ++wp_index) { bool is_hit; @@ -124,9 +125,8 @@ Status NativeRegisterContextWatchpoint_x86::GetWatchpointHitIndex( return Status(); } -Status -NativeRegisterContextWatchpoint_x86::IsWatchpointVacant(uint32_t wp_index, - bool &is_vacant) { +Status NativeRegisterContextDBReg_x86::IsWatchpointVacant(uint32_t wp_index, + bool &is_vacant) { if (wp_index >= NumSupportedHardwareWatchpoints()) return Status("Watchpoint index out of range"); @@ -140,7 +140,7 @@ NativeRegisterContextWatchpoint_x86::IsWatchpointVacant(uint32_t wp_index, return error; } -Status NativeRegisterContextWatchpoint_x86::SetHardwareWatchpointWithIndex( +Status NativeRegisterContextDBReg_x86::SetHardwareWatchpointWithIndex( lldb::addr_t addr, size_t size, uint32_t watch_flags, uint32_t wp_index) { if (wp_index >= NumSupportedHardwareWatchpoints()) @@ -202,7 +202,7 @@ Status NativeRegisterContextWatchpoint_x86::SetHardwareWatchpointWithIndex( return error; } -bool NativeRegisterContextWatchpoint_x86::ClearHardwareWatchpoint( +bool NativeRegisterContextDBReg_x86::ClearHardwareWatchpoint( uint32_t wp_index) { if (wp_index >= NumSupportedHardwareWatchpoints()) return false; @@ -217,8 +217,7 @@ bool NativeRegisterContextWatchpoint_x86::ClearHardwareWatchpoint( .Success(); } -Status -NativeRegisterContextWatchpoint_x86::ClearWatchpointHit(uint32_t wp_index) { +Status NativeRegisterContextDBReg_x86::ClearWatchpointHit(uint32_t wp_index) { if (wp_index >= NumSupportedHardwareWatchpoints()) return Status("Watchpoint index out of range"); @@ -231,7 +230,7 @@ NativeRegisterContextWatchpoint_x86::ClearWatchpointHit(uint32_t wp_index) { GetDR(6), RegisterValue(dr6.GetAsUInt64() & ~GetStatusBit(wp_index))); } -Status NativeRegisterContextWatchpoint_x86::ClearAllHardwareWatchpoints() { +Status NativeRegisterContextDBReg_x86::ClearAllHardwareWatchpoints() { RegisterValue dr7; Status error = ReadRegister(GetDR(7), dr7); if (error.Fail()) @@ -241,7 +240,7 @@ Status NativeRegisterContextWatchpoint_x86::ClearAllHardwareWatchpoints() { RegisterValue(dr7.GetAsUInt64() & ~watchpoint_all_control_bit_mask)); } -uint32_t NativeRegisterContextWatchpoint_x86::SetHardwareWatchpoint( +uint32_t NativeRegisterContextDBReg_x86::SetHardwareWatchpoint( lldb::addr_t addr, size_t size, uint32_t watch_flags) { Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_WATCHPOINTS)); const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints(); @@ -254,7 +253,7 @@ uint32_t NativeRegisterContextWatchpoint_x86::SetHardwareWatchpoint( return wp_index; } if (error.Fail() && log) { - LLDB_LOGF(log, "NativeRegisterContextWatchpoint_x86::%s Error: %s", + LLDB_LOGF(log, "NativeRegisterContextDBReg_x86::%s Error: %s", __FUNCTION__, error.AsCString()); } } @@ -262,7 +261,7 @@ uint32_t NativeRegisterContextWatchpoint_x86::SetHardwareWatchpoint( } lldb::addr_t -NativeRegisterContextWatchpoint_x86::GetWatchpointAddress(uint32_t wp_index) { +NativeRegisterContextDBReg_x86::GetWatchpointAddress(uint32_t wp_index) { if (wp_index >= NumSupportedHardwareWatchpoints()) return LLDB_INVALID_ADDRESS; RegisterValue drN; @@ -271,8 +270,7 @@ NativeRegisterContextWatchpoint_x86::GetWatchpointAddress(uint32_t wp_index) { return drN.GetAsUInt64(); } -uint32_t -NativeRegisterContextWatchpoint_x86::NumSupportedHardwareWatchpoints() { +uint32_t NativeRegisterContextDBReg_x86::NumSupportedHardwareWatchpoints() { // Available debug address registers: dr0, dr1, dr2, dr3 return 4; } diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextWatchpoint_x86.h b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_x86.h index cfb8900a4fd2..c0c6ce29eab5 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextWatchpoint_x86.h +++ b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/NativeRegisterContextDBReg_x86.h @@ -1,4 +1,4 @@ -//===-- NativeRegisterContextWatchpoint_x86.h -------------------*- C++ -*-===// +//===-- NativeRegisterContextDBReg_x86.h ------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,14 +6,14 @@ // //===----------------------------------------------------------------------===// -#ifndef lldb_NativeRegisterContextWatchpoint_x86_h -#define lldb_NativeRegisterContextWatchpoint_x86_h +#ifndef lldb_NativeRegisterContextDBReg_x86_h +#define lldb_NativeRegisterContextDBReg_x86_h #include "Plugins/Process/Utility/NativeRegisterContextRegisterInfo.h" namespace lldb_private { -class NativeRegisterContextWatchpoint_x86 +class NativeRegisterContextDBReg_x86 : public virtual NativeRegisterContextRegisterInfo { public: Status IsWatchpointHit(uint32_t wp_index, bool &is_hit) override; @@ -45,4 +45,4 @@ public: } // namespace lldb_private -#endif // #ifndef lldb_NativeRegisterContextWatchpoint_x86_h +#endif // #ifndef lldb_NativeRegisterContextDBReg_x86_h diff --git a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/RegisterInfos_powerpc.h b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/RegisterInfos_powerpc.h index 51be31f8e028..90863dfdb090 100644 --- a/contrib/llvm-project/lldb/source/Plugins/Process/Utility/RegisterInfos_powerpc.h +++ b/contrib/llvm-project/lldb/source/Plugins/Process/Utility/RegisterInfos_powerpc.h @@ -10,8 +10,8 @@ // Computes the offset of the given GPR in the user data area. #define GPR_OFFSET(regname) (offsetof(GPR, regname)) -#define FPR_OFFSET(regname) (offsetof(FPR, regname)) -#define VMX_OFFSET(regname) (offsetof(VMX, regname)) +#define FPR_OFFSET(regname) (sizeof(GPR) + offsetof(FPR, regname)) +#define VMX_OFFSET(regname) (sizeof(GPR) + sizeof(FPR) + offsetof(VMX, regname)) #define GPR_SIZE(regname) (sizeof(((GPR *)NULL)->regname)) #ifdef DECLARE_REGISTER_INFOS_POWERPC_STRUCT diff --git a/contrib/llvm-project/lldb/tools/lldb-server/lldb-gdbserver.cpp b/contrib/llvm-project/lldb/tools/lldb-server/lldb-gdbserver.cpp index 9b8c67af14db..a9db73f90098 100644 --- a/contrib/llvm-project/lldb/tools/lldb-server/lldb-gdbserver.cpp +++ b/contrib/llvm-project/lldb/tools/lldb-server/lldb-gdbserver.cpp @@ -40,7 +40,7 @@ #if defined(__linux__) #include "Plugins/Process/Linux/NativeProcessLinux.h" #elif defined(__FreeBSD__) -#include "Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.h" +#include "Plugins/Process/FreeBSD/NativeProcessFreeBSD.h" #elif defined(__NetBSD__) #include "Plugins/Process/NetBSD/NativeProcessNetBSD.h" #elif defined(_WIN32) diff --git a/contrib/llvm-project/llvm/include/llvm-c/Core.h b/contrib/llvm-project/llvm/include/llvm-c/Core.h index 8274213aa839..a78df16ca404 100644 --- a/contrib/llvm-project/llvm/include/llvm-c/Core.h +++ b/contrib/llvm-project/llvm/include/llvm-c/Core.h @@ -160,10 +160,10 @@ typedef enum { LLVMVectorTypeKind, /**< Fixed width SIMD vector type */ LLVMMetadataTypeKind, /**< Metadata */ LLVMX86_MMXTypeKind, /**< X86 MMX */ - LLVMX86_AMXTypeKind, /**< X86 AMX */ LLVMTokenTypeKind, /**< Tokens */ LLVMScalableVectorTypeKind, /**< Scalable SIMD vector type */ - LLVMBFloatTypeKind /**< 16 bit brain floating point type */ + LLVMBFloatTypeKind, /**< 16 bit brain floating point type */ + LLVMX86_AMXTypeKind /**< X86 AMX */ } LLVMTypeKind; typedef enum { @@ -270,7 +270,6 @@ typedef enum { LLVMConstantVectorValueKind, LLVMUndefValueValueKind, - LLVMPoisonValueValueKind, LLVMConstantAggregateZeroValueKind, LLVMConstantDataArrayValueKind, LLVMConstantDataVectorValueKind, @@ -283,6 +282,7 @@ typedef enum { LLVMInlineAsmValueKind, LLVMInstructionValueKind, + LLVMPoisonValueValueKind } LLVMValueKind; typedef enum { diff --git a/contrib/llvm-project/llvm/include/llvm-c/Orc.h b/contrib/llvm-project/llvm/include/llvm-c/Orc.h index 183107c148a6..9beef44c89dd 100644 --- a/contrib/llvm-project/llvm/include/llvm-c/Orc.h +++ b/contrib/llvm-project/llvm/include/llvm-c/Orc.h @@ -339,8 +339,7 @@ LLVMErrorRef LLVMOrcResourceTrackerRemove(LLVMOrcResourceTrackerRef RT); * ownership has not been passed to a JITDylib (e.g. because some error * prevented the client from calling LLVMOrcJITDylibAddGenerator). */ -void LLVMOrcDisposeDefinitionGenerator( - LLVMOrcDefinitionGeneratorRef DG); +void LLVMOrcDisposeDefinitionGenerator(LLVMOrcDefinitionGeneratorRef DG); /** * Dispose of a MaterializationUnit. @@ -388,7 +387,9 @@ LLVMOrcExecutionSessionCreateJITDylib(LLVMOrcExecutionSessionRef ES, * Returns the JITDylib with the given name, or NULL if no such JITDylib * exists. */ -LLVMOrcJITDylibRef LLVMOrcExecutionSessionGetJITDylibByName(const char *Name); +LLVMOrcJITDylibRef +LLVMOrcExecutionSessionGetJITDylibByName(LLVMOrcExecutionSessionRef ES, + const char *Name); /** * Return a reference to a newly created resource tracker associated with JD. diff --git a/contrib/llvm-project/llvm/include/llvm/Analysis/AssumptionCache.h b/contrib/llvm-project/llvm/include/llvm/Analysis/AssumptionCache.h index 0ef63dc68e1c..c4602d3449c0 100644 --- a/contrib/llvm-project/llvm/include/llvm/Analysis/AssumptionCache.h +++ b/contrib/llvm-project/llvm/include/llvm/Analysis/AssumptionCache.h @@ -45,7 +45,7 @@ public: enum : unsigned { ExprResultIdx = std::numeric_limits<unsigned>::max() }; struct ResultElem { - WeakTrackingVH Assume; + WeakVH Assume; /// contains either ExprResultIdx or the index of the operand bundle /// containing the knowledge. diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h index 81c1d6aad49a..26bf4ab2618c 100644 --- a/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h +++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h @@ -490,7 +490,10 @@ protected: /// - \c Add has a constant operand. bool canFoldAddIntoGEP(const User *GEP, const Value *Add); - /// Test whether the given value has exactly one use. + /// Test whether the register associated with this value has exactly one use, + /// in which case that single use is killing. Note that multiple IR values + /// may map onto the same register, in which case this is not the same as + /// checking that an IR value has one use. bool hasTrivialKill(const Value *V); /// Create a machine mem operand from the given instruction. diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/MachineInstr.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/MachineInstr.h index 6bbe2d03f9e5..f8d97c2c07a6 100644 --- a/contrib/llvm-project/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/MachineInstr.h @@ -1156,6 +1156,10 @@ public: return getOpcode() == TargetOpcode::CFI_INSTRUCTION; } + bool isPseudoProbe() const { + return getOpcode() == TargetOpcode::PSEUDO_PROBE; + } + // True if the instruction represents a position in the function. bool isPosition() const { return isLabel() || isCFIInstruction(); } @@ -1165,6 +1169,9 @@ public: bool isDebugInstr() const { return isDebugValue() || isDebugLabel() || isDebugRef(); } + bool isDebugOrPseudoInstr() const { + return isDebugInstr() || isPseudoProbe(); + } bool isDebugOffsetImm() const { return getDebugOffset().isImm(); } diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h index c3221aac8eea..40115fbd2f15 100644 --- a/contrib/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h @@ -2785,6 +2785,10 @@ public: return false; } + /// Does this target require the clearing of high-order bits in a register + /// passed to the fp16 to fp conversion library function. + virtual bool shouldKeepZExtForFP16Conv() const { return false; } + //===--------------------------------------------------------------------===// // Runtime Library hooks // diff --git a/contrib/llvm-project/llvm/include/llvm/Demangle/ItaniumDemangle.h b/contrib/llvm-project/llvm/include/llvm/Demangle/ItaniumDemangle.h index 6bfc02d15379..e5fca98f9271 100644 --- a/contrib/llvm-project/llvm/include/llvm/Demangle/ItaniumDemangle.h +++ b/contrib/llvm-project/llvm/include/llvm/Demangle/ItaniumDemangle.h @@ -96,7 +96,6 @@ X(InitListExpr) \ X(FoldExpr) \ X(ThrowExpr) \ - X(UUIDOfExpr) \ X(BoolExpr) \ X(StringLiteral) \ X(LambdaExpr) \ @@ -2035,21 +2034,6 @@ public: } }; -// MSVC __uuidof extension, generated by clang in -fms-extensions mode. -class UUIDOfExpr : public Node { - Node *Operand; -public: - UUIDOfExpr(Node *Operand_) : Node(KUUIDOfExpr), Operand(Operand_) {} - - template<typename Fn> void match(Fn F) const { F(Operand); } - - void printLeft(OutputStream &S) const override { - S << "__uuidof("; - Operand->print(S); - S << ")"; - } -}; - class BoolExpr : public Node { bool Value; @@ -5013,6 +4997,43 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() { } } return nullptr; + case 'u': { + ++First; + Node *Name = getDerived().parseSourceName(/*NameState=*/nullptr); + if (!Name) + return nullptr; + // Special case legacy __uuidof mangling. The 't' and 'z' appear where the + // standard encoding expects a <template-arg>, and would be otherwise be + // interpreted as <type> node 'short' or 'ellipsis'. However, neither + // __uuidof(short) nor __uuidof(...) can actually appear, so there is no + // actual conflict here. + if (Name->getBaseName() == "__uuidof") { + if (numLeft() < 2) + return nullptr; + if (*First == 't') { + ++First; + Node *Ty = getDerived().parseType(); + if (!Ty) + return nullptr; + return make<CallExpr>(Name, makeNodeArray(&Ty, &Ty + 1)); + } + if (*First == 'z') { + ++First; + Node *Ex = getDerived().parseExpr(); + if (!Ex) + return nullptr; + return make<CallExpr>(Name, makeNodeArray(&Ex, &Ex + 1)); + } + } + size_t ExprsBegin = Names.size(); + while (!consumeIf('E')) { + Node *E = getDerived().parseTemplateArg(); + if (E == nullptr) + return E; + Names.push_back(E); + } + return make<CallExpr>(Name, popTrailingNodeArray(ExprsBegin)); + } case '1': case '2': case '3': @@ -5024,21 +5045,6 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() { case '9': return getDerived().parseUnresolvedName(); } - - if (consumeIf("u8__uuidoft")) { - Node *Ty = getDerived().parseType(); - if (!Ty) - return nullptr; - return make<UUIDOfExpr>(Ty); - } - - if (consumeIf("u8__uuidofz")) { - Node *Ex = getDerived().parseExpr(); - if (!Ex) - return nullptr; - return make<UUIDOfExpr>(Ex); - } - return nullptr; } diff --git a/contrib/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/contrib/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def index 844046167975..75d360bf4237 100644 --- a/contrib/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def +++ b/contrib/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def @@ -375,7 +375,7 @@ __OMP_RTL(__kmpc_init_allocator, false, /* omp_allocator_handle_t */ VoidPtr, __OMP_RTL(__kmpc_destroy_allocator, false, Void, /* Int */ Int32, /* omp_allocator_handle_t */ VoidPtr) -__OMP_RTL(__kmpc_push_target_tripcount, false, Void, IdentPtr, Int64, Int64) +__OMP_RTL(__kmpc_push_target_tripcount_mapper, false, Void, IdentPtr, Int64, Int64) __OMP_RTL(__tgt_target_mapper, false, Int32, IdentPtr, Int64, VoidPtr, Int32, VoidPtrPtr, VoidPtrPtr, Int64Ptr, Int64Ptr, VoidPtrPtr, VoidPtrPtr) __OMP_RTL(__tgt_target_nowait_mapper, false, Int32, IdentPtr, Int64, VoidPtr, Int32, @@ -844,7 +844,7 @@ __OMP_RTL_ATTRS(__kmpc_free, AllocAttrs, AttributeSet(), {}) __OMP_RTL_ATTRS(__kmpc_init_allocator, DefaultAttrs, ReturnPtrAttrs, {}) __OMP_RTL_ATTRS(__kmpc_destroy_allocator, AllocAttrs, AttributeSet(), {}) -__OMP_RTL_ATTRS(__kmpc_push_target_tripcount, SetterAttrs, AttributeSet(), {}) +__OMP_RTL_ATTRS(__kmpc_push_target_tripcount_mapper, SetterAttrs, AttributeSet(), {}) __OMP_RTL_ATTRS(__tgt_target_mapper, ForkAttrs, AttributeSet(), {}) __OMP_RTL_ATTRS(__tgt_target_nowait_mapper, ForkAttrs, AttributeSet(), {}) __OMP_RTL_ATTRS(__tgt_target_teams_mapper, ForkAttrs, AttributeSet(), {}) diff --git a/contrib/llvm-project/llvm/include/llvm/IR/InstrTypes.h b/contrib/llvm-project/llvm/include/llvm/IR/InstrTypes.h index f42ef48de6b3..955ac8e537fe 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/InstrTypes.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/InstrTypes.h @@ -1757,9 +1757,6 @@ public: return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly); } - /// Returns true if this function is guaranteed to return. - bool willReturn() const { return hasFnAttr(Attribute::WillReturn); } - void setOnlyReadsMemory() { addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly); } diff --git a/contrib/llvm-project/llvm/include/llvm/IR/Instruction.h b/contrib/llvm-project/llvm/include/llvm/IR/Instruction.h index d2a55f89fac9..b99dc62bbb9d 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/Instruction.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/Instruction.h @@ -633,6 +633,10 @@ public: /// generated program. bool isSafeToRemove() const; + /// Return true if the instruction will return (unwinding is considered as + /// a form of returning control flow here). + bool willReturn() const; + /// Return true if the instruction is a variety of EH-block. bool isEHPad() const { switch (getOpcode()) { @@ -650,6 +654,9 @@ public: /// llvm.lifetime.end marker. bool isLifetimeStartOrEnd() const; + /// Return true if the instruction is a DbgInfoIntrinsic or PseudoProbeInst. + bool isDebugOrPseudoInst() const; + /// Return a pointer to the next non-debug instruction in the same basic /// block as 'this', or nullptr if no such instruction exists. Skip any pseudo /// operations if \c SkipPseudoOp is true. diff --git a/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h b/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h index 9d68f3fdde6c..df3a1d568756 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h @@ -981,12 +981,16 @@ public: return cast<ConstantInt>(const_cast<Value *>(getArgOperand(0))); } + ConstantInt *getIndex() const { + return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1))); + } + ConstantInt *getAttributes() const { return cast<ConstantInt>(const_cast<Value *>(getArgOperand(2))); } - ConstantInt *getIndex() const { - return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1))); + ConstantInt *getFactor() const { + return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3))); } }; diff --git a/contrib/llvm-project/llvm/include/llvm/IR/Intrinsics.td b/contrib/llvm-project/llvm/include/llvm/IR/Intrinsics.td index b2bfc6e6f9e6..21307ed1bd91 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/Intrinsics.td +++ b/contrib/llvm-project/llvm/include/llvm/IR/Intrinsics.td @@ -1298,7 +1298,7 @@ def int_sideeffect : DefaultAttrsIntrinsic<[], [], [IntrInaccessibleMemOnly, Int // Like the sideeffect intrinsic defined above, this intrinsic is treated by the // optimizer as having opaque side effects so that it won't be get rid of or moved // out of the block it probes. -def int_pseudoprobe : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty], +def int_pseudoprobe : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty, llvm_i64_ty], [IntrInaccessibleMemOnly, IntrWillReturn]>; // Intrinsics to support half precision floating point format diff --git a/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsRISCV.td b/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsRISCV.td index ab5b09b72ac3..c4056895f68e 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -790,6 +790,9 @@ let TargetPrefix = "riscv" in { defm vsoxei : RISCVIStore; defm vsuxei : RISCVIStore; + def int_riscv_vle1 : RISCVUSLoad; + def int_riscv_vse1 : RISCVUSStore; + defm vamoswap : RISCVAMO; defm vamoadd : RISCVAMO; defm vamoxor : RISCVAMO; @@ -940,8 +943,8 @@ let TargetPrefix = "riscv" in { defm vfwnmsac : RISCVTernaryWide; defm vfsqrt : RISCVUnaryAA; - defm vfrsqrte7 : RISCVUnaryAA; - defm vfrece7 : RISCVUnaryAA; + defm vfrsqrt7 : RISCVUnaryAA; + defm vfrec7 : RISCVUnaryAA; defm vfmin : RISCVBinaryAAX; defm vfmax : RISCVBinaryAAX; diff --git a/contrib/llvm-project/llvm/include/llvm/IR/Metadata.h b/contrib/llvm-project/llvm/include/llvm/IR/Metadata.h index 0b87416befe9..9a4480b75a30 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/Metadata.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/Metadata.h @@ -667,6 +667,12 @@ struct AAMDNodes { /// The tag specifying the noalias scope. MDNode *NoAlias = nullptr; + // Shift tbaa Metadata node to start off bytes later + static MDNode *ShiftTBAA(MDNode *M, size_t off); + + // Shift tbaa.struct Metadata node to start off bytes later + static MDNode *ShiftTBAAStruct(MDNode *M, size_t off); + /// Given two sets of AAMDNodes that apply to the same pointer, /// give the best AAMDNodes that are compatible with both (i.e. a set of /// nodes whose allowable aliasing conclusions are a subset of those @@ -680,6 +686,18 @@ struct AAMDNodes { Result.NoAlias = Other.NoAlias == NoAlias ? NoAlias : nullptr; return Result; } + + /// Create a new AAMDNode that describes this AAMDNode after applying a + /// constant offset to the start of the pointer + AAMDNodes shift(size_t Offset) { + AAMDNodes Result; + Result.TBAA = TBAA ? ShiftTBAA(TBAA, Offset) : nullptr; + Result.TBAAStruct = + TBAAStruct ? ShiftTBAAStruct(TBAAStruct, Offset) : nullptr; + Result.Scope = Scope; + Result.NoAlias = NoAlias; + return Result; + } }; // Specialize DenseMapInfo for AAMDNodes. diff --git a/contrib/llvm-project/llvm/include/llvm/IR/Operator.h b/contrib/llvm-project/llvm/include/llvm/IR/Operator.h index acfacbd6c74e..945f7e46e142 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/Operator.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/Operator.h @@ -568,6 +568,11 @@ public: bool accumulateConstantOffset( const DataLayout &DL, APInt &Offset, function_ref<bool(Value &, APInt &)> ExternalAnalysis = nullptr) const; + + static bool accumulateConstantOffset( + Type *SourceType, ArrayRef<const Value *> Index, const DataLayout &DL, + APInt &Offset, + function_ref<bool(Value &, APInt &)> ExternalAnalysis = nullptr); }; class PtrToIntOperator diff --git a/contrib/llvm-project/llvm/include/llvm/IR/PseudoProbe.h b/contrib/llvm-project/llvm/include/llvm/IR/PseudoProbe.h index e0370c264102..5165e80caa2d 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/PseudoProbe.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/PseudoProbe.h @@ -16,28 +16,39 @@ #include "llvm/ADT/Optional.h" #include <cassert> #include <cstdint> +#include <limits> namespace llvm { class Instruction; +class BasicBlock; constexpr const char *PseudoProbeDescMetadataName = "llvm.pseudo_probe_desc"; enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall }; +// The saturated distrution factor representing 100% for block probes. +constexpr static uint64_t PseudoProbeFullDistributionFactor = + std::numeric_limits<uint64_t>::max(); + struct PseudoProbeDwarfDiscriminator { +public: // The following APIs encodes/decodes per-probe information to/from a // 32-bit integer which is organized as: // [2:0] - 0x7, this is reserved for regular discriminator, // see DWARF discriminator encoding rule // [18:3] - probe id - // [25:19] - reserved + // [25:19] - probe distribution factor // [28:26] - probe type, see PseudoProbeType // [31:29] - reserved for probe attributes - static uint32_t packProbeData(uint32_t Index, uint32_t Type) { + static uint32_t packProbeData(uint32_t Index, uint32_t Type, uint32_t Flags, + uint32_t Factor) { assert(Index <= 0xFFFF && "Probe index too big to encode, exceeding 2^16"); assert(Type <= 0x7 && "Probe type too big to encode, exceeding 7"); - return (Index << 3) | (Type << 26) | 0x7; + assert(Flags <= 0x7); + assert(Factor <= 100 && + "Probe distribution factor too big to encode, exceeding 100"); + return (Index << 3) | (Factor << 19) | (Type << 26) | 0x7; } static uint32_t extractProbeIndex(uint32_t Value) { @@ -51,16 +62,26 @@ struct PseudoProbeDwarfDiscriminator { static uint32_t extractProbeAttributes(uint32_t Value) { return (Value >> 29) & 0x7; } + + static uint32_t extractProbeFactor(uint32_t Value) { + return (Value >> 19) & 0x7F; + } + + // The saturated distrution factor representing 100% for callsites. + constexpr static uint8_t FullDistributionFactor = 100; }; struct PseudoProbe { uint32_t Id; uint32_t Type; uint32_t Attr; + float Factor; }; Optional<PseudoProbe> extractProbe(const Instruction &Inst); +void setProbeDistributionFactor(Instruction &Inst, float Factor); + } // end namespace llvm #endif // LLVM_IR_PSEUDOPROBE_H diff --git a/contrib/llvm-project/llvm/include/llvm/Passes/StandardInstrumentations.h b/contrib/llvm-project/llvm/include/llvm/Passes/StandardInstrumentations.h index 795a980878e2..61c86b0468f2 100644 --- a/contrib/llvm-project/llvm/include/llvm/Passes/StandardInstrumentations.h +++ b/contrib/llvm-project/llvm/include/llvm/Passes/StandardInstrumentations.h @@ -22,6 +22,7 @@ #include "llvm/IR/PassTimingInfo.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Transforms/IPO/SampleProfileProbe.h" #include <string> #include <utility> @@ -273,6 +274,7 @@ class StandardInstrumentations { OptBisectInstrumentation OptBisect; PreservedCFGCheckerInstrumentation PreservedCFGChecker; IRChangedPrinter PrintChangedIR; + PseudoProbeVerifier PseudoProbeVerification; VerifyInstrumentation Verify; bool VerifyEach; diff --git a/contrib/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h b/contrib/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h index 6bb5825339ae..55b94b2e690d 100644 --- a/contrib/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h +++ b/contrib/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Error.h" #include <algorithm> #include <cstdint> @@ -89,6 +90,8 @@ public: void addRecord(const sampleprof::FunctionSamples &FS, bool isCallsiteSample = false); + std::unique_ptr<ProfileSummary> computeSummaryForProfiles( + const StringMap<sampleprof::FunctionSamples> &Profiles); std::unique_ptr<ProfileSummary> getSummary(); }; diff --git a/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h b/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h index c45ace9e68c1..25d5b2376c11 100644 --- a/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h +++ b/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h @@ -347,6 +347,16 @@ public: return SortedTargets; } + /// Prorate call targets by a distribution factor. + static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets, + float DistributionFactor) { + CallTargetMap AdjustedTargets; + for (const auto &I : Targets) { + AdjustedTargets[I.first()] = I.second * DistributionFactor; + } + return AdjustedTargets; + } + /// Merge the samples in \p Other into this record. /// Optionally scale sample counts by \p Weight. sampleprof_error merge(const SampleRecord &Other, uint64_t Weight = 1) { @@ -439,9 +449,11 @@ public: void clearState(ContextStateMask S) { State &= (uint32_t)~S; } bool hasContext() const { return State != UnknownContext; } bool isBaseContext() const { return CallingContext.empty(); } - StringRef getName() const { return Name; } + StringRef getNameWithoutContext() const { return Name; } StringRef getCallingContext() const { return CallingContext; } - StringRef getNameWithContext() const { return FullContext; } + StringRef getNameWithContext(bool WithBracket = false) const { + return WithBracket ? InputContext : FullContext; + } private: // Give a context string, decode and populate internal states like @@ -449,6 +461,7 @@ private: // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]` void setContext(StringRef ContextStr, ContextStateMask CState) { assert(!ContextStr.empty()); + InputContext = ContextStr; // Note that `[]` wrapped input indicates a full context string, otherwise // it's treated as context-less function name only. bool HasContext = ContextStr.startswith("["); @@ -480,6 +493,9 @@ private: } } + // Input context string including bracketed calling context and leaf function + // name + StringRef InputContext; // Full context string including calling context and leaf function name StringRef FullContext; // Function name for the associated sample profile @@ -676,7 +692,8 @@ public: Name = Other.getName(); if (!GUIDToFuncNameMap) GUIDToFuncNameMap = Other.GUIDToFuncNameMap; - + if (Context.getNameWithContext(true).empty()) + Context = Other.getContext(); if (FunctionHash == 0) { // Set the function hash code for the target profile. FunctionHash = Other.getFunctionHash(); @@ -743,8 +760,10 @@ public: StringRef getName() const { return Name; } /// Return function name with context. - StringRef getNameWithContext() const { - return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name; + StringRef getNameWithContext(bool WithBracket = false) const { + return FunctionSamples::ProfileIsCS + ? Context.getNameWithContext(WithBracket) + : Name; } /// Return the original function name. diff --git a/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h b/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h index 3f52a2f6163b..999e75eddffa 100644 --- a/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -488,8 +488,12 @@ protected: /// \brief Whether samples are collected based on pseudo probes. bool ProfileIsProbeBased = false; + /// Whether function profiles are context-sensitive. bool ProfileIsCS = false; + /// Number of context-sensitive profiles. + uint32_t CSProfileCount = 0; + /// \brief The format of sample. SampleProfileFormat Format = SPF_None; }; diff --git a/contrib/llvm-project/llvm/include/llvm/Support/CommandLine.h b/contrib/llvm-project/llvm/include/llvm/Support/CommandLine.h index 38f3e188be55..0706aa226c0e 100644 --- a/contrib/llvm-project/llvm/include/llvm/Support/CommandLine.h +++ b/contrib/llvm-project/llvm/include/llvm/Support/CommandLine.h @@ -369,9 +369,22 @@ public: virtual void setDefault() = 0; + // Prints the help string for an option. + // + // This maintains the Indent for multi-line descriptions. + // FirstLineIndentedBy is the count of chars of the first line + // i.e. the one containing the --<option name>. static void printHelpStr(StringRef HelpStr, size_t Indent, size_t FirstLineIndentedBy); + // Prints the help string for an enum value. + // + // This maintains the Indent for multi-line descriptions. + // FirstLineIndentedBy is the count of chars of the first line + // i.e. the one containing the =<value>. + static void printEnumValHelpStr(StringRef HelpStr, size_t Indent, + size_t FirstLineIndentedBy); + virtual void getExtraOptionNames(SmallVectorImpl<StringRef> &) {} // addOccurrence - Wrapper around handleOccurrence that enforces Flags. diff --git a/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h index 5b2600144fa3..da0bdae0eaee 100644 --- a/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h +++ b/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h @@ -18,11 +18,13 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/CallGraph.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/Instructions.h" #include "llvm/ProfileData/SampleProf.h" #include <list> #include <map> +#include <vector> using namespace llvm; using namespace sampleprof; @@ -42,7 +44,7 @@ public: CallSiteLoc(CallLoc){}; ContextTrieNode *getChildContext(const LineLocation &CallSite, StringRef CalleeName); - ContextTrieNode *getChildContext(const LineLocation &CallSite); + ContextTrieNode *getHottestChildContext(const LineLocation &CallSite); ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite, StringRef CalleeName, bool AllowCreate = true); @@ -89,16 +91,24 @@ private: // calling context and the context is identified by path from root to the node. class SampleContextTracker { public: + using ContextSamplesTy = SmallSet<FunctionSamples *, 16>; + SampleContextTracker(StringMap<FunctionSamples> &Profiles); // Query context profile for a specific callee with given name at a given // call-site. The full context is identified by location of call instruction. FunctionSamples *getCalleeContextSamplesFor(const CallBase &Inst, StringRef CalleeName); + // Get samples for indirect call targets for call site at given location. + std::vector<const FunctionSamples *> + getIndirectCalleeContextSamplesFor(const DILocation *DIL); // Query context profile for a given location. The full context // is identified by input DILocation. FunctionSamples *getContextSamplesFor(const DILocation *DIL); // Query context profile for a given sample contxt of a function. FunctionSamples *getContextSamplesFor(const SampleContext &Context); + // Get all context profile for given function. + ContextSamplesTy &getAllContextSamplesFor(const Function &Func); + ContextSamplesTy &getAllContextSamplesFor(StringRef Name); // Query base profile for a given function. A base profile is a merged view // of all context profiles for contexts that are not inlined. FunctionSamples *getBaseSamplesFor(const Function &Func, @@ -109,6 +119,9 @@ public: // This makes sure that inlined context profile will be excluded in // function's base profile. void markContextSamplesInlined(const FunctionSamples *InlinedSamples); + void promoteMergeContextSamplesTree(const Instruction &Inst, + StringRef CalleeName); + void addCallGraphEdges(CallGraph &CG, StringMap<Function *> &SymbolMap); // Dump the internal context profile trie. void dump(); @@ -122,8 +135,6 @@ private: ContextTrieNode *getTopLevelContextNode(StringRef FName); ContextTrieNode &addTopLevelContextNode(StringRef FName); ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo); - void promoteMergeContextSamplesTree(const Instruction &Inst, - StringRef CalleeName); void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode, StringRef ContextStrToRemove); ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &FromNode, @@ -131,7 +142,7 @@ private: StringRef ContextStrToRemove); // Map from function name to context profiles (excluding base profile) - StringMap<SmallSet<FunctionSamples *, 16>> FuncToCtxtProfileSet; + StringMap<ContextSamplesTy> FuncToCtxtProfileSet; // Root node for context trie tree ContextTrieNode RootContext; diff --git a/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h index 78117fd4a9c2..0fd79d8ff7f3 100644 --- a/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h +++ b/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h @@ -16,6 +16,10 @@ #define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H #include "llvm/ADT/DenseMap.h" +#include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/IR/PassInstrumentation.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/PseudoProbe.h" #include "llvm/ProfileData/SampleProf.h" @@ -29,6 +33,8 @@ class Module; using namespace sampleprof; using BlockIdMap = std::unordered_map<BasicBlock *, uint32_t>; using InstructionIdMap = std::unordered_map<Instruction *, uint32_t>; +using ProbeFactorMap = std::unordered_map<uint64_t, float>; +using FuncProbeFactorMap = StringMap<ProbeFactorMap>; enum class PseudoProbeReservedId { Invalid = 0, Last = Invalid }; @@ -43,6 +49,33 @@ public: uint64_t getFunctionHash() const { return FunctionHash; } }; +// A pseudo probe verifier that can be run after each IR passes to detect the +// violation of updating probe factors. In principle, the sum of distribution +// factor for a probe should be identical before and after a pass. For a +// function pass, the factor sum for a probe would be typically 100%. +class PseudoProbeVerifier { +public: + void registerCallbacks(PassInstrumentationCallbacks &PIC); + + // Implementation of pass instrumentation callbacks for new pass manager. + void runAfterPass(StringRef PassID, Any IR); + +private: + // Allow a little bias due the rounding to integral factors. + constexpr static float DistributionFactorVariance = 0.02f; + // Distribution factors from last pass. + FuncProbeFactorMap FunctionProbeFactors; + + void collectProbeFactors(const BasicBlock *BB, ProbeFactorMap &ProbeFactors); + void runAfterPass(const Module *M); + void runAfterPass(const LazyCallGraph::SCC *C); + void runAfterPass(const Function *F); + void runAfterPass(const Loop *L); + bool shouldVerifyFunction(const Function *F); + void verifyProbeFactors(const Function *F, + const ProbeFactorMap &ProbeFactors); +}; + // This class serves sample counts correlation for SampleProfileLoader by // analyzing pseudo probes and their function descriptors injected by // SampleProfileProber. @@ -102,5 +135,13 @@ public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; +class PseudoProbeUpdatePass : public PassInfoMixin<PseudoProbeUpdatePass> { + void runOnFunction(Function &F, FunctionAnalysisManager &FAM); + +public: + PseudoProbeUpdatePass() {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + } // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H diff --git a/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h b/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h index 56aaa5d48e2a..aa960c625630 100644 --- a/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -274,6 +274,13 @@ void updateProfileCallee( void identifyNoAliasScopesToClone( ArrayRef<BasicBlock *> BBs, SmallVectorImpl<MDNode *> &NoAliasDeclScopes); +/// Find the 'llvm.experimental.noalias.scope.decl' intrinsics in the specified +/// instruction range and extract their scope. These are candidates for +/// duplication when cloning. +void identifyNoAliasScopesToClone( + BasicBlock::iterator Start, BasicBlock::iterator End, + SmallVectorImpl<MDNode *> &NoAliasDeclScopes); + /// Duplicate the specified list of noalias decl scopes. /// The 'Ext' string is added as an extension to the name. /// Afterwards, the ClonedScopes contains the mapping of the original scope diff --git a/contrib/llvm-project/llvm/lib/Analysis/DemandedBits.cpp b/contrib/llvm-project/llvm/lib/Analysis/DemandedBits.cpp index 461fd7239905..dd11b0b02bf8 100644 --- a/contrib/llvm-project/llvm/lib/Analysis/DemandedBits.cpp +++ b/contrib/llvm-project/llvm/lib/Analysis/DemandedBits.cpp @@ -80,7 +80,7 @@ void DemandedBitsWrapperPass::print(raw_ostream &OS, const Module *M) const { static bool isAlwaysLive(Instruction *I) { return I->isTerminator() || isa<DbgInfoIntrinsic>(I) || I->isEHPad() || - I->mayHaveSideEffects(); + I->mayHaveSideEffects() || !I->willReturn(); } void DemandedBits::determineLiveOperandBits( diff --git a/contrib/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp b/contrib/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp index 7f311d8f9a2b..94a24ccf2155 100644 --- a/contrib/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp +++ b/contrib/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp @@ -243,11 +243,14 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, if (RecurrenceType->isFloatingPointTy()) { if (!isFloatingPointRecurrenceKind(Kind)) return false; - } else { + } else if (RecurrenceType->isIntegerTy()) { if (!isIntegerRecurrenceKind(Kind)) return false; if (isArithmeticRecurrenceKind(Kind)) Start = lookThroughAnd(Phi, RecurrenceType, VisitedInsts, CastInsts); + } else { + // Pointer min/max may exist, but it is not supported as a reduction op. + return false; } Worklist.push_back(Start); diff --git a/contrib/llvm-project/llvm/lib/Analysis/MemorySSA.cpp b/contrib/llvm-project/llvm/lib/Analysis/MemorySSA.cpp index 52dca7d378e1..4722b68e20e9 100644 --- a/contrib/llvm-project/llvm/lib/Analysis/MemorySSA.cpp +++ b/contrib/llvm-project/llvm/lib/Analysis/MemorySSA.cpp @@ -281,7 +281,6 @@ instructionClobbersQuery(const MemoryDef *MD, const MemoryLocation &UseLoc, // clobbers where they don't really exist at all. Please see D43269 for // context. switch (II->getIntrinsicID()) { - case Intrinsic::lifetime_end: case Intrinsic::invariant_start: case Intrinsic::invariant_end: case Intrinsic::assume: @@ -358,22 +357,6 @@ struct UpwardsMemoryQuery { } // end anonymous namespace -static bool lifetimeEndsAt(MemoryDef *MD, const MemoryLocation &Loc, - BatchAAResults &AA) { - Instruction *Inst = MD->getMemoryInst(); - if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { - switch (II->getIntrinsicID()) { - case Intrinsic::lifetime_end: { - MemoryLocation ArgLoc = MemoryLocation::getAfter(II->getArgOperand(1)); - return AA.alias(ArgLoc, Loc) == MustAlias; - } - default: - return false; - } - } - return false; -} - template <typename AliasAnalysisType> static bool isUseTriviallyOptimizableToLiveOnEntry(AliasAnalysisType &AA, const Instruction *I) { @@ -1465,15 +1448,6 @@ void MemorySSA::OptimizeUses::optimizeUsesInBlock( } MemoryDef *MD = cast<MemoryDef>(VersionStack[UpperBound]); - // If the lifetime of the pointer ends at this instruction, it's live on - // entry. - if (!UseMLOC.IsCall && lifetimeEndsAt(MD, UseMLOC.getLoc(), *AA)) { - // Reset UpperBound to liveOnEntryDef's place in the stack - UpperBound = 0; - FoundClobberResult = true; - LocInfo.AR = MustAlias; - break; - } ClobberAlias CA = instructionClobbersQuery(MD, MU, UseMLOC, *AA); if (CA.IsClobber) { FoundClobberResult = true; diff --git a/contrib/llvm-project/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp b/contrib/llvm-project/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp index 7d97fc5da9b0..268acb682cf1 100644 --- a/contrib/llvm-project/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp +++ b/contrib/llvm-project/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp @@ -737,3 +737,84 @@ bool TypeBasedAAWrapperPass::doFinalization(Module &M) { void TypeBasedAAWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); } + +MDNode *AAMDNodes::ShiftTBAA(MDNode *MD, size_t Offset) { + // Fast path if there's no offset + if (Offset == 0) + return MD; + // Fast path if there's no path tbaa node (and thus scalar) + if (!isStructPathTBAA(MD)) + return MD; + + TBAAStructTagNode Tag(MD); + SmallVector<Metadata *, 5> Sub; + Sub.push_back(MD->getOperand(0)); + Sub.push_back(MD->getOperand(1)); + ConstantInt *InnerOffset = mdconst::extract<ConstantInt>(MD->getOperand(2)); + + if (Tag.isNewFormat()) { + ConstantInt *InnerSize = mdconst::extract<ConstantInt>(MD->getOperand(3)); + + if (InnerOffset->getZExtValue() + InnerSize->getZExtValue() <= Offset) { + return nullptr; + } + + uint64_t NewSize = InnerSize->getZExtValue(); + uint64_t NewOffset = InnerOffset->getZExtValue() - Offset; + if (InnerOffset->getZExtValue() < Offset) { + NewOffset = 0; + NewSize -= Offset - InnerOffset->getZExtValue(); + } + + Sub.push_back(ConstantAsMetadata::get( + ConstantInt::get(InnerOffset->getType(), NewOffset))); + + Sub.push_back(ConstantAsMetadata::get( + ConstantInt::get(InnerSize->getType(), NewSize))); + + // immutable type + if (MD->getNumOperands() >= 5) + Sub.push_back(MD->getOperand(4)); + } else { + if (InnerOffset->getZExtValue() < Offset) + return nullptr; + + Sub.push_back(ConstantAsMetadata::get(ConstantInt::get( + InnerOffset->getType(), InnerOffset->getZExtValue() - Offset))); + + // immutable type + if (MD->getNumOperands() >= 4) + Sub.push_back(MD->getOperand(3)); + } + return MDNode::get(MD->getContext(), Sub); +} + +MDNode *AAMDNodes::ShiftTBAAStruct(MDNode *MD, size_t Offset) { + // Fast path if there's no offset + if (Offset == 0) + return MD; + SmallVector<Metadata *, 3> Sub; + for (size_t i = 0, size = MD->getNumOperands(); i < size; i += 3) { + ConstantInt *InnerOffset = mdconst::extract<ConstantInt>(MD->getOperand(i)); + ConstantInt *InnerSize = + mdconst::extract<ConstantInt>(MD->getOperand(i + 1)); + // Don't include any triples that aren't in bounds + if (InnerOffset->getZExtValue() + InnerSize->getZExtValue() <= Offset) + continue; + + uint64_t NewSize = InnerSize->getZExtValue(); + uint64_t NewOffset = InnerOffset->getZExtValue() - Offset; + if (InnerOffset->getZExtValue() < Offset) { + NewOffset = 0; + NewSize -= Offset - InnerOffset->getZExtValue(); + } + + // Shift the offset of the triple + Sub.push_back(ConstantAsMetadata::get( + ConstantInt::get(InnerOffset->getType(), NewOffset))); + Sub.push_back(ConstantAsMetadata::get( + ConstantInt::get(InnerSize->getType(), NewSize))); + Sub.push_back(MD->getOperand(i + 2)); + } + return MDNode::get(MD->getContext(), Sub); +}
\ No newline at end of file diff --git a/contrib/llvm-project/llvm/lib/Analysis/ValueTracking.cpp b/contrib/llvm-project/llvm/lib/Analysis/ValueTracking.cpp index 5600a3b33750..e174c5efe424 100644 --- a/contrib/llvm-project/llvm/lib/Analysis/ValueTracking.cpp +++ b/contrib/llvm-project/llvm/lib/Analysis/ValueTracking.cpp @@ -5018,36 +5018,14 @@ bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) { // arbitrary length of time, but programs aren't allowed to rely on that. // If there is no successor, then execution can't transfer to it. - if (const auto *CRI = dyn_cast<CleanupReturnInst>(I)) - return !CRI->unwindsToCaller(); - if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) - return !CatchSwitch->unwindsToCaller(); - if (isa<ResumeInst>(I)) - return false; if (isa<ReturnInst>(I)) return false; if (isa<UnreachableInst>(I)) return false; - // Calls can throw, or contain an infinite loop, or kill the process. - if (const auto *CB = dyn_cast<CallBase>(I)) { - // Call sites that throw have implicit non-local control flow. - if (!CB->doesNotThrow()) - return false; - - // A function which doens't throw and has "willreturn" attribute will - // always return. - if (CB->hasFnAttr(Attribute::WillReturn)) - return true; - - // FIXME: Temporarily assume that all side-effect free intrinsics will - // return. Remove this workaround once all intrinsics are appropriately - // annotated. - return isa<IntrinsicInst>(CB) && CB->onlyReadsMemory(); - } - - // Other instructions return normally. - return true; + // An instruction that returns without throwing must transfer control flow + // to a successor. + return !I->mayThrow() && I->willReturn(); } bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index df0219fcfa64..a9353bdfb780 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -968,10 +968,11 @@ bool CombinerHelper::matchOptBrCondByInvertingCond(MachineInstr &MI) { if (BrCond->getOpcode() != TargetOpcode::G_BRCOND) return false; - // Check that the next block is the conditional branch target. - if (!MBB->isLayoutSuccessor(BrCond->getOperand(1).getMBB())) - return false; - return true; + // Check that the next block is the conditional branch target. Also make sure + // that it isn't the same as the G_BR's target (otherwise, this will loop.) + MachineBasicBlock *BrCondTarget = BrCond->getOperand(1).getMBB(); + return BrCondTarget != MI.getOperand(0).getMBB() && + MBB->isLayoutSuccessor(BrCondTarget); } void CombinerHelper::applyOptBrCondByInvertingCond(MachineInstr &MI) { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index e7f40523efaf..3178ee16af2b 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -1063,6 +1063,11 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, Observer.changedInstr(MI); return Legalized; case TargetOpcode::G_PHI: { + // FIXME: add support for when SizeOp0 isn't an exact multiple of + // NarrowSize. + if (SizeOp0 % NarrowSize != 0) + return UnableToLegalize; + unsigned NumParts = SizeOp0 / NarrowSize; SmallVector<Register, 2> DstRegs(NumParts); SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeShrink.cpp b/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeShrink.cpp index 26439a656917..7fa14fd902ef 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeShrink.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeShrink.cpp @@ -156,7 +156,8 @@ bool LiveRangeShrink::runOnMachineFunction(MachineFunction &MF) { // If MI has side effects, it should become a barrier for code motion. // IOM is rebuild from the next instruction to prevent later // instructions from being moved before this MI. - if (MI.hasUnmodeledSideEffects() && Next != MBB.end()) { + if (MI.hasUnmodeledSideEffects() && !MI.isPseudoProbe() && + Next != MBB.end()) { BuildInstOrderMap(Next, IOM); SawStore = false; } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineInstr.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineInstr.cpp index 59d98054e3a2..b6cfd7dcbfbc 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/MachineInstr.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineInstr.cpp @@ -1462,7 +1462,8 @@ bool MachineInstr::hasUnmodeledSideEffects() const { } bool MachineInstr::isLoadFoldBarrier() const { - return mayStore() || isCall() || hasUnmodeledSideEffects(); + return mayStore() || isCall() || + (hasUnmodeledSideEffects() && !isPseudoProbe()); } /// allDefsAreDead - Return true if all the defs of this instruction are dead. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 615bea2a4905..6a6f83827f72 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -6517,8 +6517,11 @@ static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift, // reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate // in direction shift1 by Neg. The range [0, EltSize) means that we only need // to consider shift amounts with defined behavior. +// +// The IsRotate flag should be set when the LHS of both shifts is the same. +// Otherwise if matching a general funnel shift, it should be clear. static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize, - SelectionDAG &DAG) { + SelectionDAG &DAG, bool IsRotate) { // If EltSize is a power of 2 then: // // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1) @@ -6550,8 +6553,11 @@ static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize, // always invokes undefined behavior for 32-bit X. // // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise. + // + // NOTE: We can only do this when matching an AND and not a general + // funnel shift. unsigned MaskLoBits = 0; - if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) { + if (IsRotate && Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) { if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) { KnownBits Known = DAG.computeKnownBits(Neg.getOperand(0)); unsigned Bits = Log2_64(EltSize); @@ -6641,7 +6647,8 @@ SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos, // (srl x, (*ext y))) -> // (rotr x, y) or (rotl x, (sub 32, y)) EVT VT = Shifted.getValueType(); - if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) { + if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG, + /*IsRotate*/ true)) { bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT); return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted, HasPos ? Pos : Neg); @@ -6670,7 +6677,7 @@ SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, // fold (or (shl x0, (*ext (sub 32, y))), // (srl x1, (*ext y))) -> // (fshr x0, x1, y) or (fshl x0, x1, (sub 32, y)) - if (matchRotateSub(InnerPos, InnerNeg, EltBits, DAG)) { + if (matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, /*IsRotate*/ N0 == N1)) { bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT); return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, N0, N1, HasPos ? Pos : Neg); @@ -21174,7 +21181,7 @@ SDValue DAGCombiner::visitFP16_TO_FP(SDNode *N) { SDValue N0 = N->getOperand(0); // fold fp16_to_fp(op & 0xffff) -> fp16_to_fp(op) - if (N0->getOpcode() == ISD::AND) { + if (!TLI.shouldKeepZExtForFP16Conv() && N0->getOpcode() == ISD::AND) { ConstantSDNode *AndConst = getAsNonOpaqueConstant(N0.getOperand(1)); if (AndConst && AndConst->getAPIntValue() == 0xffff) { return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index 62f7f3d98ba6..0ff77d4ba1ab 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -261,12 +261,16 @@ bool FastISel::hasTrivialKill(const Value *V) { if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0))) return false; + // Casts and extractvalues may be trivially coalesced by fast-isel. + if (I->getOpcode() == Instruction::BitCast || + I->getOpcode() == Instruction::PtrToInt || + I->getOpcode() == Instruction::IntToPtr || + I->getOpcode() == Instruction::ExtractValue) + return false; + // Only instructions with a single use in the same basic block are considered // to have trivial kills. return I->hasOneUse() && - !(I->getOpcode() == Instruction::BitCast || - I->getOpcode() == Instruction::PtrToInt || - I->getOpcode() == Instruction::IntToPtr) && cast<Instruction>(*I->user_begin())->getParent() == I->getParent(); } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 6638ff6a6358..a6bd774934ac 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -9660,8 +9660,9 @@ findArgumentCopyElisionCandidates(const DataLayout &DL, // We will look through cast uses, so ignore them completely. if (I.isCast()) continue; - // Ignore debug info intrinsics, they don't escape or store to allocas. - if (isa<DbgInfoIntrinsic>(I)) + // Ignore debug info and pseudo op intrinsics, they don't escape or store + // to allocas. + if (I.isDebugOrPseudoInst()) continue; // This is an unknown instruction. Assume it escapes or writes to all // static alloca operands. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 5760132e44a0..b0ad86899d25 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -2012,7 +2012,7 @@ bool TargetLowering::SimplifyDemandedBits( const APInt *ShAmtC = TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts); - if (!ShAmtC) + if (!ShAmtC || ShAmtC->uge(BitWidth)) break; uint64_t ShVal = ShAmtC->getZExtValue(); @@ -5935,6 +5935,11 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, SDLoc DL(Op); + // Because getNegatedExpression can delete nodes we need a handle to keep + // temporary nodes alive in case the recursion manages to create an identical + // node. + std::list<HandleSDNode> Handles; + switch (Opcode) { case ISD::ConstantFP: { // Don't invert constant FP values after legalization unless the target says @@ -6003,11 +6008,18 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, NegatibleCost CostX = NegatibleCost::Expensive; SDValue NegX = getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth); + // Prevent this node from being deleted by the next call. + if (NegX) + Handles.emplace_back(NegX); + // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X) NegatibleCost CostY = NegatibleCost::Expensive; SDValue NegY = getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth); + // We're done with the handles. + Handles.clear(); + // Negate the X if its cost is less or equal than Y. if (NegX && (CostX <= CostY)) { Cost = CostX; @@ -6052,11 +6064,18 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, NegatibleCost CostX = NegatibleCost::Expensive; SDValue NegX = getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth); + // Prevent this node from being deleted by the next call. + if (NegX) + Handles.emplace_back(NegX); + // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y)) NegatibleCost CostY = NegatibleCost::Expensive; SDValue NegY = getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth); + // We're done with the handles. + Handles.clear(); + // Negate the X if its cost is less or equal than Y. if (NegX && (CostX <= CostY)) { Cost = CostX; @@ -6094,15 +6113,25 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, if (!NegZ) break; + // Prevent this node from being deleted by the next two calls. + Handles.emplace_back(NegZ); + // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z)) NegatibleCost CostX = NegatibleCost::Expensive; SDValue NegX = getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth); + // Prevent this node from being deleted by the next call. + if (NegX) + Handles.emplace_back(NegX); + // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z)) NegatibleCost CostY = NegatibleCost::Expensive; SDValue NegY = getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth); + // We're done with the handles. + Handles.clear(); + // Negate the X if its cost is less or equal than Y. if (NegX && (CostX <= CostY)) { Cost = std::min(CostX, CostZ); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/StackProtector.cpp b/contrib/llvm-project/llvm/lib/CodeGen/StackProtector.cpp index 0411faabbcc3..8d91afb6e99d 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/StackProtector.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/StackProtector.cpp @@ -192,7 +192,7 @@ bool StackProtector::HasAddressTaken(const Instruction *AI, // Ignore intrinsics that do not become real instructions. // TODO: Narrow this to intrinsics that have store-like effects. const auto *CI = cast<CallInst>(I); - if (!isa<DbgInfoIntrinsic>(CI) && !CI->isLifetimeStartOrEnd()) + if (!CI->isDebugOrPseudoInst() && !CI->isLifetimeStartOrEnd()) return true; break; } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/contrib/llvm-project/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp index ecee4aed7f88..2a9132bd2fe0 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -801,8 +801,8 @@ bool TwoAddressInstructionPass::rescheduleMIBelowKill( MachineBasicBlock::iterator KillPos = KillMI; ++KillPos; for (MachineInstr &OtherMI : make_range(End, KillPos)) { - // Debug instructions cannot be counted against the limit. - if (OtherMI.isDebugInstr()) + // Debug or pseudo instructions cannot be counted against the limit. + if (OtherMI.isDebugOrPseudoInstr()) continue; if (NumVisited > 10) // FIXME: Arbitrary limit to reduce compile time cost. return false; @@ -974,8 +974,8 @@ bool TwoAddressInstructionPass::rescheduleKillAboveMI( unsigned NumVisited = 0; for (MachineInstr &OtherMI : make_range(mi, MachineBasicBlock::iterator(KillMI))) { - // Debug instructions cannot be counted against the limit. - if (OtherMI.isDebugInstr()) + // Debug or pseudo instructions cannot be counted against the limit. + if (OtherMI.isDebugOrPseudoInstr()) continue; if (NumVisited > 10) // FIXME: Arbitrary limit to reduce compile time cost. return false; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index dfdd2c6c669f..834d4cc8f514 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -393,7 +393,7 @@ void LLVMOrcDisposeJITTargetMachineBuilder( delete unwrap(JTMB); } -void lLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer) { +void LLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer) { delete unwrap(ObjLayer); } diff --git a/contrib/llvm-project/llvm/lib/IR/AutoUpgrade.cpp b/contrib/llvm-project/llvm/lib/IR/AutoUpgrade.cpp index 23e7af6287b6..7d83cf5dcf1d 100644 --- a/contrib/llvm-project/llvm/lib/IR/AutoUpgrade.cpp +++ b/contrib/llvm-project/llvm/lib/IR/AutoUpgrade.cpp @@ -937,6 +937,12 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys); return true; } + } else if (Name.startswith("ptr.annotation.") && F->arg_size() == 4) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::ptr_annotation, + F->arg_begin()->getType()); + return true; } break; @@ -947,6 +953,16 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { } break; + case 'v': { + if (Name == "var.annotation" && F->arg_size() == 4) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::var_annotation); + return true; + } + break; + } + case 'x': if (UpgradeX86IntrinsicFunction(F, Name, NewFn)) return true; @@ -3730,6 +3746,32 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { CI->eraseFromParent(); return; + case Intrinsic::ptr_annotation: + // Upgrade from versions that lacked the annotation attribute argument. + assert(CI->getNumArgOperands() == 4 && + "Before LLVM 12.0 this intrinsic took four arguments"); + // Create a new call with an added null annotation attribute argument. + NewCall = Builder.CreateCall( + NewFn, + {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), + CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())}); + NewCall->takeName(CI); + CI->replaceAllUsesWith(NewCall); + CI->eraseFromParent(); + return; + + case Intrinsic::var_annotation: + // Upgrade from versions that lacked the annotation attribute argument. + assert(CI->getNumArgOperands() == 4 && + "Before LLVM 12.0 this intrinsic took four arguments"); + // Create a new call with an added null annotation attribute argument. + NewCall = Builder.CreateCall( + NewFn, + {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), + CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())}); + CI->eraseFromParent(); + return; + case Intrinsic::x86_xop_vfrcz_ss: case Intrinsic::x86_xop_vfrcz_sd: NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); diff --git a/contrib/llvm-project/llvm/lib/IR/ConstantFold.cpp b/contrib/llvm-project/llvm/lib/IR/ConstantFold.cpp index 03cb108cc485..95dd55237e5f 100644 --- a/contrib/llvm-project/llvm/lib/IR/ConstantFold.cpp +++ b/contrib/llvm-project/llvm/lib/IR/ConstantFold.cpp @@ -630,7 +630,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored)) { // Undefined behavior invoked - the destination type can't represent // the input constant. - return PoisonValue::get(DestTy); + return UndefValue::get(DestTy); } return ConstantInt::get(FPC->getContext(), IntVal); } @@ -916,7 +916,7 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val, unsigned NumElts = ValTy->getNumElements(); if (CIdx->uge(NumElts)) - return PoisonValue::get(Val->getType()); + return UndefValue::get(Val->getType()); SmallVector<Constant*, 16> Result; Result.reserve(NumElts); @@ -1151,21 +1151,23 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, } case Instruction::SDiv: case Instruction::UDiv: - // X / undef -> poison - // X / 0 -> poison - if (match(C2, m_CombineOr(m_Undef(), m_Zero()))) - return PoisonValue::get(C2->getType()); + // X / undef -> undef + if (isa<UndefValue>(C2)) + return C2; + // undef / 0 -> undef // undef / 1 -> undef - if (match(C2, m_One())) + if (match(C2, m_Zero()) || match(C2, m_One())) return C1; // undef / X -> 0 otherwise return Constant::getNullValue(C1->getType()); case Instruction::URem: case Instruction::SRem: - // X % undef -> poison - // X % 0 -> poison - if (match(C2, m_CombineOr(m_Undef(), m_Zero()))) - return PoisonValue::get(C2->getType()); + // X % undef -> undef + if (match(C2, m_Undef())) + return C2; + // undef % 0 -> undef + if (match(C2, m_Zero())) + return C1; // undef % X -> 0 otherwise return Constant::getNullValue(C1->getType()); case Instruction::Or: // X | undef -> -1 @@ -1173,28 +1175,28 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, return C1; return Constant::getAllOnesValue(C1->getType()); // undef | X -> ~0 case Instruction::LShr: - // X >>l undef -> poison + // X >>l undef -> undef if (isa<UndefValue>(C2)) - return PoisonValue::get(C2->getType()); + return C2; // undef >>l 0 -> undef if (match(C2, m_Zero())) return C1; // undef >>l X -> 0 return Constant::getNullValue(C1->getType()); case Instruction::AShr: - // X >>a undef -> poison + // X >>a undef -> undef if (isa<UndefValue>(C2)) - return PoisonValue::get(C2->getType()); + return C2; // undef >>a 0 -> undef if (match(C2, m_Zero())) return C1; - // TODO: undef >>a X -> poison if the shift is exact + // TODO: undef >>a X -> undef if the shift is exact // undef >>a X -> 0 return Constant::getNullValue(C1->getType()); case Instruction::Shl: // X << undef -> undef if (isa<UndefValue>(C2)) - return PoisonValue::get(C2->getType()); + return C2; // undef << 0 -> undef if (match(C2, m_Zero())) return C1; @@ -1247,14 +1249,14 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, if (CI2->isOne()) return C1; // X / 1 == X if (CI2->isZero()) - return PoisonValue::get(CI2->getType()); // X / 0 == poison + return UndefValue::get(CI2->getType()); // X / 0 == undef break; case Instruction::URem: case Instruction::SRem: if (CI2->isOne()) return Constant::getNullValue(CI2->getType()); // X % 1 == 0 if (CI2->isZero()) - return PoisonValue::get(CI2->getType()); // X % 0 == poison + return UndefValue::get(CI2->getType()); // X % 0 == undef break; case Instruction::And: if (CI2->isZero()) return C2; // X & 0 == 0 @@ -1368,7 +1370,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, case Instruction::SDiv: assert(!CI2->isZero() && "Div by zero handled above"); if (C2V.isAllOnesValue() && C1V.isMinSignedValue()) - return PoisonValue::get(CI1->getType()); // MIN_INT / -1 -> poison + return UndefValue::get(CI1->getType()); // MIN_INT / -1 -> undef return ConstantInt::get(CI1->getContext(), C1V.sdiv(C2V)); case Instruction::URem: assert(!CI2->isZero() && "Div by zero handled above"); @@ -1376,7 +1378,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, case Instruction::SRem: assert(!CI2->isZero() && "Div by zero handled above"); if (C2V.isAllOnesValue() && C1V.isMinSignedValue()) - return PoisonValue::get(CI1->getType()); // MIN_INT % -1 -> poison + return UndefValue::get(CI1->getType()); // MIN_INT % -1 -> undef return ConstantInt::get(CI1->getContext(), C1V.srem(C2V)); case Instruction::And: return ConstantInt::get(CI1->getContext(), C1V & C2V); @@ -1387,15 +1389,15 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, case Instruction::Shl: if (C2V.ult(C1V.getBitWidth())) return ConstantInt::get(CI1->getContext(), C1V.shl(C2V)); - return PoisonValue::get(C1->getType()); // too big shift is poison + return UndefValue::get(C1->getType()); // too big shift is undef case Instruction::LShr: if (C2V.ult(C1V.getBitWidth())) return ConstantInt::get(CI1->getContext(), C1V.lshr(C2V)); - return PoisonValue::get(C1->getType()); // too big shift is poison + return UndefValue::get(C1->getType()); // too big shift is undef case Instruction::AShr: if (C2V.ult(C1V.getBitWidth())) return ConstantInt::get(CI1->getContext(), C1V.ashr(C2V)); - return PoisonValue::get(C1->getType()); // too big shift is poison + return UndefValue::get(C1->getType()); // too big shift is undef } } @@ -1441,7 +1443,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, // Fast path for splatted constants. if (Constant *C2Splat = C2->getSplatValue()) { if (Instruction::isIntDivRem(Opcode) && C2Splat->isNullValue()) - return PoisonValue::get(VTy); + return UndefValue::get(VTy); if (Constant *C1Splat = C1->getSplatValue()) { return ConstantVector::getSplat( VTy->getElementCount(), @@ -1458,9 +1460,9 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, Constant *LHS = ConstantExpr::getExtractElement(C1, ExtractIdx); Constant *RHS = ConstantExpr::getExtractElement(C2, ExtractIdx); - // If any element of a divisor vector is zero, the whole op is poison. + // If any element of a divisor vector is zero, the whole op is undef. if (Instruction::isIntDivRem(Opcode) && RHS->isNullValue()) - return PoisonValue::get(VTy); + return UndefValue::get(VTy); Result.push_back(ConstantExpr::get(Opcode, LHS, RHS)); } @@ -2343,8 +2345,7 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, return PoisonValue::get(GEPTy); if (isa<UndefValue>(C)) - // If inbounds, we can choose an out-of-bounds pointer as a base pointer. - return InBounds ? PoisonValue::get(GEPTy) : UndefValue::get(GEPTy); + return UndefValue::get(GEPTy); Constant *Idx0 = cast<Constant>(Idxs[0]); if (Idxs.size() == 1 && (Idx0->isNullValue() || isa<UndefValue>(Idx0))) diff --git a/contrib/llvm-project/llvm/lib/IR/Instruction.cpp b/contrib/llvm-project/llvm/lib/IR/Instruction.cpp index 1e3fcd672a43..8e52dd3ddc71 100644 --- a/contrib/llvm-project/llvm/lib/IR/Instruction.cpp +++ b/contrib/llvm-project/llvm/lib/IR/Instruction.cpp @@ -633,6 +633,16 @@ bool Instruction::isSafeToRemove() const { !this->isTerminator(); } +bool Instruction::willReturn() const { + if (const auto *CB = dyn_cast<CallBase>(this)) + // FIXME: Temporarily assume that all side-effect free intrinsics will + // return. Remove this workaround once all intrinsics are appropriately + // annotated. + return CB->hasFnAttr(Attribute::WillReturn) || + (isa<IntrinsicInst>(CB) && CB->onlyReadsMemory()); + return true; +} + bool Instruction::isLifetimeStartOrEnd() const { auto II = dyn_cast<IntrinsicInst>(this); if (!II) @@ -641,6 +651,10 @@ bool Instruction::isLifetimeStartOrEnd() const { return ID == Intrinsic::lifetime_start || ID == Intrinsic::lifetime_end; } +bool Instruction::isDebugOrPseudoInst() const { + return isa<DbgInfoIntrinsic>(this) || isa<PseudoProbeInst>(this); +} + const Instruction * Instruction::getNextNonDebugInstruction(bool SkipPseudoOp) const { for (const Instruction *I = getNextNode(); I; I = I->getNextNode()) diff --git a/contrib/llvm-project/llvm/lib/IR/Operator.cpp b/contrib/llvm-project/llvm/lib/IR/Operator.cpp index 0f70fc37dee2..69181f35827b 100644 --- a/contrib/llvm-project/llvm/lib/IR/Operator.cpp +++ b/contrib/llvm-project/llvm/lib/IR/Operator.cpp @@ -61,10 +61,17 @@ Align GEPOperator::getMaxPreservedAlignment(const DataLayout &DL) const { bool GEPOperator::accumulateConstantOffset( const DataLayout &DL, APInt &Offset, function_ref<bool(Value &, APInt &)> ExternalAnalysis) const { - assert(Offset.getBitWidth() == - DL.getIndexSizeInBits(getPointerAddressSpace()) && - "The offset bit width does not match DL specification."); + assert(Offset.getBitWidth() == + DL.getIndexSizeInBits(getPointerAddressSpace()) && + "The offset bit width does not match DL specification."); + SmallVector<const Value *> Index(value_op_begin() + 1, value_op_end()); + return GEPOperator::accumulateConstantOffset(getSourceElementType(), Index, + DL, Offset, ExternalAnalysis); +} +bool GEPOperator::accumulateConstantOffset( + Type *SourceType, ArrayRef<const Value *> Index, const DataLayout &DL, + APInt &Offset, function_ref<bool(Value &, APInt &)> ExternalAnalysis) { bool UsedExternalAnalysis = false; auto AccumulateOffset = [&](APInt Index, uint64_t Size) -> bool { Index = Index.sextOrTrunc(Offset.getBitWidth()); @@ -85,9 +92,10 @@ bool GEPOperator::accumulateConstantOffset( } return true; }; - - for (gep_type_iterator GTI = gep_type_begin(this), GTE = gep_type_end(this); - GTI != GTE; ++GTI) { + auto begin = generic_gep_type_iterator<decltype(Index.begin())>::begin( + SourceType, Index.begin()); + auto end = generic_gep_type_iterator<decltype(Index.end())>::end(Index.end()); + for (auto GTI = begin, GTE = end; GTI != GTE; ++GTI) { // Scalable vectors are multiplied by a runtime constant. bool ScalableType = false; if (isa<ScalableVectorType>(GTI.getIndexedType())) diff --git a/contrib/llvm-project/llvm/lib/IR/PseudoProbe.cpp b/contrib/llvm-project/llvm/lib/IR/PseudoProbe.cpp index 804214f06e7a..80d2963938d4 100644 --- a/contrib/llvm-project/llvm/lib/IR/PseudoProbe.cpp +++ b/contrib/llvm-project/llvm/lib/IR/PseudoProbe.cpp @@ -35,6 +35,9 @@ Optional<PseudoProbe> extractProbeFromDiscriminator(const Instruction &Inst) { PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator); Probe.Attr = PseudoProbeDwarfDiscriminator::extractProbeAttributes(Discriminator); + Probe.Factor = + PseudoProbeDwarfDiscriminator::extractProbeFactor(Discriminator) / + (float)PseudoProbeDwarfDiscriminator::FullDistributionFactor; return Probe; } } @@ -47,6 +50,8 @@ Optional<PseudoProbe> extractProbe(const Instruction &Inst) { Probe.Id = II->getIndex()->getZExtValue(); Probe.Type = (uint32_t)PseudoProbeType::Block; Probe.Attr = II->getAttributes()->getZExtValue(); + Probe.Factor = II->getFactor()->getZExtValue() / + (float)PseudoProbeFullDistributionFactor; return Probe; } @@ -55,4 +60,40 @@ Optional<PseudoProbe> extractProbe(const Instruction &Inst) { return None; } + +void setProbeDistributionFactor(Instruction &Inst, float Factor) { + assert(Factor >= 0 && Factor <= 1 && + "Distribution factor must be in [0, 1.0]"); + if (auto *II = dyn_cast<PseudoProbeInst>(&Inst)) { + IRBuilder<> Builder(&Inst); + uint64_t IntFactor = PseudoProbeFullDistributionFactor; + if (Factor < 1) + IntFactor *= Factor; + auto OrigFactor = II->getFactor()->getZExtValue(); + if (IntFactor != OrigFactor) + II->replaceUsesOfWith(II->getFactor(), Builder.getInt64(IntFactor)); + } else if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst)) { + if (const DebugLoc &DLoc = Inst.getDebugLoc()) { + const DILocation *DIL = DLoc; + auto Discriminator = DIL->getDiscriminator(); + if (DILocation::isPseudoProbeDiscriminator(Discriminator)) { + auto Index = + PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator); + auto Type = + PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator); + auto Attr = PseudoProbeDwarfDiscriminator::extractProbeAttributes( + Discriminator); + // Round small factors to 0 to avoid over-counting. + uint32_t IntFactor = + PseudoProbeDwarfDiscriminator::FullDistributionFactor; + if (Factor < 1) + IntFactor *= Factor; + uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData( + Index, Type, Attr, IntFactor); + DIL = DIL->cloneWithDiscriminator(V); + Inst.setDebugLoc(DIL); + } + } + } +} } // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/IR/Verifier.cpp b/contrib/llvm-project/llvm/lib/IR/Verifier.cpp index 100e881c8fa8..6dd299ee9845 100644 --- a/contrib/llvm-project/llvm/lib/IR/Verifier.cpp +++ b/contrib/llvm-project/llvm/lib/IR/Verifier.cpp @@ -1070,12 +1070,6 @@ void Verifier::visitDICompositeType(const DICompositeType &N) { if (auto *Params = N.getRawTemplateParams()) visitTemplateParams(N, *Params); - if (N.getTag() == dwarf::DW_TAG_class_type || - N.getTag() == dwarf::DW_TAG_union_type) { - AssertDI(N.getFile() && !N.getFile()->getFilename().empty(), - "class/union requires a filename", &N, N.getFile()); - } - if (auto *D = N.getRawDiscriminator()) { AssertDI(isa<DIDerivedType>(D) && N.getTag() == dwarf::DW_TAG_variant_part, "discriminator can only appear on variant part"); diff --git a/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp b/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp index 69307b617552..2d810ffd350b 100644 --- a/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp +++ b/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp @@ -1397,6 +1397,17 @@ bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm, if (TargetObjectWriter->getEMachine() == ELF::EM_386 && Type == ELF::R_386_GOTOFF) return true; + + // ld.lld handles R_MIPS_HI16/R_MIPS_LO16 separately, not as a whole, so + // it doesn't know that an R_MIPS_HI16 with implicit addend 1 and an + // R_MIPS_LO16 with implicit addend -32768 represents 32768, which is in + // range of a MergeInputSection. We could introduce a new RelExpr member + // (like R_RISCV_PC_INDIRECT for R_RISCV_PCREL_HI20 / R_RISCV_PCREL_LO12) + // but the complexity is unnecessary given that GNU as keeps the original + // symbol for this case as well. + if (TargetObjectWriter->getEMachine() == ELF::EM_MIPS && + !hasRelocationAddend()) + return true; } // Most TLS relocations use a got, so they need the symbol. Even those that diff --git a/contrib/llvm-project/llvm/lib/Passes/PassBuilder.cpp b/contrib/llvm-project/llvm/lib/Passes/PassBuilder.cpp index d4c4c6e01ef5..6c1a7c75d30a 100644 --- a/contrib/llvm-project/llvm/lib/Passes/PassBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/Passes/PassBuilder.cpp @@ -1423,6 +1423,9 @@ PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPreLink)); + if (PGOOpt && PGOOpt->PseudoProbeForProfiling) + MPM.addPass(PseudoProbeUpdatePass()); + // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1477,6 +1480,9 @@ PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) { if (PTO.Coroutines) MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); + if (PGOOpt && PGOOpt->PseudoProbeForProfiling) + MPM.addPass(PseudoProbeUpdatePass()); + // Emit annotation remarks. addAnnotationRemarksPass(MPM); diff --git a/contrib/llvm-project/llvm/lib/Passes/PassRegistry.def b/contrib/llvm-project/llvm/lib/Passes/PassRegistry.def index 860bfade733d..877cb9ed13b3 100644 --- a/contrib/llvm-project/llvm/lib/Passes/PassRegistry.def +++ b/contrib/llvm-project/llvm/lib/Passes/PassRegistry.def @@ -119,6 +119,7 @@ MODULE_PASS("kasan-module", ModuleAddressSanitizerPass(/*CompileKernel=*/true, f MODULE_PASS("sancov-module", ModuleSanitizerCoveragePass()) MODULE_PASS("memprof-module", ModuleMemProfilerPass()) MODULE_PASS("poison-checking", PoisonCheckingPass()) +MODULE_PASS("pseudo-probe-update", PseudoProbeUpdatePass()) #undef MODULE_PASS #ifndef CGSCC_ANALYSIS diff --git a/contrib/llvm-project/llvm/lib/Passes/StandardInstrumentations.cpp b/contrib/llvm-project/llvm/lib/Passes/StandardInstrumentations.cpp index a8bfe02d4432..6795aed7b04e 100644 --- a/contrib/llvm-project/llvm/lib/Passes/StandardInstrumentations.cpp +++ b/contrib/llvm-project/llvm/lib/Passes/StandardInstrumentations.cpp @@ -882,6 +882,7 @@ void StandardInstrumentations::registerCallbacks( OptBisect.registerCallbacks(PIC); PreservedCFGChecker.registerCallbacks(PIC); PrintChangedIR.registerCallbacks(PIC); + PseudoProbeVerification.registerCallbacks(PIC); if (VerifyEach) Verify.registerCallbacks(PIC); } diff --git a/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp index a8cc308b4e3a..cdbcde50d33a 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -794,7 +794,6 @@ LineCoverageStats::LineCoverageStats( ExecutionCount = WrappedSegment->Count; if (!MinRegionCount) return; - ExecutionCount = 0; for (const auto *LS : LineSegments) if (isStartOfRegion(LS)) ExecutionCount = std::max(ExecutionCount, LS->Count); diff --git a/contrib/llvm-project/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/contrib/llvm-project/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp index d2603097c550..0e03aa50173d 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -18,9 +18,14 @@ #include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" using namespace llvm; +cl::opt<bool> UseContextLessSummary( + "profile-summary-contextless", cl::Hidden, cl::init(false), cl::ZeroOrMore, + cl::desc("Merge context profiles before calculating thresholds.")); + // A set of cutoff values. Each value, when divided by ProfileSummary::Scale // (which is 1000000) is a desired percentile of total counts. static const uint32_t DefaultCutoffsData[] = { @@ -111,6 +116,35 @@ std::unique_ptr<ProfileSummary> SampleProfileSummaryBuilder::getSummary() { MaxFunctionCount, NumCounts, NumFunctions); } +std::unique_ptr<ProfileSummary> +SampleProfileSummaryBuilder::computeSummaryForProfiles( + const StringMap<sampleprof::FunctionSamples> &Profiles) { + assert(NumFunctions == 0 && + "This can only be called on an empty summary builder"); + StringMap<sampleprof::FunctionSamples> ContextLessProfiles; + const StringMap<sampleprof::FunctionSamples> *ProfilesToUse = &Profiles; + // For CSSPGO, context-sensitive profile effectively split a function profile + // into many copies each representing the CFG profile of a particular calling + // context. That makes the count distribution looks more flat as we now have + // more function profiles each with lower counts, which in turn leads to lower + // hot thresholds. To compensate for that, by defauly we merge context + // profiles before coumputing profile summary. + if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCS && + !UseContextLessSummary.getNumOccurrences())) { + for (const auto &I : Profiles) { + ContextLessProfiles[I.second.getName()].merge(I.second); + } + ProfilesToUse = &ContextLessProfiles; + } + + for (const auto &I : *ProfilesToUse) { + const sampleprof::FunctionSamples &Profile = I.second; + addRecord(Profile); + } + + return getSummary(); +} + std::unique_ptr<ProfileSummary> InstrProfSummaryBuilder::getSummary() { computeDetailedSummary(); return std::make_unique<ProfileSummary>( diff --git a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfReader.cpp b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfReader.cpp index c42931174bc0..38cbca844c87 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfReader.cpp @@ -222,8 +222,6 @@ std::error_code SampleProfileReaderText::readImpl() { sampleprof_error Result = sampleprof_error::success; InlineCallStack InlineStack; - int CSProfileCount = 0; - int RegularProfileCount = 0; uint32_t ProbeProfileCount = 0; // SeenMetadata tracks whether we have processed metadata for the current @@ -257,11 +255,9 @@ std::error_code SampleProfileReaderText::readImpl() { SampleContext FContext(FName); if (FContext.hasContext()) ++CSProfileCount; - else - ++RegularProfileCount; Profiles[FContext] = FunctionSamples(); FunctionSamples &FProfile = Profiles[FContext]; - FProfile.setName(FContext.getName()); + FProfile.setName(FContext.getNameWithoutContext()); FProfile.setContext(FContext); MergeResult(Result, FProfile.addTotalSamples(NumSamples)); MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); @@ -324,13 +320,14 @@ std::error_code SampleProfileReaderText::readImpl() { } } - assert((RegularProfileCount == 0 || CSProfileCount == 0) && + assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile"); ProfileIsCS = (CSProfileCount > 0); assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) && "Cannot have both probe-based profiles and regular profiles"); ProfileIsProbeBased = (ProbeProfileCount > 0); FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; + FunctionSamples::ProfileIsCS = ProfileIsCS; if (Result == sampleprof_error::success) computeSummary(); @@ -546,12 +543,16 @@ SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { if (std::error_code EC = FName.getError()) return EC; - Profiles[*FName] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[*FName]; - FProfile.setName(*FName); - + SampleContext FContext(*FName); + Profiles[FContext] = FunctionSamples(); + FunctionSamples &FProfile = Profiles[FContext]; + FProfile.setName(FContext.getNameWithoutContext()); + FProfile.setContext(FContext); FProfile.addHeadSamples(*NumHeadSamples); + if (FContext.hasContext()) + CSProfileCount++; + if (std::error_code EC = readProfile(FProfile)) return EC; return sampleprof_error::success; @@ -654,40 +655,44 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { return EC; } assert(Data == End && "More data is read than expected"); - return sampleprof_error::success; - } - - if (Remapper) { - for (auto Name : FuncsToUse) { - Remapper->insert(Name); + } else { + if (Remapper) { + for (auto Name : FuncsToUse) { + Remapper->insert(Name); + } } - } - if (useMD5()) { - for (auto Name : FuncsToUse) { - auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); - if (iter == FuncOffsetTable.end()) - continue; - const uint8_t *FuncProfileAddr = Start + iter->second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; - } - } else { - for (auto NameOffset : FuncOffsetTable) { - auto FuncName = NameOffset.first; - if (!FuncsToUse.count(FuncName) && - (!Remapper || !Remapper->exist(FuncName))) - continue; - const uint8_t *FuncProfileAddr = Start + NameOffset.second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; + if (useMD5()) { + for (auto Name : FuncsToUse) { + auto GUID = std::to_string(MD5Hash(Name)); + auto iter = FuncOffsetTable.find(StringRef(GUID)); + if (iter == FuncOffsetTable.end()) + continue; + const uint8_t *FuncProfileAddr = Start + iter->second; + assert(FuncProfileAddr < End && "out of LBRProfile section"); + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } + } else { + for (auto NameOffset : FuncOffsetTable) { + SampleContext FContext(NameOffset.first); + auto FuncName = FContext.getNameWithoutContext(); + if (!FuncsToUse.count(FuncName) && + (!Remapper || !Remapper->exist(FuncName))) + continue; + const uint8_t *FuncProfileAddr = Start + NameOffset.second; + assert(FuncProfileAddr < End && "out of LBRProfile section"); + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } } + Data = End; } - Data = End; + assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && + "Cannot have both context-sensitive and regular profile"); + ProfileIsCS = (CSProfileCount > 0); + FunctionSamples::ProfileIsCS = ProfileIsCS; return sampleprof_error::success; } @@ -878,7 +883,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5) { std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() { if (!ProfileIsProbeBased) return sampleprof_error::success; - for (unsigned I = 0; I < Profiles.size(); ++I) { + while (Data < End) { auto FName(readStringFromTable()); if (std::error_code EC = FName.getError()) return EC; @@ -887,8 +892,14 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() { if (std::error_code EC = Checksum.getError()) return EC; - Profiles[*FName].setFunctionHash(*Checksum); + SampleContext FContext(*FName); + // No need to load metadata for profiles that are not loaded in the current + // module. + if (Profiles.count(FContext)) + Profiles[FContext].setFunctionHash(*Checksum); } + + assert(Data == End && "More data is read than expected"); return sampleprof_error::success; } @@ -1599,9 +1610,5 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, // profile. Binary format has the profile summary in its header. void SampleProfileReader::computeSummary() { SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); - for (const auto &I : Profiles) { - const FunctionSamples &Profile = I.second; - Builder.addRecord(Profile); - } - Summary = Builder.getSummary(); + Summary = Builder.computeSummaryForProfiles(Profiles); } diff --git a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp index 71dba6281f76..8017f2a82804 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -147,7 +147,7 @@ std::error_code SampleProfileWriterExtBinaryBase::write( std::error_code SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) { uint64_t Offset = OutputStream->tell(); - StringRef Name = S.getName(); + StringRef Name = S.getNameWithContext(true); FuncOffsetTable[Name] = Offset - SecLBRProfileStart; encodeULEB128(S.getHeadSamples(), *OutputStream); return writeBody(S); @@ -360,10 +360,7 @@ std::error_code SampleProfileWriterCompactBinary::write( /// it needs to be parsed by the SampleProfileReaderText class. std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { auto &OS = *OutputStream; - if (FunctionSamples::ProfileIsCS) - OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples(); - else - OS << S.getName() << ":" << S.getTotalSamples(); + OS << S.getNameWithContext(true) << ":" << S.getTotalSamples(); if (Indent == 0) OS << ":" << S.getHeadSamples(); OS << "\n"; @@ -635,7 +632,7 @@ std::error_code SampleProfileWriterBinary::writeSummary() { std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { auto &OS = *OutputStream; - if (std::error_code EC = writeNameIdx(S.getName())) + if (std::error_code EC = writeNameIdx(S.getNameWithContext(true))) return EC; encodeULEB128(S.getTotalSamples(), OS); @@ -752,9 +749,5 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, void SampleProfileWriter::computeSummary( const StringMap<FunctionSamples> &ProfileMap) { SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); - for (const auto &I : ProfileMap) { - const FunctionSamples &Profile = I.second; - Builder.addRecord(Profile); - } - Summary = Builder.getSummary(); + Summary = Builder.computeSummaryForProfiles(ProfileMap); } diff --git a/contrib/llvm-project/llvm/lib/Support/CommandLine.cpp b/contrib/llvm-project/llvm/lib/Support/CommandLine.cpp index 6d89481bf28a..e2f014d1815b 100644 --- a/contrib/llvm-project/llvm/lib/Support/CommandLine.cpp +++ b/contrib/llvm-project/llvm/lib/Support/CommandLine.cpp @@ -1726,6 +1726,19 @@ void Option::printHelpStr(StringRef HelpStr, size_t Indent, } } +void Option::printEnumValHelpStr(StringRef HelpStr, size_t BaseIndent, + size_t FirstLineIndentedBy) { + const StringRef ValHelpPrefix = " "; + assert(BaseIndent >= FirstLineIndentedBy + ValHelpPrefix.size()); + std::pair<StringRef, StringRef> Split = HelpStr.split('\n'); + outs().indent(BaseIndent - FirstLineIndentedBy) + << ArgHelpPrefix << ValHelpPrefix << Split.first << "\n"; + while (!Split.second.empty()) { + Split = Split.second.split('\n'); + outs().indent(BaseIndent + ValHelpPrefix.size()) << Split.first << "\n"; + } +} + // Print out the option for the alias. void alias::printOptionInfo(size_t GlobalWidth) const { outs() << PrintArg(ArgStr); @@ -1971,17 +1984,17 @@ void generic_parser_base::printOptionInfo(const Option &O, StringRef Description = getDescription(i); if (!shouldPrintOption(OptionName, Description, O)) continue; - assert(GlobalWidth >= OptionName.size() + OptionPrefixesSize); - size_t NumSpaces = GlobalWidth - OptionName.size() - OptionPrefixesSize; + size_t FirstLineIndent = OptionName.size() + OptionPrefixesSize; outs() << OptionPrefix << OptionName; if (OptionName.empty()) { outs() << EmptyOption; - assert(NumSpaces >= EmptyOption.size()); - NumSpaces -= EmptyOption.size(); + assert(FirstLineIndent >= EmptyOption.size()); + FirstLineIndent += EmptyOption.size(); } if (!Description.empty()) - outs().indent(NumSpaces) << ArgHelpPrefix << " " << Description; - outs() << '\n'; + Option::printEnumValHelpStr(Description, GlobalWidth, FirstLineIndent); + else + outs() << '\n'; } } else { if (!O.HelpStr.empty()) diff --git a/contrib/llvm-project/llvm/lib/Support/Windows/Path.inc b/contrib/llvm-project/llvm/lib/Support/Windows/Path.inc index dc9bcf868381..adcbd1b5f8f3 100644 --- a/contrib/llvm-project/llvm/lib/Support/Windows/Path.inc +++ b/contrib/llvm-project/llvm/lib/Support/Windows/Path.inc @@ -402,8 +402,22 @@ std::error_code is_local(int FD, bool &Result) { } static std::error_code setDeleteDisposition(HANDLE Handle, bool Delete) { - // First, check if the file is on a network (non-local) drive. If so, don't - // set DeleteFile to true, since it prevents opening the file for writes. + // Clear the FILE_DISPOSITION_INFO flag first, before checking if it's a + // network file. On Windows 7 the function realPathFromHandle() below fails + // if the FILE_DISPOSITION_INFO flag was already set to 'DeleteFile = true' by + // a prior call. + FILE_DISPOSITION_INFO Disposition; + Disposition.DeleteFile = false; + if (!SetFileInformationByHandle(Handle, FileDispositionInfo, &Disposition, + sizeof(Disposition))) + return mapWindowsError(::GetLastError()); + if (!Delete) + return std::error_code(); + + // Check if the file is on a network (non-local) drive. If so, don't + // continue when DeleteFile is true, since it prevents opening the file for + // writes. Note -- this will leak temporary files on disk, but only when the + // target file is on a network drive. SmallVector<wchar_t, 128> FinalPath; if (std::error_code EC = realPathFromHandle(Handle, FinalPath)) return EC; @@ -415,9 +429,9 @@ static std::error_code setDeleteDisposition(HANDLE Handle, bool Delete) { if (!IsLocal) return std::error_code(); - // The file is on a local drive, set the DeleteFile to true. - FILE_DISPOSITION_INFO Disposition; - Disposition.DeleteFile = Delete; + // The file is on a local drive, we can safely set FILE_DISPOSITION_INFO's + // flag. + Disposition.DeleteFile = true; if (!SetFileInformationByHandle(Handle, FileDispositionInfo, &Disposition, sizeof(Disposition))) return mapWindowsError(::GetLastError()); diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 1be09186dc0a..1451151f4dc5 100644 --- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1017,11 +1017,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, // Vector reductions for (MVT VT : { MVT::v4f16, MVT::v2f32, MVT::v8f16, MVT::v4f32, MVT::v2f64 }) { - setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); + if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) { + setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); + setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); - if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) setOperationAction(ISD::VECREDUCE_FADD, VT, Legal); + } } for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp index 37c924d879b1..68c721cb0d72 100644 --- a/contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp +++ b/contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp @@ -111,7 +111,7 @@ AArch64MCAsmInfoMicrosoftCOFF::AArch64MCAsmInfoMicrosoftCOFF() { SupportsDebugInformation = true; CodePointerSize = 8; - CommentString = ";"; + CommentString = "//"; ExceptionsType = ExceptionHandling::WinEH; WinEHEncodingType = WinEH::EncodingType::Itanium; } diff --git a/contrib/llvm-project/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/contrib/llvm-project/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp index 2628070f219c..cdb78aae1c4f 100644 --- a/contrib/llvm-project/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp +++ b/contrib/llvm-project/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp @@ -75,17 +75,19 @@ static bool foldImmediates(MachineInstr &MI, const SIInstrInfo *TII, MachineOperand &MovSrc = Def->getOperand(1); bool ConstantFolded = false; - if (MovSrc.isImm() && (isInt<32>(MovSrc.getImm()) || - isUInt<32>(MovSrc.getImm()))) { - Src0.ChangeToImmediate(MovSrc.getImm()); - ConstantFolded = true; - } else if (MovSrc.isFI()) { - Src0.ChangeToFrameIndex(MovSrc.getIndex()); - ConstantFolded = true; - } else if (MovSrc.isGlobal()) { - Src0.ChangeToGA(MovSrc.getGlobal(), MovSrc.getOffset(), - MovSrc.getTargetFlags()); - ConstantFolded = true; + if (TII->isOperandLegal(MI, Src0Idx, &MovSrc)) { + if (MovSrc.isImm() && + (isInt<32>(MovSrc.getImm()) || isUInt<32>(MovSrc.getImm()))) { + Src0.ChangeToImmediate(MovSrc.getImm()); + ConstantFolded = true; + } else if (MovSrc.isFI()) { + Src0.ChangeToFrameIndex(MovSrc.getIndex()); + ConstantFolded = true; + } else if (MovSrc.isGlobal()) { + Src0.ChangeToGA(MovSrc.getGlobal(), MovSrc.getOffset(), + MovSrc.getTargetFlags()); + ConstantFolded = true; + } } if (ConstantFolded) { diff --git a/contrib/llvm-project/llvm/lib/Target/ARM/ARMISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/ARM/ARMISelLowering.cpp index 397979b4ab1e..598062672a56 100644 --- a/contrib/llvm-project/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/contrib/llvm-project/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -18661,6 +18661,8 @@ ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { : AtomicExpansionKind::None; } +// Similar to shouldExpandAtomicRMWInIR, ldrex/strex can be used up to 32 +// bits, and up to 64 bits on the non-M profiles. TargetLowering::AtomicExpansionKind ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { // At -O0, fast-regalloc cannot cope with the live vregs necessary to @@ -18668,9 +18670,11 @@ ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { // on the stack and close enough to the spill slot, this can lead to a // situation where the monitor always gets cleared and the atomic operation // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. + unsigned Size = AI->getOperand(1)->getType()->getPrimitiveSizeInBits(); bool HasAtomicCmpXchg = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps(); - if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg) + if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg && + Size <= (Subtarget->isMClass() ? 32U : 64U)) return AtomicExpansionKind::LLSC; return AtomicExpansionKind::None; } diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index 693b0adaede4..2604218da160 100644 --- a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -5896,7 +5896,13 @@ bool PPCDAGToDAGISel::AllUsersSelectZero(SDNode *N) { User->getMachineOpcode() != PPC::SELECT_I8) return false; + SDNode *Op1 = User->getOperand(1).getNode(); SDNode *Op2 = User->getOperand(2).getNode(); + // If we have a degenerate select with two equal operands, swapping will + // not do anything, and we may run into an infinite loop. + if (Op1 == Op2) + return false; + if (!Op2->isMachineOpcode()) return false; diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 9215c17cb94b..929a72ac687e 100644 --- a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -8604,16 +8604,20 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, // If it is a splat of a double, check if we can shrink it to a 32 bit // non-denormal float which when converted back to double gives us the same - // double. This is to exploit the XXSPLTIDP instruction.+ // If we lose precision, we use XXSPLTI32DX. + // double. This is to exploit the XXSPLTIDP instruction. + // If we lose precision, we use XXSPLTI32DX. if (BVNIsConstantSplat && (SplatBitSize == 64) && Subtarget.hasPrefixInstrs()) { - if (convertToNonDenormSingle(APSplatBits) && - (Op->getValueType(0) == MVT::v2f64)) { + // Check the type first to short-circuit so we don't modify APSplatBits if + // this block isn't executed. + if ((Op->getValueType(0) == MVT::v2f64) && + convertToNonDenormSingle(APSplatBits)) { SDValue SplatNode = DAG.getNode( PPCISD::XXSPLTI_SP_TO_DP, dl, MVT::v2f64, DAG.getTargetConstant(APSplatBits.getZExtValue(), dl, MVT::i32)); return DAG.getBitcast(Op.getValueType(), SplatNode); - } else { // We may lose precision, so we have to use XXSPLTI32DX. + } else { + // We may lose precision, so we have to use XXSPLTI32DX. uint32_t Hi = (uint32_t)((APSplatBits.getZExtValue() & 0xFFFFFFFF00000000LL) >> 32); diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.h b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.h index 477105bd03ac..0dda2c181572 100644 --- a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -987,6 +987,9 @@ namespace llvm { shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override; + // Keep the zero-extensions for arguments to libcalls. + bool shouldKeepZExtForFP16Conv() const override { return true; } + /// createFastISel - This method returns a target-specific FastISel object, /// or null if the target does not support "fast" instruction selection. FastISel *createFastISel(FunctionLoweringInfo &FuncInfo, diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index e7e590153605..dcf7525d7458 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -2126,7 +2126,7 @@ bool RISCVAsmParser::parseDirectiveAttribute() { if (getFeatureBits(RISCV::FeatureStdExtB)) formalArchStr = (Twine(formalArchStr) + "_b0p93").str(); if (getFeatureBits(RISCV::FeatureStdExtV)) - formalArchStr = (Twine(formalArchStr) + "_v1p0").str(); + formalArchStr = (Twine(formalArchStr) + "_v0p10").str(); if (getFeatureBits(RISCV::FeatureExtZfh)) formalArchStr = (Twine(formalArchStr) + "_zfh0p1").str(); if (getFeatureBits(RISCV::FeatureExtZba)) @@ -2152,9 +2152,9 @@ bool RISCVAsmParser::parseDirectiveAttribute() { if (getFeatureBits(RISCV::FeatureExtZbt)) formalArchStr = (Twine(formalArchStr) + "_zbt0p93").str(); if (getFeatureBits(RISCV::FeatureExtZvamo)) - formalArchStr = (Twine(formalArchStr) + "_zvamo1p0").str(); + formalArchStr = (Twine(formalArchStr) + "_zvamo0p10").str(); if (getFeatureBits(RISCV::FeatureStdExtZvlsseg)) - formalArchStr = (Twine(formalArchStr) + "_zvlsseg1p0").str(); + formalArchStr = (Twine(formalArchStr) + "_zvlsseg0p10").str(); getTargetStreamer().emitTextAttribute(Tag, formalArchStr); } diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp index 72434a15bedb..13c4b84aa300 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp @@ -63,7 +63,7 @@ void RISCVTargetStreamer::emitTargetAttributes(const MCSubtargetInfo &STI) { if (STI.hasFeature(RISCV::FeatureStdExtB)) Arch += "_b0p93"; if (STI.hasFeature(RISCV::FeatureStdExtV)) - Arch += "_v1p0"; + Arch += "_v0p10"; if (STI.hasFeature(RISCV::FeatureExtZfh)) Arch += "_zfh0p1"; if (STI.hasFeature(RISCV::FeatureExtZba)) @@ -89,9 +89,9 @@ void RISCVTargetStreamer::emitTargetAttributes(const MCSubtargetInfo &STI) { if (STI.hasFeature(RISCV::FeatureExtZbt)) Arch += "_zbt0p93"; if (STI.hasFeature(RISCV::FeatureExtZvamo)) - Arch += "_zvamo1p0"; + Arch += "_zvamo0p10"; if (STI.hasFeature(RISCV::FeatureStdExtZvlsseg)) - Arch += "_zvlsseg1p0"; + Arch += "_zvlsseg0p10"; emitTextAttribute(RISCVAttrs::ARCH, Arch); } diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVCleanupVSETVLI.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVCleanupVSETVLI.cpp index 6a12f99b8903..ae32cbd1ae59 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVCleanupVSETVLI.cpp +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVCleanupVSETVLI.cpp @@ -59,7 +59,8 @@ bool RISCVCleanupVSETVLI::runOnMachineBasicBlock(MachineBasicBlock &MBB) { for (auto MII = MBB.begin(), MIE = MBB.end(); MII != MIE;) { MachineInstr &MI = *MII++; - if (MI.getOpcode() != RISCV::PseudoVSETVLI) { + if (MI.getOpcode() != RISCV::PseudoVSETVLI && + MI.getOpcode() != RISCV::PseudoVSETIVLI) { if (PrevVSETVLI && (MI.isCall() || MI.modifiesRegister(RISCV::VL) || MI.modifiesRegister(RISCV::VTYPE))) { @@ -69,26 +70,48 @@ bool RISCVCleanupVSETVLI::runOnMachineBasicBlock(MachineBasicBlock &MBB) { continue; } - // If we don't have a previous VSETVLI or the VL output isn't dead, we + // If we don't have a previous VSET{I}VLI or the VL output isn't dead, we // can't remove this VSETVLI. if (!PrevVSETVLI || !MI.getOperand(0).isDead()) { PrevVSETVLI = &MI; continue; } - Register PrevAVLReg = PrevVSETVLI->getOperand(1).getReg(); - Register AVLReg = MI.getOperand(1).getReg(); + // If a previous "set vl" instruction opcode is different from this one, we + // can't differentiate the AVL values. + if (PrevVSETVLI->getOpcode() != MI.getOpcode()) { + PrevVSETVLI = &MI; + continue; + } + + // The remaining two cases are + // 1. PrevVSETVLI = PseudoVSETVLI + // MI = PseudoVSETVLI + // + // 2. PrevVSETVLI = PseudoVSETIVLI + // MI = PseudoVSETIVLI + Register AVLReg; + bool SameAVL = false; + if (MI.getOpcode() == RISCV::PseudoVSETVLI) { + AVLReg = MI.getOperand(1).getReg(); + SameAVL = PrevVSETVLI->getOperand(1).getReg() == AVLReg; + } else { // RISCV::PseudoVSETIVLI + SameAVL = + PrevVSETVLI->getOperand(1).getImm() == MI.getOperand(1).getImm(); + } int64_t PrevVTYPEImm = PrevVSETVLI->getOperand(2).getImm(); int64_t VTYPEImm = MI.getOperand(2).getImm(); - // Does this VSETVLI use the same AVL register and VTYPE immediate? - if (PrevAVLReg != AVLReg || PrevVTYPEImm != VTYPEImm) { + // Does this VSET{I}VLI use the same AVL register/value and VTYPE immediate? + if (!SameAVL || PrevVTYPEImm != VTYPEImm) { PrevVSETVLI = &MI; continue; } // If the AVLReg is X0 we need to look at the output VL of both VSETVLIs. - if (AVLReg == RISCV::X0) { + if ((MI.getOpcode() == RISCV::PseudoVSETVLI) && (AVLReg == RISCV::X0)) { + assert((PrevVSETVLI->getOpcode() == RISCV::PseudoVSETVLI) && + "Unexpected vsetvli opcode."); Register PrevOutVL = PrevVSETVLI->getOperand(0).getReg(); Register OutVL = MI.getOperand(0).getReg(); // We can't remove if the previous VSETVLI left VL unchanged and the diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp index 5f50892ca886..ec9a39569952 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp @@ -103,6 +103,7 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB, case RISCV::PseudoLA_TLS_GD: return expandLoadTLSGDAddress(MBB, MBBI, NextMBBI); case RISCV::PseudoVSETVLI: + case RISCV::PseudoVSETIVLI: return expandVSetVL(MBB, MBBI); case RISCV::PseudoVMCLR_M_B1: case RISCV::PseudoVMCLR_M_B2: @@ -217,9 +218,15 @@ bool RISCVExpandPseudo::expandVSetVL(MachineBasicBlock &MBB, DebugLoc DL = MBBI->getDebugLoc(); - assert(MBBI->getOpcode() == RISCV::PseudoVSETVLI && + assert((MBBI->getOpcode() == RISCV::PseudoVSETVLI || + MBBI->getOpcode() == RISCV::PseudoVSETIVLI) && "Unexpected pseudo instruction"); - const MCInstrDesc &Desc = TII->get(RISCV::VSETVLI); + unsigned Opcode; + if (MBBI->getOpcode() == RISCV::PseudoVSETVLI) + Opcode = RISCV::VSETVLI; + else + Opcode = RISCV::VSETIVLI; + const MCInstrDesc &Desc = TII->get(Opcode); assert(Desc.getNumOperands() == 3 && "Unexpected instruction format"); Register DstReg = MBBI->getOperand(0).getReg(); diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index 7b0f38671f06..43bf16c53a62 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -569,12 +569,14 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { SDValue VLOperand = Node->getOperand(2); if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) { - if (C->isNullValue()) { - VLOperand = SDValue( - CurDAG->getMachineNode(RISCV::ADDI, DL, XLenVT, - CurDAG->getRegister(RISCV::X0, XLenVT), - CurDAG->getTargetConstant(0, DL, XLenVT)), - 0); + uint64_t AVL = C->getZExtValue(); + if (isUInt<5>(AVL)) { + SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT); + ReplaceNode(Node, + CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT, + MVT::Other, VLImm, VTypeIOp, + /* Chain */ Node->getOperand(0))); + return; } } @@ -824,93 +826,6 @@ bool RISCVDAGToDAGISel::MatchSRLIW(SDNode *N) const { return (Mask | maskTrailingOnes<uint64_t>(ShAmt)) == 0xffffffff; } -// Check that it is a SLOI (Shift Left Ones Immediate). A PatFrag has already -// determined it has the right structure: -// -// (OR (SHL RS1, VC2), VC1) -// -// Check that VC1, the mask used to fill with ones, is compatible -// with VC2, the shamt: -// -// VC1 == maskTrailingOnes(VC2) -// -bool RISCVDAGToDAGISel::MatchSLOI(SDNode *N) const { - assert(N->getOpcode() == ISD::OR); - assert(N->getOperand(0).getOpcode() == ISD::SHL); - assert(isa<ConstantSDNode>(N->getOperand(1))); - assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1))); - - SDValue Shl = N->getOperand(0); - if (Subtarget->is64Bit()) { - uint64_t VC1 = N->getConstantOperandVal(1); - uint64_t VC2 = Shl.getConstantOperandVal(1); - return VC1 == maskTrailingOnes<uint64_t>(VC2); - } - - uint32_t VC1 = N->getConstantOperandVal(1); - uint32_t VC2 = Shl.getConstantOperandVal(1); - return VC1 == maskTrailingOnes<uint32_t>(VC2); -} - -// Check that it is a SROI (Shift Right Ones Immediate). A PatFrag has already -// determined it has the right structure: -// -// (OR (SRL RS1, VC2), VC1) -// -// Check that VC1, the mask used to fill with ones, is compatible -// with VC2, the shamt: -// -// VC1 == maskLeadingOnes(VC2) -// -bool RISCVDAGToDAGISel::MatchSROI(SDNode *N) const { - assert(N->getOpcode() == ISD::OR); - assert(N->getOperand(0).getOpcode() == ISD::SRL); - assert(isa<ConstantSDNode>(N->getOperand(1))); - assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1))); - - SDValue Srl = N->getOperand(0); - if (Subtarget->is64Bit()) { - uint64_t VC1 = N->getConstantOperandVal(1); - uint64_t VC2 = Srl.getConstantOperandVal(1); - return VC1 == maskLeadingOnes<uint64_t>(VC2); - } - - uint32_t VC1 = N->getConstantOperandVal(1); - uint32_t VC2 = Srl.getConstantOperandVal(1); - return VC1 == maskLeadingOnes<uint32_t>(VC2); -} - -// Check that it is a SROIW (Shift Right Ones Immediate i32 on RV64). A PatFrag -// has already determined it has the right structure: -// -// (OR (SRL RS1, VC2), VC1) -// -// and then we check that VC1, the mask used to fill with ones, is compatible -// with VC2, the shamt: -// -// VC2 < 32 -// VC1 == maskTrailingZeros<uint64_t>(32 - VC2) -// -bool RISCVDAGToDAGISel::MatchSROIW(SDNode *N) const { - assert(N->getOpcode() == ISD::OR); - assert(N->getOperand(0).getOpcode() == ISD::SRL); - assert(isa<ConstantSDNode>(N->getOperand(1))); - assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1))); - - // The IsRV64 predicate is checked after PatFrag predicates so we can get - // here even on RV32. - if (!Subtarget->is64Bit()) - return false; - - SDValue Srl = N->getOperand(0); - uint64_t VC1 = N->getConstantOperandVal(1); - uint64_t VC2 = Srl.getConstantOperandVal(1); - - // Immediate range should be enforced by uimm5 predicate. - assert(VC2 < 32 && "Unexpected immediate"); - return VC1 == maskTrailingZeros<uint64_t>(32 - VC2); -} - // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32 // on RV64). // SLLIUW is the same as SLLI except for the fact that it clears the bits @@ -946,6 +861,23 @@ bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const { return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF); } +// X0 has special meaning for vsetvl/vsetvli. +// rd | rs1 | AVL value | Effect on vl +//-------------------------------------------------------------- +// !X0 | X0 | VLMAX | Set vl to VLMAX +// X0 | X0 | Value in vl | Keep current vl, just change vtype. +bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) { + // If the VL value is a constant 0, manually select it to an ADDI with 0 + // immediate to prevent the default selection path from matching it to X0. + auto *C = dyn_cast<ConstantSDNode>(N); + if (C && C->isNullValue()) + VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0); + else + VL = N; + + return true; +} + bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) { if (N.getOpcode() != ISD::SPLAT_VECTOR && N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64) diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h index 23601c3b8f06..6099586d049d 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h @@ -46,11 +46,10 @@ public: bool SelectAddrFI(SDValue Addr, SDValue &Base); bool MatchSRLIW(SDNode *N) const; - bool MatchSLOI(SDNode *N) const; - bool MatchSROI(SDNode *N) const; - bool MatchSROIW(SDNode *N) const; bool MatchSLLIUW(SDNode *N) const; + bool selectVLOp(SDValue N, SDValue &VL); + bool selectVSplat(SDValue N, SDValue &SplatVal); bool selectVSplatSimm5(SDValue N, SDValue &SplatVal); bool selectVSplatUimm5(SDValue N, SDValue &SplatVal); diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td index 147993127e78..80f46b73bfd7 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td @@ -38,9 +38,11 @@ class RISCVLSUMOP<bits<5> val> { bits<5> Value = val; } def LUMOPUnitStride : RISCVLSUMOP<0b00000>; +def LUMOPUnitStrideMask : RISCVLSUMOP<0b01011>; def LUMOPUnitStrideWholeReg : RISCVLSUMOP<0b01000>; def LUMOPUnitStrideFF: RISCVLSUMOP<0b10000>; def SUMOPUnitStride : RISCVLSUMOP<0b00000>; +def SUMOPUnitStrideMask : RISCVLSUMOP<0b01011>; def SUMOPUnitStrideWholeReg : RISCVLSUMOP<0b01000>; class RISCVAMOOP<bits<5> val> { @@ -63,10 +65,23 @@ def LSWidth8 : RISCVWidth<0b0000>; def LSWidth16 : RISCVWidth<0b0101>; def LSWidth32 : RISCVWidth<0b0110>; def LSWidth64 : RISCVWidth<0b0111>; -def LSWidth128 : RISCVWidth<0b1000>; -def LSWidth256 : RISCVWidth<0b1101>; -def LSWidth512 : RISCVWidth<0b1110>; -def LSWidth1024 : RISCVWidth<0b1111>; + +class RVInstSetiVLi<dag outs, dag ins, string opcodestr, string argstr> + : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> { + bits<5> uimm; + bits<5> rd; + bits<10> vtypei; + + let Inst{31} = 1; + let Inst{30} = 1; + let Inst{29-20} = vtypei{9-0}; + let Inst{19-15} = uimm; + let Inst{14-12} = 0b111; + let Inst{11-7} = rd; + let Opcode = OPC_OP_V.Value; + + let Defs = [VTYPE, VL]; +} class RVInstSetVLi<dag outs, dag ins, string opcodestr, string argstr> : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> { diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoB.td b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoB.td index 1bc288b5177c..7888ac7bac8e 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoB.td +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoB.td @@ -45,25 +45,6 @@ def shfl_uimm : Operand<XLenVT>, ImmLeaf<XLenVT, [{ }]; } - -// Check that it is a SLOI (Shift Left Ones Immediate). -def SLOIPat : PatFrag<(ops node:$A, node:$B), - (or (shl node:$A, node:$B), imm), [{ - return MatchSLOI(N); -}]>; - -// Check that it is a SROI (Shift Right Ones Immediate). -def SROIPat : PatFrag<(ops node:$A, node:$B), - (or (srl node:$A, node:$B), imm), [{ - return MatchSROI(N); -}]>; - -// Check that it is a SROIW (Shift Right Ones Immediate i32 on RV64). -def SROIWPat : PatFrag<(ops node:$A, node:$B), - (or (srl node:$A, node:$B), imm), [{ - return MatchSROIW(N); -}]>; - // Checks if this mask has a single 0 bit and cannot be used with ANDI. def BCLRMask : ImmLeaf<XLenVT, [{ if (Subtarget->is64Bit()) @@ -210,11 +191,6 @@ def SH2ADD : ALU_rr<0b0010000, 0b100, "sh2add">, Sched<[]>; def SH3ADD : ALU_rr<0b0010000, 0b110, "sh3add">, Sched<[]>; } // Predicates = [HasStdExtZba] -let Predicates = [HasStdExtZbp] in { -def SLO : ALU_rr<0b0010000, 0b001, "slo">, Sched<[]>; -def SRO : ALU_rr<0b0010000, 0b101, "sro">, Sched<[]>; -} // Predicates = [HasStdExtZbp] - let Predicates = [HasStdExtZbbOrZbp] in { def ROL : ALU_rr<0b0110000, 0b001, "rol">, Sched<[]>; def ROR : ALU_rr<0b0110000, 0b101, "ror">, Sched<[]>; @@ -238,11 +214,6 @@ def XPERMB : ALU_rr<0b0010100, 0b100, "xperm.b">, Sched<[]>; def XPERMH : ALU_rr<0b0010100, 0b110, "xperm.h">, Sched<[]>; } // Predicates = [HasStdExtZbp] -let Predicates = [HasStdExtZbp] in { -def SLOI : RVBShift_ri<0b00100, 0b001, OPC_OP_IMM, "sloi">, Sched<[]>; -def SROI : RVBShift_ri<0b00100, 0b101, OPC_OP_IMM, "sroi">, Sched<[]>; -} // Predicates = [HasStdExtZbp] - let Predicates = [HasStdExtZbbOrZbp] in def RORI : RVBShift_ri<0b01100, 0b101, OPC_OP_IMM, "rori">, Sched<[]>; @@ -369,11 +340,6 @@ def SH2ADDUW : ALUW_rr<0b0010000, 0b100, "sh2add.uw">, Sched<[]>; def SH3ADDUW : ALUW_rr<0b0010000, 0b110, "sh3add.uw">, Sched<[]>; } // Predicates = [HasStdExtZbb, IsRV64] -let Predicates = [HasStdExtZbp, IsRV64] in { -def SLOW : ALUW_rr<0b0010000, 0b001, "slow">, Sched<[]>; -def SROW : ALUW_rr<0b0010000, 0b101, "srow">, Sched<[]>; -} // Predicates = [HasStdExtZbp, IsRV64] - let Predicates = [HasStdExtZbbOrZbp, IsRV64] in { def ROLW : ALUW_rr<0b0110000, 0b001, "rolw">, Sched<[]>; def RORW : ALUW_rr<0b0110000, 0b101, "rorw">, Sched<[]>; @@ -395,11 +361,6 @@ let Predicates = [HasStdExtZbp, IsRV64] in { def XPERMW : ALU_rr<0b0010100, 0b000, "xperm.w">, Sched<[]>; } // Predicates = [HasStdExtZbp, IsRV64] -let Predicates = [HasStdExtZbp, IsRV64] in { -def SLOIW : RVBShiftW_ri<0b0010000, 0b001, OPC_OP_IMM_32, "sloiw">, Sched<[]>; -def SROIW : RVBShiftW_ri<0b0010000, 0b101, OPC_OP_IMM_32, "sroiw">, Sched<[]>; -} // Predicates = [HasStdExtZbp, IsRV64] - let Predicates = [HasStdExtZbbOrZbp, IsRV64] in def RORIW : RVBShiftW_ri<0b0110000, 0b101, OPC_OP_IMM_32, "roriw">, Sched<[]>; @@ -673,13 +634,6 @@ def : Pat<(or GPR:$rs1, (not GPR:$rs2)), (ORN GPR:$rs1, GPR:$rs2)>; def : Pat<(xor GPR:$rs1, (not GPR:$rs2)), (XNOR GPR:$rs1, GPR:$rs2)>; } // Predicates = [HasStdExtZbbOrZbp] -let Predicates = [HasStdExtZbp] in { -def : Pat<(not (shiftop<shl> (not GPR:$rs1), GPR:$rs2)), - (SLO GPR:$rs1, GPR:$rs2)>; -def : Pat<(not (shiftop<srl> (not GPR:$rs1), GPR:$rs2)), - (SRO GPR:$rs1, GPR:$rs2)>; -} // Predicates = [HasStdExtZbp] - let Predicates = [HasStdExtZbbOrZbp] in { def : Pat<(rotl GPR:$rs1, GPR:$rs2), (ROL GPR:$rs1, GPR:$rs2)>; def : Pat<(rotr GPR:$rs1, GPR:$rs2), (ROR GPR:$rs1, GPR:$rs2)>; @@ -710,13 +664,6 @@ def : Pat<(and (srl GPR:$rs1, uimmlog2xlen:$shamt), (XLenVT 1)), (BEXTI GPR:$rs1, uimmlog2xlen:$shamt)>; } -let Predicates = [HasStdExtZbp] in { -def : Pat<(SLOIPat GPR:$rs1, uimmlog2xlen:$shamt), - (SLOI GPR:$rs1, uimmlog2xlen:$shamt)>; -def : Pat<(SROIPat GPR:$rs1, uimmlog2xlen:$shamt), - (SROI GPR:$rs1, uimmlog2xlen:$shamt)>; -} // Predicates = [HasStdExtZbp] - // There's no encoding for roli in the the 'B' extension as it can be // implemented with rori by negating the immediate. let Predicates = [HasStdExtZbbOrZbp] in { @@ -936,13 +883,6 @@ def : Pat<(add (SLLIUWPat GPR:$rs1, (XLenVT 3)), GPR:$rs2), (SH3ADDUW GPR:$rs1, GPR:$rs2)>; } // Predicates = [HasStdExtZba, IsRV64] -let Predicates = [HasStdExtZbp, IsRV64] in { -def : Pat<(not (shiftopw<riscv_sllw> (not GPR:$rs1), GPR:$rs2)), - (SLOW GPR:$rs1, GPR:$rs2)>; -def : Pat<(not (shiftopw<riscv_srlw> (not GPR:$rs1), GPR:$rs2)), - (SROW GPR:$rs1, GPR:$rs2)>; -} // Predicates = [HasStdExtZbp, IsRV64] - let Predicates = [HasStdExtZbbOrZbp, IsRV64] in { def : Pat<(riscv_rolw GPR:$rs1, GPR:$rs2), (ROLW GPR:$rs1, GPR:$rs2)>; @@ -983,13 +923,6 @@ def : Pat<(xor (assertsexti32 GPR:$rs1), BSETINVWMask:$mask), } // Predicates = [HasStdExtZbs, IsRV64] let Predicates = [HasStdExtZbp, IsRV64] in { -def : Pat<(sext_inreg (SLOIPat GPR:$rs1, uimm5:$shamt), i32), - (SLOIW GPR:$rs1, uimm5:$shamt)>; -def : Pat<(SROIWPat GPR:$rs1, uimm5:$shamt), - (SROIW GPR:$rs1, uimm5:$shamt)>; -} // Predicates = [HasStdExtZbp, IsRV64] - -let Predicates = [HasStdExtZbp, IsRV64] in { def : Pat<(riscv_rorw (riscv_greviw GPR:$rs1, 24), (i64 16)), (GREVIW GPR:$rs1, 8)>; def : Pat<(riscv_rolw (riscv_greviw GPR:$rs1, 24), (i64 16)), (GREVIW GPR:$rs1, 8)>; def : Pat<(riscv_greviw GPR:$rs1, timm:$shamt), (GREVIW GPR:$rs1, timm:$shamt)>; diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoV.td index 4f9e9cfbdb98..b3fc76aee161 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoV.td +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoV.td @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// /// /// This file describes the RISC-V instructions from the standard 'V' Vector -/// extension, version 0.9. +/// extension, version 0.10. /// This version is still experimental as the 'V' extension hasn't been /// ratified yet. /// @@ -82,6 +82,12 @@ def simm5_plus1 : Operand<XLenVT>, ImmLeaf<XLenVT, //===----------------------------------------------------------------------===// let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in { +// load vd, (rs1) +class VUnitStrideLoadMask<string opcodestr> + : RVInstVLU<0b000, LSWidth8.Value{3}, LUMOPUnitStrideMask, LSWidth8.Value{2-0}, + (outs VR:$vd), + (ins GPR:$rs1), opcodestr, "$vd, (${rs1})">; + // load vd, (rs1), vm class VUnitStrideLoad<RISCVLSUMOP lumop, RISCVWidth width, string opcodestr> @@ -138,6 +144,12 @@ class VIndexedSegmentLoad<bits<3> nf, RISCVMOP mop, RISCVWidth width, let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in { // store vd, vs3, (rs1), vm +class VUnitStrideStoreMask<string opcodestr> + : RVInstVSU<0b000, LSWidth8.Value{3}, SUMOPUnitStrideMask, LSWidth8.Value{2-0}, + (outs), (ins VR:$vs3, GPR:$rs1), opcodestr, + "$vs3, (${rs1})">; + +// store vd, vs3, (rs1), vm class VUnitStrideStore<RISCVLSUMOP sumop, RISCVWidth width, string opcodestr> : RVInstVSU<0b000, width.Value{3}, sumop, width.Value{2-0}, @@ -423,10 +435,6 @@ multiclass VWholeLoad<bits<3> nf, string opcodestr> { def E16_V : VWholeLoad<nf, LSWidth16, opcodestr # "e16.v">; def E32_V : VWholeLoad<nf, LSWidth32, opcodestr # "e32.v">; def E64_V : VWholeLoad<nf, LSWidth64, opcodestr # "e64.v">; - def E128_V : VWholeLoad<nf, LSWidth128, opcodestr # "e128.v">; - def E256_V : VWholeLoad<nf, LSWidth256, opcodestr # "e256.v">; - def E512_V : VWholeLoad<nf, LSWidth512, opcodestr # "e512.v">; - def E1024_V : VWholeLoad<nf, LSWidth1024, opcodestr # "e1024.v">; } //===----------------------------------------------------------------------===// @@ -438,6 +446,9 @@ let hasSideEffects = 1, mayLoad = 0, mayStore = 0 in { def VSETVLI : RVInstSetVLi<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei), "vsetvli", "$rd, $rs1, $vtypei">; +def VSETIVLI : RVInstSetiVLi<(outs GPR:$rd), (ins uimm5:$uimm, VTypeIOp:$vtypei), + "vsetivli", "$rd, $uimm, $vtypei">; + def VSETVL : RVInstSetVL<(outs GPR:$rd), (ins GPR:$rs1, GPR:$rs2), "vsetvl", "$rd, $rs1, $rs2">; } // hasSideEffects = 1, mayLoad = 0, mayStore = 0 @@ -447,47 +458,30 @@ def VLE8_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth8, "vle8.v">; def VLE16_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth16, "vle16.v">; def VLE32_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth32, "vle32.v">; def VLE64_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth64, "vle64.v">; -def VLE128_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth128, "vle128.v">; -def VLE256_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth256, "vle256.v">; -def VLE512_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth512, "vle512.v">; -def VLE1024_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth1024, "vle1024.v">; def VLE8FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth8, "vle8ff.v">; def VLE16FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth16, "vle16ff.v">; def VLE32FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth32, "vle32ff.v">; def VLE64FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth64, "vle64ff.v">; -def VLE128FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth128, "vle128ff.v">; -def VLE256FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth256, "vle256ff.v">; -def VLE512FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth512, "vle512ff.v">; -def VLE1024FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth1024, "vle1024ff.v">; + +def VLE1_V : VUnitStrideLoadMask<"vle1.v">; +def VSE1_V : VUnitStrideStoreMask<"vse1.v">; def VSE8_V : VUnitStrideStore<SUMOPUnitStride, LSWidth8, "vse8.v">; def VSE16_V : VUnitStrideStore<SUMOPUnitStride, LSWidth16, "vse16.v">; def VSE32_V : VUnitStrideStore<SUMOPUnitStride, LSWidth32, "vse32.v">; def VSE64_V : VUnitStrideStore<SUMOPUnitStride, LSWidth64, "vse64.v">; -def VSE128_V : VUnitStrideStore<SUMOPUnitStride, LSWidth128, "vse128.v">; -def VSE256_V : VUnitStrideStore<SUMOPUnitStride, LSWidth256, "vse256.v">; -def VSE512_V : VUnitStrideStore<SUMOPUnitStride, LSWidth512, "vse512.v">; -def VSE1024_V : VUnitStrideStore<SUMOPUnitStride, LSWidth1024, "vse1024.v">; // Vector Strided Instructions def VLSE8_V : VStridedLoad<LSWidth8, "vlse8.v">; def VLSE16_V : VStridedLoad<LSWidth16, "vlse16.v">; def VLSE32_V : VStridedLoad<LSWidth32, "vlse32.v">; def VLSE64_V : VStridedLoad<LSWidth64, "vlse64.v">; -def VLSE128_V : VStridedLoad<LSWidth128, "vlse128.v">; -def VLSE256_V : VStridedLoad<LSWidth256, "vlse256.v">; -def VLSE512_V : VStridedLoad<LSWidth512, "vlse512.v">; -def VLSE1024_V : VStridedLoad<LSWidth1024, "vlse1024.v">; def VSSE8_V : VStridedStore<LSWidth8, "vsse8.v">; def VSSE16_V : VStridedStore<LSWidth16, "vsse16.v">; def VSSE32_V : VStridedStore<LSWidth32, "vsse32.v">; def VSSE64_V : VStridedStore<LSWidth64, "vsse64.v">; -def VSSE128_V : VStridedStore<LSWidth128, "vsse128.v">; -def VSSE256_V : VStridedStore<LSWidth256, "vsse256.v">; -def VSSE512_V : VStridedStore<LSWidth512, "vsse512.v">; -def VSSE1024_V : VStridedStore<LSWidth1024, "vsse1024.v">; // Vector Indexed Instructions def VLUXEI8_V : VIndexedLoad<MOPLDIndexedUnord, LSWidth8, "vluxei8.v">; @@ -510,19 +504,19 @@ def VSOXEI16_V : VIndexedStore<MOPSTIndexedOrder, LSWidth16, "vsoxei16.v">; def VSOXEI32_V : VIndexedStore<MOPSTIndexedOrder, LSWidth32, "vsoxei32.v">; def VSOXEI64_V : VIndexedStore<MOPSTIndexedOrder, LSWidth64, "vsoxei64.v">; -defm VL1R : VWholeLoad<1, "vl1r">; -defm VL2R : VWholeLoad<2, "vl2r">; -defm VL4R : VWholeLoad<4, "vl4r">; -defm VL8R : VWholeLoad<8, "vl8r">; +defm VL1R : VWholeLoad<0, "vl1r">; +defm VL2R : VWholeLoad<1, "vl2r">; +defm VL4R : VWholeLoad<3, "vl4r">; +defm VL8R : VWholeLoad<7, "vl8r">; def : InstAlias<"vl1r.v $vd, (${rs1})", (VL1RE8_V VR:$vd, GPR:$rs1)>; def : InstAlias<"vl2r.v $vd, (${rs1})", (VL2RE8_V VR:$vd, GPR:$rs1)>; def : InstAlias<"vl4r.v $vd, (${rs1})", (VL4RE8_V VR:$vd, GPR:$rs1)>; def : InstAlias<"vl8r.v $vd, (${rs1})", (VL8RE8_V VR:$vd, GPR:$rs1)>; -def VS1R_V : VWholeStore<1, "vs1r.v">; -def VS2R_V : VWholeStore<2, "vs2r.v">; -def VS4R_V : VWholeStore<4, "vs4r.v">; -def VS8R_V : VWholeStore<8, "vs8r.v">; +def VS1R_V : VWholeStore<0, "vs1r.v">; +def VS2R_V : VWholeStore<1, "vs2r.v">; +def VS4R_V : VWholeStore<3, "vs4r.v">; +def VS8R_V : VWholeStore<7, "vs8r.v">; // Vector Single-Width Integer Add and Subtract defm VADD_V : VALU_IV_V_X_I<"vadd", 0b000000>; @@ -806,8 +800,8 @@ defm VFWNMSAC_V : VALUr_FV_V_F<"vfwnmsac", 0b111111>; // Vector Floating-Point Square-Root Instruction defm VFSQRT_V : VALU_FV_VS2<"vfsqrt.v", 0b010011, 0b00000>; -defm VFRSQRTE7_V : VALU_FV_VS2<"vfrsqrte7.v", 0b010011, 0b00100>; -defm VFRECE7_V : VALU_FV_VS2<"vfrece7.v", 0b010011, 0b00101>; +defm VFRSQRT7_V : VALU_FV_VS2<"vfrsqrt7.v", 0b010011, 0b00100>; +defm VFREC7_V : VALU_FV_VS2<"vfrec7.v", 0b010011, 0b00101>; // Vector Floating-Point MIN/MAX Instructions defm VFMIN_V : VALU_FV_V_F<"vfmin", 0b000100>; @@ -1058,47 +1052,27 @@ let Predicates = [HasStdExtZvlsseg] in { def VLSEG#nf#E16_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStride, LSWidth16, "vlseg"#nf#"e16.v">; def VLSEG#nf#E32_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStride, LSWidth32, "vlseg"#nf#"e32.v">; def VLSEG#nf#E64_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStride, LSWidth64, "vlseg"#nf#"e64.v">; - def VLSEG#nf#E128_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStride, LSWidth128, "vlseg"#nf#"e128.v">; - def VLSEG#nf#E256_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStride, LSWidth256, "vlseg"#nf#"e256.v">; - def VLSEG#nf#E512_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStride, LSWidth512, "vlseg"#nf#"e512.v">; - def VLSEG#nf#E1024_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStride, LSWidth1024, "vlseg"#nf#"e1024.v">; def VLSEG#nf#E8FF_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStrideFF, LSWidth8, "vlseg"#nf#"e8ff.v">; def VLSEG#nf#E16FF_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStrideFF, LSWidth16, "vlseg"#nf#"e16ff.v">; def VLSEG#nf#E32FF_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStrideFF, LSWidth32, "vlseg"#nf#"e32ff.v">; def VLSEG#nf#E64FF_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStrideFF, LSWidth64, "vlseg"#nf#"e64ff.v">; - def VLSEG#nf#E128FF_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStrideFF, LSWidth128, "vlseg"#nf#"e128ff.v">; - def VLSEG#nf#E256FF_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStrideFF, LSWidth256, "vlseg"#nf#"e256ff.v">; - def VLSEG#nf#E512FF_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStrideFF, LSWidth512, "vlseg"#nf#"e512ff.v">; - def VLSEG#nf#E1024FF_V : VUnitStrideSegmentLoad<!add(nf, -1), LUMOPUnitStrideFF, LSWidth1024, "vlseg"#nf#"e1024ff.v">; def VSSEG#nf#E8_V : VUnitStrideSegmentStore<!add(nf, -1), LSWidth8, "vsseg"#nf#"e8.v">; def VSSEG#nf#E16_V : VUnitStrideSegmentStore<!add(nf, -1), LSWidth16, "vsseg"#nf#"e16.v">; def VSSEG#nf#E32_V : VUnitStrideSegmentStore<!add(nf, -1), LSWidth32, "vsseg"#nf#"e32.v">; def VSSEG#nf#E64_V : VUnitStrideSegmentStore<!add(nf, -1), LSWidth64, "vsseg"#nf#"e64.v">; - def VSSEG#nf#E128_V : VUnitStrideSegmentStore<!add(nf, -1), LSWidth128, "vsseg"#nf#"e128.v">; - def VSSEG#nf#E256_V : VUnitStrideSegmentStore<!add(nf, -1), LSWidth256, "vsseg"#nf#"e256.v">; - def VSSEG#nf#E512_V : VUnitStrideSegmentStore<!add(nf, -1), LSWidth512, "vsseg"#nf#"e512.v">; - def VSSEG#nf#E1024_V : VUnitStrideSegmentStore<!add(nf, -1), LSWidth1024, "vsseg"#nf#"e1024.v">; // Vector Strided Instructions def VLSSEG#nf#E8_V : VStridedSegmentLoad<!add(nf, -1), LSWidth8, "vlsseg"#nf#"e8.v">; def VLSSEG#nf#E16_V : VStridedSegmentLoad<!add(nf, -1), LSWidth16, "vlsseg"#nf#"e16.v">; def VLSSEG#nf#E32_V : VStridedSegmentLoad<!add(nf, -1), LSWidth32, "vlsseg"#nf#"e32.v">; def VLSSEG#nf#E64_V : VStridedSegmentLoad<!add(nf, -1), LSWidth64, "vlsseg"#nf#"e64.v">; - def VLSSEG#nf#E128_V : VStridedSegmentLoad<!add(nf, -1), LSWidth128, "vlsseg"#nf#"e128.v">; - def VLSSEG#nf#E256_V : VStridedSegmentLoad<!add(nf, -1), LSWidth256, "vlsseg"#nf#"e256.v">; - def VLSSEG#nf#E512_V : VStridedSegmentLoad<!add(nf, -1), LSWidth512, "vlsseg"#nf#"e512.v">; - def VLSSEG#nf#E1024_V : VStridedSegmentLoad<!add(nf, -1), LSWidth1024, "vlsseg"#nf#"e1024.v">; def VSSSEG#nf#E8_V : VStridedSegmentStore<!add(nf, -1), LSWidth8, "vssseg"#nf#"e8.v">; def VSSSEG#nf#E16_V : VStridedSegmentStore<!add(nf, -1), LSWidth16, "vssseg"#nf#"e16.v">; def VSSSEG#nf#E32_V : VStridedSegmentStore<!add(nf, -1), LSWidth32, "vssseg"#nf#"e32.v">; def VSSSEG#nf#E64_V : VStridedSegmentStore<!add(nf, -1), LSWidth64, "vssseg"#nf#"e64.v">; - def VSSSEG#nf#E128_V : VStridedSegmentStore<!add(nf, -1), LSWidth128, "vssseg"#nf#"e128.v">; - def VSSSEG#nf#E256_V : VStridedSegmentStore<!add(nf, -1), LSWidth256, "vssseg"#nf#"e256.v">; - def VSSSEG#nf#E512_V : VStridedSegmentStore<!add(nf, -1), LSWidth512, "vssseg"#nf#"e512.v">; - def VSSSEG#nf#E1024_V : VStridedSegmentStore<!add(nf, -1), LSWidth1024, "vssseg"#nf#"e1024.v">; // Vector Indexed Instructions def VLUXSEG#nf#EI8_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, @@ -1109,14 +1083,6 @@ let Predicates = [HasStdExtZvlsseg] in { LSWidth32, "vluxseg"#nf#"ei32.v">; def VLUXSEG#nf#EI64_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, LSWidth64, "vluxseg"#nf#"ei64.v">; - def VLUXSEG#nf#EI128_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, - LSWidth128, "vluxseg"#nf#"ei128.v">; - def VLUXSEG#nf#EI256_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, - LSWidth256, "vluxseg"#nf#"ei256.v">; - def VLUXSEG#nf#EI512_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, - LSWidth512, "vluxseg"#nf#"ei512.v">; - def VLUXSEG#nf#EI1024_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, - LSWidth1024, "vluxseg"#nf#"ei1024.v">; def VLOXSEG#nf#EI8_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, LSWidth8, "vloxseg"#nf#"ei8.v">; @@ -1126,14 +1092,6 @@ let Predicates = [HasStdExtZvlsseg] in { LSWidth32, "vloxseg"#nf#"ei32.v">; def VLOXSEG#nf#EI64_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, LSWidth64, "vloxseg"#nf#"ei64.v">; - def VLOXSEG#nf#EI128_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, - LSWidth128, "vloxseg"#nf#"ei128.v">; - def VLOXSEG#nf#EI256_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, - LSWidth256, "vloxseg"#nf#"ei256.v">; - def VLOXSEG#nf#EI512_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, - LSWidth512, "vloxseg"#nf#"ei512.v">; - def VLOXSEG#nf#EI1024_V : VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, - LSWidth1024, "vloxseg"#nf#"ei1024.v">; def VSUXSEG#nf#EI8_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, LSWidth8, "vsuxseg"#nf#"ei8.v">; @@ -1143,14 +1101,6 @@ let Predicates = [HasStdExtZvlsseg] in { LSWidth32, "vsuxseg"#nf#"ei32.v">; def VSUXSEG#nf#EI64_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, LSWidth64, "vsuxseg"#nf#"ei64.v">; - def VSUXSEG#nf#EI128_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, - LSWidth128, "vsuxseg"#nf#"ei128.v">; - def VSUXSEG#nf#EI256_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, - LSWidth256, "vsuxseg"#nf#"ei256.v">; - def VSUXSEG#nf#EI512_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, - LSWidth512, "vsuxseg"#nf#"ei512.v">; - def VSUXSEG#nf#EI1024_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, - LSWidth1024, "vsuxseg"#nf#"ei1024.v">; def VSOXSEG#nf#EI8_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, LSWidth8, "vsoxseg"#nf#"ei8.v">; @@ -1160,14 +1110,6 @@ let Predicates = [HasStdExtZvlsseg] in { LSWidth32, "vsoxseg"#nf#"ei32.v">; def VSOXSEG#nf#EI64_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, LSWidth64, "vsoxseg"#nf#"ei64.v">; - def VSOXSEG#nf#EI128_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, - LSWidth128, "vsoxseg"#nf#"ei128.v">; - def VSOXSEG#nf#EI256_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, - LSWidth256, "vsoxseg"#nf#"ei256.v">; - def VSOXSEG#nf#EI512_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, - LSWidth512, "vsoxseg"#nf#"ei512.v">; - def VSOXSEG#nf#EI1024_V : VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, - LSWidth1024, "vsoxseg"#nf#"ei1024.v">; } } // Predicates = [HasStdExtZvlsseg] diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 06e4d053d5d7..60bd1b24cab8 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// /// /// This file contains the required infrastructure to support code generation -/// for the standard 'V' (Vector) extension, version 0.9. This version is still +/// for the standard 'V' (Vector) extension, version 0.10. This version is still /// experimental as the 'V' extension hasn't been ratified yet. /// /// This file is included from RISCVInstrInfoV.td @@ -42,17 +42,7 @@ def riscv_read_vl : SDNode<"RISCVISD::READ_VL", //-------------------------------------------------------------- // !X0 | X0 | VLMAX | Set vl to VLMAX // X0 | X0 | Value in vl | Keep current vl, just change vtype. -def NoX0 : SDNodeXForm<undef, -[{ - auto *C = dyn_cast<ConstantSDNode>(N); - if (C && C->isNullValue()) { - SDLoc DL(N); - return SDValue(CurDAG->getMachineNode(RISCV::ADDI, DL, Subtarget->getXLenVT(), - CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT()), - CurDAG->getTargetConstant(0, DL, Subtarget->getXLenVT())), 0); - } - return SDValue(N, 0); -}]>; +def VLOp : ComplexPattern<XLenVT, 1, "selectVLOp">; def DecImm : SDNodeXForm<imm, [{ return CurDAG->getTargetConstant(N->getSExtValue() - 1, SDLoc(N), @@ -1228,6 +1218,14 @@ multiclass VPseudoUSLoad { } } +multiclass VPseudoLoadMask { + foreach mti = AllMasks in { + let VLMul = mti.LMul.value in { + def "_V_" # mti.BX : VPseudoUSLoadNoMask<VR>; + } + } +} + multiclass VPseudoSLoad { foreach lmul = MxList.m in { defvar LInfo = lmul.MX; @@ -1264,6 +1262,14 @@ multiclass VPseudoUSStore { } } +multiclass VPseudoStoreMask { + foreach mti = AllMasks in { + let VLMul = mti.LMul.value in { + def "_V_" # mti.BX : VPseudoUSStoreNoMask<VR>; + } + } +} + multiclass VPseudoSStore { foreach lmul = MxList.m in { defvar LInfo = lmul.MX; @@ -1951,10 +1957,10 @@ class VPatUnaryNoMask<string intrinsic_name, VReg op2_reg_class> : Pat<(result_type (!cast<Intrinsic>(intrinsic_name) (op2_type op2_reg_class:$rs2), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) (op2_type op2_reg_class:$rs2), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatUnaryMask<string intrinsic_name, string inst, @@ -1970,21 +1976,21 @@ class VPatUnaryMask<string intrinsic_name, (result_type result_reg_class:$merge), (op2_type op2_reg_class:$rs2), (mask_type V0), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX#"_MASK") (result_type result_reg_class:$merge), (op2_type op2_reg_class:$rs2), - (mask_type V0), (NoX0 GPR:$vl), sew)>; + (mask_type V0), GPR:$vl, sew)>; class VPatMaskUnaryNoMask<string intrinsic_name, string inst, MTypeInfo mti> : Pat<(mti.Mask (!cast<Intrinsic>(intrinsic_name) (mti.Mask VR:$rs2), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_M_"#mti.BX) (mti.Mask VR:$rs2), - (NoX0 GPR:$vl), mti.SEW)>; + GPR:$vl, mti.SEW)>; class VPatMaskUnaryMask<string intrinsic_name, string inst, @@ -1993,11 +1999,11 @@ class VPatMaskUnaryMask<string intrinsic_name, (mti.Mask VR:$merge), (mti.Mask VR:$rs2), (mti.Mask V0), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_M_"#mti.BX#"_MASK") (mti.Mask VR:$merge), (mti.Mask VR:$rs2), - (mti.Mask V0), (NoX0 GPR:$vl), mti.SEW)>; + (mti.Mask V0), GPR:$vl, mti.SEW)>; class VPatUnaryAnyMask<string intrinsic, string inst, @@ -2013,12 +2019,12 @@ class VPatUnaryAnyMask<string intrinsic, (result_type result_reg_class:$merge), (op1_type op1_reg_class:$rs1), (mask_type VR:$rs2), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) (result_type result_reg_class:$merge), (op1_type op1_reg_class:$rs1), (mask_type VR:$rs2), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatBinaryNoMask<string intrinsic_name, string inst, @@ -2031,11 +2037,11 @@ class VPatBinaryNoMask<string intrinsic_name, Pat<(result_type (!cast<Intrinsic>(intrinsic_name) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatBinaryMask<string intrinsic_name, string inst, @@ -2052,12 +2058,12 @@ class VPatBinaryMask<string intrinsic_name, (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), (mask_type V0), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_MASK") (result_type result_reg_class:$merge), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), (NoX0 GPR:$vl), sew)>; + (mask_type V0), GPR:$vl, sew)>; class VPatTernaryNoMask<string intrinsic, string inst, @@ -2075,12 +2081,12 @@ class VPatTernaryNoMask<string intrinsic, (result_type result_reg_class:$rs3), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) result_reg_class:$rs3, (op1_type op1_reg_class:$rs1), op2_kind:$rs2, - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatTernaryMask<string intrinsic, string inst, @@ -2099,13 +2105,13 @@ class VPatTernaryMask<string intrinsic, (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), (mask_type V0), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX # "_MASK") result_reg_class:$rs3, (op1_type op1_reg_class:$rs1), op2_kind:$rs2, (mask_type V0), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatAMOWDNoMask<string intrinsic_name, string inst, @@ -2119,10 +2125,10 @@ class VPatAMOWDNoMask<string intrinsic_name, GPR:$rs1, (op1_type op1_reg_class:$vs2), (result_type vlmul.vrclass:$vd), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst # "_WD_" # vlmul.MX # "_" # emul.MX) $rs1, $vs2, $vd, - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatAMOWDMask<string intrinsic_name, string inst, @@ -2138,10 +2144,10 @@ class VPatAMOWDMask<string intrinsic_name, (op1_type op1_reg_class:$vs2), (result_type vlmul.vrclass:$vd), (mask_type V0), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst # "_WD_" # vlmul.MX # "_" # emul.MX # "_MASK") $rs1, $vs2, $vd, - (mask_type V0), (NoX0 GPR:$vl), sew)>; + (mask_type V0), GPR:$vl, sew)>; multiclass VPatUSLoad<string intrinsic, string inst, @@ -2153,14 +2159,14 @@ multiclass VPatUSLoad<string intrinsic, { defvar Intr = !cast<Intrinsic>(intrinsic); defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(type (Intr GPR:$rs1, GPR:$vl)), - (Pseudo $rs1, (NoX0 GPR:$vl), sew)>; + def : Pat<(type (Intr GPR:$rs1, (XLenVT (VLOp GPR:$vl)))), + (Pseudo $rs1, GPR:$vl, sew)>; defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); def : Pat<(type (IntrMask (type GetVRegNoV0<reg_class>.R:$merge), - GPR:$rs1, (mask_type V0), GPR:$vl)), + GPR:$rs1, (mask_type V0), (XLenVT (VLOp GPR:$vl)))), (PseudoMask $merge, - $rs1, (mask_type V0), (NoX0 GPR:$vl), sew)>; + $rs1, (mask_type V0), GPR:$vl, sew)>; } multiclass VPatUSLoadFF<string inst, @@ -2171,13 +2177,13 @@ multiclass VPatUSLoadFF<string inst, VReg reg_class> { defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(type (riscv_vleff GPR:$rs1, GPR:$vl)), - (Pseudo $rs1, (NoX0 GPR:$vl), sew)>; + def : Pat<(type (riscv_vleff GPR:$rs1, (XLenVT (VLOp GPR:$vl)))), + (Pseudo $rs1, GPR:$vl, sew)>; defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); def : Pat<(type (riscv_vleff_mask (type GetVRegNoV0<reg_class>.R:$merge), - GPR:$rs1, (mask_type V0), GPR:$vl)), + GPR:$rs1, (mask_type V0), (XLenVT (VLOp GPR:$vl)))), (PseudoMask $merge, - $rs1, (mask_type V0), (NoX0 GPR:$vl), sew)>; + $rs1, (mask_type V0), GPR:$vl, sew)>; } multiclass VPatSLoad<string intrinsic, @@ -2190,14 +2196,14 @@ multiclass VPatSLoad<string intrinsic, { defvar Intr = !cast<Intrinsic>(intrinsic); defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(type (Intr GPR:$rs1, GPR:$rs2, GPR:$vl)), - (Pseudo $rs1, $rs2, (NoX0 GPR:$vl), sew)>; + def : Pat<(type (Intr GPR:$rs1, GPR:$rs2, (XLenVT (VLOp GPR:$vl)))), + (Pseudo $rs1, $rs2, GPR:$vl, sew)>; defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); def : Pat<(type (IntrMask (type GetVRegNoV0<reg_class>.R:$merge), - GPR:$rs1, GPR:$rs2, (mask_type V0), GPR:$vl)), + GPR:$rs1, GPR:$rs2, (mask_type V0), (XLenVT (VLOp GPR:$vl)))), (PseudoMask $merge, - $rs1, $rs2, (mask_type V0), (NoX0 GPR:$vl), sew)>; + $rs1, $rs2, (mask_type V0), GPR:$vl, sew)>; } multiclass VPatILoad<string intrinsic, @@ -2213,16 +2219,16 @@ multiclass VPatILoad<string intrinsic, { defvar Intr = !cast<Intrinsic>(intrinsic); defvar Pseudo = !cast<Instruction>(inst#"_V_"#idx_vlmul.MX#"_"#vlmul.MX); - def : Pat<(type (Intr GPR:$rs1, (idx_type idx_reg_class:$rs2), GPR:$vl)), - (Pseudo $rs1, $rs2, (NoX0 GPR:$vl), sew)>; + def : Pat<(type (Intr GPR:$rs1, (idx_type idx_reg_class:$rs2), (XLenVT (VLOp GPR:$vl)))), + (Pseudo $rs1, $rs2, GPR:$vl, sew)>; defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); defvar PseudoMask = !cast<Instruction>(inst#"_V_"#idx_vlmul.MX#"_"#vlmul.MX#"_MASK"); def : Pat<(type (IntrMask (type GetVRegNoV0<reg_class>.R:$merge), GPR:$rs1, (idx_type idx_reg_class:$rs2), - (mask_type V0), GPR:$vl)), + (mask_type V0), (XLenVT (VLOp GPR:$vl)))), (PseudoMask $merge, - $rs1, $rs2, (mask_type V0), (NoX0 GPR:$vl), sew)>; + $rs1, $rs2, (mask_type V0), GPR:$vl, sew)>; } multiclass VPatUSStore<string intrinsic, @@ -2235,12 +2241,12 @@ multiclass VPatUSStore<string intrinsic, { defvar Intr = !cast<Intrinsic>(intrinsic); defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(Intr (type reg_class:$rs3), GPR:$rs1, GPR:$vl), - (Pseudo $rs3, $rs1, (NoX0 GPR:$vl), sew)>; + def : Pat<(Intr (type reg_class:$rs3), GPR:$rs1, (XLenVT (VLOp GPR:$vl))), + (Pseudo $rs3, $rs1, GPR:$vl, sew)>; defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); - def : Pat<(IntrMask (type reg_class:$rs3), GPR:$rs1, (mask_type V0), GPR:$vl), - (PseudoMask $rs3, $rs1, (mask_type V0), (NoX0 GPR:$vl), sew)>; + def : Pat<(IntrMask (type reg_class:$rs3), GPR:$rs1, (mask_type V0), (XLenVT (VLOp GPR:$vl))), + (PseudoMask $rs3, $rs1, (mask_type V0), GPR:$vl, sew)>; } multiclass VPatSStore<string intrinsic, @@ -2253,12 +2259,12 @@ multiclass VPatSStore<string intrinsic, { defvar Intr = !cast<Intrinsic>(intrinsic); defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(Intr (type reg_class:$rs3), GPR:$rs1, GPR:$rs2, GPR:$vl), - (Pseudo $rs3, $rs1, $rs2, (NoX0 GPR:$vl), sew)>; + def : Pat<(Intr (type reg_class:$rs3), GPR:$rs1, GPR:$rs2, (XLenVT (VLOp GPR:$vl))), + (Pseudo $rs3, $rs1, $rs2, GPR:$vl, sew)>; defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); - def : Pat<(IntrMask (type reg_class:$rs3), GPR:$rs1, GPR:$rs2, (mask_type V0), GPR:$vl), - (PseudoMask $rs3, $rs1, $rs2, (mask_type V0), (NoX0 GPR:$vl), sew)>; + def : Pat<(IntrMask (type reg_class:$rs3), GPR:$rs1, GPR:$rs2, (mask_type V0), (XLenVT (VLOp GPR:$vl))), + (PseudoMask $rs3, $rs1, $rs2, (mask_type V0), GPR:$vl, sew)>; } multiclass VPatIStore<string intrinsic, @@ -2275,13 +2281,13 @@ multiclass VPatIStore<string intrinsic, defvar Intr = !cast<Intrinsic>(intrinsic); defvar Pseudo = !cast<Instruction>(inst#"_V_"#idx_vlmul.MX#"_"#vlmul.MX); def : Pat<(Intr (type reg_class:$rs3), GPR:$rs1, - (idx_type idx_reg_class:$rs2), GPR:$vl), - (Pseudo $rs3, $rs1, $rs2, (NoX0 GPR:$vl), sew)>; + (idx_type idx_reg_class:$rs2), (XLenVT (VLOp GPR:$vl))), + (Pseudo $rs3, $rs1, $rs2, GPR:$vl, sew)>; defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); defvar PseudoMask = !cast<Instruction>(inst#"_V_"#idx_vlmul.MX#"_"#vlmul.MX#"_MASK"); def : Pat<(IntrMask (type reg_class:$rs3), GPR:$rs1, - (idx_type idx_reg_class:$rs2), (mask_type V0), GPR:$vl), - (PseudoMask $rs3, $rs1, $rs2, (mask_type V0), (NoX0 GPR:$vl), sew)>; + (idx_type idx_reg_class:$rs2), (mask_type V0), (XLenVT (VLOp GPR:$vl))), + (PseudoMask $rs3, $rs1, $rs2, (mask_type V0), GPR:$vl, sew)>; } multiclass VPatUnaryS_M<string intrinsic_name, @@ -2289,13 +2295,13 @@ multiclass VPatUnaryS_M<string intrinsic_name, { foreach mti = AllMasks in { def : Pat<(XLenVT (!cast<Intrinsic>(intrinsic_name) - (mti.Mask VR:$rs1), GPR:$vl)), + (mti.Mask VR:$rs1), (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_M_"#mti.BX) $rs1, - (NoX0 GPR:$vl), mti.SEW)>; + GPR:$vl, mti.SEW)>; def : Pat<(XLenVT (!cast<Intrinsic>(intrinsic_name # "_mask") - (mti.Mask VR:$rs1), (mti.Mask V0), GPR:$vl)), + (mti.Mask VR:$rs1), (mti.Mask V0), (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_M_"#mti.BX#"_MASK") $rs1, - (mti.Mask V0), (NoX0 GPR:$vl), mti.SEW)>; + (mti.Mask V0), GPR:$vl, mti.SEW)>; } } @@ -2360,24 +2366,24 @@ multiclass VPatNullaryV<string intrinsic, string instruction> { foreach vti = AllIntegerVectors in { def : Pat<(vti.Vector (!cast<Intrinsic>(intrinsic) - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(instruction#"_V_" # vti.LMul.MX) - (NoX0 GPR:$vl), vti.SEW)>; + GPR:$vl, vti.SEW)>; def : Pat<(vti.Vector (!cast<Intrinsic>(intrinsic # "_mask") (vti.Vector vti.RegClass:$merge), - (vti.Mask V0), (XLenVT GPR:$vl))), + (vti.Mask V0), (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(instruction#"_V_" # vti.LMul.MX # "_MASK") vti.RegClass:$merge, (vti.Mask V0), - (NoX0 GPR:$vl), vti.SEW)>; + GPR:$vl, vti.SEW)>; } } multiclass VPatNullaryM<string intrinsic, string inst> { foreach mti = AllMasks in def : Pat<(mti.Mask (!cast<Intrinsic>(intrinsic) - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_M_"#mti.BX) - (NoX0 GPR:$vl), mti.SEW)>; + GPR:$vl, mti.SEW)>; } multiclass VPatBinary<string intrinsic, @@ -2414,11 +2420,11 @@ multiclass VPatBinaryCarryIn<string intrinsic, (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), (mask_type V0), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), (NoX0 GPR:$vl), sew)>; + (mask_type V0), GPR:$vl, sew)>; } multiclass VPatBinaryMaskOut<string intrinsic, @@ -2435,11 +2441,11 @@ multiclass VPatBinaryMaskOut<string intrinsic, def : Pat<(result_type (!cast<Intrinsic>(intrinsic) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (XLenVT GPR:$vl))), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; } multiclass VPatConversion<string intrinsic, @@ -3125,7 +3131,7 @@ def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins), // Pseudos. let hasSideEffects = 1, mayLoad = 0, mayStore = 0, Defs = [VL, VTYPE] in { def PseudoVSETVLI : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei), []>; - +def PseudoVSETIVLI : Pseudo<(outs GPR:$rd), (ins uimm5:$rs1, VTypeIOp:$vtypei), []>; } //===----------------------------------------------------------------------===// @@ -3142,6 +3148,9 @@ foreach eew = EEWList in { defm PseudoVSE # eew : VPseudoUSStore; } +defm PseudoVLE1 : VPseudoLoadMask; +defm PseudoVSE1 : VPseudoStoreMask; + //===----------------------------------------------------------------------===// // 7.5 Vector Strided Instructions //===----------------------------------------------------------------------===// @@ -3437,12 +3446,12 @@ defm PseudoVFSQRT : VPseudoUnaryV_V; //===----------------------------------------------------------------------===// // 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction //===----------------------------------------------------------------------===// -defm PseudoVFRSQRTE7 : VPseudoUnaryV_V; +defm PseudoVFRSQRT7 : VPseudoUnaryV_V; //===----------------------------------------------------------------------===// // 14.10. Vector Floating-Point Reciprocal Estimate Instruction //===----------------------------------------------------------------------===// -defm PseudoVFRECE7 : VPseudoUnaryV_V; +defm PseudoVFREC7 : VPseudoUnaryV_V; //===----------------------------------------------------------------------===// // 14.11. Vector Floating-Point Min/Max Instructions @@ -3719,6 +3728,15 @@ foreach vti = AllVectors in vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>; } +foreach vti = AllMasks in { + defvar PseudoVLE1 = !cast<Instruction>("PseudoVLE1_V_"#vti.BX); + def : Pat<(vti.Mask (int_riscv_vle1 GPR:$rs1, (XLenVT (VLOp GPR:$vl)))), + (PseudoVLE1 $rs1, GPR:$vl, vti.SEW)>; + defvar PseudoVSE1 = !cast<Instruction>("PseudoVSE1_V_"#vti.BX); + def : Pat<(int_riscv_vse1 (vti.Mask VR:$rs3), GPR:$rs1, (XLenVT (VLOp GPR:$vl))), + (PseudoVSE1 $rs3, $rs1, GPR:$vl, vti.SEW)>; +} + //===----------------------------------------------------------------------===// // 7.5 Vector Strided Instructions //===----------------------------------------------------------------------===// @@ -3886,62 +3904,63 @@ defm "" : VPatBinaryM_VX_VI<"int_riscv_vmsgt", "PseudoVMSGT", AllIntegerVectors> // instruction. foreach vti = AllIntegerVectors in { def : Pat<(vti.Mask (int_riscv_vmslt (vti.Vector vti.RegClass:$rs1), - (vti.Scalar simm5_plus1:$rs2), GPR:$vl)), + (vti.Scalar simm5_plus1:$rs2), (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMSLE_VI_"#vti.LMul.MX) vti.RegClass:$rs1, (DecImm simm5_plus1:$rs2), - (NoX0 GPR:$vl), + GPR:$vl, vti.SEW)>; def : Pat<(vti.Mask (int_riscv_vmslt_mask (vti.Mask V0), (vti.Vector vti.RegClass:$rs1), (vti.Scalar simm5_plus1:$rs2), (vti.Mask VR:$merge), - GPR:$vl)), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMSLE_VI_"#vti.LMul.MX#"_MASK") VR:$merge, vti.RegClass:$rs1, (DecImm simm5_plus1:$rs2), (vti.Mask V0), - (NoX0 GPR:$vl), + GPR:$vl, vti.SEW)>; - def : Pat<(vti.Mask (int_riscv_vmsltu (vti.Vector vti.RegClass:$rs1), - (vti.Scalar simm5_plus1:$rs2), GPR:$vl)), + def : Pat<(vti.Mask (int_riscv_vmsltu (vti.Vector vti.RegClass:$rs1), + (vti.Scalar simm5_plus1:$rs2), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMSLEU_VI_"#vti.LMul.MX) vti.RegClass:$rs1, (DecImm simm5_plus1:$rs2), - (NoX0 GPR:$vl), + GPR:$vl, vti.SEW)>; def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask V0), (vti.Vector vti.RegClass:$rs1), (vti.Scalar simm5_plus1:$rs2), (vti.Mask VR:$merge), - GPR:$vl)), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMSLEU_VI_"#vti.LMul.MX#"_MASK") VR:$merge, vti.RegClass:$rs1, (DecImm simm5_plus1:$rs2), (vti.Mask V0), - (NoX0 GPR:$vl), + GPR:$vl, vti.SEW)>; // Special cases to avoid matching vmsltu.vi 0 (always false) to // vmsleu.vi -1 (always true). Instead match to vmsne.vv. def : Pat<(vti.Mask (int_riscv_vmsltu (vti.Vector vti.RegClass:$rs1), - (vti.Scalar 0), GPR:$vl)), + (vti.Scalar 0), (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMSNE_VV_"#vti.LMul.MX) vti.RegClass:$rs1, vti.RegClass:$rs1, - (NoX0 GPR:$vl), + GPR:$vl, vti.SEW)>; def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask V0), (vti.Vector vti.RegClass:$rs1), (vti.Scalar 0), (vti.Mask VR:$merge), - GPR:$vl)), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMSNE_VV_"#vti.LMul.MX#"_MASK") VR:$merge, vti.RegClass:$rs1, vti.RegClass:$rs1, (vti.Mask V0), - (NoX0 GPR:$vl), + GPR:$vl, vti.SEW)>; } @@ -4002,18 +4021,18 @@ defm "" : VPatBinaryV_VM_XM_IM<"int_riscv_vmerge", "PseudoVMERGE">; //===----------------------------------------------------------------------===// foreach vti = AllVectors in { def : Pat<(vti.Vector (int_riscv_vmv_v_v (vti.Vector vti.RegClass:$rs1), - GPR:$vl)), + (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMV_V_V_"#vti.LMul.MX) - $rs1, (NoX0 GPR:$vl), vti.SEW)>; + $rs1, GPR:$vl, vti.SEW)>; } foreach vti = AllIntegerVectors in { - def : Pat<(vti.Vector (int_riscv_vmv_v_x GPR:$rs2, GPR:$vl)), + def : Pat<(vti.Vector (int_riscv_vmv_v_x GPR:$rs2, (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMV_V_X_"#vti.LMul.MX) - $rs2, (NoX0 GPR:$vl), vti.SEW)>; - def : Pat<(vti.Vector (int_riscv_vmv_v_x simm5:$imm5, GPR:$vl)), + $rs2, GPR:$vl, vti.SEW)>; + def : Pat<(vti.Vector (int_riscv_vmv_v_x simm5:$imm5, (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMV_V_I_"#vti.LMul.MX) - simm5:$imm5, (NoX0 GPR:$vl), vti.SEW)>; + simm5:$imm5, GPR:$vl, vti.SEW)>; } //===----------------------------------------------------------------------===// @@ -4109,12 +4128,12 @@ defm "" : VPatUnaryV_V<"int_riscv_vfsqrt", "PseudoVFSQRT", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction //===----------------------------------------------------------------------===// -defm "" : VPatUnaryV_V<"int_riscv_vfrsqrte7", "PseudoVFRSQRTE7", AllFloatVectors>; +defm "" : VPatUnaryV_V<"int_riscv_vfrsqrt7", "PseudoVFRSQRT7", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.10. Vector Floating-Point Reciprocal Estimate Instruction //===----------------------------------------------------------------------===// -defm "" : VPatUnaryV_V<"int_riscv_vfrece7", "PseudoVFRECE7", AllFloatVectors>; +defm "" : VPatUnaryV_V<"int_riscv_vfrec7", "PseudoVFREC7", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.11. Vector Floating-Point Min/Max Instructions @@ -4157,8 +4176,8 @@ foreach fvti = AllFloatVectors in { defvar instr = !cast<Instruction>("PseudoVMERGE_VIM_"#fvti.LMul.MX); def : Pat<(fvti.Vector (int_riscv_vfmerge (fvti.Vector fvti.RegClass:$rs2), (fvti.Scalar (fpimm0)), - (fvti.Mask V0), (XLenVT GPR:$vl))), - (instr fvti.RegClass:$rs2, 0, (fvti.Mask V0), (NoX0 GPR:$vl), fvti.SEW)>; + (fvti.Mask V0), (XLenVT (VLOp GPR:$vl)))), + (instr fvti.RegClass:$rs2, 0, (fvti.Mask V0), GPR:$vl, fvti.SEW)>; } //===----------------------------------------------------------------------===// @@ -4167,16 +4186,16 @@ foreach fvti = AllFloatVectors in { foreach fvti = AllFloatVectors in { // If we're splatting fpimm0, use vmv.v.x vd, x0. def : Pat<(fvti.Vector (int_riscv_vfmv_v_f - (fvti.Scalar (fpimm0)), GPR:$vl)), + (fvti.Scalar (fpimm0)), (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMV_V_I_"#fvti.LMul.MX) - 0, (NoX0 GPR:$vl), fvti.SEW)>; + 0, GPR:$vl, fvti.SEW)>; def : Pat<(fvti.Vector (int_riscv_vfmv_v_f - (fvti.Scalar fvti.ScalarRegClass:$rs2), GPR:$vl)), + (fvti.Scalar fvti.ScalarRegClass:$rs2), (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVFMV_V_" # fvti.ScalarSuffix # "_" # fvti.LMul.MX) (fvti.Scalar fvti.ScalarRegClass:$rs2), - (NoX0 GPR:$vl), fvti.SEW)>; + GPR:$vl, fvti.SEW)>; } //===----------------------------------------------------------------------===// @@ -4321,9 +4340,9 @@ foreach vti = AllIntegerVectors in { def : Pat<(riscv_vmv_x_s (vti.Vector vti.RegClass:$rs2)), (!cast<Instruction>("PseudoVMV_X_S_" # vti.LMul.MX) $rs2, vti.SEW)>; def : Pat<(vti.Vector (int_riscv_vmv_s_x (vti.Vector vti.RegClass:$rs1), - GPR:$rs2, GPR:$vl)), + GPR:$rs2, (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVMV_S_X_" # vti.LMul.MX) - (vti.Vector $rs1), $rs2, (NoX0 GPR:$vl), vti.SEW)>; + (vti.Vector $rs1), $rs2, GPR:$vl, vti.SEW)>; } } // Predicates = [HasStdExtV] @@ -4339,12 +4358,12 @@ foreach fvti = AllFloatVectors in { (instr $rs2, fvti.SEW)>; def : Pat<(fvti.Vector (int_riscv_vfmv_s_f (fvti.Vector fvti.RegClass:$rs1), - (fvti.Scalar fvti.ScalarRegClass:$rs2), GPR:$vl)), + (fvti.Scalar fvti.ScalarRegClass:$rs2), (XLenVT (VLOp GPR:$vl)))), (!cast<Instruction>("PseudoVFMV_S_"#fvti.ScalarSuffix#"_" # fvti.LMul.MX) (fvti.Vector $rs1), (fvti.Scalar fvti.ScalarRegClass:$rs2), - (NoX0 GPR:$vl), fvti.SEW)>; + GPR:$vl, fvti.SEW)>; } } // Predicates = [HasStdExtV, HasStdExtF] diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index aea3d0e17ccc..dee67708bed1 100644 --- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -8,7 +8,7 @@ /// /// This file contains the required infrastructure and SDNode patterns to /// support code generation for the standard 'V' (Vector) extension, version -/// 0.9. This version is still experimental as the 'V' extension hasn't been +/// 0.10. This version is still experimental as the 'V' extension hasn't been /// ratified yet. /// /// This file is included from and depends upon RISCVInstrInfoVPseudos.td @@ -384,8 +384,8 @@ defm "" : VPatBinarySDNode_VV_VX<mulhs, "PseudoVMULH">; defm "" : VPatBinarySDNode_VV_VX<mulhu, "PseudoVMULHU">; // 12.11. Vector Integer Divide Instructions -defm "" : VPatBinarySDNode_VV_VX<sdiv, "PseudoVDIVU">; -defm "" : VPatBinarySDNode_VV_VX<udiv, "PseudoVDIV">; +defm "" : VPatBinarySDNode_VV_VX<udiv, "PseudoVDIVU">; +defm "" : VPatBinarySDNode_VV_VX<sdiv, "PseudoVDIV">; defm "" : VPatBinarySDNode_VV_VX<urem, "PseudoVREMU">; defm "" : VPatBinarySDNode_VV_VX<srem, "PseudoVREM">; diff --git a/contrib/llvm-project/llvm/lib/Target/VE/VE.h b/contrib/llvm-project/llvm/lib/Target/VE/VE.h index a404f7ced70a..8c1fa840f19c 100644 --- a/contrib/llvm-project/llvm/lib/Target/VE/VE.h +++ b/contrib/llvm-project/llvm/lib/Target/VE/VE.h @@ -334,7 +334,7 @@ inline static bool isMImmVal(uint64_t Val) { return true; } // (m)1 patterns - return (Val & (1UL << 63)) && isShiftedMask_64(Val); + return (Val & (UINT64_C(1) << 63)) && isShiftedMask_64(Val); } inline static bool isMImm32Val(uint32_t Val) { @@ -347,14 +347,14 @@ inline static bool isMImm32Val(uint32_t Val) { return true; } // (m)1 patterns - return (Val & (1 << 31)) && isShiftedMask_32(Val); + return (Val & (UINT32_C(1) << 31)) && isShiftedMask_32(Val); } /// val2MImm - Convert an integer immediate value to target MImm immediate. inline static uint64_t val2MImm(uint64_t Val) { if (Val == 0) return 0; // (0)1 - if (Val & (1UL << 63)) + if (Val & (UINT64_C(1) << 63)) return countLeadingOnes(Val); // (m)1 return countLeadingZeros(Val) | 0x40; // (m)0 } @@ -364,8 +364,8 @@ inline static uint64_t mimm2Val(uint64_t Val) { if (Val == 0) return 0; // (0)1 if ((Val & 0x40) == 0) - return (uint64_t)((1L << 63) >> (Val & 0x3f)); // (m)1 - return ((uint64_t)(-1L) >> (Val & 0x3f)); // (m)0 + return (uint64_t)((INT64_C(1) << 63) >> (Val & 0x3f)); // (m)1 + return ((uint64_t)INT64_C(-1) >> (Val & 0x3f)); // (m)0 } inline unsigned M0(unsigned Val) { return Val + 64; } diff --git a/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp b/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp index 05e482a6b66e..4e6d8e8e1a54 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp @@ -492,6 +492,7 @@ static int readPrefixes(struct InternalInstruction *insn) { insn->addressSize = (insn->hasAdSize ? 4 : 8); insn->displacementSize = 4; insn->immediateSize = 4; + insn->hasOpSize = false; } else { insn->registerSize = (insn->hasOpSize ? 2 : 4); insn->addressSize = (insn->hasAdSize ? 4 : 8); diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86FastISel.cpp b/contrib/llvm-project/llvm/lib/Target/X86/X86FastISel.cpp index caf158102230..a1a16a19f5e5 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86FastISel.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86FastISel.cpp @@ -284,6 +284,14 @@ bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I, return false; } + // Make sure no potentially eflags clobbering phi moves can be inserted in + // between. + auto HasPhis = [](const BasicBlock *Succ) { + return !llvm::empty(Succ->phis()); + }; + if (I->isTerminator() && llvm::any_of(successors(I), HasPhis)) + return false; + CC = TmpCC; return true; } diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp index 0dd20235aa3c..6b816c710f98 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -36916,11 +36916,18 @@ static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V, Res = DAG.getNode(SrcOpc0, DL, SrcVT0, DAG.getBitcast(SrcVT0, Res)); return DAG.getBitcast(VT, Res); } + case X86ISD::VPERMILPI: + // TODO: Handle v4f64 permutes with different low/high lane masks. + if (SrcVT0 == MVT::v4f64) { + uint64_t Mask = Src0.getConstantOperandVal(1); + if ((Mask & 0x3) != ((Mask >> 2) & 0x3)) + break; + } + LLVM_FALLTHROUGH; case X86ISD::VSHLI: case X86ISD::VSRLI: case X86ISD::VSRAI: case X86ISD::PSHUFD: - case X86ISD::VPERMILPI: if (Src1.isUndef() || Src0.getOperand(1) == Src1.getOperand(1)) { SDValue LHS = DAG.getBitcast(VT, Src0.getOperand(0)); SDValue RHS = diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAVX512.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAVX512.td index 0c2b278fdd7b..19012797ae9a 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAVX512.td @@ -1123,10 +1123,10 @@ defm : vextract_for_mask_cast<"VEXTRACTI64x4Z", v64i8_info, v32i8x_info, EXTRACT_get_vextract256_imm, [HasAVX512]>; // vextractps - extract 32 bits from XMM -def VEXTRACTPSZrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst), +def VEXTRACTPSZrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32orGR64:$dst), (ins VR128X:$src1, u8imm:$src2), "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", - [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>, + [(set GR32orGR64:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>, EVEX, VEX_WIG, Sched<[WriteVecExtract]>; def VEXTRACTPSZmr : AVX512AIi8<0x17, MRMDestMem, (outs), diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSSE.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSSE.td index 7cf555748c46..a185a2007b72 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSSE.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSSE.td @@ -3778,7 +3778,7 @@ let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { VEX_4V, VEX_WIG; defm VPACKUSDW : sse4_pack<0x2B, "vpackusdw", v8i16, v4i32, X86Packus, VR128, i128mem, SchedWriteShuffle.XMM, load, 0>, - VEX_4V; + VEX_4V, VEX_WIG; } let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { @@ -3794,7 +3794,7 @@ let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { VEX_4V, VEX_L, VEX_WIG; defm VPACKUSDWY : sse4_pack<0x2B, "vpackusdw", v16i16, v8i32, X86Packus, VR256, i256mem, SchedWriteShuffle.YMM, load, 0>, - VEX_4V, VEX_L; + VEX_4V, VEX_L, VEX_WIG; } let Constraints = "$src1 = $dst" in { @@ -4756,7 +4756,7 @@ let isCommutable = 0 in { SchedWritePHAdd.XMM, 0>, VEX_4V, VEX_WIG; defm VPHSUBD : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v4i32, v4i32, VR128, load, i128mem, - SchedWritePHAdd.XMM, 0>, VEX_4V; + SchedWritePHAdd.XMM, 0>, VEX_4V, VEX_WIG; defm VPSIGNB : SS3I_binop_rm_int<0x08, "vpsignb", int_x86_ssse3_psign_b_128, SchedWriteVecALU.XMM, load, 0>, VEX_4V, VEX_WIG; @@ -4802,7 +4802,7 @@ let isCommutable = 0 in { SchedWritePHAdd.YMM, 0>, VEX_4V, VEX_L, VEX_WIG; defm VPHSUBDY : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v8i32, v8i32, VR256, load, i256mem, - SchedWritePHAdd.YMM, 0>, VEX_4V, VEX_L; + SchedWritePHAdd.YMM, 0>, VEX_4V, VEX_L, VEX_WIG; defm VPSIGNB : SS3I_binop_rm_int_y<0x08, "vpsignb", int_x86_avx2_psign_b, SchedWriteVecALU.YMM>, VEX_4V, VEX_L, VEX_WIG; defm VPSIGNW : SS3I_binop_rm_int_y<0x09, "vpsignw", int_x86_avx2_psign_w, @@ -6503,7 +6503,7 @@ multiclass pcmpistrm_SS42AI<string asm> { let Defs = [XMM0, EFLAGS], hasSideEffects = 0 in { let Predicates = [HasAVX] in - defm VPCMPISTRM : pcmpistrm_SS42AI<"vpcmpistrm">, VEX; + defm VPCMPISTRM : pcmpistrm_SS42AI<"vpcmpistrm">, VEX, VEX_WIG; defm PCMPISTRM : pcmpistrm_SS42AI<"pcmpistrm"> ; } @@ -6521,7 +6521,7 @@ multiclass SS42AI_pcmpestrm<string asm> { let Defs = [XMM0, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in { let Predicates = [HasAVX] in - defm VPCMPESTRM : SS42AI_pcmpestrm<"vpcmpestrm">, VEX; + defm VPCMPESTRM : SS42AI_pcmpestrm<"vpcmpestrm">, VEX, VEX_WIG; defm PCMPESTRM : SS42AI_pcmpestrm<"pcmpestrm">; } @@ -6539,7 +6539,7 @@ multiclass SS42AI_pcmpistri<string asm> { let Defs = [ECX, EFLAGS], hasSideEffects = 0 in { let Predicates = [HasAVX] in - defm VPCMPISTRI : SS42AI_pcmpistri<"vpcmpistri">, VEX; + defm VPCMPISTRI : SS42AI_pcmpistri<"vpcmpistri">, VEX, VEX_WIG; defm PCMPISTRI : SS42AI_pcmpistri<"pcmpistri">; } @@ -6557,7 +6557,7 @@ multiclass SS42AI_pcmpestri<string asm> { let Defs = [ECX, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in { let Predicates = [HasAVX] in - defm VPCMPESTRI : SS42AI_pcmpestri<"vpcmpestri">, VEX; + defm VPCMPESTRI : SS42AI_pcmpestri<"vpcmpestri">, VEX, VEX_WIG; defm PCMPESTRI : SS42AI_pcmpestri<"pcmpestri">; } diff --git a/contrib/llvm-project/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/contrib/llvm-project/llvm/lib/Transforms/IPO/FunctionAttrs.cpp index 30a1f81ad0e1..6730824e860a 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/IPO/FunctionAttrs.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/IPO/FunctionAttrs.cpp @@ -149,6 +149,13 @@ static MemoryAccessKind checkFunctionMemoryAccess(Function &F, bool ThisBody, if (isNoModRef(MRI)) continue; + // A pseudo probe call shouldn't change any function attribute since it + // doesn't translate to a real instruction. It comes with a memory access + // tag to prevent itself being removed by optimizations and not block + // other instructions being optimized. + if (isa<PseudoProbeInst>(I)) + continue; + if (!AliasAnalysis::onlyAccessesArgPointees(MRB)) { // The call could access any memory. If that includes writes, note it. if (isModSet(MRI)) @@ -1445,8 +1452,7 @@ static bool functionWillReturn(const Function &F) { // If there are no loops, then the function is willreturn if all calls in // it are willreturn. return all_of(instructions(F), [](const Instruction &I) { - const auto *CB = dyn_cast<CallBase>(&I); - return !CB || CB->hasFnAttr(Attribute::WillReturn); + return I.willReturn(); }); } diff --git a/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleContextTracker.cpp index 37fc27e91100..158fa0771c3b 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleContextTracker.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleContextTracker.cpp @@ -30,7 +30,7 @@ namespace llvm { ContextTrieNode *ContextTrieNode::getChildContext(const LineLocation &CallSite, StringRef CalleeName) { if (CalleeName.empty()) - return getChildContext(CallSite); + return getHottestChildContext(CallSite); uint32_t Hash = nodeHash(CalleeName, CallSite); auto It = AllChildContext.find(Hash); @@ -40,18 +40,22 @@ ContextTrieNode *ContextTrieNode::getChildContext(const LineLocation &CallSite, } ContextTrieNode * -ContextTrieNode::getChildContext(const LineLocation &CallSite) { +ContextTrieNode::getHottestChildContext(const LineLocation &CallSite) { // CSFDO-TODO: This could be slow, change AllChildContext so we can // do point look up for child node by call site alone. - // CSFDO-TODO: Return the child with max count for indirect call + // Retrieve the child node with max count for indirect call ContextTrieNode *ChildNodeRet = nullptr; + uint64_t MaxCalleeSamples = 0; for (auto &It : AllChildContext) { ContextTrieNode &ChildNode = It.second; - if (ChildNode.CallSiteLoc == CallSite) { - if (ChildNodeRet) - return nullptr; - else - ChildNodeRet = &ChildNode; + if (ChildNode.CallSiteLoc != CallSite) + continue; + FunctionSamples *Samples = ChildNode.getFunctionSamples(); + if (!Samples) + continue; + if (Samples->getTotalSamples() > MaxCalleeSamples) { + ChildNodeRet = &ChildNode; + MaxCalleeSamples = Samples->getTotalSamples(); } } @@ -179,7 +183,7 @@ SampleContextTracker::SampleContextTracker( SampleContext Context(FuncSample.first(), RawContext); LLVM_DEBUG(dbgs() << "Tracking Context for function: " << Context << "\n"); if (!Context.isBaseContext()) - FuncToCtxtProfileSet[Context.getName()].insert(FSamples); + FuncToCtxtProfileSet[Context.getNameWithoutContext()].insert(FSamples); ContextTrieNode *NewNode = getOrCreateContextPath(Context, true); assert(!NewNode->getFunctionSamples() && "New node can't have sample profile"); @@ -191,12 +195,12 @@ FunctionSamples * SampleContextTracker::getCalleeContextSamplesFor(const CallBase &Inst, StringRef CalleeName) { LLVM_DEBUG(dbgs() << "Getting callee context for instr: " << Inst << "\n"); - // CSFDO-TODO: We use CalleeName to differentiate indirect call - // We need to get sample for indirect callee too. DILocation *DIL = Inst.getDebugLoc(); if (!DIL) return nullptr; + // For indirect call, CalleeName will be empty, in which case the context + // profile for callee with largest total samples will be returned. ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, CalleeName); if (CalleeContext) { FunctionSamples *FSamples = CalleeContext->getFunctionSamples(); @@ -209,6 +213,26 @@ SampleContextTracker::getCalleeContextSamplesFor(const CallBase &Inst, return nullptr; } +std::vector<const FunctionSamples *> +SampleContextTracker::getIndirectCalleeContextSamplesFor( + const DILocation *DIL) { + std::vector<const FunctionSamples *> R; + if (!DIL) + return R; + + ContextTrieNode *CallerNode = getContextFor(DIL); + LineLocation CallSite = FunctionSamples::getCallSiteIdentifier(DIL); + for (auto &It : CallerNode->getAllChildContext()) { + ContextTrieNode &ChildNode = It.second; + if (ChildNode.getCallSiteLoc() != CallSite) + continue; + if (FunctionSamples *CalleeSamples = ChildNode.getFunctionSamples()) + R.push_back(CalleeSamples); + } + + return R; +} + FunctionSamples * SampleContextTracker::getContextSamplesFor(const DILocation *DIL) { assert(DIL && "Expect non-null location"); @@ -239,6 +263,17 @@ SampleContextTracker::getContextSamplesFor(const SampleContext &Context) { return Node->getFunctionSamples(); } +SampleContextTracker::ContextSamplesTy & +SampleContextTracker::getAllContextSamplesFor(const Function &Func) { + StringRef CanonName = FunctionSamples::getCanonicalFnName(Func); + return FuncToCtxtProfileSet[CanonName]; +} + +SampleContextTracker::ContextSamplesTy & +SampleContextTracker::getAllContextSamplesFor(StringRef Name) { + return FuncToCtxtProfileSet[Name]; +} + FunctionSamples *SampleContextTracker::getBaseSamplesFor(const Function &Func, bool MergeContext) { StringRef CanonName = FunctionSamples::getCanonicalFnName(Func); @@ -295,11 +330,6 @@ void SampleContextTracker::promoteMergeContextSamplesTree( const Instruction &Inst, StringRef CalleeName) { LLVM_DEBUG(dbgs() << "Promoting and merging context tree for instr: \n" << Inst << "\n"); - // CSFDO-TODO: We also need to promote context profile from indirect - // calls. We won't have callee names from those from call instr. - if (CalleeName.empty()) - return; - // Get the caller context for the call instruction, we don't use callee // name from call because there can be context from indirect calls too. DILocation *DIL = Inst.getDebugLoc(); @@ -308,8 +338,23 @@ void SampleContextTracker::promoteMergeContextSamplesTree( return; // Get the context that needs to be promoted - LineLocation CallSite(FunctionSamples::getOffset(DIL), - DIL->getBaseDiscriminator()); + LineLocation CallSite = FunctionSamples::getCallSiteIdentifier(DIL); + // For indirect call, CalleeName will be empty, in which case we need to + // promote all non-inlined child context profiles. + if (CalleeName.empty()) { + for (auto &It : CallerNode->getAllChildContext()) { + ContextTrieNode *NodeToPromo = &It.second; + if (CallSite != NodeToPromo->getCallSiteLoc()) + continue; + FunctionSamples *FromSamples = NodeToPromo->getFunctionSamples(); + if (FromSamples && FromSamples->getContext().hasState(InlinedContext)) + continue; + promoteMergeContextSamplesTree(*NodeToPromo); + } + return; + } + + // Get the context for the given callee that needs to be promoted ContextTrieNode *NodeToPromo = CallerNode->getChildContext(CallSite, CalleeName); if (!NodeToPromo) @@ -329,6 +374,8 @@ ContextTrieNode &SampleContextTracker::promoteMergeContextSamplesTree( LLVM_DEBUG(dbgs() << " Found context tree root to promote: " << FromSamples->getContext() << "\n"); + assert(!FromSamples->getContext().hasState(InlinedContext) && + "Shouldn't promote inlined context profile"); StringRef ContextStrToRemove = FromSamples->getContext().getCallingContext(); return promoteMergeContextSamplesTree(NodeToPromo, RootContext, ContextStrToRemove); @@ -361,18 +408,14 @@ SampleContextTracker::getCalleeContextFor(const DILocation *DIL, StringRef CalleeName) { assert(DIL && "Expect non-null location"); - // CSSPGO-TODO: need to support indirect callee - if (CalleeName.empty()) - return nullptr; - ContextTrieNode *CallContext = getContextFor(DIL); if (!CallContext) return nullptr; + // When CalleeName is empty, the child context profile with max + // total samples will be returned. return CallContext->getChildContext( - LineLocation(FunctionSamples::getOffset(DIL), - DIL->getBaseDiscriminator()), - CalleeName); + FunctionSamples::getCallSiteIdentifier(DIL), CalleeName); } ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) { @@ -386,8 +429,8 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) { if (Name.empty()) Name = PrevDIL->getScope()->getSubprogram()->getName(); S.push_back( - std::make_pair(LineLocation(FunctionSamples::getOffset(DIL), - DIL->getBaseDiscriminator()), Name)); + std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL), + PrevDIL->getScope()->getSubprogram()->getLinkageName())); PrevDIL = DIL; } @@ -518,4 +561,25 @@ ContextTrieNode &SampleContextTracker::promoteMergeContextSamplesTree( return *ToNode; } +// Replace call graph edges with dynamic call edges from the profile. +void SampleContextTracker::addCallGraphEdges(CallGraph &CG, + StringMap<Function *> &SymbolMap) { + // Add profile call edges to the call graph. + std::queue<ContextTrieNode *> NodeQueue; + NodeQueue.push(&RootContext); + while (!NodeQueue.empty()) { + ContextTrieNode *Node = NodeQueue.front(); + NodeQueue.pop(); + Function *F = SymbolMap.lookup(Node->getFuncName()); + for (auto &I : Node->getAllChildContext()) { + ContextTrieNode *ChildNode = &I.second; + NodeQueue.push(ChildNode); + if (F && !F->isDeclaration()) { + Function *Callee = SymbolMap.lookup(ChildNode->getFuncName()); + if (Callee && !Callee->isDeclaration()) + CG[F]->addCalledFunction(nullptr, CG[Callee]); + } + } + } +} } // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleProfile.cpp b/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleProfile.cpp index 264ac4065e8c..a6a419bfe742 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -26,6 +26,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/None.h" +#include "llvm/ADT/PriorityQueue.h" #include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" @@ -107,6 +108,16 @@ STATISTIC(NumCSNotInlined, STATISTIC(NumMismatchedProfile, "Number of functions with CFG mismatched profile"); STATISTIC(NumMatchedProfile, "Number of functions with CFG matched profile"); +STATISTIC(NumDuplicatedInlinesite, + "Number of inlined callsites with a partial distribution factor"); + +STATISTIC(NumCSInlinedHitMinLimit, + "Number of functions with FDO inline stopped due to min size limit"); +STATISTIC(NumCSInlinedHitMaxLimit, + "Number of functions with FDO inline stopped due to max size limit"); +STATISTIC( + NumCSInlinedHitGrowthLimit, + "Number of functions with FDO inline stopped due to growth size limit"); // Command line option to specify the file to read samples from. This is // mainly used for debugging. @@ -166,11 +177,53 @@ static cl::opt<bool> ProfileTopDownLoad( "order of call graph during sample profile loading. It only " "works for new pass manager. ")); +static cl::opt<bool> UseProfileIndirectCallEdges( + "use-profile-indirect-call-edges", cl::init(true), cl::Hidden, + cl::desc("Considering indirect call samples from profile when top-down " + "processing functions. Only CSSPGO is supported.")); + +static cl::opt<bool> UseProfileTopDownOrder( + "use-profile-top-down-order", cl::init(false), cl::Hidden, + cl::desc("Process functions in one SCC in a top-down order " + "based on the input profile.")); + static cl::opt<bool> ProfileSizeInline( "sample-profile-inline-size", cl::Hidden, cl::init(false), cl::desc("Inline cold call sites in profile loader if it's beneficial " "for code size.")); +static cl::opt<int> ProfileInlineGrowthLimit( + "sample-profile-inline-growth-limit", cl::Hidden, cl::init(12), + cl::desc("The size growth ratio limit for proirity-based sample profile " + "loader inlining.")); + +static cl::opt<int> ProfileInlineLimitMin( + "sample-profile-inline-limit-min", cl::Hidden, cl::init(100), + cl::desc("The lower bound of size growth limit for " + "proirity-based sample profile loader inlining.")); + +static cl::opt<int> ProfileInlineLimitMax( + "sample-profile-inline-limit-max", cl::Hidden, cl::init(10000), + cl::desc("The upper bound of size growth limit for " + "proirity-based sample profile loader inlining.")); + +static cl::opt<int> ProfileICPThreshold( + "sample-profile-icp-threshold", cl::Hidden, cl::init(5), + cl::desc( + "Relative hotness threshold for indirect " + "call promotion in proirity-based sample profile loader inlining.")); + +static cl::opt<int> SampleHotCallSiteThreshold( + "sample-profile-hot-inline-threshold", cl::Hidden, cl::init(3000), + cl::desc("Hot callsite threshold for proirity-based sample profile loader " + "inlining.")); + +static cl::opt<bool> CallsitePrioritizedInline( + "sample-profile-prioritized-inline", cl::Hidden, cl::ZeroOrMore, + cl::init(false), + cl::desc("Use call site prioritized inlining for sample profile loader." + "Currently only CSSPGO is supported.")); + static cl::opt<int> SampleColdCallSiteThreshold( "sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45), cl::desc("Threshold for inlining cold callsites")); @@ -313,6 +366,38 @@ private: DenseMap<uint64_t, StringRef> &CurrentGUIDToFuncNameMap; }; +// Inline candidate used by iterative callsite prioritized inliner +struct InlineCandidate { + CallBase *CallInstr; + const FunctionSamples *CalleeSamples; + // Prorated callsite count, which will be used to guide inlining. For example, + // if a callsite is duplicated in LTO prelink, then in LTO postlink the two + // copies will get their own distribution factors and their prorated counts + // will be used to decide if they should be inlined independently. + uint64_t CallsiteCount; + // Call site distribution factor to prorate the profile samples for a + // duplicated callsite. Default value is 1.0. + float CallsiteDistribution; +}; + +// Inline candidate comparer using call site weight +struct CandidateComparer { + bool operator()(const InlineCandidate &LHS, const InlineCandidate &RHS) { + if (LHS.CallsiteCount != RHS.CallsiteCount) + return LHS.CallsiteCount < RHS.CallsiteCount; + + // Tie breaker using GUID so we have stable/deterministic inlining order + assert(LHS.CalleeSamples && RHS.CalleeSamples && + "Expect non-null FunctionSamples"); + return LHS.CalleeSamples->getGUID(LHS.CalleeSamples->getName()) < + RHS.CalleeSamples->getGUID(RHS.CalleeSamples->getName()); + } +}; + +using CandidateQueue = + PriorityQueue<InlineCandidate, std::vector<InlineCandidate>, + CandidateComparer>; + /// Sample profile pass. /// /// This pass reads profile data from the file specified by @@ -350,9 +435,21 @@ protected: findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const; mutable DenseMap<const DILocation *, const FunctionSamples *> DILocation2SampleMap; const FunctionSamples *findFunctionSamples(const Instruction &I) const; - bool inlineCallInstruction(CallBase &CB); + // Attempt to promote indirect call and also inline the promoted call + bool tryPromoteAndInlineCandidate( + Function &F, InlineCandidate &Candidate, uint64_t SumOrigin, + uint64_t &Sum, DenseSet<Instruction *> &PromotedInsns, + SmallVector<CallBase *, 8> *InlinedCallSites = nullptr); bool inlineHotFunctions(Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs); + InlineCost shouldInlineCandidate(InlineCandidate &Candidate); + bool getInlineCandidate(InlineCandidate *NewCandidate, CallBase *CB); + bool + tryInlineCandidate(InlineCandidate &Candidate, + SmallVector<CallBase *, 8> *InlinedCallSites = nullptr); + bool + inlineHotFunctionsWithPriority(Function &F, + DenseSet<GlobalValue::GUID> &InlinedGUIDs); // Inline cold/small functions in addition to hot ones bool shouldInlineColdCallee(CallBase &CallInst); void emitOptimizationRemarksForInlineCandidates( @@ -371,6 +468,8 @@ protected: uint64_t visitEdge(Edge E, unsigned *NumUnknownEdges, Edge *UnknownEdge); void buildEdges(Function &F); std::vector<Function *> buildFunctionOrder(Module &M, CallGraph *CG); + void addCallGraphEdges(CallGraph &CG, const FunctionSamples &Samples); + void replaceCallGraphEdges(CallGraph &CG, StringMap<Function *> &SymbolMap); bool propagateThroughEdges(Function &F, bool UpdateBlockCount); void computeDominanceAndLoopInfo(Function &F); void clearFunctionData(); @@ -808,7 +907,7 @@ ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) { const ErrorOr<uint64_t> &R = FS->findSamplesAt(Probe->Id, 0); if (R) { - uint64_t Samples = R.get(); + uint64_t Samples = R.get() * Probe->Factor; bool FirstMark = CoverageTracker.markSamplesUsed(FS, Probe->Id, 0, Samples); if (FirstMark) { ORE->emit([&]() { @@ -816,13 +915,17 @@ ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) { Remark << "Applied " << ore::NV("NumSamples", Samples); Remark << " samples from profile (ProbeId="; Remark << ore::NV("ProbeId", Probe->Id); + Remark << ", Factor="; + Remark << ore::NV("Factor", Probe->Factor); + Remark << ", OriginalSamples="; + Remark << ore::NV("OriginalSamples", R.get()); Remark << ")"; return Remark; }); } - LLVM_DEBUG(dbgs() << " " << Probe->Id << ":" << Inst - << " - weight: " << R.get() << ")\n"); + << " - weight: " << R.get() << " - factor: " + << format("%0.2f", Probe->Factor) << ")\n"); return Samples; } return R; @@ -918,6 +1021,31 @@ SampleProfileLoader::findIndirectCallFunctionSamples( return R; } + auto FSCompare = [](const FunctionSamples *L, const FunctionSamples *R) { + assert(L && R && "Expect non-null FunctionSamples"); + if (L->getEntrySamples() != R->getEntrySamples()) + return L->getEntrySamples() > R->getEntrySamples(); + return FunctionSamples::getGUID(L->getName()) < + FunctionSamples::getGUID(R->getName()); + }; + + if (ProfileIsCS) { + auto CalleeSamples = + ContextTracker->getIndirectCalleeContextSamplesFor(DIL); + if (CalleeSamples.empty()) + return R; + + // For CSSPGO, we only use target context profile's entry count + // as that already includes both inlined callee and non-inlined ones.. + Sum = 0; + for (const auto *const FS : CalleeSamples) { + Sum += FS->getEntrySamples(); + R.push_back(FS); + } + llvm::sort(R, FSCompare); + return R; + } + const FunctionSamples *FS = findFunctionSamples(Inst); if (FS == nullptr) return R; @@ -935,12 +1063,7 @@ SampleProfileLoader::findIndirectCallFunctionSamples( Sum += NameFS.second.getEntrySamples(); R.push_back(&NameFS.second); } - llvm::sort(R, [](const FunctionSamples *L, const FunctionSamples *R) { - if (L->getEntrySamples() != R->getEntrySamples()) - return L->getEntrySamples() > R->getEntrySamples(); - return FunctionSamples::getGUID(L->getName()) < - FunctionSamples::getGUID(R->getName()); - }); + llvm::sort(R, FSCompare); } return R; } @@ -977,42 +1100,64 @@ SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const { return it.first->second; } -bool SampleProfileLoader::inlineCallInstruction(CallBase &CB) { - if (ExternalInlineAdvisor) { - auto Advice = ExternalInlineAdvisor->getAdvice(CB); - if (!Advice->isInliningRecommended()) { - Advice->recordUnattemptedInlining(); - return false; +/// Attempt to promote indirect call and also inline the promoted call. +/// +/// \param F Caller function. +/// \param Candidate ICP and inline candidate. +/// \param Sum Sum of target counts for indirect call. +/// \param PromotedInsns Map to keep track of indirect call already processed. +/// \param Candidate ICP and inline candidate. +/// \param InlinedCallSite Output vector for new call sites exposed after +/// inlining. +bool SampleProfileLoader::tryPromoteAndInlineCandidate( + Function &F, InlineCandidate &Candidate, uint64_t SumOrigin, uint64_t &Sum, + DenseSet<Instruction *> &PromotedInsns, + SmallVector<CallBase *, 8> *InlinedCallSite) { + const char *Reason = "Callee function not available"; + // R->getValue() != &F is to prevent promoting a recursive call. + // If it is a recursive call, we do not inline it as it could bloat + // the code exponentially. There is way to better handle this, e.g. + // clone the caller first, and inline the cloned caller if it is + // recursive. As llvm does not inline recursive calls, we will + // simply ignore it instead of handling it explicitly. + auto R = SymbolMap.find(Candidate.CalleeSamples->getFuncName()); + if (R != SymbolMap.end() && R->getValue() && + !R->getValue()->isDeclaration() && R->getValue()->getSubprogram() && + R->getValue()->hasFnAttribute("use-sample-profile") && + R->getValue() != &F && + isLegalToPromote(*Candidate.CallInstr, R->getValue(), &Reason)) { + auto *DI = + &pgo::promoteIndirectCall(*Candidate.CallInstr, R->getValue(), + Candidate.CallsiteCount, Sum, false, ORE); + if (DI) { + Sum -= Candidate.CallsiteCount; + // Prorate the indirect callsite distribution. + // Do not update the promoted direct callsite distribution at this + // point since the original distribution combined with the callee + // profile will be used to prorate callsites from the callee if + // inlined. Once not inlined, the direct callsite distribution should + // be prorated so that the it will reflect the real callsite counts. + setProbeDistributionFactor(*Candidate.CallInstr, + Candidate.CallsiteDistribution * Sum / + SumOrigin); + PromotedInsns.insert(Candidate.CallInstr); + Candidate.CallInstr = DI; + if (isa<CallInst>(DI) || isa<InvokeInst>(DI)) { + bool Inlined = tryInlineCandidate(Candidate, InlinedCallSite); + if (!Inlined) { + // Prorate the direct callsite distribution so that it reflects real + // callsite counts. + setProbeDistributionFactor(*DI, Candidate.CallsiteDistribution * + Candidate.CallsiteCount / + SumOrigin); + } + return Inlined; + } } - // Dummy record, we don't use it for replay. - Advice->recordInlining(); - } - - Function *CalledFunction = CB.getCalledFunction(); - assert(CalledFunction); - DebugLoc DLoc = CB.getDebugLoc(); - BasicBlock *BB = CB.getParent(); - InlineParams Params = getInlineParams(); - Params.ComputeFullInlineCost = true; - // Checks if there is anything in the reachable portion of the callee at - // this callsite that makes this inlining potentially illegal. Need to - // set ComputeFullInlineCost, otherwise getInlineCost may return early - // when cost exceeds threshold without checking all IRs in the callee. - // The acutal cost does not matter because we only checks isNever() to - // see if it is legal to inline the callsite. - InlineCost Cost = - getInlineCost(CB, Params, GetTTI(*CalledFunction), GetAC, GetTLI); - if (Cost.isNever()) { - ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG, "InlineFail", DLoc, BB) - << "incompatible inlining"); - return false; - } - InlineFunctionInfo IFI(nullptr, GetAC); - if (InlineFunction(CB, IFI).isSuccess()) { - // The call to InlineFunction erases I, so we can't pass it here. - emitInlinedInto(*ORE, DLoc, BB, *CalledFunction, *BB->getParent(), Cost, - true, CSINLINE_DEBUG); - return true; + } else { + LLVM_DEBUG(dbgs() << "\nFailed to promote indirect call to " + << Candidate.CalleeSamples->getFuncName() << " because " + << Reason << "\n"); } return false; } @@ -1078,10 +1223,11 @@ bool SampleProfileLoader::inlineHotFunctions( "ProfAccForSymsInList should be false when profile-sample-accurate " "is enabled"); - DenseMap<CallBase *, const FunctionSamples *> localNotInlinedCallSites; + DenseMap<CallBase *, const FunctionSamples *> LocalNotInlinedCallSites; bool Changed = false; - while (true) { - bool LocalChanged = false; + bool LocalChanged = true; + while (LocalChanged) { + LocalChanged = false; SmallVector<CallBase *, 10> CIS; for (auto &BB : F) { bool Hot = false; @@ -1095,7 +1241,7 @@ bool SampleProfileLoader::inlineHotFunctions( "GUIDToFuncNameMap has to be populated"); AllCandidates.push_back(CB); if (FS->getEntrySamples() > 0 || ProfileIsCS) - localNotInlinedCallSites.try_emplace(CB, FS); + LocalNotInlinedCallSites.try_emplace(CB, FS); if (callsiteIsHot(FS, PSI)) Hot = true; else if (shouldInlineColdCallee(*CB)) @@ -1113,6 +1259,11 @@ bool SampleProfileLoader::inlineHotFunctions( } for (CallBase *I : CIS) { Function *CalledFunction = I->getCalledFunction(); + InlineCandidate Candidate = { + I, + LocalNotInlinedCallSites.count(I) ? LocalNotInlinedCallSites[I] + : nullptr, + 0 /* dummy count */, 1.0 /* dummy distribution factor */}; // Do not inline recursive calls. if (CalledFunction == &F) continue; @@ -1121,6 +1272,7 @@ bool SampleProfileLoader::inlineHotFunctions( continue; uint64_t Sum; for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) { + uint64_t SumOrigin = Sum; if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) { FS->findInlinedFunctions(InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); @@ -1129,65 +1281,34 @@ bool SampleProfileLoader::inlineHotFunctions( if (!callsiteIsHot(FS, PSI)) continue; - const char *Reason = "Callee function not available"; - // R->getValue() != &F is to prevent promoting a recursive call. - // If it is a recursive call, we do not inline it as it could bloat - // the code exponentially. There is way to better handle this, e.g. - // clone the caller first, and inline the cloned caller if it is - // recursive. As llvm does not inline recursive calls, we will - // simply ignore it instead of handling it explicitly. - auto CalleeFunctionName = FS->getFuncName(); - auto R = SymbolMap.find(CalleeFunctionName); - if (R != SymbolMap.end() && R->getValue() && - !R->getValue()->isDeclaration() && - R->getValue()->getSubprogram() && - R->getValue()->hasFnAttribute("use-sample-profile") && - R->getValue() != &F && - isLegalToPromote(*I, R->getValue(), &Reason)) { - uint64_t C = FS->getEntrySamples(); - auto &DI = - pgo::promoteIndirectCall(*I, R->getValue(), C, Sum, false, ORE); - Sum -= C; - PromotedInsns.insert(I); - // If profile mismatches, we should not attempt to inline DI. - if ((isa<CallInst>(DI) || isa<InvokeInst>(DI)) && - inlineCallInstruction(cast<CallBase>(DI))) { - if (ProfileIsCS) - ContextTracker->markContextSamplesInlined(FS); - localNotInlinedCallSites.erase(I); - LocalChanged = true; - ++NumCSInlined; - } - } else { - LLVM_DEBUG(dbgs() - << "\nFailed to promote indirect call to " - << CalleeFunctionName << " because " << Reason << "\n"); + Candidate = {I, FS, FS->getEntrySamples(), 1.0}; + if (tryPromoteAndInlineCandidate(F, Candidate, SumOrigin, Sum, + PromotedInsns)) { + LocalNotInlinedCallSites.erase(I); + LocalChanged = true; } } } else if (CalledFunction && CalledFunction->getSubprogram() && !CalledFunction->isDeclaration()) { - if (inlineCallInstruction(*I)) { - if (ProfileIsCS) - ContextTracker->markContextSamplesInlined( - localNotInlinedCallSites[I]); - localNotInlinedCallSites.erase(I); + if (tryInlineCandidate(Candidate)) { + LocalNotInlinedCallSites.erase(I); LocalChanged = true; - ++NumCSInlined; } } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) { findCalleeFunctionSamples(*I)->findInlinedFunctions( InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); } } - if (LocalChanged) { - Changed = true; - } else { - break; - } + Changed |= LocalChanged; } + // For CS profile, profile for not inlined context will be merged when + // base profile is being trieved + if (ProfileIsCS) + return Changed; + // Accumulate not inlined callsite information into notInlinedSamples - for (const auto &Pair : localNotInlinedCallSites) { + for (const auto &Pair : LocalNotInlinedCallSites) { CallBase *I = Pair.getFirst(); Function *Callee = I->getCalledFunction(); if (!Callee || Callee->isDeclaration()) @@ -1232,6 +1353,266 @@ bool SampleProfileLoader::inlineHotFunctions( return Changed; } +bool SampleProfileLoader::tryInlineCandidate( + InlineCandidate &Candidate, SmallVector<CallBase *, 8> *InlinedCallSites) { + + CallBase &CB = *Candidate.CallInstr; + Function *CalledFunction = CB.getCalledFunction(); + assert(CalledFunction && "Expect a callee with definition"); + DebugLoc DLoc = CB.getDebugLoc(); + BasicBlock *BB = CB.getParent(); + + InlineCost Cost = shouldInlineCandidate(Candidate); + if (Cost.isNever()) { + ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG, "InlineFail", DLoc, BB) + << "incompatible inlining"); + return false; + } + + if (!Cost) + return false; + + InlineFunctionInfo IFI(nullptr, GetAC); + if (InlineFunction(CB, IFI).isSuccess()) { + // The call to InlineFunction erases I, so we can't pass it here. + emitInlinedInto(*ORE, DLoc, BB, *CalledFunction, *BB->getParent(), Cost, + true, CSINLINE_DEBUG); + + // Now populate the list of newly exposed call sites. + if (InlinedCallSites) { + InlinedCallSites->clear(); + for (auto &I : IFI.InlinedCallSites) + InlinedCallSites->push_back(I); + } + + if (ProfileIsCS) + ContextTracker->markContextSamplesInlined(Candidate.CalleeSamples); + ++NumCSInlined; + + // Prorate inlined probes for a duplicated inlining callsite which probably + // has a distribution less than 100%. Samples for an inlinee should be + // distributed among the copies of the original callsite based on each + // callsite's distribution factor for counts accuracy. Note that an inlined + // probe may come with its own distribution factor if it has been duplicated + // in the inlinee body. The two factor are multiplied to reflect the + // aggregation of duplication. + if (Candidate.CallsiteDistribution < 1) { + for (auto &I : IFI.InlinedCallSites) { + if (Optional<PseudoProbe> Probe = extractProbe(*I)) + setProbeDistributionFactor(*I, Probe->Factor * + Candidate.CallsiteDistribution); + } + NumDuplicatedInlinesite++; + } + + return true; + } + return false; +} + +bool SampleProfileLoader::getInlineCandidate(InlineCandidate *NewCandidate, + CallBase *CB) { + assert(CB && "Expect non-null call instruction"); + + if (isa<IntrinsicInst>(CB)) + return false; + + // Find the callee's profile. For indirect call, find hottest target profile. + const FunctionSamples *CalleeSamples = findCalleeFunctionSamples(*CB); + if (!CalleeSamples) + return false; + + float Factor = 1.0; + if (Optional<PseudoProbe> Probe = extractProbe(*CB)) + Factor = Probe->Factor; + + uint64_t CallsiteCount = 0; + ErrorOr<uint64_t> Weight = getBlockWeight(CB->getParent()); + if (Weight) + CallsiteCount = Weight.get(); + if (CalleeSamples) + CallsiteCount = std::max( + CallsiteCount, uint64_t(CalleeSamples->getEntrySamples() * Factor)); + + *NewCandidate = {CB, CalleeSamples, CallsiteCount, Factor}; + return true; +} + +InlineCost +SampleProfileLoader::shouldInlineCandidate(InlineCandidate &Candidate) { + std::unique_ptr<InlineAdvice> Advice = nullptr; + if (ExternalInlineAdvisor) { + Advice = ExternalInlineAdvisor->getAdvice(*Candidate.CallInstr); + if (!Advice->isInliningRecommended()) { + Advice->recordUnattemptedInlining(); + return InlineCost::getNever("not previously inlined"); + } + Advice->recordInlining(); + return InlineCost::getAlways("previously inlined"); + } + + // Adjust threshold based on call site hotness, only do this for callsite + // prioritized inliner because otherwise cost-benefit check is done earlier. + int SampleThreshold = SampleColdCallSiteThreshold; + if (CallsitePrioritizedInline) { + if (Candidate.CallsiteCount > PSI->getHotCountThreshold()) + SampleThreshold = SampleHotCallSiteThreshold; + else if (!ProfileSizeInline) + return InlineCost::getNever("cold callsite"); + } + + Function *Callee = Candidate.CallInstr->getCalledFunction(); + assert(Callee && "Expect a definition for inline candidate of direct call"); + + InlineParams Params = getInlineParams(); + Params.ComputeFullInlineCost = true; + // Checks if there is anything in the reachable portion of the callee at + // this callsite that makes this inlining potentially illegal. Need to + // set ComputeFullInlineCost, otherwise getInlineCost may return early + // when cost exceeds threshold without checking all IRs in the callee. + // The acutal cost does not matter because we only checks isNever() to + // see if it is legal to inline the callsite. + InlineCost Cost = getInlineCost(*Candidate.CallInstr, Callee, Params, + GetTTI(*Callee), GetAC, GetTLI); + + // Honor always inline and never inline from call analyzer + if (Cost.isNever() || Cost.isAlways()) + return Cost; + + // For old FDO inliner, we inline the call site as long as cost is not + // "Never". The cost-benefit check is done earlier. + if (!CallsitePrioritizedInline) { + return InlineCost::get(Cost.getCost(), INT_MAX); + } + + // Otherwise only use the cost from call analyzer, but overwite threshold with + // Sample PGO threshold. + return InlineCost::get(Cost.getCost(), SampleThreshold); +} + +bool SampleProfileLoader::inlineHotFunctionsWithPriority( + Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) { + DenseSet<Instruction *> PromotedInsns; + assert(ProfileIsCS && "Prioritiy based inliner only works with CSSPGO now"); + + // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure + // Profile symbol list is ignored when profile-sample-accurate is on. + assert((!ProfAccForSymsInList || + (!ProfileSampleAccurate && + !F.hasFnAttribute("profile-sample-accurate"))) && + "ProfAccForSymsInList should be false when profile-sample-accurate " + "is enabled"); + + // Populating worklist with initial call sites from root inliner, along + // with call site weights. + CandidateQueue CQueue; + InlineCandidate NewCandidate; + for (auto &BB : F) { + for (auto &I : BB.getInstList()) { + auto *CB = dyn_cast<CallBase>(&I); + if (!CB) + continue; + if (getInlineCandidate(&NewCandidate, CB)) + CQueue.push(NewCandidate); + } + } + + // Cap the size growth from profile guided inlining. This is needed even + // though cost of each inline candidate already accounts for callee size, + // because with top-down inlining, we can grow inliner size significantly + // with large number of smaller inlinees each pass the cost check. + assert(ProfileInlineLimitMax >= ProfileInlineLimitMin && + "Max inline size limit should not be smaller than min inline size " + "limit."); + unsigned SizeLimit = F.getInstructionCount() * ProfileInlineGrowthLimit; + SizeLimit = std::min(SizeLimit, (unsigned)ProfileInlineLimitMax); + SizeLimit = std::max(SizeLimit, (unsigned)ProfileInlineLimitMin); + if (ExternalInlineAdvisor) + SizeLimit = std::numeric_limits<unsigned>::max(); + + // Perform iterative BFS call site prioritized inlining + bool Changed = false; + while (!CQueue.empty() && F.getInstructionCount() < SizeLimit) { + InlineCandidate Candidate = CQueue.top(); + CQueue.pop(); + CallBase *I = Candidate.CallInstr; + Function *CalledFunction = I->getCalledFunction(); + + if (CalledFunction == &F) + continue; + if (I->isIndirectCall()) { + if (PromotedInsns.count(I)) + continue; + uint64_t Sum; + auto CalleeSamples = findIndirectCallFunctionSamples(*I, Sum); + uint64_t SumOrigin = Sum; + Sum *= Candidate.CallsiteDistribution; + for (const auto *FS : CalleeSamples) { + // TODO: Consider disable pre-lTO ICP for MonoLTO as well + if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) { + FS->findInlinedFunctions(InlinedGUIDs, F.getParent(), + PSI->getOrCompHotCountThreshold()); + continue; + } + uint64_t EntryCountDistributed = + FS->getEntrySamples() * Candidate.CallsiteDistribution; + // In addition to regular inline cost check, we also need to make sure + // ICP isn't introducing excessive speculative checks even if individual + // target looks beneficial to promote and inline. That means we should + // only do ICP when there's a small number dominant targets. + if (EntryCountDistributed < SumOrigin / ProfileICPThreshold) + break; + // TODO: Fix CallAnalyzer to handle all indirect calls. + // For indirect call, we don't run CallAnalyzer to get InlineCost + // before actual inlining. This is because we could see two different + // types from the same definition, which makes CallAnalyzer choke as + // it's expecting matching parameter type on both caller and callee + // side. See example from PR18962 for the triggering cases (the bug was + // fixed, but we generate different types). + if (!PSI->isHotCount(EntryCountDistributed)) + break; + SmallVector<CallBase *, 8> InlinedCallSites; + // Attach function profile for promoted indirect callee, and update + // call site count for the promoted inline candidate too. + Candidate = {I, FS, EntryCountDistributed, + Candidate.CallsiteDistribution}; + if (tryPromoteAndInlineCandidate(F, Candidate, SumOrigin, Sum, + PromotedInsns, &InlinedCallSites)) { + for (auto *CB : InlinedCallSites) { + if (getInlineCandidate(&NewCandidate, CB)) + CQueue.emplace(NewCandidate); + } + Changed = true; + } + } + } else if (CalledFunction && CalledFunction->getSubprogram() && + !CalledFunction->isDeclaration()) { + SmallVector<CallBase *, 8> InlinedCallSites; + if (tryInlineCandidate(Candidate, &InlinedCallSites)) { + for (auto *CB : InlinedCallSites) { + if (getInlineCandidate(&NewCandidate, CB)) + CQueue.emplace(NewCandidate); + } + Changed = true; + } + } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) { + findCalleeFunctionSamples(*I)->findInlinedFunctions( + InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); + } + } + + if (!CQueue.empty()) { + if (SizeLimit == (unsigned)ProfileInlineLimitMax) + ++NumCSInlinedHitMaxLimit; + else if (SizeLimit == (unsigned)ProfileInlineLimitMin) + ++NumCSInlinedHitMinLimit; + else + ++NumCSInlinedHitGrowthLimit; + } + + return Changed; +} + /// Find equivalence classes for the given block. /// /// This finds all the blocks that are guaranteed to execute the same @@ -1654,6 +2035,14 @@ void SampleProfileLoader::propagateWeights(Function &F) { auto T = FS->findCallTargetMapAt(CallSite); if (!T || T.get().empty()) continue; + // Prorate the callsite counts to reflect what is already done to the + // callsite, such as ICP or calliste cloning. + if (FunctionSamples::ProfileIsProbeBased) { + if (Optional<PseudoProbe> Probe = extractProbe(I)) { + if (Probe->Factor < 1) + T = SampleRecord::adjustCallTargets(T.get(), Probe->Factor); + } + } SmallVector<InstrProfValueData, 2> SortedCallTargets = GetSortedValueDataFromCallTargets(T.get()); uint64_t Sum; @@ -1833,7 +2222,10 @@ bool SampleProfileLoader::emitAnnotations(Function &F) { } DenseSet<GlobalValue::GUID> InlinedGUIDs; - Changed |= inlineHotFunctions(F, InlinedGUIDs); + if (ProfileIsCS && CallsitePrioritizedInline) + Changed |= inlineHotFunctionsWithPriority(F, InlinedGUIDs); + else + Changed |= inlineHotFunctions(F, InlinedGUIDs); // Compute basic block weights. Changed |= computeBlockWeights(F); @@ -1898,6 +2290,45 @@ INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass, "sample-profile", "Sample Profile loader", false, false) +// Add inlined profile call edges to the call graph. +void SampleProfileLoader::addCallGraphEdges(CallGraph &CG, + const FunctionSamples &Samples) { + Function *Caller = SymbolMap.lookup(Samples.getFuncName()); + if (!Caller || Caller->isDeclaration()) + return; + + // Skip non-inlined call edges which are not important since top down inlining + // for non-CS profile is to get more precise profile matching, not to enable + // more inlining. + + for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) { + for (const auto &InlinedSamples : CallsiteSamples.second) { + Function *Callee = SymbolMap.lookup(InlinedSamples.first); + if (Callee && !Callee->isDeclaration()) + CG[Caller]->addCalledFunction(nullptr, CG[Callee]); + addCallGraphEdges(CG, InlinedSamples.second); + } + } +} + +// Replace call graph edges with dynamic call edges from the profile. +void SampleProfileLoader::replaceCallGraphEdges( + CallGraph &CG, StringMap<Function *> &SymbolMap) { + // Remove static call edges from the call graph except for the ones from the + // root which make the call graph connected. + for (const auto &Node : CG) + if (Node.second.get() != CG.getExternalCallingNode()) + Node.second->removeAllCalledFunctions(); + + // Add profile call edges to the call graph. + if (ProfileIsCS) { + ContextTracker->addCallGraphEdges(CG, SymbolMap); + } else { + for (const auto &Samples : Reader->getProfiles()) + addCallGraphEdges(CG, Samples.second); + } +} + std::vector<Function *> SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) { std::vector<Function *> FunctionOrderList; @@ -1920,16 +2351,97 @@ SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) { } assert(&CG->getModule() == &M); + + // Add indirect call edges from profile to augment the static call graph. + // Functions will be processed in a top-down order defined by the static call + // graph. Adjusting the order by considering indirect call edges from the + // profile (which don't exist in the static call graph) can enable the + // inlining of indirect call targets by processing the caller before them. + // TODO: enable this for non-CS profile and fix the counts returning logic to + // have a full support for indirect calls. + if (UseProfileIndirectCallEdges && ProfileIsCS) { + for (auto &Entry : *CG) { + const auto *F = Entry.first; + if (!F || F->isDeclaration() || !F->hasFnAttribute("use-sample-profile")) + continue; + auto &AllContexts = ContextTracker->getAllContextSamplesFor(F->getName()); + if (AllContexts.empty()) + continue; + + for (const auto &BB : *F) { + for (const auto &I : BB.getInstList()) { + const auto *CB = dyn_cast<CallBase>(&I); + if (!CB || !CB->isIndirectCall()) + continue; + const DebugLoc &DLoc = I.getDebugLoc(); + if (!DLoc) + continue; + auto CallSite = FunctionSamples::getCallSiteIdentifier(DLoc); + for (FunctionSamples *Samples : AllContexts) { + if (auto CallTargets = Samples->findCallTargetMapAt(CallSite)) { + for (const auto &Target : CallTargets.get()) { + Function *Callee = SymbolMap.lookup(Target.first()); + if (Callee && !Callee->isDeclaration()) + Entry.second->addCalledFunction(nullptr, (*CG)[Callee]); + } + } + } + } + } + } + } + + // Compute a top-down order the profile which is used to sort functions in + // one SCC later. The static processing order computed for an SCC may not + // reflect the call contexts in the context-sensitive profile, thus may cause + // potential inlining to be overlooked. The function order in one SCC is being + // adjusted to a top-down order based on the profile to favor more inlining. + DenseMap<Function *, uint64_t> ProfileOrderMap; + if (UseProfileTopDownOrder || + (ProfileIsCS && !UseProfileTopDownOrder.getNumOccurrences())) { + // Create a static call graph. The call edges are not important since they + // will be replaced by dynamic edges from the profile. + CallGraph ProfileCG(M); + replaceCallGraphEdges(ProfileCG, SymbolMap); + scc_iterator<CallGraph *> CGI = scc_begin(&ProfileCG); + uint64_t I = 0; + while (!CGI.isAtEnd()) { + for (CallGraphNode *Node : *CGI) { + if (auto *F = Node->getFunction()) + ProfileOrderMap[F] = ++I; + } + ++CGI; + } + } + scc_iterator<CallGraph *> CGI = scc_begin(CG); while (!CGI.isAtEnd()) { - for (CallGraphNode *node : *CGI) { - auto F = node->getFunction(); + uint64_t Start = FunctionOrderList.size(); + for (CallGraphNode *Node : *CGI) { + auto *F = Node->getFunction(); if (F && !F->isDeclaration() && F->hasFnAttribute("use-sample-profile")) FunctionOrderList.push_back(F); } + + // Sort nodes in SCC based on the profile top-down order. + if (!ProfileOrderMap.empty()) { + std::stable_sort(FunctionOrderList.begin() + Start, + FunctionOrderList.end(), + [&ProfileOrderMap](Function *Left, Function *Right) { + return ProfileOrderMap[Left] < ProfileOrderMap[Right]; + }); + } + ++CGI; } + LLVM_DEBUG({ + dbgs() << "Function processing order:\n"; + for (auto F : reverse(FunctionOrderList)) { + dbgs() << F->getName() << "\n"; + } + }); + std::reverse(FunctionOrderList.begin(), FunctionOrderList.end()); return FunctionOrderList; } @@ -1978,6 +2490,12 @@ bool SampleProfileLoader::doInitialization(Module &M, ProfileIsCS = true; FunctionSamples::ProfileIsCS = true; + // Enable priority-base inliner and size inline by default for CSSPGO. + if (!ProfileSizeInline.getNumOccurrences()) + ProfileSizeInline = true; + if (!CallsitePrioritizedInline.getNumOccurrences()) + CallsitePrioritizedInline = true; + // Tracker for profiles under different context ContextTracker = std::make_unique<SampleContextTracker>(Reader->getProfiles()); @@ -2075,6 +2593,7 @@ bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) { } bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) { + LLVM_DEBUG(dbgs() << "\n\nProcessing Function " << F.getName() << "\n"); DILocation2SampleMap.clear(); // By default the entry count is initialized to -1, which will be treated // conservatively by getEntryCount as the same as unknown (None). This is diff --git a/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp index 7cecd20b78d8..a885c3ee4ded 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp @@ -12,6 +12,7 @@ #include "llvm/Transforms/IPO/SampleProfileProbe.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" @@ -25,8 +26,10 @@ #include "llvm/IR/MDBuilder.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/CRC.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/ModuleUtils.h" +#include <unordered_set> #include <vector> using namespace llvm; @@ -35,6 +38,115 @@ using namespace llvm; STATISTIC(ArtificialDbgLine, "Number of probes that have an artificial debug line"); +static cl::opt<bool> + VerifyPseudoProbe("verify-pseudo-probe", cl::init(false), cl::Hidden, + cl::desc("Do pseudo probe verification")); + +static cl::list<std::string> VerifyPseudoProbeFuncList( + "verify-pseudo-probe-funcs", cl::Hidden, + cl::desc("The option to specify the name of the functions to verify.")); + +static cl::opt<bool> + UpdatePseudoProbe("update-pseudo-probe", cl::init(true), cl::Hidden, + cl::desc("Update pseudo probe distribution factor")); + +bool PseudoProbeVerifier::shouldVerifyFunction(const Function *F) { + // Skip function declaration. + if (F->isDeclaration()) + return false; + // Skip function that will not be emitted into object file. The prevailing + // defintion will be verified instead. + if (F->hasAvailableExternallyLinkage()) + return false; + // Do a name matching. + static std::unordered_set<std::string> VerifyFuncNames( + VerifyPseudoProbeFuncList.begin(), VerifyPseudoProbeFuncList.end()); + return VerifyFuncNames.empty() || VerifyFuncNames.count(F->getName().str()); +} + +void PseudoProbeVerifier::registerCallbacks(PassInstrumentationCallbacks &PIC) { + if (VerifyPseudoProbe) { + PIC.registerAfterPassCallback( + [this](StringRef P, Any IR, const PreservedAnalyses &) { + this->runAfterPass(P, IR); + }); + } +} + +// Callback to run after each transformation for the new pass manager. +void PseudoProbeVerifier::runAfterPass(StringRef PassID, Any IR) { + std::string Banner = + "\n*** Pseudo Probe Verification After " + PassID.str() + " ***\n"; + dbgs() << Banner; + if (any_isa<const Module *>(IR)) + runAfterPass(any_cast<const Module *>(IR)); + else if (any_isa<const Function *>(IR)) + runAfterPass(any_cast<const Function *>(IR)); + else if (any_isa<const LazyCallGraph::SCC *>(IR)) + runAfterPass(any_cast<const LazyCallGraph::SCC *>(IR)); + else if (any_isa<const Loop *>(IR)) + runAfterPass(any_cast<const Loop *>(IR)); + else + llvm_unreachable("Unknown IR unit"); +} + +void PseudoProbeVerifier::runAfterPass(const Module *M) { + for (const Function &F : *M) + runAfterPass(&F); +} + +void PseudoProbeVerifier::runAfterPass(const LazyCallGraph::SCC *C) { + for (const LazyCallGraph::Node &N : *C) + runAfterPass(&N.getFunction()); +} + +void PseudoProbeVerifier::runAfterPass(const Function *F) { + if (!shouldVerifyFunction(F)) + return; + ProbeFactorMap ProbeFactors; + for (const auto &BB : *F) + collectProbeFactors(&BB, ProbeFactors); + verifyProbeFactors(F, ProbeFactors); +} + +void PseudoProbeVerifier::runAfterPass(const Loop *L) { + const Function *F = L->getHeader()->getParent(); + runAfterPass(F); +} + +void PseudoProbeVerifier::collectProbeFactors(const BasicBlock *Block, + ProbeFactorMap &ProbeFactors) { + for (const auto &I : *Block) { + if (Optional<PseudoProbe> Probe = extractProbe(I)) + ProbeFactors[Probe->Id] += Probe->Factor; + } +} + +void PseudoProbeVerifier::verifyProbeFactors( + const Function *F, const ProbeFactorMap &ProbeFactors) { + bool BannerPrinted = false; + auto &PrevProbeFactors = FunctionProbeFactors[F->getName()]; + for (const auto &I : ProbeFactors) { + float CurProbeFactor = I.second; + if (PrevProbeFactors.count(I.first)) { + float PrevProbeFactor = PrevProbeFactors[I.first]; + if (std::abs(CurProbeFactor - PrevProbeFactor) > + DistributionFactorVariance) { + if (!BannerPrinted) { + dbgs() << "Function " << F->getName() << ":\n"; + BannerPrinted = true; + } + dbgs() << "Probe " << I.first << "\tprevious factor " + << format("%0.2f", PrevProbeFactor) << "\tcurrent factor " + << format("%0.2f", CurProbeFactor) << "\n"; + } + } + + // Update + PrevProbeFactors[I.first] = I.second; + } +} + PseudoProbeManager::PseudoProbeManager(const Module &M) { if (NamedMDNode *FuncInfo = M.getNamedMetadata(PseudoProbeDescMetadataName)) { for (const auto *Operand : FuncInfo->operands()) { @@ -201,7 +313,8 @@ void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) { Function *ProbeFn = llvm::Intrinsic::getDeclaration(M, Intrinsic::pseudoprobe); Value *Args[] = {Builder.getInt64(Guid), Builder.getInt64(Index), - Builder.getInt32(0)}; + Builder.getInt32(0), + Builder.getInt64(PseudoProbeFullDistributionFactor)}; auto *Probe = Builder.CreateCall(ProbeFn, Args); AssignDebugLoc(Probe); } @@ -219,7 +332,8 @@ void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) { // Levarge the 32-bit discriminator field of debug data to store the ID and // type of a callsite probe. This gets rid of the dependency on plumbing a // customized metadata through the codegen pipeline. - uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData(Index, Type); + uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData( + Index, Type, 0, PseudoProbeDwarfDiscriminator::FullDistributionFactor); if (auto DIL = Call->getDebugLoc()) { DIL = DIL->cloneWithDiscriminator(V); Call->setDebugLoc(DIL); @@ -274,3 +388,47 @@ PreservedAnalyses SampleProfileProbePass::run(Module &M, return PreservedAnalyses::none(); } + +void PseudoProbeUpdatePass::runOnFunction(Function &F, + FunctionAnalysisManager &FAM) { + BlockFrequencyInfo &BFI = FAM.getResult<BlockFrequencyAnalysis>(F); + auto BBProfileCount = [&BFI](BasicBlock *BB) { + return BFI.getBlockProfileCount(BB) + ? BFI.getBlockProfileCount(BB).getValue() + : 0; + }; + + // Collect the sum of execution weight for each probe. + ProbeFactorMap ProbeFactors; + for (auto &Block : F) { + for (auto &I : Block) { + if (Optional<PseudoProbe> Probe = extractProbe(I)) + ProbeFactors[Probe->Id] += BBProfileCount(&Block); + } + } + + // Fix up over-counted probes. + for (auto &Block : F) { + for (auto &I : Block) { + if (Optional<PseudoProbe> Probe = extractProbe(I)) { + float Sum = ProbeFactors[Probe->Id]; + if (Sum != 0) + setProbeDistributionFactor(I, BBProfileCount(&Block) / Sum); + } + } + } +} + +PreservedAnalyses PseudoProbeUpdatePass::run(Module &M, + ModuleAnalysisManager &AM) { + if (UpdatePseudoProbe) { + for (auto &F : M) { + if (F.isDeclaration()) + continue; + FunctionAnalysisManager &FAM = + AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + runOnFunction(F, FAM); + } + } + return PreservedAnalyses::none(); +} diff --git a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index 0b53007bb6dc..07e68c44416d 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -1270,6 +1270,7 @@ Instruction *InstCombinerImpl::visitZExt(ZExtInst &CI) { ICmpInst *LHS = dyn_cast<ICmpInst>(SrcI->getOperand(0)); ICmpInst *RHS = dyn_cast<ICmpInst>(SrcI->getOperand(1)); if (LHS && RHS && LHS->hasOneUse() && RHS->hasOneUse() && + LHS->getOperand(0)->getType() == RHS->getOperand(0)->getType() && (transformZExtICmp(LHS, CI, false) || transformZExtICmp(RHS, CI, false))) { // zext (or icmp, icmp) -> or (zext icmp), (zext icmp) diff --git a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index d687ec654438..b211b0813611 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -592,8 +592,14 @@ static bool isSafeAndProfitableToSinkLoad(LoadInst *L) { BasicBlock::iterator BBI = L->getIterator(), E = L->getParent()->end(); for (++BBI; BBI != E; ++BBI) - if (BBI->mayWriteToMemory()) + if (BBI->mayWriteToMemory()) { + // Calls that only access inaccessible memory do not block sinking the + // load. + if (auto *CB = dyn_cast<CallBase>(BBI)) + if (CB->onlyAccessesInaccessibleMemory()) + continue; return false; + } // Check for non-address taken alloca. If not address-taken already, it isn't // profitable to do this xform. diff --git a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index c265516213aa..16efe863779a 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -345,10 +345,14 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, return false; // Get the constant out of the ICmp, if there is one. + // Only try this when exactly 1 operand is a constant (if both operands + // are constant, the icmp should eventually simplify). Otherwise, we may + // invert the transform that reduces set bits and infinite-loop. + Value *X; const APInt *CmpC; ICmpInst::Predicate Pred; - if (!match(I->getOperand(0), m_c_ICmp(Pred, m_APInt(CmpC), m_Value())) || - CmpC->getBitWidth() != SelC->getBitWidth()) + if (!match(I->getOperand(0), m_ICmp(Pred, m_Value(X), m_APInt(CmpC))) || + isa<Constant>(X) || CmpC->getBitWidth() != SelC->getBitWidth()) return ShrinkDemandedConstant(I, OpNo, DemandedMask); // If the constant is already the same as the ICmp, leave it as-is. diff --git a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 518e909e8ab4..828fd49524ec 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -3878,9 +3878,10 @@ static bool prepareICWorklistFromFunction(Function &F, const DataLayout &DL, } } - // Skip processing debug intrinsics in InstCombine. Processing these call instructions - // consumes non-trivial amount of time and provides no value for the optimization. - if (!isa<DbgInfoIntrinsic>(Inst)) { + // Skip processing debug and pseudo intrinsics in InstCombine. Processing + // these call instructions consumes non-trivial amount of time and + // provides no value for the optimization. + if (!Inst->isDebugOrPseudoInst()) { InstrsForInstCombineWorklist.push_back(Inst); SeenAliasScopes.analyse(Inst); } diff --git a/contrib/llvm-project/llvm/lib/Transforms/Scalar/ADCE.cpp b/contrib/llvm-project/llvm/lib/Transforms/Scalar/ADCE.cpp index 2b649732a799..ce4e5e575fbf 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Scalar/ADCE.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Scalar/ADCE.cpp @@ -325,7 +325,7 @@ void AggressiveDeadCodeElimination::initialize() { bool AggressiveDeadCodeElimination::isAlwaysLive(Instruction &I) { // TODO -- use llvm::isInstructionTriviallyDead - if (I.isEHPad() || I.mayHaveSideEffects()) { + if (I.isEHPad() || I.mayHaveSideEffects() || !I.willReturn()) { // Skip any value profile instrumentation calls if they are // instrumenting constants. if (isInstrumentsConstant(I)) diff --git a/contrib/llvm-project/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/contrib/llvm-project/llvm/lib/Transforms/Scalar/JumpThreading.cpp index 96aef90c1c1a..10b08b4e2224 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -2076,6 +2076,15 @@ JumpThreadingPass::cloneInstructions(BasicBlock::iterator BI, ValueMapping[PN] = NewPN; } + // Clone noalias scope declarations in the threaded block. When threading a + // loop exit, we would otherwise end up with two idential scope declarations + // visible at the same time. + SmallVector<MDNode *> NoAliasScopes; + DenseMap<MDNode *, MDNode *> ClonedScopes; + LLVMContext &Context = PredBB->getContext(); + identifyNoAliasScopesToClone(BI, BE, NoAliasScopes); + cloneNoAliasScopes(NoAliasScopes, ClonedScopes, "thread", Context); + // Clone the non-phi instructions of the source basic block into NewBB, // keeping track of the mapping and using it to remap operands in the cloned // instructions. @@ -2084,6 +2093,7 @@ JumpThreadingPass::cloneInstructions(BasicBlock::iterator BI, New->setName(BI->getName()); NewBB->getInstList().push_back(New); ValueMapping[&*BI] = New; + adaptNoAliasScopes(New, ClonedScopes, Context); // Remap operands to patch up intra-block references. for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i) diff --git a/contrib/llvm-project/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp b/contrib/llvm-project/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp index 18717394d384..822a786fc7c7 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp @@ -1114,12 +1114,16 @@ void LoopUnswitch::emitPreheaderBranchOnCondition( Loop *L = LI->getLoopFor(I->getParent()); auto *DefiningAccess = MemA->getDefiningAccess(); - // If the defining access is a MemoryPhi in the header, get the incoming - // value for the pre-header as defining access. - if (DefiningAccess->getBlock() == I->getParent()) { + // Get the first defining access before the loop. + while (L->contains(DefiningAccess->getBlock())) { + // If the defining access is a MemoryPhi, get the incoming + // value for the pre-header as defining access. if (auto *MemPhi = dyn_cast<MemoryPhi>(DefiningAccess)) { DefiningAccess = MemPhi->getIncomingValueForBlock(L->getLoopPreheader()); + } else { + DefiningAccess = + cast<MemoryDef>(DefiningAccess)->getDefiningAccess(); } } MSSAU->createMemoryAccessInBB(New, DefiningAccess, New->getParent(), diff --git a/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp b/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp index d111a6ba4241..af510f1a84bf 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp @@ -2524,7 +2524,7 @@ private: NewAI.getAlign(), LI.isVolatile(), LI.getName()); if (AATags) - NewLI->setAAMetadata(AATags); + NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); if (LI.isVolatile()) NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); if (NewLI->isAtomic()) @@ -2563,7 +2563,7 @@ private: IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy), getSliceAlign(), LI.isVolatile(), LI.getName()); if (AATags) - NewLI->setAAMetadata(AATags); + NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); if (LI.isVolatile()) NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); @@ -2626,7 +2626,7 @@ private: } StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign()); if (AATags) - Store->setAAMetadata(AATags); + Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); Pass.DeadInsts.push_back(&SI); LLVM_DEBUG(dbgs() << " to: " << *Store << "\n"); @@ -2650,7 +2650,7 @@ private: Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, LLVMContext::MD_access_group}); if (AATags) - Store->setAAMetadata(AATags); + Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); Pass.DeadInsts.push_back(&SI); LLVM_DEBUG(dbgs() << " to: " << *Store << "\n"); return true; @@ -2720,7 +2720,7 @@ private: NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, LLVMContext::MD_access_group}); if (AATags) - NewSI->setAAMetadata(AATags); + NewSI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); if (SI.isVolatile()) NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); if (NewSI->isAtomic()) @@ -2816,7 +2816,7 @@ private: getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size, MaybeAlign(getSliceAlign()), II.isVolatile()); if (AATags) - New->setAAMetadata(AATags); + New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); LLVM_DEBUG(dbgs() << " to: " << *New << "\n"); return false; } @@ -2885,7 +2885,7 @@ private: StoreInst *New = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign(), II.isVolatile()); if (AATags) - New->setAAMetadata(AATags); + New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); LLVM_DEBUG(dbgs() << " to: " << *New << "\n"); return !II.isVolatile(); } @@ -3006,7 +3006,7 @@ private: CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign, Size, II.isVolatile()); if (AATags) - New->setAAMetadata(AATags); + New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); LLVM_DEBUG(dbgs() << " to: " << *New << "\n"); return false; } @@ -3060,7 +3060,7 @@ private: LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign, II.isVolatile(), "copyload"); if (AATags) - Load->setAAMetadata(AATags); + Load->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); Src = Load; } @@ -3080,7 +3080,7 @@ private: StoreInst *Store = cast<StoreInst>( IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile())); if (AATags) - Store->setAAMetadata(AATags); + Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); LLVM_DEBUG(dbgs() << " to: " << *Store << "\n"); return !II.isVolatile(); } @@ -3381,8 +3381,13 @@ private: IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep"); LoadInst *Load = IRB.CreateAlignedLoad(Ty, GEP, Alignment, Name + ".load"); - if (AATags) - Load->setAAMetadata(AATags); + + APInt Offset( + DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0); + if (AATags && + GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset)) + Load->setAAMetadata(AATags.shift(Offset.getZExtValue())); + Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert"); LLVM_DEBUG(dbgs() << " to: " << *Load << "\n"); } @@ -3428,8 +3433,13 @@ private: IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep"); StoreInst *Store = IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Alignment); - if (AATags) - Store->setAAMetadata(AATags); + + APInt Offset( + DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0); + if (AATags && + GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset)) + Store->setAAMetadata(AATags.shift(Offset.getZExtValue())); + LLVM_DEBUG(dbgs() << " to: " << *Store << "\n"); } }; diff --git a/contrib/llvm-project/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/contrib/llvm-project/llvm/lib/Transforms/Utils/BuildLibCalls.cpp index f4afa3ad4623..dba5403f272a 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Utils/BuildLibCalls.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Utils/BuildLibCalls.cpp @@ -170,16 +170,6 @@ static bool setRetAndArgsNoUndef(Function &F) { return setRetNoUndef(F) | setArgsNoUndef(F); } -static bool setRetNonNull(Function &F) { - assert(F.getReturnType()->isPointerTy() && - "nonnull applies only to pointers"); - if (F.hasAttribute(AttributeList::ReturnIndex, Attribute::NonNull)) - return false; - F.addAttribute(AttributeList::ReturnIndex, Attribute::NonNull); - ++NumNonNull; - return true; -} - static bool setReturnedArg(Function &F, unsigned ArgNo) { if (F.hasParamAttribute(ArgNo, Attribute::Returned)) return false; @@ -1005,63 +995,6 @@ bool llvm::inferLibFuncAttributes(Function &F, const TargetLibraryInfo &TLI) { Changed |= setDoesNotCapture(F, 0); Changed |= setDoesNotCapture(F, 1); return Changed; - case LibFunc_ZdlPvRKSt9nothrow_t: // delete(void*, nothrow) - case LibFunc_ZdlPvSt11align_val_tRKSt9nothrow_t: // delete(void*, align_val_t, nothrow) - case LibFunc_ZdaPvRKSt9nothrow_t: // delete[](void*, nothrow) - case LibFunc_ZdaPvSt11align_val_tRKSt9nothrow_t: // delete[](void*, align_val_t, nothrow) - Changed |= setDoesNotThrow(F); - LLVM_FALLTHROUGH; - case LibFunc_ZdlPv: // delete(void*) - case LibFunc_ZdlPvj: // delete(void*, unsigned int) - case LibFunc_ZdlPvm: // delete(void*, unsigned long) - case LibFunc_ZdaPv: // delete[](void*) - case LibFunc_ZdaPvj: // delete[](void*, unsigned int) - case LibFunc_ZdaPvm: // delete[](void*, unsigned long) - case LibFunc_ZdlPvSt11align_val_t: // delete(void*, align_val_t) - case LibFunc_ZdlPvjSt11align_val_t: // delete(void*, unsigned int, align_val_t) - case LibFunc_ZdlPvmSt11align_val_t: // delete(void*, unsigned long, align_val_t) - case LibFunc_ZdaPvSt11align_val_t: // delete[](void*, align_val_t) - case LibFunc_ZdaPvjSt11align_val_t: // delete[](void*, unsigned int, align_val_t) - case LibFunc_ZdaPvmSt11align_val_t: // delete[](void*, unsigned long, align_val_t); - Changed |= setOnlyAccessesInaccessibleMemOrArgMem(F); - Changed |= setArgsNoUndef(F); - Changed |= setWillReturn(F); - Changed |= setDoesNotCapture(F, 0); - return Changed; - case LibFunc_ZnwjRKSt9nothrow_t: // new(unsigned int, nothrow) - case LibFunc_ZnwmRKSt9nothrow_t: // new(unsigned long, nothrow) - case LibFunc_ZnajRKSt9nothrow_t: // new[](unsigned int, nothrow) - case LibFunc_ZnamRKSt9nothrow_t: // new[](unsigned long, nothrow) - case LibFunc_ZnwjSt11align_val_tRKSt9nothrow_t: // new(unsigned int, align_val_t, nothrow) - case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t: // new(unsigned long, align_val_t, nothrow) - case LibFunc_ZnajSt11align_val_tRKSt9nothrow_t: // new[](unsigned int, align_val_t, nothrow) - case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t: // new[](unsigned long, align_val_t, nothrow) - // Nothrow operator new may return null pointer - Changed |= setDoesNotThrow(F); - Changed |= setOnlyAccessesInaccessibleMemory(F); - Changed |= setRetNoUndef(F); - Changed |= setRetDoesNotAlias(F); - Changed |= setWillReturn(F); - return Changed; - case LibFunc_Znwj: // new(unsigned int) - case LibFunc_Znwm: // new(unsigned long) - case LibFunc_Znaj: // new[](unsigned int) - case LibFunc_Znam: // new[](unsigned long) - case LibFunc_ZnwjSt11align_val_t: // new(unsigned int, align_val_t) - case LibFunc_ZnwmSt11align_val_t: // new(unsigned long, align_val_t) - case LibFunc_ZnajSt11align_val_t: // new[](unsigned int, align_val_t) - case LibFunc_ZnamSt11align_val_t: // new[](unsigned long, align_val_t) - case LibFunc_msvc_new_int: // new(unsigned int) - case LibFunc_msvc_new_longlong: // new(unsigned long long) - case LibFunc_msvc_new_array_int: // new[](unsigned int) - case LibFunc_msvc_new_array_longlong: // new[](unsigned long long) - Changed |= setOnlyAccessesInaccessibleMemory(F); - // Operator new always returns a nonnull noalias pointer - Changed |= setRetNoUndef(F); - Changed |= setRetNonNull(F); - Changed |= setRetDoesNotAlias(F); - Changed |= setWillReturn(F); - return Changed; // TODO: add LibFunc entries for: // case LibFunc_memset_pattern4: // case LibFunc_memset_pattern8: diff --git a/contrib/llvm-project/llvm/lib/Transforms/Utils/CloneFunction.cpp b/contrib/llvm-project/llvm/lib/Transforms/Utils/CloneFunction.cpp index 51a49574e55d..6ab061510a60 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -989,3 +989,11 @@ void llvm::identifyNoAliasScopesToClone( if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&I)) NoAliasDeclScopes.push_back(Decl->getScopeList()); } + +void llvm::identifyNoAliasScopesToClone( + BasicBlock::iterator Start, BasicBlock::iterator End, + SmallVectorImpl<MDNode *> &NoAliasDeclScopes) { + for (Instruction &I : make_range(Start, End)) + if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&I)) + NoAliasDeclScopes.push_back(Decl->getScopeList()); +} diff --git a/contrib/llvm-project/llvm/lib/Transforms/Utils/InlineFunction.cpp b/contrib/llvm-project/llvm/lib/Transforms/Utils/InlineFunction.cpp index 0ac8fa537f4e..3026342cc4a6 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -921,14 +921,20 @@ void ScopedAliasMetadataDeepCloner::remap(ValueToValueMapTy &VMap) { if (!I) continue; + // Only update scopes when we find them in the map. If they are not, it is + // because we already handled that instruction before. This is faster than + // tracking which instructions we already updated. if (MDNode *M = I->getMetadata(LLVMContext::MD_alias_scope)) - I->setMetadata(LLVMContext::MD_alias_scope, MDMap[M]); + if (MDNode *MNew = MDMap.lookup(M)) + I->setMetadata(LLVMContext::MD_alias_scope, MNew); if (MDNode *M = I->getMetadata(LLVMContext::MD_noalias)) - I->setMetadata(LLVMContext::MD_noalias, MDMap[M]); + if (MDNode *MNew = MDMap.lookup(M)) + I->setMetadata(LLVMContext::MD_noalias, MNew); if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(I)) - Decl->setScopeList(MDMap[Decl->getScopeList()]); + if (MDNode *MNew = MDMap.lookup(Decl->getScopeList())) + Decl->setScopeList(MNew); } } diff --git a/contrib/llvm-project/llvm/lib/Transforms/Utils/Local.cpp b/contrib/llvm-project/llvm/lib/Transforms/Utils/Local.cpp index 477ea458c763..ae26058c210c 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Utils/Local.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Utils/Local.cpp @@ -420,13 +420,8 @@ bool llvm::wouldInstructionBeTriviallyDead(Instruction *I, return true; } - if (auto *CB = dyn_cast<CallBase>(I)) { - // Treat calls that may not return as alive. - // TODO: Remove the intrinsic escape hatch once all intrinsics set - // willreturn properly. - if (!CB->willReturn() && !isa<IntrinsicInst>(I)) - return false; - } + if (!I->willReturn()) + return false; if (!I->mayHaveSideEffects()) return true; @@ -923,6 +918,7 @@ static void gatherIncomingValuesToPhi(PHINode *PN, /// \param IncomingValues A map from block to value. static void replaceUndefValuesInPhi(PHINode *PN, const IncomingValueMap &IncomingValues) { + SmallVector<unsigned> TrueUndefOps; for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { Value *V = PN->getIncomingValue(i); @@ -930,10 +926,31 @@ static void replaceUndefValuesInPhi(PHINode *PN, BasicBlock *BB = PN->getIncomingBlock(i); IncomingValueMap::const_iterator It = IncomingValues.find(BB); - if (It == IncomingValues.end()) continue; + // Keep track of undef/poison incoming values. Those must match, so we fix + // them up below if needed. + // Note: this is conservatively correct, but we could try harder and group + // the undef values per incoming basic block. + if (It == IncomingValues.end()) { + TrueUndefOps.push_back(i); + continue; + } + + // There is a defined value for this incoming block, so map this undef + // incoming value to the defined value. PN->setIncomingValue(i, It->second); } + + // If there are both undef and poison values incoming, then convert those + // values to undef. It is invalid to have different values for the same + // incoming block. + unsigned PoisonCount = count_if(TrueUndefOps, [&](unsigned i) { + return isa<PoisonValue>(PN->getIncomingValue(i)); + }); + if (PoisonCount != 0 && PoisonCount != TrueUndefOps.size()) { + for (unsigned i : TrueUndefOps) + PN->setIncomingValue(i, UndefValue::get(PN->getType())); + } } /// Replace a value flowing from a block to a phi with diff --git a/contrib/llvm-project/llvm/lib/Transforms/Utils/LoopPeel.cpp b/contrib/llvm-project/llvm/lib/Transforms/Utils/LoopPeel.cpp index cb5fee7d28e6..befacb591762 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Utils/LoopPeel.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Utils/LoopPeel.cpp @@ -509,7 +509,7 @@ static void cloneLoopBlocks( SmallVectorImpl<std::pair<BasicBlock *, BasicBlock *>> &ExitEdges, SmallVectorImpl<BasicBlock *> &NewBlocks, LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap, ValueToValueMapTy &LVMap, DominatorTree *DT, - LoopInfo *LI) { + LoopInfo *LI, ArrayRef<MDNode *> LoopLocalNoAliasDeclScopes) { BasicBlock *Header = L->getHeader(); BasicBlock *Latch = L->getLoopLatch(); BasicBlock *PreHeader = L->getLoopPreheader(); @@ -545,6 +545,15 @@ static void cloneLoopBlocks( } } + { + // Identify what other metadata depends on the cloned version. After + // cloning, replace the metadata with the corrected version for both + // memory instructions and noalias intrinsics. + std::string Ext = (Twine("Peel") + Twine(IterNumber)).str(); + cloneAndAdaptNoAliasScopes(LoopLocalNoAliasDeclScopes, NewBlocks, + Header->getContext(), Ext); + } + // Recursively create the new Loop objects for nested loops, if any, // to preserve LoopInfo. for (Loop *ChildLoop : *L) { @@ -769,13 +778,19 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, uint64_t ExitWeight = 0, FallThroughWeight = 0; initBranchWeights(Header, LatchBR, ExitWeight, FallThroughWeight); + // Identify what noalias metadata is inside the loop: if it is inside the + // loop, the associated metadata must be cloned for each iteration. + SmallVector<MDNode *, 6> LoopLocalNoAliasDeclScopes; + identifyNoAliasScopesToClone(L->getBlocks(), LoopLocalNoAliasDeclScopes); + // For each peeled-off iteration, make a copy of the loop. for (unsigned Iter = 0; Iter < PeelCount; ++Iter) { SmallVector<BasicBlock *, 8> NewBlocks; ValueToValueMapTy VMap; cloneLoopBlocks(L, Iter, InsertTop, InsertBot, ExitEdges, NewBlocks, - LoopBlocks, VMap, LVMap, DT, LI); + LoopBlocks, VMap, LVMap, DT, LI, + LoopLocalNoAliasDeclScopes); // Remap to use values from the current iteration instead of the // previous one. diff --git a/contrib/llvm-project/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/contrib/llvm-project/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 7cfe17618cde..de9560df9785 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -1628,6 +1628,11 @@ static bool canSinkInstructions( I->getType()->isTokenTy()) return false; + // Do not try to sink an instruction in an infinite loop - it can cause + // this algorithm to infinite loop. + if (I->getParent()->getSingleSuccessor() == I->getParent()) + return false; + // Conservatively return false if I is an inline-asm instruction. Sinking // and merging inline-asm instructions can potentially create arguments // that cannot satisfy the inline-asm constraints. @@ -1714,13 +1719,13 @@ static bool canSinkInstructions( return true; } -// Assuming canSinkLastInstruction(Blocks) has returned true, sink the last +// Assuming canSinkInstructions(Blocks) has returned true, sink the last // instruction of every block in Blocks to their common successor, commoning // into one instruction. static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) { auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0); - // canSinkLastInstruction returning true guarantees that every block has at + // canSinkInstructions returning true guarantees that every block has at // least one non-terminator instruction. SmallVector<Instruction*,4> Insts; for (auto *BB : Blocks) { @@ -1733,9 +1738,9 @@ static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) { } // The only checking we need to do now is that all users of all instructions - // are the same PHI node. canSinkLastInstruction should have checked this but - // it is slightly over-aggressive - it gets confused by commutative instructions - // so double-check it here. + // are the same PHI node. canSinkInstructions should have checked this but + // it is slightly over-aggressive - it gets confused by commutative + // instructions so double-check it here. Instruction *I0 = Insts.front(); if (!I0->user_empty()) { auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); @@ -1746,11 +1751,11 @@ static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) { return false; } - // We don't need to do any more checking here; canSinkLastInstruction should + // We don't need to do any more checking here; canSinkInstructions should // have done it all for us. SmallVector<Value*, 4> NewOperands; for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) { - // This check is different to that in canSinkLastInstruction. There, we + // This check is different to that in canSinkInstructions. There, we // cared about the global view once simplifycfg (and instcombine) have // completed - it takes into account PHIs that become trivially // simplifiable. However here we need a more local view; if an operand diff --git a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 1795470fa58c..19797e6f7858 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -142,6 +142,10 @@ public: return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS}); } + VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal) { + return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}); + } + //===--------------------------------------------------------------------===// // RAII helpers. //===--------------------------------------------------------------------===// diff --git a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index ea0d7673edf6..b456a97aa4ec 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -372,19 +372,11 @@ static Type *getMemInstValueType(Value *I) { /// A helper function that returns true if the given type is irregular. The /// type is irregular if its allocated size doesn't equal the store size of an -/// element of the corresponding vector type at the given vectorization factor. -static bool hasIrregularType(Type *Ty, const DataLayout &DL, ElementCount VF) { - // Determine if an array of VF elements of type Ty is "bitcast compatible" - // with a <VF x Ty> vector. - if (VF.isVector()) { - auto *VectorTy = VectorType::get(Ty, VF); - return TypeSize::get(VF.getKnownMinValue() * - DL.getTypeAllocSize(Ty).getFixedValue(), - VF.isScalable()) != DL.getTypeStoreSize(VectorTy); - } - - // If the vectorization factor is one, we just check if an array of type Ty - // requires padding between elements. +/// element of the corresponding vector type. +static bool hasIrregularType(Type *Ty, const DataLayout &DL) { + // Determine if an array of N elements of type Ty is "bitcast compatible" + // with a <N x Ty> vector. + // This is only true if there is no padding between the array elements. return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty); } @@ -5212,7 +5204,7 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened( // requires padding and will be scalarized. auto &DL = I->getModule()->getDataLayout(); auto *ScalarTy = getMemInstValueType(I); - if (hasIrregularType(ScalarTy, DL, VF)) + if (hasIrregularType(ScalarTy, DL)) return false; // Check if masking is required. @@ -5259,7 +5251,7 @@ bool LoopVectorizationCostModel::memoryInstructionCanBeWidened( // requires padding and will be scalarized. auto &DL = I->getModule()->getDataLayout(); auto *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType(); - if (hasIrregularType(ScalarTy, DL, VF)) + if (hasIrregularType(ScalarTy, DL)) return false; return true; @@ -5504,11 +5496,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { return None; } - ElementCount MaxVF = computeFeasibleMaxVF(TC, UserVF); - switch (ScalarEpilogueStatus) { case CM_ScalarEpilogueAllowed: - return MaxVF; + return computeFeasibleMaxVF(TC, UserVF); case CM_ScalarEpilogueNotAllowedUsePredicate: LLVM_FALLTHROUGH; case CM_ScalarEpilogueNotNeededUsePredicate: @@ -5546,7 +5536,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a " "scalar epilogue instead.\n"); ScalarEpilogueStatus = CM_ScalarEpilogueAllowed; - return MaxVF; + return computeFeasibleMaxVF(TC, UserVF); } return None; } @@ -5563,6 +5553,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { InterleaveInfo.invalidateGroupsRequiringScalarEpilogue(); } + ElementCount MaxVF = computeFeasibleMaxVF(TC, UserVF); assert(!MaxVF.isScalable() && "Scalable vectors do not yet support tail folding"); assert((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) && @@ -8196,8 +8187,15 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst, if (BI->getSuccessor(0) != Dst) EdgeMask = Builder.createNot(EdgeMask); - if (SrcMask) // Otherwise block in-mask is all-one, no need to AND. - EdgeMask = Builder.createAnd(EdgeMask, SrcMask); + if (SrcMask) { // Otherwise block in-mask is all-one, no need to AND. + // The condition is 'SrcMask && EdgeMask', which is equivalent to + // 'select i1 SrcMask, i1 EdgeMask, i1 false'. + // The select version does not introduce new UB if SrcMask is false and + // EdgeMask is poison. Using 'and' here introduces undefined behavior. + VPValue *False = Plan->getOrAddVPValue( + ConstantInt::getFalse(BI->getCondition()->getType())); + EdgeMask = Builder.createSelect(SrcMask, EdgeMask, False); + } return EdgeMaskCache[Edge] = EdgeMask; } diff --git a/contrib/llvm-project/llvm/tools/llvm-dwp/llvm-dwp.cpp b/contrib/llvm-project/llvm/tools/llvm-dwp/llvm-dwp.cpp index 9aed3526b0aa..d495bd3d4cab 100644 --- a/contrib/llvm-project/llvm/tools/llvm-dwp/llvm-dwp.cpp +++ b/contrib/llvm-project/llvm/tools/llvm-dwp/llvm-dwp.cpp @@ -526,8 +526,8 @@ getDWOFilenames(StringRef ExecFilename) { std::string DWOCompDir = dwarf::toString(Die.find(dwarf::DW_AT_comp_dir), ""); if (!DWOCompDir.empty()) { - SmallString<16> DWOPath; - sys::path::append(DWOPath, DWOCompDir, DWOName); + SmallString<16> DWOPath(std::move(DWOName)); + sys::fs::make_absolute(DWOCompDir, DWOPath); DWOPaths.emplace_back(DWOPath.data(), DWOPath.size()); } else { DWOPaths.push_back(std::move(DWOName)); diff --git a/contrib/llvm-project/llvm/tools/llvm-objdump/llvm-objdump.cpp b/contrib/llvm-project/llvm/tools/llvm-objdump/llvm-objdump.cpp index 3134f989603a..17128e95727f 100644 --- a/contrib/llvm-project/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/contrib/llvm-project/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -947,8 +947,8 @@ protected: std::unordered_map<std::string, std::vector<StringRef>> LineCache; // Keep track of missing sources. StringSet<> MissingSources; - // Only emit 'no debug info' warning once. - bool WarnedNoDebugInfo; + // Only emit 'invalid debug info' warning once. + bool WarnedInvalidDebugInfo = false; private: bool cacheSource(const DILineInfo& LineInfoFile); @@ -962,8 +962,7 @@ private: public: SourcePrinter() = default; - SourcePrinter(const ObjectFile *Obj, StringRef DefaultArch) - : Obj(Obj), WarnedNoDebugInfo(false) { + SourcePrinter(const ObjectFile *Obj, StringRef DefaultArch) : Obj(Obj) { symbolize::LLVMSymbolizer::Options SymbolizerOpts; SymbolizerOpts.PrintFunctions = DILineInfoSpecifier::FunctionNameKind::LinkageName; @@ -1018,22 +1017,17 @@ void SourcePrinter::printSourceLine(formatted_raw_ostream &OS, return; DILineInfo LineInfo = DILineInfo(); - auto ExpectedLineInfo = Symbolizer->symbolizeCode(*Obj, Address); + Expected<DILineInfo> ExpectedLineInfo = + Symbolizer->symbolizeCode(*Obj, Address); std::string ErrorMessage; - if (!ExpectedLineInfo) - ErrorMessage = toString(ExpectedLineInfo.takeError()); - else + if (ExpectedLineInfo) { LineInfo = *ExpectedLineInfo; - - if (LineInfo.FileName == DILineInfo::BadString) { - if (!WarnedNoDebugInfo) { - std::string Warning = - "failed to parse debug information for " + ObjectFilename.str(); - if (!ErrorMessage.empty()) - Warning += ": " + ErrorMessage; - reportWarning(Warning, ObjectFilename); - WarnedNoDebugInfo = true; - } + } else if (!WarnedInvalidDebugInfo) { + WarnedInvalidDebugInfo = true; + // TODO Untested. + reportWarning("failed to parse debug information: " + + toString(ExpectedLineInfo.takeError()), + ObjectFilename); } if (!Prefix.empty() && sys::path::is_absolute_gnu(LineInfo.FileName)) { diff --git a/contrib/llvm-project/llvm/tools/llvm-profdata/llvm-profdata.cpp b/contrib/llvm-project/llvm/tools/llvm-profdata/llvm-profdata.cpp index 8dc43924c067..7e53c30c7579 100644 --- a/contrib/llvm-project/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/contrib/llvm-project/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -696,7 +696,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper, Remapper ? remapSamples(I->second, *Remapper, Result) : FunctionSamples(); FunctionSamples &Samples = Remapper ? Remapped : I->second; - StringRef FName = Samples.getName(); + StringRef FName = Samples.getNameWithContext(true); MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight)); if (Result != sampleprof_error::success) { std::error_code EC = make_error_code(Result); diff --git a/contrib/llvm-project/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/contrib/llvm-project/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp index 9c68acee0ae2..8734c2d74045 100644 --- a/contrib/llvm-project/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp +++ b/contrib/llvm-project/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp @@ -181,7 +181,12 @@ static void symbolizeInput(const opt::InputArgList &Args, uint64_t AdjustVMA, // the topmost function, which suits our needs better. auto ResOrErr = Symbolizer.symbolizeInlinedCode( ModuleName, {Offset, object::SectionedAddress::UndefSection}); - Printer << (error(ResOrErr) ? DILineInfo() : ResOrErr.get().getFrame(0)); + if (!ResOrErr || ResOrErr->getNumberOfFrames() == 0) { + error(ResOrErr); + Printer << DILineInfo(); + } else { + Printer << ResOrErr->getFrame(0); + } } else { auto ResOrErr = Symbolizer.symbolizeCode( ModuleName, {Offset, object::SectionedAddress::UndefSection}); diff --git a/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp index 4be0d90a45d2..978d24c8300d 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -638,13 +638,13 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, std::max(maxArgAttrs, unsigned(intrinsic.ArgumentAttributes.size())); unsigned &N = UniqAttributes[&intrinsic]; if (N) continue; - assert(AttrNum < 256 && "Too many unique attributes for table!"); N = ++AttrNum; + assert(N < 65536 && "Too many unique attributes for table!"); } // Emit an array of AttributeList. Most intrinsics will have at least one // entry, for the function itself (index ~1), which is usually nounwind. - OS << " static const uint8_t IntrinsicsToAttributesMap[] = {\n"; + OS << " static const uint16_t IntrinsicsToAttributesMap[] = {\n"; for (unsigned i = 0, e = Ints.size(); i != e; ++i) { const CodeGenIntrinsic &intrinsic = Ints[i]; diff --git a/contrib/llvm-project/openmp/runtime/src/kmp_config.h.cmake b/contrib/llvm-project/openmp/runtime/src/kmp_config.h.cmake index 3d682c690fc7..f6aee7197ee8 100644 --- a/contrib/llvm-project/openmp/runtime/src/kmp_config.h.cmake +++ b/contrib/llvm-project/openmp/runtime/src/kmp_config.h.cmake @@ -44,8 +44,8 @@ #define OMPT_DEBUG LIBOMP_OMPT_DEBUG #cmakedefine01 LIBOMP_OMPT_SUPPORT #define OMPT_SUPPORT LIBOMP_OMPT_SUPPORT -#cmakedefine01 LIBOMPTARGET_PROFILING_SUPPORT -#define OMPTARGET_PROFILING_SUPPORT LIBOMPTARGET_PROFILING_SUPPORT +#cmakedefine01 LIBOMP_PROFILING_SUPPORT +#define OMP_PROFILING_SUPPORT LIBOMP_PROFILING_SUPPORT #cmakedefine01 LIBOMP_OMPT_OPTIONAL #define OMPT_OPTIONAL LIBOMP_OMPT_OPTIONAL #cmakedefine01 LIBOMP_USE_ADAPTIVE_LOCKS diff --git a/contrib/llvm-project/openmp/runtime/src/kmp_runtime.cpp b/contrib/llvm-project/openmp/runtime/src/kmp_runtime.cpp index 4a0634d59cff..b981f8740dbe 100644 --- a/contrib/llvm-project/openmp/runtime/src/kmp_runtime.cpp +++ b/contrib/llvm-project/openmp/runtime/src/kmp_runtime.cpp @@ -32,7 +32,7 @@ #include "ompt-specific.h" #endif -#if OMPTARGET_PROFILING_SUPPORT +#if OMP_PROFILING_SUPPORT #include "llvm/Support/TimeProfiler.h" static char *ProfileTraceFile = nullptr; #endif @@ -920,6 +920,12 @@ static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team, if (TCR_PTR(__kmp_threads[0]) == NULL) { --capacity; } + // If it is not for initializing the hidden helper team, we need to take + // __kmp_hidden_helper_threads_num out of the capacity because it is included + // in __kmp_threads_capacity. + if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) { + capacity -= __kmp_hidden_helper_threads_num; + } if (__kmp_nth + new_nthreads - (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > capacity) { @@ -3632,6 +3638,13 @@ int __kmp_register_root(int initial_thread) { --capacity; } + // If it is not for initializing the hidden helper team, we need to take + // __kmp_hidden_helper_threads_num out of the capacity because it is included + // in __kmp_threads_capacity. + if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) { + capacity -= __kmp_hidden_helper_threads_num; + } + /* see if there are too many threads */ if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) { if (__kmp_tp_cached) { @@ -3664,7 +3677,7 @@ int __kmp_register_root(int initial_thread) { /* find an available thread slot */ // Don't reassign the zero slot since we need that to only be used by // initial thread. Slots for hidden helper threads should also be skipped. - if (initial_thread && __kmp_threads[0] == NULL) { + if (initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) { gtid = 0; } else { for (gtid = __kmp_hidden_helper_threads_num + 1; @@ -5740,7 +5753,7 @@ void __kmp_free_thread(kmp_info_t *this_th) { /* ------------------------------------------------------------------------ */ void *__kmp_launch_thread(kmp_info_t *this_thr) { -#if OMPTARGET_PROFILING_SUPPORT +#if OMP_PROFILING_SUPPORT ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE"); // TODO: add a configuration option for time granularity if (ProfileTraceFile) @@ -5848,7 +5861,7 @@ void *__kmp_launch_thread(kmp_info_t *this_thr) { KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid)); KMP_MB(); -#if OMPTARGET_PROFILING_SUPPORT +#if OMP_PROFILING_SUPPORT llvm::timeTraceProfilerFinishThread(); #endif return this_thr; diff --git a/contrib/llvm-project/openmp/runtime/src/kmp_settings.cpp b/contrib/llvm-project/openmp/runtime/src/kmp_settings.cpp index a8522130f972..50f6a05faaf5 100644 --- a/contrib/llvm-project/openmp/runtime/src/kmp_settings.cpp +++ b/contrib/llvm-project/openmp/runtime/src/kmp_settings.cpp @@ -504,9 +504,10 @@ int __kmp_initial_threads_capacity(int req_nproc) { nth = (4 * __kmp_xproc); // If hidden helper task is enabled, we initialize the thread capacity with - // extra - // __kmp_hidden_helper_threads_num. - nth += __kmp_hidden_helper_threads_num; + // extra __kmp_hidden_helper_threads_num. + if (__kmp_enable_hidden_helper) { + nth += __kmp_hidden_helper_threads_num; + } if (nth > __kmp_max_nth) nth = __kmp_max_nth; @@ -3355,7 +3356,8 @@ static void __kmp_stg_parse_allocator(char const *name, char const *value, ntraits++; } } - omp_alloctrait_t traits[ntraits]; + omp_alloctrait_t *traits = + (omp_alloctrait_t *)KMP_ALLOCA(ntraits * sizeof(omp_alloctrait_t)); // Helper macros #define IS_POWER_OF_TWO(n) (((n) & ((n)-1)) == 0) diff --git a/contrib/llvm-project/openmp/runtime/src/kmp_tasking.cpp b/contrib/llvm-project/openmp/runtime/src/kmp_tasking.cpp index 3d7021128dbd..4bcd11946694 100644 --- a/contrib/llvm-project/openmp/runtime/src/kmp_tasking.cpp +++ b/contrib/llvm-project/openmp/runtime/src/kmp_tasking.cpp @@ -326,7 +326,8 @@ static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) { kmp_info_t *thread = __kmp_threads[gtid]; kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); - if (taskdata->td_flags.hidden_helper) { + // We don't need to map to shadow gtid if it is already hidden helper thread + if (taskdata->td_flags.hidden_helper && !KMP_HIDDEN_HELPER_THREAD(gtid)) { gtid = KMP_GTID_TO_SHADOW_GTID(gtid); thread = __kmp_threads[gtid]; } |