diff options
Diffstat (limited to 'contrib/llvm-project/clang')
97 files changed, 3931 insertions, 2545 deletions
diff --git a/contrib/llvm-project/clang/include/clang/AST/Decl.h b/contrib/llvm-project/clang/include/clang/AST/Decl.h index cd0878d70825..a807bcdd76b3 100644 --- a/contrib/llvm-project/clang/include/clang/AST/Decl.h +++ b/contrib/llvm-project/clang/include/clang/AST/Decl.h @@ -358,7 +358,8 @@ public: /// /// \param IsKnownNewer \c true if this declaration is known to be newer /// than \p OldD (for instance, if this declaration is newly-created). - bool declarationReplaces(NamedDecl *OldD, bool IsKnownNewer = true) const; + bool declarationReplaces(const NamedDecl *OldD, + bool IsKnownNewer = true) const; /// Determine whether this declaration has linkage. bool hasLinkage() const; @@ -4332,30 +4333,6 @@ public: return field_begin() == field_end(); } - FieldDecl *getLastField() { - FieldDecl *FD = nullptr; - for (FieldDecl *Field : fields()) - FD = Field; - return FD; - } - const FieldDecl *getLastField() const { - return const_cast<RecordDecl *>(this)->getLastField(); - } - - template <typename Functor> - const FieldDecl *findFieldIf(Functor &Pred) const { - for (const Decl *D : decls()) { - if (const auto *FD = dyn_cast<FieldDecl>(D); FD && Pred(FD)) - return FD; - - if (const auto *RD = dyn_cast<RecordDecl>(D)) - if (const FieldDecl *FD = RD->findFieldIf(Pred)) - return FD; - } - - return nullptr; - } - /// Note that the definition of this type is now complete. virtual void completeDefinition(); diff --git a/contrib/llvm-project/clang/include/clang/AST/DeclBase.h b/contrib/llvm-project/clang/include/clang/AST/DeclBase.h index 5b1038582bc6..10dcbdb262d8 100644 --- a/contrib/llvm-project/clang/include/clang/AST/DeclBase.h +++ b/contrib/llvm-project/clang/include/clang/AST/DeclBase.h @@ -19,7 +19,6 @@ #include "clang/AST/SelectorLocationsKind.h" #include "clang/Basic/IdentifierTable.h" #include "clang/Basic/LLVM.h" -#include "clang/Basic/LangOptions.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/Specifiers.h" #include "llvm/ADT/ArrayRef.h" @@ -489,15 +488,6 @@ public: // Return true if this is a FileContext Decl. bool isFileContextDecl() const; - /// Whether it resembles a flexible array member. This is a static member - /// because we want to be able to call it with a nullptr. That allows us to - /// perform non-Decl specific checks based on the object's type and strict - /// flex array level. - static bool isFlexibleArrayMemberLike( - ASTContext &Context, const Decl *D, QualType Ty, - LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, - bool IgnoreTemplateOrMacroSubstitution); - ASTContext &getASTContext() const LLVM_READONLY; /// Helper to get the language options from the ASTContext. diff --git a/contrib/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h b/contrib/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h index c501801b95bd..8f2714e142bb 100644 --- a/contrib/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h +++ b/contrib/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h @@ -2036,12 +2036,7 @@ bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper( #define DEF_TRAVERSE_TMPL_PART_SPEC_DECL(TMPLDECLKIND, DECLKIND) \ DEF_TRAVERSE_DECL(TMPLDECLKIND##TemplatePartialSpecializationDecl, { \ /* The partial specialization. */ \ - if (TemplateParameterList *TPL = D->getTemplateParameters()) { \ - for (TemplateParameterList::iterator I = TPL->begin(), E = TPL->end(); \ - I != E; ++I) { \ - TRY_TO(TraverseDecl(*I)); \ - } \ - } \ + TRY_TO(TraverseTemplateParameterListHelper(D->getTemplateParameters())); \ /* The args that remains unspecialized. */ \ TRY_TO(TraverseTemplateArgumentLocsHelper( \ D->getTemplateArgsAsWritten()->getTemplateArgs(), \ diff --git a/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h index d7e39ab2616f..e8c27d6c1203 100644 --- a/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h +++ b/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h @@ -289,6 +289,22 @@ public: /// `E` must be a glvalue or a `BuiltinType::BuiltinFn` StorageLocation *getStorageLocation(const Expr &E) const; + /// Returns the result of casting `getStorageLocation(...)` to a subclass of + /// `StorageLocation` (using `cast_or_null<T>`). + /// This assert-fails if the result of `getStorageLocation(...)` is not of + /// type `T *`; if the storage location is not guaranteed to have type `T *`, + /// consider using `dyn_cast_or_null<T>(getStorageLocation(...))` instead. + template <typename T> + std::enable_if_t<std::is_base_of_v<StorageLocation, T>, T *> + get(const ValueDecl &D) const { + return cast_or_null<T>(getStorageLocation(D)); + } + template <typename T> + std::enable_if_t<std::is_base_of_v<StorageLocation, T>, T *> + get(const Expr &E) const { + return cast_or_null<T>(getStorageLocation(E)); + } + /// Returns the storage location assigned to the `this` pointee in the /// environment or null if the `this` pointee has no assigned storage location /// in the environment. @@ -325,7 +341,8 @@ public: /// /// Requirements: /// `E` must be a prvalue of record type. - RecordStorageLocation &getResultObjectLocation(const Expr &RecordPRValue); + RecordStorageLocation & + getResultObjectLocation(const Expr &RecordPRValue) const; /// Returns the return value of the current function. This can be null if: /// - The function has a void return type @@ -434,24 +451,14 @@ public: /// Assigns `Val` as the value of the prvalue `E` in the environment. /// - /// If `E` is not yet associated with a storage location, associates it with - /// a newly created storage location. In any case, associates the storage - /// location of `E` with `Val`. - /// - /// Once the migration to strict handling of value categories is complete - /// (see https://discourse.llvm.org/t/70086), this function will be renamed to - /// `setValue()`. At this point, prvalue expressions will be associated - /// directly with `Value`s, and the legacy behavior of associating prvalue - /// expressions with storage locations (as described above) will be - /// eliminated. - /// /// Requirements: /// - /// `E` must be a prvalue - /// If `Val` is a `RecordValue`, its `RecordStorageLocation` must be the - /// same as that of any `RecordValue` that has already been associated with - /// `E`. This is to guarantee that the result object initialized by a prvalue - /// `RecordValue` has a durable storage location. + /// - `E` must be a prvalue + /// - If `Val` is a `RecordValue`, its `RecordStorageLocation` must be + /// `getResultObjectLocation(E)`. An exception to this is if `E` is an + /// expression that originally creates a `RecordValue` (such as a + /// `CXXConstructExpr` or `CallExpr`), as these establish the location of + /// the result object in the first place. void setValue(const Expr &E, Value &Val); /// Returns the value assigned to `Loc` in the environment or null if `Loc` @@ -466,6 +473,26 @@ public: /// storage location in the environment, otherwise returns null. Value *getValue(const Expr &E) const; + /// Returns the result of casting `getValue(...)` to a subclass of `Value` + /// (using `cast_or_null<T>`). + /// This assert-fails if the result of `getValue(...)` is not of type `T *`; + /// if the value is not guaranteed to have type `T *`, consider using + /// `dyn_cast_or_null<T>(getValue(...))` instead. + template <typename T> + std::enable_if_t<std::is_base_of_v<Value, T>, T *> + get(const StorageLocation &Loc) const { + return cast_or_null<T>(getValue(Loc)); + } + template <typename T> + std::enable_if_t<std::is_base_of_v<Value, T>, T *> + get(const ValueDecl &D) const { + return cast_or_null<T>(getValue(D)); + } + template <typename T> + std::enable_if_t<std::is_base_of_v<Value, T>, T *> get(const Expr &E) const { + return cast_or_null<T>(getValue(E)); + } + // FIXME: should we deprecate the following & call arena().create() directly? /// Creates a `T` (some subclass of `Value`), forwarding `args` to the @@ -608,14 +635,6 @@ private: // The copy-constructor is for use in fork() only. Environment(const Environment &) = default; - /// Internal version of `setStorageLocation()` that doesn't check if the - /// expression is a prvalue. - void setStorageLocationInternal(const Expr &E, StorageLocation &Loc); - - /// Internal version of `getStorageLocation()` that doesn't check if the - /// expression is a prvalue. - StorageLocation *getStorageLocationInternal(const Expr &E) const; - /// Creates a value appropriate for `Type`, if `Type` is supported, otherwise /// return null. /// @@ -708,20 +727,9 @@ RecordStorageLocation *getBaseObjectLocation(const MemberExpr &ME, std::vector<FieldDecl *> getFieldsForInitListExpr(const RecordDecl *RD); /// Associates a new `RecordValue` with `Loc` and returns the new value. -/// It is not defined whether the field values remain the same or not. -/// -/// This function is primarily intended for use by checks that set custom -/// properties on `RecordValue`s to model the state of these values. Such checks -/// should avoid modifying the properties of an existing `RecordValue` because -/// these changes would be visible to other `Environment`s that share the same -/// `RecordValue`. Instead, call `refreshRecordValue()`, then set the properties -/// on the new `RecordValue` that it returns. Typical usage: -/// -/// refreshRecordValue(Loc, Env).setProperty("my_prop", MyPropValue); RecordValue &refreshRecordValue(RecordStorageLocation &Loc, Environment &Env); /// Associates a new `RecordValue` with `Expr` and returns the new value. -/// See also documentation for the overload above. RecordValue &refreshRecordValue(const Expr &Expr, Environment &Env); } // namespace dataflow diff --git a/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/RecordOps.h b/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/RecordOps.h index 7b87840d626b..783e53e980aa 100644 --- a/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/RecordOps.h +++ b/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/RecordOps.h @@ -22,19 +22,13 @@ namespace dataflow { /// Copies a record (struct, class, or union) from `Src` to `Dst`. /// /// This performs a deep copy, i.e. it copies every field (including synthetic -/// fields) and recurses on fields of record type. It also copies properties -/// from the `RecordValue` associated with `Src` to the `RecordValue` associated -/// with `Dst` (if these `RecordValue`s exist). +/// fields) and recurses on fields of record type. /// /// If there is a `RecordValue` associated with `Dst` in the environment, this /// function creates a new `RecordValue` and associates it with `Dst`; clients /// need to be aware of this and must not assume that the `RecordValue` /// associated with `Dst` remains the same after the call. /// -/// We create a new `RecordValue` rather than modifying properties on the old -/// `RecordValue` because the old `RecordValue` may be shared with other -/// `Environment`s, and we don't want changes to properties to be visible there. -/// /// Requirements: /// /// `Src` and `Dst` must have the same canonical unqualified type. @@ -49,9 +43,7 @@ void copyRecord(RecordStorageLocation &Src, RecordStorageLocation &Dst, /// /// This performs a deep comparison, i.e. it compares every field (including /// synthetic fields) and recurses on fields of record type. Fields of reference -/// type compare equal if they refer to the same storage location. If -/// `RecordValue`s are associated with `Loc1` and Loc2`, it also compares the -/// properties on those `RecordValue`s. +/// type compare equal if they refer to the same storage location. /// /// Note on how to interpret the result: /// - If this returns true, the records are guaranteed to be equal at runtime. diff --git a/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/Value.h b/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/Value.h index e6c68e5b4e93..be1bf9324c87 100644 --- a/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/Value.h +++ b/contrib/llvm-project/clang/include/clang/Analysis/FlowSensitive/Value.h @@ -63,7 +63,11 @@ public: /// Assigns `Val` as the value of the synthetic property with the given /// `Name`. + /// + /// Properties may not be set on `RecordValue`s; use synthetic fields instead + /// (for details, see documentation for `RecordStorageLocation`). void setProperty(llvm::StringRef Name, Value &Val) { + assert(getKind() != Kind::Record); Properties.insert_or_assign(Name, &Val); } @@ -184,33 +188,23 @@ private: /// In C++, prvalues of class type serve only a limited purpose: They can only /// be used to initialize a result object. It is not possible to access member /// variables or call member functions on a prvalue of class type. -/// Correspondingly, `RecordValue` also serves only two limited purposes: -/// - It conveys a prvalue of class type from the place where the object is -/// constructed to the result object that it initializes. +/// Correspondingly, `RecordValue` also serves only a limited purpose: It +/// conveys a prvalue of class type from the place where the object is +/// constructed to the result object that it initializes. /// -/// When creating a prvalue of class type, we already need a storage location -/// for `this`, even though prvalues are otherwise not associated with storage -/// locations. `RecordValue` is therefore essentially a wrapper for a storage -/// location, which is then used to set the storage location for the result -/// object when we process the AST node for that result object. +/// When creating a prvalue of class type, we already need a storage location +/// for `this`, even though prvalues are otherwise not associated with storage +/// locations. `RecordValue` is therefore essentially a wrapper for a storage +/// location, which is then used to set the storage location for the result +/// object when we process the AST node for that result object. /// -/// For example: -/// MyStruct S = MyStruct(3); +/// For example: +/// MyStruct S = MyStruct(3); /// -/// In this example, `MyStruct(3) is a prvalue, which is modeled as a -/// `RecordValue` that wraps a `RecordStorageLocation`. This -// `RecordStorageLocation` is then used as the storage location for `S`. +/// In this example, `MyStruct(3) is a prvalue, which is modeled as a +/// `RecordValue` that wraps a `RecordStorageLocation`. This +/// `RecordStorageLocation` is then used as the storage location for `S`. /// -/// - It allows properties to be associated with an object of class type. -/// Note that when doing so, you should avoid mutating the properties of an -/// existing `RecordValue` in place, as these changes would be visible to -/// other `Environment`s that share the same `RecordValue`. Instead, associate -/// a new `RecordValue` with the `RecordStorageLocation` and set the -/// properties on this new `RecordValue`. (See also `refreshRecordValue()` in -/// DataflowEnvironment.h, which makes this easy.) -/// Note also that this implies that it is common for the same -/// `RecordStorageLocation` to be associated with different `RecordValue`s -/// in different environments. /// Over time, we may eliminate `RecordValue` entirely. See also the discussion /// here: https://reviews.llvm.org/D155204#inline-1503204 class RecordValue final : public Value { diff --git a/contrib/llvm-project/clang/include/clang/Basic/Attr.td b/contrib/llvm-project/clang/include/clang/Basic/Attr.td index 2b57058d3f1c..db17211747b1 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/Attr.td +++ b/contrib/llvm-project/clang/include/clang/Basic/Attr.td @@ -4331,24 +4331,6 @@ def AvailableOnlyInDefaultEvalMethod : InheritableAttr { let Documentation = [Undocumented]; } -def CountedBy : InheritableAttr { - let Spellings = [Clang<"counted_by">]; - let Subjects = SubjectList<[Field]>; - let Args = [IdentifierArgument<"CountedByField">]; - let Documentation = [CountedByDocs]; - let LangOpts = [COnly]; - // FIXME: This is ugly. Let using a DeclArgument would be nice, but a Decl - // isn't yet available due to the fact that we're still parsing the - // structure. Maybe that code could be changed sometime in the future. - code AdditionalMembers = [{ - private: - SourceRange CountedByFieldLoc; - public: - SourceRange getCountedByFieldLoc() const { return CountedByFieldLoc; } - void setCountedByFieldLoc(SourceRange Loc) { CountedByFieldLoc = Loc; } - }]; -} - def PreferredType: InheritableAttr { let Spellings = [Clang<"preferred_type">]; let Subjects = SubjectList<[BitField], ErrorDiag>; diff --git a/contrib/llvm-project/clang/include/clang/Basic/AttrDocs.td b/contrib/llvm-project/clang/include/clang/Basic/AttrDocs.td index 90041fa8dbb3..98a7ecc7fd7d 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/AttrDocs.td +++ b/contrib/llvm-project/clang/include/clang/Basic/AttrDocs.td @@ -7500,72 +7500,6 @@ attribute, they default to the value ``65535``. }]; } -def CountedByDocs : Documentation { - let Category = DocCatField; - let Content = [{ -Clang supports the ``counted_by`` attribute on the flexible array member of a -structure in C. The argument for the attribute is the name of a field member in -the same structure holding the count of elements in the flexible array. This -information can be used to improve the results of the array bound sanitizer and -the ``__builtin_dynamic_object_size`` builtin. - -For example, the following code: - -.. code-block:: c - - struct bar; - - struct foo { - size_t count; - char other; - struct bar *array[] __attribute__((counted_by(count))); - }; - -specifies that the flexible array member ``array`` has the number of elements -allocated for it stored in ``count``. This establishes a relationship between -``array`` and ``count``. Specifically, ``p->array`` must have at least -``p->count`` number of elements available. It's the user's responsibility to -ensure that this relationship is maintained through changes to the structure. - -In the following example, the allocated array erroneously has fewer elements -than what's specified by ``p->count``. This would result in an out-of-bounds -access not being detected. - -.. code-block:: c - - #define SIZE_INCR 42 - - struct foo *p; - - void foo_alloc(size_t count) { - p = malloc(MAX(sizeof(struct foo), - offsetof(struct foo, array[0]) + count * sizeof(struct bar *))); - p->count = count + SIZE_INCR; - } - -The next example updates ``p->count``, breaking the relationship requirement -that ``p->array`` must have at least ``p->count`` number of elements available: - -.. code-block:: c - - #define SIZE_INCR 42 - - struct foo *p; - - void foo_alloc(size_t count) { - p = malloc(MAX(sizeof(struct foo), - offsetof(struct foo, array[0]) + count * sizeof(struct bar *))); - p->count = count; - } - - void use_foo(int index) { - p->count += SIZE_INCR + 1; /* 'count' is now larger than the number of elements of 'array'. */ - p->array[index] = 0; /* the sanitizer can't properly check if this is an out-of-bounds access. */ - } - - }]; -} - def CoroOnlyDestroyWhenCompleteDocs : Documentation { let Category = DocCatDecl; let Content = [{ diff --git a/contrib/llvm-project/clang/include/clang/Basic/DiagnosticGroups.td b/contrib/llvm-project/clang/include/clang/Basic/DiagnosticGroups.td index 80b5680b94f6..6765721ae700 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/DiagnosticGroups.td +++ b/contrib/llvm-project/clang/include/clang/Basic/DiagnosticGroups.td @@ -348,7 +348,8 @@ def CXX98CompatPedantic : DiagGroup<"c++98-compat-pedantic", CXXPre20CompatPedantic, CXXPre23CompatPedantic]>; -def CXX11Narrowing : DiagGroup<"c++11-narrowing">; +def CXX11NarrowingConstReference : DiagGroup<"c++11-narrowing-const-reference">; +def CXX11Narrowing : DiagGroup<"c++11-narrowing", [CXX11NarrowingConstReference]>; def CXX11WarnInconsistentOverrideDestructor : DiagGroup<"inconsistent-missing-destructor-override">; @@ -1488,4 +1489,5 @@ def DXILValidation : DiagGroup<"dxil-validation">; def ReadOnlyPlacementChecks : DiagGroup<"read-only-types">; // Warnings and fixes to support the "safe buffers" programming model. -def UnsafeBufferUsage : DiagGroup<"unsafe-buffer-usage">; +def UnsafeBufferUsageInContainer : DiagGroup<"unsafe-buffer-usage-in-container">; +def UnsafeBufferUsage : DiagGroup<"unsafe-buffer-usage", [UnsafeBufferUsageInContainer]>; diff --git a/contrib/llvm-project/clang/include/clang/Basic/DiagnosticParseKinds.td b/contrib/llvm-project/clang/include/clang/Basic/DiagnosticParseKinds.td index 6150fc36430a..e4b1069cde18 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/contrib/llvm-project/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -1358,11 +1358,9 @@ def err_acc_unexpected_directive def warn_pragma_acc_unimplemented : Warning<"OpenACC directives not yet implemented, pragma ignored">, InGroup<SourceUsesOpenACC>; -def warn_pragma_acc_unimplemented_clause_parsing - : Warning<"OpenACC clause parsing not yet implemented">, - InGroup<SourceUsesOpenACC>; def err_acc_invalid_directive : Error<"invalid OpenACC directive %select{%1|'%1 %2'}0">; +def err_acc_invalid_clause : Error<"invalid OpenACC clause %0">; def err_acc_missing_directive : Error<"expected OpenACC directive">; def err_acc_invalid_open_paren : Error<"expected clause-list or newline in OpenACC directive">; diff --git a/contrib/llvm-project/clang/include/clang/Basic/DiagnosticSemaKinds.td b/contrib/llvm-project/clang/include/clang/Basic/DiagnosticSemaKinds.td index f572b6f861fd..cb68ded1f38c 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/contrib/llvm-project/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -3155,6 +3155,9 @@ def err_attribute_arm_feature_sve_bits_unsupported : Error< def warn_attribute_arm_sm_incompat_builtin : Warning< "builtin call has undefined behaviour when called from a %0 function">, InGroup<DiagGroup<"undefined-arm-streaming">>; +def warn_attribute_arm_za_builtin_no_za_state : Warning< + "builtin call is not valid when calling from a function without active ZA state">, + InGroup<DiagGroup<"undefined-arm-za">>; def err_sve_vector_in_non_sve_target : Error< "SVE vector type %0 cannot be used in a target without sve">; def err_attribute_riscv_rvv_bits_unsupported : Error< @@ -6155,12 +6158,24 @@ def err_illegal_initializer_type : Error<"illegal initializer type %0">; def ext_init_list_type_narrowing : ExtWarn< "type %0 cannot be narrowed to %1 in initializer list">, InGroup<CXX11Narrowing>, DefaultError, SFINAEFailure; +// *_narrowing_const_reference diagnostics have the same messages, but are +// controlled by -Wc++11-narrowing-const-reference for narrowing involving a +// const reference. +def ext_init_list_type_narrowing_const_reference : ExtWarn< + "type %0 cannot be narrowed to %1 in initializer list">, + InGroup<CXX11NarrowingConstReference>, DefaultError, SFINAEFailure; def ext_init_list_variable_narrowing : ExtWarn< "non-constant-expression cannot be narrowed from type %0 to %1 in " "initializer list">, InGroup<CXX11Narrowing>, DefaultError, SFINAEFailure; +def ext_init_list_variable_narrowing_const_reference : ExtWarn< + "non-constant-expression cannot be narrowed from type %0 to %1 in " + "initializer list">, InGroup<CXX11NarrowingConstReference>, DefaultError, SFINAEFailure; def ext_init_list_constant_narrowing : ExtWarn< "constant expression evaluates to %0 which cannot be narrowed to type %1">, InGroup<CXX11Narrowing>, DefaultError, SFINAEFailure; +def ext_init_list_constant_narrowing_const_reference : ExtWarn< + "constant expression evaluates to %0 which cannot be narrowed to type %1">, + InGroup<CXX11NarrowingConstReference>, DefaultError, SFINAEFailure; def warn_init_list_type_narrowing : Warning< "type %0 cannot be narrowed to %1 in initializer list in C++11">, InGroup<CXX11Narrowing>, DefaultIgnore; @@ -6426,17 +6441,6 @@ def warn_superclass_variable_sized_type_not_at_end : Warning< "field %0 can overwrite instance variable %1 with variable sized type %2" " in superclass %3">, InGroup<ObjCFlexibleArray>; -def err_counted_by_attr_not_on_flexible_array_member : Error< - "'counted_by' only applies to C99 flexible array members">; -def err_counted_by_attr_refers_to_flexible_array : Error< - "'counted_by' cannot refer to the flexible array %0">; -def err_counted_by_must_be_in_structure : Error< - "field %0 in 'counted_by' not inside structure">; -def err_flexible_array_counted_by_attr_field_not_integer : Error< - "field %0 in 'counted_by' must be a non-boolean integer type">; -def note_flexible_array_counted_by_attr_field : Note< - "field %0 declared here">; - let CategoryName = "ARC Semantic Issue" in { // ARC-mode diagnostics. @@ -11368,6 +11372,8 @@ def err_openmp_vla_in_task_untied : Error< def warn_omp_unterminated_declare_target : Warning< "expected '#pragma omp end declare target' at end of file to match '#pragma omp %0'">, InGroup<SourceUsesOpenMP>; +def err_ompx_bare_no_grid : Error< + "'ompx_bare' clauses requires explicit grid size via 'num_teams' and 'thread_limit' clauses">; } // end of OpenMP category let CategoryName = "Related Result Type Issue" in { diff --git a/contrib/llvm-project/clang/include/clang/Basic/LangOptions.def b/contrib/llvm-project/clang/include/clang/Basic/LangOptions.def index 152d9f65f86d..21abc346cf17 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/LangOptions.def +++ b/contrib/llvm-project/clang/include/clang/Basic/LangOptions.def @@ -456,6 +456,7 @@ ENUM_LANGOPT(SignReturnAddressScope, SignReturnAddressScopeKind, 2, SignReturnAd ENUM_LANGOPT(SignReturnAddressKey, SignReturnAddressKeyKind, 1, SignReturnAddressKeyKind::AKey, "Key used for return address signing") LANGOPT(BranchTargetEnforcement, 1, 0, "Branch-target enforcement enabled") +LANGOPT(BranchProtectionPAuthLR, 1, 0, "Use PC as a diversifier using PAuthLR NOP instructions.") LANGOPT(SpeculativeLoadHardening, 1, 0, "Speculative load hardening enabled") diff --git a/contrib/llvm-project/clang/include/clang/Basic/OpenACCKinds.h b/contrib/llvm-project/clang/include/clang/Basic/OpenACCKinds.h index 62c0a4c1a9de..3117d584d347 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/OpenACCKinds.h +++ b/contrib/llvm-project/clang/include/clang/Basic/OpenACCKinds.h @@ -69,6 +69,30 @@ enum class OpenACCAtomicKind { Capture, Invalid, }; + +/// Represents the kind of an OpenACC clause. +enum class OpenACCClauseKind { + // 'finalize' clause, allowed on 'exit data' directive. + Finalize, + // 'if_present' clause, allowed on 'host_data' and 'update' directives. + IfPresent, + // 'seq' clause, allowed on 'loop' and 'routine' directives. + Seq, + // 'independent' clause, allowed on 'loop' directives. + Independent, + // 'auto' clause, allowed on 'loop' directives. + Auto, + // 'worker' clause, allowed on 'loop' and 'routine' directives. + Worker, + // 'vector' clause, allowed on 'loop' and 'routine' directives. Takes no + // arguments for 'routine', so the 'loop' version is not yet implemented + // completely. + Vector, + // 'nohost' clause, allowed on 'routine' directives. + NoHost, + // Represents an invalid clause, for the purposes of parsing. + Invalid, +}; } // namespace clang #endif // LLVM_CLANG_BASIC_OPENACCKINDS_H diff --git a/contrib/llvm-project/clang/include/clang/Basic/PlistSupport.h b/contrib/llvm-project/clang/include/clang/Basic/PlistSupport.h index 557462a5b90d..d52d196019cf 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/PlistSupport.h +++ b/contrib/llvm-project/clang/include/clang/Basic/PlistSupport.h @@ -77,8 +77,7 @@ inline raw_ostream &EmitInteger(raw_ostream &o, int64_t value) { inline raw_ostream &EmitString(raw_ostream &o, StringRef s) { o << "<string>"; - for (StringRef::const_iterator I = s.begin(), E = s.end(); I != E; ++I) { - char c = *I; + for (char c : s) { switch (c) { default: o << c; diff --git a/contrib/llvm-project/clang/include/clang/Basic/TargetInfo.h b/contrib/llvm-project/clang/include/clang/Basic/TargetInfo.h index aa0f5023104a..ac3c324c6c29 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/TargetInfo.h +++ b/contrib/llvm-project/clang/include/clang/Basic/TargetInfo.h @@ -1372,6 +1372,7 @@ public: LangOptions::SignReturnAddressKeyKind SignKey = LangOptions::SignReturnAddressKeyKind::AKey; bool BranchTargetEnforcement = false; + bool BranchProtectionPAuthLR = false; }; /// Determine if the Architecture in this TargetInfo supports branch diff --git a/contrib/llvm-project/clang/include/clang/Basic/Version.h b/contrib/llvm-project/clang/include/clang/Basic/Version.h index 2881d8db954e..8e4e6928fded 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/Version.h +++ b/contrib/llvm-project/clang/include/clang/Basic/Version.h @@ -40,6 +40,9 @@ namespace clang { /// string as getClangRevision. std::string getLLVMRevision(); + /// Retrieves the Clang vendor tag. + std::string getClangVendor(); + /// Retrieves the full repository version that is an amalgamation of /// the information in getClangRepositoryPath() and getClangRevision(). std::string getClangFullRepositoryVersion(); diff --git a/contrib/llvm-project/clang/include/clang/Basic/arm_sme.td b/contrib/llvm-project/clang/include/clang/Basic/arm_sme.td index fcff6fe35b7b..aac3bd486de9 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/arm_sme.td +++ b/contrib/llvm-project/clang/include/clang/Basic/arm_sme.td @@ -299,6 +299,44 @@ multiclass ZAAddSub<string n_suffix> { defm SVADD : ZAAddSub<"add">; defm SVSUB : ZAAddSub<"sub">; +// SME2 - MOVA + +// +// Single, 2 and 4 vector-group read/write intrinsics. +// + +multiclass ZAWrite_VG<string n, string t, string i, list<ImmCheck> checks> { + def NAME # _VG2_H : Inst<"svwrite_hor_" # n # "[_{d}]_vg2", "vim2", t, MergeNone, i # "_hor_vg2", [IsSharedZA, IsStreaming], checks>; + def NAME # _VG2_V : Inst<"svwrite_ver_" # n # "[_{d}]_vg2", "vim2", t, MergeNone, i # "_ver_vg2", [IsSharedZA, IsStreaming], checks>; + def NAME # _VG4_H : Inst<"svwrite_hor_" # n # "[_{d}]_vg4", "vim4", t, MergeNone, i # "_hor_vg4", [IsSharedZA, IsStreaming], checks>; + def NAME # _VG4_V : Inst<"svwrite_ver_" # n # "[_{d}]_vg4", "vim4", t, MergeNone, i # "_ver_vg4", [IsSharedZA, IsStreaming], checks>; + def NAME # _VG1x2 : Inst<"svwrite_" # n # "[_{d}]_vg1x2", "vm2", t, MergeNone, i # "_vg1x2", [IsSharedZA, IsStreaming], []>; + def NAME # _VG1x4 : Inst<"svwrite_" # n # "[_{d}]_vg1x4", "vm4", t, MergeNone, i # "_vg1x4", [IsSharedZA, IsStreaming], []>; +} + +let TargetGuard = "sme2" in { + defm SVWRITE_ZA8 : ZAWrite_VG<"za8", "cUc", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_0>]>; + defm SVWRITE_ZA16 : ZAWrite_VG<"za16", "sUshb", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_1>]>; + defm SVWRITE_ZA32 : ZAWrite_VG<"za32", "iUif", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_3>]>; + defm SVWRITE_ZA64 : ZAWrite_VG<"za64", "lUld", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_7>]>; +} + +multiclass ZARead_VG<string n, string t, string i, list<ImmCheck> checks> { + def NAME # _VG2_H : Inst<"svread_hor_" # n # "_{d}_vg2", "2im", t, MergeNone, i # "_hor_vg2", [IsSharedZA, IsPreservesZA, IsStreaming], checks>; + def NAME # _VG2_V : Inst<"svread_ver_" # n # "_{d}_vg2", "2im", t, MergeNone, i # "_ver_vg2", [IsSharedZA, IsPreservesZA, IsStreaming], checks>; + def NAME # _VG4_H : Inst<"svread_hor_" # n # "_{d}_vg4", "4im", t, MergeNone, i # "_hor_vg4", [IsSharedZA, IsPreservesZA, IsStreaming], checks>; + def NAME # _VG4_V : Inst<"svread_ver_" # n # "_{d}_vg4", "4im", t, MergeNone, i # "_ver_vg4", [IsSharedZA, IsPreservesZA, IsStreaming], checks>; + def NAME # _VG1x2 : Inst<"svread_" # n # "_{d}_vg1x2", "2m", t, MergeNone, i # "_vg1x2", [IsSharedZA, IsPreservesZA, IsStreaming], []>; + def NAME # _VG1x4 : Inst<"svread_" # n # "_{d}_vg1x4", "4m", t, MergeNone, i # "_vg1x4", [IsSharedZA, IsPreservesZA, IsStreaming], []>; +} + +let TargetGuard = "sme2" in { + defm SVREAD_ZA8 : ZARead_VG<"za8", "cUc", "aarch64_sme_read", [ImmCheck<0, ImmCheck0_0>]>; + defm SVREAD_ZA16 : ZARead_VG<"za16", "sUshb", "aarch64_sme_read", [ImmCheck<0, ImmCheck0_1>]>; + defm SVREAD_ZA32 : ZARead_VG<"za32", "iUif", "aarch64_sme_read", [ImmCheck<0, ImmCheck0_3>]>; + defm SVREAD_ZA64 : ZARead_VG<"za64", "lUld", "aarch64_sme_read", [ImmCheck<0, ImmCheck0_7>]>; +} + // // Outer product and accumulate/subtract // @@ -313,6 +351,285 @@ let TargetGuard = "sme2" in { def SVBMOPA : Inst<"svbmopa_za32[_{d}]_m", "viPPdd", "iUi", MergeNone, "aarch64_sme_bmopa_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>; def SVBMOPS : Inst<"svbmops_za32[_{d}]_m", "viPPdd", "iUi", MergeNone, "aarch64_sme_bmops_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>; + + // VERTICAL DOT-PRODUCT + def SVVDOT_LANE_ZA32_VG1x2_S : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "s", MergeNone, "aarch64_sme_svdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVVDOT_LANE_ZA32_VG1x4_S : Inst<"svvdot_lane_za32[_{d}]_vg1x4", "vm4di", "c", MergeNone, "aarch64_sme_svdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVVDOT_LANE_ZA32_VG1x2_U : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "Us", MergeNone, "aarch64_sme_uvdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVVDOT_LANE_ZA32_VG1x4_U : Inst<"svvdot_lane_za32[_{d}]_vg1x4", "vm4di", "Uc", MergeNone, "aarch64_sme_uvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVVDOT_LANE_ZA32_VG1x2_F : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "hb", MergeNone, "aarch64_sme_fvdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVSUVDOT_LANE_ZA32_VG1x4 : Inst<"svsuvdot_lane_za32[_{d}]_vg1x4", "vm4di", "c", MergeNone, "aarch64_sme_suvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVUSVDOT_LANE_ZA32_VG1x4 : Inst<"svusvdot_lane_za32[_{d}]_vg1x4", "vm4di", "Uc", MergeNone, "aarch64_sme_usvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + + // Multi-vector signed & unsigned integer dot-product + def SVDOT_MULTI_ZA32_VG1x2_S : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "cs", MergeNone, "aarch64_sme_sdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA32_VG1x4_S : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "cs", MergeNone, "aarch64_sme_sdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA32_VG1x2_U : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "UcUs", MergeNone, "aarch64_sme_udot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA32_VG1x4_U : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "UcUs", MergeNone, "aarch64_sme_udot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x2_S : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "cs", MergeNone, "aarch64_sme_sdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x4_S : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "cs", MergeNone, "aarch64_sme_sdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x2_U : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "UcUs", MergeNone, "aarch64_sme_udot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x4_U : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "UcUs", MergeNone, "aarch64_sme_udot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_LANE_ZA32_VG1x2_S : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "cs", MergeNone, "aarch64_sme_sdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVDOT_LANE_ZA32_VG1x4_S : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "cs", MergeNone, "aarch64_sme_sdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVDOT_LANE_ZA32_VG1x2_U : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "UcUs", MergeNone, "aarch64_sme_udot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVDOT_LANE_ZA32_VG1x4_U : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "UcUs", MergeNone, "aarch64_sme_udot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + + def SVUSDOT_SINGLE_ZA32_VG1x2 : Inst<"svusdot[_single]_za32[_{d}]_vg1x2", "vm2.dx", "Uc", MergeNone, "aarch64_sme_usdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVUSDOT_SINGLE_ZA32_VG1x4 : Inst<"svusdot[_single]_za32[_{d}]_vg1x4", "vm4.dx", "Uc", MergeNone, "aarch64_sme_usdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVUSDOT_MULTI_ZA32_VG1x2 : Inst<"svusdot_za32[_{d}]_vg1x2", "vm2.d2.x", "Uc", MergeNone, "aarch64_sme_usdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVUSDOT_MULTI_ZA32_VG1x4 : Inst<"svusdot_za32[_{d}]_vg1x4", "vm4.d4.x", "Uc", MergeNone, "aarch64_sme_usdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVUSDOT_LANE_ZA32_VG1x2 : Inst<"svusdot_lane_za32[_{d}]_vg1x2", "vm2.dxi", "Uc", MergeNone, "aarch64_sme_usdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVUSDOT_LANE_ZA32_VG1x4 : Inst<"svusdot_lane_za32[_{d}]_vg1x4", "vm4.dxi", "Uc", MergeNone, "aarch64_sme_usdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + + def SVSUDOT_SINGLE_ZA32_VG1x2 : Inst<"svsudot[_single]_za32[_{d}]_vg1x2", "vm2.du", "c", MergeNone, "aarch64_sme_sudot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVSUDOT_SINGLE_ZA32_VG1x4 : Inst<"svsudot[_single]_za32[_{d}]_vg1x4", "vm4.du", "c", MergeNone, "aarch64_sme_sudot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + + // Multi-multi sudot builtins are mapped to usdot, with zn & zm operands swapped + def SVSUDOT_MULTI_ZA32_VG1x2 : Inst<"svsudot_za32[_{d}]_vg1x2", "vm2.d2.u", "c", MergeNone, "aarch64_sme_usdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVSUDOT_MULTI_ZA32_VG1x4 : Inst<"svsudot_za32[_{d}]_vg1x4", "vm4.d4.u", "c", MergeNone, "aarch64_sme_usdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVSUDOT_LANE_ZA32_VG1x2 : Inst<"svsudot_lane_za32[_{d}]_vg1x2", "vm2.dui", "c", MergeNone, "aarch64_sme_sudot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVSUDOT_LANE_ZA32_VG1x4 : Inst<"svsudot_lane_za32[_{d}]_vg1x4", "vm4.dui", "c", MergeNone, "aarch64_sme_sudot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + + // Multi-vector half-precision/BFloat16 floating-point dot-product + def SVDOT_MULTI_ZA32_VG1x2_F16 : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "bh", MergeNone, "aarch64_sme_fdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA32_VG1x4_F16 : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "bh", MergeNone, "aarch64_sme_fdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x2_F16 : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "bh", MergeNone, "aarch64_sme_fdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x4_F16 : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "bh", MergeNone, "aarch64_sme_fdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_LANE_ZA32_VG1x2_F16 : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "bh", MergeNone, "aarch64_sme_fdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVDOT_LANE_ZA32_VG1x4_F16 : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "bh", MergeNone, "aarch64_sme_fdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; +} + +let TargetGuard = "sme2,sme-i16i64" in { + def SVVDOT_LANE_ZA64_VG1x4_S : Inst<"svvdot_lane_za64[_{d}]_vg1x4", "vm4di", "s", MergeNone, "aarch64_sme_svdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVVDOT_LANE_ZA64_VG1x4_U : Inst<"svvdot_lane_za64[_{d}]_vg1x4", "vm4di", "Us", MergeNone, "aarch64_sme_uvdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + + def SVDOT_MULTI_ZA64_VG1x2_S16 : Inst<"svdot_za64[_{d}]_vg1x2", "vm22", "s", MergeNone, "aarch64_sme_sdot_za64_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA64_VG1x4_S16 : Inst<"svdot_za64[_{d}]_vg1x4", "vm44", "s", MergeNone, "aarch64_sme_sdot_za64_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA64_VG1x2_U16 : Inst<"svdot_za64[_{d}]_vg1x2", "vm22", "Us", MergeNone, "aarch64_sme_udot_za64_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA64_VG1x4_U16 : Inst<"svdot_za64[_{d}]_vg1x4", "vm44", "Us", MergeNone, "aarch64_sme_udot_za64_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA64_VG1x2_S16 : Inst<"svdot[_single]_za64[_{d}]_vg1x2", "vm2d", "s", MergeNone, "aarch64_sme_sdot_single_za64_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA64_VG1x4_S16 : Inst<"svdot[_single]_za64[_{d}]_vg1x4", "vm4d", "s", MergeNone, "aarch64_sme_sdot_single_za64_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA64_VG1x2_U16 : Inst<"svdot[_single]_za64[_{d}]_vg1x2", "vm2d", "Us", MergeNone, "aarch64_sme_udot_single_za64_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA64_VG1x4_U16 : Inst<"svdot[_single]_za64[_{d}]_vg1x4", "vm4d", "Us", MergeNone, "aarch64_sme_udot_single_za64_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_LANE_ZA64_VG1x2_S16 : Inst<"svdot_lane_za64[_{d}]_vg1x2", "vm2di", "s", MergeNone, "aarch64_sme_sdot_lane_za64_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVDOT_LANE_ZA64_VG1x4_S16 : Inst<"svdot_lane_za64[_{d}]_vg1x4", "vm4di", "s", MergeNone, "aarch64_sme_sdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVDOT_LANE_ZA64_VG1x2_U16 : Inst<"svdot_lane_za64[_{d}]_vg1x2", "vm2di", "Us", MergeNone, "aarch64_sme_udot_lane_za64_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVDOT_LANE_ZA64_VG1x4_U16 : Inst<"svdot_lane_za64[_{d}]_vg1x4", "vm4di", "Us", MergeNone, "aarch64_sme_udot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; +} + +// FMLA/FMLS +let TargetGuard = "sme2" in { + def SVMLA_MULTI_VG1x2_F32 : Inst<"svmla_za32[_{d}]_vg1x2", "vm22", "f", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLA_MULTI_VG1x4_F32 : Inst<"svmla_za32[_{d}]_vg1x4", "vm44", "f", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVMLS_MULTI_VG1x2_F32 : Inst<"svmls_za32[_{d}]_vg1x2", "vm22", "f", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLS_MULTI_VG1x4_F32 : Inst<"svmls_za32[_{d}]_vg1x4", "vm44", "f", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVMLA_SINGLE_VG1x2_F32 : Inst<"svmla[_single]_za32[_{d}]_vg1x2", "vm2d", "f", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLA_SINGLE_VG1x4_F32 : Inst<"svmla[_single]_za32[_{d}]_vg1x4", "vm4d", "f", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVMLS_SINGLE_VG1x2_F32 : Inst<"svmls[_single]_za32[_{d}]_vg1x2", "vm2d", "f", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLS_SINGLE_VG1x4_F32 : Inst<"svmls[_single]_za32[_{d}]_vg1x4", "vm4d", "f", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVMLA_LANE_VG1x2_F32 : Inst<"svmla_lane_za32[_{d}]_vg1x2", "vm2di", "f", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVMLA_LANE_VG1x4_F32 : Inst<"svmla_lane_za32[_{d}]_vg1x4", "vm4di", "f", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVMLS_LANE_VG1x2_F32 : Inst<"svmls_lane_za32[_{d}]_vg1x2", "vm2di", "f", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVMLS_LANE_VG1x4_F32 : Inst<"svmls_lane_za32[_{d}]_vg1x4", "vm4di", "f", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; +} + +let TargetGuard = "sme2,sme-f64f64" in { + def SVMLA_MULTI_VG1x2_F64 : Inst<"svmla_za64[_{d}]_vg1x2", "vm22", "d", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLA_MULTI_VG1x4_F64 : Inst<"svmla_za64[_{d}]_vg1x4", "vm44", "d", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVMLS_MULTI_VG1x2_F64 : Inst<"svmls_za64[_{d}]_vg1x2", "vm22", "d", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLS_MULTI_VG1x4_F64 : Inst<"svmls_za64[_{d}]_vg1x4", "vm44", "d", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVMLA_SINGLE_VG1x2_F64 : Inst<"svmla[_single]_za64[_{d}]_vg1x2", "vm2d", "d", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLA_SINGLE_VG1x4_F64 : Inst<"svmla[_single]_za64[_{d}]_vg1x4", "vm4d", "d", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVMLS_SINGLE_VG1x2_F64 : Inst<"svmls[_single]_za64[_{d}]_vg1x2", "vm2d", "d", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLS_SINGLE_VG1x4_F64 : Inst<"svmls[_single]_za64[_{d}]_vg1x4", "vm4d", "d", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVMLA_LANE_VG1x2_F64 : Inst<"svmla_lane_za64[_{d}]_vg1x2", "vm2di", "d", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVMLA_LANE_VG1x4_F64 : Inst<"svmla_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVMLS_LANE_VG1x2_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x2", "vm2di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVMLS_LANE_VG1x4_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; +} + +// FMLAL/FMLSL/UMLAL/SMLAL +// SMLALL/UMLALL/USMLALL/SUMLALL +let TargetGuard = "sme2" in { + // MULTI MLAL + def SVMLAL_MULTI_VG2x2_F16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "bh", MergeNone, "aarch64_sme_fmlal_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x4_F16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "bh", MergeNone, "aarch64_sme_fmlal_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x2_S16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "s", MergeNone, "aarch64_sme_smlal_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x4_S16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "s", MergeNone, "aarch64_sme_smlal_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x2_U16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "Us", MergeNone, "aarch64_sme_umlal_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x4_U16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "Us", MergeNone, "aarch64_sme_umlal_vg2x4", [IsStreaming, IsSharedZA], []>; + + def SVMLAL_MULTI_VG4x2_S8 : Inst<"svmla_za32[_{d}]_vg4x2", "vm22", "c", MergeNone, "aarch64_sme_smla_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x2_U8 : Inst<"svmla_za32[_{d}]_vg4x2", "vm22", "Uc", MergeNone, "aarch64_sme_umla_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x4_S8 : Inst<"svmla_za32[_{d}]_vg4x4", "vm44", "c", MergeNone, "aarch64_sme_smla_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x4_U8 : Inst<"svmla_za32[_{d}]_vg4x4", "vm44", "Uc", MergeNone, "aarch64_sme_umla_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + + // MULTI MLSL + def SVMLSL_MULTI_VG2x2_F16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "bh", MergeNone, "aarch64_sme_fmlsl_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x4_F16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "bh", MergeNone, "aarch64_sme_fmlsl_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x2_S16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "s", MergeNone, "aarch64_sme_smlsl_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x4_S16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "s", MergeNone, "aarch64_sme_smlsl_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x2_U16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "Us", MergeNone, "aarch64_sme_umlsl_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x4_U16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "Us", MergeNone, "aarch64_sme_umlsl_vg2x4", [IsStreaming, IsSharedZA], []>; + + def SVMLSL_MULTI_VG4x2_S8 : Inst<"svmls_za32[_{d}]_vg4x2", "vm22", "c", MergeNone, "aarch64_sme_smls_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x2_U8 : Inst<"svmls_za32[_{d}]_vg4x2", "vm22", "Uc", MergeNone, "aarch64_sme_umls_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x4_S8 : Inst<"svmls_za32[_{d}]_vg4x4", "vm44", "c", MergeNone, "aarch64_sme_smls_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x4_U8 : Inst<"svmls_za32[_{d}]_vg4x4", "vm44", "Uc", MergeNone, "aarch64_sme_umls_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + + // SINGLE MLAL + def SVMLAL_SINGLE_VG2x1_F16 : Inst<"svmla_za32[_{d}]_vg2x1", "vmdd", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x2_F16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x4_F16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x1_S16 : Inst<"svmla_za32[_{d}]_vg2x1", "vmdd", "s", MergeNone, "aarch64_sme_smlal_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x2_S16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "s", MergeNone, "aarch64_sme_smlal_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x4_S16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "s", MergeNone, "aarch64_sme_smlal_single_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x1_U16 : Inst<"svmla_za32[_{d}]_vg2x1", "vmdd", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x2_U16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x4_U16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x4", [IsStreaming, IsSharedZA], []>; + + def SVMLAL_SINGLE_VG4x1_S8 : Inst<"svmla_za32[_{d}]_vg4x1", "vmdd", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x1_U8 : Inst<"svmla_za32[_{d}]_vg4x1", "vmdd", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x2_S8 : Inst<"svmla[_single]_za32[_{d}]_vg4x2", "vm2d", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x2_U8 : Inst<"svmla[_single]_za32[_{d}]_vg4x2", "vm2d", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x4_S8 : Inst<"svmla[_single]_za32[_{d}]_vg4x4", "vm4d", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x4_U8 : Inst<"svmla[_single]_za32[_{d}]_vg4x4", "vm4d", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // SINGLE MLSL + def SVMLSL_SINGLE_VG2x1_F16 : Inst<"svmls_za32[_{d}]_vg2x1", "vmdd", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x2_F16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x4_F16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x1_S16 : Inst<"svmls_za32[_{d}]_vg2x1", "vmdd", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x2_S16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x4_S16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x1_U16 : Inst<"svmls_za32[_{d}]_vg2x1", "vmdd", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x2_U16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x4_U16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>; + + def SVMLSL_SINGLE_VG4x1_S8 : Inst<"svmls_za32[_{d}]_vg4x1", "vmdd", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x1_U8 : Inst<"svmls_za32[_{d}]_vg4x1", "vmdd", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x2_S8 : Inst<"svmls[_single]_za32[_{d}]_vg4x2", "vm2d", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x2_U8 : Inst<"svmls[_single]_za32[_{d}]_vg4x2", "vm2d", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x4_S8 : Inst<"svmls[_single]_za32[_{d}]_vg4x4", "vm4d", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x4_U8 : Inst<"svmls[_single]_za32[_{d}]_vg4x4", "vm4d", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // INDEXED MLAL + def SVMLAL_LANE_VG2x1_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x2_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x4_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x1_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x2_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x4_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x1_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x2_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x4_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + + def SVMLAL_LANE_VG4x1_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x1", "vmddi", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x1_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x1", "vmddi", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x2_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x2", "vm2di", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x2_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x2", "vm2di", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x4_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x4", "vm4di", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x4_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x4", "vm4di", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + + // INDEXED MLSL + def SVMLSL_LANE_VG2x1_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x2_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x4_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x1_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x2_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x4_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x1_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x2_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x4_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + + def SVMLSL_LANE_VG4x1_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x1", "vmddi", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x1_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x1", "vmddi", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x2_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x2", "vm2di", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x2_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x2", "vm2di", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x4_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x4", "vm4di", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x4_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x4", "vm4di", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + + // SINGLE SUMLALL + // Single sumla maps to usmla, with zn & zm operands swapped + def SVSUMLALL_SINGLE_VG4x1 : Inst<"svsumla_za32[_{d}]_vg4x1", "vmdu", "c", MergeNone, "aarch64_sme_usmla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + + def SVSUMLALL_SINGLE_VG4x2 : Inst<"svsumla[_single]_za32[_{d}]_vg4x2", "vm2.du", "c", MergeNone, "aarch64_sme_sumla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVSUMLALL_SINGLE_VG4x4 : Inst<"svsumla[_single]_za32[_{d}]_vg4x4", "vm4.du", "c", MergeNone, "aarch64_sme_sumla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // Multi-multi sumla builtins are mapped to usmla, with zn & zm operands swapped + def SVSUMLALL_MULTI_VG4x2 : Inst<"svsumla_za32[_{d}]_vg4x2", "vm2.d2.u", "c", MergeNone, "aarch64_sme_usmla_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVSUMLALL_MULTI_VG4x4 : Inst<"svsumla_za32[_{d}]_vg4x4", "vm4.d4.u", "c", MergeNone, "aarch64_sme_usmla_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + + // INDEXED SUMLALL + def SVSUMLALL_LANE_VG4x1 : Inst<"svsumla_lane_za32[_{d}]_vg4x1", "vmdui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVSUMLALL_LANE_VG4x2 : Inst<"svsumla_lane_za32[_{d}]_vg4x2", "vm2ui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVSUMLALL_LANE_VG4x4 : Inst<"svsumla_lane_za32[_{d}]_vg4x4", "vm4ui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + + // SINGLE USMLALL + def SVUSMLALL_SINGLE_VG4x1 : Inst<"svusmla_za32[_{d}]_vg4x1", "vmdx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVUSMLALL_SINGLE_VG4x2 : Inst<"svusmla[_single]_za32[_{d}]_vg4x2", "vm2.dx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVUSMLALL_SINGLE_VG4x4 : Inst<"svusmla[_single]_za32[_{d}]_vg4x4", "vm4.dx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // MULTI USMLALL + def SVUSMLALL_MULTI_VG4x2 : Inst<"svusmla_za32[_{d}]_vg4x2", "vm2.d2.x", "Uc", MergeNone, "aarch64_sme_usmla_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVUSMLALL_MULTI_VG4x4 : Inst<"svusmla_za32[_{d}]_vg4x4", "vm4.d4.x", "Uc", MergeNone, "aarch64_sme_usmla_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + + // INDEXED USMLALL + def SVUSMLALL_LANE_VG4x1 : Inst<"svusmla_lane_za32[_{d}]_vg4x1", "vmdxi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVUSMLALL_LANE_VG4x2 : Inst<"svusmla_lane_za32[_{d}]_vg4x2", "vm2xi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVUSMLALL_LANE_VG4x4 : Inst<"svusmla_lane_za32[_{d}]_vg4x4", "vm4xi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; +} + +let TargetGuard = "sme2,sme-i16i64" in { + // MULTI MLAL + def SVMLAL_MULTI_VG4x2_S16 : Inst<"svmla_za64[_{d}]_vg4x2", "vm22", "s", MergeNone, "aarch64_sme_smla_za64_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x2_U16 : Inst<"svmla_za64[_{d}]_vg4x2", "vm22", "Us", MergeNone, "aarch64_sme_umla_za64_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x4_S16 : Inst<"svmla_za64[_{d}]_vg4x4", "vm44", "s", MergeNone, "aarch64_sme_smla_za64_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x4_U16 : Inst<"svmla_za64[_{d}]_vg4x4", "vm44", "Us", MergeNone, "aarch64_sme_umla_za64_vg4x4", [IsStreaming, IsSharedZA], []>; + + // MULTI MLSL + def SVMLSL_MULTI_VG4x2_S16 : Inst<"svmls_za64[_{d}]_vg4x2", "vm22", "s", MergeNone, "aarch64_sme_smls_za64_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x2_U16 : Inst<"svmls_za64[_{d}]_vg4x2", "vm22", "Us", MergeNone, "aarch64_sme_umls_za64_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x4_S16 : Inst<"svmls_za64[_{d}]_vg4x4", "vm44", "s", MergeNone, "aarch64_sme_smls_za64_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x4_U16 : Inst<"svmls_za64[_{d}]_vg4x4", "vm44", "Us", MergeNone, "aarch64_sme_umls_za64_vg4x4", [IsStreaming, IsSharedZA], []>; + + // SINGLE MLAL + def SVMLAL_SINGLE_VG4x1_S16 : Inst<"svmla_za64[_{d}]_vg4x1", "vmdd", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x1_U16 : Inst<"svmla_za64[_{d}]_vg4x1", "vmdd", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x2_S16 : Inst<"svmla[_single]_za64[_{d}]_vg4x2", "vm2d", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x2_U16 : Inst<"svmla[_single]_za64[_{d}]_vg4x2", "vm2d", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x4_S16 : Inst<"svmla[_single]_za64[_{d}]_vg4x4", "vm4d", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x4_U16 : Inst<"svmla[_single]_za64[_{d}]_vg4x4", "vm4d", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // SINGLE MLSL + def SVMLSL_SINGLE_VG4x1_S16 : Inst<"svmls_za64[_{d}]_vg4x1", "vmdd", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x1_U16 : Inst<"svmls_za64[_{d}]_vg4x1", "vmdd", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x2_S16 : Inst<"svmls[_single]_za64[_{d}]_vg4x2", "vm2d", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x2_U16 : Inst<"svmls[_single]_za64[_{d}]_vg4x2", "vm2d", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x4_S16 : Inst<"svmls[_single]_za64[_{d}]_vg4x4", "vm4d", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x4_U16 : Inst<"svmls[_single]_za64[_{d}]_vg4x4", "vm4d", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // INDEXED MLAL + def SVMLAL_LANE_VG4x1_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x1", "vmddi", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x1_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x1", "vmddi", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x2_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x2", "vm2di", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x2_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x2", "vm2di", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x4_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x4", "vm4di", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x4_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x4", "vm4di", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + + // INDEXED MLSL + def SVMLSL_LANE_VG4x1_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x1", "vmddi", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x1_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x1", "vmddi", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x2_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x2", "vm2di", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x2_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x2", "vm2di", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x4_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x4", "vm4di", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x4_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x4", "vm4di", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; } // diff --git a/contrib/llvm-project/clang/include/clang/Basic/arm_sve.td b/contrib/llvm-project/clang/include/clang/Basic/arm_sve.td index 55fd35c3b6c2..91f62c4c7633 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/arm_sve.td +++ b/contrib/llvm-project/clang/include/clang/Basic/arm_sve.td @@ -19,27 +19,27 @@ include "arm_sve_sme_incl.td" // Loads // Load one vector (scalar base) -def SVLD1 : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">; -def SVLD1SB : MInst<"svld1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ld1">; -def SVLD1UB : MInst<"svld1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1">; -def SVLD1SH : MInst<"svld1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ld1">; -def SVLD1UH : MInst<"svld1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1">; -def SVLD1SW : MInst<"svld1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1">; -def SVLD1UW : MInst<"svld1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1">; +def SVLD1 : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ld1">; +def SVLD1SB : MInst<"svld1sb_{d}", "dPS", "silUsUiUl", [IsLoad, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_ld1">; +def SVLD1UB : MInst<"svld1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_ld1">; +def SVLD1SH : MInst<"svld1sh_{d}", "dPT", "ilUiUl", [IsLoad, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_ld1">; +def SVLD1UH : MInst<"svld1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_ld1">; +def SVLD1SW : MInst<"svld1sw_{d}", "dPU", "lUl", [IsLoad, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_ld1">; +def SVLD1UW : MInst<"svld1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_ld1">; let TargetGuard = "sve,bf16" in { - def SVLD1_BF : MInst<"svld1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">; - def SVLD1_VNUM_BF : MInst<"svld1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">; + def SVLD1_BF : MInst<"svld1[_{2}]", "dPc", "b", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ld1">; + def SVLD1_VNUM_BF : MInst<"svld1_vnum[_{2}]", "dPcl", "b", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ld1">; } // Load one vector (scalar base, VL displacement) -def SVLD1_VNUM : MInst<"svld1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">; -def SVLD1SB_VNUM : MInst<"svld1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ld1">; -def SVLD1UB_VNUM : MInst<"svld1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1">; -def SVLD1SH_VNUM : MInst<"svld1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ld1">; -def SVLD1UH_VNUM : MInst<"svld1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1">; -def SVLD1SW_VNUM : MInst<"svld1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1">; -def SVLD1UW_VNUM : MInst<"svld1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1">; +def SVLD1_VNUM : MInst<"svld1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ld1">; +def SVLD1SB_VNUM : MInst<"svld1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_ld1">; +def SVLD1UB_VNUM : MInst<"svld1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_ld1">; +def SVLD1SH_VNUM : MInst<"svld1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_ld1">; +def SVLD1UH_VNUM : MInst<"svld1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_ld1">; +def SVLD1SW_VNUM : MInst<"svld1sw_vnum_{d}", "dPUl", "lUl", [IsLoad, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_ld1">; +def SVLD1UW_VNUM : MInst<"svld1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_ld1">; // Load one vector (vector base) def SVLD1_GATHER_BASES_U : MInst<"svld1_gather[_{2}base]_{d}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">; @@ -243,27 +243,27 @@ let TargetGuard = "sve,bf16" in { } // Load one vector, unextended load, non-temporal (scalar base) -def SVLDNT1 : MInst<"svldnt1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">; +def SVLDNT1 : MInst<"svldnt1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ldnt1">; // Load one vector, unextended load, non-temporal (scalar base, VL displacement) -def SVLDNT1_VNUM : MInst<"svldnt1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">; +def SVLDNT1_VNUM : MInst<"svldnt1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ldnt1">; let TargetGuard = "sve,bf16" in { - def SVLDNT1_BF : MInst<"svldnt1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">; - def SVLDNT1_VNUM_BF : MInst<"svldnt1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">; + def SVLDNT1_BF : MInst<"svldnt1[_{2}]", "dPc", "b", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ldnt1">; + def SVLDNT1_VNUM_BF : MInst<"svldnt1_vnum[_{2}]", "dPcl", "b", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ldnt1">; } // Load one quadword and replicate (scalar base) -def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq">; +def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1rq">; + def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1rq", [IsStreamingCompatible]>; } multiclass StructLoad<string name, string proto, string i> { - def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructLoad]>; + def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructLoad, IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def: SInst<name, proto, "b", MergeNone, i, [IsStructLoad]>; + def: SInst<name, proto, "b", MergeNone, i, [IsStructLoad, IsStreamingCompatible]>; } } @@ -286,16 +286,16 @@ let TargetGuard = "sve,f64mm,bf16" in { } let TargetGuard = "sve,bf16" in { - def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone]>; - def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone]>; - def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone]>; - def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone]>; - def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone]>; - def SVBFMLAL_N : SInst<"svbfmlalb[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone]>; - def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone]>; - def SVBFDOT_LANE : SInst<"svbfdot_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfdot_lane_v2", [IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>; - def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalb_lane_v2", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; - def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalt_lane_v2", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMLAL_N : SInst<"svbfmlalb[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFDOT_LANE : SInst<"svbfdot_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfdot_lane_v2", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_3>]>; + def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalb_lane_v2", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalt_lane_v2", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; } let TargetGuard = "sve2p1" in { @@ -334,26 +334,26 @@ let TargetGuard = "sve2p1" in { // Stores // Store one vector (scalar base) -def SVST1 : MInst<"svst1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_st1">; -def SVST1B_S : MInst<"svst1b[_{d}]", "vPAd", "sil", [IsStore], MemEltTyInt8, "aarch64_sve_st1">; -def SVST1B_U : MInst<"svst1b[_{d}]", "vPEd", "UsUiUl", [IsStore], MemEltTyInt8, "aarch64_sve_st1">; -def SVST1H_S : MInst<"svst1h[_{d}]", "vPBd", "il", [IsStore], MemEltTyInt16, "aarch64_sve_st1">; -def SVST1H_U : MInst<"svst1h[_{d}]", "vPFd", "UiUl", [IsStore], MemEltTyInt16, "aarch64_sve_st1">; -def SVST1W_S : MInst<"svst1w[_{d}]", "vPCd", "l", [IsStore], MemEltTyInt32, "aarch64_sve_st1">; -def SVST1W_U : MInst<"svst1w[_{d}]", "vPGd", "Ul", [IsStore], MemEltTyInt32, "aarch64_sve_st1">; +def SVST1 : MInst<"svst1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_st1">; +def SVST1B_S : MInst<"svst1b[_{d}]", "vPAd", "sil", [IsStore, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_st1">; +def SVST1B_U : MInst<"svst1b[_{d}]", "vPEd", "UsUiUl", [IsStore, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_st1">; +def SVST1H_S : MInst<"svst1h[_{d}]", "vPBd", "il", [IsStore, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_st1">; +def SVST1H_U : MInst<"svst1h[_{d}]", "vPFd", "UiUl", [IsStore, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_st1">; +def SVST1W_S : MInst<"svst1w[_{d}]", "vPCd", "l", [IsStore, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_st1">; +def SVST1W_U : MInst<"svst1w[_{d}]", "vPGd", "Ul", [IsStore, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_st1">; // Store one vector (scalar base, VL displacement) -def SVST1_VNUM : MInst<"svst1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_st1">; -def SVST1B_VNUM_S : MInst<"svst1b_vnum[_{d}]", "vPAld", "sil", [IsStore], MemEltTyInt8, "aarch64_sve_st1">; -def SVST1B_VNUM_U : MInst<"svst1b_vnum[_{d}]", "vPEld", "UsUiUl", [IsStore], MemEltTyInt8, "aarch64_sve_st1">; -def SVST1H_VNUM_S : MInst<"svst1h_vnum[_{d}]", "vPBld", "il", [IsStore], MemEltTyInt16, "aarch64_sve_st1">; -def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl", [IsStore], MemEltTyInt16, "aarch64_sve_st1">; -def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l", [IsStore], MemEltTyInt32, "aarch64_sve_st1">; -def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul", [IsStore], MemEltTyInt32, "aarch64_sve_st1">; +def SVST1_VNUM : MInst<"svst1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_st1">; +def SVST1B_VNUM_S : MInst<"svst1b_vnum[_{d}]", "vPAld", "sil", [IsStore, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_st1">; +def SVST1B_VNUM_U : MInst<"svst1b_vnum[_{d}]", "vPEld", "UsUiUl", [IsStore, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_st1">; +def SVST1H_VNUM_S : MInst<"svst1h_vnum[_{d}]", "vPBld", "il", [IsStore, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_st1">; +def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl", [IsStore, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_st1">; +def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l", [IsStore, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_st1">; +def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul", [IsStore, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_st1">; let TargetGuard = "sve,bf16" in { - def SVST1_BF : MInst<"svst1[_{d}]", "vPpd", "b", [IsStore], MemEltTyDefault, "aarch64_sve_st1">; - def SVST1_VNUM_BF : MInst<"svst1_vnum[_{d}]", "vPpld", "b", [IsStore], MemEltTyDefault, "aarch64_sve_st1">; + def SVST1_BF : MInst<"svst1[_{d}]", "vPpd", "b", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_st1">; + def SVST1_VNUM_BF : MInst<"svst1_vnum[_{d}]", "vPpld", "b", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_st1">; } // Store one vector (vector base) @@ -426,9 +426,9 @@ def SVST1H_SCATTER_INDEX_S : MInst<"svst1h_scatter[_{2}base]_index[_{d}]", "v def SVST1W_SCATTER_INDEX_S : MInst<"svst1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">; multiclass StructStore<string name, string proto, string i> { - def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructStore]>; + def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructStore, IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def: SInst<name, proto, "b", MergeNone, i, [IsStructStore]>; + def: SInst<name, proto, "b", MergeNone, i, [IsStructStore, IsStreamingCompatible]>; } } // Store N vectors into N-element structure (scalar base) @@ -442,14 +442,14 @@ defm SVST3_VNUM : StructStore<"svst3_vnum[_{d}]", "vPpl3", "aarch64_sve_st3">; defm SVST4_VNUM : StructStore<"svst4_vnum[_{d}]", "vPpl4", "aarch64_sve_st4">; // Store one vector, with no truncation, non-temporal (scalar base) -def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; +def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_stnt1">; // Store one vector, with no truncation, non-temporal (scalar base, VL displacement) -def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; +def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_stnt1">; let TargetGuard = "sve,bf16" in { - def SVSTNT1_BF : MInst<"svstnt1[_{d}]", "vPpd", "b", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; - def SVSTNT1_VNUM_BF : MInst<"svstnt1_vnum[_{d}]", "vPpld", "b", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; + def SVSTNT1_BF : MInst<"svstnt1[_{d}]", "vPpd", "b", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_stnt1">; + def SVSTNT1_VNUM_BF : MInst<"svstnt1_vnum[_{d}]", "vPpld", "b", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_stnt1">; } let TargetGuard = "sve2p1" in { @@ -488,16 +488,16 @@ let TargetGuard = "sve2p1" in { // Prefetches // Prefetch (Scalar base) -def SVPRFB : MInst<"svprfb", "vPQJ", "c", [IsPrefetch], MemEltTyInt8, "aarch64_sve_prf">; -def SVPRFH : MInst<"svprfh", "vPQJ", "s", [IsPrefetch], MemEltTyInt16, "aarch64_sve_prf">; -def SVPRFW : MInst<"svprfw", "vPQJ", "i", [IsPrefetch], MemEltTyInt32, "aarch64_sve_prf">; -def SVPRFD : MInst<"svprfd", "vPQJ", "l", [IsPrefetch], MemEltTyInt64, "aarch64_sve_prf">; +def SVPRFB : MInst<"svprfb", "vPQJ", "c", [IsPrefetch, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_prf">; +def SVPRFH : MInst<"svprfh", "vPQJ", "s", [IsPrefetch, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_prf">; +def SVPRFW : MInst<"svprfw", "vPQJ", "i", [IsPrefetch, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_prf">; +def SVPRFD : MInst<"svprfd", "vPQJ", "l", [IsPrefetch, IsStreamingCompatible], MemEltTyInt64, "aarch64_sve_prf">; // Prefetch (Scalar base, VL displacement) -def SVPRFB_VNUM : MInst<"svprfb_vnum", "vPQlJ", "c", [IsPrefetch], MemEltTyInt8, "aarch64_sve_prf">; -def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPQlJ", "s", [IsPrefetch], MemEltTyInt16, "aarch64_sve_prf">; -def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPQlJ", "i", [IsPrefetch], MemEltTyInt32, "aarch64_sve_prf">; -def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPQlJ", "l", [IsPrefetch], MemEltTyInt64, "aarch64_sve_prf">; +def SVPRFB_VNUM : MInst<"svprfb_vnum", "vPQlJ", "c", [IsPrefetch, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_prf">; +def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPQlJ", "s", [IsPrefetch, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_prf">; +def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPQlJ", "i", [IsPrefetch, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_prf">; +def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPQlJ", "l", [IsPrefetch, IsStreamingCompatible], MemEltTyInt64, "aarch64_sve_prf">; // Prefetch (Vector bases) def SVPRFB_GATHER_BASES : MInst<"svprfb_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_scalar_offset">; @@ -552,9 +552,9 @@ def SVDUPQ_32 : SInst<"svdupq[_n]_{d}", "dssss", "iUif", MergeNone>; def SVDUPQ_64 : SInst<"svdupq[_n]_{d}", "dss", "lUld", MergeNone>; multiclass svdup_base<string n, string p, MergeType mt, string i> { - def NAME : SInst<n, p, "csilUcUsUiUlhfd", mt, i>; + def NAME : SInst<n, p, "csilUcUsUiUlhfd", mt, i, [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def _BF16: SInst<n, p, "b", mt, i>; + def _BF16: SInst<n, p, "b", mt, i, [IsStreamingCompatible]>; } } @@ -563,14 +563,14 @@ defm SVDUP_M : svdup_base<"svdup[_n]_{d}", "ddPs", MergeOp1, "aarch64_sve_du defm SVDUP_X : svdup_base<"svdup[_n]_{d}", "dPs", MergeAnyExp, "aarch64_sve_dup">; defm SVDUP_Z : svdup_base<"svdup[_n]_{d}", "dPs", MergeZeroExp, "aarch64_sve_dup">; -def SVINDEX : SInst<"svindex_{d}", "dss", "csilUcUsUiUl", MergeNone, "aarch64_sve_index">; +def SVINDEX : SInst<"svindex_{d}", "dss", "csilUcUsUiUl", MergeNone, "aarch64_sve_index", [IsStreamingCompatible]>; // Integer arithmetic -multiclass SInstZPZ<string name, string types, string intrinsic, list<FlagType> flags=[]> { - def _M : SInst<name # "[_{d}]", "ddPd", types, MergeOp1, intrinsic, flags>; - def _X : SInst<name # "[_{d}]", "dPd", types, MergeAnyExp, intrinsic, flags>; - def _Z : SInst<name # "[_{d}]", "dPd", types, MergeZeroExp, intrinsic, flags>; +multiclass SInstZPZ<string name, string types, string intrinsic> { + def _M : SInst<name # "[_{d}]", "ddPd", types, MergeOp1, intrinsic, [IsStreamingCompatible]>; + def _X : SInst<name # "[_{d}]", "dPd", types, MergeAnyExp, intrinsic, [IsStreamingCompatible]>; + def _Z : SInst<name # "[_{d}]", "dPd", types, MergeZeroExp, intrinsic, [IsStreamingCompatible]>; } defm SVABS : SInstZPZ<"svabs", "csil", "aarch64_sve_abs">; @@ -579,13 +579,13 @@ defm SVNEG : SInstZPZ<"svneg", "csil", "aarch64_sve_neg">; //------------------------------------------------------------------------------ multiclass SInstZPZZ<string name, string types, string m_intrinsic, string x_intrinsic, list<FlagType> flags=[]> { - def _M : SInst<name # "[_{d}]", "dPdd", types, MergeOp1, m_intrinsic, flags>; - def _X : SInst<name # "[_{d}]", "dPdd", types, MergeAny, x_intrinsic, flags>; - def _Z : SInst<name # "[_{d}]", "dPdd", types, MergeZero, m_intrinsic, flags>; + def _M : SInst<name # "[_{d}]", "dPdd", types, MergeOp1, m_intrinsic, !listconcat(flags, [IsStreamingCompatible])>; + def _X : SInst<name # "[_{d}]", "dPdd", types, MergeAny, x_intrinsic, !listconcat(flags, [IsStreamingCompatible])>; + def _Z : SInst<name # "[_{d}]", "dPdd", types, MergeZero, m_intrinsic, !listconcat(flags, [IsStreamingCompatible])>; - def _N_M : SInst<name # "[_n_{d}]", "dPda", types, MergeOp1, m_intrinsic, flags>; - def _N_X : SInst<name # "[_n_{d}]", "dPda", types, MergeAny, x_intrinsic, flags>; - def _N_Z : SInst<name # "[_n_{d}]", "dPda", types, MergeZero, m_intrinsic, flags>; + def _N_M : SInst<name # "[_n_{d}]", "dPda", types, MergeOp1, m_intrinsic, !listconcat(flags, [IsStreamingCompatible])>; + def _N_X : SInst<name # "[_n_{d}]", "dPda", types, MergeAny, x_intrinsic, !listconcat(flags, [IsStreamingCompatible])>; + def _N_Z : SInst<name # "[_n_{d}]", "dPda", types, MergeZero, m_intrinsic, !listconcat(flags, [IsStreamingCompatible])>; } defm SVABD_S : SInstZPZZ<"svabd", "csil", "aarch64_sve_sabd", "aarch64_sve_sabd_u">; @@ -617,26 +617,26 @@ multiclass SInstZPZZZ<string name, string types, string m_intrinsic, string x_in def _N_Z : SInst<name # "[_n_{d}]", "dPdda", types, MergeZero, m_intrinsic, flags>; } -defm SVMAD : SInstZPZZZ<"svmad", "csilUcUsUiUl", "aarch64_sve_mad", "aarch64_sve_mla_u", [ReverseMergeAnyAccOp]>; -defm SVMLA : SInstZPZZZ<"svmla", "csilUcUsUiUl", "aarch64_sve_mla", "aarch64_sve_mla_u">; -defm SVMLS : SInstZPZZZ<"svmls", "csilUcUsUiUl", "aarch64_sve_mls", "aarch64_sve_mls_u">; -defm SVMSB : SInstZPZZZ<"svmsb", "csilUcUsUiUl", "aarch64_sve_msb", "aarch64_sve_mls_u", [ReverseMergeAnyAccOp]>; +defm SVMAD : SInstZPZZZ<"svmad", "csilUcUsUiUl", "aarch64_sve_mad", "aarch64_sve_mla_u", [ReverseMergeAnyAccOp, IsStreamingCompatible]>; +defm SVMLA : SInstZPZZZ<"svmla", "csilUcUsUiUl", "aarch64_sve_mla", "aarch64_sve_mla_u", [IsStreamingCompatible]>; +defm SVMLS : SInstZPZZZ<"svmls", "csilUcUsUiUl", "aarch64_sve_mls", "aarch64_sve_mls_u", [IsStreamingCompatible]>; +defm SVMSB : SInstZPZZZ<"svmsb", "csilUcUsUiUl", "aarch64_sve_msb", "aarch64_sve_mls_u", [ReverseMergeAnyAccOp, IsStreamingCompatible]>; //------------------------------------------------------------------------------ -def SVDOT_S : SInst<"svdot[_{0}]", "ddqq", "il", MergeNone, "aarch64_sve_sdot">; -def SVDOT_U : SInst<"svdot[_{0}]", "ddqq", "UiUl", MergeNone, "aarch64_sve_udot">; -def SVQADD_S : SInst<"svqadd[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqadd_x">; -def SVQADD_U : SInst<"svqadd[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x">; -def SVQSUB_S : SInst<"svqsub[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqsub_x">; -def SVQSUB_U : SInst<"svqsub[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x">; +def SVDOT_S : SInst<"svdot[_{0}]", "ddqq", "il", MergeNone, "aarch64_sve_sdot", [IsStreamingCompatible]>; +def SVDOT_U : SInst<"svdot[_{0}]", "ddqq", "UiUl", MergeNone, "aarch64_sve_udot", [IsStreamingCompatible]>; +def SVQADD_S : SInst<"svqadd[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqadd_x", [IsStreamingCompatible]>; +def SVQADD_U : SInst<"svqadd[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x", [IsStreamingCompatible]>; +def SVQSUB_S : SInst<"svqsub[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqsub_x", [IsStreamingCompatible]>; +def SVQSUB_U : SInst<"svqsub[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [IsStreamingCompatible]>; -def SVDOT_N_S : SInst<"svdot[_n_{0}]", "ddqr", "il", MergeNone, "aarch64_sve_sdot">; -def SVDOT_N_U : SInst<"svdot[_n_{0}]", "ddqr", "UiUl", MergeNone, "aarch64_sve_udot">; -def SVQADD_N_S : SInst<"svqadd[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqadd_x">; -def SVQADD_N_U : SInst<"svqadd[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x">; -def SVQSUB_N_S : SInst<"svqsub[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqsub_x">; -def SVQSUB_N_U : SInst<"svqsub[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x">; +def SVDOT_N_S : SInst<"svdot[_n_{0}]", "ddqr", "il", MergeNone, "aarch64_sve_sdot", [IsStreamingCompatible]>; +def SVDOT_N_U : SInst<"svdot[_n_{0}]", "ddqr", "UiUl", MergeNone, "aarch64_sve_udot", [IsStreamingCompatible]>; +def SVQADD_N_S : SInst<"svqadd[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqadd_x", [IsStreamingCompatible]>; +def SVQADD_N_U : SInst<"svqadd[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x", [IsStreamingCompatible]>; +def SVQSUB_N_S : SInst<"svqsub[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqsub_x", [IsStreamingCompatible]>; +def SVQSUB_N_U : SInst<"svqsub[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [IsStreamingCompatible]>; def SVDOT_LANE_S : SInst<"svdot_lane[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_sdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; def SVDOT_LANE_U : SInst<"svdot_lane[_{d}]", "ddqqi", "UiUl", MergeNone, "aarch64_sve_udot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; @@ -656,107 +656,107 @@ defm SVNOT : SInstZPZ<"svnot", "csilUcUsUiUl", "aarch64_sve_not">; // Shifts multiclass SInst_SHIFT<string name, string intrinsic, string ts, string wide_ts> { - def _M : SInst<name # "[_{d}]", "dPdu", ts, MergeOp1, intrinsic>; - def _X : SInst<name # "[_{d}]", "dPdu", ts, MergeAny, intrinsic # _u>; - def _Z : SInst<name # "[_{d}]", "dPdu", ts, MergeZero, intrinsic>; + def _M : SInst<name # "[_{d}]", "dPdu", ts, MergeOp1, intrinsic, [IsStreamingCompatible]>; + def _X : SInst<name # "[_{d}]", "dPdu", ts, MergeAny, intrinsic # _u, [IsStreamingCompatible]>; + def _Z : SInst<name # "[_{d}]", "dPdu", ts, MergeZero, intrinsic, [IsStreamingCompatible]>; - def _N_M : SInst<name # "[_n_{d}]", "dPdL", ts, MergeOp1, intrinsic>; - def _N_X : SInst<name # "[_n_{d}]", "dPdL", ts, MergeAny, intrinsic # _u>; - def _N_Z : SInst<name # "[_n_{d}]", "dPdL", ts, MergeZero, intrinsic>; + def _N_M : SInst<name # "[_n_{d}]", "dPdL", ts, MergeOp1, intrinsic, [IsStreamingCompatible]>; + def _N_X : SInst<name # "[_n_{d}]", "dPdL", ts, MergeAny, intrinsic # _u, [IsStreamingCompatible]>; + def _N_Z : SInst<name # "[_n_{d}]", "dPdL", ts, MergeZero, intrinsic, [IsStreamingCompatible]>; - def _WIDE_M : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeOp1, intrinsic # _wide>; - def _WIDE_X : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeAny, intrinsic # _wide>; - def _WIDE_Z : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeZero, intrinsic # _wide>; + def _WIDE_M : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeOp1, intrinsic # _wide, [IsStreamingCompatible]>; + def _WIDE_X : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeAny, intrinsic # _wide, [IsStreamingCompatible]>; + def _WIDE_Z : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeZero, intrinsic # _wide, [IsStreamingCompatible]>; - def _WIDE_N_M : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeOp1, intrinsic # _wide>; - def _WIDE_N_X : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeAny, intrinsic # _wide>; - def _WIDE_N_Z : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeZero, intrinsic # _wide>; + def _WIDE_N_M : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeOp1, intrinsic # _wide, [IsStreamingCompatible]>; + def _WIDE_N_X : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeAny, intrinsic # _wide, [IsStreamingCompatible]>; + def _WIDE_N_Z : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeZero, intrinsic # _wide, [IsStreamingCompatible]>; } defm SVASR : SInst_SHIFT<"svasr", "aarch64_sve_asr", "csil", "csi">; defm SVLSL : SInst_SHIFT<"svlsl", "aarch64_sve_lsl", "csilUcUsUiUl", "csiUcUsUi">; defm SVLSR : SInst_SHIFT<"svlsr", "aarch64_sve_lsr", "UcUsUiUl", "UcUsUi">; -def SVASRD_M : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVASRD_X : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVASRD_Z : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVASRD_M : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_asrd", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVASRD_X : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_asrd", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVASRD_Z : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_asrd", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVINSR : SInst<"svinsr[_n_{d}]", "dds", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr">; +def SVINSR : SInst<"svinsr[_n_{d}]", "dds", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def SVINSR_BF16 : SInst<"svinsr[_n_{d}]", "dds", "b", MergeNone, "aarch64_sve_insr">; + def SVINSR_BF16 : SInst<"svinsr[_n_{d}]", "dds", "b", MergeNone, "aarch64_sve_insr", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// // Integer reductions -def SVADDV_S : SInst<"svaddv[_{d}]", "lPd", "csil", MergeNone, "aarch64_sve_saddv">; -def SVADDV_U : SInst<"svaddv[_{d}]", "nPd", "UcUsUiUl", MergeNone, "aarch64_sve_uaddv">; -def SVANDV : SInst<"svandv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andv">; -def SVEORV : SInst<"sveorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorv">; -def SVMAXV_S : SInst<"svmaxv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_smaxv">; -def SVMAXV_U : SInst<"svmaxv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxv">; -def SVMINV_S : SInst<"svminv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_sminv">; -def SVMINV_U : SInst<"svminv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_uminv">; -def SVORV : SInst<"svorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orv">; +def SVADDV_S : SInst<"svaddv[_{d}]", "lPd", "csil", MergeNone, "aarch64_sve_saddv", [IsStreamingCompatible]>; +def SVADDV_U : SInst<"svaddv[_{d}]", "nPd", "UcUsUiUl", MergeNone, "aarch64_sve_uaddv", [IsStreamingCompatible]>; +def SVANDV : SInst<"svandv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andv", [IsStreamingCompatible]>; +def SVEORV : SInst<"sveorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorv", [IsStreamingCompatible]>; +def SVMAXV_S : SInst<"svmaxv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_smaxv", [IsStreamingCompatible]>; +def SVMAXV_U : SInst<"svmaxv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxv", [IsStreamingCompatible]>; +def SVMINV_S : SInst<"svminv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_sminv", [IsStreamingCompatible]>; +def SVMINV_U : SInst<"svminv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_uminv", [IsStreamingCompatible]>; +def SVORV : SInst<"svorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orv", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Integer comparisons -def SVCMPEQ : SInst<"svcmpeq[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq">; -def SVCMPNE : SInst<"svcmpne[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne">; -def SVCMPGE : SInst<"svcmpge[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge">; -def SVCMPGT : SInst<"svcmpgt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt">; -def SVCMPLE : SInst<"svcmple[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare]>; -def SVCMPLT : SInst<"svcmplt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare]>; -def SVCMPHI : SInst<"svcmpgt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi">; -def SVCMPHS : SInst<"svcmpge[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs">; -def SVCMPLO : SInst<"svcmplt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare]>; -def SVCMPLS : SInst<"svcmple[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare]>; - -def SVCMPEQ_N : SInst<"svcmpeq[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq">; -def SVCMPNE_N : SInst<"svcmpne[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne">; -def SVCMPGE_N : SInst<"svcmpge[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge">; -def SVCMPGT_N : SInst<"svcmpgt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt">; -def SVCMPLE_N : SInst<"svcmple[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare]>; -def SVCMPLT_N : SInst<"svcmplt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare]>; -def SVCMPHS_N : SInst<"svcmpge[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs">; -def SVCMPHI_N : SInst<"svcmpgt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi">; -def SVCMPLS_N : SInst<"svcmple[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare]>; -def SVCMPLO_N : SInst<"svcmplt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare]>; - -def SVCMPEQ_WIDE : SInst<"svcmpeq_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpeq_wide">; -def SVCMPNE_WIDE : SInst<"svcmpne_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpne_wide">; -def SVCMPGE_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpge_wide">; -def SVCMPGT_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpgt_wide">; -def SVCMPLE_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmple_wide">; -def SVCMPLT_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmplt_wide">; -def SVCMPHI_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide">; -def SVCMPHS_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide">; -def SVCMPLO_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide">; -def SVCMPLS_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide">; - -def SVCMPEQ_WIDE_N : SInst<"svcmpeq_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpeq_wide">; -def SVCMPNE_WIDE_N : SInst<"svcmpne_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpne_wide">; -def SVCMPGE_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpge_wide">; -def SVCMPGT_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpgt_wide">; -def SVCMPLE_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmple_wide">; -def SVCMPLT_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmplt_wide">; -def SVCMPHS_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide">; -def SVCMPHI_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide">; -def SVCMPLO_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide">; -def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide">; +def SVCMPEQ : SInst<"svcmpeq[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq", [IsStreamingCompatible]>; +def SVCMPNE : SInst<"svcmpne[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne", [IsStreamingCompatible]>; +def SVCMPGE : SInst<"svcmpge[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [IsStreamingCompatible]>; +def SVCMPGT : SInst<"svcmpgt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [IsStreamingCompatible]>; +def SVCMPLE : SInst<"svcmple[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLT : SInst<"svcmplt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPHI : SInst<"svcmpgt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [IsStreamingCompatible]>; +def SVCMPHS : SInst<"svcmpge[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [IsStreamingCompatible]>; +def SVCMPLO : SInst<"svcmplt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLS : SInst<"svcmple[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare, IsStreamingCompatible]>; + +def SVCMPEQ_N : SInst<"svcmpeq[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq", [IsStreamingCompatible]>; +def SVCMPNE_N : SInst<"svcmpne[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne", [IsStreamingCompatible]>; +def SVCMPGE_N : SInst<"svcmpge[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [IsStreamingCompatible]>; +def SVCMPGT_N : SInst<"svcmpgt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [IsStreamingCompatible]>; +def SVCMPLE_N : SInst<"svcmple[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLT_N : SInst<"svcmplt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPHS_N : SInst<"svcmpge[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [IsStreamingCompatible]>; +def SVCMPHI_N : SInst<"svcmpgt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [IsStreamingCompatible]>; +def SVCMPLS_N : SInst<"svcmple[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLO_N : SInst<"svcmplt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare, IsStreamingCompatible]>; + +def SVCMPEQ_WIDE : SInst<"svcmpeq_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpeq_wide", [IsStreamingCompatible]>; +def SVCMPNE_WIDE : SInst<"svcmpne_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpne_wide", [IsStreamingCompatible]>; +def SVCMPGE_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpge_wide", [IsStreamingCompatible]>; +def SVCMPGT_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpgt_wide", [IsStreamingCompatible]>; +def SVCMPLE_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmple_wide", [IsStreamingCompatible]>; +def SVCMPLT_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmplt_wide", [IsStreamingCompatible]>; +def SVCMPHI_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide", [IsStreamingCompatible]>; +def SVCMPHS_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide", [IsStreamingCompatible]>; +def SVCMPLO_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide", [IsStreamingCompatible]>; +def SVCMPLS_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide", [IsStreamingCompatible]>; + +def SVCMPEQ_WIDE_N : SInst<"svcmpeq_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpeq_wide", [IsStreamingCompatible]>; +def SVCMPNE_WIDE_N : SInst<"svcmpne_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpne_wide", [IsStreamingCompatible]>; +def SVCMPGE_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpge_wide", [IsStreamingCompatible]>; +def SVCMPGT_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpgt_wide", [IsStreamingCompatible]>; +def SVCMPLE_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmple_wide", [IsStreamingCompatible]>; +def SVCMPLT_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmplt_wide", [IsStreamingCompatible]>; +def SVCMPHS_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide", [IsStreamingCompatible]>; +def SVCMPHI_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide", [IsStreamingCompatible]>; +def SVCMPLO_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide", [IsStreamingCompatible]>; +def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // While comparisons -def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile]>; -def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile]>; -def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile]>; -def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile]>; -def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile]>; -def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile]>; -def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile]>; -def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile]>; +def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile, IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Counting bit @@ -767,12 +767,12 @@ multiclass SInstCLS<string name, string types, string intrinsic, list<FlagType> def _Z : SInst<name # "[_{d}]", "uPd", types, MergeZeroExp, intrinsic, flags>; } -defm SVCLS : SInstCLS<"svcls", "csil", "aarch64_sve_cls">; -defm SVCLZ : SInstCLS<"svclz", "csilUcUsUiUl", "aarch64_sve_clz">; -defm SVCNT : SInstCLS<"svcnt", "csilUcUsUiUlhfd", "aarch64_sve_cnt">; +defm SVCLS : SInstCLS<"svcls", "csil", "aarch64_sve_cls", [IsStreamingCompatible]>; +defm SVCLZ : SInstCLS<"svclz", "csilUcUsUiUl", "aarch64_sve_clz", [IsStreamingCompatible]>; +defm SVCNT : SInstCLS<"svcnt", "csilUcUsUiUlhfd", "aarch64_sve_cnt", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - defm SVCNT_BF16 : SInstCLS<"svcnt", "b", "aarch64_sve_cnt">; + defm SVCNT_BF16 : SInstCLS<"svcnt", "b", "aarch64_sve_cnt", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -827,13 +827,13 @@ def SVTMAD : SInst<"svtmad[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_ftma def SVTSMUL : SInst<"svtsmul[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftsmul_x">; def SVTSSEL : SInst<"svtssel[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftssel_x">; -def SVSCALE_M : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeOp1, "aarch64_sve_fscale">; -def SVSCALE_X : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeAny, "aarch64_sve_fscale">; -def SVSCALE_Z : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeZero, "aarch64_sve_fscale">; +def SVSCALE_M : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeOp1, "aarch64_sve_fscale", [IsStreamingCompatible]>; +def SVSCALE_X : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeAny, "aarch64_sve_fscale", [IsStreamingCompatible]>; +def SVSCALE_Z : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeZero, "aarch64_sve_fscale", [IsStreamingCompatible]>; -def SVSCALE_N_M : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeOp1, "aarch64_sve_fscale">; -def SVSCALE_N_X : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeAny, "aarch64_sve_fscale">; -def SVSCALE_N_Z : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeZero, "aarch64_sve_fscale">; +def SVSCALE_N_M : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeOp1, "aarch64_sve_fscale", [IsStreamingCompatible]>; +def SVSCALE_N_X : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeAny, "aarch64_sve_fscale", [IsStreamingCompatible]>; +def SVSCALE_N_Z : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeZero, "aarch64_sve_fscale", [IsStreamingCompatible]>; defm SVMAD_F : SInstZPZZZ<"svmad", "hfd", "aarch64_sve_fmad", "aarch64_sve_fmla_u", [ReverseMergeAnyAccOp]>; defm SVMLA_F : SInstZPZZZ<"svmla", "hfd", "aarch64_sve_fmla", "aarch64_sve_fmla_u">; @@ -844,42 +844,42 @@ defm SVNMLA_F : SInstZPZZZ<"svnmla", "hfd", "aarch64_sve_fnmla", "aarch64_sve_fn defm SVNMLS_F : SInstZPZZZ<"svnmls", "hfd", "aarch64_sve_fnmls", "aarch64_sve_fnmls_u">; defm SVNMSB_F : SInstZPZZZ<"svnmsb", "hfd", "aarch64_sve_fnmsb", "aarch64_sve_fnmls_u", [ReverseMergeAnyAccOp]>; -def SVCADD_M : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeOp1, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>; -def SVCADD_X : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeAny, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>; -def SVCADD_Z : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeZero, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>; -def SVCMLA_M : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeOp1, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVCMLA_X : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeAny, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVCMLA_Z : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeZero, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>; +def SVCADD_M : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeOp1, "aarch64_sve_fcadd", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRot90_270>]>; +def SVCADD_X : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeAny, "aarch64_sve_fcadd", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRot90_270>]>; +def SVCADD_Z : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeZero, "aarch64_sve_fcadd", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRot90_270>]>; +def SVCMLA_M : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeOp1, "aarch64_sve_fcmla", [IsStreamingCompatible], [ImmCheck<4, ImmCheckComplexRotAll90>]>; +def SVCMLA_X : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeAny, "aarch64_sve_fcmla", [IsStreamingCompatible], [ImmCheck<4, ImmCheckComplexRotAll90>]>; +def SVCMLA_Z : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeZero, "aarch64_sve_fcmla", [IsStreamingCompatible], [ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVCMLA_LANE : SInst<"svcmla_lane[_{d}]", "ddddii", "hf", MergeNone, "aarch64_sve_fcmla_lane", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, +def SVCMLA_LANE : SInst<"svcmla_lane[_{d}]", "ddddii", "hf", MergeNone, "aarch64_sve_fcmla_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVMLA_LANE : SInst<"svmla_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmla_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLS_LANE : SInst<"svmls_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmls_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMUL_LANE : SInst<"svmul_lane[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_fmul_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMLA_LANE : SInst<"svmla_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmla_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLS_LANE : SInst<"svmls_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmls_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMUL_LANE : SInst<"svmul_lane[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_fmul_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVRECPE : SInst<"svrecpe[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frecpe_x">; -def SVRECPS : SInst<"svrecps[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frecps_x">; -def SVRSQRTE : SInst<"svrsqrte[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frsqrte_x">; -def SVRSQRTS : SInst<"svrsqrts[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frsqrts_x">; +def SVRECPE : SInst<"svrecpe[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frecpe_x", [IsStreamingCompatible]>; +def SVRECPS : SInst<"svrecps[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frecps_x", [IsStreamingCompatible]>; +def SVRSQRTE : SInst<"svrsqrte[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frsqrte_x", [IsStreamingCompatible]>; +def SVRSQRTS : SInst<"svrsqrts[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frsqrts_x", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Floating-point reductions -def SVFADDA : SInst<"svadda[_{d}]", "sPsd", "hfd", MergeNone, "aarch64_sve_fadda">; -def SVFADDV : SInst<"svaddv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_faddv">; -def SVFMAXV : SInst<"svmaxv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxv">; -def SVFMAXNMV : SInst<"svmaxnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxnmv">; -def SVFMINV : SInst<"svminv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminv">; -def SVFMINNMV : SInst<"svminnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminnmv">; +def SVFADDA : SInst<"svadda[_{d}]", "sPsd", "hfd", MergeNone, "aarch64_sve_fadda", [IsStreamingCompatible]>; +def SVFADDV : SInst<"svaddv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_faddv", [IsStreamingCompatible]>; +def SVFMAXV : SInst<"svmaxv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxv", [IsStreamingCompatible]>; +def SVFMAXNMV : SInst<"svmaxnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxnmv", [IsStreamingCompatible]>; +def SVFMINV : SInst<"svminv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminv", [IsStreamingCompatible]>; +def SVFMINNMV : SInst<"svminnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminnmv", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Floating-point comparisons -def SVACGE : SInst<"svacge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge">; -def SVACGT : SInst<"svacgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt">; -def SVACLE : SInst<"svacle[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare]>; -def SVACLT : SInst<"svaclt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare]>; -def SVCMPUO : SInst<"svcmpuo[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpuo">; +def SVACGE : SInst<"svacge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [IsStreamingCompatible]>; +def SVACGT : SInst<"svacgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [IsStreamingCompatible]>; +def SVACLE : SInst<"svacle[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare, IsStreamingCompatible]>; +def SVACLT : SInst<"svaclt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPUO : SInst<"svcmpuo[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpuo", [IsStreamingCompatible]>; def SVACGE_N : SInst<"svacge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facge">; def SVACGT_N : SInst<"svacgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt">; @@ -887,19 +887,19 @@ def SVACLE_N : SInst<"svacle[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_ def SVACLT_N : SInst<"svaclt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare]>; def SVCMPUO_N : SInst<"svcmpuo[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpuo">; -def SVCMPEQ_F : SInst<"svcmpeq[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpeq">; -def SVCMPNE_F : SInst<"svcmpne[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpne">; -def SVCMPGE_F : SInst<"svcmpge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge">; -def SVCMPGT_F : SInst<"svcmpgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt">; -def SVCMPLE_F : SInst<"svcmple[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare]>; -def SVCMPLT_F : SInst<"svcmplt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare]>; +def SVCMPEQ_F : SInst<"svcmpeq[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpeq", [IsStreamingCompatible]>; +def SVCMPNE_F : SInst<"svcmpne[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpne", [IsStreamingCompatible]>; +def SVCMPGE_F : SInst<"svcmpge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [IsStreamingCompatible]>; +def SVCMPGT_F : SInst<"svcmpgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [IsStreamingCompatible]>; +def SVCMPLE_F : SInst<"svcmple[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLT_F : SInst<"svcmplt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare, IsStreamingCompatible]>; -def SVCMPEQ_F_N : SInst<"svcmpeq[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpeq">; -def SVCMPNE_F_N : SInst<"svcmpne[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpne">; -def SVCMPGE_F_N : SInst<"svcmpge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge">; -def SVCMPGT_F_N : SInst<"svcmpgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt">; -def SVCMPLE_F_N : SInst<"svcmple[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare]>; -def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare]>; +def SVCMPEQ_F_N : SInst<"svcmpeq[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpeq", [IsStreamingCompatible]>; +def SVCMPNE_F_N : SInst<"svcmpne[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpne", [IsStreamingCompatible]>; +def SVCMPGE_F_N : SInst<"svcmpge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [IsStreamingCompatible]>; +def SVCMPGT_F_N : SInst<"svcmpgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [IsStreamingCompatible]>; +def SVCMPLE_F_N : SInst<"svcmple[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare, IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Floating-point conversions @@ -907,16 +907,16 @@ def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sv multiclass SInstCvtMXZ< string name, string m_types, string xz_types, string types, string intrinsic, list<FlagType> flags = [IsOverloadNone]> { - def _M : SInst<name, m_types, types, MergeOp1, intrinsic, flags>; - def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, flags>; - def _Z : SInst<name, xz_types, types, MergeZeroExp, intrinsic, flags>; + def _M : SInst<name, m_types, types, MergeOp1, intrinsic, !listconcat(flags, [IsStreamingCompatible])>; + def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, !listconcat(flags, [IsStreamingCompatible])>; + def _Z : SInst<name, xz_types, types, MergeZeroExp, intrinsic, !listconcat(flags, [IsStreamingCompatible])>; } multiclass SInstCvtMX<string name, string m_types, string xz_types, string types, string intrinsic, list<FlagType> flags = [IsOverloadNone]> { - def _M : SInst<name, m_types, types, MergeOp1, intrinsic, flags>; - def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, flags>; + def _M : SInst<name, m_types, types, MergeOp1, intrinsic, !listconcat(flags, [IsStreamingCompatible])>; + def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, !listconcat(flags, [IsStreamingCompatible])>; } // svcvt_s##_f16 @@ -930,7 +930,7 @@ defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l", "aar let TargetGuard = "sve,bf16" in { defm SVCVT_BF16_F32 : SInstCvtMXZ<"svcvt_bf16[_f32]", "ddPM", "dPM", "b", "aarch64_sve_fcvt_bf16f32">; - def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "ddPM", "b", MergeOp1, "aarch64_sve_fcvtnt_bf16f32", [IsOverloadNone]>; + def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "ddPM", "b", MergeOp1, "aarch64_sve_fcvtnt_bf16f32", [IsOverloadNone, IsStreamingCompatible]>; } // svcvt_s##_f64 @@ -994,11 +994,11 @@ defm SVCVTLT_F64 : SInstCvtMX<"svcvtlt_f64[_f32]", "ddPh", "dPh", "d", "aarc defm SVCVTX_F32 : SInstCvtMXZ<"svcvtx_f32[_f64]", "MMPd", "MPd", "d", "aarch64_sve_fcvtx_f32f64">; -def SVCVTNT_F32 : SInst<"svcvtnt_f16[_f32]", "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone]>; -def SVCVTNT_F64 : SInst<"svcvtnt_f32[_f64]", "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone]>; +def SVCVTNT_F32 : SInst<"svcvtnt_f16[_f32]", "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone, IsStreamingCompatible]>; +def SVCVTNT_F64 : SInst<"svcvtnt_f32[_f64]", "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone, IsStreamingCompatible]>; // SVCVTNT_X : Implemented as macro by SveEmitter.cpp -def SVCVTXNT_F32 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone]>; +def SVCVTXNT_F32 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone, IsStreamingCompatible]>; // SVCVTXNT_X_F32 : Implemented as macro by SveEmitter.cpp } @@ -1007,9 +1007,9 @@ def SVCVTXNT_F32 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch6 // Permutations and selection multiclass SVEPerm<string name, string proto, string i> { - def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i>; + def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def: SInst<name, proto, "b", MergeNone, i>; + def: SInst<name, proto, "b", MergeNone, i, [IsStreamingCompatible]>; } } @@ -1033,81 +1033,81 @@ def SVDUPQ_LANE : SInst<"svdupq_lane[_{d}]", "ddn", "csilUcUsUiUlhfd", MergeNo let TargetGuard = "sve,bf16" in { def SVDUPQ_LANE_BF16 : SInst<"svdupq_lane[_{d}]", "ddn", "b", MergeNone, "aarch64_sve_dupq_lane">; } -def SVEXT : SInst<"svext[_{d}]", "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [], [ImmCheck<2, ImmCheckExtract, 1>]>; +def SVEXT : SInst<"svext[_{d}]", "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [IsStreamingCompatible], [ImmCheck<2, ImmCheckExtract, 1>]>; defm SVLASTA : SVEPerm<"svlasta[_{d}]", "sPd", "aarch64_sve_lasta">; defm SVLASTB : SVEPerm<"svlastb[_{d}]", "sPd", "aarch64_sve_lastb">; -def SVREV : SInst<"svrev[_{d}]", "dd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_rev">; -def SVSEL : SInst<"svsel[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_sel">; -def SVSPLICE : SInst<"svsplice[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_splice">; -def SVTBL : SInst<"svtbl[_{d}]", "ddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl">; +def SVREV : SInst<"svrev[_{d}]", "dd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_rev", [IsStreamingCompatible]>; +def SVSEL : SInst<"svsel[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_sel", [IsStreamingCompatible]>; +def SVSPLICE : SInst<"svsplice[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_splice", [IsStreamingCompatible]>; +def SVTBL : SInst<"svtbl[_{d}]", "ddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def SVTBL_BF16 : SInst<"svtbl[_{d}]", "ddu", "b", MergeNone, "aarch64_sve_tbl">; + def SVTBL_BF16 : SInst<"svtbl[_{d}]", "ddu", "b", MergeNone, "aarch64_sve_tbl", [IsStreamingCompatible]>; } -def SVTRN1 : SInst<"svtrn1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1">; -def SVTRN2 : SInst<"svtrn2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2">; -def SVUNPKHI_S : SInst<"svunpkhi[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpkhi">; -def SVUNPKHI_U : SInst<"svunpkhi[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpkhi">; -def SVUNPKLO_S : SInst<"svunpklo[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpklo">; -def SVUNPKLO_U : SInst<"svunpklo[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpklo">; -def SVUZP1 : SInst<"svuzp1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1">; -def SVUZP2 : SInst<"svuzp2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2">; -def SVZIP1 : SInst<"svzip1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1">; -def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2">; +def SVTRN1 : SInst<"svtrn1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1", [IsStreamingCompatible]>; +def SVTRN2 : SInst<"svtrn2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2", [IsStreamingCompatible]>; +def SVUNPKHI_S : SInst<"svunpkhi[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpkhi", [IsStreamingCompatible]>; +def SVUNPKHI_U : SInst<"svunpkhi[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpkhi", [IsStreamingCompatible]>; +def SVUNPKLO_S : SInst<"svunpklo[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpklo", [IsStreamingCompatible]>; +def SVUNPKLO_U : SInst<"svunpklo[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpklo", [IsStreamingCompatible]>; +def SVUZP1 : SInst<"svuzp1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1", [IsStreamingCompatible]>; +def SVUZP2 : SInst<"svuzp2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2", [IsStreamingCompatible]>; +def SVZIP1 : SInst<"svzip1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1", [IsStreamingCompatible]>; +def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { -def SVEXT_BF16 : SInst<"svext[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_ext", [], [ImmCheck<2, ImmCheckExtract, 1>]>; -def SVREV_BF16 : SInst<"svrev[_{d}]", "dd", "b", MergeNone, "aarch64_sve_rev">; -def SVSEL_BF16 : SInst<"svsel[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_sel">; -def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice">; -def SVTRN1_BF16 : SInst<"svtrn1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1">; -def SVTRN2_BF16 : SInst<"svtrn2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2">; -def SVUZP1_BF16 : SInst<"svuzp1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1">; -def SVUZP2_BF16 : SInst<"svuzp2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2">; -def SVZIP1_BF16 : SInst<"svzip1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1">; -def SVZIP2_BF16 : SInst<"svzip2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2">; -} - -def SVREV_B8 : SInst<"svrev_b8", "PP", "Pc", MergeNone, "aarch64_sve_rev">; -def SVREV_B16 : SInst<"svrev_b16", "PP", "Pc", MergeNone, "aarch64_sve_rev_b16", [IsOverloadNone]>; -def SVREV_B32 : SInst<"svrev_b32", "PP", "Pc", MergeNone, "aarch64_sve_rev_b32", [IsOverloadNone]>; -def SVREV_B64 : SInst<"svrev_b64", "PP", "Pc", MergeNone, "aarch64_sve_rev_b64", [IsOverloadNone]>; -def SVSEL_B : SInst<"svsel[_b]", "PPPP", "Pc", MergeNone, "aarch64_sve_sel">; -def SVTRN1_B8 : SInst<"svtrn1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn1">; -def SVTRN1_B16 : SInst<"svtrn1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b16", [IsOverloadNone]>; -def SVTRN1_B32 : SInst<"svtrn1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b32", [IsOverloadNone]>; -def SVTRN1_B64 : SInst<"svtrn1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b64", [IsOverloadNone]>; -def SVTRN2_B8 : SInst<"svtrn2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn2">; -def SVTRN2_B16 : SInst<"svtrn2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b16", [IsOverloadNone]>; -def SVTRN2_B32 : SInst<"svtrn2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b32", [IsOverloadNone]>; -def SVTRN2_B64 : SInst<"svtrn2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b64", [IsOverloadNone]>; -def SVPUNPKHI : SInst<"svunpkhi[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpkhi">; -def SVPUNPKLO : SInst<"svunpklo[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpklo">; -def SVUZP1_B8 : SInst<"svuzp1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1">; -def SVUZP1_B16 : SInst<"svuzp1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b16", [IsOverloadNone]>; -def SVUZP1_B32 : SInst<"svuzp1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b32", [IsOverloadNone]>; -def SVUZP1_B64 : SInst<"svuzp1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b64", [IsOverloadNone]>; -def SVUZP2_B8 : SInst<"svuzp2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2">; -def SVUZP2_B16 : SInst<"svuzp2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b16", [IsOverloadNone]>; -def SVUZP2_B32 : SInst<"svuzp2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b32", [IsOverloadNone]>; -def SVUZP2_B64 : SInst<"svuzp2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b64", [IsOverloadNone]>; -def SVZIP1_B8 : SInst<"svzip1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip1">; -def SVZIP1_B16 : SInst<"svzip1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b16", [IsOverloadNone]>; -def SVZIP1_B32 : SInst<"svzip1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b32", [IsOverloadNone]>; -def SVZIP1_B64 : SInst<"svzip1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b64", [IsOverloadNone]>; -def SVZIP2_B : SInst<"svzip2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip2">; -def SVZIP2_B16 : SInst<"svzip2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b16", [IsOverloadNone]>; -def SVZIP2_B32 : SInst<"svzip2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b32", [IsOverloadNone]>; -def SVZIP2_B64 : SInst<"svzip2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b64", [IsOverloadNone]>; +def SVEXT_BF16 : SInst<"svext[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_ext", [IsStreamingCompatible], [ImmCheck<2, ImmCheckExtract, 1>]>; +def SVREV_BF16 : SInst<"svrev[_{d}]", "dd", "b", MergeNone, "aarch64_sve_rev", [IsStreamingCompatible]>; +def SVSEL_BF16 : SInst<"svsel[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_sel", [IsStreamingCompatible]>; +def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice", [IsStreamingCompatible]>; +def SVTRN1_BF16 : SInst<"svtrn1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1", [IsStreamingCompatible]>; +def SVTRN2_BF16 : SInst<"svtrn2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2", [IsStreamingCompatible]>; +def SVUZP1_BF16 : SInst<"svuzp1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1", [IsStreamingCompatible]>; +def SVUZP2_BF16 : SInst<"svuzp2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2", [IsStreamingCompatible]>; +def SVZIP1_BF16 : SInst<"svzip1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1", [IsStreamingCompatible]>; +def SVZIP2_BF16 : SInst<"svzip2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2", [IsStreamingCompatible]>; +} + +def SVREV_B8 : SInst<"svrev_b8", "PP", "Pc", MergeNone, "aarch64_sve_rev", [IsStreamingCompatible]>; +def SVREV_B16 : SInst<"svrev_b16", "PP", "Pc", MergeNone, "aarch64_sve_rev_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVREV_B32 : SInst<"svrev_b32", "PP", "Pc", MergeNone, "aarch64_sve_rev_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVREV_B64 : SInst<"svrev_b64", "PP", "Pc", MergeNone, "aarch64_sve_rev_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVSEL_B : SInst<"svsel[_b]", "PPPP", "Pc", MergeNone, "aarch64_sve_sel", [IsStreamingCompatible]>; +def SVTRN1_B8 : SInst<"svtrn1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn1", [IsStreamingCompatible]>; +def SVTRN1_B16 : SInst<"svtrn1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN1_B32 : SInst<"svtrn1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN1_B64 : SInst<"svtrn1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN2_B8 : SInst<"svtrn2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn2", [IsStreamingCompatible]>; +def SVTRN2_B16 : SInst<"svtrn2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN2_B32 : SInst<"svtrn2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN2_B64 : SInst<"svtrn2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVPUNPKHI : SInst<"svunpkhi[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpkhi", [IsStreamingCompatible]>; +def SVPUNPKLO : SInst<"svunpklo[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpklo", [IsStreamingCompatible]>; +def SVUZP1_B8 : SInst<"svuzp1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1", [IsStreamingCompatible]>; +def SVUZP1_B16 : SInst<"svuzp1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP1_B32 : SInst<"svuzp1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP1_B64 : SInst<"svuzp1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP2_B8 : SInst<"svuzp2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2", [IsStreamingCompatible]>; +def SVUZP2_B16 : SInst<"svuzp2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP2_B32 : SInst<"svuzp2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP2_B64 : SInst<"svuzp2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP1_B8 : SInst<"svzip1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip1", [IsStreamingCompatible]>; +def SVZIP1_B16 : SInst<"svzip1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP1_B32 : SInst<"svzip1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP1_B64 : SInst<"svzip1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP2_B : SInst<"svzip2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip2", [IsStreamingCompatible]>; +def SVZIP2_B16 : SInst<"svzip2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP2_B32 : SInst<"svzip2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP2_B64 : SInst<"svzip2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b64", [IsOverloadNone, IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Predicate creation -def SVPFALSE : SInst<"svpfalse[_b]", "Pv", "", MergeNone, "", [IsOverloadNone]>; +def SVPFALSE : SInst<"svpfalse[_b]", "Pv", "", MergeNone, "", [IsOverloadNone, IsStreamingCompatible]>; -def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue">; -def SVPTRUE : SInst<"svptrue_{d}", "Pv", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL]>; +def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsStreamingCompatible]>; +def SVPTRUE : SInst<"svptrue_{d}", "Pv", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL, IsStreamingCompatible]>; def SVDUPQ_B8 : SInst<"svdupq[_n]_{d}", "Pssssssssssssssss", "Pc", MergeNone>; def SVDUPQ_B16 : SInst<"svdupq[_n]_{d}", "Pssssssss", "Ps", MergeNone>; @@ -1119,33 +1119,33 @@ def SVDUP_N_B : SInst<"svdup[_n]_{d}", "Ps", "PcPsPiPl", MergeNone>; //////////////////////////////////////////////////////////////////////////////// // Predicate operations -def SVAND_B_Z : SInst<"svand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_and_z">; -def SVBIC_B_Z : SInst<"svbic[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_bic_z">; -def SVEOR_B_Z : SInst<"sveor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_eor_z">; -def SVMOV_B_Z : SInst<"svmov[_b]_z", "PPP", "Pc", MergeNone>; // Uses custom expansion -def SVNAND_B_Z : SInst<"svnand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nand_z">; -def SVNOR_B_Z : SInst<"svnor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nor_z">; -def SVNOT_B_Z : SInst<"svnot[_b]_z", "PPP", "Pc", MergeNone>; // Uses custom expansion -def SVORN_B_Z : SInst<"svorn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orn_z">; -def SVORR_B_Z : SInst<"svorr[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orr_z">; - -def SVBRKA : SInst<"svbrka[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brka">; -def SVBRKA_Z : SInst<"svbrka[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brka_z">; -def SVBRKB : SInst<"svbrkb[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brkb">; -def SVBRKB_Z : SInst<"svbrkb[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brkb_z">; -def SVBRKN_Z : SInst<"svbrkn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkn_z">; -def SVBRKPA_Z : SInst<"svbrkpa[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpa_z">; -def SVBRKPB_Z : SInst<"svbrkpb[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpb_z">; - -def SVPFIRST : SInst<"svpfirst[_b]", "PPP", "Pc", MergeNone, "aarch64_sve_pfirst">; -def SVPNEXT : SInst<"svpnext_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_pnext">; +def SVAND_B_Z : SInst<"svand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_and_z", [IsStreamingCompatible]>; +def SVBIC_B_Z : SInst<"svbic[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_bic_z", [IsStreamingCompatible]>; +def SVEOR_B_Z : SInst<"sveor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_eor_z", [IsStreamingCompatible]>; +def SVMOV_B_Z : SInst<"svmov[_b]_z", "PPP", "Pc", MergeNone, "", [IsStreamingCompatible]>; // Uses custom expansion +def SVNAND_B_Z : SInst<"svnand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nand_z", [IsStreamingCompatible]>; +def SVNOR_B_Z : SInst<"svnor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nor_z", [IsStreamingCompatible]>; +def SVNOT_B_Z : SInst<"svnot[_b]_z", "PPP", "Pc", MergeNone, "", [IsStreamingCompatible]>; // Uses custom expansion +def SVORN_B_Z : SInst<"svorn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orn_z", [IsStreamingCompatible]>; +def SVORR_B_Z : SInst<"svorr[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orr_z", [IsStreamingCompatible]>; + +def SVBRKA : SInst<"svbrka[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brka", [IsStreamingCompatible]>; +def SVBRKA_Z : SInst<"svbrka[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brka_z", [IsStreamingCompatible]>; +def SVBRKB : SInst<"svbrkb[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brkb", [IsStreamingCompatible]>; +def SVBRKB_Z : SInst<"svbrkb[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brkb_z", [IsStreamingCompatible]>; +def SVBRKN_Z : SInst<"svbrkn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkn_z", [IsStreamingCompatible]>; +def SVBRKPA_Z : SInst<"svbrkpa[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpa_z", [IsStreamingCompatible]>; +def SVBRKPB_Z : SInst<"svbrkpb[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpb_z", [IsStreamingCompatible]>; + +def SVPFIRST : SInst<"svpfirst[_b]", "PPP", "Pc", MergeNone, "aarch64_sve_pfirst", [IsStreamingCompatible]>; +def SVPNEXT : SInst<"svpnext_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_pnext", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Testing predicates -def SVPTEST_ANY : SInst<"svptest_any", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_any">; -def SVPTEST_FIRST : SInst<"svptest_first", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_first">; -def SVPTEST_LAST : SInst<"svptest_last", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_last">; +def SVPTEST_ANY : SInst<"svptest_any", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_any", [IsStreamingCompatible]>; +def SVPTEST_FIRST : SInst<"svptest_first", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_first", [IsStreamingCompatible]>; +def SVPTEST_LAST : SInst<"svptest_last", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_last", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // FFR manipulation @@ -1158,21 +1158,21 @@ def SVWRFFR : SInst<"svwrffr", "vP", "Pc", MergeNone, "", [IsOverloadNone]>; //////////////////////////////////////////////////////////////////////////////// // Counting elements -def SVCNTB_PAT : SInst<"svcntb_pat", "nI", "", MergeNone, "aarch64_sve_cntb", [IsOverloadNone]>; -def SVCNTH_PAT : SInst<"svcnth_pat", "nI", "", MergeNone, "aarch64_sve_cnth", [IsOverloadNone]>; -def SVCNTW_PAT : SInst<"svcntw_pat", "nI", "", MergeNone, "aarch64_sve_cntw", [IsOverloadNone]>; -def SVCNTD_PAT : SInst<"svcntd_pat", "nI", "", MergeNone, "aarch64_sve_cntd", [IsOverloadNone]>; +def SVCNTB_PAT : SInst<"svcntb_pat", "nI", "", MergeNone, "aarch64_sve_cntb", [IsOverloadNone, IsStreamingCompatible]>; +def SVCNTH_PAT : SInst<"svcnth_pat", "nI", "", MergeNone, "aarch64_sve_cnth", [IsOverloadNone, IsStreamingCompatible]>; +def SVCNTW_PAT : SInst<"svcntw_pat", "nI", "", MergeNone, "aarch64_sve_cntw", [IsOverloadNone, IsStreamingCompatible]>; +def SVCNTD_PAT : SInst<"svcntd_pat", "nI", "", MergeNone, "aarch64_sve_cntd", [IsOverloadNone, IsStreamingCompatible]>; -def SVCNTB : SInst<"svcntb", "nv", "", MergeNone, "aarch64_sve_cntb", [IsAppendSVALL, IsOverloadNone]>; -def SVCNTH : SInst<"svcnth", "nv", "", MergeNone, "aarch64_sve_cnth", [IsAppendSVALL, IsOverloadNone]>; -def SVCNTW : SInst<"svcntw", "nv", "", MergeNone, "aarch64_sve_cntw", [IsAppendSVALL, IsOverloadNone]>; -def SVCNTD : SInst<"svcntd", "nv", "", MergeNone, "aarch64_sve_cntd", [IsAppendSVALL, IsOverloadNone]>; +def SVCNTB : SInst<"svcntb", "nv", "", MergeNone, "aarch64_sve_cntb", [IsAppendSVALL, IsOverloadNone, IsStreamingCompatible]>; +def SVCNTH : SInst<"svcnth", "nv", "", MergeNone, "aarch64_sve_cnth", [IsAppendSVALL, IsOverloadNone, IsStreamingCompatible]>; +def SVCNTW : SInst<"svcntw", "nv", "", MergeNone, "aarch64_sve_cntw", [IsAppendSVALL, IsOverloadNone, IsStreamingCompatible]>; +def SVCNTD : SInst<"svcntd", "nv", "", MergeNone, "aarch64_sve_cntd", [IsAppendSVALL, IsOverloadNone, IsStreamingCompatible]>; -def SVCNTP : SInst<"svcntp_{d}", "nPP", "PcPsPiPl", MergeNone, "aarch64_sve_cntp">; -def SVLEN : SInst<"svlen[_{d}]", "nd", "csilUcUsUiUlhfd", MergeNone>; +def SVCNTP : SInst<"svcntp_{d}", "nPP", "PcPsPiPl", MergeNone, "aarch64_sve_cntp", [IsStreamingCompatible]>; +def SVLEN : SInst<"svlen[_{d}]", "nd", "csilUcUsUiUlhfd", MergeNone, "", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { -def SVLEN_BF16 : SInst<"svlen[_{d}]", "nd", "b", MergeNone>; +def SVLEN_BF16 : SInst<"svlen[_{d}]", "nd", "b", MergeNone, "", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1189,20 +1189,20 @@ def UnsignedWord : sat_type<"U", "Ui">; def UnsignedDoubleWord : sat_type<"U", "Ul">; multiclass SInst_SAT1<string name, string intrinsic, sat_type type> { - def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone], [ImmCheck<2, ImmCheck1_16>]>; - def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone], [ImmCheck<2, ImmCheck1_16>]>; - def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>; - def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>; + def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<2, ImmCheck1_16>]>; + def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<2, ImmCheck1_16>]>; + def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL, IsStreamingCompatible], [ImmCheck<1, ImmCheck1_16>]>; + def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL, IsStreamingCompatible], [ImmCheck<1, ImmCheck1_16>]>; } multiclass SInst_SAT2<string name, string intrinsic, sat_type type> { - def "" : SInst<name # "_pat[_{d}]", "ddIi", type.T, MergeNone, intrinsic, [], [ImmCheck<2, ImmCheck1_16>]>; - def _ALL : SInst<name # "[_{d}]", "ddi", type.T, MergeNone, intrinsic, [IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>; + def "" : SInst<name # "_pat[_{d}]", "ddIi", type.T, MergeNone, intrinsic, [IsStreamingCompatible], [ImmCheck<2, ImmCheck1_16>]>; + def _ALL : SInst<name # "[_{d}]", "ddi", type.T, MergeNone, intrinsic, [IsInsertOp1SVALL, IsStreamingCompatible], [ImmCheck<1, ImmCheck1_16>]>; - def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone], [ImmCheck<2, ImmCheck1_16>]>; - def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone], [ImmCheck<2, ImmCheck1_16>]>; - def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>; - def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>; + def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<2, ImmCheck1_16>]>; + def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<2, ImmCheck1_16>]>; + def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL, IsStreamingCompatible], [ImmCheck<1, ImmCheck1_16>]>; + def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL, IsStreamingCompatible], [ImmCheck<1, ImmCheck1_16>]>; } defm SVQDECB_S : SInst_SAT1<"svqdecb", "aarch64_sve_sqdecb", SignedByte>; @@ -1223,32 +1223,32 @@ defm SVQINCW_U : SInst_SAT2<"svqincw", "aarch64_sve_uqincw", UnsignedWord>; defm SVQINCD_S : SInst_SAT2<"svqincd", "aarch64_sve_sqincd", SignedDoubleWord>; defm SVQINCD_U : SInst_SAT2<"svqincd", "aarch64_sve_uqincd", UnsignedDoubleWord>; -def SVQDECP_S : SInst<"svqdecp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqdecp">; -def SVQDECP_U : SInst<"svqdecp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqdecp">; -def SVQINCP_S : SInst<"svqincp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqincp">; -def SVQINCP_U : SInst<"svqincp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqincp">; +def SVQDECP_S : SInst<"svqdecp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqdecp", [IsStreamingCompatible]>; +def SVQDECP_U : SInst<"svqdecp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqdecp", [IsStreamingCompatible]>; +def SVQINCP_S : SInst<"svqincp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqincp", [IsStreamingCompatible]>; +def SVQINCP_U : SInst<"svqincp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqincp", [IsStreamingCompatible]>; -def SVQDECP_N_S32 : SInst<"svqdecp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n32">; -def SVQDECP_N_S64 : SInst<"svqdecp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n64">; -def SVQDECP_N_U32 : SInst<"svqdecp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n32">; -def SVQDECP_N_U64 : SInst<"svqdecp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n64">; -def SVQINCP_N_S32 : SInst<"svqincp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n32">; -def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n64">; -def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32">; -def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64">; +def SVQDECP_N_S32 : SInst<"svqdecp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n32", [IsStreamingCompatible]>; +def SVQDECP_N_S64 : SInst<"svqdecp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n64", [IsStreamingCompatible]>; +def SVQDECP_N_U32 : SInst<"svqdecp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n32", [IsStreamingCompatible]>; +def SVQDECP_N_U64 : SInst<"svqdecp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n64", [IsStreamingCompatible]>; +def SVQINCP_N_S32 : SInst<"svqincp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n32", [IsStreamingCompatible]>; +def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n64", [IsStreamingCompatible]>; +def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32", [IsStreamingCompatible]>; +def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64", [IsStreamingCompatible]>; let TargetGuard = "sve,i8mm" in { def SVMLLA_S32 : SInst<"svmmla[_s32]", "ddqq","i", MergeNone, "aarch64_sve_smmla">; def SVMLLA_U32 : SInst<"svmmla[_u32]", "ddqq","Ui", MergeNone, "aarch64_sve_ummla">; def SVUSMLLA_S32 : SInst<"svusmmla[_s32]", "ddbq","i", MergeNone, "aarch64_sve_usmmla">; -def SVUSDOT_S : SInst<"svusdot[_s32]", "ddbq", "i", MergeNone, "aarch64_sve_usdot">; -def SVUSDOT_N_S : SInst<"svusdot[_n_s32]", "ddbr", "i", MergeNone, "aarch64_sve_usdot">; -def SVSUDOT_S : SInst<"svsudot[_s32]", "ddqb", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>; -def SVSUDOT_N_S : SInst<"svsudot[_n_s32]", "ddq@", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>; +def SVUSDOT_S : SInst<"svusdot[_s32]", "ddbq", "i", MergeNone, "aarch64_sve_usdot", [IsStreamingCompatible]>; +def SVUSDOT_N_S : SInst<"svusdot[_n_s32]", "ddbr", "i", MergeNone, "aarch64_sve_usdot", [IsStreamingCompatible]>; +def SVSUDOT_S : SInst<"svsudot[_s32]", "ddqb", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, IsStreamingCompatible]>; +def SVSUDOT_N_S : SInst<"svsudot[_n_s32]", "ddq@", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, IsStreamingCompatible]>; -def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]", "ddbqi", "i", MergeNone, "aarch64_sve_usdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; -def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]", "ddqbi", "i", MergeNone, "aarch64_sve_sudot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; +def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]", "ddbqi", "i", MergeNone, "aarch64_sve_usdot_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; +def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]", "ddqbi", "i", MergeNone, "aarch64_sve_sudot_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; } let TargetGuard = "sve,f32mm" in { @@ -1257,12 +1257,12 @@ def SVMLLA_F32 : SInst<"svmmla[_f32]", "dddd","f", MergeNone, "aarch64_sve_fmmla let TargetGuard = "sve,f64mm" in { def SVMLLA_F64 : SInst<"svmmla[_f64]", "dddd","d", MergeNone, "aarch64_sve_fmmla">; -def SVTRN1Q : SInst<"svtrn1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1q">; -def SVTRN2Q : SInst<"svtrn2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2q">; -def SVUZP1Q : SInst<"svuzp1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1q">; -def SVUZP2Q : SInst<"svuzp2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2q">; -def SVZIP1Q : SInst<"svzip1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1q">; -def SVZIP2Q : SInst<"svzip2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q">; +def SVTRN1Q : SInst<"svtrn1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1q", [IsStreamingCompatible]>; +def SVTRN2Q : SInst<"svtrn2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2q", [IsStreamingCompatible]>; +def SVUZP1Q : SInst<"svuzp1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1q", [IsStreamingCompatible]>; +def SVUZP2Q : SInst<"svuzp2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2q", [IsStreamingCompatible]>; +def SVZIP1Q : SInst<"svzip1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1q", [IsStreamingCompatible]>; +def SVZIP2Q : SInst<"svzip2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q", [IsStreamingCompatible]>; } let TargetGuard = "sve,bf16,f64mm" in { @@ -1276,20 +1276,20 @@ def SVZIP2Q_BF16 : SInst<"svzip2q[_{d}]", "ddd", "b", MergeNone, "aarc //////////////////////////////////////////////////////////////////////////////// // Vector creation -def SVUNDEF_1 : SInst<"svundef_{d}", "dv", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; -def SVUNDEF_2 : SInst<"svundef2_{d}", "2v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; -def SVUNDEF_3 : SInst<"svundef3_{d}", "3v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; -def SVUNDEF_4 : SInst<"svundef4_{d}", "4v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; +def SVUNDEF_1 : SInst<"svundef_{d}", "dv", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_2 : SInst<"svundef2_{d}", "2v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_3 : SInst<"svundef3_{d}", "3v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_4 : SInst<"svundef4_{d}", "4v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>; def SVCREATE_2 : SInst<"svcreate2[_{d}]", "2dd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>; def SVCREATE_3 : SInst<"svcreate3[_{d}]", "3ddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>; def SVCREATE_4 : SInst<"svcreate4[_{d}]", "4dddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>; let TargetGuard = "sve,bf16" in { -def SVUNDEF_1_BF16 : SInst<"svundef_{d}", "dv", "b", MergeNone, "", [IsUndef]>; -def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2v", "b", MergeNone, "", [IsUndef]>; -def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3v", "b", MergeNone, "", [IsUndef]>; -def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4v", "b", MergeNone, "", [IsUndef]>; +def SVUNDEF_1_BF16 : SInst<"svundef_{d}", "dv", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2v", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3v", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4v", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>; def SVCREATE_2_BF16 : SInst<"svcreate2[_{d}]", "2dd", "b", MergeNone, "", [IsTupleCreate]>; def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "", [IsTupleCreate]>; @@ -1331,14 +1331,26 @@ let TargetGuard = "sve2p1" in { //////////////////////////////////////////////////////////////////////////////// // SVE2 WhileGE/GT let TargetGuard = "sve2" in { -def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile]>; -def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile]>; -def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile]>; -def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile]>; -def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile]>; -def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile]>; -def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile]>; -def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile]>; +def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile, IsStreamingCompatible]>; +} + +let TargetGuard = "sve2p1|sme2" in { + def SVWHILEGE_S64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege_x2">; + def SVWHILEGT_S64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt_x2">; + def SVWHILEHI_U64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi_x2">; + def SVWHILEHS_U64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs_x2">; + def SVWHILELE_S64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele_x2">; + def SVWHILELT_S64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt_x2">; + def SVWHILELO_U64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo_x2">; + def SVWHILELS_U64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilels_x2">; + } //////////////////////////////////////////////////////////////////////////////// @@ -1380,49 +1392,49 @@ multiclass SInstZPZxZ<string name, string types, string pat_v, string pat_n, str } let TargetGuard = "sve2" in { -defm SVQRSHL_S : SInstZPZxZ<"svqrshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqrshl">; -defm SVQRSHL_U : SInstZPZxZ<"svqrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqrshl">; -defm SVQSHL_S : SInstZPZxZ<"svqshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqshl">; -defm SVQSHL_U : SInstZPZxZ<"svqshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqshl">; -defm SVRSHL_S : SInstZPZxZ<"svrshl", "csil", "dPdx", "dPdK", "aarch64_sve_srshl">; -defm SVRSHL_U : SInstZPZxZ<"svrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_urshl">; -defm SVSQADD : SInstZPZxZ<"svsqadd", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_usqadd">; -defm SVUQADD : SInstZPZxZ<"svuqadd", "csil", "dPdu", "dPdL", "aarch64_sve_suqadd">; - -def SVABA_S : SInst<"svaba[_{d}]", "dddd", "csil" , MergeNone, "aarch64_sve_saba">; -def SVABA_U : SInst<"svaba[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uaba">; -def SVQDMULH : SInst<"svqdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqdmulh">; -def SVQRDMULH : SInst<"svqrdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqrdmulh">; -def SVQRDMLAH : SInst<"svqrdmlah[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlah">; -def SVQRDMLSH : SInst<"svqrdmlsh[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlsh">; - -def SVABA_S_N : SInst<"svaba[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_saba">; -def SVABA_U_N : SInst<"svaba[_n_{d}]", "ddda", "UcUsUiUl", MergeNone, "aarch64_sve_uaba">; -def SVQDMULH_N : SInst<"svqdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqdmulh">; -def SVQRDMULH_N : SInst<"svqrdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqrdmulh">; -def SVQRDMLAH_N : SInst<"svqrdmlah[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlah">; -def SVQRDMLSH_N : SInst<"svqrdmlsh[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlsh">; - -def SVQDMULH_LANE : SInst<"svqdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqdmulh_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVQRDMULH_LANE : SInst<"svqrdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqrdmulh_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVQRDMLAH_LANE : SInst<"svqrdmlah_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlah_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQRDMLSH_LANE : SInst<"svqrdmlsh_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlsh_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; - -def SVQSHLU_M : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeOp1, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; -def SVQSHLU_X : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeAny, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; -def SVQSHLU_Z : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeZero, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; -def SVRSHR_M_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_M_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeOp1, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_X_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_X_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeAny, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_Z_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_Z_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeZero, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSRA_S : SInst<"svrsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_srsra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSRA_U : SInst<"svrsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_ursra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVSLI : SInst<"svsli[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sli", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; -def SVSRA_S : SInst<"svsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_ssra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVSRA_U : SInst<"svsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_usra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVSRI : SInst<"svsri[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sri", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +defm SVQRSHL_S : SInstZPZxZ<"svqrshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqrshl", [IsStreamingCompatible]>; +defm SVQRSHL_U : SInstZPZxZ<"svqrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqrshl", [IsStreamingCompatible]>; +defm SVQSHL_S : SInstZPZxZ<"svqshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqshl", [IsStreamingCompatible]>; +defm SVQSHL_U : SInstZPZxZ<"svqshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqshl", [IsStreamingCompatible]>; +defm SVRSHL_S : SInstZPZxZ<"svrshl", "csil", "dPdx", "dPdK", "aarch64_sve_srshl", [IsStreamingCompatible]>; +defm SVRSHL_U : SInstZPZxZ<"svrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_urshl", [IsStreamingCompatible]>; +defm SVSQADD : SInstZPZxZ<"svsqadd", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_usqadd", [IsStreamingCompatible]>; +defm SVUQADD : SInstZPZxZ<"svuqadd", "csil", "dPdu", "dPdL", "aarch64_sve_suqadd", [IsStreamingCompatible]>; + +def SVABA_S : SInst<"svaba[_{d}]", "dddd", "csil" , MergeNone, "aarch64_sve_saba", [IsStreamingCompatible]>; +def SVABA_U : SInst<"svaba[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uaba", [IsStreamingCompatible]>; +def SVQDMULH : SInst<"svqdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqdmulh", [IsStreamingCompatible]>; +def SVQRDMULH : SInst<"svqrdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqrdmulh", [IsStreamingCompatible]>; +def SVQRDMLAH : SInst<"svqrdmlah[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlah", [IsStreamingCompatible]>; +def SVQRDMLSH : SInst<"svqrdmlsh[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlsh", [IsStreamingCompatible]>; + +def SVABA_S_N : SInst<"svaba[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_saba", [IsStreamingCompatible]>; +def SVABA_U_N : SInst<"svaba[_n_{d}]", "ddda", "UcUsUiUl", MergeNone, "aarch64_sve_uaba", [IsStreamingCompatible]>; +def SVQDMULH_N : SInst<"svqdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqdmulh", [IsStreamingCompatible]>; +def SVQRDMULH_N : SInst<"svqrdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqrdmulh", [IsStreamingCompatible]>; +def SVQRDMLAH_N : SInst<"svqrdmlah[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlah", [IsStreamingCompatible]>; +def SVQRDMLSH_N : SInst<"svqrdmlsh[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlsh", [IsStreamingCompatible]>; + +def SVQDMULH_LANE : SInst<"svqdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqdmulh_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVQRDMULH_LANE : SInst<"svqrdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqrdmulh_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVQRDMLAH_LANE : SInst<"svqrdmlah_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlah_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQRDMLSH_LANE : SInst<"svqrdmlsh_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlsh_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; + +def SVQSHLU_M : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeOp1, "aarch64_sve_sqshlu", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; +def SVQSHLU_X : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeAny, "aarch64_sve_sqshlu", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; +def SVQSHLU_Z : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeZero, "aarch64_sve_sqshlu", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; +def SVRSHR_M_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_srshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_M_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeOp1, "aarch64_sve_urshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_X_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_srshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_X_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeAny, "aarch64_sve_urshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_Z_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_srshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_Z_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeZero, "aarch64_sve_urshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSRA_S : SInst<"svrsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_srsra", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSRA_U : SInst<"svrsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_ursra", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVSLI : SInst<"svsli[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sli", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; +def SVSRA_S : SInst<"svsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_ssra", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVSRA_U : SInst<"svsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_usra", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVSRI : SInst<"svsri[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sri", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1434,29 +1446,29 @@ multiclass SInstPairwise<string name, string types, string intrinsic, list<FlagT } let TargetGuard = "sve2" in { -defm SVADDP : SInstPairwise<"svaddp", "csliUcUsUiUl", "aarch64_sve_addp">; -defm SVADDP_F : SInstPairwise<"svaddp", "hfd", "aarch64_sve_faddp">; -defm SVMAXNMP : SInstPairwise<"svmaxnmp", "hfd", "aarch64_sve_fmaxnmp">; -defm SVMAXP_F : SInstPairwise<"svmaxp", "hfd", "aarch64_sve_fmaxp">; -defm SVMAXP_S : SInstPairwise<"svmaxp", "csli", "aarch64_sve_smaxp">; -defm SVMAXP_U : SInstPairwise<"svmaxp", "UcUsUiUl", "aarch64_sve_umaxp">; -defm SVMINNMP : SInstPairwise<"svminnmp", "hfd", "aarch64_sve_fminnmp">; -defm SVMINP_F : SInstPairwise<"svminp", "hfd", "aarch64_sve_fminp">; -defm SVMINP_S : SInstPairwise<"svminp", "csli", "aarch64_sve_sminp">; -defm SVMINP_U : SInstPairwise<"svminp", "UcUsUiUl", "aarch64_sve_uminp">; +defm SVADDP : SInstPairwise<"svaddp", "csliUcUsUiUl", "aarch64_sve_addp", [IsStreamingCompatible]>; +defm SVADDP_F : SInstPairwise<"svaddp", "hfd", "aarch64_sve_faddp", [IsStreamingCompatible]>; +defm SVMAXNMP : SInstPairwise<"svmaxnmp", "hfd", "aarch64_sve_fmaxnmp", [IsStreamingCompatible]>; +defm SVMAXP_F : SInstPairwise<"svmaxp", "hfd", "aarch64_sve_fmaxp", [IsStreamingCompatible]>; +defm SVMAXP_S : SInstPairwise<"svmaxp", "csli", "aarch64_sve_smaxp", [IsStreamingCompatible]>; +defm SVMAXP_U : SInstPairwise<"svmaxp", "UcUsUiUl", "aarch64_sve_umaxp", [IsStreamingCompatible]>; +defm SVMINNMP : SInstPairwise<"svminnmp", "hfd", "aarch64_sve_fminnmp", [IsStreamingCompatible]>; +defm SVMINP_F : SInstPairwise<"svminp", "hfd", "aarch64_sve_fminp", [IsStreamingCompatible]>; +defm SVMINP_S : SInstPairwise<"svminp", "csli", "aarch64_sve_sminp", [IsStreamingCompatible]>; +defm SVMINP_U : SInstPairwise<"svminp", "UcUsUiUl", "aarch64_sve_uminp", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Widening pairwise arithmetic let TargetGuard = "sve2" in { -def SVADALP_S_M : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeOp1, "aarch64_sve_sadalp">; -def SVADALP_S_X : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeAny, "aarch64_sve_sadalp">; -def SVADALP_S_Z : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeZero, "aarch64_sve_sadalp">; +def SVADALP_S_M : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeOp1, "aarch64_sve_sadalp", [IsStreamingCompatible]>; +def SVADALP_S_X : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeAny, "aarch64_sve_sadalp", [IsStreamingCompatible]>; +def SVADALP_S_Z : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeZero, "aarch64_sve_sadalp", [IsStreamingCompatible]>; -def SVADALP_U_M : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeOp1, "aarch64_sve_uadalp">; -def SVADALP_U_X : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeAny, "aarch64_sve_uadalp">; -def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_sve_uadalp">; +def SVADALP_U_M : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeOp1, "aarch64_sve_uadalp", [IsStreamingCompatible]>; +def SVADALP_U_X : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeAny, "aarch64_sve_uadalp", [IsStreamingCompatible]>; +def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_sve_uadalp", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1464,56 +1476,56 @@ def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_s // let TargetGuard = "sve2" in { -def SVBCAX : SInst<"svbcax[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax">; -def SVBSL : SInst<"svbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl">; -def SVBSL1N : SInst<"svbsl1n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n">; -def SVBSL2N : SInst<"svbsl2n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n">; -def SVEOR3 : SInst<"sveor3[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3">; -def SVNBSL : SInst<"svnbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl">; - -def SVBCAX_N : SInst<"svbcax[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax">; -def SVBSL_N : SInst<"svbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl">; -def SVBSL1N_N : SInst<"svbsl1n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n">; -def SVBSL2N_N : SInst<"svbsl2n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n">; -def SVEOR3_N : SInst<"sveor3[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3">; -def SVNBSL_N : SInst<"svnbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl">; -def SVXAR_N : SInst<"svxar[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_xar", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVBCAX : SInst<"svbcax[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [IsStreamingCompatible]>; +def SVBSL : SInst<"svbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [IsStreamingCompatible]>; +def SVBSL1N : SInst<"svbsl1n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [IsStreamingCompatible]>; +def SVBSL2N : SInst<"svbsl2n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n", [IsStreamingCompatible]>; +def SVEOR3 : SInst<"sveor3[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3", [IsStreamingCompatible]>; +def SVNBSL : SInst<"svnbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl", [IsStreamingCompatible]>; + +def SVBCAX_N : SInst<"svbcax[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [IsStreamingCompatible]>; +def SVBSL_N : SInst<"svbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [IsStreamingCompatible]>; +def SVBSL1N_N : SInst<"svbsl1n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [IsStreamingCompatible]>; +def SVBSL2N_N : SInst<"svbsl2n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n", [IsStreamingCompatible]>; +def SVEOR3_N : SInst<"sveor3[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3", [IsStreamingCompatible]>; +def SVNBSL_N : SInst<"svnbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl", [IsStreamingCompatible]>; +def SVXAR_N : SInst<"svxar[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_xar", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Large integer arithmetic let TargetGuard = "sve2" in { -def SVADCLB : SInst<"svadclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclb">; -def SVADCLT : SInst<"svadclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclt">; -def SVSBCLB : SInst<"svsbclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclb">; -def SVSBCLT : SInst<"svsbclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclt">; +def SVADCLB : SInst<"svadclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclb", [IsStreamingCompatible]>; +def SVADCLT : SInst<"svadclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclt", [IsStreamingCompatible]>; +def SVSBCLB : SInst<"svsbclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclb", [IsStreamingCompatible]>; +def SVSBCLT : SInst<"svsbclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclt", [IsStreamingCompatible]>; -def SVADCLB_N : SInst<"svadclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclb">; -def SVADCLT_N : SInst<"svadclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclt">; -def SVSBCLB_N : SInst<"svsbclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclb">; -def SVSBCLT_N : SInst<"svsbclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclt">; +def SVADCLB_N : SInst<"svadclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclb", [IsStreamingCompatible]>; +def SVADCLT_N : SInst<"svadclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclt", [IsStreamingCompatible]>; +def SVSBCLB_N : SInst<"svsbclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclb", [IsStreamingCompatible]>; +def SVSBCLT_N : SInst<"svsbclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclt", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Multiplication by indexed elements let TargetGuard = "sve2" in { -def SVMLA_LANE_2 : SInst<"svmla_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mla_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLS_LANE_2 : SInst<"svmls_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mls_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi", "silUsUiUl", MergeNone, "aarch64_sve_mul_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMLA_LANE_2 : SInst<"svmla_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mla_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLS_LANE_2 : SInst<"svmls_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mls_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi", "silUsUiUl", MergeNone, "aarch64_sve_mul_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Uniform complex integer arithmetic let TargetGuard = "sve2" in { -def SVCADD : SInst<"svcadd[_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cadd_x", [], [ImmCheck<2, ImmCheckComplexRot90_270>]>; -def SVSQCADD : SInst<"svqcadd[_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_sqcadd_x", [], [ImmCheck<2, ImmCheckComplexRot90_270>]>; -def SVCMLA : SInst<"svcmla[_{d}]", "ddddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmla_x", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>; -def SVCMLA_LANE_X : SInst<"svcmla_lane[_{d}]", "ddddii", "siUsUi", MergeNone, "aarch64_sve_cmla_lane_x", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, +def SVCADD : SInst<"svcadd[_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cadd_x", [IsStreamingCompatible], [ImmCheck<2, ImmCheckComplexRot90_270>]>; +def SVSQCADD : SInst<"svqcadd[_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_sqcadd_x", [IsStreamingCompatible], [ImmCheck<2, ImmCheckComplexRot90_270>]>; +def SVCMLA : SInst<"svcmla[_{d}]", "ddddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmla_x", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRotAll90>]>; +def SVCMLA_LANE_X : SInst<"svcmla_lane[_{d}]", "ddddii", "siUsUi", MergeNone, "aarch64_sve_cmla_lane_x", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVSQRDCMLAH_X : SInst<"svqrdcmlah[_{d}]", "ddddi", "csil", MergeNone, "aarch64_sve_sqrdcmlah_x", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>; -def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", MergeNone, "aarch64_sve_sqrdcmlah_lane_x", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, +def SVSQRDCMLAH_X : SInst<"svqrdcmlah[_{d}]", "ddddi", "csil", MergeNone, "aarch64_sve_sqrdcmlah_x", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRotAll90>]>; +def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", MergeNone, "aarch64_sve_sqrdcmlah_lane_x", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, ImmCheck<4, ImmCheckComplexRotAll90>]>; } @@ -1521,18 +1533,18 @@ def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", // SVE2 - Widening DSP operations multiclass SInstWideDSPAcc<string name, string types, string intrinsic> { - def : SInst<name # "[_{d}]", "ddhh", types, MergeNone, intrinsic>; - def _N : SInst<name # "[_n_{d}]", "ddhR", types, MergeNone, intrinsic>; + def : SInst<name # "[_{d}]", "ddhh", types, MergeNone, intrinsic, [IsStreamingCompatible]>; + def _N : SInst<name # "[_n_{d}]", "ddhR", types, MergeNone, intrinsic, [IsStreamingCompatible]>; } multiclass SInstWideDSPLong<string name, string types, string intrinsic> { - def : SInst<name # "[_{d}]", "dhh", types, MergeNone, intrinsic>; - def _N : SInst<name # "[_n_{d}]", "dhR", types, MergeNone, intrinsic>; + def : SInst<name # "[_{d}]", "dhh", types, MergeNone, intrinsic, [IsStreamingCompatible]>; + def _N : SInst<name # "[_n_{d}]", "dhR", types, MergeNone, intrinsic, [IsStreamingCompatible]>; } multiclass SInstWideDSPWide<string name, string types, string intrinsic> { - def : SInst<name # "[_{d}]", "ddh", types, MergeNone, intrinsic>; - def _N : SInst<name # "[_n_{d}]", "ddR", types, MergeNone, intrinsic>; + def : SInst<name # "[_{d}]", "ddh", types, MergeNone, intrinsic, [IsStreamingCompatible]>; + def _N : SInst<name # "[_n_{d}]", "ddR", types, MergeNone, intrinsic, [IsStreamingCompatible]>; } let TargetGuard = "sve2" in { @@ -1581,87 +1593,87 @@ defm SVSUBWB_U : SInstWideDSPWide<"svsubwb", "UsUiUl", "aarch64_sve_usubwb">; defm SVSUBWT_S : SInstWideDSPWide<"svsubwt", "sil", "aarch64_sve_ssubwt">; defm SVSUBWT_U : SInstWideDSPWide<"svsubwt", "UsUiUl", "aarch64_sve_usubwt">; -def SVSHLLB_S_N : SInst<"svshllb[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllb", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; -def SVSHLLB_U_N : SInst<"svshllb[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllb", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; -def SVSHLLT_S_N : SInst<"svshllt[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllt", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; -def SVSHLLT_U_N : SInst<"svshllt[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllt", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; - -def SVMOVLB_S_N : SInst<"svmovlb[_{d}]", "dh", "sil", MergeNone>; -def SVMOVLB_U_N : SInst<"svmovlb[_{d}]", "dh", "UsUiUl", MergeNone>; -def SVMOVLT_S_N : SInst<"svmovlt[_{d}]", "dh", "sil", MergeNone>; -def SVMOVLT_U_N : SInst<"svmovlt[_{d}]", "dh", "UsUiUl", MergeNone>; - -def SVMLALB_S_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLALB_U_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLALT_S_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLALT_U_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLB_S_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLB_U_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLT_S_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLT_U_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMULLB_S_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVMULLB_U_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVMULLT_S_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVMULLT_U_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVQDMLALB_LANE : SInst<"svqdmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQDMLALT_LANE : SInst<"svqdmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQDMLSLB_LANE : SInst<"svqdmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQDMLSLT_LANE : SInst<"svqdmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQDMULLB_LANE : SInst<"svqdmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVQDMULLT_LANE : SInst<"svqdmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVSHLLB_S_N : SInst<"svshllb[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; +def SVSHLLB_U_N : SInst<"svshllb[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; +def SVSHLLT_S_N : SInst<"svshllt[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllt", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; +def SVSHLLT_U_N : SInst<"svshllt[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllt", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; + +def SVMOVLB_S_N : SInst<"svmovlb[_{d}]", "dh", "sil", MergeNone, "", [IsStreamingCompatible]>; +def SVMOVLB_U_N : SInst<"svmovlb[_{d}]", "dh", "UsUiUl", MergeNone, "", [IsStreamingCompatible]>; +def SVMOVLT_S_N : SInst<"svmovlt[_{d}]", "dh", "sil", MergeNone, "", [IsStreamingCompatible]>; +def SVMOVLT_U_N : SInst<"svmovlt[_{d}]", "dh", "UsUiUl", MergeNone, "", [IsStreamingCompatible]>; + +def SVMLALB_S_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALB_U_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALT_S_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALT_U_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLB_S_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLB_U_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLT_S_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLT_U_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMULLB_S_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullb_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMULLB_U_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullb_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMULLT_S_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullt_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMULLT_U_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullt_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVQDMLALB_LANE : SInst<"svqdmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQDMLALT_LANE : SInst<"svqdmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQDMLSLB_LANE : SInst<"svqdmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQDMLSLT_LANE : SInst<"svqdmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQDMULLB_LANE : SInst<"svqdmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullb_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVQDMULLT_LANE : SInst<"svqdmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullt_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Narrowing DSP operations let TargetGuard = "sve2" in { -def SVADDHNB : SInst<"svaddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnb">; -def SVADDHNT : SInst<"svaddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnt">; -def SVRADDHNB : SInst<"svraddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb">; -def SVRADDHNT : SInst<"svraddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt">; -def SVRSUBHNB : SInst<"svrsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb">; -def SVRSUBHNT : SInst<"svrsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt">; -def SVSUBHNB : SInst<"svsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnb">; -def SVSUBHNT : SInst<"svsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnt">; - -def SVADDHNB_N : SInst<"svaddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_addhnb">; -def SVADDHNT_N : SInst<"svaddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_addhnt">; -def SVRADDHNB_N : SInst<"svraddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb">; -def SVRADDHNT_N : SInst<"svraddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt">; -def SVRSUBHNB_N : SInst<"svrsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb">; -def SVRSUBHNT_N : SInst<"svrsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt">; -def SVSUBHNB_N : SInst<"svsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_subhnb">; -def SVSUBHNT_N : SInst<"svsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_subhnt">; - -def SVSHRNB : SInst<"svshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVRSHRNB : SInst<"svrshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQSHRUNB : SInst<"svqshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqshrunb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQRSHRUNB : SInst<"svqrshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqrshrunb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQSHRNB_S : SInst<"svqshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQSHRNB_U : SInst<"svqshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQRSHRNB_S : SInst<"svqrshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqrshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQRSHRNB_U : SInst<"svqrshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; - -def SVSHRNT : SInst<"svshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVRSHRNT : SInst<"svrshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQSHRUNT : SInst<"svqshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqshrunt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQRSHRUNT : SInst<"svqrshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqrshrunt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQSHRNT_S : SInst<"svqshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQSHRNT_U : SInst<"svqshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQRSHRNT_S : SInst<"svqrshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqrshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQRSHRNT_U : SInst<"svqrshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVADDHNB : SInst<"svaddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [IsStreamingCompatible]>; +def SVADDHNT : SInst<"svaddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [IsStreamingCompatible]>; +def SVRADDHNB : SInst<"svraddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [IsStreamingCompatible]>; +def SVRADDHNT : SInst<"svraddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt", [IsStreamingCompatible]>; +def SVRSUBHNB : SInst<"svrsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb", [IsStreamingCompatible]>; +def SVRSUBHNT : SInst<"svrsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt", [IsStreamingCompatible]>; +def SVSUBHNB : SInst<"svsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnb", [IsStreamingCompatible]>; +def SVSUBHNT : SInst<"svsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnt", [IsStreamingCompatible]>; + +def SVADDHNB_N : SInst<"svaddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [IsStreamingCompatible]>; +def SVADDHNT_N : SInst<"svaddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [IsStreamingCompatible]>; +def SVRADDHNB_N : SInst<"svraddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [IsStreamingCompatible]>; +def SVRADDHNT_N : SInst<"svraddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt", [IsStreamingCompatible]>; +def SVRSUBHNB_N : SInst<"svrsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb", [IsStreamingCompatible]>; +def SVRSUBHNT_N : SInst<"svrsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt", [IsStreamingCompatible]>; +def SVSUBHNB_N : SInst<"svsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_subhnb", [IsStreamingCompatible]>; +def SVSUBHNT_N : SInst<"svsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_subhnt", [IsStreamingCompatible]>; + +def SVSHRNB : SInst<"svshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVRSHRNB : SInst<"svrshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQSHRUNB : SInst<"svqshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqshrunb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQRSHRUNB : SInst<"svqrshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqrshrunb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQSHRNB_S : SInst<"svqshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQSHRNB_U : SInst<"svqshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQRSHRNB_S : SInst<"svqrshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqrshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQRSHRNB_U : SInst<"svqrshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; + +def SVSHRNT : SInst<"svshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVRSHRNT : SInst<"svrshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQSHRUNT : SInst<"svqshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqshrunt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQRSHRUNT : SInst<"svqrshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqrshrunt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQSHRNT_S : SInst<"svqshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQSHRNT_U : SInst<"svqshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQRSHRNT_S : SInst<"svqrshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqrshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQRSHRNT_U : SInst<"svqrshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Unary narrowing operations let TargetGuard = "sve2" in { -def SVQXTNB_S : SInst<"svqxtnb[_{d}]", "hd", "sil", MergeNone, "aarch64_sve_sqxtnb">; -def SVQXTNB_U : SInst<"svqxtnb[_{d}]", "hd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnb">; -def SVQXTUNB_S : SInst<"svqxtunb[_{d}]", "ed", "sil", MergeNone, "aarch64_sve_sqxtunb">; +def SVQXTNB_S : SInst<"svqxtnb[_{d}]", "hd", "sil", MergeNone, "aarch64_sve_sqxtnb", [IsStreamingCompatible]>; +def SVQXTNB_U : SInst<"svqxtnb[_{d}]", "hd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnb", [IsStreamingCompatible]>; +def SVQXTUNB_S : SInst<"svqxtunb[_{d}]", "ed", "sil", MergeNone, "aarch64_sve_sqxtunb", [IsStreamingCompatible]>; -def SVQXTNT_S : SInst<"svqxtnt[_{d}]", "hhd", "sil", MergeNone, "aarch64_sve_sqxtnt">; -def SVQXTNT_U : SInst<"svqxtnt[_{d}]", "hhd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnt">; -def SVQXTUNT_S : SInst<"svqxtunt[_{d}]", "eed", "sil", MergeNone, "aarch64_sve_sqxtunt">; +def SVQXTNT_S : SInst<"svqxtnt[_{d}]", "hhd", "sil", MergeNone, "aarch64_sve_sqxtnt", [IsStreamingCompatible]>; +def SVQXTNT_U : SInst<"svqxtnt[_{d}]", "hhd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnt", [IsStreamingCompatible]>; +def SVQXTUNT_S : SInst<"svqxtunt[_{d}]", "eed", "sil", MergeNone, "aarch64_sve_sqxtunt", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1802,18 +1814,18 @@ def SVSTNT1W_SCATTER_INDEX_S : MInst<"svstnt1w_scatter[_{2}base]_index[_{d}]", " // SVE2 - Polynomial arithmetic let TargetGuard = "sve2" in { -def SVEORBT : SInst<"sveorbt[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt">; -def SVEORBT_N : SInst<"sveorbt[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt">; -def SVEORTB : SInst<"sveortb[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb">; -def SVEORTB_N : SInst<"sveortb[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb">; -def SVPMUL : SInst<"svpmul[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_pmul">; -def SVPMUL_N : SInst<"svpmul[_n_{d}]", "dda", "Uc", MergeNone, "aarch64_sve_pmul">; -def SVPMULLB : SInst<"svpmullb[_{d}]", "dhh", "UsUl", MergeNone>; -def SVPMULLB_N : SInst<"svpmullb[_n_{d}]", "dhR", "UsUl", MergeNone>; -def SVPMULLB_PAIR : SInst<"svpmullb_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullb_pair">; -def SVPMULLB_PAIR_N : SInst<"svpmullb_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullb_pair">; -def SVPMULLT : SInst<"svpmullt[_{d}]", "dhh", "UsUl", MergeNone>; -def SVPMULLT_N : SInst<"svpmullt[_n_{d}]", "dhR", "UsUl", MergeNone>; +def SVEORBT : SInst<"sveorbt[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [IsStreamingCompatible]>; +def SVEORBT_N : SInst<"sveorbt[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [IsStreamingCompatible]>; +def SVEORTB : SInst<"sveortb[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [IsStreamingCompatible]>; +def SVEORTB_N : SInst<"sveortb[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [IsStreamingCompatible]>; +def SVPMUL : SInst<"svpmul[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_pmul", [IsStreamingCompatible]>; +def SVPMUL_N : SInst<"svpmul[_n_{d}]", "dda", "Uc", MergeNone, "aarch64_sve_pmul", [IsStreamingCompatible]>; +def SVPMULLB : SInst<"svpmullb[_{d}]", "dhh", "UsUl", MergeNone, "", [IsStreamingCompatible]>; +def SVPMULLB_N : SInst<"svpmullb[_n_{d}]", "dhR", "UsUl", MergeNone, "", [IsStreamingCompatible]>; +def SVPMULLB_PAIR : SInst<"svpmullb_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullb_pair", [IsStreamingCompatible]>; +def SVPMULLB_PAIR_N : SInst<"svpmullb_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullb_pair", [IsStreamingCompatible]>; +def SVPMULLT : SInst<"svpmullt[_{d}]", "dhh", "UsUl", MergeNone, "", [IsStreamingCompatible]>; +def SVPMULLT_N : SInst<"svpmullt[_n_{d}]", "dhR", "UsUl", MergeNone, "", [IsStreamingCompatible]>; def SVPMULLT_PAIR : SInst<"svpmullt_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullt_pair">; def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullt_pair">; } @@ -1822,8 +1834,8 @@ def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda", "UcUi", Mer // SVE2 - Complex integer dot product let TargetGuard = "sve2" in { -def SVCDOT : SInst<"svcdot[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_cdot", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>; -def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch64_sve_cdot_lane", [], [ImmCheck<4, ImmCheckComplexRotAll90>, +def SVCDOT : SInst<"svcdot[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_cdot", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRotAll90>]>; +def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch64_sve_cdot_lane", [IsStreamingCompatible], [ImmCheck<4, ImmCheckComplexRotAll90>, ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; } @@ -1831,27 +1843,27 @@ def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch // SVE2 - Floating-point widening multiply-accumulate let TargetGuard = "sve2" in { -def SVMLALB_F : SInst<"svmlalb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalb">; -def SVMLALB_F_N : SInst<"svmlalb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalb">; -def SVMLALB_F_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLALT_F : SInst<"svmlalt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalt">; -def SVMLALT_F_N : SInst<"svmlalt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalt">; -def SVMLALT_F_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLB_F : SInst<"svmlslb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslb">; -def SVMLSLB_F_N : SInst<"svmlslb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslb">; -def SVMLSLB_F_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLT_F : SInst<"svmlslt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslt">; -def SVMLSLT_F_N : SInst<"svmlslt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslt">; -def SVMLSLT_F_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALB_F : SInst<"svmlalb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalb", [IsStreamingCompatible]>; +def SVMLALB_F_N : SInst<"svmlalb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalb", [IsStreamingCompatible]>; +def SVMLALB_F_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALT_F : SInst<"svmlalt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalt", [IsStreamingCompatible]>; +def SVMLALT_F_N : SInst<"svmlalt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalt", [IsStreamingCompatible]>; +def SVMLALT_F_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLB_F : SInst<"svmlslb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslb", [IsStreamingCompatible]>; +def SVMLSLB_F_N : SInst<"svmlslb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslb", [IsStreamingCompatible]>; +def SVMLSLB_F_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLT_F : SInst<"svmlslt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslt", [IsStreamingCompatible]>; +def SVMLSLT_F_N : SInst<"svmlslt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslt", [IsStreamingCompatible]>; +def SVMLSLT_F_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Floating-point integer binary logarithm let TargetGuard = "sve2" in { -def SVLOGB_M : SInst<"svlogb[_{d}]", "xxPd", "hfd", MergeOp1, "aarch64_sve_flogb">; -def SVLOGB_X : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeAnyExp, "aarch64_sve_flogb">; -def SVLOGB_Z : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeZeroExp, "aarch64_sve_flogb">; +def SVLOGB_M : SInst<"svlogb[_{d}]", "xxPd", "hfd", MergeOp1, "aarch64_sve_flogb", [IsStreamingCompatible]>; +def SVLOGB_X : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeAnyExp, "aarch64_sve_flogb", [IsStreamingCompatible]>; +def SVLOGB_Z : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeZeroExp, "aarch64_sve_flogb", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1873,32 +1885,32 @@ def SVNMATCH : SInst<"svnmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve //////////////////////////////////////////////////////////////////////////////// // SVE2 - Contiguous conflict detection let TargetGuard = "sve2" in { -def SVWHILERW_B : SInst<"svwhilerw[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilerw_b", [IsOverloadWhileRW]>; -def SVWHILERW_H : SInst<"svwhilerw[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW]>; -def SVWHILERW_S : SInst<"svwhilerw[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilerw_s", [IsOverloadWhileRW]>; -def SVWHILERW_D : SInst<"svwhilerw[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilerw_d", [IsOverloadWhileRW]>; +def SVWHILERW_B : SInst<"svwhilerw[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilerw_b", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILERW_H : SInst<"svwhilerw[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILERW_S : SInst<"svwhilerw[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilerw_s", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILERW_D : SInst<"svwhilerw[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilerw_d", [IsOverloadWhileRW, IsStreamingCompatible]>; -def SVWHILEWR_B : SInst<"svwhilewr[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilewr_b", [IsOverloadWhileRW]>; -def SVWHILEWR_H : SInst<"svwhilewr[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW]>; -def SVWHILEWR_S : SInst<"svwhilewr[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilewr_s", [IsOverloadWhileRW]>; -def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilewr_d", [IsOverloadWhileRW]>; +def SVWHILEWR_B : SInst<"svwhilewr[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilewr_b", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILEWR_H : SInst<"svwhilewr[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILEWR_S : SInst<"svwhilewr[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilewr_s", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilewr_d", [IsOverloadWhileRW, IsStreamingCompatible]>; } let TargetGuard = "sve2,bf16" in { -def SVWHILERW_H_BF16 : SInst<"svwhilerw[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW]>; -def SVWHILEWR_H_BF16 : SInst<"svwhilewr[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW]>; +def SVWHILERW_H_BF16 : SInst<"svwhilerw[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILEWR_H_BF16 : SInst<"svwhilewr[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW, IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Extended table lookup/permute let TargetGuard = "sve2" in { -def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u", "csilUcUsUiUlhfd", MergeNone>; -def SVTBX : SInst<"svtbx[_{d}]", "dddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbx">; +def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u", "csilUcUsUiUlhfd", MergeNone, "", [IsStreamingCompatible]>; +def SVTBX : SInst<"svtbx[_{d}]", "dddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbx", [IsStreamingCompatible]>; } let TargetGuard = "sve2,bf16" in { -def SVTBL2_BF16 : SInst<"svtbl2[_{d}]", "d2u", "b", MergeNone>; -def SVTBX_BF16 : SInst<"svtbx[_{d}]", "dddu", "b", MergeNone, "aarch64_sve_tbx">; +def SVTBL2_BF16 : SInst<"svtbl2[_{d}]", "d2u", "b", MergeNone, "", [IsStreamingCompatible]>; +def SVTBX_BF16 : SInst<"svtbx[_{d}]", "dddu", "b", MergeNone, "aarch64_sve_tbx", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1967,109 +1979,83 @@ let TargetGuard = "sve2p1|sme2" in { //FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>; def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>; + +def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +} + +multiclass MultiVecLoad<string i> { + // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; } -let TargetGuard = "sve2p1" in { -def SVWHILEGE_COUNT : SInst<"svwhilege_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELE_COUNT : SInst<"svwhilele_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELT_COUNT : SInst<"svwhilelt_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELO_COUNT : SInst<"svwhilelo_{d}", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELS_COUNT : SInst<"svwhilels_{d}", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHI_COUNT : SInst<"svwhilehi_{d}", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHS_COUNT : SInst<"svwhilehs_{d}", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; - -def SVLD1B_X2 : MInst<"svld1[_{2}]_x2", "2}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1H_X2 : MInst<"svld1[_{2}]_x2", "2}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1W_X2 : MInst<"svld1[_{2}]_x2", "2}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1D_X2 : MInst<"svld1[_{2}]_x2", "2}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1B_X4 : MInst<"svld1[_{2}]_x4", "4}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1H_X4 : MInst<"svld1[_{2}]_x4", "4}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1W_X4 : MInst<"svld1[_{2}]_x4", "4}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1D_X4 : MInst<"svld1[_{2}]_x4", "4}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; - -def SVLDNT1B_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1H_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1W_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1D_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1B_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1H_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1W_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1D_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; - -def SVLD1B_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1H_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1W_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1D_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1B_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1H_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1W_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1D_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; - -def SVLDNT1B_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1H_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1W_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1D_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1B_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1H_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1W_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1D_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; - -def SVST1B_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1H_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1W_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1D_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1B_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1H_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1W_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1D_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; - -def SVST1B_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1H_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1W_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1D_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1B_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1H_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1W_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1D_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; - -def SVSTNT1B_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1H_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1W_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1D_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1B_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1H_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1W_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1D_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; - -def SVSTNT1B_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1H_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1W_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1D_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1B_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1H_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1W_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1D_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; +let TargetGuard = "sve2p1|sme2" in { + defm LD1 : MultiVecLoad<"ld1">; + defm LDNT1 : MultiVecLoad<"ldnt1">; +} + +multiclass MultiVecStore<string i> { + // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; +} +let TargetGuard = "sve2p1|sme2" in { + defm ST1 : MultiVecStore<"st1">; + defm STNT1 : MultiVecStore<"stnt1">; +} + +let TargetGuard = "sve2p1" in { def SVDOT_X2_S : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "i", MergeNone, "aarch64_sve_sdot_x2", [], []>; def SVDOT_X2_U : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "Ui", MergeNone, "aarch64_sve_udot_x2", [], []>; def SVDOT_X2_F : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "f", MergeNone, "aarch64_sve_fdot_x2", [], []>; def SVDOT_LANE_X2_S : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "i", MergeNone, "aarch64_sve_sdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "Ui", MergeNone, "aarch64_sve_udot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "f", MergeNone, "aarch64_sve_fdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; - -def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone], []>; -def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone], []>; - -def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; -def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; } -let TargetGuard = "sve2p1" in { +let TargetGuard = "sve2p1|sme" in { def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [], []>; def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>; -defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUl", "aarch64_sve_revd">; +defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">; } let TargetGuard = "sve2p1|sme2" in { @@ -2082,21 +2068,21 @@ let TargetGuard = "sve2p1|sme2" in { def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<1, ImmCheck2_4_Mul2>]>; } -let TargetGuard = "sve2p1,b16b16" in { -defm SVMUL_BF : SInstZPZZ<"svmul", "b", "aarch64_sve_fmul", "aarch64_sve_fmul_u">; -defm SVADD_BF : SInstZPZZ<"svadd", "b", "aarch64_sve_fadd", "aarch64_sve_fadd_u">; -defm SVSUB_BF : SInstZPZZ<"svsub", "b", "aarch64_sve_fsub", "aarch64_sve_fsub_u">; -defm SVMAXNM_BF : SInstZPZZ<"svmaxnm","b", "aarch64_sve_fmaxnm", "aarch64_sve_fmaxnm_u">; -defm SVMINNM_BF : SInstZPZZ<"svminnm","b", "aarch64_sve_fminnm", "aarch64_sve_fminnm_u">; -defm SVMAX_BF : SInstZPZZ<"svmax", "b", "aarch64_sve_fmax", "aarch64_sve_fmax_u">; -defm SVMIN_BF : SInstZPZZ<"svmin", "b", "aarch64_sve_fmin", "aarch64_sve_fmin_u">; -defm SVMLA_BF : SInstZPZZZ<"svmla", "b", "aarch64_sve_fmla", "aarch64_sve_fmla_u", []>; -defm SVMLS_BF : SInstZPZZZ<"svmls", "b", "aarch64_sve_fmls", "aarch64_sve_fmls_u", []>; -def SVMLA_LANE_BF : SInst<"svmla_lane[_{d}]", "ddddi", "b", MergeNone, "aarch64_sve_fmla_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLS_LANE_BF : SInst<"svmls_lane[_{d}]", "ddddi", "b", MergeNone, "aarch64_sve_fmls_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMUL_LANE_BF : SInst<"svmul_lane[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_fmul_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVFCLAMP_BF : SInst<"svclamp[_{d}]", "dddd", "b", MergeNone, "aarch64_sve_fclamp", [], []>; -} //sve2p1,b16b16 +let TargetGuard = "(sve2|sme2),b16b16" in { +defm SVMUL_BF : SInstZPZZ<"svmul", "b", "aarch64_sve_fmul", "aarch64_sve_fmul_u", [IsStreamingCompatible]>; +defm SVADD_BF : SInstZPZZ<"svadd", "b", "aarch64_sve_fadd", "aarch64_sve_fadd_u", [IsStreamingCompatible]>; +defm SVSUB_BF : SInstZPZZ<"svsub", "b", "aarch64_sve_fsub", "aarch64_sve_fsub_u", [IsStreamingCompatible]>; +defm SVMAXNM_BF : SInstZPZZ<"svmaxnm","b", "aarch64_sve_fmaxnm", "aarch64_sve_fmaxnm_u", [IsStreamingCompatible]>; +defm SVMINNM_BF : SInstZPZZ<"svminnm","b", "aarch64_sve_fminnm", "aarch64_sve_fminnm_u", [IsStreamingCompatible]>; +defm SVMAX_BF : SInstZPZZ<"svmax", "b", "aarch64_sve_fmax", "aarch64_sve_fmax_u", [IsStreamingCompatible]>; +defm SVMIN_BF : SInstZPZZ<"svmin", "b", "aarch64_sve_fmin", "aarch64_sve_fmin_u", [IsStreamingCompatible]>; +defm SVMLA_BF : SInstZPZZZ<"svmla", "b", "aarch64_sve_fmla", "aarch64_sve_fmla_u", [IsStreamingCompatible]>; +defm SVMLS_BF : SInstZPZZZ<"svmls", "b", "aarch64_sve_fmls", "aarch64_sve_fmls_u", [IsStreamingCompatible]>; +def SVMLA_LANE_BF : SInst<"svmla_lane[_{d}]", "ddddi", "b", MergeNone, "aarch64_sve_fmla_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLS_LANE_BF : SInst<"svmls_lane[_{d}]", "ddddi", "b", MergeNone, "aarch64_sve_fmls_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMUL_LANE_BF : SInst<"svmul_lane[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_fmul_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVFCLAMP_BF : SInst<"svclamp[_{d}]", "dddd", "b", MergeNone, "aarch64_sve_fclamp", [IsStreamingCompatible], []>; +} // SME2 @@ -2138,6 +2124,21 @@ let TargetGuard = "sme2" in { } let TargetGuard = "sme2" in { + // FRINTA / FRINTM / FRINTN / FRINTP + def SVRINTA_X2 : SInst<"svrinta[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frinta_x2", [IsStreaming], []>; + def SVRINTA_X4 : SInst<"svrinta[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frinta_x4", [IsStreaming], []>; + + def SVRINTM_X2 : SInst<"svrintm[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintm_x2", [IsStreaming], []>; + def SVRINTM_X4 : SInst<"svrintm[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintm_x4", [IsStreaming], []>; + + def SVRINTN_X2 : SInst<"svrintn[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintn_x2", [IsStreaming], []>; + def SVRINTN_X4 : SInst<"svrintn[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintn_x4", [IsStreaming], []>; + + def SVRINTP_X2 : SInst<"svrintp[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintp_x2", [IsStreaming], []>; + def SVRINTP_X4 : SInst<"svrintp[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintp_x4", [IsStreaming], []>; +} + +let TargetGuard = "sme2" in { def SVSCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "csil", MergeNone, "aarch64_sve_sclamp_single_x2", [IsStreaming], []>; def SVUCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp_single_x2", [IsStreaming], []>; def SVFCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "hfd", MergeNone, "aarch64_sve_fclamp_single_x2", [IsStreaming], []>; @@ -2297,6 +2298,22 @@ let TargetGuard = "sme2" in { } // +// Multi-vector zip/unzip +// + +let TargetGuard = "sme2" in { + def SVZIP_X2 : SInst<"svzip[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x2", [IsStreaming], []>; + def SVZIPQ_X2 : SInst<"svzipq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x2", [IsStreaming], []>; + def SVZIP_X4 : SInst<"svzip[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x4", [IsStreaming], []>; + def SVZIPQ_X4 : SInst<"svzipq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x4", [IsStreaming], []>; + + def SVUZP_X2 : SInst<"svuzp[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x2", [IsStreaming], []>; + def SVUZPQ_X2 : SInst<"svuzpq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x2", [IsStreaming], []>; + def SVUZP_X4 : SInst<"svuzp[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x4", [IsStreaming], []>; + def SVUZPQ_X4 : SInst<"svuzpq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x4", [IsStreaming], []>; +} + +// // Multi-vector unpack // @@ -2306,3 +2323,13 @@ let TargetGuard = "sme2" in { def SVSUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "sil", MergeNone, "aarch64_sve_sunpk_x4", [IsStreaming], []>; def SVUUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "UsUiUl", MergeNone, "aarch64_sve_uunpk_x4", [IsStreaming], []>; } + +let TargetGuard = "sve2p1|sme2" in { +// == BFloat16 multiply-subtract == +// FIXME: Make all of these IsStreamingOrSVE2p1 once that is added + def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingCompatible], []>; + def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingCompatible], []>; + + def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; +} diff --git a/contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td b/contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td index 682f1d5c8af6..f2dde7f540fb 100644 --- a/contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td +++ b/contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td @@ -2540,7 +2540,7 @@ multiclass RVVSignedWidenBinBuiltinSetVwsll let UnMaskedPolicyScheme = HasPassthruOperand in { // zvkb - let RequiredFeatures = ["Zvkb"] in { + let RequiredFeatures = ["Zvkb", "Experimental"] in { defm vandn : RVVUnsignedBinBuiltinSet; defm vbrev8 : RVVOutBuiltinSetZvbb; defm vrev8 : RVVOutBuiltinSetZvbb; @@ -2549,7 +2549,7 @@ let UnMaskedPolicyScheme = HasPassthruOperand in { } // zvbb - let RequiredFeatures = ["Zvbb"] in { + let RequiredFeatures = ["Zvbb", "Experimental"] in { defm vbrev : RVVOutBuiltinSetZvbb; defm vclz : RVVOutBuiltinSetZvbb; defm vctz : RVVOutBuiltinSetZvbb; @@ -2559,7 +2559,7 @@ let UnMaskedPolicyScheme = HasPassthruOperand in { } // zvbc - let RequiredFeatures = ["Zvbc"] in { + let RequiredFeatures = ["Zvbc", "Experimental"] in { defm vclmul : RVVInt64BinBuiltinSet; defm vclmulh : RVVInt64BinBuiltinSet; } @@ -2567,13 +2567,13 @@ let UnMaskedPolicyScheme = HasPassthruOperand in { let UnMaskedPolicyScheme = HasPolicyOperand, HasMasked = false in { // zvkg - let RequiredFeatures = ["Zvkg"] in { + let RequiredFeatures = ["Zvkg", "Experimental"] in { defm vghsh : RVVOutOp2BuiltinSetVVZvk; defm vgmul : RVVOutBuiltinSetZvk<HasVV=1, HasVS=0>; } // zvkned - let RequiredFeatures = ["Zvkned"] in { + let RequiredFeatures = ["Zvkned", "Experimental"] in { defm vaesdf : RVVOutBuiltinSetZvk; defm vaesdm : RVVOutBuiltinSetZvk; defm vaesef : RVVOutBuiltinSetZvk; @@ -2585,28 +2585,28 @@ let UnMaskedPolicyScheme = HasPolicyOperand, HasMasked = false in { } // zvknha - let RequiredFeatures = ["Zvknha"] in { + let RequiredFeatures = ["Zvknha", "Experimental"] in { defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"i">; defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"i">; defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"i">; } // zvknhb - let RequiredFeatures = ["Zvknhb"] in { + let RequiredFeatures = ["Zvknhb", "Experimental"] in { defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"il">; defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"il">; defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"il">; } // zvksed - let RequiredFeatures = ["Zvksed"] in { + let RequiredFeatures = ["Zvksed", "Experimental"] in { let UnMaskedPolicyScheme = HasPassthruOperand in defm vsm4k : RVVOutOp1BuiltinSet<"vsm4k", "i", [["vi", "Uv", "UvUvKz"]]>; defm vsm4r : RVVOutBuiltinSetZvk; } // zvksh - let RequiredFeatures = ["Zvksh"] in { + let RequiredFeatures = ["Zvksh", "Experimental"] in { defm vsm3c : RVVOutOp2BuiltinSetVIZvk; let UnMaskedPolicyScheme = HasPassthruOperand in defm vsm3me : RVVOutOp1BuiltinSet<"vsm3me", "i", [["vv", "Uv", "UvUvUv"]]>; diff --git a/contrib/llvm-project/clang/include/clang/Driver/Options.td b/contrib/llvm-project/clang/include/clang/Driver/Options.td index 1b02087425b7..2b93ddf03349 100644 --- a/contrib/llvm-project/clang/include/clang/Driver/Options.td +++ b/contrib/llvm-project/clang/include/clang/Driver/Options.td @@ -5308,7 +5308,8 @@ def rewrite_objc : Flag<["-"], "rewrite-objc">, Flags<[NoXarchOption]>, def rewrite_legacy_objc : Flag<["-"], "rewrite-legacy-objc">, Flags<[NoXarchOption]>, HelpText<"Rewrite Legacy Objective-C source to C++">; -def rdynamic : Flag<["-"], "rdynamic">, Group<Link_Group>; +def rdynamic : Flag<["-"], "rdynamic">, Group<Link_Group>, + Visibility<[ClangOption, FlangOption]>; def resource_dir : Separate<["-"], "resource-dir">, Flags<[NoXarchOption, HelpHidden]>, Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>, @@ -6999,6 +7000,8 @@ def msign_return_address_key_EQ : Joined<["-"], "msign-return-address-key=">, Values<"a_key,b_key">; def mbranch_target_enforce : Flag<["-"], "mbranch-target-enforce">, MarshallingInfoFlag<LangOpts<"BranchTargetEnforcement">>; +def mbranch_protection_pauth_lr : Flag<["-"], "mbranch-protection-pauth-lr">, + MarshallingInfoFlag<LangOpts<"BranchProtectionPAuthLR">>; def fno_dllexport_inlines : Flag<["-"], "fno-dllexport-inlines">, MarshallingInfoNegativeFlag<LangOpts<"DllExportInlines">>; def cfguard_no_checks : Flag<["-"], "cfguard-no-checks">, diff --git a/contrib/llvm-project/clang/include/clang/Interpreter/CodeCompletion.h b/contrib/llvm-project/clang/include/clang/Interpreter/CodeCompletion.h index 9adcdf0dc3af..c64aa899759f 100644 --- a/contrib/llvm-project/clang/include/clang/Interpreter/CodeCompletion.h +++ b/contrib/llvm-project/clang/include/clang/Interpreter/CodeCompletion.h @@ -23,8 +23,27 @@ namespace clang { class CodeCompletionResult; class CompilerInstance; -void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content, - unsigned Line, unsigned Col, const CompilerInstance *ParentCI, - std::vector<std::string> &CCResults); +struct ReplCodeCompleter { + ReplCodeCompleter() = default; + std::string Prefix; + + /// \param InterpCI [in] The compiler instance that is used to trigger code + /// completion + + /// \param Content [in] The string where code completion is triggered. + + /// \param Line [in] The line number of the code completion point. + + /// \param Col [in] The column number of the code completion point. + + /// \param ParentCI [in] The running interpreter compiler instance that + /// provides ASTContexts. + + /// \param CCResults [out] The completion results. + void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content, + unsigned Line, unsigned Col, + const CompilerInstance *ParentCI, + std::vector<std::string> &CCResults); +}; } // namespace clang #endif diff --git a/contrib/llvm-project/clang/include/clang/Interpreter/Interpreter.h b/contrib/llvm-project/clang/include/clang/Interpreter/Interpreter.h index 43573fb1a4b8..01858dfcc90a 100644 --- a/contrib/llvm-project/clang/include/clang/Interpreter/Interpreter.h +++ b/contrib/llvm-project/clang/include/clang/Interpreter/Interpreter.h @@ -101,6 +101,7 @@ public: const ASTContext &getASTContext() const; ASTContext &getASTContext(); const CompilerInstance *getCompilerInstance() const; + CompilerInstance *getCompilerInstance(); llvm::Expected<llvm::orc::LLJIT &> getExecutionEngine(); llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Code); diff --git a/contrib/llvm-project/clang/include/clang/Sema/Sema.h b/contrib/llvm-project/clang/include/clang/Sema/Sema.h index 20228da15ade..5e3b57ea3322 100644 --- a/contrib/llvm-project/clang/include/clang/Sema/Sema.h +++ b/contrib/llvm-project/clang/include/clang/Sema/Sema.h @@ -4799,8 +4799,6 @@ public: bool CheckAlwaysInlineAttr(const Stmt *OrigSt, const Stmt *CurSt, const AttributeCommonInfo &A); - bool CheckCountedByAttr(Scope *Scope, const FieldDecl *FD); - /// Adjust the calling convention of a method to be the ABI default if it /// wasn't specified explicitly. This handles method types formed from /// function type typedefs and typename template arguments. @@ -5644,7 +5642,6 @@ public: CorrectionCandidateCallback &CCC, TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr, ArrayRef<Expr *> Args = std::nullopt, - DeclContext *LookupCtx = nullptr, TypoExpr **Out = nullptr); DeclResult LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S, @@ -9357,8 +9354,7 @@ public: QualType DeduceTemplateSpecializationFromInitializer( TypeSourceInfo *TInfo, const InitializedEntity &Entity, - const InitializationKind &Kind, MultiExprArg Init, - ParenListExpr *PL = nullptr); + const InitializationKind &Kind, MultiExprArg Init); QualType deduceVarTypeFromInitializer(VarDecl *VDecl, DeclarationName Name, QualType Type, TypeSourceInfo *TSI, @@ -13851,6 +13847,7 @@ private: bool CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool ParseSVEImmChecks(CallExpr *TheCall, SmallVector<std::tuple<int, int, int>, 3> &ImmChecks); + bool CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool CheckCDEBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, CallExpr *TheCall); bool CheckARMCoprocessorImmediate(const TargetInfo &TI, const Expr *CoprocArg, diff --git a/contrib/llvm-project/clang/include/clang/Sema/TypoCorrection.h b/contrib/llvm-project/clang/include/clang/Sema/TypoCorrection.h index 09de164297e7..e0f8d152dbe5 100644 --- a/contrib/llvm-project/clang/include/clang/Sema/TypoCorrection.h +++ b/contrib/llvm-project/clang/include/clang/Sema/TypoCorrection.h @@ -282,7 +282,7 @@ class CorrectionCandidateCallback { public: static const unsigned InvalidDistance = TypoCorrection::InvalidDistance; - explicit CorrectionCandidateCallback(const IdentifierInfo *Typo = nullptr, + explicit CorrectionCandidateCallback(IdentifierInfo *Typo = nullptr, NestedNameSpecifier *TypoNNS = nullptr) : Typo(Typo), TypoNNS(TypoNNS) {} @@ -319,7 +319,7 @@ public: /// this method. virtual std::unique_ptr<CorrectionCandidateCallback> clone() = 0; - void setTypoName(const IdentifierInfo *II) { Typo = II; } + void setTypoName(IdentifierInfo *II) { Typo = II; } void setTypoNNS(NestedNameSpecifier *NNS) { TypoNNS = NNS; } // Flags for context-dependent keywords. WantFunctionLikeCasts is only @@ -345,13 +345,13 @@ protected: candidate.getCorrectionSpecifier() == TypoNNS; } - const IdentifierInfo *Typo; + IdentifierInfo *Typo; NestedNameSpecifier *TypoNNS; }; class DefaultFilterCCC final : public CorrectionCandidateCallback { public: - explicit DefaultFilterCCC(const IdentifierInfo *Typo = nullptr, + explicit DefaultFilterCCC(IdentifierInfo *Typo = nullptr, NestedNameSpecifier *TypoNNS = nullptr) : CorrectionCandidateCallback(Typo, TypoNNS) {} @@ -365,10 +365,6 @@ public: template <class C> class DeclFilterCCC final : public CorrectionCandidateCallback { public: - explicit DeclFilterCCC(const IdentifierInfo *Typo = nullptr, - NestedNameSpecifier *TypoNNS = nullptr) - : CorrectionCandidateCallback(Typo, TypoNNS) {} - bool ValidateCandidate(const TypoCorrection &candidate) override { return candidate.getCorrectionDeclAs<C>(); } diff --git a/contrib/llvm-project/clang/include/clang/Serialization/ASTReader.h b/contrib/llvm-project/clang/include/clang/Serialization/ASTReader.h index 59358e77edb0..21d791f5cd89 100644 --- a/contrib/llvm-project/clang/include/clang/Serialization/ASTReader.h +++ b/contrib/llvm-project/clang/include/clang/Serialization/ASTReader.h @@ -2422,6 +2422,8 @@ public: CurrentBitsIndex = 0; } + void advance(uint32_t BitsWidth) { CurrentBitsIndex += BitsWidth; } + bool getNextBit() { assert(isValid()); return Value & (1 << CurrentBitsIndex++); diff --git a/contrib/llvm-project/clang/include/clang/Serialization/ASTWriter.h b/contrib/llvm-project/clang/include/clang/Serialization/ASTWriter.h index a56929ef0245..de69f99003d8 100644 --- a/contrib/llvm-project/clang/include/clang/Serialization/ASTWriter.h +++ b/contrib/llvm-project/clang/include/clang/Serialization/ASTWriter.h @@ -564,11 +564,25 @@ private: unsigned DeclEnumAbbrev = 0; unsigned DeclObjCIvarAbbrev = 0; unsigned DeclCXXMethodAbbrev = 0; + unsigned DeclDependentNonTemplateCXXMethodAbbrev = 0; + unsigned DeclTemplateCXXMethodAbbrev = 0; + unsigned DeclMemberSpecializedCXXMethodAbbrev = 0; + unsigned DeclTemplateSpecializedCXXMethodAbbrev = 0; + unsigned DeclDependentSpecializationCXXMethodAbbrev = 0; + unsigned DeclTemplateTypeParmAbbrev = 0; + unsigned DeclUsingShadowAbbrev = 0; unsigned DeclRefExprAbbrev = 0; unsigned CharacterLiteralAbbrev = 0; unsigned IntegerLiteralAbbrev = 0; unsigned ExprImplicitCastAbbrev = 0; + unsigned BinaryOperatorAbbrev = 0; + unsigned CompoundAssignOperatorAbbrev = 0; + unsigned CallExprAbbrev = 0; + unsigned CXXOperatorCallExprAbbrev = 0; + unsigned CXXMemberCallExprAbbrev = 0; + + unsigned CompoundStmtAbbrev = 0; void WriteDeclAbbrevs(); void WriteDecl(ASTContext &Context, Decl *D); @@ -735,12 +749,41 @@ public: unsigned getDeclFieldAbbrev() const { return DeclFieldAbbrev; } unsigned getDeclEnumAbbrev() const { return DeclEnumAbbrev; } unsigned getDeclObjCIvarAbbrev() const { return DeclObjCIvarAbbrev; } - unsigned getDeclCXXMethodAbbrev() const { return DeclCXXMethodAbbrev; } + unsigned getDeclCXXMethodAbbrev(FunctionDecl::TemplatedKind Kind) const { + switch (Kind) { + case FunctionDecl::TK_NonTemplate: + return DeclCXXMethodAbbrev; + case FunctionDecl::TK_FunctionTemplate: + return DeclTemplateCXXMethodAbbrev; + case FunctionDecl::TK_MemberSpecialization: + return DeclMemberSpecializedCXXMethodAbbrev; + case FunctionDecl::TK_FunctionTemplateSpecialization: + return DeclTemplateSpecializedCXXMethodAbbrev; + case FunctionDecl::TK_DependentNonTemplate: + return DeclDependentNonTemplateCXXMethodAbbrev; + case FunctionDecl::TK_DependentFunctionTemplateSpecialization: + return DeclDependentSpecializationCXXMethodAbbrev; + } + llvm_unreachable("Unknwon Template Kind!"); + } + unsigned getDeclTemplateTypeParmAbbrev() const { + return DeclTemplateTypeParmAbbrev; + } + unsigned getDeclUsingShadowAbbrev() const { return DeclUsingShadowAbbrev; } unsigned getDeclRefExprAbbrev() const { return DeclRefExprAbbrev; } unsigned getCharacterLiteralAbbrev() const { return CharacterLiteralAbbrev; } unsigned getIntegerLiteralAbbrev() const { return IntegerLiteralAbbrev; } unsigned getExprImplicitCastAbbrev() const { return ExprImplicitCastAbbrev; } + unsigned getBinaryOperatorAbbrev() const { return BinaryOperatorAbbrev; } + unsigned getCompoundAssignOperatorAbbrev() const { + return CompoundAssignOperatorAbbrev; + } + unsigned getCallExprAbbrev() const { return CallExprAbbrev; } + unsigned getCXXOperatorCallExprAbbrev() { return CXXOperatorCallExprAbbrev; } + unsigned getCXXMemberCallExprAbbrev() { return CXXMemberCallExprAbbrev; } + + unsigned getCompoundStmtAbbrev() const { return CompoundStmtAbbrev; } bool hasChain() const { return Chain; } ASTReader *getChain() const { return Chain; } @@ -841,46 +884,33 @@ public: BitsPacker(BitsPacker &&) = delete; BitsPacker operator=(const BitsPacker &) = delete; BitsPacker operator=(BitsPacker &&) = delete; - ~BitsPacker() { - assert(!hasUnconsumedValues() && "There are unprocessed bits!"); + ~BitsPacker() = default; + + bool canWriteNextNBits(uint32_t BitsWidth) const { + return CurrentBitIndex + BitsWidth < BitIndexUpbound; + } + + void reset(uint32_t Value) { + UnderlyingValue = Value; + CurrentBitIndex = 0; } void addBit(bool Value) { addBits(Value, 1); } void addBits(uint32_t Value, uint32_t BitsWidth) { assert(BitsWidth < BitIndexUpbound); assert((Value < (1u << BitsWidth)) && "Passing narrower bit width!"); + assert(canWriteNextNBits(BitsWidth) && + "Inserting too much bits into a value!"); - if (CurrentBitIndex + BitsWidth >= BitIndexUpbound) { - Values.push_back(0); - CurrentBitIndex = 0; - } - - assert(CurrentBitIndex < BitIndexUpbound); - Values.back() |= Value << CurrentBitIndex; + UnderlyingValue |= Value << CurrentBitIndex; CurrentBitIndex += BitsWidth; } - bool hasUnconsumedValues() const { - return ConsumingValueIndex < Values.size(); - } - uint32_t getNextValue() { - assert(hasUnconsumedValues()); - return Values[ConsumingValueIndex++]; - } - - // We can convert the packer to an uint32_t if there is only one values. - operator uint32_t() { - assert(Values.size() == 1); - return getNextValue(); - } + operator uint32_t() { return UnderlyingValue; } private: - SmallVector<uint64_t, 4> Values; - uint16_t ConsumingValueIndex = 0; - // Initialize CurrentBitIndex with an invalid value - // to make it easier to update Values. See the implementation - // of `addBits` to see the details. - uint16_t CurrentBitIndex = BitIndexUpbound; + uint32_t UnderlyingValue = 0; + uint32_t CurrentBitIndex = 0; }; } // namespace clang diff --git a/contrib/llvm-project/clang/include/clang/Support/RISCVVIntrinsicUtils.h b/contrib/llvm-project/clang/include/clang/Support/RISCVVIntrinsicUtils.h index 49ce32553da8..c525d3443331 100644 --- a/contrib/llvm-project/clang/include/clang/Support/RISCVVIntrinsicUtils.h +++ b/contrib/llvm-project/clang/include/clang/Support/RISCVVIntrinsicUtils.h @@ -485,7 +485,7 @@ public: // RVVRequire should be sync'ed with target features, but only // required features used in riscv_vector.td. -enum RVVRequire : uint16_t { +enum RVVRequire : uint32_t { RVV_REQ_None = 0, RVV_REQ_RV64 = 1 << 0, RVV_REQ_ZvfhminOrZvfh = 1 << 1, @@ -503,8 +503,9 @@ enum RVVRequire : uint16_t { RVV_REQ_Zvknhb = 1 << 13, RVV_REQ_Zvksed = 1 << 14, RVV_REQ_Zvksh = 1 << 15, + RVV_REQ_Experimental = 1 << 16, - LLVM_MARK_AS_BITMASK_ENUM(RVV_REQ_Zvksh) + LLVM_MARK_AS_BITMASK_ENUM(RVV_REQ_Experimental) }; // Raw RVV intrinsic info, used to expand later. @@ -536,7 +537,7 @@ struct RVVIntrinsicRecord { uint8_t OverloadedSuffixSize; // Required target features for this intrinsic. - uint16_t RequiredExtensions; + uint32_t RequiredExtensions; // Supported type, mask of BasicType. uint8_t TypeRangeMask; diff --git a/contrib/llvm-project/clang/lib/APINotes/APINotesManager.cpp b/contrib/llvm-project/clang/lib/APINotes/APINotesManager.cpp index a921c8b9fce3..d3aef09dac91 100644 --- a/contrib/llvm-project/clang/lib/APINotes/APINotesManager.cpp +++ b/contrib/llvm-project/clang/lib/APINotes/APINotesManager.cpp @@ -125,7 +125,7 @@ APINotesManager::loadAPINotes(StringRef Buffer) { bool APINotesManager::loadAPINotes(const DirectoryEntry *HeaderDir, FileEntryRef APINotesFile) { - assert(Readers.find(HeaderDir) == Readers.end()); + assert(!Readers.contains(HeaderDir)); if (auto Reader = loadAPINotes(APINotesFile)) { Readers[HeaderDir] = Reader.release(); return false; diff --git a/contrib/llvm-project/clang/lib/AST/ASTImporter.cpp b/contrib/llvm-project/clang/lib/AST/ASTImporter.cpp index f1f335118f37..b61180c4f349 100644 --- a/contrib/llvm-project/clang/lib/AST/ASTImporter.cpp +++ b/contrib/llvm-project/clang/lib/AST/ASTImporter.cpp @@ -2771,9 +2771,11 @@ ASTNodeImporter::VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *D) { for (auto *FoundDecl : FoundDecls) { if (!FoundDecl->isInIdentifierNamespace(IDNS)) continue; - if (auto *FoundAlias = dyn_cast<TypeAliasTemplateDecl>(FoundDecl)) - return Importer.MapImported(D, FoundAlias); - ConflictingDecls.push_back(FoundDecl); + if (auto *FoundAlias = dyn_cast<TypeAliasTemplateDecl>(FoundDecl)) { + if (IsStructuralMatch(D, FoundAlias)) + return Importer.MapImported(D, FoundAlias); + ConflictingDecls.push_back(FoundDecl); + } } if (!ConflictingDecls.empty()) { @@ -3418,10 +3420,16 @@ static bool isAncestorDeclContextOf(const DeclContext *DC, const Stmt *S) { while (!ToProcess.empty()) { const Stmt *CurrentS = ToProcess.pop_back_val(); ToProcess.append(CurrentS->child_begin(), CurrentS->child_end()); - if (const auto *DeclRef = dyn_cast<DeclRefExpr>(CurrentS)) + if (const auto *DeclRef = dyn_cast<DeclRefExpr>(CurrentS)) { if (const Decl *D = DeclRef->getDecl()) if (isAncestorDeclContextOf(DC, D)) return true; + } else if (const auto *E = + dyn_cast_or_null<SubstNonTypeTemplateParmExpr>(CurrentS)) { + if (const Decl *D = E->getAssociatedDecl()) + if (isAncestorDeclContextOf(DC, D)) + return true; + } } return false; } @@ -7820,6 +7828,18 @@ ExpectedStmt ASTNodeImporter::VisitExplicitCastExpr(ExplicitCastExpr *E) { *ToLParenLocOrErr, OCE->getBridgeKind(), E->getCastKind(), *ToBridgeKeywordLocOrErr, ToTypeInfoAsWritten, ToSubExpr); } + case Stmt::BuiltinBitCastExprClass: { + auto *BBC = cast<BuiltinBitCastExpr>(E); + ExpectedSLoc ToKWLocOrErr = import(BBC->getBeginLoc()); + if (!ToKWLocOrErr) + return ToKWLocOrErr.takeError(); + ExpectedSLoc ToRParenLocOrErr = import(BBC->getEndLoc()); + if (!ToRParenLocOrErr) + return ToRParenLocOrErr.takeError(); + return new (Importer.getToContext()) BuiltinBitCastExpr( + ToType, E->getValueKind(), E->getCastKind(), ToSubExpr, + ToTypeInfoAsWritten, *ToKWLocOrErr, *ToRParenLocOrErr); + } default: llvm_unreachable("Cast expression of unsupported type!"); return make_error<ASTImportError>(ASTImportError::UnsupportedConstruct); @@ -9003,10 +9023,6 @@ class AttrImporter { public: AttrImporter(ASTImporter &I) : Importer(I), NImporter(I) {} - // Useful for accessing the imported attribute. - template <typename T> T *castAttrAs() { return cast<T>(ToAttr); } - template <typename T> const T *castAttrAs() const { return cast<T>(ToAttr); } - // Create an "importer" for an attribute parameter. // Result of the 'value()' of that object is to be passed to the function // 'importAttr', in the order that is expected by the attribute class. @@ -9101,6 +9117,12 @@ Expected<Attr *> ASTImporter::Import(const Attr *FromAttr) { break; } + case attr::AlignValue: { + auto *From = cast<AlignValueAttr>(FromAttr); + AI.importAttr(From, AI.importArg(From->getAlignment()).value()); + break; + } + case attr::Format: { const auto *From = cast<FormatAttr>(FromAttr); AI.importAttr(From, Import(From->getType()), From->getFormatIdx(), @@ -9214,15 +9236,6 @@ Expected<Attr *> ASTImporter::Import(const Attr *FromAttr) { From->args_size()); break; } - case attr::CountedBy: { - AI.cloneAttr(FromAttr); - const auto *CBA = cast<CountedByAttr>(FromAttr); - Expected<SourceRange> SR = Import(CBA->getCountedByFieldLoc()).get(); - if (!SR) - return SR.takeError(); - AI.castAttrAs<CountedByAttr>()->setCountedByFieldLoc(SR.get()); - break; - } default: { // The default branch works for attributes that have no arguments to import. @@ -9391,7 +9404,6 @@ Expected<Decl *> ASTImporter::Import(Decl *FromD) { setImportDeclError(FromD, *Error); return make_error<ASTImportError>(*Error); } - // Make sure that ImportImpl registered the imported decl. assert(ImportedDecls.count(FromD) != 0 && "Missing call to MapImported?"); if (auto Error = ImportAttrs(ToD, FromD)) diff --git a/contrib/llvm-project/clang/lib/AST/ASTStructuralEquivalence.cpp b/contrib/llvm-project/clang/lib/AST/ASTStructuralEquivalence.cpp index 6bb4bf14b873..1f492b051e03 100644 --- a/contrib/llvm-project/clang/lib/AST/ASTStructuralEquivalence.cpp +++ b/contrib/llvm-project/clang/lib/AST/ASTStructuralEquivalence.cpp @@ -1978,6 +1978,18 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, } static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, + TypeAliasTemplateDecl *D1, + TypeAliasTemplateDecl *D2) { + // Check template parameters. + if (!IsTemplateDeclCommonStructurallyEquivalent(Context, D1, D2)) + return false; + + // Check the templated declaration. + return IsStructurallyEquivalent(Context, D1->getTemplatedDecl(), + D2->getTemplatedDecl()); +} + +static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, ConceptDecl *D1, ConceptDecl *D2) { // Check template parameters. diff --git a/contrib/llvm-project/clang/lib/AST/Decl.cpp b/contrib/llvm-project/clang/lib/AST/Decl.cpp index 527ea6042daa..12e0a6faa4c3 100644 --- a/contrib/llvm-project/clang/lib/AST/Decl.cpp +++ b/contrib/llvm-project/clang/lib/AST/Decl.cpp @@ -1088,11 +1088,11 @@ bool NamedDecl::isPlaceholderVar(const LangOptions &LangOpts) const { return false; if (isa<FieldDecl>(this)) return true; - if (auto *IFD = dyn_cast<IndirectFieldDecl>(this)) { + if (const auto *IFD = dyn_cast<IndirectFieldDecl>(this)) { if (!getDeclContext()->isFunctionOrMethod() && !getDeclContext()->isRecord()) return false; - VarDecl *VD = IFD->getVarDecl(); + const VarDecl *VD = IFD->getVarDecl(); return !VD || VD->getStorageDuration() == SD_Automatic; } // and it declares a variable with automatic storage duration @@ -1105,7 +1105,7 @@ bool NamedDecl::isPlaceholderVar(const LangOptions &LangOpts) const { } if (const auto *BD = dyn_cast<BindingDecl>(this); BD && getDeclContext()->isFunctionOrMethod()) { - VarDecl *VD = BD->getHoldingVar(); + const VarDecl *VD = BD->getHoldingVar(); return !VD || VD->getStorageDuration() == StorageDuration::SD_Automatic; } return false; @@ -1843,7 +1843,8 @@ static bool isRedeclarable(Decl::Kind K) { llvm_unreachable("unknown decl kind"); } -bool NamedDecl::declarationReplaces(NamedDecl *OldD, bool IsKnownNewer) const { +bool NamedDecl::declarationReplaces(const NamedDecl *OldD, + bool IsKnownNewer) const { assert(getDeclName() == OldD->getDeclName() && "Declaration name mismatch"); // Never replace one imported declaration with another; we need both results @@ -1873,13 +1874,13 @@ bool NamedDecl::declarationReplaces(NamedDecl *OldD, bool IsKnownNewer) const { // Using declarations can be replaced if they import the same name from the // same context. - if (auto *UD = dyn_cast<UsingDecl>(this)) { + if (const auto *UD = dyn_cast<UsingDecl>(this)) { ASTContext &Context = getASTContext(); return Context.getCanonicalNestedNameSpecifier(UD->getQualifier()) == Context.getCanonicalNestedNameSpecifier( cast<UsingDecl>(OldD)->getQualifier()); } - if (auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(this)) { + if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(this)) { ASTContext &Context = getASTContext(); return Context.getCanonicalNestedNameSpecifier(UUVD->getQualifier()) == Context.getCanonicalNestedNameSpecifier( @@ -1896,7 +1897,7 @@ bool NamedDecl::declarationReplaces(NamedDecl *OldD, bool IsKnownNewer) const { // Check whether this is actually newer than OldD. We want to keep the // newer declaration. This loop will usually only iterate once, because // OldD is usually the previous declaration. - for (auto *D : redecls()) { + for (const auto *D : redecls()) { if (D == OldD) break; @@ -2199,8 +2200,7 @@ static LanguageLinkage getDeclLanguageLinkage(const T &D) { // Language linkage is a C++ concept, but saying that everything else in C has // C language linkage fits the implementation nicely. - ASTContext &Context = D.getASTContext(); - if (!Context.getLangOpts().CPlusPlus) + if (!D.getASTContext().getLangOpts().CPlusPlus) return CLanguageLinkage; // C++ [dcl.link]p4: A C language linkage is ignored in determining the @@ -2943,7 +2943,7 @@ bool ParmVarDecl::isDestroyedInCallee() const { // FIXME: isParamDestroyedInCallee() should probably imply // isDestructedType() - auto *RT = getType()->getAs<RecordType>(); + const auto *RT = getType()->getAs<RecordType>(); if (RT && RT->getDecl()->isParamDestroyedInCallee() && getType().isDestructedType()) return true; @@ -3105,7 +3105,7 @@ FunctionDecl::getDefaultedFunctionInfo() const { } bool FunctionDecl::hasBody(const FunctionDecl *&Definition) const { - for (auto *I : redecls()) { + for (const auto *I : redecls()) { if (I->doesThisDeclarationHaveABody()) { Definition = I; return true; @@ -3116,7 +3116,7 @@ bool FunctionDecl::hasBody(const FunctionDecl *&Definition) const { } bool FunctionDecl::hasTrivialBody() const { - Stmt *S = getBody(); + const Stmt *S = getBody(); if (!S) { // Since we don't have a body for this function, we don't know if it's // trivial or not. @@ -3212,7 +3212,7 @@ void FunctionDecl::setPure(bool P) { template<std::size_t Len> static bool isNamed(const NamedDecl *ND, const char (&Str)[Len]) { - IdentifierInfo *II = ND->getIdentifier(); + const IdentifierInfo *II = ND->getIdentifier(); return II && II->isStr(Str); } @@ -3305,9 +3305,9 @@ bool FunctionDecl::isReservedGlobalPlacementOperator() const { if (proto->getNumParams() != 2 || proto->isVariadic()) return false; - ASTContext &Context = - cast<TranslationUnitDecl>(getDeclContext()->getRedeclContext()) - ->getASTContext(); + const ASTContext &Context = + cast<TranslationUnitDecl>(getDeclContext()->getRedeclContext()) + ->getASTContext(); // The result type and first argument type are constant across all // these operators. The second argument must be exactly void*. @@ -3342,7 +3342,7 @@ bool FunctionDecl::isReplaceableGlobalAllocationFunction( unsigned Params = 1; QualType Ty = FPT->getParamType(Params); - ASTContext &Ctx = getASTContext(); + const ASTContext &Ctx = getASTContext(); auto Consume = [&] { ++Params; @@ -3388,7 +3388,8 @@ bool FunctionDecl::isReplaceableGlobalAllocationFunction( QualType T = Ty; while (const auto *TD = T->getAs<TypedefType>()) T = TD->getDecl()->getUnderlyingType(); - IdentifierInfo *II = T->castAs<EnumType>()->getDecl()->getIdentifier(); + const IdentifierInfo *II = + T->castAs<EnumType>()->getDecl()->getIdentifier(); if (II && II->isStr("__hot_cold_t")) Consume(); } @@ -3586,7 +3587,7 @@ unsigned FunctionDecl::getBuiltinID(bool ConsiderWrapperFunctions) const { (!hasAttr<ArmBuiltinAliasAttr>() && !hasAttr<BuiltinAliasAttr>())) return 0; - ASTContext &Context = getASTContext(); + const ASTContext &Context = getASTContext(); if (!Context.BuiltinInfo.isPredefinedLibFunction(BuiltinID)) return BuiltinID; @@ -3745,7 +3746,7 @@ bool FunctionDecl::doesDeclarationForceExternallyVisibleDefinition() const { assert(!doesThisDeclarationHaveABody() && "Must have a declaration without a body."); - ASTContext &Context = getASTContext(); + const ASTContext &Context = getASTContext(); if (Context.getLangOpts().MSVCCompat) { const FunctionDecl *Definition; diff --git a/contrib/llvm-project/clang/lib/AST/DeclBase.cpp b/contrib/llvm-project/clang/lib/AST/DeclBase.cpp index e4d7169752bc..5e03f0223d31 100644 --- a/contrib/llvm-project/clang/lib/AST/DeclBase.cpp +++ b/contrib/llvm-project/clang/lib/AST/DeclBase.cpp @@ -29,6 +29,7 @@ #include "clang/AST/Type.h" #include "clang/Basic/IdentifierTable.h" #include "clang/Basic/LLVM.h" +#include "clang/Basic/LangOptions.h" #include "clang/Basic/Module.h" #include "clang/Basic/ObjCRuntime.h" #include "clang/Basic/PartialDiagnostic.h" @@ -410,79 +411,6 @@ bool Decl::isFileContextDecl() const { return DC && DC->isFileContext(); } -bool Decl::isFlexibleArrayMemberLike( - ASTContext &Ctx, const Decl *D, QualType Ty, - LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, - bool IgnoreTemplateOrMacroSubstitution) { - // For compatibility with existing code, we treat arrays of length 0 or - // 1 as flexible array members. - const auto *CAT = Ctx.getAsConstantArrayType(Ty); - if (CAT) { - using FAMKind = LangOptions::StrictFlexArraysLevelKind; - - llvm::APInt Size = CAT->getSize(); - if (StrictFlexArraysLevel == FAMKind::IncompleteOnly) - return false; - - // GCC extension, only allowed to represent a FAM. - if (Size.isZero()) - return true; - - if (StrictFlexArraysLevel == FAMKind::ZeroOrIncomplete && Size.uge(1)) - return false; - - if (StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete && Size.uge(2)) - return false; - } else if (!Ctx.getAsIncompleteArrayType(Ty)) { - return false; - } - - if (const auto *OID = dyn_cast_if_present<ObjCIvarDecl>(D)) - return OID->getNextIvar() == nullptr; - - const auto *FD = dyn_cast_if_present<FieldDecl>(D); - if (!FD) - return false; - - if (CAT) { - // GCC treats an array memeber of a union as an FAM if the size is one or - // zero. - llvm::APInt Size = CAT->getSize(); - if (FD->getParent()->isUnion() && (Size.isZero() || Size.isOne())) - return true; - } - - // Don't consider sizes resulting from macro expansions or template argument - // substitution to form C89 tail-padded arrays. - if (IgnoreTemplateOrMacroSubstitution) { - TypeSourceInfo *TInfo = FD->getTypeSourceInfo(); - while (TInfo) { - TypeLoc TL = TInfo->getTypeLoc(); - - // Look through typedefs. - if (TypedefTypeLoc TTL = TL.getAsAdjusted<TypedefTypeLoc>()) { - const TypedefNameDecl *TDL = TTL.getTypedefNameDecl(); - TInfo = TDL->getTypeSourceInfo(); - continue; - } - - if (auto CTL = TL.getAs<ConstantArrayTypeLoc>()) { - if (const Expr *SizeExpr = - dyn_cast_if_present<IntegerLiteral>(CTL.getSizeExpr()); - !SizeExpr || SizeExpr->getExprLoc().isMacroID()) - return false; - } - - break; - } - } - - // Test that the field is the last in the structure. - RecordDecl::field_iterator FI( - DeclContext::decl_iterator(const_cast<FieldDecl *>(FD))); - return ++FI == FD->getParent()->field_end(); -} - TranslationUnitDecl *Decl::getTranslationUnitDecl() { if (auto *TUD = dyn_cast<TranslationUnitDecl>(this)) return TUD; diff --git a/contrib/llvm-project/clang/lib/AST/Expr.cpp b/contrib/llvm-project/clang/lib/AST/Expr.cpp index b125fc676da8..a90f92d07f86 100644 --- a/contrib/llvm-project/clang/lib/AST/Expr.cpp +++ b/contrib/llvm-project/clang/lib/AST/Expr.cpp @@ -205,22 +205,85 @@ bool Expr::isKnownToHaveBooleanValue(bool Semantic) const { } bool Expr::isFlexibleArrayMemberLike( - ASTContext &Ctx, + ASTContext &Context, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution) const { + + // For compatibility with existing code, we treat arrays of length 0 or + // 1 as flexible array members. + const auto *CAT = Context.getAsConstantArrayType(getType()); + if (CAT) { + llvm::APInt Size = CAT->getSize(); + + using FAMKind = LangOptions::StrictFlexArraysLevelKind; + + if (StrictFlexArraysLevel == FAMKind::IncompleteOnly) + return false; + + // GCC extension, only allowed to represent a FAM. + if (Size == 0) + return true; + + if (StrictFlexArraysLevel == FAMKind::ZeroOrIncomplete && Size.uge(1)) + return false; + + if (StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete && Size.uge(2)) + return false; + } else if (!Context.getAsIncompleteArrayType(getType())) + return false; + const Expr *E = IgnoreParens(); - const Decl *D = nullptr; - if (const auto *ME = dyn_cast<MemberExpr>(E)) - D = ME->getMemberDecl(); - else if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) - D = DRE->getDecl(); + const NamedDecl *ND = nullptr; + if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) + ND = DRE->getDecl(); + else if (const auto *ME = dyn_cast<MemberExpr>(E)) + ND = ME->getMemberDecl(); else if (const auto *IRE = dyn_cast<ObjCIvarRefExpr>(E)) - D = IRE->getDecl(); + return IRE->getDecl()->getNextIvar() == nullptr; + + if (!ND) + return false; - return Decl::isFlexibleArrayMemberLike(Ctx, D, E->getType(), - StrictFlexArraysLevel, - IgnoreTemplateOrMacroSubstitution); + // A flexible array member must be the last member in the class. + // FIXME: If the base type of the member expr is not FD->getParent(), + // this should not be treated as a flexible array member access. + if (const auto *FD = dyn_cast<FieldDecl>(ND)) { + // GCC treats an array memeber of a union as an FAM if the size is one or + // zero. + if (CAT) { + llvm::APInt Size = CAT->getSize(); + if (FD->getParent()->isUnion() && (Size.isZero() || Size.isOne())) + return true; + } + + // Don't consider sizes resulting from macro expansions or template argument + // substitution to form C89 tail-padded arrays. + if (IgnoreTemplateOrMacroSubstitution) { + TypeSourceInfo *TInfo = FD->getTypeSourceInfo(); + while (TInfo) { + TypeLoc TL = TInfo->getTypeLoc(); + // Look through typedefs. + if (TypedefTypeLoc TTL = TL.getAsAdjusted<TypedefTypeLoc>()) { + const TypedefNameDecl *TDL = TTL.getTypedefNameDecl(); + TInfo = TDL->getTypeSourceInfo(); + continue; + } + if (ConstantArrayTypeLoc CTL = TL.getAs<ConstantArrayTypeLoc>()) { + const Expr *SizeExpr = dyn_cast<IntegerLiteral>(CTL.getSizeExpr()); + if (!SizeExpr || SizeExpr->getExprLoc().isMacroID()) + return false; + } + break; + } + } + + RecordDecl::field_iterator FI( + DeclContext::decl_iterator(const_cast<FieldDecl *>(FD))); + return ++FI == FD->getParent()->field_end(); + } + + return false; } const ValueDecl * diff --git a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp index b98037b73645..96fe6df88dbb 100644 --- a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp +++ b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp @@ -726,27 +726,70 @@ void Environment::setStorageLocation(const Expr &E, StorageLocation &Loc) { // so allow these as an exception. assert(E.isGLValue() || E.getType()->isSpecificBuiltinType(BuiltinType::BuiltinFn)); - setStorageLocationInternal(E, Loc); + const Expr &CanonE = ignoreCFGOmittedNodes(E); + assert(!ExprToLoc.contains(&CanonE)); + ExprToLoc[&CanonE] = &Loc; } StorageLocation *Environment::getStorageLocation(const Expr &E) const { // See comment in `setStorageLocation()`. assert(E.isGLValue() || E.getType()->isSpecificBuiltinType(BuiltinType::BuiltinFn)); - return getStorageLocationInternal(E); + auto It = ExprToLoc.find(&ignoreCFGOmittedNodes(E)); + return It == ExprToLoc.end() ? nullptr : &*It->second; +} + +// Returns whether a prvalue of record type is the one that originally +// constructs the object (i.e. it doesn't propagate it from one of its +// children). +static bool isOriginalRecordConstructor(const Expr &RecordPRValue) { + if (auto *Init = dyn_cast<InitListExpr>(&RecordPRValue)) + return !Init->isSemanticForm() || !Init->isTransparent(); + return isa<CXXConstructExpr>(RecordPRValue) || isa<CallExpr>(RecordPRValue) || + isa<LambdaExpr>(RecordPRValue) || + // The framework currently does not propagate the objects created in + // the two branches of a `ConditionalOperator` because there is no way + // to reconcile their storage locations, which are different. We + // therefore claim that the `ConditionalOperator` is the expression + // that originally constructs the object. + // Ultimately, this will be fixed by propagating locations down from + // the result object, rather than up from the original constructor as + // we do now (see also the FIXME in the documentation for + // `getResultObjectLocation()`). + isa<ConditionalOperator>(RecordPRValue); } RecordStorageLocation & -Environment::getResultObjectLocation(const Expr &RecordPRValue) { +Environment::getResultObjectLocation(const Expr &RecordPRValue) const { assert(RecordPRValue.getType()->isRecordType()); assert(RecordPRValue.isPRValue()); - if (StorageLocation *ExistingLoc = getStorageLocationInternal(RecordPRValue)) - return *cast<RecordStorageLocation>(ExistingLoc); - auto &Loc = cast<RecordStorageLocation>( - DACtx->getStableStorageLocation(RecordPRValue)); - setStorageLocationInternal(RecordPRValue, Loc); - return Loc; + // Returns a storage location that we can use if assertions fail. + auto FallbackForAssertFailure = + [this, &RecordPRValue]() -> RecordStorageLocation & { + return cast<RecordStorageLocation>( + DACtx->getStableStorageLocation(RecordPRValue)); + }; + + if (isOriginalRecordConstructor(RecordPRValue)) { + auto *Val = cast_or_null<RecordValue>(getValue(RecordPRValue)); + // The builtin transfer function should have created a `RecordValue` for all + // original record constructors. + assert(Val); + if (!Val) + return FallbackForAssertFailure(); + return Val->getLoc(); + } + + // Expression nodes that propagate a record prvalue should have exactly one + // child. + llvm::SmallVector<const Stmt *> children(RecordPRValue.child_begin(), + RecordPRValue.child_end()); + assert(children.size() == 1); + if (children.empty()) + return FallbackForAssertFailure(); + + return getResultObjectLocation(*cast<Expr>(children[0])); } PointerValue &Environment::getOrCreateNullPointerValue(QualType PointeeType) { @@ -760,6 +803,11 @@ void Environment::setValue(const StorageLocation &Loc, Value &Val) { } void Environment::setValue(const Expr &E, Value &Val) { + if (auto *RecordVal = dyn_cast<RecordValue>(&Val)) { + assert(isOriginalRecordConstructor(E) || + &RecordVal->getLoc() == &getResultObjectLocation(E)); + } + assert(E.isPRValue()); ExprToVal[&E] = &Val; } @@ -799,18 +847,6 @@ Value *Environment::createValue(QualType Type) { return Val; } -void Environment::setStorageLocationInternal(const Expr &E, - StorageLocation &Loc) { - const Expr &CanonE = ignoreCFGOmittedNodes(E); - assert(!ExprToLoc.contains(&CanonE)); - ExprToLoc[&CanonE] = &Loc; -} - -StorageLocation *Environment::getStorageLocationInternal(const Expr &E) const { - auto It = ExprToLoc.find(&ignoreCFGOmittedNodes(E)); - return It == ExprToLoc.end() ? nullptr : &*It->second; -} - Value *Environment::createValueUnlessSelfReferential( QualType Type, llvm::DenseSet<QualType> &Visited, int Depth, int &CreatedValuesCount) { @@ -998,7 +1034,7 @@ RecordStorageLocation *getImplicitObjectLocation(const CXXMemberCallExpr &MCE, if (ImplicitObject == nullptr) return nullptr; if (ImplicitObject->getType()->isPointerType()) { - if (auto *Val = cast_or_null<PointerValue>(Env.getValue(*ImplicitObject))) + if (auto *Val = Env.get<PointerValue>(*ImplicitObject)) return &cast<RecordStorageLocation>(Val->getPointeeLoc()); return nullptr; } @@ -1012,11 +1048,11 @@ RecordStorageLocation *getBaseObjectLocation(const MemberExpr &ME, if (Base == nullptr) return nullptr; if (ME.isArrow()) { - if (auto *Val = cast_or_null<PointerValue>(Env.getValue(*Base))) + if (auto *Val = Env.get<PointerValue>(*Base)) return &cast<RecordStorageLocation>(Val->getPointeeLoc()); return nullptr; } - return cast_or_null<RecordStorageLocation>(Env.getStorageLocation(*Base)); + return Env.get<RecordStorageLocation>(*Base); } std::vector<FieldDecl *> getFieldsForInitListExpr(const RecordDecl *RD) { @@ -1041,9 +1077,10 @@ RecordValue &refreshRecordValue(const Expr &Expr, Environment &Env) { assert(Expr.getType()->isRecordType()); if (Expr.isPRValue()) { - if (auto *ExistingVal = cast_or_null<RecordValue>(Env.getValue(Expr))) { + if (auto *ExistingVal = Env.get<RecordValue>(Expr)) { auto &NewVal = Env.create<RecordValue>(ExistingVal->getLoc()); Env.setValue(Expr, NewVal); + Env.setValue(NewVal.getLoc(), NewVal); return NewVal; } @@ -1052,8 +1089,7 @@ RecordValue &refreshRecordValue(const Expr &Expr, Environment &Env) { return NewVal; } - if (auto *Loc = - cast_or_null<RecordStorageLocation>(Env.getStorageLocation(Expr))) { + if (auto *Loc = Env.get<RecordStorageLocation>(Expr)) { auto &NewVal = Env.create<RecordValue>(*Loc); Env.setValue(*Loc, NewVal); return NewVal; diff --git a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp index 69ac2c2b82cf..1d31b22b6d25 100644 --- a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp +++ b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp @@ -226,7 +226,7 @@ auto isComparisonOperatorCall(L lhs_arg_matcher, R rhs_arg_matcher) { /// Ensures that `Expr` is mapped to a `BoolValue` and returns its formula. const Formula &forceBoolValue(Environment &Env, const Expr &Expr) { - auto *Value = cast_or_null<BoolValue>(Env.getValue(Expr)); + auto *Value = Env.get<BoolValue>(Expr); if (Value != nullptr) return Value->formula(); @@ -267,7 +267,7 @@ BoolValue *getHasValue(Environment &Env, RecordStorageLocation *OptionalLoc) { if (OptionalLoc == nullptr) return nullptr; StorageLocation &HasValueLoc = locForHasValue(*OptionalLoc); - auto *HasValueVal = cast_or_null<BoolValue>(Env.getValue(HasValueLoc)); + auto *HasValueVal = Env.get<BoolValue>(HasValueLoc); if (HasValueVal == nullptr) { HasValueVal = &Env.makeAtomicBoolValue(); Env.setValue(HasValueLoc, *HasValueVal); @@ -406,7 +406,7 @@ void transferCallReturningOptional(const CallExpr *E, if (E->isPRValue()) { Loc = &State.Env.getResultObjectLocation(*E); } else { - Loc = cast_or_null<RecordStorageLocation>(State.Env.getStorageLocation(*E)); + Loc = State.Env.get<RecordStorageLocation>(*E); if (Loc == nullptr) { Loc = &cast<RecordStorageLocation>(State.Env.createStorageLocation(*E)); State.Env.setStorageLocation(*E, *Loc); @@ -449,8 +449,7 @@ BoolValue &valueOrConversionHasValue(const FunctionDecl &F, const Expr &E, // This is a constructor/assignment call for `optional<T>` with argument of // type `optional<U>` such that `T` is constructible from `U`. - auto *Loc = - cast_or_null<RecordStorageLocation>(State.Env.getStorageLocation(E)); + auto *Loc = State.Env.get<RecordStorageLocation>(E); if (auto *HasValueVal = getHasValue(State.Env, Loc)) return *HasValueVal; return State.Env.makeAtomicBoolValue(); @@ -471,8 +470,7 @@ void transferAssignment(const CXXOperatorCallExpr *E, BoolValue &HasValueVal, LatticeTransferState &State) { assert(E->getNumArgs() > 0); - if (auto *Loc = cast_or_null<RecordStorageLocation>( - State.Env.getStorageLocation(*E->getArg(0)))) { + if (auto *Loc = State.Env.get<RecordStorageLocation>(*E->getArg(0))) { createOptionalValue(*Loc, HasValueVal, State.Env); // Assign a storage location for the whole expression. @@ -534,18 +532,15 @@ void transferSwapCall(const CXXMemberCallExpr *E, const MatchFinder::MatchResult &, LatticeTransferState &State) { assert(E->getNumArgs() == 1); - auto *OtherLoc = cast_or_null<RecordStorageLocation>( - State.Env.getStorageLocation(*E->getArg(0))); + auto *OtherLoc = State.Env.get<RecordStorageLocation>(*E->getArg(0)); transferSwap(getImplicitObjectLocation(*E, State.Env), OtherLoc, State.Env); } void transferStdSwapCall(const CallExpr *E, const MatchFinder::MatchResult &, LatticeTransferState &State) { assert(E->getNumArgs() == 2); - auto *Arg0Loc = cast_or_null<RecordStorageLocation>( - State.Env.getStorageLocation(*E->getArg(0))); - auto *Arg1Loc = cast_or_null<RecordStorageLocation>( - State.Env.getStorageLocation(*E->getArg(1))); + auto *Arg0Loc = State.Env.get<RecordStorageLocation>(*E->getArg(0)); + auto *Arg1Loc = State.Env.get<RecordStorageLocation>(*E->getArg(1)); transferSwap(Arg0Loc, Arg1Loc, State.Env); } @@ -585,11 +580,9 @@ void transferOptionalAndOptionalCmp(const clang::CXXOperatorCallExpr *CmpExpr, Environment &Env = State.Env; auto &A = Env.arena(); auto *CmpValue = &forceBoolValue(Env, *CmpExpr); - auto *Arg0Loc = cast_or_null<RecordStorageLocation>( - Env.getStorageLocation(*CmpExpr->getArg(0))); + auto *Arg0Loc = Env.get<RecordStorageLocation>(*CmpExpr->getArg(0)); if (auto *LHasVal = getHasValue(Env, Arg0Loc)) { - auto *Arg1Loc = cast_or_null<RecordStorageLocation>( - Env.getStorageLocation(*CmpExpr->getArg(1))); + auto *Arg1Loc = Env.get<RecordStorageLocation>(*CmpExpr->getArg(1)); if (auto *RHasVal = getHasValue(Env, Arg1Loc)) { if (CmpExpr->getOperator() == clang::OO_ExclaimEqual) CmpValue = &A.makeNot(*CmpValue); @@ -603,7 +596,7 @@ void transferOptionalAndValueCmp(const clang::CXXOperatorCallExpr *CmpExpr, const clang::Expr *E, Environment &Env) { auto &A = Env.arena(); auto *CmpValue = &forceBoolValue(Env, *CmpExpr); - auto *Loc = cast_or_null<RecordStorageLocation>(Env.getStorageLocation(*E)); + auto *Loc = Env.get<RecordStorageLocation>(*E); if (auto *HasVal = getHasValue(Env, Loc)) { if (CmpExpr->getOperator() == clang::OO_ExclaimEqual) CmpValue = &A.makeNot(*CmpValue); @@ -616,7 +609,7 @@ void transferOptionalAndNulloptCmp(const clang::CXXOperatorCallExpr *CmpExpr, const clang::Expr *E, Environment &Env) { auto &A = Env.arena(); auto *CmpValue = &forceBoolValue(Env, *CmpExpr); - auto *Loc = cast_or_null<RecordStorageLocation>(Env.getStorageLocation(*E)); + auto *Loc = Env.get<RecordStorageLocation>(*E); if (auto *HasVal = getHasValue(Env, Loc)) { if (CmpExpr->getOperator() == clang::OO_ExclaimEqual) CmpValue = &A.makeNot(*CmpValue); diff --git a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/RecordOps.cpp b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/RecordOps.cpp index caaf443382b0..da4dd6dc0785 100644 --- a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/RecordOps.cpp +++ b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/RecordOps.cpp @@ -66,19 +66,8 @@ void clang::dataflow::copyRecord(RecordStorageLocation &Src, } } - RecordValue *SrcVal = cast_or_null<RecordValue>(Env.getValue(Src)); - RecordValue *DstVal = cast_or_null<RecordValue>(Env.getValue(Dst)); - - DstVal = &Env.create<RecordValue>(Dst); + RecordValue *DstVal = &Env.create<RecordValue>(Dst); Env.setValue(Dst, *DstVal); - - if (SrcVal == nullptr) - return; - - for (const auto &[Name, Value] : SrcVal->properties()) { - if (Value != nullptr) - DstVal->setProperty(Name, *Value); - } } bool clang::dataflow::recordsEqual(const RecordStorageLocation &Loc1, @@ -125,25 +114,5 @@ bool clang::dataflow::recordsEqual(const RecordStorageLocation &Loc1, } } - llvm::StringMap<Value *> Props1, Props2; - - if (RecordValue *Val1 = cast_or_null<RecordValue>(Env1.getValue(Loc1))) - for (const auto &[Name, Value] : Val1->properties()) - Props1[Name] = Value; - if (RecordValue *Val2 = cast_or_null<RecordValue>(Env2.getValue(Loc2))) - for (const auto &[Name, Value] : Val2->properties()) - Props2[Name] = Value; - - if (Props1.size() != Props2.size()) - return false; - - for (const auto &[Name, Value] : Props1) { - auto It = Props2.find(Name); - if (It == Props2.end()) - return false; - if (Value != It->second) - return false; - } - return true; } diff --git a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/Transfer.cpp b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/Transfer.cpp index bbf5f12359bc..55093c2e2cda 100644 --- a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/Transfer.cpp +++ b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/Transfer.cpp @@ -339,8 +339,7 @@ public: switch (S->getOpcode()) { case UO_Deref: { - const auto *SubExprVal = - cast_or_null<PointerValue>(Env.getValue(*SubExpr)); + const auto *SubExprVal = Env.get<PointerValue>(*SubExpr); if (SubExprVal == nullptr) break; @@ -467,8 +466,7 @@ public: const Expr *Arg = S->getArg(0); assert(Arg != nullptr); - auto *ArgLoc = - cast_or_null<RecordStorageLocation>(Env.getStorageLocation(*Arg)); + auto *ArgLoc = Env.get<RecordStorageLocation>(*Arg); if (ArgLoc == nullptr) return; @@ -489,7 +487,6 @@ public: if (S->getType()->isRecordType()) { auto &InitialVal = *cast<RecordValue>(Env.createValue(S->getType())); Env.setValue(*S, InitialVal); - copyRecord(InitialVal.getLoc(), Env.getResultObjectLocation(*S), Env); } transferInlineCall(S, ConstructorDecl); @@ -516,14 +513,12 @@ public: RecordStorageLocation *LocSrc = nullptr; if (Arg1->isPRValue()) { - if (auto *Val = cast_or_null<RecordValue>(Env.getValue(*Arg1))) + if (auto *Val = Env.get<RecordValue>(*Arg1)) LocSrc = &Val->getLoc(); } else { - LocSrc = - cast_or_null<RecordStorageLocation>(Env.getStorageLocation(*Arg1)); + LocSrc = Env.get<RecordStorageLocation>(*Arg1); } - auto *LocDst = - cast_or_null<RecordStorageLocation>(Env.getStorageLocation(*Arg0)); + auto *LocDst = Env.get<RecordStorageLocation>(*Arg0); if (LocSrc == nullptr || LocDst == nullptr) return; @@ -582,6 +577,14 @@ public: Env.setValue(*S, *ArgVal); } else if (const FunctionDecl *F = S->getDirectCallee()) { transferInlineCall(S, F); + + // If this call produces a prvalue of record type, make sure that we have + // a `RecordValue` for it. This is required so that + // `Environment::getResultObjectLocation()` is able to return a location + // for this `CallExpr`. + if (S->getType()->isRecordType() && S->isPRValue()) + if (Env.getValue(*S) == nullptr) + refreshRecordValue(*S, Env); } } @@ -669,7 +672,7 @@ public: auto Init = Inits[InitIdx++]; assert(Base.getType().getCanonicalType() == Init->getType().getCanonicalType()); - auto* BaseVal = cast_or_null<RecordValue>(Env.getValue(*Init)); + auto *BaseVal = Env.get<RecordValue>(*Init); if (!BaseVal) BaseVal = cast<RecordValue>(Env.createValue(Init->getType())); // Take ownership of the fields of the `RecordValue` for the base class diff --git a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp index 8c9360235da7..faf83a8920d4 100644 --- a/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp +++ b/contrib/llvm-project/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp @@ -130,7 +130,7 @@ private: if (Env.getValue(Cond) == nullptr) transfer(StmtToEnv, Cond, Env); - auto *Val = cast_or_null<BoolValue>(Env.getValue(Cond)); + auto *Val = Env.get<BoolValue>(Cond); // Value merging depends on flow conditions from different environments // being mutually exclusive -- that is, they cannot both be true in their // entirety (even if they may share some clauses). So, we need *some* value diff --git a/contrib/llvm-project/clang/lib/Basic/Targets/AArch64.cpp b/contrib/llvm-project/clang/lib/Basic/Targets/AArch64.cpp index def16c032c86..3ee39133fcee 100644 --- a/contrib/llvm-project/clang/lib/Basic/Targets/AArch64.cpp +++ b/contrib/llvm-project/clang/lib/Basic/Targets/AArch64.cpp @@ -225,6 +225,7 @@ bool AArch64TargetInfo::validateBranchProtection(StringRef Spec, StringRef, BPI.SignKey = LangOptions::SignReturnAddressKeyKind::BKey; BPI.BranchTargetEnforcement = PBP.BranchTargetEnforcement; + BPI.BranchProtectionPAuthLR = PBP.BranchProtectionPAuthLR; return true; } diff --git a/contrib/llvm-project/clang/lib/Basic/Targets/ARM.cpp b/contrib/llvm-project/clang/lib/Basic/Targets/ARM.cpp index ce7e4d4639ce..6e1842fc64e5 100644 --- a/contrib/llvm-project/clang/lib/Basic/Targets/ARM.cpp +++ b/contrib/llvm-project/clang/lib/Basic/Targets/ARM.cpp @@ -419,6 +419,7 @@ bool ARMTargetInfo::validateBranchProtection(StringRef Spec, StringRef Arch, BPI.SignKey = LangOptions::SignReturnAddressKeyKind::AKey; BPI.BranchTargetEnforcement = PBP.BranchTargetEnforcement; + BPI.BranchProtectionPAuthLR = PBP.BranchProtectionPAuthLR; return true; } diff --git a/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.cpp b/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.cpp index 60a4e0ed69c3..685462961ee3 100644 --- a/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.cpp +++ b/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.cpp @@ -350,6 +350,7 @@ bool RISCVTargetInfo::hasFeature(StringRef Feature) const { .Case("riscv64", Is64Bit) .Case("32bit", !Is64Bit) .Case("64bit", Is64Bit) + .Case("experimental", HasExperimental) .Default(std::nullopt); if (Result) return *Result; @@ -382,6 +383,9 @@ bool RISCVTargetInfo::handleTargetFeatures(std::vector<std::string> &Features, FastUnalignedAccess = llvm::is_contained(Features, "+fast-unaligned-access"); + if (llvm::is_contained(Features, "+experimental")) + HasExperimental = true; + return true; } diff --git a/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.h b/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.h index a893cae914ce..f98c88cd45f8 100644 --- a/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.h +++ b/contrib/llvm-project/clang/lib/Basic/Targets/RISCV.h @@ -31,6 +31,7 @@ protected: private: bool FastUnalignedAccess; + bool HasExperimental = false; public: RISCVTargetInfo(const llvm::Triple &Triple, const TargetOptions &) diff --git a/contrib/llvm-project/clang/lib/Basic/Version.cpp b/contrib/llvm-project/clang/lib/Basic/Version.cpp index e205da7adec1..4823f566bd77 100644 --- a/contrib/llvm-project/clang/lib/Basic/Version.cpp +++ b/contrib/llvm-project/clang/lib/Basic/Version.cpp @@ -57,6 +57,14 @@ std::string getLLVMRevision() { #endif } +std::string getClangVendor() { +#ifdef CLANG_VENDOR + return CLANG_VENDOR; +#else + return ""; +#endif +} + std::string getClangFullRepositoryVersion() { std::string buf; llvm::raw_string_ostream OS(buf); @@ -92,10 +100,7 @@ std::string getClangFullVersion() { std::string getClangToolFullVersion(StringRef ToolName) { std::string buf; llvm::raw_string_ostream OS(buf); -#ifdef CLANG_VENDOR - OS << CLANG_VENDOR; -#endif - OS << ToolName << " version " CLANG_VERSION_STRING; + OS << getClangVendor() << ToolName << " version " CLANG_VERSION_STRING; std::string repo = getClangFullRepositoryVersion(); if (!repo.empty()) { @@ -110,10 +115,7 @@ std::string getClangFullCPPVersion() { // the one we report on the command line. std::string buf; llvm::raw_string_ostream OS(buf); -#ifdef CLANG_VENDOR - OS << CLANG_VENDOR; -#endif - OS << "Clang " CLANG_VERSION_STRING; + OS << getClangVendor() << "Clang " CLANG_VERSION_STRING; std::string repo = getClangFullRepositoryVersion(); if (!repo.empty()) { diff --git a/contrib/llvm-project/clang/lib/Basic/Warnings.cpp b/contrib/llvm-project/clang/lib/Basic/Warnings.cpp index cb23d844ef8f..bab1af4f03b6 100644 --- a/contrib/llvm-project/clang/lib/Basic/Warnings.cpp +++ b/contrib/llvm-project/clang/lib/Basic/Warnings.cpp @@ -198,8 +198,7 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags, } } - for (unsigned i = 0, e = Opts.Remarks.size(); i != e; ++i) { - StringRef Opt = Opts.Remarks[i]; + for (StringRef Opt : Opts.Remarks) { const auto Flavor = diag::Flavor::Remark; // Check to see if this warning starts with "no-", if so, this is a diff --git a/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp b/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp index 7d16de33763a..a6142d99f3b6 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp @@ -881,7 +881,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline( << PluginFN << toString(PassPlugin.takeError()); } } - for (auto PassCallback : CodeGenOpts.PassBuilderCallbacks) + for (const auto &PassCallback : CodeGenOpts.PassBuilderCallbacks) PassCallback(PB); #define HANDLE_EXTENSION(Ext) \ get##Ext##PluginInfo().RegisterPassBuilderCallbacks(PB); @@ -1068,11 +1068,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline( } } if (CodeGenOpts.FatLTO) { - // Set module flags, like EnableSplitLTOUnit and UnifiedLTO, since FatLTO + // Set the EnableSplitLTOUnit and UnifiedLTO module flags, since FatLTO // uses a different action than Backend_EmitBC or Backend_EmitLL. - if (!TheModule->getModuleFlag("ThinLTO")) - TheModule->addModuleFlag(llvm::Module::Error, "ThinLTO", - uint32_t(CodeGenOpts.PrepareForThinLTO)); if (!TheModule->getModuleFlag("EnableSplitLTOUnit")) TheModule->addModuleFlag(llvm::Module::Error, "EnableSplitLTOUnit", uint32_t(CodeGenOpts.EnableSplitLTOUnit)); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp index 3327866d2b96..5081062da286 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp @@ -25,7 +25,6 @@ #include "clang/AST/Attr.h" #include "clang/AST/Decl.h" #include "clang/AST/OSLog.h" -#include "clang/AST/OperationKinds.h" #include "clang/Basic/TargetBuiltins.h" #include "clang/Basic/TargetInfo.h" #include "clang/Basic/TargetOptions.h" @@ -232,19 +231,19 @@ static Value *MakeBinaryAtomicValue( static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) { Value *Val = CGF.EmitScalarExpr(E->getArg(0)); - Value *Address = CGF.EmitScalarExpr(E->getArg(1)); + Address Addr = CGF.EmitPointerWithAlignment(E->getArg(1)); Val = CGF.EmitToMemory(Val, E->getArg(0)->getType()); - LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getArg(0)->getType()); + LValue LV = CGF.MakeAddrLValue(Addr, E->getArg(0)->getType()); LV.setNontemporal(true); CGF.EmitStoreOfScalar(Val, LV, false); return nullptr; } static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) { - Value *Address = CGF.EmitScalarExpr(E->getArg(0)); + Address Addr = CGF.EmitPointerWithAlignment(E->getArg(0)); - LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType()); + LValue LV = CGF.MakeAddrLValue(Addr, E->getType()); LV.setNontemporal(true); return CGF.EmitLoadOfScalar(LV, E->getExprLoc()); } @@ -819,165 +818,6 @@ CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type, return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true); } -llvm::Value * -CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type, - llvm::IntegerType *ResType) { - // The code generated here calculates the size of a struct with a flexible - // array member that uses the counted_by attribute. There are two instances - // we handle: - // - // struct s { - // unsigned long flags; - // int count; - // int array[] __attribute__((counted_by(count))); - // } - // - // 1) bdos of the flexible array itself: - // - // __builtin_dynamic_object_size(p->array, 1) == - // p->count * sizeof(*p->array) - // - // 2) bdos of a pointer into the flexible array: - // - // __builtin_dynamic_object_size(&p->array[42], 1) == - // (p->count - 42) * sizeof(*p->array) - // - // 2) bdos of the whole struct, including the flexible array: - // - // __builtin_dynamic_object_size(p, 1) == - // max(sizeof(struct s), - // offsetof(struct s, array) + p->count * sizeof(*p->array)) - // - ASTContext &Ctx = getContext(); - const Expr *Base = E->IgnoreParenImpCasts(); - const Expr *Idx = nullptr; - - if (const auto *UO = dyn_cast<UnaryOperator>(Base); - UO && UO->getOpcode() == UO_AddrOf) { - Expr *SubExpr = UO->getSubExpr()->IgnoreParenImpCasts(); - if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) { - Base = ASE->getBase()->IgnoreParenImpCasts(); - Idx = ASE->getIdx()->IgnoreParenImpCasts(); - - if (const auto *IL = dyn_cast<IntegerLiteral>(Idx)) { - int64_t Val = IL->getValue().getSExtValue(); - if (Val < 0) - // __bdos returns 0 for negative indexes into an array in a struct. - return getDefaultBuiltinObjectSizeResult(Type, ResType); - - if (Val == 0) - // The index is 0, so we don't need to take it into account. - Idx = nullptr; - } - } else { - // Potential pointer to another element in the struct. - Base = SubExpr; - } - } - - // Get the flexible array member Decl. - const ValueDecl *FAMDecl = nullptr; - if (const auto *ME = dyn_cast<MemberExpr>(Base)) { - // Check if \p Base is referencing the FAM itself. - if (const ValueDecl *MD = ME->getMemberDecl()) { - const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = - getLangOpts().getStrictFlexArraysLevel(); - if (!Decl::isFlexibleArrayMemberLike( - Ctx, MD, MD->getType(), StrictFlexArraysLevel, - /*IgnoreTemplateOrMacroSubstitution=*/true)) - return nullptr; - - FAMDecl = MD; - } - } else if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) { - // Check if we're pointing to the whole struct. - QualType Ty = DRE->getDecl()->getType(); - if (Ty->isPointerType()) - Ty = Ty->getPointeeType(); - - if (const auto *RD = Ty->getAsRecordDecl()) - // Don't use the outer lexical record because the FAM might be in a - // different RecordDecl. - FAMDecl = FindFlexibleArrayMemberField(Ctx, RD); - } - - if (!FAMDecl || !FAMDecl->hasAttr<CountedByAttr>()) - // No flexible array member found or it doesn't have the "counted_by" - // attribute. - return nullptr; - - const ValueDecl *CountedByFD = FindCountedByField(Base); - if (!CountedByFD) - // Can't find the field referenced by the "counted_by" attribute. - return nullptr; - - // Build a load of the counted_by field. - bool IsSigned = CountedByFD->getType()->isSignedIntegerType(); - const Expr *CountedByExpr = BuildCountedByFieldExpr(Base, CountedByFD); - Value *CountedByInst = EmitAnyExprToTemp(CountedByExpr).getScalarVal(); - llvm::Type *CountedByTy = CountedByInst->getType(); - - // Build a load of the index and subtract it from the count. - Value *IdxInst = nullptr; - if (Idx) { - bool IdxSigned = Idx->getType()->isSignedIntegerType(); - IdxInst = EmitAnyExprToTemp(Idx).getScalarVal(); - IdxInst = IdxSigned ? Builder.CreateSExtOrTrunc(IdxInst, CountedByTy) - : Builder.CreateZExtOrTrunc(IdxInst, CountedByTy); - - // We go ahead with the calculation here. If the index turns out to be - // negative, we'll catch it at the end. - CountedByInst = - Builder.CreateSub(CountedByInst, IdxInst, "", !IsSigned, IsSigned); - } - - // Calculate how large the flexible array member is in bytes. - const ArrayType *ArrayTy = Ctx.getAsArrayType(FAMDecl->getType()); - CharUnits Size = Ctx.getTypeSizeInChars(ArrayTy->getElementType()); - llvm::Constant *ElemSize = - llvm::ConstantInt::get(CountedByTy, Size.getQuantity(), IsSigned); - Value *FAMSize = - Builder.CreateMul(CountedByInst, ElemSize, "", !IsSigned, IsSigned); - FAMSize = IsSigned ? Builder.CreateSExtOrTrunc(FAMSize, ResType) - : Builder.CreateZExtOrTrunc(FAMSize, ResType); - Value *Res = FAMSize; - - if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) { - // The whole struct is specificed in the __bdos. - const RecordDecl *OuterRD = - CountedByFD->getDeclContext()->getOuterLexicalRecordContext(); - const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(OuterRD); - - // Get the offset of the FAM. - CharUnits Offset = Ctx.toCharUnitsFromBits(Ctx.getFieldOffset(FAMDecl)); - llvm::Constant *FAMOffset = - ConstantInt::get(ResType, Offset.getQuantity(), IsSigned); - Value *OffsetAndFAMSize = - Builder.CreateAdd(FAMOffset, Res, "", !IsSigned, IsSigned); - - // Get the full size of the struct. - llvm::Constant *SizeofStruct = - ConstantInt::get(ResType, Layout.getSize().getQuantity(), IsSigned); - - // max(sizeof(struct s), - // offsetof(struct s, array) + p->count * sizeof(*p->array)) - Res = IsSigned - ? Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax, - OffsetAndFAMSize, SizeofStruct) - : Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax, - OffsetAndFAMSize, SizeofStruct); - } - - // A negative \p IdxInst or \p CountedByInst means that the index lands - // outside of the flexible array member. If that's the case, we want to - // return 0. - Value *Cmp = Builder.CreateIsNotNeg(CountedByInst); - if (IdxInst) - Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(IdxInst), Cmp); - - return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned)); -} - /// Returns a Value corresponding to the size of the given expression. /// This Value may be either of the following: /// - A llvm::Argument (if E is a param with the pass_object_size attribute on @@ -1010,13 +850,6 @@ CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type, } } - if (IsDynamic) { - // Emit special code for a flexible array member with the "counted_by" - // attribute. - if (Value *V = emitFlexibleArrayMemberSize(E, Type, ResType)) - return V; - } - // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't // evaluate E for side-effects. In either case, we shouldn't lower to // @llvm.objectsize. @@ -3214,7 +3047,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue); if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment)) - AlignmentCI = ConstantInt::get(AlignmentCI->getType(), + AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(), llvm::Value::MaximumAlignment); emitAlignmentAssumption(PtrValue, Ptr, @@ -10485,6 +10318,30 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, return nullptr; } +static void swapCommutativeSMEOperands(unsigned BuiltinID, + SmallVectorImpl<Value *> &Ops) { + unsigned MultiVec; + switch (BuiltinID) { + default: + return; + case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1: + MultiVec = 1; + break; + case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2: + case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2: + MultiVec = 2; + break; + case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4: + case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4: + MultiVec = 4; + break; + } + + if (MultiVec > 0) + for (unsigned I = 0; I < MultiVec; ++I) + std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]); +} + Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E) { auto *Builtin = findARMVectorIntrinsicInMap(AArch64SMEIntrinsicMap, BuiltinID, @@ -10507,6 +10364,9 @@ Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, BuiltinID == SME::BI__builtin_sme_svstr_za) return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic); + // Handle builtins which require their multi-vector operands to be swapped + swapCommutativeSMEOperands(BuiltinID, Ops); + // Should not happen! if (Builtin->LLVMIntrinsic == 0) return nullptr; @@ -17034,7 +16894,7 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, Value *Op1 = EmitScalarExpr(E->getArg(1)); ConstantInt *AlignmentCI = cast<ConstantInt>(Op0); if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment)) - AlignmentCI = ConstantInt::get(AlignmentCI->getType(), + AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(), llvm::Value::MaximumAlignment); emitAlignmentAssumption(Op1, E->getArg(1), @@ -17272,7 +17132,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, Op0, llvm::FixedVectorType::get(ConvertType(E->getType()), 2)); if (getTarget().isLittleEndian()) - Index = ConstantInt::get(Index->getType(), 1 - Index->getZExtValue()); + Index = + ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue()); return Builder.CreateExtractElement(Unpacked, Index); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp index ed9aaa28c257..3f277725d9e7 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp @@ -30,7 +30,6 @@ #include "clang/Basic/CodeGenOptions.h" #include "clang/Basic/SourceManager.h" #include "llvm/ADT/Hashing.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Intrinsics.h" @@ -926,27 +925,16 @@ static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF, if (CE->getCastKind() == CK_ArrayToPointerDecay && !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(), StrictFlexArraysLevel)) { - CodeGenFunction::SanitizerScope SanScope(&CGF); - IndexedType = CE->getSubExpr()->getType(); const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe(); if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) return CGF.Builder.getInt(CAT->getSize()); - - if (const auto *VAT = dyn_cast<VariableArrayType>(AT)) + else if (const auto *VAT = dyn_cast<VariableArrayType>(AT)) return CGF.getVLASize(VAT).NumElts; // Ignore pass_object_size here. It's not applicable on decayed pointers. } - - if (const ValueDecl *VD = CGF.FindCountedByField(Base)) { - IndexedType = Base->getType(); - const Expr *E = CGF.BuildCountedByFieldExpr(Base, VD); - return CGF.EmitAnyExprToTemp(E).getScalarVal(); - } } - CodeGenFunction::SanitizerScope SanScope(&CGF); - QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0}; if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) { IndexedType = Base->getType(); @@ -956,122 +944,13 @@ static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF, return nullptr; } -const Expr * -CodeGenFunction::BuildCountedByFieldExpr(const Expr *Base, - const ValueDecl *CountedByVD) { - // Find the outer struct expr (i.e. p in p->a.b.c.d). - Expr *CountedByExpr = const_cast<Expr *>(Base)->IgnoreParenImpCasts(); - - // Work our way up the expression until we reach the DeclRefExpr. - while (!isa<DeclRefExpr>(CountedByExpr)) - if (const auto *ME = dyn_cast<MemberExpr>(CountedByExpr)) - CountedByExpr = ME->getBase()->IgnoreParenImpCasts(); - - // Add back an implicit cast to create the required pr-value. - CountedByExpr = ImplicitCastExpr::Create( - getContext(), CountedByExpr->getType(), CK_LValueToRValue, CountedByExpr, - nullptr, VK_PRValue, FPOptionsOverride()); - - if (const auto *IFD = dyn_cast<IndirectFieldDecl>(CountedByVD)) { - // The counted_by field is inside an anonymous struct / union. The - // IndirectFieldDecl has the correct order of FieldDecls to build this - // easily. (Yay!) - for (NamedDecl *ND : IFD->chain()) { - auto *VD = cast<ValueDecl>(ND); - CountedByExpr = - MemberExpr::CreateImplicit(getContext(), CountedByExpr, - CountedByExpr->getType()->isPointerType(), - VD, VD->getType(), VK_LValue, OK_Ordinary); - } - } else { - CountedByExpr = MemberExpr::CreateImplicit( - getContext(), const_cast<Expr *>(CountedByExpr), - CountedByExpr->getType()->isPointerType(), - const_cast<ValueDecl *>(CountedByVD), CountedByVD->getType(), VK_LValue, - OK_Ordinary); - } - - return CountedByExpr; -} - -const ValueDecl * -CodeGenFunction::FindFlexibleArrayMemberField(ASTContext &Ctx, - const RecordDecl *RD) { - const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = - getLangOpts().getStrictFlexArraysLevel(); - - for (const Decl *D : RD->decls()) { - if (const auto *VD = dyn_cast<ValueDecl>(D); - VD && Decl::isFlexibleArrayMemberLike( - Ctx, VD, VD->getType(), StrictFlexArraysLevel, - /*IgnoreTemplateOrMacroSubstitution=*/true)) - return VD; - - if (const auto *Record = dyn_cast<RecordDecl>(D)) - if (const ValueDecl *VD = FindFlexibleArrayMemberField(Ctx, Record)) - return VD; - } - - return nullptr; -} - -const ValueDecl *CodeGenFunction::FindCountedByField(const Expr *Base) { - ASTContext &Ctx = getContext(); - const RecordDecl *OuterRD = nullptr; - const FieldDecl *FD = nullptr; - - Base = Base->IgnoreParenImpCasts(); - - // Get the outer-most lexical RecordDecl. - if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) { - QualType Ty = DRE->getDecl()->getType(); - if (Ty->isPointerType()) - Ty = Ty->getPointeeType(); - - if (const auto *RD = Ty->getAsRecordDecl()) - OuterRD = RD->getOuterLexicalRecordContext(); - } else if (const auto *ME = dyn_cast<MemberExpr>(Base)) { - if (const ValueDecl *MD = ME->getMemberDecl()) { - OuterRD = MD->getDeclContext()->getOuterLexicalRecordContext(); - - const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = - getLangOpts().getStrictFlexArraysLevel(); - if (Decl::isFlexibleArrayMemberLike( - Ctx, MD, MD->getType(), StrictFlexArraysLevel, - /*IgnoreTemplateOrMacroSubstitution=*/true)) - // Base is referencing the FAM itself. - FD = dyn_cast<FieldDecl>(MD); - } - } - - if (!OuterRD) - return nullptr; - - if (!FD) { - const ValueDecl *VD = FindFlexibleArrayMemberField(Ctx, OuterRD); - FD = dyn_cast_if_present<FieldDecl>(VD); - if (!FD) - return nullptr; - } - - const auto *CBA = FD->getAttr<CountedByAttr>(); - if (!CBA) - return nullptr; - - DeclarationName DName(CBA->getCountedByField()); - DeclContext::lookup_result Lookup = OuterRD->lookup(DName); - - if (Lookup.empty()) - return nullptr; - - return dyn_cast<ValueDecl>(Lookup.front()); -} - void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index, QualType IndexType, bool Accessed) { assert(SanOpts.has(SanitizerKind::ArrayBounds) && "should not be called unless adding bounds checks"); + SanitizerScope SanScope(this); + const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = getLangOpts().getStrictFlexArraysLevel(); @@ -1081,8 +960,6 @@ void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base, if (!Bound) return; - SanitizerScope SanScope(this); - bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType(); llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned); llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp index 41ad2ddac30d..6adf99531e30 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp @@ -1894,8 +1894,8 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { // initializer, since LLVM optimizers generally do not want to touch // shuffles. unsigned CurIdx = 0; - bool VIsUndefShuffle = false; - llvm::Value *V = llvm::UndefValue::get(VType); + bool VIsPoisonShuffle = false; + llvm::Value *V = llvm::PoisonValue::get(VType); for (unsigned i = 0; i != NumInitElements; ++i) { Expr *IE = E->getInit(i); Value *Init = Visit(IE); @@ -1915,16 +1915,16 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { llvm::ConstantInt *C = cast<llvm::ConstantInt>(EI->getIndexOperand()); Value *LHS = nullptr, *RHS = nullptr; if (CurIdx == 0) { - // insert into undef -> shuffle (src, undef) + // insert into poison -> shuffle (src, poison) // shufflemask must use an i32 Args.push_back(getAsInt32(C, CGF.Int32Ty)); Args.resize(ResElts, -1); LHS = EI->getVectorOperand(); RHS = V; - VIsUndefShuffle = true; - } else if (VIsUndefShuffle) { - // insert into undefshuffle && size match -> shuffle (v, src) + VIsPoisonShuffle = true; + } else if (VIsPoisonShuffle) { + // insert into poison shuffle && size match -> shuffle (v, src) llvm::ShuffleVectorInst *SVV = cast<llvm::ShuffleVectorInst>(V); for (unsigned j = 0; j != CurIdx; ++j) Args.push_back(getMaskElt(SVV, j, 0)); @@ -1933,7 +1933,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { LHS = cast<llvm::ShuffleVectorInst>(V)->getOperand(0); RHS = EI->getVectorOperand(); - VIsUndefShuffle = false; + VIsPoisonShuffle = false; } if (!Args.empty()) { V = Builder.CreateShuffleVector(LHS, RHS, Args); @@ -1944,7 +1944,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { } V = Builder.CreateInsertElement(V, Init, Builder.getInt32(CurIdx), "vecinit"); - VIsUndefShuffle = false; + VIsPoisonShuffle = false; ++CurIdx; continue; } @@ -1962,9 +1962,9 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { if (OpTy->getNumElements() == ResElts) { for (unsigned j = 0; j != CurIdx; ++j) { - // If the current vector initializer is a shuffle with undef, merge + // If the current vector initializer is a shuffle with poison, merge // this shuffle directly into it. - if (VIsUndefShuffle) { + if (VIsPoisonShuffle) { Args.push_back(getMaskElt(cast<llvm::ShuffleVectorInst>(V), j, 0)); } else { Args.push_back(j); @@ -1974,7 +1974,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { Args.push_back(getMaskElt(SVI, j, Offset)); Args.resize(ResElts, -1); - if (VIsUndefShuffle) + if (VIsPoisonShuffle) V = cast<llvm::ShuffleVectorInst>(V)->getOperand(0); Init = SVOp; @@ -1997,12 +1997,12 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { Args.resize(ResElts, -1); } - // If V is undef, make sure it ends up on the RHS of the shuffle to aid + // If V is poison, make sure it ends up on the RHS of the shuffle to aid // merging subsequent shuffles into this one. if (CurIdx == 0) std::swap(V, Init); V = Builder.CreateShuffleVector(V, Init, Args, "vecinit"); - VIsUndefShuffle = isa<llvm::UndefValue>(Init); + VIsPoisonShuffle = isa<llvm::PoisonValue>(Init); CurIdx += InitElts; } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp index 3e8a40e7540b..e887d35198b3 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp @@ -182,10 +182,8 @@ void CGHLSLRuntime::finishCodeGen() { llvm::hlsl::ResourceKind RK = Buf.IsCBuffer ? llvm::hlsl::ResourceKind::CBuffer : llvm::hlsl::ResourceKind::TBuffer; - std::string TyName = - Buf.Name.str() + (Buf.IsCBuffer ? ".cb." : ".tb.") + "ty"; - addBufferResourceAnnotation(GV, TyName, RC, RK, /*IsROV=*/false, - Buf.Binding); + addBufferResourceAnnotation(GV, RC, RK, /*IsROV=*/false, + llvm::hlsl::ElementType::Invalid, Buf.Binding); } } @@ -194,10 +192,10 @@ CGHLSLRuntime::Buffer::Buffer(const HLSLBufferDecl *D) Binding(D->getAttr<HLSLResourceBindingAttr>()) {} void CGHLSLRuntime::addBufferResourceAnnotation(llvm::GlobalVariable *GV, - llvm::StringRef TyName, llvm::hlsl::ResourceClass RC, llvm::hlsl::ResourceKind RK, bool IsROV, + llvm::hlsl::ElementType ET, BufferResBinding &Binding) { llvm::Module &M = CGM.getModule(); @@ -216,15 +214,62 @@ void CGHLSLRuntime::addBufferResourceAnnotation(llvm::GlobalVariable *GV, assert(false && "Unsupported buffer type!"); return; } - assert(ResourceMD != nullptr && "ResourceMD must have been set by the switch above."); llvm::hlsl::FrontendResource Res( - GV, TyName, RK, IsROV, Binding.Reg.value_or(UINT_MAX), Binding.Space); + GV, RK, ET, IsROV, Binding.Reg.value_or(UINT_MAX), Binding.Space); ResourceMD->addOperand(Res.getMetadata()); } +static llvm::hlsl::ElementType +calculateElementType(const ASTContext &Context, const clang::Type *ResourceTy) { + using llvm::hlsl::ElementType; + + // TODO: We may need to update this when we add things like ByteAddressBuffer + // that don't have a template parameter (or, indeed, an element type). + const auto *TST = ResourceTy->getAs<TemplateSpecializationType>(); + assert(TST && "Resource types must be template specializations"); + ArrayRef<TemplateArgument> Args = TST->template_arguments(); + assert(!Args.empty() && "Resource has no element type"); + + // At this point we have a resource with an element type, so we can assume + // that it's valid or we would have diagnosed the error earlier. + QualType ElTy = Args[0].getAsType(); + + // We should either have a basic type or a vector of a basic type. + if (const auto *VecTy = ElTy->getAs<clang::VectorType>()) + ElTy = VecTy->getElementType(); + + if (ElTy->isSignedIntegerType()) { + switch (Context.getTypeSize(ElTy)) { + case 16: + return ElementType::I16; + case 32: + return ElementType::I32; + case 64: + return ElementType::I64; + } + } else if (ElTy->isUnsignedIntegerType()) { + switch (Context.getTypeSize(ElTy)) { + case 16: + return ElementType::U16; + case 32: + return ElementType::U32; + case 64: + return ElementType::U64; + } + } else if (ElTy->isSpecificBuiltinType(BuiltinType::Half)) + return ElementType::F16; + else if (ElTy->isSpecificBuiltinType(BuiltinType::Float)) + return ElementType::F32; + else if (ElTy->isSpecificBuiltinType(BuiltinType::Double)) + return ElementType::F64; + + // TODO: We need to handle unorm/snorm float types here once we support them + llvm_unreachable("Invalid element type for resource"); +} + void CGHLSLRuntime::annotateHLSLResource(const VarDecl *D, GlobalVariable *GV) { const Type *Ty = D->getType()->getPointeeOrArrayElementType(); if (!Ty) @@ -239,10 +284,10 @@ void CGHLSLRuntime::annotateHLSLResource(const VarDecl *D, GlobalVariable *GV) { llvm::hlsl::ResourceClass RC = Attr->getResourceClass(); llvm::hlsl::ResourceKind RK = Attr->getResourceKind(); bool IsROV = Attr->getIsROV(); + llvm::hlsl::ElementType ET = calculateElementType(CGM.getContext(), Ty); - QualType QT(Ty, 0); BufferResBinding Binding(D->getAttr<HLSLResourceBindingAttr>()); - addBufferResourceAnnotation(GV, QT.getAsString(), RC, RK, IsROV, Binding); + addBufferResourceAnnotation(GV, RC, RK, IsROV, ET, Binding); } CGHLSLRuntime::BufferResBinding::BufferResBinding( diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h b/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h index bb500cb5c979..bffefb66740a 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h @@ -90,9 +90,9 @@ public: private: void addBufferResourceAnnotation(llvm::GlobalVariable *GV, - llvm::StringRef TyName, llvm::hlsl::ResourceClass RC, llvm::hlsl::ResourceKind RK, bool IsROV, + llvm::hlsl::ElementType ET, BufferResBinding &Binding); void addConstant(VarDecl *D, Buffer &CB); void addBufferDecls(const DeclContext *DC, Buffer &CB); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 7f7e6f530666..ea6645a39e83 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -6811,8 +6811,10 @@ private: OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers, ArrayRef<OpenMPMotionModifierKind> MotionModifiers, OMPClauseMappableExprCommon::MappableExprComponentListRef Components, - MapCombinedInfoTy &CombinedInfo, StructRangeInfoTy &PartialStruct, - bool IsFirstComponentList, bool IsImplicit, + MapCombinedInfoTy &CombinedInfo, + MapCombinedInfoTy &StructBaseCombinedInfo, + StructRangeInfoTy &PartialStruct, bool IsFirstComponentList, + bool IsImplicit, bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr, bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr, const Expr *MapExpr = nullptr, ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef> @@ -7098,6 +7100,25 @@ private: bool IsNonContiguous = CombinedInfo.NonContigInfo.IsNonContiguous; bool IsPrevMemberReference = false; + // We need to check if we will be encountering any MEs. If we do not + // encounter any ME expression it means we will be mapping the whole struct. + // In that case we need to skip adding an entry for the struct to the + // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo + // list only when generating all info for clauses. + bool IsMappingWholeStruct = true; + if (!GenerateAllInfoForClauses) { + IsMappingWholeStruct = false; + } else { + for (auto TempI = I; TempI != CE; ++TempI) { + const MemberExpr *PossibleME = + dyn_cast<MemberExpr>(TempI->getAssociatedExpression()); + if (PossibleME) { + IsMappingWholeStruct = false; + break; + } + } + } + for (; I != CE; ++I) { // If the current component is member of a struct (parent struct) mark it. if (!EncounteredME) { @@ -7317,21 +7338,41 @@ private: break; } llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression()); + // Skip adding an entry in the CurInfo of this combined entry if the + // whole struct is currently being mapped. The struct needs to be added + // in the first position before any data internal to the struct is being + // mapped. if (!IsMemberPointerOrAddr || (Next == CE && MapType != OMPC_MAP_unknown)) { - CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); - CombinedInfo.DevicePtrDecls.push_back(nullptr); - CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); - CombinedInfo.Sizes.push_back( - CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); - CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize - : 1); + if (!IsMappingWholeStruct) { + CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); + CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.DevicePtrDecls.push_back(nullptr); + CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); + CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( + Size, CGF.Int64Ty, /*isSigned=*/true)); + CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize + : 1); + } else { + StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); + StructBaseCombinedInfo.BasePointers.push_back(BP.getPointer()); + StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr); + StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); + StructBaseCombinedInfo.Pointers.push_back(LB.getPointer()); + StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( + Size, CGF.Int64Ty, /*isSigned=*/true)); + StructBaseCombinedInfo.NonContigInfo.Dims.push_back( + IsNonContiguous ? DimSize : 1); + } // If Mapper is valid, the last component inherits the mapper. bool HasMapper = Mapper && Next == CE; - CombinedInfo.Mappers.push_back(HasMapper ? Mapper : nullptr); + if (!IsMappingWholeStruct) + CombinedInfo.Mappers.push_back(HasMapper ? Mapper : nullptr); + else + StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper + : nullptr); // We need to add a pointer flag for each map that comes from the // same expression except for the first one. We also need to signal @@ -7363,7 +7404,10 @@ private: } } - CombinedInfo.Types.push_back(Flags); + if (!IsMappingWholeStruct) + CombinedInfo.Types.push_back(Flags); + else + StructBaseCombinedInfo.Types.push_back(Flags); } // If we have encountered a member expression so far, keep track of the @@ -7954,8 +7998,10 @@ private: for (const auto &Data : Info) { StructRangeInfoTy PartialStruct; - // Temporary generated information. + // Current struct information: MapCombinedInfoTy CurInfo; + // Current struct base information: + MapCombinedInfoTy StructBaseCurInfo; const Decl *D = Data.first; const ValueDecl *VD = cast_or_null<ValueDecl>(D); for (const auto &M : Data.second) { @@ -7965,29 +8011,55 @@ private: // Remember the current base pointer index. unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size(); + unsigned StructBasePointersIdx = + StructBaseCurInfo.BasePointers.size(); CurInfo.NonContigInfo.IsNonContiguous = L.Components.back().isNonContiguous(); generateInfoForComponentList( L.MapType, L.MapModifiers, L.MotionModifiers, L.Components, - CurInfo, PartialStruct, /*IsFirstComponentList=*/false, - L.IsImplicit, L.Mapper, L.ForDeviceAddr, VD, L.VarRef); + CurInfo, StructBaseCurInfo, PartialStruct, + /*IsFirstComponentList=*/false, L.IsImplicit, + /*GenerateAllInfoForClauses*/ true, L.Mapper, L.ForDeviceAddr, VD, + L.VarRef); - // If this entry relates with a device pointer, set the relevant + // If this entry relates to a device pointer, set the relevant // declaration and add the 'return pointer' flag. if (L.ReturnDevicePointer) { - assert(CurInfo.BasePointers.size() > CurrentBasePointersIdx && + // Check whether a value was added to either CurInfo or + // StructBaseCurInfo and error if no value was added to either of + // them: + assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() || + StructBasePointersIdx < + StructBaseCurInfo.BasePointers.size()) && "Unexpected number of mapped base pointers."); + // Choose a base pointer index which is always valid: const ValueDecl *RelevantVD = L.Components.back().getAssociatedDeclaration(); assert(RelevantVD && "No relevant declaration related with device pointer??"); - CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD; - CurInfo.DevicePointers[CurrentBasePointersIdx] = - L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer; - CurInfo.Types[CurrentBasePointersIdx] |= - OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; + // If StructBaseCurInfo has been updated this iteration then work on + // the first new entry added to it i.e. make sure that when multiple + // values are added to any of the lists, the first value added is + // being modified by the assignments below (not the last value + // added). + if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) { + StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] = + RelevantVD; + StructBaseCurInfo.DevicePointers[StructBasePointersIdx] = + L.ForDeviceAddr ? DeviceInfoTy::Address + : DeviceInfoTy::Pointer; + StructBaseCurInfo.Types[StructBasePointersIdx] |= + OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; + } else { + CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD; + CurInfo.DevicePointers[CurrentBasePointersIdx] = + L.ForDeviceAddr ? DeviceInfoTy::Address + : DeviceInfoTy::Pointer; + CurInfo.Types[CurrentBasePointersIdx] |= + OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; + } } } } @@ -8034,17 +8106,24 @@ private: CurInfo.Mappers.push_back(nullptr); } } + + // Unify entries in one list making sure the struct mapping precedes the + // individual fields: + MapCombinedInfoTy UnionCurInfo; + UnionCurInfo.append(StructBaseCurInfo); + UnionCurInfo.append(CurInfo); + // If there is an entry in PartialStruct it means we have a struct with // individual members mapped. Emit an extra combined entry. if (PartialStruct.Base.isValid()) { - CurInfo.NonContigInfo.Dims.push_back(0); - emitCombinedEntry(CombinedInfo, CurInfo.Types, PartialStruct, + UnionCurInfo.NonContigInfo.Dims.push_back(0); + // Emit a combined entry: + emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct, /*IsMapThis*/ !VD, OMPBuilder, VD); } - // We need to append the results of this capture to what we already - // have. - CombinedInfo.append(CurInfo); + // We need to append the results of this capture to what we already have. + CombinedInfo.append(UnionCurInfo); } // Append data for use_device_ptr clauses. CombinedInfo.append(UseDeviceDataCombinedInfo); @@ -8554,6 +8633,7 @@ public: // Associated with a capture, because the mapping flags depend on it. // Go through all of the elements with the overlapped elements. bool IsFirstComponentList = true; + MapCombinedInfoTy StructBaseCombinedInfo; for (const auto &Pair : OverlappedData) { const MapData &L = *Pair.getFirst(); OMPClauseMappableExprCommon::MappableExprComponentListRef Components; @@ -8568,7 +8648,8 @@ public: OverlappedComponents = Pair.getSecond(); generateInfoForComponentList( MapType, MapModifiers, std::nullopt, Components, CombinedInfo, - PartialStruct, IsFirstComponentList, IsImplicit, Mapper, + StructBaseCombinedInfo, PartialStruct, IsFirstComponentList, + IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper, /*ForDeviceAddr=*/false, VD, VarRef, OverlappedComponents); IsFirstComponentList = false; } @@ -8584,10 +8665,11 @@ public: L; auto It = OverlappedData.find(&L); if (It == OverlappedData.end()) - generateInfoForComponentList(MapType, MapModifiers, std::nullopt, - Components, CombinedInfo, PartialStruct, - IsFirstComponentList, IsImplicit, Mapper, - /*ForDeviceAddr=*/false, VD, VarRef); + generateInfoForComponentList( + MapType, MapModifiers, std::nullopt, Components, CombinedInfo, + StructBaseCombinedInfo, PartialStruct, IsFirstComponentList, + IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper, + /*ForDeviceAddr=*/false, VD, VarRef); IsFirstComponentList = false; } } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h index 618e78809db4..751d8110b13d 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h @@ -3022,19 +3022,6 @@ public: void EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index, QualType IndexType, bool Accessed); - // Find a struct's flexible array member. It may be embedded inside multiple - // sub-structs, but must still be the last field. - const ValueDecl *FindFlexibleArrayMemberField(ASTContext &Ctx, - const RecordDecl *RD); - - /// Find the FieldDecl specified in a FAM's "counted_by" attribute. Returns - /// \p nullptr if either the attribute or the field doesn't exist. - const ValueDecl *FindCountedByField(const Expr *Base); - - /// Build an expression accessing the "counted_by" field. - const Expr *BuildCountedByFieldExpr(const Expr *Base, - const ValueDecl *CountedByVD); - llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, bool isInc, bool isPre); ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV, @@ -4830,9 +4817,6 @@ private: llvm::Value *EmittedE, bool IsDynamic); - llvm::Value *emitFlexibleArrayMemberSize(const Expr *E, unsigned Type, - llvm::IntegerType *ResType); - void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D, Address Loc); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp index 7ad26ace328a..d78f2594a237 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp @@ -995,12 +995,7 @@ void CodeGenModule::Release() { uint32_t(CLANG_VERSION_MINOR)); getModule().addModuleFlag(llvm::Module::Warning, "zos_product_patchlevel", uint32_t(CLANG_VERSION_PATCHLEVEL)); - std::string ProductId; -#ifdef CLANG_VENDOR - ProductId = #CLANG_VENDOR; -#else - ProductId = "clang"; -#endif + std::string ProductId = getClangVendor() + "clang"; getModule().addModuleFlag(llvm::Module::Error, "zos_product_id", llvm::MDString::get(VMContext, ProductId)); @@ -1111,6 +1106,9 @@ void CodeGenModule::Release() { if (LangOpts.BranchTargetEnforcement) getModule().addModuleFlag(llvm::Module::Min, "branch-target-enforcement", 1); + if (LangOpts.BranchProtectionPAuthLR) + getModule().addModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", + 1); if (LangOpts.hasSignReturnAddress()) getModule().addModuleFlag(llvm::Module::Min, "sign-return-address", 1); if (LangOpts.isSignReturnAddressScopeAll()) diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp index be5145daa00b..7102d190fe00 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp @@ -136,6 +136,8 @@ public: Fn->addFnAttr("branch-target-enforcement", BPI.BranchTargetEnforcement ? "true" : "false"); + Fn->addFnAttr("branch-protection-pauth-lr", + BPI.BranchProtectionPAuthLR ? "true" : "false"); } bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF, diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/AIX.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/AIX.cpp index f9670ea6f251..e6126ff62db3 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/AIX.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/AIX.cpp @@ -328,6 +328,12 @@ void aix::Linker::ConstructJob(Compilation &C, const JobAction &JA, } } + if (D.IsFlangMode()) { + addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs); + addFortranRuntimeLibs(ToolChain, Args, CmdArgs); + CmdArgs.push_back("-lm"); + CmdArgs.push_back("-lpthread"); + } const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath()); C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(), Exec, CmdArgs, Inputs, Output)); diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp index 2e39fc29a931..80c0f88105d1 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp @@ -45,6 +45,10 @@ static bool getArchFeatures(const Driver &D, StringRef Arch, (*ISAInfo)->toFeatures( Features, [&Args](const Twine &Str) { return Args.MakeArgString(Str); }, /*AddAllExtensions=*/true); + + if (EnableExperimentalExtensions) + Features.push_back(Args.MakeArgString("+experimental")); + return true; } diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp index de9fd5eaa1e0..70dc7e54aca1 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp @@ -1497,7 +1497,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, << Triple.getArchName(); StringRef Scope, Key; - bool IndirectBranches; + bool IndirectBranches, BranchProtectionPAuthLR; if (A->getOption().matches(options::OPT_msign_return_address_EQ)) { Scope = A->getValue(); @@ -1506,6 +1506,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, << A->getSpelling() << Scope; Key = "a_key"; IndirectBranches = false; + BranchProtectionPAuthLR = false; } else { StringRef DiagMsg; llvm::ARM::ParsedBranchProtection PBP; @@ -1517,6 +1518,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, << "b-key" << A->getAsString(Args); Scope = PBP.Scope; Key = PBP.Key; + BranchProtectionPAuthLR = PBP.BranchProtectionPAuthLR; IndirectBranches = PBP.BranchTargetEnforcement; } @@ -1525,6 +1527,9 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, if (!Scope.equals("none")) CmdArgs.push_back( Args.MakeArgString(Twine("-msign-return-address-key=") + Key)); + if (BranchProtectionPAuthLR) + CmdArgs.push_back( + Args.MakeArgString(Twine("-mbranch-protection-pauth-lr"))); if (IndirectBranches) CmdArgs.push_back("-mbranch-target-enforce"); } @@ -3198,13 +3203,13 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, options::OPT_fstrict_float_cast_overflow, false)) CmdArgs.push_back("-fno-strict-float-cast-overflow"); - if (const Arg *A = Args.getLastArg(options::OPT_fcx_limited_range)) + if (Args.hasArg(options::OPT_fcx_limited_range)) CmdArgs.push_back("-fcx-limited-range"); - if (const Arg *A = Args.getLastArg(options::OPT_fcx_fortran_rules)) + if (Args.hasArg(options::OPT_fcx_fortran_rules)) CmdArgs.push_back("-fcx-fortran-rules"); - if (const Arg *A = Args.getLastArg(options::OPT_fno_cx_limited_range)) + if (Args.hasArg(options::OPT_fno_cx_limited_range)) CmdArgs.push_back("-fno-cx-limited-range"); - if (const Arg *A = Args.getLastArg(options::OPT_fno_cx_fortran_rules)) + if (Args.hasArg(options::OPT_fno_cx_fortran_rules)) CmdArgs.push_back("-fno-cx-fortran-rules"); } diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp index 45901ee7157f..6eb0ed8f3fed 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1174,8 +1174,9 @@ static void addFortranMain(const ToolChain &TC, const ArgList &Args, // The --whole-archive option needs to be part of the link line to make // sure that the main() function from Fortran_main.a is pulled in by the // linker. However, it shouldn't be used if it's already active. - // TODO: Find an equivalent of `--whole-archive` for Darwin. - if (!isWholeArchivePresent(Args) && !TC.getTriple().isMacOSX()) { + // TODO: Find an equivalent of `--whole-archive` for Darwin and AIX. + if (!isWholeArchivePresent(Args) && !TC.getTriple().isMacOSX() && + !TC.getTriple().isOSAIX()) { CmdArgs.push_back("--whole-archive"); CmdArgs.push_back("-lFortran_main"); CmdArgs.push_back("--no-whole-archive"); @@ -1317,28 +1318,28 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args, const SanitizerArgs &SanArgs = TC.getSanitizerArgs(Args); // Collect shared runtimes. if (SanArgs.needsSharedRt()) { - if (SanArgs.needsAsanRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsAsanRt()) { SharedRuntimes.push_back("asan"); if (!Args.hasArg(options::OPT_shared) && !TC.getTriple().isAndroid()) HelperStaticRuntimes.push_back("asan-preinit"); } - if (SanArgs.needsMemProfRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsMemProfRt()) { SharedRuntimes.push_back("memprof"); if (!Args.hasArg(options::OPT_shared) && !TC.getTriple().isAndroid()) HelperStaticRuntimes.push_back("memprof-preinit"); } - if (SanArgs.needsUbsanRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsUbsanRt()) { if (SanArgs.requiresMinimalRuntime()) SharedRuntimes.push_back("ubsan_minimal"); else SharedRuntimes.push_back("ubsan_standalone"); } - if (SanArgs.needsScudoRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsScudoRt()) { SharedRuntimes.push_back("scudo_standalone"); } - if (SanArgs.needsTsanRt() && SanArgs.linkRuntimes()) + if (SanArgs.needsTsanRt()) SharedRuntimes.push_back("tsan"); - if (SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsHwasanRt()) { if (SanArgs.needsHwasanAliasesRt()) SharedRuntimes.push_back("hwasan_aliases"); else @@ -1349,7 +1350,7 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args, } // The stats_client library is also statically linked into DSOs. - if (SanArgs.needsStatsRt() && SanArgs.linkRuntimes()) + if (SanArgs.needsStatsRt()) StaticRuntimes.push_back("stats_client"); // Always link the static runtime regardless of DSO or executable. @@ -1365,20 +1366,19 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args, // Each static runtime that has a DSO counterpart above is excluded below, // but runtimes that exist only as static are not affected by needsSharedRt. - if (!SanArgs.needsSharedRt() && SanArgs.needsAsanRt() && SanArgs.linkRuntimes()) { + if (!SanArgs.needsSharedRt() && SanArgs.needsAsanRt()) { StaticRuntimes.push_back("asan"); if (SanArgs.linkCXXRuntimes()) StaticRuntimes.push_back("asan_cxx"); } - if (!SanArgs.needsSharedRt() && SanArgs.needsMemProfRt() && - SanArgs.linkRuntimes()) { + if (!SanArgs.needsSharedRt() && SanArgs.needsMemProfRt()) { StaticRuntimes.push_back("memprof"); if (SanArgs.linkCXXRuntimes()) StaticRuntimes.push_back("memprof_cxx"); } - if (!SanArgs.needsSharedRt() && SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) { + if (!SanArgs.needsSharedRt() && SanArgs.needsHwasanRt()) { if (SanArgs.needsHwasanAliasesRt()) { StaticRuntimes.push_back("hwasan_aliases"); if (SanArgs.linkCXXRuntimes()) @@ -1389,22 +1389,21 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args, StaticRuntimes.push_back("hwasan_cxx"); } } - if (SanArgs.needsDfsanRt() && SanArgs.linkRuntimes()) + if (SanArgs.needsDfsanRt()) StaticRuntimes.push_back("dfsan"); - if (SanArgs.needsLsanRt() && SanArgs.linkRuntimes()) + if (SanArgs.needsLsanRt()) StaticRuntimes.push_back("lsan"); - if (SanArgs.needsMsanRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsMsanRt()) { StaticRuntimes.push_back("msan"); if (SanArgs.linkCXXRuntimes()) StaticRuntimes.push_back("msan_cxx"); } - if (!SanArgs.needsSharedRt() && SanArgs.needsTsanRt() && - SanArgs.linkRuntimes()) { + if (!SanArgs.needsSharedRt() && SanArgs.needsTsanRt()) { StaticRuntimes.push_back("tsan"); if (SanArgs.linkCXXRuntimes()) StaticRuntimes.push_back("tsan_cxx"); } - if (!SanArgs.needsSharedRt() && SanArgs.needsUbsanRt() && SanArgs.linkRuntimes()) { + if (!SanArgs.needsSharedRt() && SanArgs.needsUbsanRt()) { if (SanArgs.requiresMinimalRuntime()) { StaticRuntimes.push_back("ubsan_minimal"); } else { @@ -1413,24 +1412,24 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args, StaticRuntimes.push_back("ubsan_standalone_cxx"); } } - if (SanArgs.needsSafeStackRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsSafeStackRt()) { NonWholeStaticRuntimes.push_back("safestack"); RequiredSymbols.push_back("__safestack_init"); } - if (!(SanArgs.needsSharedRt() && SanArgs.needsUbsanRt() && SanArgs.linkRuntimes())) { - if (SanArgs.needsCfiRt() && SanArgs.linkRuntimes()) + if (!(SanArgs.needsSharedRt() && SanArgs.needsUbsanRt())) { + if (SanArgs.needsCfiRt()) StaticRuntimes.push_back("cfi"); - if (SanArgs.needsCfiDiagRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsCfiDiagRt()) { StaticRuntimes.push_back("cfi_diag"); if (SanArgs.linkCXXRuntimes()) StaticRuntimes.push_back("ubsan_standalone_cxx"); } } - if (SanArgs.needsStatsRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsStatsRt()) { NonWholeStaticRuntimes.push_back("stats"); RequiredSymbols.push_back("__sanitizer_stats_register"); } - if (!SanArgs.needsSharedRt() && SanArgs.needsScudoRt() && SanArgs.linkRuntimes()) { + if (!SanArgs.needsSharedRt() && SanArgs.needsScudoRt()) { StaticRuntimes.push_back("scudo_standalone"); if (SanArgs.linkCXXRuntimes()) StaticRuntimes.push_back("scudo_standalone_cxx"); @@ -1441,13 +1440,15 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args, // C runtime, etc). Returns true if sanitizer system deps need to be linked in. bool tools::addSanitizerRuntimes(const ToolChain &TC, const ArgList &Args, ArgStringList &CmdArgs) { + const SanitizerArgs &SanArgs = TC.getSanitizerArgs(Args); SmallVector<StringRef, 4> SharedRuntimes, StaticRuntimes, NonWholeStaticRuntimes, HelperStaticRuntimes, RequiredSymbols; - collectSanitizerRuntimes(TC, Args, SharedRuntimes, StaticRuntimes, - NonWholeStaticRuntimes, HelperStaticRuntimes, - RequiredSymbols); + if (SanArgs.linkRuntimes()) { + collectSanitizerRuntimes(TC, Args, SharedRuntimes, StaticRuntimes, + NonWholeStaticRuntimes, HelperStaticRuntimes, + RequiredSymbols); + } - const SanitizerArgs &SanArgs = TC.getSanitizerArgs(Args); // Inject libfuzzer dependencies. if (SanArgs.needsFuzzer() && SanArgs.linkRuntimes() && !Args.hasArg(options::OPT_shared)) { diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp index 835215a83c40..38361d6889a1 100644 --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp @@ -30,6 +30,7 @@ #include "llvm/Option/ArgList.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Path.h" +#include "llvm/Support/RISCVISAInfo.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/TargetParser/TargetParser.h" #include <system_error> @@ -1715,6 +1716,129 @@ static void findCSKYMultilibs(const Driver &D, const llvm::Triple &TargetTriple, Result.Multilibs = CSKYMultilibs; } +/// Extend the multi-lib re-use selection mechanism for RISC-V. +/// This function will try to re-use multi-lib if they are compatible. +/// Definition of compatible: +/// - ABI must be the same. +/// - multi-lib is a subset of current arch, e.g. multi-lib=march=rv32im +/// is a subset of march=rv32imc. +/// - march that contains atomic extension can't reuse multi-lib that +/// doesn't have atomic, vice versa. e.g. multi-lib=march=rv32im and +/// march=rv32ima are not compatible, because software and hardware +/// atomic operation can't work together correctly. +static bool +selectRISCVMultilib(const MultilibSet &RISCVMultilibSet, StringRef Arch, + const Multilib::flags_list &Flags, + llvm::SmallVectorImpl<Multilib> &SelectedMultilibs) { + // Try to find the perfect matching multi-lib first. + if (RISCVMultilibSet.select(Flags, SelectedMultilibs)) + return true; + + Multilib::flags_list NewFlags; + std::vector<MultilibBuilder> NewMultilibs; + + llvm::Expected<std::unique_ptr<llvm::RISCVISAInfo>> ParseResult = + llvm::RISCVISAInfo::parseArchString( + Arch, /*EnableExperimentalExtension=*/true, + /*ExperimentalExtensionVersionCheck=*/false); + if (!ParseResult) { + // Ignore any error here, we assume it will be handled in another place. + consumeError(ParseResult.takeError()); + return false; + } + + auto &ISAInfo = *ParseResult; + + addMultilibFlag(ISAInfo->getXLen() == 32, "-m32", NewFlags); + addMultilibFlag(ISAInfo->getXLen() == 64, "-m64", NewFlags); + + // Collect all flags except march=* + for (StringRef Flag : Flags) { + if (Flag.starts_with("!march=") || Flag.starts_with("-march=")) + continue; + + NewFlags.push_back(Flag.str()); + } + + llvm::StringSet<> AllArchExts; + // Reconstruct multi-lib list, and break march option into separated + // extension. e.g. march=rv32im -> +i +m + for (const auto &M : RISCVMultilibSet) { + bool Skip = false; + + MultilibBuilder NewMultilib = + MultilibBuilder(M.gccSuffix(), M.osSuffix(), M.includeSuffix()); + for (StringRef Flag : M.flags()) { + // Add back all flags except -march. + if (!Flag.consume_front("-march=")) { + NewMultilib.flag(Flag); + continue; + } + + // Break down -march into individual extension. + llvm::Expected<std::unique_ptr<llvm::RISCVISAInfo>> MLConfigParseResult = + llvm::RISCVISAInfo::parseArchString( + Flag, /*EnableExperimentalExtension=*/true, + /*ExperimentalExtensionVersionCheck=*/false); + if (!MLConfigParseResult) { + // Ignore any error here, we assume it will handled in another place. + llvm::consumeError(MLConfigParseResult.takeError()); + + // We might get a parsing error if rv32e in the list, we could just skip + // that and process the rest of multi-lib configs. + Skip = true; + continue; + } + auto &MLConfigISAInfo = *MLConfigParseResult; + + const llvm::RISCVISAInfo::OrderedExtensionMap &MLConfigArchExts = + MLConfigISAInfo->getExtensions(); + for (auto MLConfigArchExt : MLConfigArchExts) { + auto ExtName = MLConfigArchExt.first; + NewMultilib.flag(Twine("-", ExtName).str()); + + if (AllArchExts.insert(ExtName).second) { + addMultilibFlag(ISAInfo->hasExtension(ExtName), + Twine("-", ExtName).str(), NewFlags); + } + } + + // Check the XLEN explicitly. + if (MLConfigISAInfo->getXLen() == 32) { + NewMultilib.flag("-m32"); + NewMultilib.flag("-m64", /*Disallow*/ true); + } else { + NewMultilib.flag("-m32", /*Disallow*/ true); + NewMultilib.flag("-m64"); + } + + // Atomic extension must be explicitly checked, soft and hard atomic + // operation never co-work correctly. + if (!MLConfigISAInfo->hasExtension("a")) + NewMultilib.flag("-a", /*Disallow*/ true); + } + + if (Skip) + continue; + + NewMultilibs.emplace_back(NewMultilib); + } + + // Build an internal used only multi-lib list, used for checking any + // compatible multi-lib. + MultilibSet NewRISCVMultilibs = + MultilibSetBuilder().Either(NewMultilibs).makeMultilibSet(); + + if (NewRISCVMultilibs.select(NewFlags, SelectedMultilibs)) + for (const Multilib &NewSelectedM : SelectedMultilibs) + for (const auto &M : RISCVMultilibSet) + // Look up the corresponding multi-lib entry in original multi-lib set. + if (M.gccSuffix() == NewSelectedM.gccSuffix()) + return true; + + return false; +} + static void findRISCVBareMetalMultilibs(const Driver &D, const llvm::Triple &TargetTriple, StringRef Path, const ArgList &Args, @@ -1766,7 +1890,8 @@ static void findRISCVBareMetalMultilibs(const Driver &D, } } - if (RISCVMultilibs.select(Flags, Result.SelectedMultilibs)) + if (selectRISCVMultilib(RISCVMultilibs, MArch, Flags, + Result.SelectedMultilibs)) Result.Multilibs = RISCVMultilibs; } diff --git a/contrib/llvm-project/clang/lib/Format/ContinuationIndenter.cpp b/contrib/llvm-project/clang/lib/Format/ContinuationIndenter.cpp index bd319f21b05f..8489a30dd34a 100644 --- a/contrib/llvm-project/clang/lib/Format/ContinuationIndenter.cpp +++ b/contrib/llvm-project/clang/lib/Format/ContinuationIndenter.cpp @@ -583,17 +583,15 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { return true; } - // If the return type spans multiple lines, wrap before the function name. - if (((Current.is(TT_FunctionDeclarationName) && - !State.Line->ReturnTypeWrapped && - // Don't break before a C# function when no break after return type. - (!Style.isCSharp() || - Style.AlwaysBreakAfterReturnType != FormatStyle::RTBS_None) && - // Don't always break between a JavaScript `function` and the function - // name. - !Style.isJavaScript()) || - (Current.is(tok::kw_operator) && Previous.isNot(tok::coloncolon))) && - Previous.isNot(tok::kw_template) && CurrentState.BreakBeforeParameter) { + if (Current.is(TT_FunctionDeclarationName) && + !State.Line->ReturnTypeWrapped && + // Don't break before a C# function when no break after return type. + (!Style.isCSharp() || + Style.AlwaysBreakAfterReturnType != FormatStyle::RTBS_None) && + // Don't always break between a JavaScript `function` and the function + // name. + !Style.isJavaScript() && Previous.isNot(tok::kw_template) && + CurrentState.BreakBeforeParameter) { return true; } diff --git a/contrib/llvm-project/clang/lib/Format/Format.cpp b/contrib/llvm-project/clang/lib/Format/Format.cpp index 668e959a9416..f798d555bf99 100644 --- a/contrib/llvm-project/clang/lib/Format/Format.cpp +++ b/contrib/llvm-project/clang/lib/Format/Format.cpp @@ -1315,7 +1315,6 @@ static void expandPresetsBraceWrapping(FormatStyle &Expanded) { Expanded.BraceWrapping.AfterStruct = true; Expanded.BraceWrapping.AfterUnion = true; Expanded.BraceWrapping.AfterExternBlock = true; - Expanded.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; Expanded.BraceWrapping.SplitEmptyFunction = true; Expanded.BraceWrapping.SplitEmptyRecord = false; break; @@ -1335,7 +1334,6 @@ static void expandPresetsBraceWrapping(FormatStyle &Expanded) { Expanded.BraceWrapping.AfterStruct = true; Expanded.BraceWrapping.AfterUnion = true; Expanded.BraceWrapping.AfterExternBlock = true; - Expanded.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; Expanded.BraceWrapping.BeforeCatch = true; Expanded.BraceWrapping.BeforeElse = true; Expanded.BraceWrapping.BeforeLambdaBody = true; @@ -1350,7 +1348,6 @@ static void expandPresetsBraceWrapping(FormatStyle &Expanded) { Expanded.BraceWrapping.AfterObjCDeclaration = true; Expanded.BraceWrapping.AfterStruct = true; Expanded.BraceWrapping.AfterExternBlock = true; - Expanded.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; Expanded.BraceWrapping.BeforeCatch = true; Expanded.BraceWrapping.BeforeElse = true; Expanded.BraceWrapping.BeforeLambdaBody = true; @@ -1375,7 +1372,6 @@ static void expandPresetsBraceWrapping(FormatStyle &Expanded) { /*SplitEmptyFunction=*/true, /*SplitEmptyRecord=*/true, /*SplitEmptyNamespace=*/true}; - Expanded.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; break; case FormatStyle::BS_WebKit: Expanded.BraceWrapping.AfterFunction = true; @@ -1702,6 +1698,9 @@ FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) { /*BasedOnStyle=*/"google", }, }; + GoogleStyle.AttributeMacros.push_back("GUARDED_BY"); + GoogleStyle.AttributeMacros.push_back("ABSL_GUARDED_BY"); + GoogleStyle.SpacesBeforeTrailingComments = 2; GoogleStyle.Standard = FormatStyle::LS_Auto; @@ -1909,7 +1908,6 @@ FormatStyle getMicrosoftStyle(FormatStyle::LanguageKind Language) { Style.BraceWrapping.AfterObjCDeclaration = true; Style.BraceWrapping.AfterStruct = true; Style.BraceWrapping.AfterExternBlock = true; - Style.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; Style.BraceWrapping.BeforeCatch = true; Style.BraceWrapping.BeforeElse = true; Style.BraceWrapping.BeforeWhile = false; diff --git a/contrib/llvm-project/clang/lib/Format/MatchFilePath.cpp b/contrib/llvm-project/clang/lib/Format/MatchFilePath.cpp new file mode 100644 index 000000000000..412ee4954587 --- /dev/null +++ b/contrib/llvm-project/clang/lib/Format/MatchFilePath.cpp @@ -0,0 +1,122 @@ +//===--- MatchFilePath.cpp - Match file path with pattern -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements the functionality of matching a file path name to +/// a pattern, similar to the POSIX fnmatch() function. +/// +//===----------------------------------------------------------------------===// + +#include "MatchFilePath.h" + +using namespace llvm; + +namespace clang { +namespace format { + +// Check whether `FilePath` matches `Pattern` based on POSIX (1003.1-2008) +// 2.13.1, 2.13.2, and Rule 1 of 2.13.3. +bool matchFilePath(StringRef Pattern, StringRef FilePath) { + assert(!Pattern.empty()); + assert(!FilePath.empty()); + + // No match if `Pattern` ends with a non-meta character not equal to the last + // character of `FilePath`. + if (const auto C = Pattern.back(); !strchr("?*]", C) && C != FilePath.back()) + return false; + + constexpr auto Separator = '/'; + const auto EOP = Pattern.size(); // End of `Pattern`. + const auto End = FilePath.size(); // End of `FilePath`. + unsigned I = 0; // Index to `Pattern`. + + for (unsigned J = 0; J < End; ++J) { + if (I == EOP) + return false; + + switch (const auto F = FilePath[J]; Pattern[I]) { + case '\\': + if (++I == EOP || F != Pattern[I]) + return false; + break; + case '?': + if (F == Separator) + return false; + break; + case '*': { + while (++I < EOP && Pattern[I] == '*') { // Skip consecutive stars. + } + const auto K = FilePath.find(Separator, J); // Index of next `Separator`. + const bool NoMoreSeparatorsInFilePath = K == StringRef::npos; + if (I == EOP) // `Pattern` ends with a star. + return NoMoreSeparatorsInFilePath; + // `Pattern` ends with a lone backslash. + if (Pattern[I] == '\\' && ++I == EOP) + return false; + // The star is followed by a (possibly escaped) `Separator`. + if (Pattern[I] == Separator) { + if (NoMoreSeparatorsInFilePath) + return false; + J = K; // Skip to next `Separator` in `FilePath`. + break; + } + // Recurse. + for (auto Pat = Pattern.substr(I); J < End && FilePath[J] != Separator; + ++J) { + if (matchFilePath(Pat, FilePath.substr(J))) + return true; + } + return false; + } + case '[': + // Skip e.g. `[!]`. + if (I + 3 < EOP || (I + 3 == EOP && Pattern[I + 1] != '!')) { + // Skip unpaired `[`, brackets containing slashes, and `[]`. + if (const auto K = Pattern.find_first_of("]/", I + 1); + K != StringRef::npos && Pattern[K] == ']' && K > I + 1) { + if (F == Separator) + return false; + ++I; // After the `[`. + bool Negated = false; + if (Pattern[I] == '!') { + Negated = true; + ++I; // After the `!`. + } + bool Match = false; + do { + if (I + 2 < K && Pattern[I + 1] == '-') { + Match = Pattern[I] <= F && F <= Pattern[I + 2]; + I += 3; // After the range, e.g. `A-Z`. + } else { + Match = F == Pattern[I++]; + } + } while (!Match && I < K); + if (Negated ? Match : !Match) + return false; + I = K + 1; // After the `]`. + continue; + } + } + [[fallthrough]]; // Match `[` literally. + default: + if (F != Pattern[I]) + return false; + } + + ++I; + } + + // Match trailing stars with null strings. + while (I < EOP && Pattern[I] == '*') + ++I; + + return I == EOP; +} + +} // namespace format +} // namespace clang diff --git a/contrib/llvm-project/clang/lib/Format/MatchFilePath.h b/contrib/llvm-project/clang/lib/Format/MatchFilePath.h new file mode 100644 index 000000000000..482dab7c748e --- /dev/null +++ b/contrib/llvm-project/clang/lib/Format/MatchFilePath.h @@ -0,0 +1,22 @@ +//===--- MatchFilePath.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_FORMAT_MATCHFILEPATH_H +#define LLVM_CLANG_LIB_FORMAT_MATCHFILEPATH_H + +#include "llvm/ADT/StringRef.h" + +namespace clang { +namespace format { + +bool matchFilePath(llvm::StringRef Pattern, llvm::StringRef FilePath); + +} // end namespace format +} // end namespace clang + +#endif diff --git a/contrib/llvm-project/clang/lib/Format/TokenAnnotator.cpp b/contrib/llvm-project/clang/lib/Format/TokenAnnotator.cpp index f3551af34243..3ac3aa3c5e3a 100644 --- a/contrib/llvm-project/clang/lib/Format/TokenAnnotator.cpp +++ b/contrib/llvm-project/clang/lib/Format/TokenAnnotator.cpp @@ -3403,7 +3403,8 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current, continue; } if (Tok->is(tok::kw_const) || Tok->isSimpleTypeSpecifier() || - Tok->isOneOf(TT_PointerOrReference, TT_StartOfName, tok::ellipsis)) { + Tok->isOneOf(TT_PointerOrReference, TT_StartOfName, tok::ellipsis, + TT_TypeName)) { return true; } if (Tok->isOneOf(tok::l_brace, TT_ObjCMethodExpr) || Tok->Tok.isLiteral()) diff --git a/contrib/llvm-project/clang/lib/Format/UnwrappedLineParser.cpp b/contrib/llvm-project/clang/lib/Format/UnwrappedLineParser.cpp index c38b4c884070..684609747a55 100644 --- a/contrib/llvm-project/clang/lib/Format/UnwrappedLineParser.cpp +++ b/contrib/llvm-project/clang/lib/Format/UnwrappedLineParser.cpp @@ -1650,8 +1650,10 @@ void UnwrappedLineParser::parseStructuralElement( return; } // In Verilog labels can be any expression, so we don't do them here. - if (!Style.isVerilog() && Tokens->peekNextToken()->is(tok::colon) && - !Line->MustBeDeclaration) { + // JS doesn't have macros, and within classes colons indicate fields, not + // labels. + if (!Style.isJavaScript() && !Style.isVerilog() && + Tokens->peekNextToken()->is(tok::colon) && !Line->MustBeDeclaration) { nextToken(); Line->Tokens.begin()->Tok->MustBreakBefore = true; FormatTok->setFinalizedType(TT_GotoLabelColon); diff --git a/contrib/llvm-project/clang/lib/Headers/adcintrin.h b/contrib/llvm-project/clang/lib/Headers/adcintrin.h new file mode 100644 index 000000000000..0065a1b543f8 --- /dev/null +++ b/contrib/llvm-project/clang/lib/Headers/adcintrin.h @@ -0,0 +1,160 @@ +/*===---- adcintrin.h - ADC intrinsics -------------------------------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ + +#ifndef __ADCINTRIN_H +#define __ADCINTRIN_H + +#if !defined(__i386__) && !defined(__x86_64__) +#error "This header is only meant to be used on x86 and x64 architecture" +#endif + +/* Define the default attributes for the functions in this file. */ +#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) + +/* Use C++ inline semantics in C++, GNU inline for C mode. */ +#if defined(__cplusplus) +#define __INLINE __inline +#else +#define __INLINE static __inline +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + +/// Adds unsigned 32-bit integers \a __x and \a __y, plus 0 or 1 as indicated +/// by the carry flag \a __cf. Stores the unsigned 32-bit sum in the memory +/// at \a __p, and returns the 8-bit carry-out (carry flag). +/// +/// \code{.operation} +/// temp := (__cf == 0) ? 0 : 1 +/// Store32(__p, __x + __y + temp) +/// result := CF +/// \endcode +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the \c ADC instruction. +/// +/// \param __cf +/// The 8-bit unsigned carry flag; any non-zero value indicates carry. +/// \param __x +/// A 32-bit unsigned addend. +/// \param __y +/// A 32-bit unsigned addend. +/// \param __p +/// Pointer to memory for storing the sum. +/// \returns The 8-bit unsigned carry-out value. +__INLINE unsigned char __DEFAULT_FN_ATTRS _addcarry_u32(unsigned char __cf, + unsigned int __x, + unsigned int __y, + unsigned int *__p) { + return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p); +} + +/// Adds unsigned 32-bit integer \a __y to 0 or 1 as indicated by the carry +/// flag \a __cf, and subtracts the result from unsigned 32-bit integer +/// \a __x. Stores the unsigned 32-bit difference in the memory at \a __p, +/// and returns the 8-bit carry-out (carry or overflow flag). +/// +/// \code{.operation} +/// temp := (__cf == 0) ? 0 : 1 +/// Store32(__p, __x - (__y + temp)) +/// result := CF +/// \endcode +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the \c SBB instruction. +/// +/// \param __cf +/// The 8-bit unsigned carry flag; any non-zero value indicates carry. +/// \param __x +/// The 32-bit unsigned minuend. +/// \param __y +/// The 32-bit unsigned subtrahend. +/// \param __p +/// Pointer to memory for storing the difference. +/// \returns The 8-bit unsigned carry-out value. +__INLINE unsigned char __DEFAULT_FN_ATTRS _subborrow_u32(unsigned char __cf, + unsigned int __x, + unsigned int __y, + unsigned int *__p) { + return __builtin_ia32_subborrow_u32(__cf, __x, __y, __p); +} + +#ifdef __x86_64__ +/// Adds unsigned 64-bit integers \a __x and \a __y, plus 0 or 1 as indicated +/// by the carry flag \a __cf. Stores the unsigned 64-bit sum in the memory +/// at \a __p, and returns the 8-bit carry-out (carry flag). +/// +/// \code{.operation} +/// temp := (__cf == 0) ? 0 : 1 +/// Store64(__p, __x + __y + temp) +/// result := CF +/// \endcode +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the \c ADC instruction. +/// +/// \param __cf +/// The 8-bit unsigned carry flag; any non-zero value indicates carry. +/// \param __x +/// A 64-bit unsigned addend. +/// \param __y +/// A 64-bit unsigned addend. +/// \param __p +/// Pointer to memory for storing the sum. +/// \returns The 8-bit unsigned carry-out value. +__INLINE unsigned char __DEFAULT_FN_ATTRS +_addcarry_u64(unsigned char __cf, unsigned long long __x, + unsigned long long __y, unsigned long long *__p) { + return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); +} + +/// Adds unsigned 64-bit integer \a __y to 0 or 1 as indicated by the carry +/// flag \a __cf, and subtracts the result from unsigned 64-bit integer +/// \a __x. Stores the unsigned 64-bit difference in the memory at \a __p, +/// and returns the 8-bit carry-out (carry or overflow flag). +/// +/// \code{.operation} +/// temp := (__cf == 0) ? 0 : 1 +/// Store64(__p, __x - (__y + temp)) +/// result := CF +/// \endcode +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the \c ADC instruction. +/// +/// \param __cf +/// The 8-bit unsigned carry flag; any non-zero value indicates carry. +/// \param __x +/// The 64-bit unsigned minuend. +/// \param __y +/// The 64-bit unsigned subtrahend. +/// \param __p +/// Pointer to memory for storing the difference. +/// \returns The 8-bit unsigned carry-out value. +__INLINE unsigned char __DEFAULT_FN_ATTRS +_subborrow_u64(unsigned char __cf, unsigned long long __x, + unsigned long long __y, unsigned long long *__p) { + return __builtin_ia32_subborrow_u64(__cf, __x, __y, __p); +} +#endif + +#if defined(__cplusplus) +} +#endif + +#undef __INLINE +#undef __DEFAULT_FN_ATTRS + +#endif /* __ADCINTRIN_H */ diff --git a/contrib/llvm-project/clang/lib/Headers/adxintrin.h b/contrib/llvm-project/clang/lib/Headers/adxintrin.h index 20f6211e567b..bc6a4caf3533 100644 --- a/contrib/llvm-project/clang/lib/Headers/adxintrin.h +++ b/contrib/llvm-project/clang/lib/Headers/adxintrin.h @@ -15,7 +15,8 @@ #define __ADXINTRIN_H /* Define the default attributes for the functions in this file. */ -#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) +#define __DEFAULT_FN_ATTRS \ + __attribute__((__always_inline__, __nodebug__, __target__("adx"))) /* Use C++ inline semantics in C++, GNU inline for C mode. */ #if defined(__cplusplus) @@ -53,10 +54,10 @@ extern "C" { /// \param __p /// Pointer to memory for storing the sum. /// \returns The 8-bit unsigned carry-out value. -__INLINE unsigned char - __attribute__((__always_inline__, __nodebug__, __target__("adx"))) - _addcarryx_u32(unsigned char __cf, unsigned int __x, unsigned int __y, - unsigned int *__p) { +__INLINE unsigned char __DEFAULT_FN_ATTRS _addcarryx_u32(unsigned char __cf, + unsigned int __x, + unsigned int __y, + unsigned int *__p) { return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p); } @@ -84,137 +85,10 @@ __INLINE unsigned char /// \param __p /// Pointer to memory for storing the sum. /// \returns The 8-bit unsigned carry-out value. -__INLINE unsigned char - __attribute__((__always_inline__, __nodebug__, __target__("adx"))) - _addcarryx_u64(unsigned char __cf, unsigned long long __x, - unsigned long long __y, unsigned long long *__p) { - return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); -} -#endif - -/* Intrinsics that are also available if __ADX__ is undefined. */ - -/// Adds unsigned 32-bit integers \a __x and \a __y, plus 0 or 1 as indicated -/// by the carry flag \a __cf. Stores the unsigned 32-bit sum in the memory -/// at \a __p, and returns the 8-bit carry-out (carry flag). -/// -/// \code{.operation} -/// temp := (__cf == 0) ? 0 : 1 -/// Store32(__p, __x + __y + temp) -/// result := CF -/// \endcode -/// -/// \headerfile <immintrin.h> -/// -/// This intrinsic corresponds to the \c ADC instruction. -/// -/// \param __cf -/// The 8-bit unsigned carry flag; any non-zero value indicates carry. -/// \param __x -/// A 32-bit unsigned addend. -/// \param __y -/// A 32-bit unsigned addend. -/// \param __p -/// Pointer to memory for storing the sum. -/// \returns The 8-bit unsigned carry-out value. -__INLINE unsigned char __DEFAULT_FN_ATTRS _addcarry_u32(unsigned char __cf, - unsigned int __x, - unsigned int __y, - unsigned int *__p) { - return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p); -} - -#ifdef __x86_64__ -/// Adds unsigned 64-bit integers \a __x and \a __y, plus 0 or 1 as indicated -/// by the carry flag \a __cf. Stores the unsigned 64-bit sum in the memory -/// at \a __p, and returns the 8-bit carry-out (carry flag). -/// -/// \code{.operation} -/// temp := (__cf == 0) ? 0 : 1 -/// Store64(__p, __x + __y + temp) -/// result := CF -/// \endcode -/// -/// \headerfile <immintrin.h> -/// -/// This intrinsic corresponds to the \c ADC instruction. -/// -/// \param __cf -/// The 8-bit unsigned carry flag; any non-zero value indicates carry. -/// \param __x -/// A 64-bit unsigned addend. -/// \param __y -/// A 64-bit unsigned addend. -/// \param __p -/// Pointer to memory for storing the sum. -/// \returns The 8-bit unsigned carry-out value. __INLINE unsigned char __DEFAULT_FN_ATTRS -_addcarry_u64(unsigned char __cf, unsigned long long __x, - unsigned long long __y, unsigned long long *__p) { - return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); -} -#endif - -/// Adds unsigned 32-bit integer \a __y to 0 or 1 as indicated by the carry -/// flag \a __cf, and subtracts the result from unsigned 32-bit integer -/// \a __x. Stores the unsigned 32-bit difference in the memory at \a __p, -/// and returns the 8-bit carry-out (carry or overflow flag). -/// -/// \code{.operation} -/// temp := (__cf == 0) ? 0 : 1 -/// Store32(__p, __x - (__y + temp)) -/// result := CF -/// \endcode -/// -/// \headerfile <immintrin.h> -/// -/// This intrinsic corresponds to the \c SBB instruction. -/// -/// \param __cf -/// The 8-bit unsigned carry flag; any non-zero value indicates carry. -/// \param __x -/// The 32-bit unsigned minuend. -/// \param __y -/// The 32-bit unsigned subtrahend. -/// \param __p -/// Pointer to memory for storing the difference. -/// \returns The 8-bit unsigned carry-out value. -__INLINE unsigned char __DEFAULT_FN_ATTRS _subborrow_u32(unsigned char __cf, - unsigned int __x, - unsigned int __y, - unsigned int *__p) { - return __builtin_ia32_subborrow_u32(__cf, __x, __y, __p); -} - -#ifdef __x86_64__ -/// Adds unsigned 64-bit integer \a __y to 0 or 1 as indicated by the carry -/// flag \a __cf, and subtracts the result from unsigned 64-bit integer -/// \a __x. Stores the unsigned 64-bit difference in the memory at \a __p, -/// and returns the 8-bit carry-out (carry or overflow flag). -/// -/// \code{.operation} -/// temp := (__cf == 0) ? 0 : 1 -/// Store64(__p, __x - (__y + temp)) -/// result := CF -/// \endcode -/// -/// \headerfile <immintrin.h> -/// -/// This intrinsic corresponds to the \c ADC instruction. -/// -/// \param __cf -/// The 8-bit unsigned carry flag; any non-zero value indicates carry. -/// \param __x -/// The 64-bit unsigned minuend. -/// \param __y -/// The 64-bit unsigned subtrahend. -/// \param __p -/// Pointer to memory for storing the difference. -/// \returns The 8-bit unsigned carry-out value. -__INLINE unsigned char __DEFAULT_FN_ATTRS -_subborrow_u64(unsigned char __cf, unsigned long long __x, +_addcarryx_u64(unsigned char __cf, unsigned long long __x, unsigned long long __y, unsigned long long *__p) { - return __builtin_ia32_subborrow_u64(__cf, __x, __y, __p); + return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); } #endif @@ -222,6 +96,7 @@ _subborrow_u64(unsigned char __cf, unsigned long long __x, } #endif +#undef __INLINE #undef __DEFAULT_FN_ATTRS #endif /* __ADXINTRIN_H */ diff --git a/contrib/llvm-project/clang/lib/Headers/immintrin.h b/contrib/llvm-project/clang/lib/Headers/immintrin.h index 9bfe2fcdabdb..0149a1cdea63 100644 --- a/contrib/llvm-project/clang/lib/Headers/immintrin.h +++ b/contrib/llvm-project/clang/lib/Headers/immintrin.h @@ -580,9 +580,13 @@ _storebe_i64(void * __P, long long __D) { #include <cetintrin.h> #endif -/* Some intrinsics inside adxintrin.h are available only on processors with ADX, - * whereas others are also available at all times. */ +/* Intrinsics inside adcintrin.h are available at all times. */ +#include <adcintrin.h> + +#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ + defined(__ADX__) #include <adxintrin.h> +#endif #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ defined(__RDSEED__) diff --git a/contrib/llvm-project/clang/lib/Headers/riscv_bitmanip.h b/contrib/llvm-project/clang/lib/Headers/riscv_bitmanip.h index 1a81cc8618c9..2bc7ee022a96 100644 --- a/contrib/llvm-project/clang/lib/Headers/riscv_bitmanip.h +++ b/contrib/llvm-project/clang/lib/Headers/riscv_bitmanip.h @@ -34,7 +34,7 @@ __riscv_ctz_32(uint32_t __x) { static __inline__ unsigned __attribute__((__always_inline__, __nodebug__)) __riscv_cpop_32(uint32_t __x) { - return __builtin_riscv_cpop_32(__x); + return __builtin_popcount(__x); } #if __riscv_xlen == 64 @@ -55,7 +55,7 @@ __riscv_ctz_64(uint64_t __x) { static __inline__ unsigned __attribute__((__always_inline__, __nodebug__)) __riscv_cpop_64(uint64_t __x) { - return __builtin_riscv_cpop_64(__x); + return __builtin_popcountll(__x); } #endif #endif // defined(__riscv_zbb) @@ -120,7 +120,23 @@ __riscv_zip_32(uint32_t __x) { #endif #endif // defined(__riscv_zbkb) -#if defined(__riscv_zbkc) +#if defined(__riscv_zbc) +#if __riscv_xlen == 32 +static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__)) +__riscv_clmulr_32(uint32_t __x, uint32_t __y) { + return __builtin_riscv_clmulr_32(__x, __y); +} +#endif + +#if __riscv_xlen == 64 +static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__)) +__riscv_clmulr_64(uint64_t __x, uint64_t __y) { + return __builtin_riscv_clmulr_64(__x, __y); +} +#endif +#endif // defined(__riscv_zbc) + +#if defined(__riscv_zbkc) || defined(__riscv_zbc) static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__)) __riscv_clmul_32(uint32_t __x, uint32_t __y) { return __builtin_riscv_clmul_32(__x, __y); @@ -144,7 +160,7 @@ __riscv_clmulh_64(uint64_t __x, uint64_t __y) { return __builtin_riscv_clmulh_64(__x, __y); } #endif -#endif // defined(__riscv_zbkc) +#endif // defined(__riscv_zbkc) || defined(__riscv_zbc) #if defined(__riscv_zbkx) #if __riscv_xlen == 32 diff --git a/contrib/llvm-project/clang/lib/Interpreter/CodeCompletion.cpp b/contrib/llvm-project/clang/lib/Interpreter/CodeCompletion.cpp index c40e11b9d1ec..25183ae9eeb9 100644 --- a/contrib/llvm-project/clang/lib/Interpreter/CodeCompletion.cpp +++ b/contrib/llvm-project/clang/lib/Interpreter/CodeCompletion.cpp @@ -12,6 +12,7 @@ #include "clang/Interpreter/CodeCompletion.h" #include "clang/AST/ASTImporter.h" +#include "clang/AST/DeclLookups.h" #include "clang/AST/DeclarationName.h" #include "clang/AST/ExternalASTSource.h" #include "clang/Basic/IdentifierTable.h" @@ -23,6 +24,8 @@ #include "clang/Sema/CodeCompleteConsumer.h" #include "clang/Sema/CodeCompleteOptions.h" #include "clang/Sema/Sema.h" +#include "llvm/Support/Debug.h" +#define DEBUG_TYPE "REPLCC" namespace clang { @@ -39,11 +42,15 @@ clang::CodeCompleteOptions getClangCompleteOpts() { class ReplCompletionConsumer : public CodeCompleteConsumer { public: - ReplCompletionConsumer(std::vector<std::string> &Results) + ReplCompletionConsumer(std::vector<std::string> &Results, + ReplCodeCompleter &CC) : CodeCompleteConsumer(getClangCompleteOpts()), CCAllocator(std::make_shared<GlobalCodeCompletionAllocator>()), - CCTUInfo(CCAllocator), Results(Results){}; + CCTUInfo(CCAllocator), Results(Results), CC(CC) {} + // The entry of handling code completion. When the function is called, we + // create a `Context`-based handler (see classes defined below) to handle each + // completion result. void ProcessCodeCompleteResults(class Sema &S, CodeCompletionContext Context, CodeCompletionResult *InResults, unsigned NumResults) final; @@ -56,26 +63,147 @@ private: std::shared_ptr<GlobalCodeCompletionAllocator> CCAllocator; CodeCompletionTUInfo CCTUInfo; std::vector<std::string> &Results; + ReplCodeCompleter &CC; +}; + +/// The class CompletionContextHandler contains four interfaces, each of +/// which handles one type of completion result. +/// Its derived classes are used to create concrete handlers based on +/// \c CodeCompletionContext. +class CompletionContextHandler { +protected: + CodeCompletionContext CCC; + std::vector<std::string> &Results; + +private: + Sema &S; + +public: + CompletionContextHandler(Sema &S, CodeCompletionContext CCC, + std::vector<std::string> &Results) + : CCC(CCC), Results(Results), S(S) {} + + virtual ~CompletionContextHandler() = default; + /// Converts a Declaration completion result to a completion string, and then + /// stores it in Results. + virtual void handleDeclaration(const CodeCompletionResult &Result) { + auto PreferredType = CCC.getPreferredType(); + if (PreferredType.isNull()) { + Results.push_back(Result.Declaration->getName().str()); + return; + } + + if (auto *VD = dyn_cast<VarDecl>(Result.Declaration)) { + auto ArgumentType = VD->getType(); + if (PreferredType->isReferenceType()) { + QualType RT = PreferredType->castAs<ReferenceType>()->getPointeeType(); + Sema::ReferenceConversions RefConv; + Sema::ReferenceCompareResult RefRelationship = + S.CompareReferenceRelationship(SourceLocation(), RT, ArgumentType, + &RefConv); + switch (RefRelationship) { + case Sema::Ref_Compatible: + case Sema::Ref_Related: + Results.push_back(VD->getName().str()); + break; + case Sema::Ref_Incompatible: + break; + } + } else if (S.Context.hasSameType(ArgumentType, PreferredType)) { + Results.push_back(VD->getName().str()); + } + } + } + + /// Converts a Keyword completion result to a completion string, and then + /// stores it in Results. + virtual void handleKeyword(const CodeCompletionResult &Result) { + auto Prefix = S.getPreprocessor().getCodeCompletionFilter(); + // Add keyword to the completion results only if we are in a type-aware + // situation. + if (!CCC.getBaseType().isNull() || !CCC.getPreferredType().isNull()) + return; + if (StringRef(Result.Keyword).starts_with(Prefix)) + Results.push_back(Result.Keyword); + } + + /// Converts a Pattern completion result to a completion string, and then + /// stores it in Results. + virtual void handlePattern(const CodeCompletionResult &Result) {} + + /// Converts a Macro completion result to a completion string, and then stores + /// it in Results. + virtual void handleMacro(const CodeCompletionResult &Result) {} +}; + +class DotMemberAccessHandler : public CompletionContextHandler { +public: + DotMemberAccessHandler(Sema &S, CodeCompletionContext CCC, + std::vector<std::string> &Results) + : CompletionContextHandler(S, CCC, Results) {} + void handleDeclaration(const CodeCompletionResult &Result) override { + auto *ID = Result.Declaration->getIdentifier(); + if (!ID) + return; + if (!isa<CXXMethodDecl>(Result.Declaration)) + return; + const auto *Fun = cast<CXXMethodDecl>(Result.Declaration); + if (Fun->getParent()->getCanonicalDecl() == + CCC.getBaseType()->getAsCXXRecordDecl()->getCanonicalDecl()) { + LLVM_DEBUG(llvm::dbgs() << "[In HandleCodeCompleteDOT] Name : " + << ID->getName() << "\n"); + Results.push_back(ID->getName().str()); + } + } + + void handleKeyword(const CodeCompletionResult &Result) override {} }; void ReplCompletionConsumer::ProcessCodeCompleteResults( class Sema &S, CodeCompletionContext Context, CodeCompletionResult *InResults, unsigned NumResults) { - for (unsigned I = 0; I < NumResults; ++I) { + + auto Prefix = S.getPreprocessor().getCodeCompletionFilter(); + CC.Prefix = Prefix; + + std::unique_ptr<CompletionContextHandler> CCH; + + // initialize fine-grained code completion handler based on the code + // completion context. + switch (Context.getKind()) { + case CodeCompletionContext::CCC_DotMemberAccess: + CCH.reset(new DotMemberAccessHandler(S, Context, this->Results)); + break; + default: + CCH.reset(new CompletionContextHandler(S, Context, this->Results)); + }; + + for (unsigned I = 0; I < NumResults; I++) { auto &Result = InResults[I]; switch (Result.Kind) { case CodeCompletionResult::RK_Declaration: - if (auto *ID = Result.Declaration->getIdentifier()) { - Results.push_back(ID->getName().str()); + if (Result.Hidden) { + break; + } + if (!Result.Declaration->getDeclName().isIdentifier() || + !Result.Declaration->getName().starts_with(Prefix)) { + break; } + CCH->handleDeclaration(Result); break; case CodeCompletionResult::RK_Keyword: - Results.push_back(Result.Keyword); + CCH->handleKeyword(Result); break; - default: + case CodeCompletionResult::RK_Macro: + CCH->handleMacro(Result); + break; + case CodeCompletionResult::RK_Pattern: + CCH->handlePattern(Result); break; } } + + std::sort(Results.begin(), Results.end()); } class IncrementalSyntaxOnlyAction : public SyntaxOnlyAction { @@ -118,6 +246,16 @@ void IncrementalSyntaxOnlyAction::ExecuteAction() { CI.getASTContext().getTranslationUnitDecl()->setHasExternalVisibleStorage( true); + // Load all external decls into current context. Under the hood, it calls + // ExternalSource::completeVisibleDeclsMap, which make all decls on the redecl + // chain visible. + // + // This is crucial to code completion on dot members, since a bound variable + // before "." would be otherwise treated out-of-scope. + // + // clang-repl> Foo f1; + // clang-repl> f1.<tab> + CI.getASTContext().getTranslationUnitDecl()->lookups(); SyntaxOnlyAction::ExecuteAction(); } @@ -134,6 +272,7 @@ ExternalSource::ExternalSource(ASTContext &ChildASTCtxt, FileManager &ChildFM, bool ExternalSource::FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name) { + IdentifierTable &ParentIdTable = ParentASTCtxt.Idents; auto ParentDeclName = @@ -159,29 +298,67 @@ void ExternalSource::completeVisibleDeclsMap( for (auto *DeclCtxt = ParentTUDeclCtxt; DeclCtxt != nullptr; DeclCtxt = DeclCtxt->getPreviousDecl()) { for (auto &IDeclContext : DeclCtxt->decls()) { - if (NamedDecl *Decl = llvm::dyn_cast<NamedDecl>(IDeclContext)) { - if (auto DeclOrErr = Importer->Import(Decl)) { - if (NamedDecl *importedNamedDecl = - llvm::dyn_cast<NamedDecl>(*DeclOrErr)) { - SetExternalVisibleDeclsForName(ChildDeclContext, - importedNamedDecl->getDeclName(), - importedNamedDecl); - } - - } else { - llvm::consumeError(DeclOrErr.takeError()); - } + if (!llvm::isa<NamedDecl>(IDeclContext)) + continue; + + NamedDecl *Decl = llvm::cast<NamedDecl>(IDeclContext); + + auto DeclOrErr = Importer->Import(Decl); + if (!DeclOrErr) { + // if an error happens, it usually means the decl has already been + // imported or the decl is a result of a failed import. But in our + // case, every import is fresh each time code completion is + // triggered. So Import usually doesn't fail. If it does, it just means + // the related decl can't be used in code completion and we can safely + // drop it. + llvm::consumeError(DeclOrErr.takeError()); + continue; } + + if (!llvm::isa<NamedDecl>(*DeclOrErr)) + continue; + + NamedDecl *importedNamedDecl = llvm::cast<NamedDecl>(*DeclOrErr); + + SetExternalVisibleDeclsForName(ChildDeclContext, + importedNamedDecl->getDeclName(), + importedNamedDecl); + + if (!llvm::isa<CXXRecordDecl>(importedNamedDecl)) + continue; + + auto *Record = llvm::cast<CXXRecordDecl>(importedNamedDecl); + + if (auto Err = Importer->ImportDefinition(Decl)) { + // the same as above + consumeError(std::move(Err)); + continue; + } + + Record->setHasLoadedFieldsFromExternalStorage(true); + LLVM_DEBUG(llvm::dbgs() + << "\nCXXRecrod : " << Record->getName() << " size(methods): " + << std::distance(Record->method_begin(), Record->method_end()) + << " has def?: " << Record->hasDefinition() + << " # (methods): " + << std::distance(Record->getDefinition()->method_begin(), + Record->getDefinition()->method_end()) + << "\n"); + for (auto *Meth : Record->methods()) + SetExternalVisibleDeclsForName(ChildDeclContext, Meth->getDeclName(), + Meth); } ChildDeclContext->setHasExternalLexicalStorage(false); } } -void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content, - unsigned Line, unsigned Col, const CompilerInstance *ParentCI, - std::vector<std::string> &CCResults) { +void ReplCodeCompleter::codeComplete(CompilerInstance *InterpCI, + llvm::StringRef Content, unsigned Line, + unsigned Col, + const CompilerInstance *ParentCI, + std::vector<std::string> &CCResults) { auto DiagOpts = DiagnosticOptions(); - auto consumer = ReplCompletionConsumer(CCResults); + auto consumer = ReplCompletionConsumer(CCResults, *this); auto diag = InterpCI->getDiagnosticsPtr(); std::unique_ptr<ASTUnit> AU(ASTUnit::LoadFromCompilerInvocationAction( diff --git a/contrib/llvm-project/clang/lib/Interpreter/Interpreter.cpp b/contrib/llvm-project/clang/lib/Interpreter/Interpreter.cpp index 7968c62cbd3e..c9fcef5b5b5a 100644 --- a/contrib/llvm-project/clang/lib/Interpreter/Interpreter.cpp +++ b/contrib/llvm-project/clang/lib/Interpreter/Interpreter.cpp @@ -319,6 +319,10 @@ const CompilerInstance *Interpreter::getCompilerInstance() const { return IncrParser->getCI(); } +CompilerInstance *Interpreter::getCompilerInstance() { + return IncrParser->getCI(); +} + llvm::Expected<llvm::orc::LLJIT &> Interpreter::getExecutionEngine() { if (!IncrExecutor) { if (auto Err = CreateExecutor()) diff --git a/contrib/llvm-project/clang/lib/Lex/PPDirectives.cpp b/contrib/llvm-project/clang/lib/Lex/PPDirectives.cpp index 112bc8dc572c..9f82a6d073e3 100644 --- a/contrib/llvm-project/clang/lib/Lex/PPDirectives.cpp +++ b/contrib/llvm-project/clang/lib/Lex/PPDirectives.cpp @@ -1858,11 +1858,18 @@ static void diagnoseAutoModuleImport( // path to the file, build a properly-cased replacement in the vector, // and return true if the replacement should be suggested. static bool trySimplifyPath(SmallVectorImpl<StringRef> &Components, - StringRef RealPathName) { + StringRef RealPathName, + llvm::sys::path::Style Separator) { auto RealPathComponentIter = llvm::sys::path::rbegin(RealPathName); auto RealPathComponentEnd = llvm::sys::path::rend(RealPathName); int Cnt = 0; bool SuggestReplacement = false; + + auto IsSep = [Separator](StringRef Component) { + return Component.size() == 1 && + llvm::sys::path::is_separator(Component[0], Separator); + }; + // Below is a best-effort to handle ".." in paths. It is admittedly // not 100% correct in the presence of symlinks. for (auto &Component : llvm::reverse(Components)) { @@ -1872,10 +1879,11 @@ static bool trySimplifyPath(SmallVectorImpl<StringRef> &Components, } else if (Cnt) { --Cnt; } else if (RealPathComponentIter != RealPathComponentEnd) { - if (Component != *RealPathComponentIter) { - // If these path components differ by more than just case, then we - // may be looking at symlinked paths. Bail on this diagnostic to avoid - // noisy false positives. + if (!IsSep(Component) && !IsSep(*RealPathComponentIter) && + Component != *RealPathComponentIter) { + // If these non-separator path components differ by more than just case, + // then we may be looking at symlinked paths. Bail on this diagnostic to + // avoid noisy false positives. SuggestReplacement = RealPathComponentIter->equals_insensitive(Component); if (!SuggestReplacement) @@ -2451,7 +2459,7 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport( } #endif - if (trySimplifyPath(Components, RealPathName)) { + if (trySimplifyPath(Components, RealPathName, BackslashStyle)) { SmallString<128> Path; Path.reserve(Name.size()+2); Path.push_back(isAngled ? '<' : '"'); @@ -2474,7 +2482,7 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport( // got copied when the C: was processed and we want to skip that entry. if (!(Component.size() == 1 && IsSep(Component[0]))) Path.append(Component); - else if (!Path.empty()) + else if (Path.size() != 1) continue; // Append the separator(s) the user used, or the close quote diff --git a/contrib/llvm-project/clang/lib/Parse/ParseOpenACC.cpp b/contrib/llvm-project/clang/lib/Parse/ParseOpenACC.cpp index f7f096762e91..67325f0a286a 100644 --- a/contrib/llvm-project/clang/lib/Parse/ParseOpenACC.cpp +++ b/contrib/llvm-project/clang/lib/Parse/ParseOpenACC.cpp @@ -69,6 +69,29 @@ OpenACCDirectiveKindEx getOpenACCDirectiveKind(Token Tok) { .Default(OpenACCDirectiveKindEx::Invalid); } +// Translate single-token string representations to the OpenCC Clause Kind. +OpenACCClauseKind getOpenACCClauseKind(Token Tok) { + // auto is a keyword in some language modes, so make sure we parse it + // correctly. + if (Tok.is(tok::kw_auto)) + return OpenACCClauseKind::Auto; + + if (!Tok.is(tok::identifier)) + return OpenACCClauseKind::Invalid; + + return llvm::StringSwitch<OpenACCClauseKind>( + Tok.getIdentifierInfo()->getName()) + .Case("auto", OpenACCClauseKind::Auto) + .Case("finalize", OpenACCClauseKind::Finalize) + .Case("if_present", OpenACCClauseKind::IfPresent) + .Case("independent", OpenACCClauseKind::Independent) + .Case("nohost", OpenACCClauseKind::NoHost) + .Case("seq", OpenACCClauseKind::Seq) + .Case("vector", OpenACCClauseKind::Vector) + .Case("worker", OpenACCClauseKind::Worker) + .Default(OpenACCClauseKind::Invalid); +} + // Since 'atomic' is effectively a compound directive, this will decode the // second part of the directive. OpenACCAtomicKind getOpenACCAtomicKind(Token Tok) { @@ -164,6 +187,10 @@ ParseOpenACCEnterExitDataDirective(Parser &P, Token FirstTok, return OpenACCDirectiveKind::Invalid; } + // Consume the second name anyway, this way we can continue on without making + // this oddly look like a clause. + P.ConsumeAnyToken(); + if (!isOpenACCDirectiveKind(OpenACCDirectiveKind::Data, SecondTok)) { if (!SecondTok.is(tok::identifier)) P.Diag(SecondTok, diag::err_expected) << tok::identifier; @@ -174,8 +201,6 @@ ParseOpenACCEnterExitDataDirective(Parser &P, Token FirstTok, return OpenACCDirectiveKind::Invalid; } - P.ConsumeToken(); - return ExtDirKind == OpenACCDirectiveKindEx::Enter ? OpenACCDirectiveKind::EnterData : OpenACCDirectiveKind::ExitData; @@ -208,6 +233,10 @@ OpenACCDirectiveKind ParseOpenACCDirectiveKind(Parser &P) { // introspect on the spelling before then. if (FirstTok.isNot(tok::identifier)) { P.Diag(FirstTok, diag::err_acc_missing_directive); + + if (P.getCurToken().isNot(tok::annot_pragma_openacc_end)) + P.ConsumeAnyToken(); + return OpenACCDirectiveKind::Invalid; } @@ -262,12 +291,57 @@ OpenACCDirectiveKind ParseOpenACCDirectiveKind(Parser &P) { return DirKind; } +// The OpenACC Clause List is a comma or space-delimited list of clauses (see +// the comment on ParseOpenACCClauseList). The concept of a 'clause' doesn't +// really have its owner grammar and each individual one has its own definition. +// However, they all are named with a single-identifier (or auto!) token, +// followed in some cases by either braces or parens. +bool ParseOpenACCClause(Parser &P) { + if (!P.getCurToken().isOneOf(tok::identifier, tok::kw_auto)) + return P.Diag(P.getCurToken(), diag::err_expected) << tok::identifier; + + OpenACCClauseKind Kind = getOpenACCClauseKind(P.getCurToken()); + + if (Kind == OpenACCClauseKind::Invalid) + return P.Diag(P.getCurToken(), diag::err_acc_invalid_clause) + << P.getCurToken().getIdentifierInfo(); + + // Consume the clause name. + P.ConsumeToken(); + + // FIXME: For future clauses, we need to handle parens/etc below. + return false; +} + +// Skip until we see the end of pragma token, but don't consume it. This is us +// just giving up on the rest of the pragma so we can continue executing. We +// have to do this because 'SkipUntil' considers paren balancing, which isn't +// what we want. +void SkipUntilEndOfDirective(Parser &P) { + while (P.getCurToken().isNot(tok::annot_pragma_openacc_end)) + P.ConsumeAnyToken(); +} + +// OpenACC 3.3, section 1.7: +// To simplify the specification and convey appropriate constraint information, +// a pqr-list is a comma-separated list of pdr items. The one exception is a +// clause-list, which is a list of one or more clauses optionally separated by +// commas. void ParseOpenACCClauseList(Parser &P) { - // FIXME: In the future, we'll start parsing the clauses here, but for now we - // haven't implemented that, so just emit the unimplemented diagnostic and - // fail reasonably. - if (P.getCurToken().isNot(tok::annot_pragma_openacc_end)) - P.Diag(P.getCurToken(), diag::warn_pragma_acc_unimplemented_clause_parsing); + bool FirstClause = true; + while (P.getCurToken().isNot(tok::annot_pragma_openacc_end)) { + // Comma is optional in a clause-list. + if (!FirstClause && P.getCurToken().is(tok::comma)) + P.ConsumeToken(); + FirstClause = false; + + // Recovering from a bad clause is really difficult, so we just give up on + // error. + if (ParseOpenACCClause(P)) { + SkipUntilEndOfDirective(P); + return; + } + } } } // namespace @@ -499,7 +573,9 @@ void Parser::ParseOpenACCDirective() { ParseOpenACCClauseList(*this); Diag(getCurToken(), diag::warn_pragma_acc_unimplemented); - SkipUntil(tok::annot_pragma_openacc_end); + assert(Tok.is(tok::annot_pragma_openacc_end) && + "Didn't parse all OpenACC Clauses"); + ConsumeAnnotationToken(); } // Parse OpenACC directive on a declaration. diff --git a/contrib/llvm-project/clang/lib/Sema/SemaChecking.cpp b/contrib/llvm-project/clang/lib/Sema/SemaChecking.cpp index d4a40b850cea..2a69325f0295 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaChecking.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaChecking.cpp @@ -3156,7 +3156,6 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall, const FunctionDecl *FD, ArmStreamingType BuiltinType) { ArmStreamingType FnType = getArmStreamingFnType(FD); - if (FnType == ArmStreaming && BuiltinType == ArmNonStreaming) { S.Diag(TheCall->getBeginLoc(), diag::warn_attribute_arm_sm_incompat_builtin) << TheCall->getSourceRange() << "streaming"; @@ -3168,9 +3167,77 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall, << TheCall->getSourceRange() << "streaming compatible"; return; } + + if (FnType == ArmNonStreaming && BuiltinType == ArmStreaming) { + S.Diag(TheCall->getBeginLoc(), diag::warn_attribute_arm_sm_incompat_builtin) + << TheCall->getSourceRange() << "non-streaming"; + } +} + +static bool hasSMEZAState(const FunctionDecl *FD) { + if (FD->hasAttr<ArmNewZAAttr>()) + return true; + if (const auto *T = FD->getType()->getAs<FunctionProtoType>()) + if (T->getAArch64SMEAttributes() & FunctionType::SME_PStateZASharedMask) + return true; + return false; +} + +static bool hasSMEZAState(unsigned BuiltinID) { + switch (BuiltinID) { + default: + return false; +#define GET_SME_BUILTIN_HAS_ZA_STATE +#include "clang/Basic/arm_sme_builtins_za_state.inc" +#undef GET_SME_BUILTIN_HAS_ZA_STATE + } +} + +bool Sema::CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { + if (const FunctionDecl *FD = getCurFunctionDecl()) { + std::optional<ArmStreamingType> BuiltinType; + + switch (BuiltinID) { +#define GET_SME_STREAMING_ATTRS +#include "clang/Basic/arm_sme_streaming_attrs.inc" +#undef GET_SME_STREAMING_ATTRS + } + + if (BuiltinType) + checkArmStreamingBuiltin(*this, TheCall, FD, *BuiltinType); + + if (hasSMEZAState(BuiltinID) && !hasSMEZAState(FD)) + Diag(TheCall->getBeginLoc(), + diag::warn_attribute_arm_za_builtin_no_za_state) + << TheCall->getSourceRange(); + } + + // Range check SME intrinsics that take immediate values. + SmallVector<std::tuple<int, int, int>, 3> ImmChecks; + + switch (BuiltinID) { + default: + return false; +#define GET_SME_IMMEDIATE_CHECK +#include "clang/Basic/arm_sme_sema_rangechecks.inc" +#undef GET_SME_IMMEDIATE_CHECK + } + + return ParseSVEImmChecks(TheCall, ImmChecks); } bool Sema::CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { + if (const FunctionDecl *FD = getCurFunctionDecl()) { + std::optional<ArmStreamingType> BuiltinType; + + switch (BuiltinID) { +#define GET_SVE_STREAMING_ATTRS +#include "clang/Basic/arm_sve_streaming_attrs.inc" +#undef GET_SVE_STREAMING_ATTRS + } + if (BuiltinType) + checkArmStreamingBuiltin(*this, TheCall, FD, *BuiltinType); + } // Range check SVE intrinsics that take immediate values. SmallVector<std::tuple<int, int, int>, 3> ImmChecks; @@ -3180,9 +3247,6 @@ bool Sema::CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { #define GET_SVE_IMMEDIATE_CHECK #include "clang/Basic/arm_sve_sema_rangechecks.inc" #undef GET_SVE_IMMEDIATE_CHECK -#define GET_SME_IMMEDIATE_CHECK -#include "clang/Basic/arm_sme_sema_rangechecks.inc" -#undef GET_SME_IMMEDIATE_CHECK } return ParseSVEImmChecks(TheCall, ImmChecks); @@ -3569,6 +3633,9 @@ bool Sema::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI, if (CheckSVEBuiltinFunctionCall(BuiltinID, TheCall)) return true; + if (CheckSMEBuiltinFunctionCall(BuiltinID, TheCall)) + return true; + // For intrinsics which take an immediate value as part of the instruction, // range check them here. unsigned i = 0, l = 0, u = 0; @@ -5322,7 +5389,7 @@ bool Sema::CheckRISCVBuiltinFunctionCall(const TargetInfo &TI, QualType Op2Type = TheCall->getArg(1)->getType(); QualType Op3Type = TheCall->getArg(2)->getType(); uint64_t ElemSize = Op1Type->isRVVType(32, false) ? 32 : 64; - if (ElemSize == 64 && !TI.hasFeature("experimental-zvknhb")) + if (ElemSize == 64 && !TI.hasFeature("zvknhb")) return Diag(TheCall->getBeginLoc(), diag::err_riscv_type_requires_extension) << Op1Type << "zvknhb"; diff --git a/contrib/llvm-project/clang/lib/Sema/SemaDecl.cpp b/contrib/llvm-project/clang/lib/Sema/SemaDecl.cpp index be6a136ef37b..ffbe317d5599 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaDecl.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaDecl.cpp @@ -2005,12 +2005,12 @@ static bool ShouldDiagnoseUnusedDecl(const LangOptions &LangOpts, if (D->isInvalidDecl()) return false; - if (auto *DD = dyn_cast<DecompositionDecl>(D)) { + if (const auto *DD = dyn_cast<DecompositionDecl>(D)) { // For a decomposition declaration, warn if none of the bindings are // referenced, instead of if the variable itself is referenced (which // it is, by the bindings' expressions). bool IsAllPlaceholders = true; - for (auto *BD : DD->bindings()) { + for (const auto *BD : DD->bindings()) { if (BD->isReferenced()) return false; IsAllPlaceholders = IsAllPlaceholders && BD->isPlaceholderVar(LangOpts); @@ -2054,7 +2054,7 @@ static bool ShouldDiagnoseUnusedDecl(const LangOptions &LangOpts, if (const VarDecl *VD = dyn_cast<VarDecl>(D)) { const Expr *Init = VD->getInit(); - if (const auto *Cleanups = dyn_cast_or_null<ExprWithCleanups>(Init)) + if (const auto *Cleanups = dyn_cast_if_present<ExprWithCleanups>(Init)) Init = Cleanups->getSubExpr(); const auto *Ty = VD->getType().getTypePtr(); @@ -2068,11 +2068,10 @@ static bool ShouldDiagnoseUnusedDecl(const LangOptions &LangOpts, // Warn for reference variables whose initializtion performs lifetime // extension. - if (const auto *MTE = dyn_cast_or_null<MaterializeTemporaryExpr>(Init)) { - if (MTE->getExtendingDecl()) { - Ty = VD->getType().getNonReferenceType().getTypePtr(); - Init = MTE->getSubExpr()->IgnoreImplicitAsWritten(); - } + if (const auto *MTE = dyn_cast_if_present<MaterializeTemporaryExpr>(Init); + MTE && MTE->getExtendingDecl()) { + Ty = VD->getType().getNonReferenceType().getTypePtr(); + Init = MTE->getSubExpr()->IgnoreImplicitAsWritten(); } // If we failed to complete the type for some reason, or if the type is @@ -2089,15 +2088,14 @@ static bool ShouldDiagnoseUnusedDecl(const LangOptions &LangOpts, if (Tag->hasAttr<UnusedAttr>()) return false; - if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(Tag)) { + if (const auto *RD = dyn_cast<CXXRecordDecl>(Tag)) { if (!RD->hasTrivialDestructor() && !RD->hasAttr<WarnUnusedAttr>()) return false; if (Init) { - const CXXConstructExpr *Construct = - dyn_cast<CXXConstructExpr>(Init); + const auto *Construct = dyn_cast<CXXConstructExpr>(Init); if (Construct && !Construct->isElidable()) { - CXXConstructorDecl *CD = Construct->getConstructor(); + const CXXConstructorDecl *CD = Construct->getConstructor(); if (!CD->isTrivial() && !RD->hasAttr<WarnUnusedAttr>() && (VD->getInit()->isValueDependent() || !VD->evaluateValue())) return false; @@ -2211,10 +2209,9 @@ void Sema::DiagnoseUnusedButSetDecl(const VarDecl *VD, return; // In C++, don't warn for record types that don't have WarnUnusedAttr, to // mimic gcc's behavior. - if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(Tag)) { - if (!RD->hasAttr<WarnUnusedAttr>()) - return; - } + if (const auto *RD = dyn_cast<CXXRecordDecl>(Tag); + RD && !RD->hasAttr<WarnUnusedAttr>()) + return; } // Don't warn about __block Objective-C pointer variables, as they might @@ -12957,7 +12954,7 @@ QualType Sema::deduceVarTypeFromInitializer(VarDecl *VDecl, // FIXME: Initialization should not be taking a mutable list of inits. SmallVector<Expr*, 8> InitsCopy(DeduceInits.begin(), DeduceInits.end()); return DeduceTemplateSpecializationFromInitializer(TSI, Entity, Kind, - InitsCopy, PL); + InitsCopy); } if (DirectInit) { @@ -19518,20 +19515,6 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl, CDecl->setIvarRBraceLoc(RBrac); } } - - // Check the "counted_by" attribute to ensure that the count field exists in - // the struct. Make sure we're performing this check on the outer-most - // record. This is a C-only feature. - if (!getLangOpts().CPlusPlus && Record && - !isa<RecordDecl>(Record->getParent())) { - auto Pred = [](const Decl *D) { - if (const auto *FD = dyn_cast_if_present<FieldDecl>(D)) - return FD->hasAttr<CountedByAttr>(); - return false; - }; - if (const FieldDecl *FD = Record->findFieldIf(Pred)) - CheckCountedByAttr(S, FD); - } } /// Determine whether the given integral value is representable within diff --git a/contrib/llvm-project/clang/lib/Sema/SemaDeclAttr.cpp b/contrib/llvm-project/clang/lib/Sema/SemaDeclAttr.cpp index 5b29b05dee54..af8b90ecfed9 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaDeclAttr.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaDeclAttr.cpp @@ -8445,92 +8445,6 @@ static void handleZeroCallUsedRegsAttr(Sema &S, Decl *D, const ParsedAttr &AL) { D->addAttr(ZeroCallUsedRegsAttr::Create(S.Context, Kind, AL)); } -static void handleCountedByAttr(Sema &S, Decl *D, const ParsedAttr &AL) { - if (!AL.isArgIdent(0)) { - S.Diag(AL.getLoc(), diag::err_attribute_argument_type) - << AL << AANT_ArgumentIdentifier; - return; - } - - IdentifierLoc *IL = AL.getArgAsIdent(0); - CountedByAttr *CBA = - ::new (S.Context) CountedByAttr(S.Context, AL, IL->Ident); - CBA->setCountedByFieldLoc(IL->Loc); - D->addAttr(CBA); -} - -bool Sema::CheckCountedByAttr(Scope *S, const FieldDecl *FD) { - const auto *CBA = FD->getAttr<CountedByAttr>(); - const IdentifierInfo *FieldName = CBA->getCountedByField(); - DeclarationNameInfo NameInfo(FieldName, - CBA->getCountedByFieldLoc().getBegin()); - - LookupResult MemResult(*this, NameInfo, Sema::LookupMemberName); - LookupName(MemResult, S); - - if (MemResult.empty()) { - // The "counted_by" field needs to exist within the struct. - LookupResult OrdResult(*this, NameInfo, Sema::LookupOrdinaryName); - LookupName(OrdResult, S); - - if (!OrdResult.empty()) { - SourceRange SR = FD->getLocation(); - Diag(SR.getBegin(), diag::err_counted_by_must_be_in_structure) - << FieldName << SR; - - if (auto *ND = OrdResult.getAsSingle<NamedDecl>()) { - SR = ND->getLocation(); - Diag(SR.getBegin(), diag::note_flexible_array_counted_by_attr_field) - << ND << SR; - } - return true; - } - - CXXScopeSpec SS; - DeclFilterCCC<FieldDecl> Filter(FieldName); - return DiagnoseEmptyLookup(S, SS, MemResult, Filter, nullptr, std::nullopt, - const_cast<DeclContext *>(FD->getDeclContext())); - } - - LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = - LangOptions::StrictFlexArraysLevelKind::IncompleteOnly; - - if (!Decl::isFlexibleArrayMemberLike(Context, FD, FD->getType(), - StrictFlexArraysLevel, true)) { - // The "counted_by" attribute must be on a flexible array member. - SourceRange SR = FD->getLocation(); - Diag(SR.getBegin(), diag::err_counted_by_attr_not_on_flexible_array_member) - << SR; - return true; - } - - if (const FieldDecl *Field = MemResult.getAsSingle<FieldDecl>()) { - if (Field->hasAttr<CountedByAttr>()) { - // The "counted_by" field can't point to the flexible array member. - SourceRange SR = CBA->getCountedByFieldLoc(); - Diag(SR.getBegin(), diag::err_counted_by_attr_refers_to_flexible_array) - << CBA->getCountedByField() << SR; - return true; - } - - if (!Field->getType()->isIntegerType() || - Field->getType()->isBooleanType()) { - // The "counted_by" field must have an integer type. - SourceRange SR = CBA->getCountedByFieldLoc(); - Diag(SR.getBegin(), - diag::err_flexible_array_counted_by_attr_field_not_integer) - << CBA->getCountedByField() << SR; - - SR = Field->getLocation(); - Diag(SR.getBegin(), diag::note_flexible_array_counted_by_attr_field) - << Field << SR; - return true; - } - } - - return false; -} - static void handleFunctionReturnThunksAttr(Sema &S, Decl *D, const ParsedAttr &AL) { StringRef KindStr; @@ -9488,10 +9402,6 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, handleAvailableOnlyInDefaultEvalMethod(S, D, AL); break; - case ParsedAttr::AT_CountedBy: - handleCountedByAttr(S, D, AL); - break; - // Microsoft attributes: case ParsedAttr::AT_LayoutVersion: handleLayoutVersion(S, D, AL); diff --git a/contrib/llvm-project/clang/lib/Sema/SemaExpr.cpp b/contrib/llvm-project/clang/lib/Sema/SemaExpr.cpp index c7185d56cc99..960f513d1111 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaExpr.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaExpr.cpp @@ -2469,8 +2469,7 @@ bool Sema::DiagnoseDependentMemberLookup(const LookupResult &R) { bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R, CorrectionCandidateCallback &CCC, TemplateArgumentListInfo *ExplicitTemplateArgs, - ArrayRef<Expr *> Args, DeclContext *LookupCtx, - TypoExpr **Out) { + ArrayRef<Expr *> Args, TypoExpr **Out) { DeclarationName Name = R.getLookupName(); unsigned diagnostic = diag::err_undeclared_var_use; @@ -2486,8 +2485,7 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R, // unqualified lookup. This is useful when (for example) the // original lookup would not have found something because it was a // dependent name. - DeclContext *DC = - LookupCtx ? LookupCtx : (SS.isEmpty() ? CurContext : nullptr); + DeclContext *DC = SS.isEmpty() ? CurContext : nullptr; while (DC) { if (isa<CXXRecordDecl>(DC)) { LookupQualifiedName(R, DC); @@ -2530,12 +2528,12 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R, emitEmptyLookupTypoDiagnostic(TC, *this, SS, Name, TypoLoc, Args, diagnostic, diagnostic_suggest); }, - nullptr, CTK_ErrorRecovery, LookupCtx); + nullptr, CTK_ErrorRecovery); if (*Out) return true; - } else if (S && (Corrected = - CorrectTypo(R.getLookupNameInfo(), R.getLookupKind(), S, - &SS, CCC, CTK_ErrorRecovery, LookupCtx))) { + } else if (S && + (Corrected = CorrectTypo(R.getLookupNameInfo(), R.getLookupKind(), + S, &SS, CCC, CTK_ErrorRecovery))) { std::string CorrectedStr(Corrected.getAsString(getLangOpts())); bool DroppedSpecifier = Corrected.WillReplaceSpecifier() && Name.getAsString() == CorrectedStr; @@ -2825,7 +2823,7 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS, // a template name, but we happen to have always already looked up the name // before we get here if it must be a template name. if (DiagnoseEmptyLookup(S, SS, R, CCC ? *CCC : DefaultValidator, nullptr, - std::nullopt, nullptr, &TE)) { + std::nullopt, &TE)) { if (TE && KeywordReplacement) { auto &State = getTypoExprState(TE); auto BestTC = State.Consumer->getNextCorrection(); diff --git a/contrib/llvm-project/clang/lib/Sema/SemaExprCXX.cpp b/contrib/llvm-project/clang/lib/Sema/SemaExprCXX.cpp index 081b568762ae..4ae04358d5df 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaExprCXX.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaExprCXX.cpp @@ -843,21 +843,21 @@ Sema::ActOnCXXThrow(Scope *S, SourceLocation OpLoc, Expr *Ex) { // operation from the operand to the exception object (15.1) can be // omitted by constructing the automatic object directly into the // exception object - if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Ex->IgnoreParens())) - if (VarDecl *Var = dyn_cast<VarDecl>(DRE->getDecl())) { - if (Var->hasLocalStorage() && !Var->getType().isVolatileQualified()) { - for( ; S; S = S->getParent()) { - if (S->isDeclScope(Var)) { - IsThrownVarInScope = true; - break; - } - - // FIXME: Many of the scope checks here seem incorrect. - if (S->getFlags() & - (Scope::FnScope | Scope::ClassScope | Scope::BlockScope | - Scope::ObjCMethodScope | Scope::TryScope)) - break; + if (const auto *DRE = dyn_cast<DeclRefExpr>(Ex->IgnoreParens())) + if (const auto *Var = dyn_cast<VarDecl>(DRE->getDecl()); + Var && Var->hasLocalStorage() && + !Var->getType().isVolatileQualified()) { + for (; S; S = S->getParent()) { + if (S->isDeclScope(Var)) { + IsThrownVarInScope = true; + break; } + + // FIXME: Many of the scope checks here seem incorrect. + if (S->getFlags() & + (Scope::FnScope | Scope::ClassScope | Scope::BlockScope | + Scope::ObjCMethodScope | Scope::TryScope)) + break; } } } diff --git a/contrib/llvm-project/clang/lib/Sema/SemaExprMember.cpp b/contrib/llvm-project/clang/lib/Sema/SemaExprMember.cpp index 473eea55bb6b..2abec3d86a27 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaExprMember.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaExprMember.cpp @@ -253,7 +253,9 @@ static void diagnoseInstanceReference(Sema &SemaRef, SemaRef.Diag(Loc, diag::err_member_call_without_object) << Range << /*static*/ 0; else { - const auto *Callee = dyn_cast<CXXMethodDecl>(Rep); + if (const auto *Tpl = dyn_cast<FunctionTemplateDecl>(Rep)) + Rep = Tpl->getTemplatedDecl(); + const auto *Callee = cast<CXXMethodDecl>(Rep); auto Diag = SemaRef.Diag(Loc, diag::err_member_call_without_object) << Range << Callee->isExplicitObjectMemberFunction(); if (!Replacement.empty()) diff --git a/contrib/llvm-project/clang/lib/Sema/SemaInit.cpp b/contrib/llvm-project/clang/lib/Sema/SemaInit.cpp index 035eaae58965..61d244f3bb97 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaInit.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaInit.cpp @@ -25,6 +25,7 @@ #include "clang/Sema/EnterExpressionEvaluationContext.h" #include "clang/Sema/Initialization.h" #include "clang/Sema/Lookup.h" +#include "clang/Sema/Ownership.h" #include "clang/Sema/SemaInternal.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/FoldingSet.h" @@ -465,8 +466,7 @@ class InitListChecker { void FillInEmptyInitForField(unsigned Init, FieldDecl *Field, const InitializedEntity &ParentEntity, InitListExpr *ILE, bool &RequiresSecondPass, - bool FillWithNoInit = false, - bool WarnIfMissing = false); + bool FillWithNoInit = false); void FillInEmptyInitializations(const InitializedEntity &Entity, InitListExpr *ILE, bool &RequiresSecondPass, InitListExpr *OuterILE, unsigned OuterIndex, @@ -655,16 +655,11 @@ void InitListChecker::FillInEmptyInitForBase( } } -static bool hasAnyDesignatedInits(const InitListExpr *IL) { - return llvm::any_of(*IL, [=](const Stmt *Init) { - return isa_and_nonnull<DesignatedInitExpr>(Init); - }); -} - -void InitListChecker::FillInEmptyInitForField( - unsigned Init, FieldDecl *Field, const InitializedEntity &ParentEntity, - InitListExpr *ILE, bool &RequiresSecondPass, bool FillWithNoInit, - bool WarnIfMissing) { +void InitListChecker::FillInEmptyInitForField(unsigned Init, FieldDecl *Field, + const InitializedEntity &ParentEntity, + InitListExpr *ILE, + bool &RequiresSecondPass, + bool FillWithNoInit) { SourceLocation Loc = ILE->getEndLoc(); unsigned NumInits = ILE->getNumInits(); InitializedEntity MemberEntity @@ -732,52 +727,15 @@ void InitListChecker::FillInEmptyInitForField( if (hadError || VerifyOnly) { // Do nothing - } else { - if (WarnIfMissing) { - auto CheckAnonMember = [&](const FieldDecl *FD, - auto &&CheckAnonMember) -> FieldDecl * { - FieldDecl *Uninitialized = nullptr; - RecordDecl *RD = FD->getType()->getAsRecordDecl(); - assert(RD && "Not anonymous member checked?"); - for (auto *F : RD->fields()) { - FieldDecl *UninitializedFieldInF = nullptr; - if (F->isAnonymousStructOrUnion()) - UninitializedFieldInF = CheckAnonMember(F, CheckAnonMember); - else if (!F->isUnnamedBitfield() && - !F->getType()->isIncompleteArrayType() && - !F->hasInClassInitializer()) - UninitializedFieldInF = F; - - if (RD->isUnion() && !UninitializedFieldInF) - return nullptr; - if (!Uninitialized) - Uninitialized = UninitializedFieldInF; - } - return Uninitialized; - }; - - FieldDecl *FieldToDiagnose = nullptr; - if (Field->isAnonymousStructOrUnion()) - FieldToDiagnose = CheckAnonMember(Field, CheckAnonMember); - else if (!Field->isUnnamedBitfield() && - !Field->getType()->isIncompleteArrayType()) - FieldToDiagnose = Field; - - if (FieldToDiagnose) - SemaRef.Diag(Loc, diag::warn_missing_field_initializers) - << FieldToDiagnose; - } - - if (Init < NumInits) { - ILE->setInit(Init, MemberInit.getAs<Expr>()); - } else if (!isa<ImplicitValueInitExpr>(MemberInit.get())) { - // Empty initialization requires a constructor call, so - // extend the initializer list to include the constructor - // call and make a note that we'll need to take another pass - // through the initializer list. - ILE->updateInit(SemaRef.Context, Init, MemberInit.getAs<Expr>()); - RequiresSecondPass = true; - } + } else if (Init < NumInits) { + ILE->setInit(Init, MemberInit.getAs<Expr>()); + } else if (!isa<ImplicitValueInitExpr>(MemberInit.get())) { + // Empty initialization requires a constructor call, so + // extend the initializer list to include the constructor + // call and make a note that we'll need to take another pass + // through the initializer list. + ILE->updateInit(SemaRef.Context, Init, MemberInit.getAs<Expr>()); + RequiresSecondPass = true; } } else if (InitListExpr *InnerILE = dyn_cast<InitListExpr>(ILE->getInit(Init))) { @@ -845,36 +803,9 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity, } } } else { - InitListExpr *SForm = - ILE->isSyntacticForm() ? ILE : ILE->getSyntacticForm(); // The fields beyond ILE->getNumInits() are default initialized, so in // order to leave them uninitialized, the ILE is expanded and the extra // fields are then filled with NoInitExpr. - - // Some checks that are required for missing fields warning are bound to - // how many elements the initializer list originally was provided; perform - // them before the list is expanded. - bool WarnIfMissingField = - !SForm->isIdiomaticZeroInitializer(SemaRef.getLangOpts()) && - ILE->getNumInits(); - - // Disable check for missing fields when designators are used in C to - // match gcc behaviour. - // FIXME: Should we emulate possible gcc warning bug? - WarnIfMissingField &= - SemaRef.getLangOpts().CPlusPlus || !hasAnyDesignatedInits(SForm); - - if (OuterILE) { - // When nested designators are present, there might be two nested init - // lists created and only outer will contain designated initializer - // expression, so check outer list as well. - InitListExpr *OuterSForm = OuterILE->isSyntacticForm() - ? OuterILE - : OuterILE->getSyntacticForm(); - WarnIfMissingField &= SemaRef.getLangOpts().CPlusPlus || - !hasAnyDesignatedInits(OuterSForm); - } - unsigned NumElems = numStructUnionElements(ILE->getType()); if (!RDecl->isUnion() && RDecl->hasFlexibleArrayMember()) ++NumElems; @@ -902,7 +833,7 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity, return; FillInEmptyInitForField(Init, Field, Entity, ILE, RequiresSecondPass, - FillWithNoInit, WarnIfMissingField); + FillWithNoInit); if (hadError) return; @@ -1017,6 +948,13 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity, } } +static bool hasAnyDesignatedInits(const InitListExpr *IL) { + for (const Stmt *Init : *IL) + if (isa_and_nonnull<DesignatedInitExpr>(Init)) + return true; + return false; +} + InitListChecker::InitListChecker( Sema &S, const InitializedEntity &Entity, InitListExpr *IL, QualType &T, bool VerifyOnly, bool TreatUnavailableAsInvalid, bool InOverloadResolution, @@ -2288,8 +2226,12 @@ void InitListChecker::CheckStructUnionTypes( size_t NumRecordDecls = llvm::count_if(RD->decls(), [&](const Decl *D) { return isa<FieldDecl>(D) || isa<RecordDecl>(D); }); + bool CheckForMissingFields = + !IList->isIdiomaticZeroInitializer(SemaRef.getLangOpts()); bool HasDesignatedInit = false; + llvm::SmallPtrSet<FieldDecl *, 4> InitializedFields; + while (Index < IList->getNumInits()) { Expr *Init = IList->getInit(Index); SourceLocation InitLoc = Init->getBeginLoc(); @@ -2313,17 +2255,24 @@ void InitListChecker::CheckStructUnionTypes( // Find the field named by the designated initializer. DesignatedInitExpr::Designator *D = DIE->getDesignator(0); - if (!VerifyOnly && D->isFieldDesignator() && !DesignatedInitFailed) { + if (!VerifyOnly && D->isFieldDesignator()) { FieldDecl *F = D->getFieldDecl(); - QualType ET = SemaRef.Context.getBaseElementType(F->getType()); - if (checkDestructorReference(ET, InitLoc, SemaRef)) { - hadError = true; - return; + InitializedFields.insert(F); + if (!DesignatedInitFailed) { + QualType ET = SemaRef.Context.getBaseElementType(F->getType()); + if (checkDestructorReference(ET, InitLoc, SemaRef)) { + hadError = true; + return; + } } } InitializedSomething = true; + // Disable check for missing fields when designators are used. + // This matches gcc behaviour. + if (!SemaRef.getLangOpts().CPlusPlus) + CheckForMissingFields = false; continue; } @@ -2402,6 +2351,7 @@ void InitListChecker::CheckStructUnionTypes( CheckSubElementType(MemberEntity, IList, Field->getType(), Index, StructuredList, StructuredIndex); InitializedSomething = true; + InitializedFields.insert(*Field); if (RD->isUnion() && StructuredList) { // Initialize the first field within the union. @@ -2411,6 +2361,28 @@ void InitListChecker::CheckStructUnionTypes( ++Field; } + // Emit warnings for missing struct field initializers. + if (!VerifyOnly && InitializedSomething && CheckForMissingFields && + !RD->isUnion()) { + // It is possible we have one or more unnamed bitfields remaining. + // Find first (if any) named field and emit warning. + for (RecordDecl::field_iterator it = HasDesignatedInit ? RD->field_begin() + : Field, + end = RD->field_end(); + it != end; ++it) { + if (HasDesignatedInit && InitializedFields.count(*it)) + continue; + + if (!it->isUnnamedBitfield() && !it->hasInClassInitializer() && + !it->getType()->isIncompleteArrayType()) { + SemaRef.Diag(IList->getSourceRange().getEnd(), + diag::warn_missing_field_initializers) + << *it; + break; + } + } + } + // Check that any remaining fields can be value-initialized if we're not // building a structured list. (If we are, we'll check this later.) if (!StructuredList && Field != FieldEnd && !RD->isUnion() && @@ -5458,18 +5430,12 @@ static void TryOrBuildParenListInitialization( auto HandleInitializedEntity = [&](const InitializedEntity &SubEntity, const InitializationKind &SubKind, Expr *Arg, Expr **InitExpr = nullptr) { - InitializationSequence IS = [&]() { - if (Arg) - return InitializationSequence(S, SubEntity, SubKind, Arg); - return InitializationSequence(S, SubEntity, SubKind, std::nullopt); - }(); + InitializationSequence IS = InitializationSequence( + S, SubEntity, SubKind, Arg ? MultiExprArg(Arg) : std::nullopt); if (IS.Failed()) { if (!VerifyOnly) { - if (Arg) - IS.Diagnose(S, SubEntity, SubKind, Arg); - else - IS.Diagnose(S, SubEntity, SubKind, std::nullopt); + IS.Diagnose(S, SubEntity, SubKind, Arg ? ArrayRef(Arg) : std::nullopt); } else { Sequence.SetFailed( InitializationSequence::FK_ParenthesizedListInitFailed); @@ -5479,10 +5445,8 @@ static void TryOrBuildParenListInitialization( } if (!VerifyOnly) { ExprResult ER; - if (Arg) - ER = IS.Perform(S, SubEntity, SubKind, Arg); - else - ER = IS.Perform(S, SubEntity, SubKind, std::nullopt); + ER = IS.Perform(S, SubEntity, SubKind, + Arg ? MultiExprArg(Arg) : std::nullopt); if (InitExpr) *InitExpr = ER.get(); else @@ -10439,40 +10403,53 @@ static void DiagnoseNarrowingInInitList(Sema &S, // No narrowing occurred. return; - case NK_Type_Narrowing: + case NK_Type_Narrowing: { // This was a floating-to-integer conversion, which is always considered a // narrowing conversion even if the value is a constant and can be // represented exactly as an integer. - S.Diag(PostInit->getBeginLoc(), NarrowingErrs(S.getLangOpts()) - ? diag::ext_init_list_type_narrowing - : diag::warn_init_list_type_narrowing) + QualType T = EntityType.getNonReferenceType(); + S.Diag(PostInit->getBeginLoc(), + NarrowingErrs(S.getLangOpts()) + ? (T == EntityType + ? diag::ext_init_list_type_narrowing + : diag::ext_init_list_type_narrowing_const_reference) + : diag::warn_init_list_type_narrowing) << PostInit->getSourceRange() << PreNarrowingType.getLocalUnqualifiedType() - << EntityType.getNonReferenceType().getLocalUnqualifiedType(); + << T.getLocalUnqualifiedType(); break; + } - case NK_Constant_Narrowing: + case NK_Constant_Narrowing: { // A constant value was narrowed. + QualType T = EntityType.getNonReferenceType(); S.Diag(PostInit->getBeginLoc(), NarrowingErrs(S.getLangOpts()) - ? diag::ext_init_list_constant_narrowing + ? (T == EntityType + ? diag::ext_init_list_constant_narrowing + : diag::ext_init_list_constant_narrowing_const_reference) : diag::warn_init_list_constant_narrowing) << PostInit->getSourceRange() << ConstantValue.getAsString(S.getASTContext(), ConstantType) << EntityType.getNonReferenceType().getLocalUnqualifiedType(); break; + } - case NK_Variable_Narrowing: + case NK_Variable_Narrowing: { // A variable's value may have been narrowed. + QualType T = EntityType.getNonReferenceType(); S.Diag(PostInit->getBeginLoc(), NarrowingErrs(S.getLangOpts()) - ? diag::ext_init_list_variable_narrowing + ? (T == EntityType + ? diag::ext_init_list_variable_narrowing + : diag::ext_init_list_variable_narrowing_const_reference) : diag::warn_init_list_variable_narrowing) << PostInit->getSourceRange() << PreNarrowingType.getLocalUnqualifiedType() << EntityType.getNonReferenceType().getLocalUnqualifiedType(); break; } + } SmallString<128> StaticCast; llvm::raw_svector_ostream OS(StaticCast); @@ -10590,7 +10567,7 @@ static bool isOrIsDerivedFromSpecializationOf(CXXRecordDecl *RD, QualType Sema::DeduceTemplateSpecializationFromInitializer( TypeSourceInfo *TSInfo, const InitializedEntity &Entity, - const InitializationKind &Kind, MultiExprArg Inits, ParenListExpr *PL) { + const InitializationKind &Kind, MultiExprArg Inits) { auto *DeducedTST = dyn_cast<DeducedTemplateSpecializationType>( TSInfo->getType()->getContainedDeducedType()); assert(DeducedTST && "not a deduced template specialization type"); @@ -10821,9 +10798,12 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer( if (getLangOpts().CPlusPlus20 && !HasAnyDeductionGuide) { if (ListInit && ListInit->getNumInits()) { SynthesizeAggrGuide(ListInit); - } else if (PL && PL->getNumExprs()) { - InitListExpr TempListInit(getASTContext(), PL->getLParenLoc(), - PL->exprs(), PL->getRParenLoc()); + } else if (Inits.size()) { // parenthesized expression-list + // Inits are expressions inside the parentheses. We don't have + // the parentheses source locations, use the begin/end of Inits as the + // best heuristic. + InitListExpr TempListInit(getASTContext(), Inits.front()->getBeginLoc(), + Inits, Inits.back()->getEndLoc()); SynthesizeAggrGuide(&TempListInit); } } diff --git a/contrib/llvm-project/clang/lib/Sema/SemaOpenMP.cpp b/contrib/llvm-project/clang/lib/Sema/SemaOpenMP.cpp index e400f248d15a..3826994ef212 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaOpenMP.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaOpenMP.cpp @@ -14658,6 +14658,19 @@ StmtResult Sema::ActOnOpenMPTargetTeamsDirective(ArrayRef<OMPClause *> Clauses, } setFunctionHasBranchProtectedScope(); + const OMPClause *BareClause = nullptr; + bool HasThreadLimitAndNumTeamsClause = hasClauses(Clauses, OMPC_num_teams) && + hasClauses(Clauses, OMPC_thread_limit); + bool HasBareClause = llvm::any_of(Clauses, [&](const OMPClause *C) { + BareClause = C; + return C->getClauseKind() == OMPC_ompx_bare; + }); + + if (HasBareClause && !HasThreadLimitAndNumTeamsClause) { + Diag(BareClause->getBeginLoc(), diag::err_ompx_bare_no_grid); + return StmtError(); + } + return OMPTargetTeamsDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt); } diff --git a/contrib/llvm-project/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/contrib/llvm-project/clang/lib/Sema/SemaRISCVVectorLookup.cpp index e4642e4da016..3ed3e6195441 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaRISCVVectorLookup.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaRISCVVectorLookup.cpp @@ -206,15 +206,16 @@ void RISCVIntrinsicManagerImpl::ConstructRVVIntrinsics( {"xsfvfwmaccqqq", RVV_REQ_Xsfvfwmaccqqq}, {"xsfvqmaccdod", RVV_REQ_Xsfvqmaccdod}, {"xsfvqmaccqoq", RVV_REQ_Xsfvqmaccqoq}, - {"experimental-zvbb", RVV_REQ_Zvbb}, - {"experimental-zvbc", RVV_REQ_Zvbc}, - {"experimental-zvkb", RVV_REQ_Zvkb}, - {"experimental-zvkg", RVV_REQ_Zvkg}, - {"experimental-zvkned", RVV_REQ_Zvkned}, - {"experimental-zvknha", RVV_REQ_Zvknha}, - {"experimental-zvknhb", RVV_REQ_Zvknhb}, - {"experimental-zvksed", RVV_REQ_Zvksed}, - {"experimental-zvksh", RVV_REQ_Zvksh}}; + {"zvbb", RVV_REQ_Zvbb}, + {"zvbc", RVV_REQ_Zvbc}, + {"zvkb", RVV_REQ_Zvkb}, + {"zvkg", RVV_REQ_Zvkg}, + {"zvkned", RVV_REQ_Zvkned}, + {"zvknha", RVV_REQ_Zvknha}, + {"zvknhb", RVV_REQ_Zvknhb}, + {"zvksed", RVV_REQ_Zvksed}, + {"zvksh", RVV_REQ_Zvksh}, + {"experimental", RVV_REQ_Experimental}}; // Construction of RVVIntrinsicRecords need to sync with createRVVIntrinsics // in RISCVVEmitter.cpp. diff --git a/contrib/llvm-project/clang/lib/Sema/SemaStmt.cpp b/contrib/llvm-project/clang/lib/Sema/SemaStmt.cpp index 63348d27a8c9..f0b03db69084 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaStmt.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaStmt.cpp @@ -1271,6 +1271,9 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch, bool CaseListIsErroneous = false; + // FIXME: We'd better diagnose missing or duplicate default labels even + // in the dependent case. Because default labels themselves are never + // dependent. for (SwitchCase *SC = SS->getSwitchCaseList(); SC && !HasDependentValue; SC = SC->getNextSwitchCase()) { @@ -1327,9 +1330,6 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch, } } - if (!TheDefaultStmt) - Diag(SwitchLoc, diag::warn_switch_default); - if (!HasDependentValue) { // If we don't have a default statement, check whether the // condition is constant. @@ -1344,6 +1344,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch, assert(!HasConstantCond || (ConstantCondValue.getBitWidth() == CondWidth && ConstantCondValue.isSigned() == CondIsSigned)); + Diag(SwitchLoc, diag::warn_switch_default); } bool ShouldCheckConstantCond = HasConstantCond; diff --git a/contrib/llvm-project/clang/lib/Sema/SemaTemplate.cpp b/contrib/llvm-project/clang/lib/Sema/SemaTemplate.cpp index f10abeaba0d4..5fcc39ec7005 100644 --- a/contrib/llvm-project/clang/lib/Sema/SemaTemplate.cpp +++ b/contrib/llvm-project/clang/lib/Sema/SemaTemplate.cpp @@ -1824,6 +1824,15 @@ static void SetNestedNameSpecifier(Sema &S, TagDecl *T, T->setQualifierInfo(SS.getWithLocInContext(S.Context)); } +// Returns the template parameter list with all default template argument +// information. +static TemplateParameterList *GetTemplateParameterList(TemplateDecl *TD) { + // Make sure we get the template parameter list from the most + // recent declaration, since that is the only one that is guaranteed to + // have all the default template argument information. + return cast<TemplateDecl>(TD->getMostRecentDecl())->getTemplateParameters(); +} + DeclResult Sema::CheckClassTemplate( Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc, CXXScopeSpec &SS, IdentifierInfo *Name, SourceLocation NameLoc, @@ -2061,13 +2070,13 @@ DeclResult Sema::CheckClassTemplate( if (!(TUK == TUK_Friend && CurContext->isDependentContext()) && CheckTemplateParameterList( TemplateParams, - PrevClassTemplate - ? PrevClassTemplate->getMostRecentDecl()->getTemplateParameters() - : nullptr, + PrevClassTemplate ? GetTemplateParameterList(PrevClassTemplate) + : nullptr, (SS.isSet() && SemanticContext && SemanticContext->isRecord() && SemanticContext->isDependentContext()) ? TPC_ClassTemplateMember - : TUK == TUK_Friend ? TPC_FriendClassTemplate : TPC_ClassTemplate, + : TUK == TUK_Friend ? TPC_FriendClassTemplate + : TPC_ClassTemplate, SkipBody)) Invalid = true; @@ -2298,7 +2307,7 @@ struct ConvertConstructorToDeductionGuideTransform { // -- The template parameters are the template parameters of the class // template followed by the template parameters (including default // template arguments) of the constructor, if any. - TemplateParameterList *TemplateParams = Template->getTemplateParameters(); + TemplateParameterList *TemplateParams = GetTemplateParameterList(Template); if (FTD) { TemplateParameterList *InnerParams = FTD->getTemplateParameters(); SmallVector<NamedDecl *, 16> AllParams; @@ -2424,7 +2433,7 @@ struct ConvertConstructorToDeductionGuideTransform { Params.push_back(NewParam); } - return buildDeductionGuide(Template->getTemplateParameters(), nullptr, + return buildDeductionGuide(GetTemplateParameterList(Template), nullptr, ExplicitSpecifier(), TSI, Loc, Loc, Loc); } @@ -5956,12 +5965,7 @@ bool Sema::CheckTemplateArgumentList( // template. TemplateArgumentListInfo NewArgs = TemplateArgs; - // Make sure we get the template parameter list from the most - // recent declaration, since that is the only one that is guaranteed to - // have all the default template argument information. - TemplateParameterList *Params = - cast<TemplateDecl>(Template->getMostRecentDecl()) - ->getTemplateParameters(); + TemplateParameterList *Params = GetTemplateParameterList(Template); SourceLocation RAngleLoc = NewArgs.getRAngleLoc(); diff --git a/contrib/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp b/contrib/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp index 7140a14aefbf..d989707d5575 100644 --- a/contrib/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/contrib/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp @@ -584,7 +584,18 @@ void ASTDeclReader::Visit(Decl *D) { void ASTDeclReader::VisitDecl(Decl *D) { BitsUnpacker DeclBits(Record.readInt()); + auto ModuleOwnership = + (Decl::ModuleOwnershipKind)DeclBits.getNextBits(/*Width=*/3); + D->setReferenced(DeclBits.getNextBit()); + D->Used = DeclBits.getNextBit(); + IsDeclMarkedUsed |= D->Used; + D->setAccess((AccessSpecifier)DeclBits.getNextBits(/*Width=*/2)); + D->setImplicit(DeclBits.getNextBit()); bool HasStandaloneLexicalDC = DeclBits.getNextBit(); + bool HasAttrs = DeclBits.getNextBit(); + D->setTopLevelDeclInObjCContainer(DeclBits.getNextBit()); + D->InvalidDecl = DeclBits.getNextBit(); + D->FromASTFile = true; if (D->isTemplateParameter() || D->isTemplateParameterPack() || isa<ParmVarDecl, ObjCTypeParamDecl>(D)) { @@ -623,20 +634,6 @@ void ASTDeclReader::VisitDecl(Decl *D) { } D->setLocation(ThisDeclLoc); - D->InvalidDecl = DeclBits.getNextBit(); - bool HasAttrs = DeclBits.getNextBit(); - D->setImplicit(DeclBits.getNextBit()); - D->Used = DeclBits.getNextBit(); - IsDeclMarkedUsed |= D->Used; - D->setReferenced(DeclBits.getNextBit()); - D->setTopLevelDeclInObjCContainer(DeclBits.getNextBit()); - D->setAccess((AccessSpecifier)DeclBits.getNextBits(/*Width=*/2)); - D->FromASTFile = true; - auto ModuleOwnership = - (Decl::ModuleOwnershipKind)DeclBits.getNextBits(/*Width=*/3); - bool ModulePrivate = - (ModuleOwnership == Decl::ModuleOwnershipKind::ModulePrivate); - if (HasAttrs) { AttrVec Attrs; Record.readAttributes(Attrs); @@ -647,8 +644,9 @@ void ASTDeclReader::VisitDecl(Decl *D) { // Determine whether this declaration is part of a (sub)module. If so, it // may not yet be visible. + bool ModulePrivate = + (ModuleOwnership == Decl::ModuleOwnershipKind::ModulePrivate); if (unsigned SubmoduleID = readSubmoduleID()) { - switch (ModuleOwnership) { case Decl::ModuleOwnershipKind::Visible: ModuleOwnership = Decl::ModuleOwnershipKind::VisibleWhenImported; @@ -1065,9 +1063,11 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) { // after everything else is read. BitsUnpacker FunctionDeclBits(Record.readInt()); + FD->setCachedLinkage((Linkage)FunctionDeclBits.getNextBits(/*Width=*/3)); FD->setStorageClass((StorageClass)FunctionDeclBits.getNextBits(/*Width=*/3)); FD->setInlineSpecified(FunctionDeclBits.getNextBit()); FD->setImplicitlyInline(FunctionDeclBits.getNextBit()); + FD->setHasSkippedBody(FunctionDeclBits.getNextBit()); FD->setVirtualAsWritten(FunctionDeclBits.getNextBit()); // We defer calling `FunctionDecl::setPure()` here as for methods of // `CXXTemplateSpecializationDecl`s, we may not have connected up the @@ -1081,16 +1081,14 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) { FD->setDefaulted(FunctionDeclBits.getNextBit()); FD->setExplicitlyDefaulted(FunctionDeclBits.getNextBit()); FD->setIneligibleOrNotSelected(FunctionDeclBits.getNextBit()); - FD->setHasImplicitReturnZero(FunctionDeclBits.getNextBit()); FD->setConstexprKind( (ConstexprSpecKind)FunctionDeclBits.getNextBits(/*Width=*/2)); - FD->setUsesSEHTry(FunctionDeclBits.getNextBit()); - FD->setHasSkippedBody(FunctionDeclBits.getNextBit()); + FD->setHasImplicitReturnZero(FunctionDeclBits.getNextBit()); FD->setIsMultiVersion(FunctionDeclBits.getNextBit()); FD->setLateTemplateParsed(FunctionDeclBits.getNextBit()); FD->setFriendConstraintRefersToEnclosingTemplate( FunctionDeclBits.getNextBit()); - FD->setCachedLinkage((Linkage)FunctionDeclBits.getNextBits(/*Width=*/3)); + FD->setUsesSEHTry(FunctionDeclBits.getNextBit()); FD->EndRangeLoc = readSourceLocation(); if (FD->isExplicitlyDefaulted()) @@ -1597,6 +1595,8 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) { VisitDeclaratorDecl(VD); BitsUnpacker VarDeclBits(Record.readInt()); + auto VarLinkage = Linkage(VarDeclBits.getNextBits(/*Width=*/3)); + bool DefGeneratedInModule = VarDeclBits.getNextBit(); VD->VarDeclBits.SClass = (StorageClass)VarDeclBits.getNextBits(/*Width=*/3); VD->VarDeclBits.TSCSpec = VarDeclBits.getNextBits(/*Width=*/2); VD->VarDeclBits.InitStyle = VarDeclBits.getNextBits(/*Width=*/2); @@ -1608,17 +1608,20 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) { VD->NonParmVarDeclBits.ExceptionVar = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.NRVOVariable = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.CXXForRangeDecl = VarDeclBits.getNextBit(); - VD->NonParmVarDeclBits.ObjCForDecl = VarDeclBits.getNextBit(); + VD->NonParmVarDeclBits.IsInline = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.IsInlineSpecified = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.IsConstexpr = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.IsInitCapture = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.PreviousDeclInSameBlockScope = VarDeclBits.getNextBit(); - VD->NonParmVarDeclBits.ImplicitParamKind = - VarDeclBits.getNextBits(/*Width*/ 3); + VD->NonParmVarDeclBits.EscapingByref = VarDeclBits.getNextBit(); HasDeducedType = VarDeclBits.getNextBit(); + VD->NonParmVarDeclBits.ImplicitParamKind = + VarDeclBits.getNextBits(/*Width*/ 3); + + VD->NonParmVarDeclBits.ObjCForDecl = VarDeclBits.getNextBit(); } // If this variable has a deduced type, defer reading that type until we are @@ -1630,7 +1633,6 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) { VD->setType(Reader.GetType(DeferredTypeID)); DeferredTypeID = 0; - auto VarLinkage = Linkage(VarDeclBits.getNextBits(/*Width=*/3)); VD->setCachedLinkage(VarLinkage); // Reconstruct the one piece of the IdentifierNamespace that we need. @@ -1638,7 +1640,7 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) { VD->getLexicalDeclContext()->isFunctionOrMethod()) VD->setLocalExternDecl(); - if (VarDeclBits.getNextBit()) { + if (DefGeneratedInModule) { Reader.DefinitionSource[VD] = Loc.F->Kind == ModuleKind::MK_MainFile || Reader.getContext().getLangOpts().BuildingPCHWithObjectFile; @@ -2660,7 +2662,7 @@ void ASTDeclReader::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) { D->setDeclaredWithTypename(Record.readInt()); - if (Record.readBool()) { + if (D->hasTypeConstraint()) { ConceptReference *CR = nullptr; if (Record.readBool()) CR = Record.readConceptReference(); diff --git a/contrib/llvm-project/clang/lib/Serialization/ASTReaderStmt.cpp b/contrib/llvm-project/clang/lib/Serialization/ASTReaderStmt.cpp index b3a6f619372b..21aed570ba26 100644 --- a/contrib/llvm-project/clang/lib/Serialization/ASTReaderStmt.cpp +++ b/contrib/llvm-project/clang/lib/Serialization/ASTReaderStmt.cpp @@ -73,6 +73,8 @@ namespace clang { ASTRecordReader &Record; llvm::BitstreamCursor &DeclsCursor; + std::optional<BitsUnpacker> CurrentUnpackingBits; + SourceLocation readSourceLocation() { return Record.readSourceLocation(); } @@ -110,6 +112,9 @@ namespace clang { /// itself. static const unsigned NumExprFields = NumStmtFields + 2; + /// The number of bits required for the packing bits for the Expr class. + static const unsigned NumExprBits = 10; + /// Read and initialize a ExplicitTemplateArgumentList structure. void ReadTemplateKWAndArgsInfo(ASTTemplateKWAndArgsInfo &Args, TemplateArgumentLoc *ArgsLocArray, @@ -214,9 +219,11 @@ void ASTStmtReader::VisitAttributedStmt(AttributedStmt *S) { void ASTStmtReader::VisitIfStmt(IfStmt *S) { VisitStmt(S); - bool HasElse = Record.readInt(); - bool HasVar = Record.readInt(); - bool HasInit = Record.readInt(); + CurrentUnpackingBits.emplace(Record.readInt()); + + bool HasElse = CurrentUnpackingBits->getNextBit(); + bool HasVar = CurrentUnpackingBits->getNextBit(); + bool HasInit = CurrentUnpackingBits->getNextBit(); S->setStatementKind(static_cast<IfStatementKind>(Record.readInt())); S->setCond(Record.readSubExpr()); @@ -523,14 +530,15 @@ void ASTStmtReader::VisitCapturedStmt(CapturedStmt *S) { void ASTStmtReader::VisitExpr(Expr *E) { VisitStmt(E); + CurrentUnpackingBits.emplace(Record.readInt()); + E->setDependence(static_cast<ExprDependence>( + CurrentUnpackingBits->getNextBits(/*Width=*/5))); + E->setValueKind(static_cast<ExprValueKind>( + CurrentUnpackingBits->getNextBits(/*Width=*/2))); + E->setObjectKind(static_cast<ExprObjectKind>( + CurrentUnpackingBits->getNextBits(/*Width=*/3))); + E->setType(Record.readType()); - BitsUnpacker ExprBits(Record.readInt()); - E->setDependence( - static_cast<ExprDependence>(ExprBits.getNextBits(/*Width=*/5))); - E->setValueKind( - static_cast<ExprValueKind>(ExprBits.getNextBits(/*Width=*/2))); - E->setObjectKind( - static_cast<ExprObjectKind>(ExprBits.getNextBits(/*Width=*/3))); assert(Record.getIdx() == NumExprFields && "Incorrect expression field count"); } @@ -591,13 +599,17 @@ void ASTStmtReader::VisitPredefinedExpr(PredefinedExpr *E) { void ASTStmtReader::VisitDeclRefExpr(DeclRefExpr *E) { VisitExpr(E); - E->DeclRefExprBits.HasQualifier = Record.readInt(); - E->DeclRefExprBits.HasFoundDecl = Record.readInt(); - E->DeclRefExprBits.HasTemplateKWAndArgsInfo = Record.readInt(); - E->DeclRefExprBits.HadMultipleCandidates = Record.readInt(); - E->DeclRefExprBits.RefersToEnclosingVariableOrCapture = Record.readInt(); - E->DeclRefExprBits.NonOdrUseReason = Record.readInt(); - E->DeclRefExprBits.IsImmediateEscalating = Record.readInt(); + CurrentUnpackingBits.emplace(Record.readInt()); + E->DeclRefExprBits.HadMultipleCandidates = CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.RefersToEnclosingVariableOrCapture = + CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.NonOdrUseReason = + CurrentUnpackingBits->getNextBits(/*Width=*/2); + E->DeclRefExprBits.IsImmediateEscalating = CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.HasFoundDecl = CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.HasQualifier = CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.HasTemplateKWAndArgsInfo = + CurrentUnpackingBits->getNextBit(); E->DeclRefExprBits.CapturedByCopyInLambdaWithExplicitObjectParameter = false; unsigned NumTemplateArgs = 0; if (E->hasTemplateKWAndArgsInfo()) @@ -706,12 +718,13 @@ void ASTStmtReader::VisitParenListExpr(ParenListExpr *E) { void ASTStmtReader::VisitUnaryOperator(UnaryOperator *E) { VisitExpr(E); - bool hasFP_Features = Record.readInt(); + bool hasFP_Features = CurrentUnpackingBits->getNextBit(); assert(hasFP_Features == E->hasStoredFPFeatures()); E->setSubExpr(Record.readSubExpr()); - E->setOpcode((UnaryOperator::Opcode)Record.readInt()); + E->setOpcode( + (UnaryOperator::Opcode)CurrentUnpackingBits->getNextBits(/*Width=*/5)); E->setOperatorLoc(readSourceLocation()); - E->setCanOverflow(Record.readInt()); + E->setCanOverflow(CurrentUnpackingBits->getNextBit()); if (hasFP_Features) E->setStoredFPFeatures( FPOptionsOverride::getFromOpaqueInt(Record.readInt())); @@ -1000,12 +1013,11 @@ void ASTStmtReader::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtReader::VisitCallExpr(CallExpr *E) { VisitExpr(E); - BitsUnpacker CallExprBits = Record.readInt(); - - unsigned NumArgs = CallExprBits.getNextBits(/*Width=*/16); - bool HasFPFeatures = CallExprBits.getNextBit(); + unsigned NumArgs = Record.readInt(); + CurrentUnpackingBits.emplace(Record.readInt()); E->setADLCallKind( - static_cast<CallExpr::ADLCallKind>(CallExprBits.getNextBit())); + static_cast<CallExpr::ADLCallKind>(CurrentUnpackingBits->getNextBit())); + bool HasFPFeatures = CurrentUnpackingBits->getNextBit(); assert((NumArgs == E->getNumArgs()) && "Wrong NumArgs!"); E->setRParenLoc(readSourceLocation()); E->setCallee(Record.readSubExpr()); @@ -1024,27 +1036,29 @@ void ASTStmtReader::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) { void ASTStmtReader::VisitMemberExpr(MemberExpr *E) { VisitExpr(E); - bool HasQualifier = Record.readInt(); - bool HasFoundDecl = Record.readInt(); - bool HasTemplateInfo = Record.readInt(); + CurrentUnpackingBits.emplace(Record.readInt()); + bool HasQualifier = CurrentUnpackingBits->getNextBit(); + bool HasFoundDecl = CurrentUnpackingBits->getNextBit(); + bool HasTemplateInfo = CurrentUnpackingBits->getNextBit(); unsigned NumTemplateArgs = Record.readInt(); E->Base = Record.readSubExpr(); E->MemberDecl = Record.readDeclAs<ValueDecl>(); E->MemberDNLoc = Record.readDeclarationNameLoc(E->MemberDecl->getDeclName()); E->MemberLoc = Record.readSourceLocation(); - E->MemberExprBits.IsArrow = Record.readInt(); + E->MemberExprBits.IsArrow = CurrentUnpackingBits->getNextBit(); E->MemberExprBits.HasQualifierOrFoundDecl = HasQualifier || HasFoundDecl; E->MemberExprBits.HasTemplateKWAndArgsInfo = HasTemplateInfo; - E->MemberExprBits.HadMultipleCandidates = Record.readInt(); - E->MemberExprBits.NonOdrUseReason = Record.readInt(); + E->MemberExprBits.HadMultipleCandidates = CurrentUnpackingBits->getNextBit(); + E->MemberExprBits.NonOdrUseReason = + CurrentUnpackingBits->getNextBits(/*Width=*/2); E->MemberExprBits.OperatorLoc = Record.readSourceLocation(); if (HasQualifier || HasFoundDecl) { DeclAccessPair FoundDecl; if (HasFoundDecl) { auto *FoundD = Record.readDeclAs<NamedDecl>(); - auto AS = (AccessSpecifier)Record.readInt(); + auto AS = (AccessSpecifier)CurrentUnpackingBits->getNextBits(/*Width=*/2); FoundDecl = DeclAccessPair::make(FoundD, AS); } else { FoundDecl = DeclAccessPair::make(E->MemberDecl, @@ -1091,10 +1105,14 @@ void ASTStmtReader::VisitCastExpr(CastExpr *E) { VisitExpr(E); unsigned NumBaseSpecs = Record.readInt(); assert(NumBaseSpecs == E->path_size()); - unsigned HasFPFeatures = Record.readInt(); + + CurrentUnpackingBits.emplace(Record.readInt()); + E->setCastKind((CastKind)CurrentUnpackingBits->getNextBits(/*Width=*/7)); + unsigned HasFPFeatures = CurrentUnpackingBits->getNextBit(); assert(E->hasStoredFPFeatures() == HasFPFeatures); + E->setSubExpr(Record.readSubExpr()); - E->setCastKind((CastKind)Record.readInt()); + CastExpr::path_iterator BaseI = E->path_begin(); while (NumBaseSpecs--) { auto *BaseSpec = new (Record.getContext()) CXXBaseSpecifier; @@ -1107,10 +1125,12 @@ void ASTStmtReader::VisitCastExpr(CastExpr *E) { } void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) { - bool hasFP_Features; VisitExpr(E); - E->setHasStoredFPFeatures(hasFP_Features = Record.readInt()); - E->setOpcode((BinaryOperator::Opcode)Record.readInt()); + CurrentUnpackingBits.emplace(Record.readInt()); + E->setOpcode( + (BinaryOperator::Opcode)CurrentUnpackingBits->getNextBits(/*Width=*/6)); + bool hasFP_Features = CurrentUnpackingBits->getNextBit(); + E->setHasStoredFPFeatures(hasFP_Features); E->setLHS(Record.readSubExpr()); E->setRHS(Record.readSubExpr()); E->setOperatorLoc(readSourceLocation()); @@ -1148,7 +1168,7 @@ ASTStmtReader::VisitBinaryConditionalOperator(BinaryConditionalOperator *E) { void ASTStmtReader::VisitImplicitCastExpr(ImplicitCastExpr *E) { VisitCastExpr(E); - E->setIsPartOfExplicitCast(Record.readInt()); + E->setIsPartOfExplicitCast(CurrentUnpackingBits->getNextBit()); } void ASTStmtReader::VisitExplicitCastExpr(ExplicitCastExpr *E) { @@ -1764,8 +1784,8 @@ void ASTStmtReader::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) { SourceRange R = readSourceRange(); E->Loc = R.getBegin(); E->RParenLoc = R.getEnd(); - R = readSourceRange(); - E->AngleBrackets = R; + if (CurrentUnpackingBits->getNextBit()) + E->AngleBrackets = readSourceRange(); } void ASTStmtReader::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) { @@ -1961,9 +1981,10 @@ void ASTStmtReader::VisitCXXDependentScopeMemberExpr( CXXDependentScopeMemberExpr *E) { VisitExpr(E); - bool HasTemplateKWAndArgsInfo = Record.readInt(); unsigned NumTemplateArgs = Record.readInt(); - bool HasFirstQualifierFoundInScope = Record.readInt(); + CurrentUnpackingBits.emplace(Record.readInt()); + bool HasTemplateKWAndArgsInfo = CurrentUnpackingBits->getNextBit(); + bool HasFirstQualifierFoundInScope = CurrentUnpackingBits->getNextBit(); assert((HasTemplateKWAndArgsInfo == E->hasTemplateKWAndArgsInfo()) && "Wrong HasTemplateKWAndArgsInfo!"); @@ -1979,11 +2000,18 @@ void ASTStmtReader::VisitCXXDependentScopeMemberExpr( assert((NumTemplateArgs == E->getNumTemplateArgs()) && "Wrong NumTemplateArgs!"); - E->CXXDependentScopeMemberExprBits.IsArrow = Record.readInt(); - E->CXXDependentScopeMemberExprBits.OperatorLoc = readSourceLocation(); + E->CXXDependentScopeMemberExprBits.IsArrow = + CurrentUnpackingBits->getNextBit(); + E->BaseType = Record.readType(); E->QualifierLoc = Record.readNestedNameSpecifierLoc(); - E->Base = Record.readSubExpr(); + // not ImplicitAccess + if (CurrentUnpackingBits->getNextBit()) + E->Base = Record.readSubExpr(); + else + E->Base = nullptr; + + E->CXXDependentScopeMemberExprBits.OperatorLoc = readSourceLocation(); if (HasFirstQualifierFoundInScope) *E->getTrailingObjects<NamedDecl *>() = readDeclAs<NamedDecl>(); @@ -1995,11 +2023,11 @@ void ASTStmtReader::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) { VisitExpr(E); - if (Record.readInt()) // HasTemplateKWAndArgsInfo + if (CurrentUnpackingBits->getNextBit()) // HasTemplateKWAndArgsInfo ReadTemplateKWAndArgsInfo( *E->getTrailingObjects<ASTTemplateKWAndArgsInfo>(), E->getTrailingObjects<TemplateArgumentLoc>(), - /*NumTemplateArgs=*/Record.readInt()); + /*NumTemplateArgs=*/CurrentUnpackingBits->getNextBits(/*Width=*/16)); E->QualifierLoc = Record.readNestedNameSpecifierLoc(); E->NameInfo = Record.readDeclarationNameInfo(); @@ -2022,15 +2050,15 @@ ASTStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); - BitsUnpacker OverloadExprBits = Record.readInt(); - unsigned NumResults = OverloadExprBits.getNextBits(/*Width=*/14); - bool HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); + unsigned NumResults = Record.readInt(); + CurrentUnpackingBits.emplace(Record.readInt()); + bool HasTemplateKWAndArgsInfo = CurrentUnpackingBits->getNextBit(); assert((E->getNumDecls() == NumResults) && "Wrong NumResults!"); assert((E->hasTemplateKWAndArgsInfo() == HasTemplateKWAndArgsInfo) && "Wrong HasTemplateKWAndArgsInfo!"); if (HasTemplateKWAndArgsInfo) { - unsigned NumTemplateArgs = OverloadExprBits.getNextBits(/*Width=*/14); + unsigned NumTemplateArgs = Record.readInt(); ReadTemplateKWAndArgsInfo(*E->getTrailingASTTemplateKWAndArgsInfo(), E->getTrailingTemplateArgumentLoc(), NumTemplateArgs); @@ -2057,17 +2085,24 @@ void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) { void ASTStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) { VisitOverloadExpr(E); - E->UnresolvedMemberExprBits.IsArrow = Record.readInt(); - E->UnresolvedMemberExprBits.HasUnresolvedUsing = Record.readInt(); - E->Base = Record.readSubExpr(); - E->BaseType = Record.readType(); + E->UnresolvedMemberExprBits.IsArrow = CurrentUnpackingBits->getNextBit(); + E->UnresolvedMemberExprBits.HasUnresolvedUsing = + CurrentUnpackingBits->getNextBit(); + + if (/*!isImplicitAccess=*/CurrentUnpackingBits->getNextBit()) + E->Base = Record.readSubExpr(); + else + E->Base = nullptr; + E->OperatorLoc = readSourceLocation(); + + E->BaseType = Record.readType(); } void ASTStmtReader::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) { VisitOverloadExpr(E); - E->UnresolvedLookupExprBits.RequiresADL = Record.readInt(); - E->UnresolvedLookupExprBits.Overloaded = Record.readInt(); + E->UnresolvedLookupExprBits.RequiresADL = CurrentUnpackingBits->getNextBit(); + E->UnresolvedLookupExprBits.Overloaded = CurrentUnpackingBits->getNextBit(); E->NamingClass = readDeclAs<CXXRecordDecl>(); } @@ -2142,9 +2177,12 @@ void ASTStmtReader::VisitSubstNonTypeTemplateParmExpr( SubstNonTypeTemplateParmExpr *E) { VisitExpr(E); E->AssociatedDeclAndRef.setPointer(readDeclAs<Decl>()); - E->AssociatedDeclAndRef.setInt(Record.readInt()); - E->Index = Record.readInt(); - E->PackIndex = Record.readInt(); + E->AssociatedDeclAndRef.setInt(CurrentUnpackingBits->getNextBit()); + E->Index = CurrentUnpackingBits->getNextBits(/*Width=*/12); + if (CurrentUnpackingBits->getNextBit()) + E->PackIndex = Record.readInt(); + else + E->PackIndex = 0; E->SubstNonTypeTemplateParmExprBits.NameLoc = readSourceLocation(); E->Replacement = Record.readSubExpr(); } @@ -2836,11 +2874,12 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) NullStmt(Empty); break; - case STMT_COMPOUND: - S = CompoundStmt::CreateEmpty( - Context, /*NumStmts=*/Record[ASTStmtReader::NumStmtFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumStmtFields + 1]); + case STMT_COMPOUND: { + unsigned NumStmts = Record[ASTStmtReader::NumStmtFields]; + bool HasFPFeatures = Record[ASTStmtReader::NumStmtFields + 1]; + S = CompoundStmt::CreateEmpty(Context, NumStmts, HasFPFeatures); break; + } case STMT_CASE: S = CaseStmt::CreateEmpty( @@ -2862,13 +2901,14 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { /*NumAttrs*/Record[ASTStmtReader::NumStmtFields]); break; - case STMT_IF: - S = IfStmt::CreateEmpty( - Context, - /* HasElse=*/Record[ASTStmtReader::NumStmtFields], - /* HasVar=*/Record[ASTStmtReader::NumStmtFields + 1], - /* HasInit=*/Record[ASTStmtReader::NumStmtFields + 2]); + case STMT_IF: { + BitsUnpacker IfStmtBits(Record[ASTStmtReader::NumStmtFields]); + bool HasElse = IfStmtBits.getNextBit(); + bool HasVar = IfStmtBits.getNextBit(); + bool HasInit = IfStmtBits.getNextBit(); + S = IfStmt::CreateEmpty(Context, HasElse, HasVar, HasInit); break; + } case STMT_SWITCH: S = SwitchStmt::CreateEmpty( @@ -2945,17 +2985,19 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { /*HasFunctionName*/ Record[ASTStmtReader::NumExprFields]); break; - case EXPR_DECL_REF: - S = DeclRefExpr::CreateEmpty( - Context, - /*HasQualifier=*/Record[ASTStmtReader::NumExprFields], - /*HasFoundDecl=*/Record[ASTStmtReader::NumExprFields + 1], - /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields + 2], - /*NumTemplateArgs=*/ - Record[ASTStmtReader::NumExprFields + 2] - ? Record[ASTStmtReader::NumExprFields + 7] - : 0); + case EXPR_DECL_REF: { + BitsUnpacker DeclRefExprBits(Record[ASTStmtReader::NumExprFields]); + DeclRefExprBits.advance(5); + bool HasFoundDecl = DeclRefExprBits.getNextBit(); + bool HasQualifier = DeclRefExprBits.getNextBit(); + bool HasTemplateKWAndArgsInfo = DeclRefExprBits.getNextBit(); + unsigned NumTemplateArgs = HasTemplateKWAndArgsInfo + ? Record[ASTStmtReader::NumExprFields + 1] + : 0; + S = DeclRefExpr::CreateEmpty(Context, HasQualifier, HasFoundDecl, + HasTemplateKWAndArgsInfo, NumTemplateArgs); break; + } case EXPR_INTEGER_LITERAL: S = IntegerLiteral::Create(Context, Empty); @@ -2995,10 +3037,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { /* NumExprs=*/Record[ASTStmtReader::NumExprFields]); break; - case EXPR_UNARY_OPERATOR: - S = UnaryOperator::CreateEmpty(Context, - Record[ASTStmtReader::NumExprFields]); + case EXPR_UNARY_OPERATOR: { + BitsUnpacker UnaryOperatorBits(Record[ASTStmtReader::NumStmtFields]); + UnaryOperatorBits.advance(ASTStmtReader::NumExprBits); + bool HasFPFeatures = UnaryOperatorBits.getNextBit(); + S = UnaryOperator::CreateEmpty(Context, HasFPFeatures); break; + } case EXPR_OFFSETOF: S = OffsetOfExpr::CreateEmpty(Context, @@ -3033,8 +3078,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = CallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); break; @@ -3045,22 +3091,32 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields]); break; - case EXPR_MEMBER: - S = MemberExpr::CreateEmpty(Context, Record[ASTStmtReader::NumExprFields], - Record[ASTStmtReader::NumExprFields + 1], - Record[ASTStmtReader::NumExprFields + 2], - Record[ASTStmtReader::NumExprFields + 3]); + case EXPR_MEMBER: { + BitsUnpacker ExprMemberBits(Record[ASTStmtReader::NumExprFields]); + bool HasQualifier = ExprMemberBits.getNextBit(); + bool HasFoundDecl = ExprMemberBits.getNextBit(); + bool HasTemplateInfo = ExprMemberBits.getNextBit(); + unsigned NumTemplateArgs = Record[ASTStmtReader::NumExprFields + 1]; + S = MemberExpr::CreateEmpty(Context, HasQualifier, HasFoundDecl, + HasTemplateInfo, NumTemplateArgs); break; + } - case EXPR_BINARY_OPERATOR: - S = BinaryOperator::CreateEmpty(Context, - Record[ASTStmtReader::NumExprFields]); + case EXPR_BINARY_OPERATOR: { + BitsUnpacker BinaryOperatorBits(Record[ASTStmtReader::NumExprFields]); + BinaryOperatorBits.advance(/*Size of opcode*/ 6); + bool HasFPFeatures = BinaryOperatorBits.getNextBit(); + S = BinaryOperator::CreateEmpty(Context, HasFPFeatures); break; + } - case EXPR_COMPOUND_ASSIGN_OPERATOR: - S = CompoundAssignOperator::CreateEmpty( - Context, Record[ASTStmtReader::NumExprFields]); + case EXPR_COMPOUND_ASSIGN_OPERATOR: { + BitsUnpacker BinaryOperatorBits(Record[ASTStmtReader::NumExprFields]); + BinaryOperatorBits.advance(/*Size of opcode*/ 6); + bool HasFPFeatures = BinaryOperatorBits.getNextBit(); + S = CompoundAssignOperator::CreateEmpty(Context, HasFPFeatures); break; + } case EXPR_CONDITIONAL_OPERATOR: S = new (Context) ConditionalOperator(Empty); @@ -3070,19 +3126,23 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) BinaryConditionalOperator(Empty); break; - case EXPR_IMPLICIT_CAST: - S = ImplicitCastExpr::CreateEmpty( - Context, - /*PathSize*/ Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures*/ Record[ASTStmtReader::NumExprFields + 1]); + case EXPR_IMPLICIT_CAST: { + unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CastExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CastExprBits.advance(7); + bool HasFPFeatures = CastExprBits.getNextBit(); + S = ImplicitCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); break; + } - case EXPR_CSTYLE_CAST: - S = CStyleCastExpr::CreateEmpty( - Context, - /*PathSize*/ Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures*/ Record[ASTStmtReader::NumExprFields + 1]); + case EXPR_CSTYLE_CAST: { + unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CastExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CastExprBits.advance(7); + bool HasFPFeatures = CastExprBits.getNextBit(); + S = CStyleCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); break; + } case EXPR_COMPOUND_LITERAL: S = new (Context) CompoundLiteralExpr(Empty); @@ -3777,8 +3837,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { } case EXPR_CXX_OPERATOR_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = CXXOperatorCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -3786,8 +3847,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { } case EXPR_CXX_MEMBER_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = CXXMemberCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -3814,22 +3876,26 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { /* NumArgs=*/Record[ASTStmtReader::NumExprFields]); break; - case EXPR_CXX_STATIC_CAST: - S = CXXStaticCastExpr::CreateEmpty( - Context, - /*PathSize*/ Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures*/ Record[ASTStmtReader::NumExprFields + 1]); + case EXPR_CXX_STATIC_CAST: { + unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CastExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CastExprBits.advance(7); + bool HasFPFeatures = CastExprBits.getNextBit(); + S = CXXStaticCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); break; + } - case EXPR_CXX_DYNAMIC_CAST: - S = CXXDynamicCastExpr::CreateEmpty(Context, - /*PathSize*/ Record[ASTStmtReader::NumExprFields]); + case EXPR_CXX_DYNAMIC_CAST: { + unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + S = CXXDynamicCastExpr::CreateEmpty(Context, PathSize); break; + } - case EXPR_CXX_REINTERPRET_CAST: - S = CXXReinterpretCastExpr::CreateEmpty(Context, - /*PathSize*/ Record[ASTStmtReader::NumExprFields]); + case EXPR_CXX_REINTERPRET_CAST: { + unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + S = CXXReinterpretCastExpr::CreateEmpty(Context, PathSize); break; + } case EXPR_CXX_CONST_CAST: S = CXXConstCastExpr::CreateEmpty(Context); @@ -3839,21 +3905,28 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = CXXAddrspaceCastExpr::CreateEmpty(Context); break; - case EXPR_CXX_FUNCTIONAL_CAST: - S = CXXFunctionalCastExpr::CreateEmpty( - Context, - /*PathSize*/ Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures*/ Record[ASTStmtReader::NumExprFields + 1]); + case EXPR_CXX_FUNCTIONAL_CAST: { + unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CastExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CastExprBits.advance(7); + bool HasFPFeatures = CastExprBits.getNextBit(); + S = CXXFunctionalCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); break; + } - case EXPR_BUILTIN_BIT_CAST: - assert(Record[ASTStmtReader::NumExprFields] == 0 && "Wrong PathSize!"); + case EXPR_BUILTIN_BIT_CAST: { +#ifndef NDEBUG + unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + assert(PathSize == 0 && "Wrong PathSize!"); +#endif S = new (Context) BuiltinBitCastExpr(Empty); break; + } case EXPR_USER_DEFINED_LITERAL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = UserDefinedLiteral::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -3944,47 +4017,62 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { Record[ASTStmtReader::NumExprFields]); break; - case EXPR_CXX_DEPENDENT_SCOPE_MEMBER: + case EXPR_CXX_DEPENDENT_SCOPE_MEMBER: { + unsigned NumTemplateArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker DependentScopeMemberBits( + Record[ASTStmtReader::NumExprFields + 1]); + bool HasTemplateKWAndArgsInfo = DependentScopeMemberBits.getNextBit(); + + bool HasFirstQualifierFoundInScope = + DependentScopeMemberBits.getNextBit(); S = CXXDependentScopeMemberExpr::CreateEmpty( - Context, - /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields], - /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields + 1], - /*HasFirstQualifierFoundInScope=*/ - Record[ASTStmtReader::NumExprFields + 2]); + Context, HasTemplateKWAndArgsInfo, NumTemplateArgs, + HasFirstQualifierFoundInScope); break; + } - case EXPR_CXX_DEPENDENT_SCOPE_DECL_REF: - S = DependentScopeDeclRefExpr::CreateEmpty(Context, - /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields], - /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] - ? Record[ASTStmtReader::NumExprFields + 1] - : 0); + case EXPR_CXX_DEPENDENT_SCOPE_DECL_REF: { + BitsUnpacker DependentScopeDeclRefBits( + Record[ASTStmtReader::NumStmtFields]); + DependentScopeDeclRefBits.advance(ASTStmtReader::NumExprBits); + bool HasTemplateKWAndArgsInfo = DependentScopeDeclRefBits.getNextBit(); + unsigned NumTemplateArgs = + HasTemplateKWAndArgsInfo + ? DependentScopeDeclRefBits.getNextBits(/*Width=*/16) + : 0; + S = DependentScopeDeclRefExpr::CreateEmpty( + Context, HasTemplateKWAndArgsInfo, NumTemplateArgs); break; + } case EXPR_CXX_UNRESOLVED_CONSTRUCT: S = CXXUnresolvedConstructExpr::CreateEmpty(Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields]); break; - case EXPR_CXX_UNRESOLVED_MEMBER: + case EXPR_CXX_UNRESOLVED_MEMBER: { + auto NumResults = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker OverloadExprBits(Record[ASTStmtReader::NumExprFields + 1]); + auto HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); + auto NumTemplateArgs = HasTemplateKWAndArgsInfo + ? Record[ASTStmtReader::NumExprFields + 2] + : 0; S = UnresolvedMemberExpr::CreateEmpty( - Context, - /*NumResults=*/Record[ASTStmtReader::NumExprFields] & ((1 << 14) - 1), - /*HasTemplateKWAndArgsInfo=*/ - (Record[ASTStmtReader::NumExprFields] >> 14) & (0x1), - /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] >> 14 & - ((1 << 14) - 1)); + Context, NumResults, HasTemplateKWAndArgsInfo, NumTemplateArgs); break; + } - case EXPR_CXX_UNRESOLVED_LOOKUP: + case EXPR_CXX_UNRESOLVED_LOOKUP: { + auto NumResults = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker OverloadExprBits(Record[ASTStmtReader::NumExprFields + 1]); + auto HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); + auto NumTemplateArgs = HasTemplateKWAndArgsInfo + ? Record[ASTStmtReader::NumExprFields + 2] + : 0; S = UnresolvedLookupExpr::CreateEmpty( - Context, - /*NumResults=*/Record[ASTStmtReader::NumExprFields] & ((1 << 14) - 1), - /*HasTemplateKWAndArgsInfo=*/ - (Record[ASTStmtReader::NumExprFields] >> 14) & (0x1), - /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] >> 14 & - ((1 << 14) - 1)); + Context, NumResults, HasTemplateKWAndArgsInfo, NumTemplateArgs); break; + } case EXPR_TYPE_TRAIT: S = TypeTraitExpr::CreateDeserialized(Context, @@ -4044,8 +4132,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CUDA_KERNEL_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = CUDAKernelCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); diff --git a/contrib/llvm-project/clang/lib/Serialization/ASTWriter.cpp b/contrib/llvm-project/clang/lib/Serialization/ASTWriter.cpp index 91eb2af8f8ad..78939bfd533f 100644 --- a/contrib/llvm-project/clang/lib/Serialization/ASTWriter.cpp +++ b/contrib/llvm-project/clang/lib/Serialization/ASTWriter.cpp @@ -6003,12 +6003,17 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) { BitsPacker DefinitionBits; -#define FIELD(Name, Width, Merge) DefinitionBits.addBits(Data.Name, Width); +#define FIELD(Name, Width, Merge) \ + if (!DefinitionBits.canWriteNextNBits(Width)) { \ + Record->push_back(DefinitionBits); \ + DefinitionBits.reset(0); \ + } \ + DefinitionBits.addBits(Data.Name, Width); + #include "clang/AST/CXXRecordDeclDefinitionBits.def" #undef FIELD - while (DefinitionBits.hasUnconsumedValues()) - Record->push_back(DefinitionBits.getNextValue()); + Record->push_back(DefinitionBits); // getODRHash will compute the ODRHash if it has not been previously computed. Record->push_back(D->getODRHash()); @@ -6047,7 +6052,7 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) { LambdaBits.addBits(Lambda.CaptureDefault, /*Width=*/2); LambdaBits.addBits(Lambda.NumCaptures, /*Width=*/15); LambdaBits.addBit(Lambda.HasKnownInternalLinkage); - Record->push_back(LambdaBits.getNextValue()); + Record->push_back(LambdaBits); Record->push_back(Lambda.NumExplicitCaptures); Record->push_back(Lambda.ManglingNumber); @@ -6058,10 +6063,12 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) { for (unsigned I = 0, N = Lambda.NumCaptures; I != N; ++I) { const LambdaCapture &Capture = Lambda.Captures.front()[I]; AddSourceLocation(Capture.getLocation()); + BitsPacker CaptureBits; CaptureBits.addBit(Capture.isImplicit()); CaptureBits.addBits(Capture.getCaptureKind(), /*Width=*/3); Record->push_back(CaptureBits); + switch (Capture.getCaptureKind()) { case LCK_StarThis: case LCK_This: diff --git a/contrib/llvm-project/clang/lib/Serialization/ASTWriterDecl.cpp b/contrib/llvm-project/clang/lib/Serialization/ASTWriterDecl.cpp index 43169b2befc6..2554abc682a1 100644 --- a/contrib/llvm-project/clang/lib/Serialization/ASTWriterDecl.cpp +++ b/contrib/llvm-project/clang/lib/Serialization/ASTWriterDecl.cpp @@ -321,15 +321,25 @@ void ASTDeclWriter::Visit(Decl *D) { void ASTDeclWriter::VisitDecl(Decl *D) { BitsPacker DeclBits; + + // The order matters here. It will be better to put the bit with higher + // probability to be 0 in the end of the bits. + // + // Since we're using VBR6 format to store it. + // It will be pretty effient if all the higher bits are 0. + // For example, if we need to pack 8 bits into a value and the stored value + // is 0xf0, the actual stored value will be 0b000111'110000, which takes 12 + // bits actually. However, if we changed the order to be 0x0f, then we can + // store it as 0b001111, which takes 6 bits only now. + DeclBits.addBits((uint64_t)D->getModuleOwnershipKind(), /*BitWidth=*/3); + DeclBits.addBit(D->isReferenced()); + DeclBits.addBit(D->isUsed(false)); + DeclBits.addBits(D->getAccess(), /*BitWidth=*/2); + DeclBits.addBit(D->isImplicit()); DeclBits.addBit(D->getDeclContext() != D->getLexicalDeclContext()); - DeclBits.addBit(D->isInvalidDecl()); DeclBits.addBit(D->hasAttrs()); - DeclBits.addBit(D->isImplicit()); - DeclBits.addBit(D->isUsed(false)); - DeclBits.addBit(D->isReferenced()); DeclBits.addBit(D->isTopLevelDeclInObjCContainer()); - DeclBits.addBits(D->getAccess(), /*BitWidth=*/2); - DeclBits.addBits((uint64_t)D->getModuleOwnershipKind(), /*BitWidth=*/3); + DeclBits.addBit(D->isInvalidDecl()); Record.push_back(DeclBits); Record.AddDeclRef(cast_or_null<Decl>(D->getDeclContext())); @@ -493,21 +503,13 @@ void ASTDeclWriter::VisitEnumDecl(EnumDecl *D) { Record.AddDeclRef(nullptr); } - if (D->getDeclContext() == D->getLexicalDeclContext() && - !D->hasAttrs() && - !D->isImplicit() && - !D->isUsed(false) && - !D->hasExtInfo() && + if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && + !D->isInvalidDecl() && !D->isImplicit() && !D->hasExtInfo() && !D->getTypedefNameForAnonDecl() && D->getFirstDecl() == D->getMostRecentDecl() && - !D->isInvalidDecl() && - !D->isReferenced() && !D->isTopLevelDeclInObjCContainer() && - D->getAccess() == AS_none && - !D->isModulePrivate() && !CXXRecordDecl::classofKind(D->getKind()) && - !D->getIntegerTypeSourceInfo() && - !D->getMemberSpecializationInfo() && + !D->getIntegerTypeSourceInfo() && !D->getMemberSpecializationInfo() && !needsAnonymousDeclarationNumber(D) && D->getDeclName().getNameKind() == DeclarationName::Identifier) AbbrevToUse = Writer.getDeclEnumAbbrev(); @@ -542,18 +544,11 @@ void ASTDeclWriter::VisitRecordDecl(RecordDecl *D) { if (!isa<CXXRecordDecl>(D)) Record.push_back(D->getODRHash()); - if (D->getDeclContext() == D->getLexicalDeclContext() && - !D->hasAttrs() && - !D->isImplicit() && - !D->isUsed(false) && - !D->hasExtInfo() && + if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && + !D->isImplicit() && !D->isInvalidDecl() && !D->hasExtInfo() && !D->getTypedefNameForAnonDecl() && D->getFirstDecl() == D->getMostRecentDecl() && - !D->isInvalidDecl() && - !D->isReferenced() && !D->isTopLevelDeclInObjCContainer() && - D->getAccess() == AS_none && - !D->isModulePrivate() && !CXXRecordDecl::classofKind(D->getKind()) && !needsAnonymousDeclarationNumber(D) && D->getDeclName().getNameKind() == DeclarationName::Identifier) @@ -674,11 +669,16 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) { Record.AddDeclarationNameLoc(D->DNLoc, D->getDeclName()); Record.push_back(D->getIdentifierNamespace()); + // The order matters here. It will be better to put the bit with higher + // probability to be 0 in the end of the bits. See the comments in VisitDecl + // for details. BitsPacker FunctionDeclBits; // FIXME: stable encoding + FunctionDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), 3); FunctionDeclBits.addBits((uint32_t)D->getStorageClass(), /*BitWidth=*/3); FunctionDeclBits.addBit(D->isInlineSpecified()); FunctionDeclBits.addBit(D->isInlined()); + FunctionDeclBits.addBit(D->hasSkippedBody()); FunctionDeclBits.addBit(D->isVirtualAsWritten()); FunctionDeclBits.addBit(D->isPure()); FunctionDeclBits.addBit(D->hasInheritedPrototype()); @@ -689,14 +689,12 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) { FunctionDeclBits.addBit(D->isDefaulted()); FunctionDeclBits.addBit(D->isExplicitlyDefaulted()); FunctionDeclBits.addBit(D->isIneligibleOrNotSelected()); - FunctionDeclBits.addBit(D->hasImplicitReturnZero()); FunctionDeclBits.addBits((uint64_t)(D->getConstexprKind()), /*BitWidth=*/2); - FunctionDeclBits.addBit(D->usesSEHTry()); - FunctionDeclBits.addBit(D->hasSkippedBody()); + FunctionDeclBits.addBit(D->hasImplicitReturnZero()); FunctionDeclBits.addBit(D->isMultiVersion()); FunctionDeclBits.addBit(D->isLateTemplateParsed()); FunctionDeclBits.addBit(D->FriendConstraintRefersToEnclosingTemplate()); - FunctionDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), 3); + FunctionDeclBits.addBit(D->usesSEHTry()); Record.push_back(FunctionDeclBits); Record.AddSourceLocation(D->getEndLoc()); @@ -1060,7 +1058,28 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) { VisitRedeclarable(D); VisitDeclaratorDecl(D); + // The order matters here. It will be better to put the bit with higher + // probability to be 0 in the end of the bits. See the comments in VisitDecl + // for details. BitsPacker VarDeclBits; + VarDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), + /*BitWidth=*/3); + + bool ModulesCodegen = false; + if (Writer.WritingModule && D->getStorageDuration() == SD_Static && + !D->getDescribedVarTemplate()) { + // When building a C++20 module interface unit or a partition unit, a + // strong definition in the module interface is provided by the + // compilation of that unit, not by its users. (Inline variables are still + // emitted in module users.) + ModulesCodegen = + (Writer.WritingModule->isInterfaceOrPartition() || + (D->hasAttr<DLLExportAttr>() && + Writer.Context->getLangOpts().BuildingPCHWithObjectFile)) && + Writer.Context->GetGVALinkageForVariable(D) >= GVA_StrongExternal; + } + VarDeclBits.addBit(ModulesCodegen); + VarDeclBits.addBits(D->getStorageClass(), /*BitWidth=*/3); VarDeclBits.addBits(D->getTSCSpec(), /*BitWidth=*/2); VarDeclBits.addBits(D->getInitStyle(), /*BitWidth=*/2); @@ -1072,41 +1091,26 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) { VarDeclBits.addBit(D->isExceptionVariable()); VarDeclBits.addBit(D->isNRVOVariable()); VarDeclBits.addBit(D->isCXXForRangeDecl()); - VarDeclBits.addBit(D->isObjCForDecl()); + VarDeclBits.addBit(D->isInline()); VarDeclBits.addBit(D->isInlineSpecified()); VarDeclBits.addBit(D->isConstexpr()); VarDeclBits.addBit(D->isInitCapture()); VarDeclBits.addBit(D->isPreviousDeclInSameBlockScope()); + VarDeclBits.addBit(D->isEscapingByref()); + HasDeducedType = D->getType()->getContainedDeducedType(); + VarDeclBits.addBit(HasDeducedType); + if (const auto *IPD = dyn_cast<ImplicitParamDecl>(D)) VarDeclBits.addBits(llvm::to_underlying(IPD->getParameterKind()), /*Width=*/3); else VarDeclBits.addBits(0, /*Width=*/3); - VarDeclBits.addBit(D->isEscapingByref()); - HasDeducedType = D->getType()->getContainedDeducedType(); - VarDeclBits.addBit(HasDeducedType); - } - - VarDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), /*BitWidth=*/3); - - bool ModulesCodegen = false; - if (Writer.WritingModule && D->getStorageDuration() == SD_Static && - !D->getDescribedVarTemplate()) { - // When building a C++20 module interface unit or a partition unit, a - // strong definition in the module interface is provided by the - // compilation of that unit, not by its users. (Inline variables are still - // emitted in module users.) - ModulesCodegen = - (Writer.WritingModule->isInterfaceOrPartition() || - (D->hasAttr<DLLExportAttr>() && - Writer.Context->getLangOpts().BuildingPCHWithObjectFile)) && - Writer.Context->GetGVALinkageForVariable(D) >= GVA_StrongExternal; + VarDeclBits.addBit(D->isObjCForDecl()); } - VarDeclBits.addBit(ModulesCodegen); Record.push_back(VarDeclBits); if (ModulesCodegen) @@ -1135,29 +1139,17 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) { Record.push_back(VarNotTemplate); } - if (D->getDeclContext() == D->getLexicalDeclContext() && - !D->hasAttrs() && - !D->isImplicit() && - !D->isUsed(false) && - !D->isInvalidDecl() && - !D->isReferenced() && + if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && !D->isTopLevelDeclInObjCContainer() && - D->getAccess() == AS_none && - !D->isModulePrivate() && !needsAnonymousDeclarationNumber(D) && D->getDeclName().getNameKind() == DeclarationName::Identifier && - !D->hasExtInfo() && - D->getFirstDecl() == D->getMostRecentDecl() && - D->getKind() == Decl::Var && - !D->isInline() && - !D->isConstexpr() && - !D->isInitCapture() && - !D->isPreviousDeclInSameBlockScope() && - !D->isEscapingByref() && - !HasDeducedType && - D->getStorageDuration() != SD_Static && - !D->getDescribedVarTemplate() && - !D->getMemberSpecializationInfo()) + !D->hasExtInfo() && D->getFirstDecl() == D->getMostRecentDecl() && + D->getKind() == Decl::Var && !D->isInline() && !D->isConstexpr() && + !D->isInitCapture() && !D->isPreviousDeclInSameBlockScope() && + !D->isEscapingByref() && !HasDeducedType && + D->getStorageDuration() != SD_Static && !D->getDescribedVarTemplate() && + !D->getMemberSpecializationInfo() && !D->isObjCForDecl() && + !isa<ImplicitParamDecl>(D) && !D->isEscapingByref()) AbbrevToUse = Writer.getDeclVarAbbrev(); Code = serialization::DECL_VAR; @@ -1193,14 +1185,10 @@ void ASTDeclWriter::VisitParmVarDecl(ParmVarDecl *D) { // we dynamically check for the properties that we optimize for, but don't // know are true of all PARM_VAR_DECLs. if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && - !D->hasExtInfo() && !D->isImplicit() && !D->isUsed(false) && - !D->isInvalidDecl() && !D->isReferenced() && D->getAccess() == AS_none && - !D->isModulePrivate() && D->getStorageClass() == 0 && + !D->hasExtInfo() && D->getStorageClass() == 0 && !D->isInvalidDecl() && + !D->isTopLevelDeclInObjCContainer() && D->getInitStyle() == VarDecl::CInit && // Can params have anything else? - D->getFunctionScopeDepth() == 0 && D->getObjCDeclQualifier() == 0 && - !D->isKNRPromoted() && !D->isExplicitObjectParameter() && - !D->hasInheritedDefaultArg() && D->getInit() == nullptr && - !D->hasUninstantiatedDefaultArg()) // No default expr. + D->getInit() == nullptr) // No default expr. AbbrevToUse = Writer.getDeclParmVarAbbrev(); // Check things we know are true of *every* PARM_VAR_DECL, which is more than @@ -1403,6 +1391,13 @@ void ASTDeclWriter::VisitUsingShadowDecl(UsingShadowDecl *D) { Record.push_back(D->getIdentifierNamespace()); Record.AddDeclRef(D->UsingOrNextShadow); Record.AddDeclRef(Context.getInstantiatedFromUsingShadowDecl(D)); + + if (D->getDeclContext() == D->getLexicalDeclContext() && + D->getFirstDecl() == D->getMostRecentDecl() && !D->hasAttrs() && + !needsAnonymousDeclarationNumber(D) && + D->getDeclName().getNameKind() == DeclarationName::Identifier) + AbbrevToUse = Writer.getDeclUsingShadowAbbrev(); + Code = serialization::DECL_USING_SHADOW; } @@ -1507,10 +1502,32 @@ void ASTDeclWriter::VisitCXXMethodDecl(CXXMethodDecl *D) { D->getFirstDecl() == D->getMostRecentDecl() && !D->isInvalidDecl() && !D->hasAttrs() && !D->isTopLevelDeclInObjCContainer() && D->getDeclName().getNameKind() == DeclarationName::Identifier && - !D->hasExtInfo() && !D->hasInheritedPrototype() && - D->hasWrittenPrototype() && - D->getTemplatedKind() == FunctionDecl::TK_NonTemplate) - AbbrevToUse = Writer.getDeclCXXMethodAbbrev(); + !D->hasExtInfo() && !D->isExplicitlyDefaulted()) { + if (D->getTemplatedKind() == FunctionDecl::TK_NonTemplate || + D->getTemplatedKind() == FunctionDecl::TK_FunctionTemplate || + D->getTemplatedKind() == FunctionDecl::TK_MemberSpecialization || + D->getTemplatedKind() == FunctionDecl::TK_DependentNonTemplate) + AbbrevToUse = Writer.getDeclCXXMethodAbbrev(D->getTemplatedKind()); + else if (D->getTemplatedKind() == + FunctionDecl::TK_FunctionTemplateSpecialization) { + FunctionTemplateSpecializationInfo *FTSInfo = + D->getTemplateSpecializationInfo(); + + if (FTSInfo->TemplateArguments->size() == 1) { + const TemplateArgument &TA = FTSInfo->TemplateArguments->get(0); + if (TA.getKind() == TemplateArgument::Type && + !FTSInfo->TemplateArgumentsAsWritten && + !FTSInfo->getMemberSpecializationInfo()) + AbbrevToUse = Writer.getDeclCXXMethodAbbrev(D->getTemplatedKind()); + } + } else if (D->getTemplatedKind() == + FunctionDecl::TK_DependentFunctionTemplateSpecialization) { + DependentFunctionTemplateSpecializationInfo *DFTSInfo = + D->getDependentSpecializationInfo(); + if (!DFTSInfo->TemplateArgumentsAsWritten) + AbbrevToUse = Writer.getDeclCXXMethodAbbrev(D->getTemplatedKind()); + } + } Code = serialization::DECL_CXX_METHOD; } @@ -1782,7 +1799,7 @@ void ASTDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) { Record.push_back(D->wasDeclaredWithTypename()); const TypeConstraint *TC = D->getTypeConstraint(); - Record.push_back(TC != nullptr); + assert((bool)TC == D->hasTypeConstraint()); if (TC) { auto *CR = TC->getConceptReference(); Record.push_back(CR != nullptr); @@ -1800,6 +1817,13 @@ void ASTDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) { if (OwnsDefaultArg) Record.AddTypeSourceInfo(D->getDefaultArgumentInfo()); + if (!TC && !OwnsDefaultArg && + D->getDeclContext() == D->getLexicalDeclContext() && + !D->isInvalidDecl() && !D->hasAttrs() && + !D->isTopLevelDeclInObjCContainer() && !D->isImplicit() && + D->getDeclName().getNameKind() == DeclarationName::Identifier) + AbbrevToUse = Writer.getDeclTemplateTypeParmAbbrev(); + Code = serialization::DECL_TEMPLATE_TYPE_PARM; } @@ -2031,6 +2055,106 @@ void ASTDeclWriter::VisitOMPCapturedExprDecl(OMPCapturedExprDecl *D) { // ASTWriter Implementation //===----------------------------------------------------------------------===// +namespace { +template <FunctionDecl::TemplatedKind Kind> +std::shared_ptr<llvm::BitCodeAbbrev> +getFunctionDeclAbbrev(serialization::DeclCode Code) { + using namespace llvm; + + auto Abv = std::make_shared<BitCodeAbbrev>(); + Abv->Add(BitCodeAbbrevOp(Code)); + // RedeclarableDecl + Abv->Add(BitCodeAbbrevOp(0)); // CanonicalDecl + Abv->Add(BitCodeAbbrevOp(Kind)); + if constexpr (Kind == FunctionDecl::TK_NonTemplate) { + + } else if constexpr (Kind == FunctionDecl::TK_FunctionTemplate) { + // DescribedFunctionTemplate + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + } else if constexpr (Kind == FunctionDecl::TK_DependentNonTemplate) { + // Instantiated From Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + } else if constexpr (Kind == FunctionDecl::TK_MemberSpecialization) { + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // InstantiatedFrom + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 3)); // TemplateSpecializationKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Specialized Location + } else if constexpr (Kind == + FunctionDecl::TK_FunctionTemplateSpecialization) { + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Template + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 3)); // TemplateSpecializationKind + Abv->Add(BitCodeAbbrevOp(1)); // Template Argument Size + Abv->Add(BitCodeAbbrevOp(TemplateArgument::Type)); // Template Argument Kind + Abv->Add( + BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Template Argument Type + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Is Defaulted + Abv->Add(BitCodeAbbrevOp(0)); // TemplateArgumentsAsWritten + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation + Abv->Add(BitCodeAbbrevOp(0)); + Abv->Add( + BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Canonical Decl of template + } else if constexpr (Kind == FunctionDecl:: + TK_DependentFunctionTemplateSpecialization) { + // Candidates of specialization + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abv->Add(BitCodeAbbrevOp(0)); // TemplateArgumentsAsWritten + } else { + llvm_unreachable("Unknown templated kind?"); + } + // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 8)); // Packed DeclBits: ModuleOwnershipKind, + // isUsed, isReferenced, AccessSpecifier, + // isImplicit + // + // The following bits should be 0: + // HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, + // isInvalidDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID + // NamedDecl + Abv->Add(BitCodeAbbrevOp(DeclarationName::Identifier)); // NameKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Identifier + Abv->Add(BitCodeAbbrevOp(0)); // AnonDeclNumber + // ValueDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // DeclaratorDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // InnerLocStart + Abv->Add(BitCodeAbbrevOp(0)); // HasExtInfo + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // TSIType + // FunctionDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 11)); // IDNS + Abv->Add(BitCodeAbbrevOp( + BitCodeAbbrevOp::Fixed, + 27)); // Packed Function Bits: StorageClass, Inline, InlineSpecified, + // VirtualAsWritten, Pure, HasInheritedProto, HasWrittenProto, + // Deleted, Trivial, TrivialForCall, Defaulted, ExplicitlyDefaulted, + // IsIneligibleOrNotSelected, ImplicitReturnZero, Constexpr, + // UsesSEHTry, SkippedBody, MultiVersion, LateParsed, + // FriendConstraintRefersToEnclosingTemplate, Linkage + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LocEnd + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // ODRHash + // This Array slurps the rest of the record. Fortunately we want to encode + // (nearly) all the remaining (variable number of) fields in the same way. + // + // This is: + // NumParams and Params[] from FunctionDecl, and + // NumOverriddenMethods, OverriddenMethods[] from CXXMethodDecl. + // + // Add an AbbrevOp for 'size then elements' and use it here. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + return Abv; +} + +template <FunctionDecl::TemplatedKind Kind> +std::shared_ptr<llvm::BitCodeAbbrev> getCXXMethodAbbrev() { + return getFunctionDeclAbbrev<Kind>(serialization::DECL_CXX_METHOD); +} +} // namespace + void ASTWriter::WriteDeclAbbrevs() { using namespace llvm; @@ -2041,10 +2165,13 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::DECL_FIELD)); // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isUsed, isReferenced, AccessSpecifier, + // + // The following bits should be 0: + // isImplicit, HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, + // isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2104,10 +2231,13 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isUsed, isReferenced, AccessSpecifier, + // + // The following bits should be 0: + // isImplicit, HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, + // isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2145,10 +2275,13 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isUsed, isReferenced, AccessSpecifier, + // + // The following bits should be 0: + // isImplicit, HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, + // isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2193,10 +2326,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 8)); // Packed DeclBits: ModuleOwnershipKind, isUsed, + // isReferenced, AccessSpecifier, + // HasStandaloneLexicalDC, HasAttrs, isImplicit, + // TopLevelDeclInObjCContainer, + // isInvalidDecl, Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2233,10 +2367,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isReferenced, isUsed, AccessSpecifier. Other + // higher bits should be 0: isImplicit, + // HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2277,12 +2412,13 @@ void ASTWriter::WriteDeclAbbrevs() { // VarDecl Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, - 27)); // Packed Var Decl bits: SClass, TSCSpec, InitStyle, + 21)); // Packed Var Decl bits: Linkage, ModulesCodegen, + // SClass, TSCSpec, InitStyle, // isARCPseudoStrong, IsThisDeclarationADemotedDefinition, // isExceptionVariable, isNRVOVariable, isCXXForRangeDecl, - // isObjCForDecl, isInline, isInlineSpecified, isConstexpr, - // isInitCapture, isPrevDeclInSameScope, ImplicitParamKind, - // EscapingByref, HasDeducedType, Linkage, ModulesCodegen + // isInline, isInlineSpecified, isConstexpr, + // isInitCapture, isPrevDeclInSameScope, + // EscapingByref, HasDeducedType, ImplicitParamKind, isObjCForDecl Abv->Add(BitCodeAbbrevOp(0)); // VarKind (local enum) // Type Source Info Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -2290,71 +2426,84 @@ void ASTWriter::WriteDeclAbbrevs() { DeclVarAbbrev = Stream.EmitAbbrev(std::move(Abv)); // Abbreviation for DECL_CXX_METHOD + DeclCXXMethodAbbrev = + Stream.EmitAbbrev(getCXXMethodAbbrev<FunctionDecl::TK_NonTemplate>()); + DeclTemplateCXXMethodAbbrev = Stream.EmitAbbrev( + getCXXMethodAbbrev<FunctionDecl::TK_FunctionTemplate>()); + DeclDependentNonTemplateCXXMethodAbbrev = Stream.EmitAbbrev( + getCXXMethodAbbrev<FunctionDecl::TK_DependentNonTemplate>()); + DeclMemberSpecializedCXXMethodAbbrev = Stream.EmitAbbrev( + getCXXMethodAbbrev<FunctionDecl::TK_MemberSpecialization>()); + DeclTemplateSpecializedCXXMethodAbbrev = Stream.EmitAbbrev( + getCXXMethodAbbrev<FunctionDecl::TK_FunctionTemplateSpecialization>()); + DeclDependentSpecializationCXXMethodAbbrev = Stream.EmitAbbrev( + getCXXMethodAbbrev< + FunctionDecl::TK_DependentFunctionTemplateSpecialization>()); + + // Abbreviation for DECL_TEMPLATE_TYPE_PARM Abv = std::make_shared<BitCodeAbbrev>(); - Abv->Add(BitCodeAbbrevOp(serialization::DECL_CXX_METHOD)); - // RedeclarableDecl - Abv->Add(BitCodeAbbrevOp(0)); // CanonicalDecl - // FIXME: Implement abbreviation for other template kinds. - Abv->Add(BitCodeAbbrevOp(FunctionDecl::TK_NonTemplate)); // TemplateKind + Abv->Add(BitCodeAbbrevOp(serialization::DECL_TEMPLATE_TYPE_PARM)); + Abv->Add(BitCodeAbbrevOp(0)); // hasTypeConstraint + // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isReferenced, isUsed, AccessSpecifier. Other + // higher bits should be 0: isImplicit, + // HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, isInvalidDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID + // NamedDecl + Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name + Abv->Add(BitCodeAbbrevOp(0)); + // TypeDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type Ref + // TemplateTypeParmDecl + Abv->Add( + BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // wasDeclaredWithTypename + Abv->Add(BitCodeAbbrevOp(0)); // OwnsDefaultArg + DeclTemplateTypeParmAbbrev = Stream.EmitAbbrev(std::move(Abv)); + + // Abbreviation for DECL_USING_SHADOW + Abv = std::make_shared<BitCodeAbbrev>(); + Abv->Add(BitCodeAbbrevOp(serialization::DECL_USING_SHADOW)); + // Redeclarable + Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 12)); // Packed DeclBits: HasStandaloneLexicalDC, // isInvalidDecl, HasAttrs, isImplicit, isUsed, // isReferenced, TopLevelDeclInObjCContainer, // AccessSpecifier, ModuleOwnershipKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl - Abv->Add(BitCodeAbbrevOp(DeclarationName::Identifier)); // NameKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Identifier - Abv->Add(BitCodeAbbrevOp(0)); // AnonDeclNumber - // ValueDecl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // DeclaratorDecl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // InnerLocStart - Abv->Add(BitCodeAbbrevOp(0)); // HasExtInfo - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // TSIType - // FunctionDecl + Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name + Abv->Add(BitCodeAbbrevOp(0)); + // UsingShadowDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // TargetDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 11)); // IDNS - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 27)); // Packed Function Bits: StorageClass, Inline, InlineSpecified, - // VirtualAsWritten, Pure, HasInheritedProto, HasWrittenProto, - // Deleted, Trivial, TrivialForCall, Defaulted, ExplicitlyDefaulted, - // IsIneligibleOrNotSelected, ImplicitReturnZero, Constexpr, - // UsesSEHTry, SkippedBody, MultiVersion, LateParsed, - // FriendConstraintRefersToEnclosingTemplate, Linkage - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LocEnd - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Default - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // ODRHash - // This Array slurps the rest of the record. Fortunately we want to encode - // (nearly) all the remaining (variable number of) fields in the same way. - // - // This is: - // NumParams and Params[] from FunctionDecl, and - // NumOverriddenMethods, OverriddenMethods[] from CXXMethodDecl. - // - // Add an AbbrevOp for 'size then elements' and use it here. - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - DeclCXXMethodAbbrev = Stream.EmitAbbrev(std::move(Abv)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // UsingOrNextShadow + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, + 6)); // InstantiatedFromUsingShadowDecl + DeclUsingShadowAbbrev = Stream.EmitAbbrev(std::move(Abv)); // Abbreviation for EXPR_DECL_REF Abv = std::make_shared<BitCodeAbbrev>(); Abv->Add(BitCodeAbbrevOp(serialization::EXPR_DECL_REF)); - //Stmt - // Expr + // Stmt + // Expr + // PackingBits: DependenceKind, ValueKind. ObjectKind should be 0. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // DependenceKind, ValueKind, ObjectKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); - //DeclRefExpr - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //HasQualifier - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //GetDeclFound - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //ExplicitTemplateArgs - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //HadMultipleCandidates - Abv->Add(BitCodeAbbrevOp(0)); // RefersToEnclosingVariableOrCapture - Abv->Add(BitCodeAbbrevOp(0)); // NonOdrUseReason - Abv->Add(BitCodeAbbrevOp(0)); // IsImmediateEscalating + // DeclRefExpr + // Packing Bits: , HadMultipleCandidates, RefersToEnclosingVariableOrCapture, + // IsImmediateEscalating, NonOdrUseReason. + // GetDeclFound, HasQualifier and ExplicitTemplateArgs should be 0. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclRef Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location DeclRefExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); @@ -2364,10 +2513,10 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_INTEGER_LITERAL)); //Stmt // Expr - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // DependenceKind, ValueKind, ObjectKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); - //Integer Literal + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // Integer Literal Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location Abv->Add(BitCodeAbbrevOp(32)); // Bit Width Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Value @@ -2378,10 +2527,10 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_CHARACTER_LITERAL)); //Stmt // Expr - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // DependenceKind, ValueKind, ObjectKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); - //Character Literal + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // Character Literal Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // getValue Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // getKind @@ -2392,17 +2541,108 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_IMPLICIT_CAST)); // Stmt // Expr - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // DependenceKind, ValueKind, ObjectKind + // Packing Bits: DependenceKind, ValueKind, ObjectKind, Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // CastExpr Abv->Add(BitCodeAbbrevOp(0)); // PathSize - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // HasFPFeatures - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // CastKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // PartOfExplicitCast + // Packing Bits: CastKind, StoredFPFeatures, isPartOfExplicitCast + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 9)); // ImplicitCastExpr ExprImplicitCastAbbrev = Stream.EmitAbbrev(std::move(Abv)); + // Abbreviation for EXPR_BINARY_OPERATOR + Abv = std::make_shared<BitCodeAbbrev>(); + Abv->Add(BitCodeAbbrevOp(serialization::EXPR_BINARY_OPERATOR)); + // Stmt + // Expr + // Packing Bits: DependenceKind. ValueKind and ObjectKind should + // be 0 in this case. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // BinaryOperator + Abv->Add( + BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpCode and HasFPFeatures + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + BinaryOperatorAbbrev = Stream.EmitAbbrev(std::move(Abv)); + + // Abbreviation for EXPR_COMPOUND_ASSIGN_OPERATOR + Abv = std::make_shared<BitCodeAbbrev>(); + Abv->Add(BitCodeAbbrevOp(serialization::EXPR_COMPOUND_ASSIGN_OPERATOR)); + // Stmt + // Expr + // Packing Bits: DependenceKind. ValueKind and ObjectKind should + // be 0 in this case. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // BinaryOperator + // Packing Bits: OpCode. The HasFPFeatures bit should be 0 + Abv->Add( + BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpCode and HasFPFeatures + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + // CompoundAssignOperator + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHSType + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Result Type + CompoundAssignOperatorAbbrev = Stream.EmitAbbrev(std::move(Abv)); + + // Abbreviation for EXPR_CALL + Abv = std::make_shared<BitCodeAbbrev>(); + Abv->Add(BitCodeAbbrevOp(serialization::EXPR_CALL)); + // Stmt + // Expr + // Packing Bits: DependenceKind, ValueKind, ObjectKind, + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // CallExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // NumArgs + Abv->Add(BitCodeAbbrevOp(0)); // ADLCallKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + CallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); + + // Abbreviation for EXPR_CXX_OPERATOR_CALL + Abv = std::make_shared<BitCodeAbbrev>(); + Abv->Add(BitCodeAbbrevOp(serialization::EXPR_CXX_OPERATOR_CALL)); + // Stmt + // Expr + // Packing Bits: DependenceKind, ValueKind, ObjectKind, + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // CallExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // NumArgs + Abv->Add(BitCodeAbbrevOp(0)); // ADLCallKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + // CXXOperatorCallExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Operator Kind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + CXXOperatorCallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); + + // Abbreviation for EXPR_CXX_MEMBER_CALL + Abv = std::make_shared<BitCodeAbbrev>(); + Abv->Add(BitCodeAbbrevOp(serialization::EXPR_CXX_MEMBER_CALL)); + // Stmt + // Expr + // Packing Bits: DependenceKind, ValueKind, ObjectKind, + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // CallExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // NumArgs + Abv->Add(BitCodeAbbrevOp(0)); // ADLCallKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + // CXXMemberCallExpr + CXXMemberCallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); + + // Abbreviation for STMT_COMPOUND + Abv = std::make_shared<BitCodeAbbrev>(); + Abv->Add(BitCodeAbbrevOp(serialization::STMT_COMPOUND)); + // Stmt + // CompoundStmt + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Num Stmts + Abv->Add(BitCodeAbbrevOp(0)); // hasStoredFPFeatures + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location + CompoundStmtAbbrev = Stream.EmitAbbrev(std::move(Abv)); + Abv = std::make_shared<BitCodeAbbrev>(); Abv->Add(BitCodeAbbrevOp(serialization::DECL_CONTEXT_LEXICAL)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); diff --git a/contrib/llvm-project/clang/lib/Serialization/ASTWriterStmt.cpp b/contrib/llvm-project/clang/lib/Serialization/ASTWriterStmt.cpp index 8524484ea8a0..7f888e44dde1 100644 --- a/contrib/llvm-project/clang/lib/Serialization/ASTWriterStmt.cpp +++ b/contrib/llvm-project/clang/lib/Serialization/ASTWriterStmt.cpp @@ -37,15 +37,70 @@ namespace clang { serialization::StmtCode Code; unsigned AbbrevToUse; + /// A helper that can help us to write a packed bit across function + /// calls. For example, we may write seperate bits in seperate functions: + /// + /// void VisitA(A* a) { + /// Record.push_back(a->isSomething()); + /// } + /// + /// void Visitb(B *b) { + /// VisitA(b); + /// Record.push_back(b->isAnother()); + /// } + /// + /// In such cases, it'll be better if we can pack these 2 bits. We achieve + /// this by writing a zero value in `VisitA` and recorded that first and add + /// the new bit to the recorded value. + class PakedBitsWriter { + public: + PakedBitsWriter(ASTRecordWriter &Record) : RecordRef(Record) {} + ~PakedBitsWriter() { assert(!CurrentIndex); } + + void addBit(bool Value) { + assert(CurrentIndex && "Writing Bits without recording first!"); + PackingBits.addBit(Value); + } + void addBits(uint32_t Value, uint32_t BitsWidth) { + assert(CurrentIndex && "Writing Bits without recording first!"); + PackingBits.addBits(Value, BitsWidth); + } + + void writeBits() { + if (!CurrentIndex) + return; + + RecordRef[*CurrentIndex] = (uint32_t)PackingBits; + CurrentIndex = std::nullopt; + PackingBits.reset(0); + } + + void updateBits() { + writeBits(); + + CurrentIndex = RecordRef.size(); + RecordRef.push_back(0); + } + + private: + BitsPacker PackingBits; + ASTRecordWriter &RecordRef; + std::optional<unsigned> CurrentIndex; + }; + + PakedBitsWriter CurrentPackingBits; + public: ASTStmtWriter(ASTWriter &Writer, ASTWriter::RecordData &Record) : Writer(Writer), Record(Writer, Record), - Code(serialization::STMT_NULL_PTR), AbbrevToUse(0) {} + Code(serialization::STMT_NULL_PTR), AbbrevToUse(0), + CurrentPackingBits(this->Record) {} ASTStmtWriter(const ASTStmtWriter&) = delete; ASTStmtWriter &operator=(const ASTStmtWriter &) = delete; uint64_t Emit() { + CurrentPackingBits.writeBits(); assert(Code != serialization::STMT_NULL_PTR && "unhandled sub-statement writing AST file"); return Record.EmitStmt(Code, AbbrevToUse); @@ -82,14 +137,20 @@ void ASTStmtWriter::VisitNullStmt(NullStmt *S) { void ASTStmtWriter::VisitCompoundStmt(CompoundStmt *S) { VisitStmt(S); + Record.push_back(S->size()); Record.push_back(S->hasStoredFPFeatures()); + for (auto *CS : S->body()) Record.AddStmt(CS); if (S->hasStoredFPFeatures()) Record.push_back(S->getStoredFPFeatures().getAsOpaqueInt()); Record.AddSourceLocation(S->getLBracLoc()); Record.AddSourceLocation(S->getRBracLoc()); + + if (!S->hasStoredFPFeatures()) + AbbrevToUse = Writer.getCompoundStmtAbbrev(); + Code = serialization::STMT_COMPOUND; } @@ -143,9 +204,11 @@ void ASTStmtWriter::VisitIfStmt(IfStmt *S) { bool HasVar = S->getConditionVariableDeclStmt() != nullptr; bool HasInit = S->getInit() != nullptr; - Record.push_back(HasElse); - Record.push_back(HasVar); - Record.push_back(HasInit); + CurrentPackingBits.updateBits(); + + CurrentPackingBits.addBit(HasElse); + CurrentPackingBits.addBit(HasVar); + CurrentPackingBits.addBit(HasInit); Record.push_back(static_cast<uint64_t>(S->getStatementKind())); Record.AddStmt(S->getCond()); Record.AddStmt(S->getThen()); @@ -548,15 +611,13 @@ void ASTStmtWriter::VisitCapturedStmt(CapturedStmt *S) { void ASTStmtWriter::VisitExpr(Expr *E) { VisitStmt(E); - Record.AddTypeRef(E->getType()); - BitsPacker ExprBits; + CurrentPackingBits.updateBits(); + CurrentPackingBits.addBits(E->getDependence(), /*BitsWidth=*/5); + CurrentPackingBits.addBits(E->getValueKind(), /*BitsWidth=*/2); + CurrentPackingBits.addBits(E->getObjectKind(), /*BitsWidth=*/3); - ExprBits.addBits(E->getDependence(), /*BitsWidth=*/5); - ExprBits.addBits(E->getValueKind(), /*BitsWidth=*/2); - ExprBits.addBits(E->getObjectKind(), /*BitsWidth=*/3); - - Record.push_back(ExprBits); + Record.AddTypeRef(E->getType()); } void ASTStmtWriter::VisitConstantExpr(ConstantExpr *E) { @@ -612,13 +673,15 @@ void ASTStmtWriter::VisitPredefinedExpr(PredefinedExpr *E) { void ASTStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) { VisitExpr(E); - Record.push_back(E->hasQualifier()); - Record.push_back(E->getDecl() != E->getFoundDecl()); - Record.push_back(E->hasTemplateKWAndArgsInfo()); - Record.push_back(E->hadMultipleCandidates()); - Record.push_back(E->refersToEnclosingVariableOrCapture()); - Record.push_back(E->isNonOdrUse()); - Record.push_back(E->isImmediateEscalating()); + CurrentPackingBits.updateBits(); + + CurrentPackingBits.addBit(E->hadMultipleCandidates()); + CurrentPackingBits.addBit(E->refersToEnclosingVariableOrCapture()); + CurrentPackingBits.addBits(E->isNonOdrUse(), /*Width=*/2); + CurrentPackingBits.addBit(E->isImmediateEscalating()); + CurrentPackingBits.addBit(E->getDecl() != E->getFoundDecl()); + CurrentPackingBits.addBit(E->hasQualifier()); + CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); if (E->hasTemplateKWAndArgsInfo()) { unsigned NumTemplateArgs = E->getNumTemplateArgs(); @@ -629,9 +692,7 @@ void ASTStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) { if ((!E->hasTemplateKWAndArgsInfo()) && (!E->hasQualifier()) && (E->getDecl() == E->getFoundDecl()) && - nk == DeclarationName::Identifier && - !E->refersToEnclosingVariableOrCapture() && !E->isNonOdrUse() && - !E->isImmediateEscalating()) { + nk == DeclarationName::Identifier && E->getObjectKind() == OK_Ordinary) { AbbrevToUse = Writer.getDeclRefExprAbbrev(); } @@ -742,11 +803,13 @@ void ASTStmtWriter::VisitUnaryOperator(UnaryOperator *E) { bool HasFPFeatures = E->hasStoredFPFeatures(); // Write this first for easy access when deserializing, as they affect the // size of the UnaryOperator. - Record.push_back(HasFPFeatures); + CurrentPackingBits.addBit(HasFPFeatures); Record.AddStmt(E->getSubExpr()); - Record.push_back(E->getOpcode()); // FIXME: stable encoding + CurrentPackingBits.addBits(E->getOpcode(), + /*Width=*/5); // FIXME: stable encoding Record.AddSourceLocation(E->getOperatorLoc()); - Record.push_back(E->canOverflow()); + CurrentPackingBits.addBit(E->canOverflow()); + if (HasFPFeatures) Record.push_back(E->getStoredFPFeatures().getAsOpaqueInt()); Code = serialization::EXPR_UNARY_OPERATOR; @@ -872,12 +935,10 @@ void ASTStmtWriter::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtWriter::VisitCallExpr(CallExpr *E) { VisitExpr(E); - BitsPacker CallExprBits; - // 16 bits should be sufficient to store the number args; - CallExprBits.addBits(E->getNumArgs(), /*BitsWidth=*/16); - CallExprBits.addBit(E->hasStoredFPFeatures()); - CallExprBits.addBit(static_cast<bool>(E->getADLCallKind())); - Record.push_back(CallExprBits); + Record.push_back(E->getNumArgs()); + CurrentPackingBits.updateBits(); + CurrentPackingBits.addBit(static_cast<bool>(E->getADLCallKind())); + CurrentPackingBits.addBit(E->hasStoredFPFeatures()); Record.AddSourceLocation(E->getRParenLoc()); Record.AddStmt(E->getCallee()); @@ -887,6 +948,11 @@ void ASTStmtWriter::VisitCallExpr(CallExpr *E) { if (E->hasStoredFPFeatures()) Record.push_back(E->getFPFeatures().getAsOpaqueInt()); + + if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind()) && + E->getStmtClass() == Stmt::CallExprClass) + AbbrevToUse = Writer.getCallExprAbbrev(); + Code = serialization::EXPR_CALL; } @@ -913,9 +979,10 @@ void ASTStmtWriter::VisitMemberExpr(MemberExpr *E) { // Write these first for easy access when deserializing, as they affect the // size of the MemberExpr. - Record.push_back(HasQualifier); - Record.push_back(HasFoundDecl); - Record.push_back(HasTemplateInfo); + CurrentPackingBits.updateBits(); + CurrentPackingBits.addBit(HasQualifier); + CurrentPackingBits.addBit(HasFoundDecl); + CurrentPackingBits.addBit(HasTemplateInfo); Record.push_back(NumTemplateArgs); Record.AddStmt(E->getBase()); @@ -923,15 +990,15 @@ void ASTStmtWriter::VisitMemberExpr(MemberExpr *E) { Record.AddDeclarationNameLoc(E->MemberDNLoc, E->getMemberDecl()->getDeclName()); Record.AddSourceLocation(E->getMemberLoc()); - Record.push_back(E->isArrow()); - Record.push_back(E->hadMultipleCandidates()); - Record.push_back(E->isNonOdrUse()); + CurrentPackingBits.addBit(E->isArrow()); + CurrentPackingBits.addBit(E->hadMultipleCandidates()); + CurrentPackingBits.addBits(E->isNonOdrUse(), /*Width=*/2); Record.AddSourceLocation(E->getOperatorLoc()); if (HasFoundDecl) { DeclAccessPair FoundDecl = E->getFoundDecl(); Record.AddDeclRef(FoundDecl.getDecl()); - Record.push_back(FoundDecl.getAccess()); + CurrentPackingBits.addBits(FoundDecl.getAccess(), /*BitWidth=*/2); } if (HasQualifier) @@ -971,10 +1038,13 @@ void ASTStmtWriter::VisitObjCBridgedCastExpr(ObjCBridgedCastExpr *E) { void ASTStmtWriter::VisitCastExpr(CastExpr *E) { VisitExpr(E); + Record.push_back(E->path_size()); - Record.push_back(E->hasStoredFPFeatures()); + CurrentPackingBits.updateBits(); + // 7 bits should be enough to store the casting kinds. + CurrentPackingBits.addBits(E->getCastKind(), /*Width=*/7); + CurrentPackingBits.addBit(E->hasStoredFPFeatures()); Record.AddStmt(E->getSubExpr()); - Record.push_back(E->getCastKind()); // FIXME: stable encoding for (CastExpr::path_iterator PI = E->path_begin(), PE = E->path_end(); PI != PE; ++PI) @@ -986,16 +1056,23 @@ void ASTStmtWriter::VisitCastExpr(CastExpr *E) { void ASTStmtWriter::VisitBinaryOperator(BinaryOperator *E) { VisitExpr(E); - bool HasFPFeatures = E->hasStoredFPFeatures(); + // Write this first for easy access when deserializing, as they affect the // size of the UnaryOperator. - Record.push_back(HasFPFeatures); - Record.push_back(E->getOpcode()); // FIXME: stable encoding + CurrentPackingBits.updateBits(); + CurrentPackingBits.addBits(E->getOpcode(), /*Width=*/6); + bool HasFPFeatures = E->hasStoredFPFeatures(); + CurrentPackingBits.addBit(HasFPFeatures); Record.AddStmt(E->getLHS()); Record.AddStmt(E->getRHS()); Record.AddSourceLocation(E->getOperatorLoc()); if (HasFPFeatures) Record.push_back(E->getStoredFPFeatures().getAsOpaqueInt()); + + if (!HasFPFeatures && E->getValueKind() == VK_PRValue && + E->getObjectKind() == OK_Ordinary) + AbbrevToUse = Writer.getBinaryOperatorAbbrev(); + Code = serialization::EXPR_BINARY_OPERATOR; } @@ -1003,6 +1080,11 @@ void ASTStmtWriter::VisitCompoundAssignOperator(CompoundAssignOperator *E) { VisitBinaryOperator(E); Record.AddTypeRef(E->getComputationLHSType()); Record.AddTypeRef(E->getComputationResultType()); + + if (!E->hasStoredFPFeatures() && E->getValueKind() == VK_PRValue && + E->getObjectKind() == OK_Ordinary) + AbbrevToUse = Writer.getCompoundAssignOperatorAbbrev(); + Code = serialization::EXPR_COMPOUND_ASSIGN_OPERATOR; } @@ -1031,7 +1113,7 @@ ASTStmtWriter::VisitBinaryConditionalOperator(BinaryConditionalOperator *E) { void ASTStmtWriter::VisitImplicitCastExpr(ImplicitCastExpr *E) { VisitCastExpr(E); - Record.push_back(E->isPartOfExplicitCast()); + CurrentPackingBits.addBit(E->isPartOfExplicitCast()); if (E->path_size() == 0 && !E->hasStoredFPFeatures()) AbbrevToUse = Writer.getExprImplicitCastAbbrev(); @@ -1588,11 +1670,19 @@ void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) { VisitCallExpr(E); Record.push_back(E->getOperator()); Record.AddSourceRange(E->Range); + + if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind())) + AbbrevToUse = Writer.getCXXOperatorCallExprAbbrev(); + Code = serialization::EXPR_CXX_OPERATOR_CALL; } void ASTStmtWriter::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) { VisitCallExpr(E); + + if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind())) + AbbrevToUse = Writer.getCXXMemberCallExprAbbrev(); + Code = serialization::EXPR_CXX_MEMBER_CALL; } @@ -1673,7 +1763,9 @@ void ASTStmtWriter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) void ASTStmtWriter::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) { VisitExplicitCastExpr(E); Record.AddSourceRange(SourceRange(E->getOperatorLoc(), E->getRParenLoc())); - Record.AddSourceRange(E->getAngleBrackets()); + CurrentPackingBits.addBit(E->getAngleBrackets().isValid()); + if (E->getAngleBrackets().isValid()) + Record.AddSourceRange(E->getAngleBrackets()); } void ASTStmtWriter::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) { @@ -1750,6 +1842,7 @@ void ASTStmtWriter::VisitCXXThisExpr(CXXThisExpr *E) { VisitExpr(E); Record.AddSourceLocation(E->getLocation()); Record.push_back(E->isImplicit()); + Code = serialization::EXPR_CXX_THIS; } @@ -1883,10 +1976,10 @@ void ASTStmtWriter::VisitCXXDependentScopeMemberExpr( // Don't emit anything here (or if you do you will have to update // the corresponding deserialization function). - - Record.push_back(E->hasTemplateKWAndArgsInfo()); Record.push_back(E->getNumTemplateArgs()); - Record.push_back(E->hasFirstQualifierFoundInScope()); + CurrentPackingBits.updateBits(); + CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); + CurrentPackingBits.addBit(E->hasFirstQualifierFoundInScope()); if (E->hasTemplateKWAndArgsInfo()) { const ASTTemplateKWAndArgsInfo &ArgInfo = @@ -1895,14 +1988,15 @@ void ASTStmtWriter::VisitCXXDependentScopeMemberExpr( E->getTrailingObjects<TemplateArgumentLoc>()); } - Record.push_back(E->isArrow()); - Record.AddSourceLocation(E->getOperatorLoc()); + CurrentPackingBits.addBit(E->isArrow()); + Record.AddTypeRef(E->getBaseType()); Record.AddNestedNameSpecifierLoc(E->getQualifierLoc()); + CurrentPackingBits.addBit(!E->isImplicitAccess()); if (!E->isImplicitAccess()) Record.AddStmt(E->getBase()); - else - Record.AddStmt(nullptr); + + Record.AddSourceLocation(E->getOperatorLoc()); if (E->hasFirstQualifierFoundInScope()) Record.AddDeclRef(E->getFirstQualifierFoundInScope()); @@ -1917,12 +2011,14 @@ ASTStmtWriter::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) { // Don't emit anything here, HasTemplateKWAndArgsInfo must be // emitted first. + CurrentPackingBits.addBit( + E->DependentScopeDeclRefExprBits.HasTemplateKWAndArgsInfo); - Record.push_back(E->DependentScopeDeclRefExprBits.HasTemplateKWAndArgsInfo); if (E->DependentScopeDeclRefExprBits.HasTemplateKWAndArgsInfo) { const ASTTemplateKWAndArgsInfo &ArgInfo = *E->getTrailingObjects<ASTTemplateKWAndArgsInfo>(); - Record.push_back(ArgInfo.NumTemplateArgs); + // 16 bits should be enought to store the number of args + CurrentPackingBits.addBits(ArgInfo.NumTemplateArgs, /*Width=*/16); AddTemplateKWAndArgsInfo(ArgInfo, E->getTrailingObjects<TemplateArgumentLoc>()); } @@ -1949,19 +2045,16 @@ ASTStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); - BitsPacker OverloadExprBits; - // 14 Bits should enough to store the number of decls. - OverloadExprBits.addBits(E->getNumDecls(), /*BitWidth=*/14); - OverloadExprBits.addBit(E->hasTemplateKWAndArgsInfo()); + Record.push_back(E->getNumDecls()); + + CurrentPackingBits.updateBits(); + CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); if (E->hasTemplateKWAndArgsInfo()) { const ASTTemplateKWAndArgsInfo &ArgInfo = *E->getTrailingASTTemplateKWAndArgsInfo(); - // 14 Bits should enough to store the number of template args. - OverloadExprBits.addBits(ArgInfo.NumTemplateArgs, /*BitWidth=*/14); - Record.push_back(OverloadExprBits); + Record.push_back(ArgInfo.NumTemplateArgs); AddTemplateKWAndArgsInfo(ArgInfo, E->getTrailingTemplateArgumentLoc()); - } else - Record.push_back(OverloadExprBits); + } for (OverloadExpr::decls_iterator OvI = E->decls_begin(), OvE = E->decls_end(); @@ -1976,18 +2069,22 @@ void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) { void ASTStmtWriter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) { VisitOverloadExpr(E); - Record.push_back(E->isArrow()); - Record.push_back(E->hasUnresolvedUsing()); - Record.AddStmt(!E->isImplicitAccess() ? E->getBase() : nullptr); - Record.AddTypeRef(E->getBaseType()); + CurrentPackingBits.addBit(E->isArrow()); + CurrentPackingBits.addBit(E->hasUnresolvedUsing()); + CurrentPackingBits.addBit(!E->isImplicitAccess()); + if (!E->isImplicitAccess()) + Record.AddStmt(E->getBase()); + Record.AddSourceLocation(E->getOperatorLoc()); + + Record.AddTypeRef(E->getBaseType()); Code = serialization::EXPR_CXX_UNRESOLVED_MEMBER; } void ASTStmtWriter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) { VisitOverloadExpr(E); - Record.push_back(E->requiresADL()); - Record.push_back(E->isOverloaded()); + CurrentPackingBits.addBit(E->requiresADL()); + CurrentPackingBits.addBit(E->isOverloaded()); Record.AddDeclRef(E->getNamingClass()); Code = serialization::EXPR_CXX_UNRESOLVED_LOOKUP; } @@ -2059,12 +2156,12 @@ void ASTStmtWriter::VisitSubstNonTypeTemplateParmExpr( SubstNonTypeTemplateParmExpr *E) { VisitExpr(E); Record.AddDeclRef(E->getAssociatedDecl()); - Record.push_back(E->isReferenceParameter()); - Record.push_back(E->getIndex()); + CurrentPackingBits.addBit(E->isReferenceParameter()); + CurrentPackingBits.addBits(E->getIndex(), /*Width=*/12); + CurrentPackingBits.addBit((bool)E->getPackIndex()); if (auto PackIndex = E->getPackIndex()) Record.push_back(*PackIndex + 1); - else - Record.push_back(0); + Record.AddSourceLocation(E->getNameLoc()); Record.AddStmt(E->getReplacement()); Code = serialization::EXPR_SUBST_NON_TYPE_TEMPLATE_PARM; diff --git a/contrib/llvm-project/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/contrib/llvm-project/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp index 925fc90e3554..254b36ed0396 100644 --- a/contrib/llvm-project/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp +++ b/contrib/llvm-project/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp @@ -266,6 +266,8 @@ private: {&StreamChecker::preFseek, &StreamChecker::evalFseek, 0}}, {{{"ftell"}, 1}, {&StreamChecker::preDefault, &StreamChecker::evalFtell, 0}}, + {{{"fflush"}, 1}, + {&StreamChecker::preFflush, &StreamChecker::evalFflush, 0}}, {{{"rewind"}, 1}, {&StreamChecker::preDefault, &StreamChecker::evalRewind, 0}}, {{{"fgetpos"}, 2}, @@ -360,6 +362,12 @@ private: CheckerContext &C, const StreamErrorState &ErrorKind) const; + void preFflush(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const; + + void evalFflush(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const; + /// Check that the stream (in StreamVal) is not NULL. /// If it can only be NULL a fatal error is emitted and nullptr returned. /// Otherwise the return value is a new state where the stream is constrained @@ -1191,6 +1199,84 @@ void StreamChecker::evalSetFeofFerror(const FnDescription *Desc, C.addTransition(State); } +void StreamChecker::preFflush(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const { + ProgramStateRef State = C.getState(); + SVal StreamVal = getStreamArg(Desc, Call); + std::optional<DefinedSVal> Stream = StreamVal.getAs<DefinedSVal>(); + if (!Stream) + return; + + ProgramStateRef StateNotNull, StateNull; + std::tie(StateNotNull, StateNull) = + C.getConstraintManager().assumeDual(State, *Stream); + if (StateNotNull && !StateNull) + ensureStreamOpened(StreamVal, C, StateNotNull); +} + +void StreamChecker::evalFflush(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const { + ProgramStateRef State = C.getState(); + SVal StreamVal = getStreamArg(Desc, Call); + std::optional<DefinedSVal> Stream = StreamVal.getAs<DefinedSVal>(); + if (!Stream) + return; + + // Skip if the stream can be both NULL and non-NULL. + ProgramStateRef StateNotNull, StateNull; + std::tie(StateNotNull, StateNull) = + C.getConstraintManager().assumeDual(State, *Stream); + if (StateNotNull && StateNull) + return; + if (StateNotNull && !StateNull) + State = StateNotNull; + else + State = StateNull; + + const CallExpr *CE = dyn_cast_or_null<CallExpr>(Call.getOriginExpr()); + if (!CE) + return; + + // `fflush` returns EOF on failure, otherwise returns 0. + ProgramStateRef StateFailed = bindInt(*EofVal, State, C, CE); + ProgramStateRef StateNotFailed = bindInt(0, State, C, CE); + + // Clear error states if `fflush` returns 0, but retain their EOF flags. + auto ClearErrorInNotFailed = [&StateNotFailed, Desc](SymbolRef Sym, + const StreamState *SS) { + if (SS->ErrorState & ErrorFError) { + StreamErrorState NewES = + (SS->ErrorState & ErrorFEof) ? ErrorFEof : ErrorNone; + StreamState NewSS = StreamState::getOpened(Desc, NewES, false); + StateNotFailed = StateNotFailed->set<StreamMap>(Sym, NewSS); + } + }; + + if (StateNotNull && !StateNull) { + // Skip if the input stream's state is unknown, open-failed or closed. + if (SymbolRef StreamSym = StreamVal.getAsSymbol()) { + const StreamState *SS = State->get<StreamMap>(StreamSym); + if (SS) { + assert(SS->isOpened() && "Stream is expected to be opened"); + ClearErrorInNotFailed(StreamSym, SS); + } else + return; + } + } else { + // Clear error states for all streams. + const StreamMapTy &Map = StateNotFailed->get<StreamMap>(); + for (const auto &I : Map) { + SymbolRef Sym = I.first; + const StreamState &SS = I.second; + if (SS.isOpened()) + ClearErrorInNotFailed(Sym, &SS); + } + } + + C.addTransition(StateNotFailed); + C.addTransition(StateFailed); +} + ProgramStateRef StreamChecker::ensureStreamNonNull(SVal StreamVal, const Expr *StreamE, CheckerContext &C, diff --git a/contrib/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp b/contrib/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp index bb9f7dc7e7e3..bf47461b59e0 100644 --- a/contrib/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp +++ b/contrib/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp @@ -1217,7 +1217,7 @@ raw_ostream &operator<<(raw_ostream &OS, const RVVIntrinsicRecord &Record) { OS << (int)Record.PrototypeLength << ","; OS << (int)Record.SuffixLength << ","; OS << (int)Record.OverloadedSuffixSize << ","; - OS << (int)Record.RequiredExtensions << ","; + OS << Record.RequiredExtensions << ","; OS << (int)Record.TypeRangeMask << ","; OS << (int)Record.Log2LMULMask << ","; OS << (int)Record.NF << ","; diff --git a/contrib/llvm-project/clang/tools/clang-repl/ClangRepl.cpp b/contrib/llvm-project/clang/tools/clang-repl/ClangRepl.cpp index b9b287127015..5bad8145324d 100644 --- a/contrib/llvm-project/clang/tools/clang-repl/ClangRepl.cpp +++ b/contrib/llvm-project/clang/tools/clang-repl/ClangRepl.cpp @@ -15,6 +15,8 @@ #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Interpreter/CodeCompletion.h" #include "clang/Interpreter/Interpreter.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Sema/Sema.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/LineEditor/LineEditor.h" @@ -123,22 +125,14 @@ ReplListCompleter::operator()(llvm::StringRef Buffer, size_t Pos, return {}; } - - codeComplete( - const_cast<clang::CompilerInstance *>((*Interp)->getCompilerInstance()), - Buffer, Lines, Pos + 1, MainInterp.getCompilerInstance(), Results); - - size_t space_pos = Buffer.rfind(" "); - llvm::StringRef Prefix; - if (space_pos == llvm::StringRef::npos) { - Prefix = Buffer; - } else { - Prefix = Buffer.substr(space_pos + 1); - } - + auto *MainCI = (*Interp)->getCompilerInstance(); + auto CC = clang::ReplCodeCompleter(); + CC.codeComplete(MainCI, Buffer, Lines, Pos + 1, + MainInterp.getCompilerInstance(), Results); for (auto c : Results) { - if (c.find(Prefix) == 0) - Comps.push_back(llvm::LineEditor::Completion(c.substr(Prefix.size()), c)); + if (c.find(CC.Prefix) == 0) + Comps.push_back( + llvm::LineEditor::Completion(c.substr(CC.Prefix.size()), c)); } return Comps; } diff --git a/contrib/llvm-project/clang/utils/TableGen/NeonEmitter.cpp b/contrib/llvm-project/clang/utils/TableGen/NeonEmitter.cpp index e5f79ba99c5c..53334016c180 100644 --- a/contrib/llvm-project/clang/utils/TableGen/NeonEmitter.cpp +++ b/contrib/llvm-project/clang/utils/TableGen/NeonEmitter.cpp @@ -550,6 +550,8 @@ class NeonEmitter { void createIntrinsic(Record *R, SmallVectorImpl<Intrinsic *> &Out); void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs); + void genStreamingSVECompatibleList(raw_ostream &OS, + SmallVectorImpl<Intrinsic *> &Defs); void genOverloadTypeCheckCode(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs); void genIntrinsicRangeCheckCode(raw_ostream &OS, @@ -2041,6 +2043,30 @@ void NeonEmitter::genBuiltinsDef(raw_ostream &OS, OS << "#endif\n\n"; } +void NeonEmitter::genStreamingSVECompatibleList( + raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs) { + OS << "#ifdef GET_NEON_STREAMING_COMPAT_FLAG\n"; + + std::set<std::string> Emitted; + for (auto *Def : Defs) { + // If the def has a body (that is, it has Operation DAGs), it won't call + // __builtin_neon_* so we don't need to generate a definition for it. + if (Def->hasBody()) + continue; + + std::string Name = Def->getMangledName(); + if (Emitted.find(Name) != Emitted.end()) + continue; + + // FIXME: We should make exceptions here for some NEON builtins that are + // permitted in streaming mode. + OS << "case NEON::BI__builtin_neon_" << Name + << ": BuiltinType = ArmNonStreaming; break;\n"; + Emitted.insert(Name); + } + OS << "#endif\n\n"; +} + /// Generate the ARM and AArch64 overloaded type checking code for /// SemaChecking.cpp, checking for unique builtin declarations. void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS, @@ -2224,6 +2250,8 @@ void NeonEmitter::runHeader(raw_ostream &OS) { // Generate ARM overloaded type checking code for SemaChecking.cpp genOverloadTypeCheckCode(OS, Defs); + genStreamingSVECompatibleList(OS, Defs); + // Generate ARM range checking code for shift/lane immediates. genIntrinsicRangeCheckCode(OS, Defs); } diff --git a/contrib/llvm-project/clang/utils/TableGen/RISCVVEmitter.cpp b/contrib/llvm-project/clang/utils/TableGen/RISCVVEmitter.cpp index 1fb41805a047..da2a885ce851 100644 --- a/contrib/llvm-project/clang/utils/TableGen/RISCVVEmitter.cpp +++ b/contrib/llvm-project/clang/utils/TableGen/RISCVVEmitter.cpp @@ -46,7 +46,7 @@ struct SemaRecord { unsigned Log2LMULMask; // Required extensions for this intrinsic. - unsigned RequiredExtensions; + uint32_t RequiredExtensions; // Prototype for this intrinsic. SmallVector<PrototypeDescriptor> Prototype; @@ -653,24 +653,26 @@ void RVVEmitter::createRVVIntrinsics( SR.RequiredExtensions = 0; for (auto RequiredFeature : RequiredFeatures) { - RVVRequire RequireExt = StringSwitch<RVVRequire>(RequiredFeature) - .Case("RV64", RVV_REQ_RV64) - .Case("ZvfhminOrZvfh", RVV_REQ_ZvfhminOrZvfh) - .Case("Xsfvcp", RVV_REQ_Xsfvcp) - .Case("Xsfvfnrclipxfqf", RVV_REQ_Xsfvfnrclipxfqf) - .Case("Xsfvfwmaccqqq", RVV_REQ_Xsfvfwmaccqqq) - .Case("Xsfvqmaccdod", RVV_REQ_Xsfvqmaccdod) - .Case("Xsfvqmaccqoq", RVV_REQ_Xsfvqmaccqoq) - .Case("Zvbb", RVV_REQ_Zvbb) - .Case("Zvbc", RVV_REQ_Zvbc) - .Case("Zvkb", RVV_REQ_Zvkb) - .Case("Zvkg", RVV_REQ_Zvkg) - .Case("Zvkned", RVV_REQ_Zvkned) - .Case("Zvknha", RVV_REQ_Zvknha) - .Case("Zvknhb", RVV_REQ_Zvknhb) - .Case("Zvksed", RVV_REQ_Zvksed) - .Case("Zvksh", RVV_REQ_Zvksh) - .Default(RVV_REQ_None); + RVVRequire RequireExt = + StringSwitch<RVVRequire>(RequiredFeature) + .Case("RV64", RVV_REQ_RV64) + .Case("ZvfhminOrZvfh", RVV_REQ_ZvfhminOrZvfh) + .Case("Xsfvcp", RVV_REQ_Xsfvcp) + .Case("Xsfvfnrclipxfqf", RVV_REQ_Xsfvfnrclipxfqf) + .Case("Xsfvfwmaccqqq", RVV_REQ_Xsfvfwmaccqqq) + .Case("Xsfvqmaccdod", RVV_REQ_Xsfvqmaccdod) + .Case("Xsfvqmaccqoq", RVV_REQ_Xsfvqmaccqoq) + .Case("Zvbb", RVV_REQ_Zvbb) + .Case("Zvbc", RVV_REQ_Zvbc) + .Case("Zvkb", RVV_REQ_Zvkb) + .Case("Zvkg", RVV_REQ_Zvkg) + .Case("Zvkned", RVV_REQ_Zvkned) + .Case("Zvknha", RVV_REQ_Zvknha) + .Case("Zvknhb", RVV_REQ_Zvknhb) + .Case("Zvksed", RVV_REQ_Zvksed) + .Case("Zvksh", RVV_REQ_Zvksh) + .Case("Experimental", RVV_REQ_Experimental) + .Default(RVV_REQ_None); assert(RequireExt != RVV_REQ_None && "Unrecognized required feature?"); SR.RequiredExtensions |= RequireExt; } diff --git a/contrib/llvm-project/clang/utils/TableGen/SveEmitter.cpp b/contrib/llvm-project/clang/utils/TableGen/SveEmitter.cpp index 9361b9950637..311c6b09dc79 100644 --- a/contrib/llvm-project/clang/utils/TableGen/SveEmitter.cpp +++ b/contrib/llvm-project/clang/utils/TableGen/SveEmitter.cpp @@ -379,9 +379,15 @@ public: /// Emit all the information needed to map builtin -> LLVM IR intrinsic. void createSMECodeGenMap(raw_ostream &o); + /// Create a table for a builtin's requirement for PSTATE.SM. + void createStreamingAttrs(raw_ostream &o, ACLEKind Kind); + /// Emit all the range checks for the immediates. void createSMERangeChecks(raw_ostream &o); + /// Create a table for a builtin's requirement for PSTATE.ZA. + void createBuiltinZAState(raw_ostream &OS); + /// Create intrinsic and add it to \p Out void createIntrinsic(Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out); @@ -1702,6 +1708,76 @@ void SVEEmitter::createSMERangeChecks(raw_ostream &OS) { OS << "#endif\n\n"; } +void SVEEmitter::createBuiltinZAState(raw_ostream &OS) { + std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); + SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; + for (auto *R : RV) + createIntrinsic(R, Defs); + + std::map<bool, std::set<std::string>> DefsZAState; + + uint64_t IsSharedZAFlag = getEnumValueForFlag("IsSharedZA"); + for (auto &Def : Defs) { + bool HasZAState = Def->isFlagSet(IsSharedZAFlag); + DefsZAState[HasZAState].insert(Def->getMangledName()); + } + + OS << "#ifdef GET_SME_BUILTIN_HAS_ZA_STATE\n"; + + for (auto HasZA : {true, false}) { + auto Names = DefsZAState[HasZA]; + for (auto Name : Names) + OS << "case SME::BI__builtin_sme_" << Name << ":\n"; + OS << " return " << (HasZA ? "true" : "false") << ";\n"; + } + OS << "#endif\n\n"; +} + +void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) { + std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); + SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; + for (auto *R : RV) + createIntrinsic(R, Defs); + + StringRef ExtensionKind; + switch (Kind) { + case ACLEKind::SME: + ExtensionKind = "SME"; + break; + case ACLEKind::SVE: + ExtensionKind = "SVE"; + break; + } + + OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n"; + + llvm::StringMap<std::set<std::string>> StreamingMap; + + uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming"); + uint64_t IsStreamingCompatibleFlag = + getEnumValueForFlag("IsStreamingCompatible"); + for (auto &Def : Defs) { + if (Def->isFlagSet(IsStreamingFlag)) + StreamingMap["ArmStreaming"].insert(Def->getMangledName()); + else if (Def->isFlagSet(IsStreamingCompatibleFlag)) + StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName()); + else + StreamingMap["ArmNonStreaming"].insert(Def->getMangledName()); + } + + for (auto BuiltinType : StreamingMap.keys()) { + for (auto Name : StreamingMap[BuiltinType]) { + OS << "case " << ExtensionKind << "::BI__builtin_" + << ExtensionKind.lower() << "_"; + OS << Name << ":\n"; + } + OS << " BuiltinType = " << BuiltinType << ";\n"; + OS << " break;\n"; + } + + OS << "#endif\n\n"; +} + namespace clang { void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createHeader(OS); @@ -1723,6 +1799,10 @@ void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createTypeFlags(OS); } +void EmitSveStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) { + SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE); +} + void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createSMEHeader(OS); } @@ -1739,4 +1819,11 @@ void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createSMERangeChecks(OS); } +void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) { + SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME); +} + +void EmitSmeBuiltinZAState(RecordKeeper &Records, raw_ostream &OS) { + SVEEmitter(Records).createBuiltinZAState(OS); +} } // End namespace clang diff --git a/contrib/llvm-project/clang/utils/TableGen/TableGen.cpp b/contrib/llvm-project/clang/utils/TableGen/TableGen.cpp index 3ad46b95984e..c1f2ca15b595 100644 --- a/contrib/llvm-project/clang/utils/TableGen/TableGen.cpp +++ b/contrib/llvm-project/clang/utils/TableGen/TableGen.cpp @@ -86,10 +86,13 @@ enum ActionType { GenArmSveBuiltinCG, GenArmSveTypeFlags, GenArmSveRangeChecks, + GenArmSveStreamingAttrs, GenArmSmeHeader, GenArmSmeBuiltins, GenArmSmeBuiltinCG, GenArmSmeRangeChecks, + GenArmSmeStreamingAttrs, + GenArmSmeBuiltinZAState, GenArmCdeHeader, GenArmCdeBuiltinDef, GenArmCdeBuiltinSema, @@ -246,6 +249,8 @@ cl::opt<ActionType> Action( "Generate arm_sve_typeflags.inc for clang"), clEnumValN(GenArmSveRangeChecks, "gen-arm-sve-sema-rangechecks", "Generate arm_sve_sema_rangechecks.inc for clang"), + clEnumValN(GenArmSveStreamingAttrs, "gen-arm-sve-streaming-attrs", + "Generate arm_sve_streaming_attrs.inc for clang"), clEnumValN(GenArmSmeHeader, "gen-arm-sme-header", "Generate arm_sme.h for clang"), clEnumValN(GenArmSmeBuiltins, "gen-arm-sme-builtins", @@ -254,6 +259,10 @@ cl::opt<ActionType> Action( "Generate arm_sme_builtin_cg_map.inc for clang"), clEnumValN(GenArmSmeRangeChecks, "gen-arm-sme-sema-rangechecks", "Generate arm_sme_sema_rangechecks.inc for clang"), + clEnumValN(GenArmSmeStreamingAttrs, "gen-arm-sme-streaming-attrs", + "Generate arm_sme_streaming_attrs.inc for clang"), + clEnumValN(GenArmSmeBuiltinZAState, "gen-arm-sme-builtin-za-state", + "Generate arm_sme_builtins_za_state.inc for clang"), clEnumValN(GenArmMveHeader, "gen-arm-mve-header", "Generate arm_mve.h for clang"), clEnumValN(GenArmMveBuiltinDef, "gen-arm-mve-builtin-def", @@ -494,6 +503,9 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenArmSveRangeChecks: EmitSveRangeChecks(Records, OS); break; + case GenArmSveStreamingAttrs: + EmitSveStreamingAttrs(Records, OS); + break; case GenArmSmeHeader: EmitSmeHeader(Records, OS); break; @@ -506,6 +518,12 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenArmSmeRangeChecks: EmitSmeRangeChecks(Records, OS); break; + case GenArmSmeStreamingAttrs: + EmitSmeStreamingAttrs(Records, OS); + break; + case GenArmSmeBuiltinZAState: + EmitSmeBuiltinZAState(Records, OS); + break; case GenArmCdeHeader: EmitCdeHeader(Records, OS); break; diff --git a/contrib/llvm-project/clang/utils/TableGen/TableGenBackends.h b/contrib/llvm-project/clang/utils/TableGen/TableGenBackends.h index ef255612f4b8..35f2f04c1e81 100644 --- a/contrib/llvm-project/clang/utils/TableGen/TableGenBackends.h +++ b/contrib/llvm-project/clang/utils/TableGen/TableGenBackends.h @@ -105,11 +105,14 @@ void EmitSveBuiltins(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSveBuiltinCG(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSveTypeFlags(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSveRangeChecks(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); +void EmitSveStreamingAttrs(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeBuiltins(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeBuiltinCG(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeRangeChecks(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); +void EmitSmeStreamingAttrs(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); +void EmitSmeBuiltinZAState(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitMveHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitMveBuiltinDef(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); |