summaryrefslogtreecommitdiff
path: root/llvm/lib/MCA
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/MCA')
-rw-r--r--llvm/lib/MCA/CodeEmitter.cpp2
-rw-r--r--llvm/lib/MCA/HardwareUnits/LSUnit.cpp84
-rw-r--r--llvm/lib/MCA/InstrBuilder.cpp25
3 files changed, 74 insertions, 37 deletions
diff --git a/llvm/lib/MCA/CodeEmitter.cpp b/llvm/lib/MCA/CodeEmitter.cpp
index 294107219cb00..dcb92d253bae8 100644
--- a/llvm/lib/MCA/CodeEmitter.cpp
+++ b/llvm/lib/MCA/CodeEmitter.cpp
@@ -25,7 +25,7 @@ CodeEmitter::getOrCreateEncodingInfo(unsigned MCID) {
const MCInst &Inst = Sequence[MCID];
MCInst Relaxed(Sequence[MCID]);
if (MAB.mayNeedRelaxation(Inst, STI))
- MAB.relaxInstruction(Inst, STI, Relaxed);
+ MAB.relaxInstruction(Relaxed, STI);
EI.first = Code.size();
MCE.encodeInstruction(Relaxed, VecOS, Fixups, STI);
diff --git a/llvm/lib/MCA/HardwareUnits/LSUnit.cpp b/llvm/lib/MCA/HardwareUnits/LSUnit.cpp
index 0ee084c7ce1a9..e945e8cecce93 100644
--- a/llvm/lib/MCA/HardwareUnits/LSUnit.cpp
+++ b/llvm/lib/MCA/HardwareUnits/LSUnit.cpp
@@ -77,9 +77,6 @@ unsigned LSUnit::dispatch(const InstRef &IR) {
acquireSQSlot();
if (Desc.MayStore) {
- // Always create a new group for store operations.
-
- // A store may not pass a previous store or store barrier.
unsigned NewGID = createMemoryGroup();
MemoryGroup &NewGroup = getGroup(NewGID);
NewGroup.addInstruction();
@@ -91,16 +88,32 @@ unsigned LSUnit::dispatch(const InstRef &IR) {
MemoryGroup &IDom = getGroup(ImmediateLoadDominator);
LLVM_DEBUG(dbgs() << "[LSUnit]: GROUP DEP: (" << ImmediateLoadDominator
<< ") --> (" << NewGID << ")\n");
- IDom.addSuccessor(&NewGroup);
+ IDom.addSuccessor(&NewGroup, !assumeNoAlias());
+ }
+
+ // A store may not pass a previous store barrier.
+ if (CurrentStoreBarrierGroupID) {
+ MemoryGroup &StoreGroup = getGroup(CurrentStoreBarrierGroupID);
+ LLVM_DEBUG(dbgs() << "[LSUnit]: GROUP DEP: ("
+ << CurrentStoreBarrierGroupID
+ << ") --> (" << NewGID << ")\n");
+ StoreGroup.addSuccessor(&NewGroup, true);
}
- if (CurrentStoreGroupID) {
+
+ // A store may not pass a previous store.
+ if (CurrentStoreGroupID &&
+ (CurrentStoreGroupID != CurrentStoreBarrierGroupID)) {
MemoryGroup &StoreGroup = getGroup(CurrentStoreGroupID);
LLVM_DEBUG(dbgs() << "[LSUnit]: GROUP DEP: (" << CurrentStoreGroupID
<< ") --> (" << NewGID << ")\n");
- StoreGroup.addSuccessor(&NewGroup);
+ StoreGroup.addSuccessor(&NewGroup, !assumeNoAlias());
}
+
CurrentStoreGroupID = NewGID;
+ if (IsMemBarrier)
+ CurrentStoreBarrierGroupID = NewGID;
+
if (Desc.MayLoad) {
CurrentLoadGroupID = NewGID;
if (IsMemBarrier)
@@ -112,31 +125,59 @@ unsigned LSUnit::dispatch(const InstRef &IR) {
assert(Desc.MayLoad && "Expected a load!");
- // Always create a new memory group if this is the first load of the sequence.
+ unsigned ImmediateLoadDominator =
+ std::max(CurrentLoadGroupID, CurrentLoadBarrierGroupID);
+
+ // A new load group is created if we are in one of the following situations:
+ // 1) This is a load barrier (by construction, a load barrier is always
+ // assigned to a different memory group).
+ // 2) There is no load in flight (by construction we always keep loads and
+ // stores into separate memory groups).
+ // 3) There is a load barrier in flight. This load depends on it.
+ // 4) There is an intervening store between the last load dispatched to the
+ // LSU and this load. We always create a new group even if this load
+ // does not alias the last dispatched store.
+ // 5) There is no intervening store and there is an active load group.
+ // However that group has already started execution, so we cannot add
+ // this load to it.
+ bool ShouldCreateANewGroup =
+ IsMemBarrier || !ImmediateLoadDominator ||
+ CurrentLoadBarrierGroupID == ImmediateLoadDominator ||
+ ImmediateLoadDominator <= CurrentStoreGroupID ||
+ getGroup(ImmediateLoadDominator).isExecuting();
- // A load may not pass a previous store unless flag 'NoAlias' is set.
- // A load may pass a previous load.
- // A younger load cannot pass a older load barrier.
- // A load barrier cannot pass a older load.
- bool ShouldCreateANewGroup = !CurrentLoadGroupID || IsMemBarrier ||
- CurrentLoadGroupID <= CurrentStoreGroupID ||
- CurrentLoadGroupID <= CurrentLoadBarrierGroupID;
if (ShouldCreateANewGroup) {
unsigned NewGID = createMemoryGroup();
MemoryGroup &NewGroup = getGroup(NewGID);
NewGroup.addInstruction();
+ // A load may not pass a previous store or store barrier
+ // unless flag 'NoAlias' is set.
if (!assumeNoAlias() && CurrentStoreGroupID) {
- MemoryGroup &StGroup = getGroup(CurrentStoreGroupID);
+ MemoryGroup &StoreGroup = getGroup(CurrentStoreGroupID);
LLVM_DEBUG(dbgs() << "[LSUnit]: GROUP DEP: (" << CurrentStoreGroupID
<< ") --> (" << NewGID << ")\n");
- StGroup.addSuccessor(&NewGroup);
+ StoreGroup.addSuccessor(&NewGroup, true);
}
- if (CurrentLoadBarrierGroupID) {
- MemoryGroup &LdGroup = getGroup(CurrentLoadBarrierGroupID);
- LLVM_DEBUG(dbgs() << "[LSUnit]: GROUP DEP: (" << CurrentLoadBarrierGroupID
- << ") --> (" << NewGID << ")\n");
- LdGroup.addSuccessor(&NewGroup);
+
+ // A load barrier may not pass a previous load or load barrier.
+ if (IsMemBarrier) {
+ if (ImmediateLoadDominator) {
+ MemoryGroup &LoadGroup = getGroup(ImmediateLoadDominator);
+ LLVM_DEBUG(dbgs() << "[LSUnit]: GROUP DEP: ("
+ << ImmediateLoadDominator
+ << ") --> (" << NewGID << ")\n");
+ LoadGroup.addSuccessor(&NewGroup, true);
+ }
+ } else {
+ // A younger load cannot pass a older load barrier.
+ if (CurrentLoadBarrierGroupID) {
+ MemoryGroup &LoadGroup = getGroup(CurrentLoadBarrierGroupID);
+ LLVM_DEBUG(dbgs() << "[LSUnit]: GROUP DEP: ("
+ << CurrentLoadBarrierGroupID
+ << ") --> (" << NewGID << ")\n");
+ LoadGroup.addSuccessor(&NewGroup, true);
+ }
}
CurrentLoadGroupID = NewGID;
@@ -145,6 +186,7 @@ unsigned LSUnit::dispatch(const InstRef &IR) {
return NewGID;
}
+ // A load may pass a previous load.
MemoryGroup &Group = getGroup(CurrentLoadGroupID);
Group.addInstruction();
return CurrentLoadGroupID;
diff --git a/llvm/lib/MCA/InstrBuilder.cpp b/llvm/lib/MCA/InstrBuilder.cpp
index c137f1da8a44b..24e2a9d2f0ced 100644
--- a/llvm/lib/MCA/InstrBuilder.cpp
+++ b/llvm/lib/MCA/InstrBuilder.cpp
@@ -160,8 +160,11 @@ static void initializeUsedResources(InstrDesc &ID,
if (countPopulation(RPC.first) > 1 && !RPC.second.isReserved()) {
// Remove the leading 1 from the resource group mask.
uint64_t Mask = RPC.first ^ PowerOf2Floor(RPC.first);
- if ((Mask & UsedResourceUnits) == Mask)
+ uint64_t MaxResourceUnits = countPopulation(Mask);
+ if (RPC.second.NumUnits > countPopulation(Mask)) {
RPC.second.setReserved();
+ RPC.second.NumUnits = MaxResourceUnits;
+ }
}
}
@@ -485,24 +488,16 @@ Error InstrBuilder::verifyInstrDesc(const InstrDesc &ID,
if (ID.NumMicroOps != 0)
return ErrorSuccess();
- bool UsesMemory = ID.MayLoad || ID.MayStore;
bool UsesBuffers = ID.UsedBuffers;
bool UsesResources = !ID.Resources.empty();
- if (!UsesMemory && !UsesBuffers && !UsesResources)
+ if (!UsesBuffers && !UsesResources)
return ErrorSuccess();
- StringRef Message;
- if (UsesMemory) {
- Message = "found an inconsistent instruction that decodes "
- "into zero opcodes and that consumes load/store "
- "unit resources.";
- } else {
- Message = "found an inconsistent instruction that decodes "
- "to zero opcodes and that consumes scheduler "
- "resources.";
- }
-
- return make_error<InstructionError<MCInst>>(Message, MCI);
+ // FIXME: see PR44797. We should revisit these checks and possibly move them
+ // in CodeGenSchedule.cpp.
+ StringRef Message = "found an inconsistent instruction that decodes to zero "
+ "opcodes and that consumes scheduler resources.";
+ return make_error<InstructionError<MCInst>>(std::string(Message), MCI);
}
Expected<const InstrDesc &>
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-06 02:55:55 +0000