summaryrefslogtreecommitdiff
path: root/lib/MCA/HardwareUnits
diff options
context:
space:
mode:
Diffstat (limited to 'lib/MCA/HardwareUnits')
-rw-r--r--lib/MCA/HardwareUnits/LSUnit.cpp28
-rw-r--r--lib/MCA/HardwareUnits/RegisterFile.cpp16
-rw-r--r--lib/MCA/HardwareUnits/ResourceManager.cpp59
-rw-r--r--lib/MCA/HardwareUnits/RetireControlUnit.cpp65
-rw-r--r--lib/MCA/HardwareUnits/Scheduler.cpp12
5 files changed, 102 insertions, 78 deletions
diff --git a/lib/MCA/HardwareUnits/LSUnit.cpp b/lib/MCA/HardwareUnits/LSUnit.cpp
index ac1a6a36547b..0ee084c7ce1a 100644
--- a/lib/MCA/HardwareUnits/LSUnit.cpp
+++ b/lib/MCA/HardwareUnits/LSUnit.cpp
@@ -29,12 +29,12 @@ LSUnitBase::LSUnitBase(const MCSchedModel &SM, unsigned LQ, unsigned SQ,
const MCExtraProcessorInfo &EPI = SM.getExtraProcessorInfo();
if (!LQSize && EPI.LoadQueueID) {
const MCProcResourceDesc &LdQDesc = *SM.getProcResource(EPI.LoadQueueID);
- LQSize = LdQDesc.BufferSize;
+ LQSize = std::max(0, LdQDesc.BufferSize);
}
if (!SQSize && EPI.StoreQueueID) {
const MCProcResourceDesc &StQDesc = *SM.getProcResource(EPI.StoreQueueID);
- SQSize = StQDesc.BufferSize;
+ SQSize = std::max(0, StQDesc.BufferSize);
}
}
}
@@ -72,9 +72,9 @@ unsigned LSUnit::dispatch(const InstRef &IR) {
assert((Desc.MayLoad || Desc.MayStore) && "Not a memory operation!");
if (Desc.MayLoad)
- assignLQSlot();
+ acquireLQSlot();
if (Desc.MayStore)
- assignSQSlot();
+ acquireSQSlot();
if (Desc.MayStore) {
// Always create a new group for store operations.
@@ -160,26 +160,28 @@ LSUnit::Status LSUnit::isAvailable(const InstRef &IR) const {
}
void LSUnitBase::onInstructionExecuted(const InstRef &IR) {
- const InstrDesc &Desc = IR.getInstruction()->getDesc();
- bool IsALoad = Desc.MayLoad;
- bool IsAStore = Desc.MayStore;
- assert((IsALoad || IsAStore) && "Expected a memory operation!");
-
unsigned GroupID = IR.getInstruction()->getLSUTokenID();
auto It = Groups.find(GroupID);
+ assert(It != Groups.end() && "Instruction not dispatched to the LS unit");
It->second->onInstructionExecuted();
- if (It->second->isExecuted()) {
+ if (It->second->isExecuted())
Groups.erase(It);
- }
+}
+
+void LSUnitBase::onInstructionRetired(const InstRef &IR) {
+ const InstrDesc &Desc = IR.getInstruction()->getDesc();
+ bool IsALoad = Desc.MayLoad;
+ bool IsAStore = Desc.MayStore;
+ assert((IsALoad || IsAStore) && "Expected a memory operation!");
if (IsALoad) {
- UsedLQEntries--;
+ releaseLQSlot();
LLVM_DEBUG(dbgs() << "[LSUnit]: Instruction idx=" << IR.getSourceIndex()
<< " has been removed from the load queue.\n");
}
if (IsAStore) {
- UsedSQEntries--;
+ releaseSQSlot();
LLVM_DEBUG(dbgs() << "[LSUnit]: Instruction idx=" << IR.getSourceIndex()
<< " has been removed from the store queue.\n");
}
diff --git a/lib/MCA/HardwareUnits/RegisterFile.cpp b/lib/MCA/HardwareUnits/RegisterFile.cpp
index 86a888ea8cae..7ea5506f11d6 100644
--- a/lib/MCA/HardwareUnits/RegisterFile.cpp
+++ b/lib/MCA/HardwareUnits/RegisterFile.cpp
@@ -147,7 +147,7 @@ void RegisterFile::freePhysRegs(const RegisterRenamingInfo &Entry,
void RegisterFile::addRegisterWrite(WriteRef Write,
MutableArrayRef<unsigned> UsedPhysRegs) {
WriteState &WS = *Write.getWriteState();
- unsigned RegID = WS.getRegisterID();
+ MCPhysReg RegID = WS.getRegisterID();
assert(RegID && "Adding an invalid register definition?");
LLVM_DEBUG({
@@ -194,7 +194,7 @@ void RegisterFile::addRegisterWrite(WriteRef Write,
}
// Update zero registers.
- unsigned ZeroRegisterID =
+ MCPhysReg ZeroRegisterID =
WS.clearsSuperRegisters() ? RegID : WS.getRegisterID();
if (IsWriteZero) {
ZeroRegisters.setBit(ZeroRegisterID);
@@ -247,7 +247,7 @@ void RegisterFile::removeRegisterWrite(
if (WS.isEliminated())
return;
- unsigned RegID = WS.getRegisterID();
+ MCPhysReg RegID = WS.getRegisterID();
assert(RegID != 0 && "Invalidating an already invalid register?");
assert(WS.getCyclesLeft() != UNKNOWN_CYCLES &&
@@ -255,7 +255,7 @@ void RegisterFile::removeRegisterWrite(
assert(WS.getCyclesLeft() <= 0 && "Invalid cycles left for this write!");
bool ShouldFreePhysRegs = !WS.isWriteZero();
- unsigned RenameAs = RegisterMappings[RegID].second.RenameAs;
+ MCPhysReg RenameAs = RegisterMappings[RegID].second.RenameAs;
if (RenameAs && RenameAs != RegID) {
RegID = RenameAs;
@@ -355,7 +355,7 @@ bool RegisterFile::tryEliminateMove(WriteState &WS, ReadState &RS) {
void RegisterFile::collectWrites(const ReadState &RS,
SmallVectorImpl<WriteRef> &Writes) const {
- unsigned RegID = RS.getRegisterID();
+ MCPhysReg RegID = RS.getRegisterID();
assert(RegID && RegID < RegisterMappings.size());
LLVM_DEBUG(dbgs() << "RegisterFile: collecting writes for register "
<< MRI.getName(RegID) << '\n');
@@ -397,7 +397,7 @@ void RegisterFile::collectWrites(const ReadState &RS,
void RegisterFile::addRegisterRead(ReadState &RS,
const MCSubtargetInfo &STI) const {
- unsigned RegID = RS.getRegisterID();
+ MCPhysReg RegID = RS.getRegisterID();
const RegisterRenamingInfo &RRI = RegisterMappings[RegID].second;
RS.setPRF(RRI.IndexPlusCost.first);
if (RS.isIndependentFromDef())
@@ -424,11 +424,11 @@ void RegisterFile::addRegisterRead(ReadState &RS,
}
}
-unsigned RegisterFile::isAvailable(ArrayRef<unsigned> Regs) const {
+unsigned RegisterFile::isAvailable(ArrayRef<MCPhysReg> Regs) const {
SmallVector<unsigned, 4> NumPhysRegs(getNumRegisterFiles());
// Find how many new mappings must be created for each register file.
- for (const unsigned RegID : Regs) {
+ for (const MCPhysReg RegID : Regs) {
const RegisterRenamingInfo &RRI = RegisterMappings[RegID].second;
const IndexPlusCostPairTy &Entry = RRI.IndexPlusCost;
if (Entry.first)
diff --git a/lib/MCA/HardwareUnits/ResourceManager.cpp b/lib/MCA/HardwareUnits/ResourceManager.cpp
index 06f2476353d6..088aea3e23c6 100644
--- a/lib/MCA/HardwareUnits/ResourceManager.cpp
+++ b/lib/MCA/HardwareUnits/ResourceManager.cpp
@@ -104,7 +104,7 @@ void ResourceState::dump() const {
static std::unique_ptr<ResourceStrategy>
getStrategyFor(const ResourceState &RS) {
if (RS.isAResourceGroup() || RS.getNumUnits() > 1)
- return llvm::make_unique<DefaultResourceStrategy>(RS.getReadyMask());
+ return std::make_unique<DefaultResourceStrategy>(RS.getReadyMask());
return std::unique_ptr<ResourceStrategy>(nullptr);
}
@@ -114,7 +114,8 @@ ResourceManager::ResourceManager(const MCSchedModel &SM)
Resource2Groups(SM.getNumProcResourceKinds() - 1, 0),
ProcResID2Mask(SM.getNumProcResourceKinds(), 0),
ResIndex2ProcResID(SM.getNumProcResourceKinds() - 1, 0),
- ProcResUnitMask(0), ReservedResourceGroups(0) {
+ ProcResUnitMask(0), ReservedResourceGroups(0),
+ AvailableBuffers(~0ULL), ReservedBuffers(0) {
computeProcResourceMasks(SM, ProcResID2Mask);
// initialize vector ResIndex2ProcResID.
@@ -127,7 +128,7 @@ ResourceManager::ResourceManager(const MCSchedModel &SM)
uint64_t Mask = ProcResID2Mask[I];
unsigned Index = getResourceStateIndex(Mask);
Resources[Index] =
- llvm::make_unique<ResourceState>(*SM.getProcResource(I), I, Mask);
+ std::make_unique<ResourceState>(*SM.getProcResource(I), I, Mask);
Strategies[Index] = getStrategyFor(*Resources[Index]);
}
@@ -241,33 +242,41 @@ void ResourceManager::release(const ResourceRef &RR) {
}
ResourceStateEvent
-ResourceManager::canBeDispatched(ArrayRef<uint64_t> Buffers) const {
- ResourceStateEvent Result = ResourceStateEvent::RS_BUFFER_AVAILABLE;
- for (uint64_t Buffer : Buffers) {
- ResourceState &RS = *Resources[getResourceStateIndex(Buffer)];
- Result = RS.isBufferAvailable();
- if (Result != ResourceStateEvent::RS_BUFFER_AVAILABLE)
- break;
- }
- return Result;
+ResourceManager::canBeDispatched(uint64_t ConsumedBuffers) const {
+ if (ConsumedBuffers & ReservedBuffers)
+ return ResourceStateEvent::RS_RESERVED;
+ if (ConsumedBuffers & (~AvailableBuffers))
+ return ResourceStateEvent::RS_BUFFER_UNAVAILABLE;
+ return ResourceStateEvent::RS_BUFFER_AVAILABLE;
}
-void ResourceManager::reserveBuffers(ArrayRef<uint64_t> Buffers) {
- for (const uint64_t Buffer : Buffers) {
- ResourceState &RS = *Resources[getResourceStateIndex(Buffer)];
+void ResourceManager::reserveBuffers(uint64_t ConsumedBuffers) {
+ while (ConsumedBuffers) {
+ uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);
+ ResourceState &RS = *Resources[getResourceStateIndex(CurrentBuffer)];
+ ConsumedBuffers ^= CurrentBuffer;
assert(RS.isBufferAvailable() == ResourceStateEvent::RS_BUFFER_AVAILABLE);
- RS.reserveBuffer();
-
+ if (!RS.reserveBuffer())
+ AvailableBuffers ^= CurrentBuffer;
if (RS.isADispatchHazard()) {
- assert(!RS.isReserved());
- RS.setReserved();
+ // Reserve this buffer now, and release it once pipeline resources
+ // consumed by the instruction become available again.
+ // We do this to simulate an in-order dispatch/issue of instructions.
+ ReservedBuffers ^= CurrentBuffer;
}
}
}
-void ResourceManager::releaseBuffers(ArrayRef<uint64_t> Buffers) {
- for (const uint64_t R : Buffers)
- Resources[getResourceStateIndex(R)]->releaseBuffer();
+void ResourceManager::releaseBuffers(uint64_t ConsumedBuffers) {
+ AvailableBuffers |= ConsumedBuffers;
+ while (ConsumedBuffers) {
+ uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);
+ ResourceState &RS = *Resources[getResourceStateIndex(CurrentBuffer)];
+ ConsumedBuffers ^= CurrentBuffer;
+ RS.releaseBuffer();
+ // Do not unreserve dispatch hazard resource buffers. Wait until all
+ // pipeline resources have been freed too.
+ }
}
uint64_t ResourceManager::checkAvailability(const InstrDesc &Desc) const {
@@ -322,7 +331,6 @@ void ResourceManager::cycleEvent(SmallVectorImpl<ResourceRef> &ResourcesFreed) {
if (countPopulation(RR.first) == 1)
release(RR);
-
releaseResource(RR.first);
ResourcesFreed.push_back(RR);
}
@@ -336,7 +344,7 @@ void ResourceManager::reserveResource(uint64_t ResourceID) {
const unsigned Index = getResourceStateIndex(ResourceID);
ResourceState &Resource = *Resources[Index];
assert(Resource.isAResourceGroup() && !Resource.isReserved() &&
- "Unexpected resource found!");
+ "Unexpected resource state found!");
Resource.setReserved();
ReservedResourceGroups ^= 1ULL << Index;
}
@@ -347,6 +355,9 @@ void ResourceManager::releaseResource(uint64_t ResourceID) {
Resource.clearReserved();
if (Resource.isAResourceGroup())
ReservedResourceGroups ^= 1ULL << Index;
+ // Now it is safe to release dispatch/issue resources.
+ if (Resource.isADispatchHazard())
+ ReservedBuffers ^= 1ULL << Index;
}
} // namespace mca
diff --git a/lib/MCA/HardwareUnits/RetireControlUnit.cpp b/lib/MCA/HardwareUnits/RetireControlUnit.cpp
index 068c5062ccdf..de519d7fd94a 100644
--- a/lib/MCA/HardwareUnits/RetireControlUnit.cpp
+++ b/lib/MCA/HardwareUnits/RetireControlUnit.cpp
@@ -21,65 +21,78 @@ namespace mca {
RetireControlUnit::RetireControlUnit(const MCSchedModel &SM)
: NextAvailableSlotIdx(0), CurrentInstructionSlotIdx(0),
- AvailableSlots(SM.MicroOpBufferSize), MaxRetirePerCycle(0) {
+ NumROBEntries(SM.MicroOpBufferSize),
+ AvailableEntries(SM.MicroOpBufferSize), MaxRetirePerCycle(0) {
// Check if the scheduling model provides extra information about the machine
// processor. If so, then use that information to set the reorder buffer size
// and the maximum number of instructions retired per cycle.
if (SM.hasExtraProcessorInfo()) {
const MCExtraProcessorInfo &EPI = SM.getExtraProcessorInfo();
if (EPI.ReorderBufferSize)
- AvailableSlots = EPI.ReorderBufferSize;
+ AvailableEntries = EPI.ReorderBufferSize;
MaxRetirePerCycle = EPI.MaxRetirePerCycle;
}
-
- assert(AvailableSlots && "Invalid reorder buffer size!");
- Queue.resize(AvailableSlots);
+ NumROBEntries = AvailableEntries;
+ assert(NumROBEntries && "Invalid reorder buffer size!");
+ Queue.resize(2 * NumROBEntries);
}
// Reserves a number of slots, and returns a new token.
-unsigned RetireControlUnit::reserveSlot(const InstRef &IR,
- unsigned NumMicroOps) {
- assert(isAvailable(NumMicroOps) && "Reorder Buffer unavailable!");
- unsigned NormalizedQuantity =
- std::min(NumMicroOps, static_cast<unsigned>(Queue.size()));
- // Zero latency instructions may have zero uOps. Artificially bump this
- // value to 1. Although zero latency instructions don't consume scheduler
- // resources, they still consume one slot in the retire queue.
- NormalizedQuantity = std::max(NormalizedQuantity, 1U);
+unsigned RetireControlUnit::dispatch(const InstRef &IR) {
+ const Instruction &Inst = *IR.getInstruction();
+ unsigned Entries = normalizeQuantity(Inst.getNumMicroOps());
+ assert((AvailableEntries >= Entries) && "Reorder Buffer unavailable!");
+
unsigned TokenID = NextAvailableSlotIdx;
- Queue[NextAvailableSlotIdx] = {IR, NormalizedQuantity, false};
- NextAvailableSlotIdx += NormalizedQuantity;
+ Queue[NextAvailableSlotIdx] = {IR, Entries, false};
+ NextAvailableSlotIdx += std::max(1U, Entries);
NextAvailableSlotIdx %= Queue.size();
- AvailableSlots -= NormalizedQuantity;
+
+ AvailableEntries -= Entries;
return TokenID;
}
-const RetireControlUnit::RUToken &RetireControlUnit::peekCurrentToken() const {
- return Queue[CurrentInstructionSlotIdx];
+const RetireControlUnit::RUToken &RetireControlUnit::getCurrentToken() const {
+ const RetireControlUnit::RUToken &Current = Queue[CurrentInstructionSlotIdx];
+#ifndef NDEBUG
+ const Instruction *Inst = Current.IR.getInstruction();
+ assert(Inst && "Invalid RUToken in the RCU queue.");
+#endif
+ return Current;
+}
+
+unsigned RetireControlUnit::computeNextSlotIdx() const {
+ const RetireControlUnit::RUToken &Current = getCurrentToken();
+ unsigned NextSlotIdx = CurrentInstructionSlotIdx + std::max(1U, Current.NumSlots);
+ return NextSlotIdx % Queue.size();
+}
+
+const RetireControlUnit::RUToken &RetireControlUnit::peekNextToken() const {
+ return Queue[computeNextSlotIdx()];
}
void RetireControlUnit::consumeCurrentToken() {
RetireControlUnit::RUToken &Current = Queue[CurrentInstructionSlotIdx];
- assert(Current.NumSlots && "Reserved zero slots?");
- assert(Current.IR && "Invalid RUToken in the RCU queue.");
Current.IR.getInstruction()->retire();
// Update the slot index to be the next item in the circular queue.
- CurrentInstructionSlotIdx += Current.NumSlots;
+ CurrentInstructionSlotIdx += std::max(1U, Current.NumSlots);
CurrentInstructionSlotIdx %= Queue.size();
- AvailableSlots += Current.NumSlots;
+ AvailableEntries += Current.NumSlots;
+ Current = { InstRef(), 0U, false };
}
void RetireControlUnit::onInstructionExecuted(unsigned TokenID) {
assert(Queue.size() > TokenID);
- assert(Queue[TokenID].Executed == false && Queue[TokenID].IR);
+ assert(Queue[TokenID].IR.getInstruction() && "Instruction was not dispatched!");
+ assert(Queue[TokenID].Executed == false && "Instruction already executed!");
Queue[TokenID].Executed = true;
}
#ifndef NDEBUG
void RetireControlUnit::dump() const {
- dbgs() << "Retire Unit: { Total Slots=" << Queue.size()
- << ", Available Slots=" << AvailableSlots << " }\n";
+ dbgs() << "Retire Unit: { Total ROB Entries =" << NumROBEntries
+ << ", Available ROB entries=" << AvailableEntries << " }\n";
}
#endif
diff --git a/lib/MCA/HardwareUnits/Scheduler.cpp b/lib/MCA/HardwareUnits/Scheduler.cpp
index 0f0f2ffb8325..8730336c6669 100644
--- a/lib/MCA/HardwareUnits/Scheduler.cpp
+++ b/lib/MCA/HardwareUnits/Scheduler.cpp
@@ -21,7 +21,7 @@ namespace mca {
void Scheduler::initializeStrategy(std::unique_ptr<SchedulerStrategy> S) {
// Ensure we have a valid (non-null) strategy object.
- Strategy = S ? std::move(S) : llvm::make_unique<DefaultSchedulerStrategy>();
+ Strategy = S ? std::move(S) : std::make_unique<DefaultSchedulerStrategy>();
}
// Anchor the vtable of SchedulerStrategy and DefaultSchedulerStrategy.
@@ -38,9 +38,8 @@ void Scheduler::dump() const {
#endif
Scheduler::Status Scheduler::isAvailable(const InstRef &IR) {
- const InstrDesc &Desc = IR.getInstruction()->getDesc();
-
- ResourceStateEvent RSE = Resources->canBeDispatched(Desc.Buffers);
+ ResourceStateEvent RSE =
+ Resources->canBeDispatched(IR.getInstruction()->getUsedBuffers());
HadTokenStall = RSE != RS_BUFFER_AVAILABLE;
switch (RSE) {
@@ -106,7 +105,7 @@ void Scheduler::issueInstruction(
bool HasDependentUsers = Inst.hasDependentUsers();
HasDependentUsers |= Inst.isMemOp() && LSU.hasDependentUsers(IR);
- Resources->releaseBuffers(Inst.getDesc().Buffers);
+ Resources->releaseBuffers(Inst.getUsedBuffers());
issueInstructionImpl(IR, UsedResources);
// Instructions that have been issued during this cycle might have unblocked
// other dependent instructions. Dependent instructions may be issued during
@@ -300,8 +299,7 @@ bool Scheduler::mustIssueImmediately(const InstRef &IR) const {
bool Scheduler::dispatch(InstRef &IR) {
Instruction &IS = *IR.getInstruction();
- const InstrDesc &Desc = IS.getDesc();
- Resources->reserveBuffers(Desc.Buffers);
+ Resources->reserveBuffers(IS.getUsedBuffers());
// If necessary, reserve queue entries in the load-store unit (LSU).
if (IS.isMemOp())
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-31 06:27:44 +0000