diff options
Diffstat (limited to 'lib/MCA/HardwareUnits')
-rw-r--r-- | lib/MCA/HardwareUnits/LSUnit.cpp | 28 | ||||
-rw-r--r-- | lib/MCA/HardwareUnits/RegisterFile.cpp | 16 | ||||
-rw-r--r-- | lib/MCA/HardwareUnits/ResourceManager.cpp | 59 | ||||
-rw-r--r-- | lib/MCA/HardwareUnits/RetireControlUnit.cpp | 65 | ||||
-rw-r--r-- | lib/MCA/HardwareUnits/Scheduler.cpp | 12 |
5 files changed, 102 insertions, 78 deletions
diff --git a/lib/MCA/HardwareUnits/LSUnit.cpp b/lib/MCA/HardwareUnits/LSUnit.cpp index ac1a6a36547b..0ee084c7ce1a 100644 --- a/lib/MCA/HardwareUnits/LSUnit.cpp +++ b/lib/MCA/HardwareUnits/LSUnit.cpp @@ -29,12 +29,12 @@ LSUnitBase::LSUnitBase(const MCSchedModel &SM, unsigned LQ, unsigned SQ, const MCExtraProcessorInfo &EPI = SM.getExtraProcessorInfo(); if (!LQSize && EPI.LoadQueueID) { const MCProcResourceDesc &LdQDesc = *SM.getProcResource(EPI.LoadQueueID); - LQSize = LdQDesc.BufferSize; + LQSize = std::max(0, LdQDesc.BufferSize); } if (!SQSize && EPI.StoreQueueID) { const MCProcResourceDesc &StQDesc = *SM.getProcResource(EPI.StoreQueueID); - SQSize = StQDesc.BufferSize; + SQSize = std::max(0, StQDesc.BufferSize); } } } @@ -72,9 +72,9 @@ unsigned LSUnit::dispatch(const InstRef &IR) { assert((Desc.MayLoad || Desc.MayStore) && "Not a memory operation!"); if (Desc.MayLoad) - assignLQSlot(); + acquireLQSlot(); if (Desc.MayStore) - assignSQSlot(); + acquireSQSlot(); if (Desc.MayStore) { // Always create a new group for store operations. @@ -160,26 +160,28 @@ LSUnit::Status LSUnit::isAvailable(const InstRef &IR) const { } void LSUnitBase::onInstructionExecuted(const InstRef &IR) { - const InstrDesc &Desc = IR.getInstruction()->getDesc(); - bool IsALoad = Desc.MayLoad; - bool IsAStore = Desc.MayStore; - assert((IsALoad || IsAStore) && "Expected a memory operation!"); - unsigned GroupID = IR.getInstruction()->getLSUTokenID(); auto It = Groups.find(GroupID); + assert(It != Groups.end() && "Instruction not dispatched to the LS unit"); It->second->onInstructionExecuted(); - if (It->second->isExecuted()) { + if (It->second->isExecuted()) Groups.erase(It); - } +} + +void LSUnitBase::onInstructionRetired(const InstRef &IR) { + const InstrDesc &Desc = IR.getInstruction()->getDesc(); + bool IsALoad = Desc.MayLoad; + bool IsAStore = Desc.MayStore; + assert((IsALoad || IsAStore) && "Expected a memory operation!"); if (IsALoad) { - UsedLQEntries--; + releaseLQSlot(); LLVM_DEBUG(dbgs() << "[LSUnit]: Instruction idx=" << IR.getSourceIndex() << " has been removed from the load queue.\n"); } if (IsAStore) { - UsedSQEntries--; + releaseSQSlot(); LLVM_DEBUG(dbgs() << "[LSUnit]: Instruction idx=" << IR.getSourceIndex() << " has been removed from the store queue.\n"); } diff --git a/lib/MCA/HardwareUnits/RegisterFile.cpp b/lib/MCA/HardwareUnits/RegisterFile.cpp index 86a888ea8cae..7ea5506f11d6 100644 --- a/lib/MCA/HardwareUnits/RegisterFile.cpp +++ b/lib/MCA/HardwareUnits/RegisterFile.cpp @@ -147,7 +147,7 @@ void RegisterFile::freePhysRegs(const RegisterRenamingInfo &Entry, void RegisterFile::addRegisterWrite(WriteRef Write, MutableArrayRef<unsigned> UsedPhysRegs) { WriteState &WS = *Write.getWriteState(); - unsigned RegID = WS.getRegisterID(); + MCPhysReg RegID = WS.getRegisterID(); assert(RegID && "Adding an invalid register definition?"); LLVM_DEBUG({ @@ -194,7 +194,7 @@ void RegisterFile::addRegisterWrite(WriteRef Write, } // Update zero registers. - unsigned ZeroRegisterID = + MCPhysReg ZeroRegisterID = WS.clearsSuperRegisters() ? RegID : WS.getRegisterID(); if (IsWriteZero) { ZeroRegisters.setBit(ZeroRegisterID); @@ -247,7 +247,7 @@ void RegisterFile::removeRegisterWrite( if (WS.isEliminated()) return; - unsigned RegID = WS.getRegisterID(); + MCPhysReg RegID = WS.getRegisterID(); assert(RegID != 0 && "Invalidating an already invalid register?"); assert(WS.getCyclesLeft() != UNKNOWN_CYCLES && @@ -255,7 +255,7 @@ void RegisterFile::removeRegisterWrite( assert(WS.getCyclesLeft() <= 0 && "Invalid cycles left for this write!"); bool ShouldFreePhysRegs = !WS.isWriteZero(); - unsigned RenameAs = RegisterMappings[RegID].second.RenameAs; + MCPhysReg RenameAs = RegisterMappings[RegID].second.RenameAs; if (RenameAs && RenameAs != RegID) { RegID = RenameAs; @@ -355,7 +355,7 @@ bool RegisterFile::tryEliminateMove(WriteState &WS, ReadState &RS) { void RegisterFile::collectWrites(const ReadState &RS, SmallVectorImpl<WriteRef> &Writes) const { - unsigned RegID = RS.getRegisterID(); + MCPhysReg RegID = RS.getRegisterID(); assert(RegID && RegID < RegisterMappings.size()); LLVM_DEBUG(dbgs() << "RegisterFile: collecting writes for register " << MRI.getName(RegID) << '\n'); @@ -397,7 +397,7 @@ void RegisterFile::collectWrites(const ReadState &RS, void RegisterFile::addRegisterRead(ReadState &RS, const MCSubtargetInfo &STI) const { - unsigned RegID = RS.getRegisterID(); + MCPhysReg RegID = RS.getRegisterID(); const RegisterRenamingInfo &RRI = RegisterMappings[RegID].second; RS.setPRF(RRI.IndexPlusCost.first); if (RS.isIndependentFromDef()) @@ -424,11 +424,11 @@ void RegisterFile::addRegisterRead(ReadState &RS, } } -unsigned RegisterFile::isAvailable(ArrayRef<unsigned> Regs) const { +unsigned RegisterFile::isAvailable(ArrayRef<MCPhysReg> Regs) const { SmallVector<unsigned, 4> NumPhysRegs(getNumRegisterFiles()); // Find how many new mappings must be created for each register file. - for (const unsigned RegID : Regs) { + for (const MCPhysReg RegID : Regs) { const RegisterRenamingInfo &RRI = RegisterMappings[RegID].second; const IndexPlusCostPairTy &Entry = RRI.IndexPlusCost; if (Entry.first) diff --git a/lib/MCA/HardwareUnits/ResourceManager.cpp b/lib/MCA/HardwareUnits/ResourceManager.cpp index 06f2476353d6..088aea3e23c6 100644 --- a/lib/MCA/HardwareUnits/ResourceManager.cpp +++ b/lib/MCA/HardwareUnits/ResourceManager.cpp @@ -104,7 +104,7 @@ void ResourceState::dump() const { static std::unique_ptr<ResourceStrategy> getStrategyFor(const ResourceState &RS) { if (RS.isAResourceGroup() || RS.getNumUnits() > 1) - return llvm::make_unique<DefaultResourceStrategy>(RS.getReadyMask()); + return std::make_unique<DefaultResourceStrategy>(RS.getReadyMask()); return std::unique_ptr<ResourceStrategy>(nullptr); } @@ -114,7 +114,8 @@ ResourceManager::ResourceManager(const MCSchedModel &SM) Resource2Groups(SM.getNumProcResourceKinds() - 1, 0), ProcResID2Mask(SM.getNumProcResourceKinds(), 0), ResIndex2ProcResID(SM.getNumProcResourceKinds() - 1, 0), - ProcResUnitMask(0), ReservedResourceGroups(0) { + ProcResUnitMask(0), ReservedResourceGroups(0), + AvailableBuffers(~0ULL), ReservedBuffers(0) { computeProcResourceMasks(SM, ProcResID2Mask); // initialize vector ResIndex2ProcResID. @@ -127,7 +128,7 @@ ResourceManager::ResourceManager(const MCSchedModel &SM) uint64_t Mask = ProcResID2Mask[I]; unsigned Index = getResourceStateIndex(Mask); Resources[Index] = - llvm::make_unique<ResourceState>(*SM.getProcResource(I), I, Mask); + std::make_unique<ResourceState>(*SM.getProcResource(I), I, Mask); Strategies[Index] = getStrategyFor(*Resources[Index]); } @@ -241,33 +242,41 @@ void ResourceManager::release(const ResourceRef &RR) { } ResourceStateEvent -ResourceManager::canBeDispatched(ArrayRef<uint64_t> Buffers) const { - ResourceStateEvent Result = ResourceStateEvent::RS_BUFFER_AVAILABLE; - for (uint64_t Buffer : Buffers) { - ResourceState &RS = *Resources[getResourceStateIndex(Buffer)]; - Result = RS.isBufferAvailable(); - if (Result != ResourceStateEvent::RS_BUFFER_AVAILABLE) - break; - } - return Result; +ResourceManager::canBeDispatched(uint64_t ConsumedBuffers) const { + if (ConsumedBuffers & ReservedBuffers) + return ResourceStateEvent::RS_RESERVED; + if (ConsumedBuffers & (~AvailableBuffers)) + return ResourceStateEvent::RS_BUFFER_UNAVAILABLE; + return ResourceStateEvent::RS_BUFFER_AVAILABLE; } -void ResourceManager::reserveBuffers(ArrayRef<uint64_t> Buffers) { - for (const uint64_t Buffer : Buffers) { - ResourceState &RS = *Resources[getResourceStateIndex(Buffer)]; +void ResourceManager::reserveBuffers(uint64_t ConsumedBuffers) { + while (ConsumedBuffers) { + uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers); + ResourceState &RS = *Resources[getResourceStateIndex(CurrentBuffer)]; + ConsumedBuffers ^= CurrentBuffer; assert(RS.isBufferAvailable() == ResourceStateEvent::RS_BUFFER_AVAILABLE); - RS.reserveBuffer(); - + if (!RS.reserveBuffer()) + AvailableBuffers ^= CurrentBuffer; if (RS.isADispatchHazard()) { - assert(!RS.isReserved()); - RS.setReserved(); + // Reserve this buffer now, and release it once pipeline resources + // consumed by the instruction become available again. + // We do this to simulate an in-order dispatch/issue of instructions. + ReservedBuffers ^= CurrentBuffer; } } } -void ResourceManager::releaseBuffers(ArrayRef<uint64_t> Buffers) { - for (const uint64_t R : Buffers) - Resources[getResourceStateIndex(R)]->releaseBuffer(); +void ResourceManager::releaseBuffers(uint64_t ConsumedBuffers) { + AvailableBuffers |= ConsumedBuffers; + while (ConsumedBuffers) { + uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers); + ResourceState &RS = *Resources[getResourceStateIndex(CurrentBuffer)]; + ConsumedBuffers ^= CurrentBuffer; + RS.releaseBuffer(); + // Do not unreserve dispatch hazard resource buffers. Wait until all + // pipeline resources have been freed too. + } } uint64_t ResourceManager::checkAvailability(const InstrDesc &Desc) const { @@ -322,7 +331,6 @@ void ResourceManager::cycleEvent(SmallVectorImpl<ResourceRef> &ResourcesFreed) { if (countPopulation(RR.first) == 1) release(RR); - releaseResource(RR.first); ResourcesFreed.push_back(RR); } @@ -336,7 +344,7 @@ void ResourceManager::reserveResource(uint64_t ResourceID) { const unsigned Index = getResourceStateIndex(ResourceID); ResourceState &Resource = *Resources[Index]; assert(Resource.isAResourceGroup() && !Resource.isReserved() && - "Unexpected resource found!"); + "Unexpected resource state found!"); Resource.setReserved(); ReservedResourceGroups ^= 1ULL << Index; } @@ -347,6 +355,9 @@ void ResourceManager::releaseResource(uint64_t ResourceID) { Resource.clearReserved(); if (Resource.isAResourceGroup()) ReservedResourceGroups ^= 1ULL << Index; + // Now it is safe to release dispatch/issue resources. + if (Resource.isADispatchHazard()) + ReservedBuffers ^= 1ULL << Index; } } // namespace mca diff --git a/lib/MCA/HardwareUnits/RetireControlUnit.cpp b/lib/MCA/HardwareUnits/RetireControlUnit.cpp index 068c5062ccdf..de519d7fd94a 100644 --- a/lib/MCA/HardwareUnits/RetireControlUnit.cpp +++ b/lib/MCA/HardwareUnits/RetireControlUnit.cpp @@ -21,65 +21,78 @@ namespace mca { RetireControlUnit::RetireControlUnit(const MCSchedModel &SM) : NextAvailableSlotIdx(0), CurrentInstructionSlotIdx(0), - AvailableSlots(SM.MicroOpBufferSize), MaxRetirePerCycle(0) { + NumROBEntries(SM.MicroOpBufferSize), + AvailableEntries(SM.MicroOpBufferSize), MaxRetirePerCycle(0) { // Check if the scheduling model provides extra information about the machine // processor. If so, then use that information to set the reorder buffer size // and the maximum number of instructions retired per cycle. if (SM.hasExtraProcessorInfo()) { const MCExtraProcessorInfo &EPI = SM.getExtraProcessorInfo(); if (EPI.ReorderBufferSize) - AvailableSlots = EPI.ReorderBufferSize; + AvailableEntries = EPI.ReorderBufferSize; MaxRetirePerCycle = EPI.MaxRetirePerCycle; } - - assert(AvailableSlots && "Invalid reorder buffer size!"); - Queue.resize(AvailableSlots); + NumROBEntries = AvailableEntries; + assert(NumROBEntries && "Invalid reorder buffer size!"); + Queue.resize(2 * NumROBEntries); } // Reserves a number of slots, and returns a new token. -unsigned RetireControlUnit::reserveSlot(const InstRef &IR, - unsigned NumMicroOps) { - assert(isAvailable(NumMicroOps) && "Reorder Buffer unavailable!"); - unsigned NormalizedQuantity = - std::min(NumMicroOps, static_cast<unsigned>(Queue.size())); - // Zero latency instructions may have zero uOps. Artificially bump this - // value to 1. Although zero latency instructions don't consume scheduler - // resources, they still consume one slot in the retire queue. - NormalizedQuantity = std::max(NormalizedQuantity, 1U); +unsigned RetireControlUnit::dispatch(const InstRef &IR) { + const Instruction &Inst = *IR.getInstruction(); + unsigned Entries = normalizeQuantity(Inst.getNumMicroOps()); + assert((AvailableEntries >= Entries) && "Reorder Buffer unavailable!"); + unsigned TokenID = NextAvailableSlotIdx; - Queue[NextAvailableSlotIdx] = {IR, NormalizedQuantity, false}; - NextAvailableSlotIdx += NormalizedQuantity; + Queue[NextAvailableSlotIdx] = {IR, Entries, false}; + NextAvailableSlotIdx += std::max(1U, Entries); NextAvailableSlotIdx %= Queue.size(); - AvailableSlots -= NormalizedQuantity; + + AvailableEntries -= Entries; return TokenID; } -const RetireControlUnit::RUToken &RetireControlUnit::peekCurrentToken() const { - return Queue[CurrentInstructionSlotIdx]; +const RetireControlUnit::RUToken &RetireControlUnit::getCurrentToken() const { + const RetireControlUnit::RUToken &Current = Queue[CurrentInstructionSlotIdx]; +#ifndef NDEBUG + const Instruction *Inst = Current.IR.getInstruction(); + assert(Inst && "Invalid RUToken in the RCU queue."); +#endif + return Current; +} + +unsigned RetireControlUnit::computeNextSlotIdx() const { + const RetireControlUnit::RUToken &Current = getCurrentToken(); + unsigned NextSlotIdx = CurrentInstructionSlotIdx + std::max(1U, Current.NumSlots); + return NextSlotIdx % Queue.size(); +} + +const RetireControlUnit::RUToken &RetireControlUnit::peekNextToken() const { + return Queue[computeNextSlotIdx()]; } void RetireControlUnit::consumeCurrentToken() { RetireControlUnit::RUToken &Current = Queue[CurrentInstructionSlotIdx]; - assert(Current.NumSlots && "Reserved zero slots?"); - assert(Current.IR && "Invalid RUToken in the RCU queue."); Current.IR.getInstruction()->retire(); // Update the slot index to be the next item in the circular queue. - CurrentInstructionSlotIdx += Current.NumSlots; + CurrentInstructionSlotIdx += std::max(1U, Current.NumSlots); CurrentInstructionSlotIdx %= Queue.size(); - AvailableSlots += Current.NumSlots; + AvailableEntries += Current.NumSlots; + Current = { InstRef(), 0U, false }; } void RetireControlUnit::onInstructionExecuted(unsigned TokenID) { assert(Queue.size() > TokenID); - assert(Queue[TokenID].Executed == false && Queue[TokenID].IR); + assert(Queue[TokenID].IR.getInstruction() && "Instruction was not dispatched!"); + assert(Queue[TokenID].Executed == false && "Instruction already executed!"); Queue[TokenID].Executed = true; } #ifndef NDEBUG void RetireControlUnit::dump() const { - dbgs() << "Retire Unit: { Total Slots=" << Queue.size() - << ", Available Slots=" << AvailableSlots << " }\n"; + dbgs() << "Retire Unit: { Total ROB Entries =" << NumROBEntries + << ", Available ROB entries=" << AvailableEntries << " }\n"; } #endif diff --git a/lib/MCA/HardwareUnits/Scheduler.cpp b/lib/MCA/HardwareUnits/Scheduler.cpp index 0f0f2ffb8325..8730336c6669 100644 --- a/lib/MCA/HardwareUnits/Scheduler.cpp +++ b/lib/MCA/HardwareUnits/Scheduler.cpp @@ -21,7 +21,7 @@ namespace mca { void Scheduler::initializeStrategy(std::unique_ptr<SchedulerStrategy> S) { // Ensure we have a valid (non-null) strategy object. - Strategy = S ? std::move(S) : llvm::make_unique<DefaultSchedulerStrategy>(); + Strategy = S ? std::move(S) : std::make_unique<DefaultSchedulerStrategy>(); } // Anchor the vtable of SchedulerStrategy and DefaultSchedulerStrategy. @@ -38,9 +38,8 @@ void Scheduler::dump() const { #endif Scheduler::Status Scheduler::isAvailable(const InstRef &IR) { - const InstrDesc &Desc = IR.getInstruction()->getDesc(); - - ResourceStateEvent RSE = Resources->canBeDispatched(Desc.Buffers); + ResourceStateEvent RSE = + Resources->canBeDispatched(IR.getInstruction()->getUsedBuffers()); HadTokenStall = RSE != RS_BUFFER_AVAILABLE; switch (RSE) { @@ -106,7 +105,7 @@ void Scheduler::issueInstruction( bool HasDependentUsers = Inst.hasDependentUsers(); HasDependentUsers |= Inst.isMemOp() && LSU.hasDependentUsers(IR); - Resources->releaseBuffers(Inst.getDesc().Buffers); + Resources->releaseBuffers(Inst.getUsedBuffers()); issueInstructionImpl(IR, UsedResources); // Instructions that have been issued during this cycle might have unblocked // other dependent instructions. Dependent instructions may be issued during @@ -300,8 +299,7 @@ bool Scheduler::mustIssueImmediately(const InstRef &IR) const { bool Scheduler::dispatch(InstRef &IR) { Instruction &IS = *IR.getInstruction(); - const InstrDesc &Desc = IS.getDesc(); - Resources->reserveBuffers(Desc.Buffers); + Resources->reserveBuffers(IS.getUsedBuffers()); // If necessary, reserve queue entries in the load-store unit (LSU). if (IS.isMemOp()) |