summaryrefslogtreecommitdiff
path: root/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp')
-rw-r--r--llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp730
1 files changed, 375 insertions, 355 deletions
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
index 18c7790a17cc1..2788b86181e27 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
@@ -105,62 +105,30 @@ unsigned HexagonResource::setWeight(unsigned s) {
return Weight;
}
-void HexagonCVIResource::SetupTUL(TypeUnitsAndLanes *TUL, StringRef CPU) {
- (*TUL)[HexagonII::TypeCVI_VA] =
- UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_VA_DV] = UnitsAndLanes(CVI_XLANE | CVI_MPY0, 2);
- (*TUL)[HexagonII::TypeCVI_VX] = UnitsAndLanes(CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_VX_LATE] = UnitsAndLanes(CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_VX_DV] = UnitsAndLanes(CVI_MPY0, 2);
- (*TUL)[HexagonII::TypeCVI_VP] = UnitsAndLanes(CVI_XLANE, 1);
- (*TUL)[HexagonII::TypeCVI_VP_VS] = UnitsAndLanes(CVI_XLANE, 2);
- (*TUL)[HexagonII::TypeCVI_VS] = UnitsAndLanes(CVI_SHIFT, 1);
- (*TUL)[HexagonII::TypeCVI_VS_VX] = UnitsAndLanes(CVI_XLANE | CVI_SHIFT, 1);
- (*TUL)[HexagonII::TypeCVI_VINLANESAT] =
- (CPU == "hexagonv60")
- ? UnitsAndLanes(CVI_SHIFT, 1)
- : UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_VM_LD] =
- UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_VM_TMP_LD] = UnitsAndLanes(CVI_NONE, 0);
- (*TUL)[HexagonII::TypeCVI_VM_VP_LDU] = UnitsAndLanes(CVI_XLANE, 1);
- (*TUL)[HexagonII::TypeCVI_VM_ST] =
- UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_VM_NEW_ST] = UnitsAndLanes(CVI_NONE, 0);
- (*TUL)[HexagonII::TypeCVI_VM_STU] = UnitsAndLanes(CVI_XLANE, 1);
- (*TUL)[HexagonII::TypeCVI_HIST] = UnitsAndLanes(CVI_XLANE, 4);
- (*TUL)[HexagonII::TypeCVI_GATHER] =
- UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_SCATTER] =
- UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_SCATTER_DV] =
- UnitsAndLanes(CVI_XLANE | CVI_MPY0, 2);
- (*TUL)[HexagonII::TypeCVI_SCATTER_NEW_ST] =
- UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
- (*TUL)[HexagonII::TypeCVI_4SLOT_MPY] = UnitsAndLanes(CVI_XLANE, 4);
- (*TUL)[HexagonII::TypeCVI_ZW] = UnitsAndLanes(CVI_ZW, 1);
-}
-
-HexagonCVIResource::HexagonCVIResource(TypeUnitsAndLanes *TUL,
- MCInstrInfo const &MCII, unsigned s,
+HexagonCVIResource::HexagonCVIResource(MCInstrInfo const &MCII,
+ MCSubtargetInfo const &STI,
+ unsigned s,
MCInst const *id)
: HexagonResource(s) {
- unsigned T = HexagonMCInstrInfo::getType(MCII, *id);
- if (TUL->count(T)) {
- // For an HVX insn.
- Valid = true;
- setUnits((*TUL)[T].first);
- setLanes((*TUL)[T].second);
- setLoad(HexagonMCInstrInfo::getDesc(MCII, *id).mayLoad());
- setStore(HexagonMCInstrInfo::getDesc(MCII, *id).mayStore());
- } else {
+ const unsigned ItinUnits = HexagonMCInstrInfo::getCVIResources(MCII, STI, *id);
+ unsigned Lanes;
+ const unsigned Units = HexagonConvertUnits(ItinUnits, &Lanes);
+
+ if (Units == 0 && Lanes == 0) {
// For core insns.
Valid = false;
setUnits(0);
setLanes(0);
setLoad(false);
setStore(false);
+ } else {
+ // For an HVX insn.
+ Valid = true;
+ setUnits(Units);
+ setLanes(Lanes);
+ setLoad(HexagonMCInstrInfo::getDesc(MCII, *id).mayLoad());
+ setStore(HexagonMCInstrInfo::getDesc(MCII, *id).mayStore());
}
}
@@ -201,124 +169,293 @@ HexagonShuffler::HexagonShuffler(MCContext &Context, bool ReportErrors,
MCSubtargetInfo const &STI)
: Context(Context), MCII(MCII), STI(STI), ReportErrors(ReportErrors) {
reset();
- HexagonCVIResource::SetupTUL(&TUL, STI.getCPU());
}
void HexagonShuffler::reset() {
Packet.clear();
BundleFlags = 0;
+ CheckFailure = false;
}
void HexagonShuffler::append(MCInst const &ID, MCInst const *Extender,
unsigned S) {
- HexagonInstr PI(&TUL, MCII, &ID, Extender, S);
+ HexagonInstr PI(MCII, STI, &ID, Extender, S);
Packet.push_back(PI);
}
-static struct {
- unsigned first;
- unsigned second;
-} jumpSlots[] = {{8, 4}, {8, 2}, {8, 1}, {4, 2}, {4, 1}, {2, 1}};
-#define MAX_JUMP_SLOTS (sizeof(jumpSlots) / sizeof(jumpSlots[0]))
-void HexagonShuffler::restrictSlot1AOK() {
- bool HasRestrictSlot1AOK = false;
- SMLoc RestrictLoc;
- for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
- MCInst const &Inst = ISJ->getDesc();
- if (HexagonMCInstrInfo::isRestrictSlot1AOK(MCII, Inst)) {
- HasRestrictSlot1AOK = true;
- RestrictLoc = Inst.getLoc();
- }
- }
- if (HasRestrictSlot1AOK)
- for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
- MCInst const &Inst = ISJ->getDesc();
- unsigned Type = HexagonMCInstrInfo::getType(MCII, Inst);
+static const unsigned Slot0Mask = 1 << 0;
+static const unsigned Slot1Mask = 1 << 1;
+static const unsigned Slot3Mask = 1 << 3;
+static const unsigned slotSingleLoad = Slot0Mask;
+static const unsigned slotSingleStore = Slot0Mask;
+
+void HexagonShuffler::restrictSlot1AOK(HexagonPacketSummary const &Summary) {
+ if (Summary.Slot1AOKLoc)
+ for (HexagonInstr &ISJ : insts()) {
+ MCInst const &Inst = ISJ.getDesc();
+ const unsigned Type = HexagonMCInstrInfo::getType(MCII, Inst);
if (Type != HexagonII::TypeALU32_2op &&
Type != HexagonII::TypeALU32_3op &&
Type != HexagonII::TypeALU32_ADDI) {
- unsigned Units = ISJ->Core.getUnits();
- if (Units & 2U) {
+ const unsigned Units = ISJ.Core.getUnits();
+
+ if (Units & Slot1Mask) {
AppliedRestrictions.push_back(std::make_pair(
Inst.getLoc(),
"Instruction was restricted from being in slot 1"));
- AppliedRestrictions.push_back(
- std::make_pair(RestrictLoc, "Instruction can only be combine "
- "with an ALU instruction in slot 1"));
- ISJ->Core.setUnits(Units & ~2U);
+ AppliedRestrictions.push_back(std::make_pair(
+ *Summary.Slot1AOKLoc, "Instruction can only be combined "
+ "with an ALU instruction in slot 1"));
+ ISJ.Core.setUnits(Units & ~Slot1Mask);
}
}
}
}
-void HexagonShuffler::restrictNoSlot1Store() {
- bool HasRestrictNoSlot1Store = false;
- SMLoc RestrictLoc;
- for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
- MCInst const &Inst = ISJ->getDesc();
- if (HexagonMCInstrInfo::isRestrictNoSlot1Store(MCII, Inst)) {
- HasRestrictNoSlot1Store = true;
- RestrictLoc = Inst.getLoc();
+void HexagonShuffler::restrictNoSlot1Store(
+ HexagonPacketSummary const &Summary) {
+ // If this packet contains an instruction that bars slot-1 stores,
+ // we should mask off slot 1 from all of the store instructions in
+ // this packet.
+
+ if (!Summary.NoSlot1StoreLoc)
+ return;
+
+ bool AppliedRestriction = false;
+
+ for (HexagonInstr &ISJ : insts()) {
+ MCInst const &Inst = ISJ.getDesc();
+ if (HexagonMCInstrInfo::getDesc(MCII, Inst).mayStore()) {
+ unsigned Units = ISJ.Core.getUnits();
+ if (Units & Slot1Mask) {
+ AppliedRestriction = true;
+ AppliedRestrictions.push_back(std::make_pair(
+ Inst.getLoc(), "Instruction was restricted from being in slot 1"));
+ ISJ.Core.setUnits(Units & ~Slot1Mask);
+ }
}
}
- if (HasRestrictNoSlot1Store) {
- bool AppliedRestriction = false;
- for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
- MCInst const &Inst = ISJ->getDesc();
- if (HexagonMCInstrInfo::getDesc(MCII, Inst).mayStore()) {
- unsigned Units = ISJ->Core.getUnits();
- if (Units & 2U) {
- AppliedRestriction = true;
- AppliedRestrictions.push_back(std::make_pair(
- Inst.getLoc(),
- "Instruction was restricted from being in slot 1"));
- ISJ->Core.setUnits(Units & ~2U);
+
+ if (AppliedRestriction)
+ AppliedRestrictions.push_back(
+ std::make_pair(*Summary.NoSlot1StoreLoc,
+ "Instruction does not allow a store in slot 1"));
+}
+
+bool HexagonShuffler::applySlotRestrictions(
+ HexagonPacketSummary const &Summary) {
+ // These restrictions can modify the slot masks in the instructions
+ // in the Packet member. They should run unconditionally and their
+ // order does not matter.
+ restrictSlot1AOK(Summary);
+ restrictNoSlot1Store(Summary);
+
+ permitNonSlot();
+
+ // These restrictions can modify the slot masks in the instructions
+ // in the Packet member, but they can also detect constraint failures
+ // which are fatal.
+ if (!CheckFailure)
+ restrictStoreLoadOrder(Summary);
+ if (!CheckFailure)
+ restrictBranchOrder(Summary);
+ if (!CheckFailure)
+ restrictPreferSlot3(Summary);
+ return !CheckFailure;
+}
+
+void HexagonShuffler::restrictBranchOrder(HexagonPacketSummary const &Summary) {
+ // preserve branch order
+ const bool HasMultipleBranches = Summary.branchInsts.size() > 1;
+ if (!HasMultipleBranches)
+ return;
+
+ if (Summary.branchInsts.size() > 2) {
+ reportError(Twine("too many branches in packet"));
+ return;
+ }
+
+ const static std::pair<unsigned, unsigned> jumpSlots[] = {
+ {8, 4}, {8, 2}, {8, 1}, {4, 2}, {4, 1}, {2, 1}};
+ // try all possible choices
+ for (std::pair<unsigned, unsigned> jumpSlot : jumpSlots) {
+ // validate first jump with this slot rule
+ if (!(jumpSlot.first & Summary.branchInsts[0]->Core.getUnits()))
+ continue;
+
+ // validate second jump with this slot rule
+ if (!(jumpSlot.second & Summary.branchInsts[1]->Core.getUnits()))
+ continue;
+
+ // both valid for this configuration, set new slot rules
+ const HexagonPacket PacketSave = Packet;
+ Summary.branchInsts[0]->Core.setUnits(jumpSlot.first);
+ Summary.branchInsts[1]->Core.setUnits(jumpSlot.second);
+
+ const bool HasShuffledPacket = tryAuction(Summary).hasValue();
+ if (HasShuffledPacket)
+ return;
+
+ // if yes, great, if not then restore original slot mask
+ // restore original values
+ Packet = PacketSave;
+ }
+
+ reportError("invalid instruction packet: out of slots");
+}
+
+
+void HexagonShuffler::permitNonSlot() {
+ for (HexagonInstr &ISJ : insts()) {
+ const bool RequiresSlot = HexagonMCInstrInfo::requiresSlot(STI, *ISJ.ID);
+ if (!RequiresSlot)
+ ISJ.Core.setAllUnits();
+ }
+}
+
+bool HexagonShuffler::ValidResourceUsage(HexagonPacketSummary const &Summary) {
+ Optional<HexagonPacket> ShuffledPacket = tryAuction(Summary);
+
+ if (!ShuffledPacket) {
+ reportError("invalid instruction packet: slot error");
+ return false;
+ } else {
+ Packet = *ShuffledPacket;
+ }
+
+ // Verify the CVI slot subscriptions.
+ std::stable_sort(begin(), end(), HexagonInstr::lessCVI);
+ // create vector of hvx instructions to check
+ HVXInstsT hvxInsts;
+ hvxInsts.clear();
+ for (const_iterator I = cbegin(); I != cend(); ++I) {
+ struct CVIUnits inst;
+ inst.Units = I->CVI.getUnits();
+ inst.Lanes = I->CVI.getLanes();
+ if (inst.Units == 0)
+ continue; // not an hvx inst or an hvx inst that doesn't uses any pipes
+ hvxInsts.push_back(inst);
+ }
+
+ // if there are any hvx instructions in this packet, check pipe usage
+ if (hvxInsts.size() > 0) {
+ unsigned startIdx, usedUnits;
+ startIdx = usedUnits = 0x0;
+ if (!checkHVXPipes(hvxInsts, startIdx, usedUnits)) {
+ // too many pipes used to be valid
+ reportError(Twine("invalid instruction packet: slot error"));
+ return false;
+ }
+ }
+ return true;
+}
+
+bool HexagonShuffler::restrictStoreLoadOrder(
+ HexagonPacketSummary const &Summary) {
+ // Modify packet accordingly.
+ // TODO: need to reserve slots #0 and #1 for duplex insns.
+ static const unsigned slotFirstLoadStore = Slot1Mask;
+ static const unsigned slotLastLoadStore = Slot0Mask;
+ unsigned slotLoadStore = slotFirstLoadStore;
+
+ for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
+ MCInst const &ID = ISJ->getDesc();
+
+ if (!ISJ->Core.getUnits())
+ // Error if insn may not be executed in any slot.
+ return false;
+
+ // A single load must use slot #0.
+ if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
+ if (Summary.loads == 1 && Summary.loads == Summary.memory &&
+ Summary.memops == 0)
+ // Pin the load to slot #0.
+ switch (ID.getOpcode()) {
+ case Hexagon::V6_vgathermw:
+ case Hexagon::V6_vgathermh:
+ case Hexagon::V6_vgathermhw:
+ case Hexagon::V6_vgathermwq:
+ case Hexagon::V6_vgathermhq:
+ case Hexagon::V6_vgathermhwq:
+ // Slot1 only loads
+ break;
+ default:
+ ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleLoad);
+ break;
+ }
+ else if (Summary.loads >= 1 && isMemReorderDisabled()) { // }:mem_noshuf
+ // Loads must keep the original order ONLY if
+ // isMemReorderDisabled() == true
+ if (slotLoadStore < slotLastLoadStore) {
+ // Error if no more slots available for loads.
+ reportError("invalid instruction packet: too many loads");
+ return false;
+ }
+ // Pin the load to the highest slot available to it.
+ ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
+ // Update the next highest slot available to loads.
+ slotLoadStore >>= 1;
+ }
+ }
+
+ // A single store must use slot #0.
+ if (HexagonMCInstrInfo::getDesc(MCII, ID).mayStore()) {
+ if (!Summary.store0) {
+ const bool PacketHasNoOnlySlot0 =
+ llvm::none_of(insts(), [&](HexagonInstr const &I) {
+ return I.Core.getUnits() == Slot0Mask &&
+ I.ID->getOpcode() != ID.getOpcode();
+ });
+ const bool SafeToMoveToSlot0 =
+ (Summary.loads == 0) ||
+ (!isMemReorderDisabled() && PacketHasNoOnlySlot0);
+
+ if (Summary.stores == 1 && SafeToMoveToSlot0)
+ // Pin the store to slot #0 only if isMemReorderDisabled() == false
+ ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleStore);
+ else if (Summary.stores >= 1) {
+ if (slotLoadStore < slotLastLoadStore) {
+ // Error if no more slots available for stores.
+ reportError("invalid instruction packet: too many stores");
+ return false;
+ }
+ // Pin the store to the highest slot available to it.
+ ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
+ // Update the next highest slot available to stores.
+ slotLoadStore >>= 1;
}
}
+ if (Summary.store1 && Summary.stores > 1) {
+ // Error if a single store with another store.
+ reportError("invalid instruction packet: too many stores");
+ return false;
+ }
}
- if (AppliedRestriction)
- AppliedRestrictions.push_back(std::make_pair(
- RestrictLoc, "Instruction does not allow a store in slot 1"));
}
-}
-void HexagonShuffler::applySlotRestrictions() {
- restrictSlot1AOK();
- restrictNoSlot1Store();
+ return true;
}
-/// Check that the packet is legal and enforce relative insn order.
-bool HexagonShuffler::check() {
- // Descriptive slot masks.
- const unsigned slotSingleLoad = 0x1, slotSingleStore = 0x1,
- slotThree = 0x8, // slotFirstJump = 0x8,
- slotFirstLoadStore = 0x2, slotLastLoadStore = 0x1;
- // Highest slots for branches and stores used to keep their original order.
- // unsigned slotJump = slotFirstJump;
- unsigned slotLoadStore = slotFirstLoadStore;
- // Number of memory operations, loads, solo loads, stores, solo stores, single
- // stores.
- unsigned memory = 0, loads = 0, load0 = 0, stores = 0, store0 = 0, store1 = 0;
- unsigned NonZCVIloads = 0, AllCVIloads = 0, CVIstores = 0;
- // Number of duplex insns
- unsigned duplex = 0;
- unsigned pSlot3Cnt = 0;
- unsigned memops = 0;
- iterator slot3ISJ = end();
- std::vector<iterator> foundBranches;
- unsigned reservedSlots = 0;
+HexagonShuffler::HexagonPacketSummary HexagonShuffler::GetPacketSummary() {
+ HexagonPacketSummary Summary = HexagonPacketSummary();
// Collect information from the insns in the packet.
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
MCInst const &ID = ISJ->getDesc();
+ if (HexagonMCInstrInfo::isRestrictSlot1AOK(MCII, ID))
+ Summary.Slot1AOKLoc = ID.getLoc();
+ if (HexagonMCInstrInfo::isRestrictNoSlot1Store(MCII, ID))
+ Summary.NoSlot1StoreLoc = ID.getLoc();
+
if (HexagonMCInstrInfo::prefersSlot3(MCII, ID)) {
- ++pSlot3Cnt;
- slot3ISJ = ISJ;
+ ++Summary.pSlot3Cnt;
+ Summary.PrefSlot3Inst = ISJ;
}
- reservedSlots |= HexagonMCInstrInfo::getOtherReservedSlots(MCII, STI, ID);
+ Summary.ReservedSlotMask |=
+ HexagonMCInstrInfo::getOtherReservedSlots(MCII, STI, ID);
switch (HexagonMCInstrInfo::getType(MCII, ID)) {
case HexagonII::TypeS_2op:
@@ -326,26 +463,27 @@ bool HexagonShuffler::check() {
case HexagonII::TypeALU64:
break;
case HexagonII::TypeJ:
- foundBranches.push_back(ISJ);
+ Summary.branchInsts.push_back(ISJ);
break;
case HexagonII::TypeCVI_VM_VP_LDU:
case HexagonII::TypeCVI_VM_LD:
case HexagonII::TypeCVI_VM_TMP_LD:
case HexagonII::TypeCVI_GATHER:
+ case HexagonII::TypeCVI_GATHER_DV:
case HexagonII::TypeCVI_GATHER_RST:
- ++NonZCVIloads;
+ ++Summary.NonZCVIloads;
LLVM_FALLTHROUGH;
case HexagonII::TypeCVI_ZW:
- ++AllCVIloads;
+ ++Summary.AllCVIloads;
LLVM_FALLTHROUGH;
case HexagonII::TypeLD:
- ++loads;
- ++memory;
+ ++Summary.loads;
+ ++Summary.memory;
if (ISJ->Core.getUnits() == slotSingleLoad ||
HexagonMCInstrInfo::getType(MCII, ID) == HexagonII::TypeCVI_VM_VP_LDU)
- ++load0;
+ ++Summary.load0;
if (HexagonMCInstrInfo::getDesc(MCII, ID).isReturn())
- foundBranches.push_back(ISJ);
+ Summary.branchInsts.push_back(ISJ);
break;
case HexagonII::TypeCVI_VM_STU:
case HexagonII::TypeCVI_VM_ST:
@@ -355,266 +493,143 @@ bool HexagonShuffler::check() {
case HexagonII::TypeCVI_SCATTER_RST:
case HexagonII::TypeCVI_SCATTER_NEW_RST:
case HexagonII::TypeCVI_SCATTER_NEW_ST:
- ++CVIstores;
+ ++Summary.CVIstores;
LLVM_FALLTHROUGH;
case HexagonII::TypeST:
- ++stores;
- ++memory;
+ ++Summary.stores;
+ ++Summary.memory;
if (ISJ->Core.getUnits() == slotSingleStore ||
HexagonMCInstrInfo::getType(MCII, ID) == HexagonII::TypeCVI_VM_STU)
- ++store0;
+ ++Summary.store0;
break;
case HexagonII::TypeV4LDST:
- ++loads;
- ++stores;
- ++store1;
- ++memops;
- ++memory;
+ ++Summary.loads;
+ ++Summary.stores;
+ ++Summary.store1;
+ ++Summary.memops;
+ ++Summary.memory;
break;
case HexagonII::TypeNCJ:
- ++memory; // NV insns are memory-like.
- foundBranches.push_back(ISJ);
+ ++Summary.memory; // NV insns are memory-like.
+ Summary.branchInsts.push_back(ISJ);
break;
case HexagonII::TypeV2LDST:
if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
- ++loads;
- ++memory;
+ ++Summary.loads;
+ ++Summary.memory;
if (ISJ->Core.getUnits() == slotSingleLoad ||
HexagonMCInstrInfo::getType(MCII, ID) ==
HexagonII::TypeCVI_VM_VP_LDU)
- ++load0;
+ ++Summary.load0;
} else {
assert(HexagonMCInstrInfo::getDesc(MCII, ID).mayStore());
- ++memory;
- ++stores;
+ ++Summary.memory;
+ ++Summary.stores;
}
break;
case HexagonII::TypeCR:
// Legacy conditional branch predicated on a register.
case HexagonII::TypeCJ:
if (HexagonMCInstrInfo::getDesc(MCII, ID).isBranch())
- foundBranches.push_back(ISJ);
+ Summary.branchInsts.push_back(ISJ);
break;
case HexagonII::TypeDUPLEX: {
- ++duplex;
+ ++Summary.duplex;
MCInst const &Inst0 = *ID.getOperand(0).getInst();
MCInst const &Inst1 = *ID.getOperand(1).getInst();
if (HexagonMCInstrInfo::getDesc(MCII, Inst0).isBranch())
- foundBranches.push_back(ISJ);
+ Summary.branchInsts.push_back(ISJ);
if (HexagonMCInstrInfo::getDesc(MCII, Inst1).isBranch())
- foundBranches.push_back(ISJ);
+ Summary.branchInsts.push_back(ISJ);
if (HexagonMCInstrInfo::getDesc(MCII, Inst0).isReturn())
- foundBranches.push_back(ISJ);
+ Summary.branchInsts.push_back(ISJ);
if (HexagonMCInstrInfo::getDesc(MCII, Inst1).isReturn())
- foundBranches.push_back(ISJ);
+ Summary.branchInsts.push_back(ISJ);
break;
}
}
}
- applySlotRestrictions();
+ return Summary;
+}
+bool HexagonShuffler::ValidPacketMemoryOps(
+ HexagonPacketSummary const &Summary) const {
// Check if the packet is legal.
- const unsigned ZCVIloads = AllCVIloads - NonZCVIloads;
+ const unsigned ZCVIloads = Summary.AllCVIloads - Summary.NonZCVIloads;
const bool ValidHVXMem =
- NonZCVIloads <= 1 && ZCVIloads <= 1 && CVIstores <= 1;
- if ((load0 > 1 || store0 > 1 || !ValidHVXMem) ||
- (duplex > 1 || (duplex && memory))) {
- reportError(llvm::Twine("invalid instruction packet"));
- return false;
- }
-
- // Modify packet accordingly.
- // TODO: need to reserve slots #0 and #1 for duplex insns.
- bool bOnlySlot3 = false;
- for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
- MCInst const &ID = ISJ->getDesc();
-
- if (!ISJ->Core.getUnits()) {
- // Error if insn may not be executed in any slot.
- return false;
- }
-
- // A single load must use slot #0.
- if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
- if (loads == 1 && loads == memory && memops == 0)
- // Pin the load to slot #0.
- switch (ID.getOpcode()) {
- case Hexagon::V6_vgathermw:
- case Hexagon::V6_vgathermh:
- case Hexagon::V6_vgathermhw:
- case Hexagon::V6_vgathermwq:
- case Hexagon::V6_vgathermhq:
- case Hexagon::V6_vgathermhwq:
- // Slot1 only loads
- break;
- default:
- ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleLoad);
- break;
- }
- else if (loads >= 1 && isMemReorderDisabled()) { // }:mem_noshuf
- // Loads must keep the original order ONLY if
- // isMemReorderDisabled() == true
- if (slotLoadStore < slotLastLoadStore) {
- // Error if no more slots available for loads.
- reportError(
- llvm::Twine("invalid instruction packet: too many loads"));
- return false;
- }
- // Pin the load to the highest slot available to it.
- ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
- // Update the next highest slot available to loads.
- slotLoadStore >>= 1;
- }
- }
+ Summary.NonZCVIloads <= 1 && ZCVIloads <= 1 && Summary.CVIstores <= 1;
+ const bool InvalidPacket =
+ ((Summary.load0 > 1 || Summary.store0 > 1 || !ValidHVXMem) ||
+ (Summary.duplex > 1 || (Summary.duplex && Summary.memory)));
- // A single store must use slot #0.
- if (HexagonMCInstrInfo::getDesc(MCII, ID).mayStore()) {
- if (!store0) {
- if (stores == 1 && (loads == 0 || !isMemReorderDisabled()))
- // Pin the store to slot #0 only if isMemReorderDisabled() == false
- ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleStore);
- else if (stores >= 1) {
- if (slotLoadStore < slotLastLoadStore) {
- // Error if no more slots available for stores.
- reportError(Twine("invalid instruction packet: too many stores"));
- return false;
- }
- // Pin the store to the highest slot available to it.
- ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
- // Update the next highest slot available to stores.
- slotLoadStore >>= 1;
- }
- }
- if (store1 && stores > 1) {
- // Error if a single store with another store.
- reportError(Twine("invalid instruction packet: too many stores"));
- return false;
- }
- }
-
- // flag if an instruction requires to be in slot 3
- if (ISJ->Core.getUnits() == slotThree)
- bOnlySlot3 = true;
-
- if (!ISJ->Core.getUnits()) {
- // Error if insn may not be executed in any slot.
- reportError(Twine("invalid instruction packet: out of slots"));
- return false;
- }
- }
-
- // preserve branch order
- bool validateSlots = true;
- if (foundBranches.size() > 1) {
- if (foundBranches.size() > 2) {
- reportError(Twine("too many branches in packet"));
- return false;
- }
-
- // try all possible choices
- for (unsigned int i = 0; i < MAX_JUMP_SLOTS; ++i) {
- // validate first jump with this slot rule
- if (!(jumpSlots[i].first & foundBranches[0]->Core.getUnits()))
- continue;
-
- // validate second jump with this slot rule
- if (!(jumpSlots[i].second & foundBranches[1]->Core.getUnits()))
- continue;
-
- // both valid for this configuration, set new slot rules
- PacketSave = Packet;
- foundBranches[0]->Core.setUnits(jumpSlots[i].first);
- foundBranches[1]->Core.setUnits(jumpSlots[i].second);
-
- HexagonUnitAuction AuctionCore(reservedSlots);
- std::stable_sort(begin(), end(), HexagonInstr::lessCore);
-
- // see if things ok with that instruction being pinned to slot "slotJump"
- bool bFail = false;
- for (iterator I = begin(); I != end() && !bFail; ++I)
- if (!AuctionCore.bid(I->Core.getUnits()))
- bFail = true;
-
- // if yes, great, if not then restore original slot mask
- if (!bFail) {
- validateSlots = false; // all good, no need to re-do auction
- break;
- } else
- // restore original values
- Packet = PacketSave;
- }
- if (validateSlots) {
- reportError(Twine("invalid instruction packet: out of slots"));
- return false;
- }
- }
-
- if (foundBranches.size() <= 1 && bOnlySlot3 == false && pSlot3Cnt == 1 &&
- slot3ISJ != end()) {
- validateSlots = true;
- // save off slot mask of instruction marked with A_PREFER_SLOT3
- // and then pin it to slot #3
- unsigned saveUnits = slot3ISJ->Core.getUnits();
- slot3ISJ->Core.setUnits(saveUnits & slotThree);
+ return !InvalidPacket;
+}
- HexagonUnitAuction AuctionCore(reservedSlots);
- std::stable_sort(begin(), end(), HexagonInstr::lessCore);
+void HexagonShuffler::restrictPreferSlot3(HexagonPacketSummary const &Summary) {
+ // flag if an instruction requires to be in slot 3
+ const bool HasOnlySlot3 = llvm::any_of(insts(), [&](HexagonInstr const &I) {
+ return (I.Core.getUnits() == Slot3Mask);
+ });
+ const bool NeedsPrefSlot3Shuffle =
+ (Summary.branchInsts.size() <= 1 && !HasOnlySlot3 &&
+ Summary.pSlot3Cnt == 1 && Summary.PrefSlot3Inst);
+
+ if (!NeedsPrefSlot3Shuffle)
+ return;
+
+ HexagonInstr *PrefSlot3Inst = *Summary.PrefSlot3Inst;
+ // save off slot mask of instruction marked with A_PREFER_SLOT3
+ // and then pin it to slot #3
+ const unsigned saveUnits = PrefSlot3Inst->Core.getUnits();
+ PrefSlot3Inst->Core.setUnits(saveUnits & Slot3Mask);
+ const bool HasShuffledPacket = tryAuction(Summary).hasValue();
+ if (HasShuffledPacket)
+ return;
+
+ PrefSlot3Inst->Core.setUnits(saveUnits);
+}
- // see if things ok with that instruction being pinned to slot #3
- bool bFail = false;
- for (iterator I = begin(); I != end() && !bFail; ++I)
- if (!AuctionCore.bid(I->Core.getUnits()))
- bFail = true;
+/// Check that the packet is legal and enforce relative insn order.
+bool HexagonShuffler::check() {
+ const HexagonPacketSummary Summary = GetPacketSummary();
+ if (!applySlotRestrictions(Summary))
+ return false;
- // if yes, great, if not then restore original slot mask
- if (!bFail)
- validateSlots = false; // all good, no need to re-do auction
- else
- for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
- MCInst const &ID = ISJ->getDesc();
- if (HexagonMCInstrInfo::prefersSlot3(MCII, ID))
- ISJ->Core.setUnits(saveUnits);
- }
+ if (!ValidPacketMemoryOps(Summary)) {
+ reportError("invalid instruction packet");
+ return false;
}
- // Check if any slot, core or CVI, is over-subscribed.
- // Verify the core slot subscriptions.
- if (validateSlots) {
- HexagonUnitAuction AuctionCore(reservedSlots);
+ ValidResourceUsage(Summary);
- std::stable_sort(begin(), end(), HexagonInstr::lessCore);
-
- for (iterator I = begin(); I != end(); ++I)
- if (!AuctionCore.bid(I->Core.getUnits())) {
- reportError(Twine("invalid instruction packet: slot error"));
- return false;
- }
- }
- // Verify the CVI slot subscriptions.
- std::stable_sort(begin(), end(), HexagonInstr::lessCVI);
- // create vector of hvx instructions to check
- HVXInstsT hvxInsts;
- hvxInsts.clear();
- for (iterator I = begin(); I != end(); ++I) {
- struct CVIUnits inst;
- inst.Units = I->CVI.getUnits();
- inst.Lanes = I->CVI.getLanes();
- if (inst.Units == 0)
- continue; // not an hvx inst or an hvx inst that doesn't uses any pipes
- hvxInsts.push_back(inst);
- }
- // if there are any hvx instructions in this packet, check pipe usage
- if (hvxInsts.size() > 0) {
- unsigned startIdx, usedUnits;
- startIdx = usedUnits = 0x0;
- if (!checkHVXPipes(hvxInsts, startIdx, usedUnits)) {
- // too many pipes used to be valid
- reportError(Twine("invalid instruction packet: slot error"));
- return false;
- }
- }
+ return !CheckFailure;
+}
- return true;
+llvm::Optional<HexagonShuffler::HexagonPacket>
+HexagonShuffler::tryAuction(HexagonPacketSummary const &Summary) const {
+ HexagonPacket PacketResult = Packet;
+ HexagonUnitAuction AuctionCore(Summary.ReservedSlotMask);
+ std::stable_sort(PacketResult.begin(), PacketResult.end(),
+ HexagonInstr::lessCore);
+
+ const bool ValidSlots =
+ llvm::all_of(insts(PacketResult), [&AuctionCore](HexagonInstr const &I) {
+ return AuctionCore.bid(I.Core.getUnits());
+ });
+
+ LLVM_DEBUG(
+ dbgs() << "Shuffle attempt: " << (ValidSlots ? "passed" : "failed")
+ << "\n";
+ for (HexagonInstr const &ISJ : insts(PacketResult))
+ dbgs() << "\t" << HexagonMCInstrInfo::getName(MCII, *ISJ.ID) << ": "
+ << llvm::format_hex(ISJ.Core.getUnits(), 4, true) << "\n";
+ );
+
+ Optional<HexagonPacket> Res;
+ if (ValidSlots)
+ Res = PacketResult;
+
+ return Res;
}
bool HexagonShuffler::shuffle() {
@@ -653,20 +668,25 @@ bool HexagonShuffler::shuffle() {
++emptySlots;
}
- for (iterator ISJ = begin(); ISJ != end(); ++ISJ)
- LLVM_DEBUG(dbgs().write_hex(ISJ->Core.getUnits()); if (ISJ->CVI.isValid()) {
- dbgs() << '/';
- dbgs().write_hex(ISJ->CVI.getUnits()) << '|';
- dbgs() << ISJ->CVI.getLanes();
- } dbgs() << ':'
- << HexagonMCInstrInfo::getDesc(MCII, ISJ->getDesc()).getOpcode();
- dbgs() << '\n');
- LLVM_DEBUG(dbgs() << '\n');
+ LLVM_DEBUG(
+ for (HexagonInstr const &ISJ : insts()) {
+ dbgs().write_hex(ISJ.Core.getUnits());
+ if (ISJ.CVI.isValid()) {
+ dbgs() << '/';
+ dbgs().write_hex(ISJ.CVI.getUnits()) << '|';
+ dbgs() << ISJ.CVI.getLanes();
+ }
+ dbgs() << ':'
+ << HexagonMCInstrInfo::getDesc(MCII, ISJ.getDesc()).getOpcode()
+ << '\n';
+ } dbgs() << '\n';
+ );
return Ok;
}
void HexagonShuffler::reportError(Twine const &Msg) {
+ CheckFailure = true;
if (ReportErrors) {
for (auto const &I : AppliedRestrictions) {
auto SM = Context.getSourceManager();
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-19 20:12:42 +0000