diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues')
5 files changed, 8254 insertions, 0 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp new file mode 100644 index 000000000000..ba417322d4f6 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp @@ -0,0 +1,4228 @@ +//===- InstrRefBasedImpl.cpp - Tracking Debug Value MIs -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file InstrRefBasedImpl.cpp +/// +/// This is a separate implementation of LiveDebugValues, see +/// LiveDebugValues.cpp and VarLocBasedImpl.cpp for more information. +/// +/// This pass propagates variable locations between basic blocks, resolving +/// control flow conflicts between them. The problem is SSA construction, where +/// each debug instruction assigns the *value* that a variable has, and every +/// instruction where the variable is in scope uses that variable. The resulting +/// map of instruction-to-value is then translated into a register (or spill) +/// location for each variable over each instruction. +/// +/// The primary difference from normal SSA construction is that we cannot +/// _create_ PHI values that contain variable values. CodeGen has already +/// completed, and we can't alter it just to make debug-info complete. Thus: +/// we can identify function positions where we would like a PHI value for a +/// variable, but must search the MachineFunction to see whether such a PHI is +/// available. If no such PHI exists, the variable location must be dropped. +/// +/// To achieve this, we perform two kinds of analysis. First, we identify +/// every value defined by every instruction (ignoring those that only move +/// another value), then re-compute an SSA-form representation of the +/// MachineFunction, using value propagation to eliminate any un-necessary +/// PHI values. This gives us a map of every value computed in the function, +/// and its location within the register file / stack. +/// +/// Secondly, for each variable we perform the same analysis, where each debug +/// instruction is considered a def, and every instruction where the variable +/// is in lexical scope as a use. Value propagation is used again to eliminate +/// any un-necessary PHIs. This gives us a map of each variable to the value +/// it should have in a block. +/// +/// Once both are complete, we have two maps for each block: +/// * Variables to the values they should have, +/// * Values to the register / spill slot they are located in. +/// After which we can marry-up variable values with a location, and emit +/// DBG_VALUE instructions specifying those locations. Variable locations may +/// be dropped in this process due to the desired variable value not being +/// resident in any machine location, or because there is no PHI value in any +/// location that accurately represents the desired value. The building of +/// location lists for each block is left to DbgEntityHistoryCalculator. +/// +/// This pass is kept efficient because the size of the first SSA problem +/// is proportional to the working-set size of the function, which the compiler +/// tries to keep small. (It's also proportional to the number of blocks). +/// Additionally, we repeatedly perform the second SSA problem analysis with +/// only the variables and blocks in a single lexical scope, exploiting their +/// locality. +/// +/// ### Terminology +/// +/// A machine location is a register or spill slot, a value is something that's +/// defined by an instruction or PHI node, while a variable value is the value +/// assigned to a variable. A variable location is a machine location, that must +/// contain the appropriate variable value. A value that is a PHI node is +/// occasionally called an mphi. +/// +/// The first SSA problem is the "machine value location" problem, +/// because we're determining which machine locations contain which values. +/// The "locations" are constant: what's unknown is what value they contain. +/// +/// The second SSA problem (the one for variables) is the "variable value +/// problem", because it's determining what values a variable has, rather than +/// what location those values are placed in. +/// +/// TODO: +/// Overlapping fragments +/// Entry values +/// Add back DEBUG statements for debugging this +/// Collect statistics +/// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/LexicalScopes.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineInstrBundle.h" +#include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/CodeGen/TargetFrameLowering.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Function.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/GenericIteratedDominanceFrontier.h" +#include "llvm/Support/TypeSize.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/Utils/SSAUpdaterImpl.h" +#include <algorithm> +#include <cassert> +#include <climits> +#include <cstdint> +#include <functional> +#include <queue> +#include <tuple> +#include <utility> +#include <vector> + +#include "InstrRefBasedImpl.h" +#include "LiveDebugValues.h" +#include <optional> + +using namespace llvm; +using namespace LiveDebugValues; + +// SSAUpdaterImple sets DEBUG_TYPE, change it. +#undef DEBUG_TYPE +#define DEBUG_TYPE "livedebugvalues" + +// Act more like the VarLoc implementation, by propagating some locations too +// far and ignoring some transfers. +static cl::opt<bool> EmulateOldLDV("emulate-old-livedebugvalues", cl::Hidden, + cl::desc("Act like old LiveDebugValues did"), + cl::init(false)); + +// Limit for the maximum number of stack slots we should track, past which we +// will ignore any spills. InstrRefBasedLDV gathers detailed information on all +// stack slots which leads to high memory consumption, and in some scenarios +// (such as asan with very many locals) the working set of the function can be +// very large, causing many spills. In these scenarios, it is very unlikely that +// the developer has hundreds of variables live at the same time that they're +// carefully thinking about -- instead, they probably autogenerated the code. +// When this happens, gracefully stop tracking excess spill slots, rather than +// consuming all the developer's memory. +static cl::opt<unsigned> + StackWorkingSetLimit("livedebugvalues-max-stack-slots", cl::Hidden, + cl::desc("livedebugvalues-stack-ws-limit"), + cl::init(250)); + +DbgOpID DbgOpID::UndefID = DbgOpID(0xffffffff); + +/// Tracker for converting machine value locations and variable values into +/// variable locations (the output of LiveDebugValues), recorded as DBG_VALUEs +/// specifying block live-in locations and transfers within blocks. +/// +/// Operating on a per-block basis, this class takes a (pre-loaded) MLocTracker +/// and must be initialized with the set of variable values that are live-in to +/// the block. The caller then repeatedly calls process(). TransferTracker picks +/// out variable locations for the live-in variable values (if there _is_ a +/// location) and creates the corresponding DBG_VALUEs. Then, as the block is +/// stepped through, transfers of values between machine locations are +/// identified and if profitable, a DBG_VALUE created. +/// +/// This is where debug use-before-defs would be resolved: a variable with an +/// unavailable value could materialize in the middle of a block, when the +/// value becomes available. Or, we could detect clobbers and re-specify the +/// variable in a backup location. (XXX these are unimplemented). +class TransferTracker { +public: + const TargetInstrInfo *TII; + const TargetLowering *TLI; + /// This machine location tracker is assumed to always contain the up-to-date + /// value mapping for all machine locations. TransferTracker only reads + /// information from it. (XXX make it const?) + MLocTracker *MTracker; + MachineFunction &MF; + bool ShouldEmitDebugEntryValues; + + /// Record of all changes in variable locations at a block position. Awkwardly + /// we allow inserting either before or after the point: MBB != nullptr + /// indicates it's before, otherwise after. + struct Transfer { + MachineBasicBlock::instr_iterator Pos; /// Position to insert DBG_VALUes + MachineBasicBlock *MBB; /// non-null if we should insert after. + SmallVector<MachineInstr *, 4> Insts; /// Vector of DBG_VALUEs to insert. + }; + + /// Stores the resolved operands (machine locations and constants) and + /// qualifying meta-information needed to construct a concrete DBG_VALUE-like + /// instruction. + struct ResolvedDbgValue { + SmallVector<ResolvedDbgOp> Ops; + DbgValueProperties Properties; + + ResolvedDbgValue(SmallVectorImpl<ResolvedDbgOp> &Ops, + DbgValueProperties Properties) + : Ops(Ops.begin(), Ops.end()), Properties(Properties) {} + + /// Returns all the LocIdx values used in this struct, in the order in which + /// they appear as operands in the debug value; may contain duplicates. + auto loc_indices() const { + return map_range( + make_filter_range( + Ops, [](const ResolvedDbgOp &Op) { return !Op.IsConst; }), + [](const ResolvedDbgOp &Op) { return Op.Loc; }); + } + }; + + /// Collection of transfers (DBG_VALUEs) to be inserted. + SmallVector<Transfer, 32> Transfers; + + /// Local cache of what-value-is-in-what-LocIdx. Used to identify differences + /// between TransferTrackers view of variable locations and MLocTrackers. For + /// example, MLocTracker observes all clobbers, but TransferTracker lazily + /// does not. + SmallVector<ValueIDNum, 32> VarLocs; + + /// Map from LocIdxes to which DebugVariables are based that location. + /// Mantained while stepping through the block. Not accurate if + /// VarLocs[Idx] != MTracker->LocIdxToIDNum[Idx]. + DenseMap<LocIdx, SmallSet<DebugVariable, 4>> ActiveMLocs; + + /// Map from DebugVariable to it's current location and qualifying meta + /// information. To be used in conjunction with ActiveMLocs to construct + /// enough information for the DBG_VALUEs for a particular LocIdx. + DenseMap<DebugVariable, ResolvedDbgValue> ActiveVLocs; + + /// Temporary cache of DBG_VALUEs to be entered into the Transfers collection. + SmallVector<MachineInstr *, 4> PendingDbgValues; + + /// Record of a use-before-def: created when a value that's live-in to the + /// current block isn't available in any machine location, but it will be + /// defined in this block. + struct UseBeforeDef { + /// Value of this variable, def'd in block. + SmallVector<DbgOp> Values; + /// Identity of this variable. + DebugVariable Var; + /// Additional variable properties. + DbgValueProperties Properties; + UseBeforeDef(ArrayRef<DbgOp> Values, const DebugVariable &Var, + const DbgValueProperties &Properties) + : Values(Values.begin(), Values.end()), Var(Var), + Properties(Properties) {} + }; + + /// Map from instruction index (within the block) to the set of UseBeforeDefs + /// that become defined at that instruction. + DenseMap<unsigned, SmallVector<UseBeforeDef, 1>> UseBeforeDefs; + + /// The set of variables that are in UseBeforeDefs and can become a location + /// once the relevant value is defined. An element being erased from this + /// collection prevents the use-before-def materializing. + DenseSet<DebugVariable> UseBeforeDefVariables; + + const TargetRegisterInfo &TRI; + const BitVector &CalleeSavedRegs; + + TransferTracker(const TargetInstrInfo *TII, MLocTracker *MTracker, + MachineFunction &MF, const TargetRegisterInfo &TRI, + const BitVector &CalleeSavedRegs, const TargetPassConfig &TPC) + : TII(TII), MTracker(MTracker), MF(MF), TRI(TRI), + CalleeSavedRegs(CalleeSavedRegs) { + TLI = MF.getSubtarget().getTargetLowering(); + auto &TM = TPC.getTM<TargetMachine>(); + ShouldEmitDebugEntryValues = TM.Options.ShouldEmitDebugEntryValues(); + } + + bool isCalleeSaved(LocIdx L) const { + unsigned Reg = MTracker->LocIdxToLocID[L]; + if (Reg >= MTracker->NumRegs) + return false; + for (MCRegAliasIterator RAI(Reg, &TRI, true); RAI.isValid(); ++RAI) + if (CalleeSavedRegs.test(*RAI)) + return true; + return false; + }; + + // An estimate of the expected lifespan of values at a machine location, with + // a greater value corresponding to a longer expected lifespan, i.e. spill + // slots generally live longer than callee-saved registers which generally + // live longer than non-callee-saved registers. The minimum value of 0 + // corresponds to an illegal location that cannot have a "lifespan" at all. + enum class LocationQuality : unsigned char { + Illegal = 0, + Register, + CalleeSavedRegister, + SpillSlot, + Best = SpillSlot + }; + + class LocationAndQuality { + unsigned Location : 24; + unsigned Quality : 8; + + public: + LocationAndQuality() : Location(0), Quality(0) {} + LocationAndQuality(LocIdx L, LocationQuality Q) + : Location(L.asU64()), Quality(static_cast<unsigned>(Q)) {} + LocIdx getLoc() const { + if (!Quality) + return LocIdx::MakeIllegalLoc(); + return LocIdx(Location); + } + LocationQuality getQuality() const { return LocationQuality(Quality); } + bool isIllegal() const { return !Quality; } + bool isBest() const { return getQuality() == LocationQuality::Best; } + }; + + // Returns the LocationQuality for the location L iff the quality of L is + // is strictly greater than the provided minimum quality. + std::optional<LocationQuality> + getLocQualityIfBetter(LocIdx L, LocationQuality Min) const { + if (L.isIllegal()) + return std::nullopt; + if (Min >= LocationQuality::SpillSlot) + return std::nullopt; + if (MTracker->isSpill(L)) + return LocationQuality::SpillSlot; + if (Min >= LocationQuality::CalleeSavedRegister) + return std::nullopt; + if (isCalleeSaved(L)) + return LocationQuality::CalleeSavedRegister; + if (Min >= LocationQuality::Register) + return std::nullopt; + return LocationQuality::Register; + } + + /// For a variable \p Var with the live-in value \p Value, attempts to resolve + /// the DbgValue to a concrete DBG_VALUE, emitting that value and loading the + /// tracking information to track Var throughout the block. + /// \p ValueToLoc is a map containing the best known location for every + /// ValueIDNum that Value may use. + /// \p MBB is the basic block that we are loading the live-in value for. + /// \p DbgOpStore is the map containing the DbgOpID->DbgOp mapping needed to + /// determine the values used by Value. + void loadVarInloc(MachineBasicBlock &MBB, DbgOpIDMap &DbgOpStore, + const DenseMap<ValueIDNum, LocationAndQuality> &ValueToLoc, + DebugVariable Var, DbgValue Value) { + SmallVector<DbgOp> DbgOps; + SmallVector<ResolvedDbgOp> ResolvedDbgOps; + bool IsValueValid = true; + unsigned LastUseBeforeDef = 0; + + // If every value used by the incoming DbgValue is available at block + // entry, ResolvedDbgOps will contain the machine locations/constants for + // those values and will be used to emit a debug location. + // If one or more values are not yet available, but will all be defined in + // this block, then LastUseBeforeDef will track the instruction index in + // this BB at which the last of those values is defined, DbgOps will + // contain the values that we will emit when we reach that instruction. + // If one or more values are undef or not available throughout this block, + // and we can't recover as an entry value, we set IsValueValid=false and + // skip this variable. + for (DbgOpID ID : Value.getDbgOpIDs()) { + DbgOp Op = DbgOpStore.find(ID); + DbgOps.push_back(Op); + if (ID.isUndef()) { + IsValueValid = false; + break; + } + if (ID.isConst()) { + ResolvedDbgOps.push_back(Op.MO); + continue; + } + + // If the value has no location, we can't make a variable location. + const ValueIDNum &Num = Op.ID; + auto ValuesPreferredLoc = ValueToLoc.find(Num); + if (ValuesPreferredLoc->second.isIllegal()) { + // If it's a def that occurs in this block, register it as a + // use-before-def to be resolved as we step through the block. + // Continue processing values so that we add any other UseBeforeDef + // entries needed for later. + if (Num.getBlock() == (unsigned)MBB.getNumber() && !Num.isPHI()) { + LastUseBeforeDef = std::max(LastUseBeforeDef, + static_cast<unsigned>(Num.getInst())); + continue; + } + recoverAsEntryValue(Var, Value.Properties, Num); + IsValueValid = false; + break; + } + + // Defer modifying ActiveVLocs until after we've confirmed we have a + // live range. + LocIdx M = ValuesPreferredLoc->second.getLoc(); + ResolvedDbgOps.push_back(M); + } + + // If we cannot produce a valid value for the LiveIn value within this + // block, skip this variable. + if (!IsValueValid) + return; + + // Add UseBeforeDef entry for the last value to be defined in this block. + if (LastUseBeforeDef) { + addUseBeforeDef(Var, Value.Properties, DbgOps, + LastUseBeforeDef); + return; + } + + // The LiveIn value is available at block entry, begin tracking and record + // the transfer. + for (const ResolvedDbgOp &Op : ResolvedDbgOps) + if (!Op.IsConst) + ActiveMLocs[Op.Loc].insert(Var); + auto NewValue = ResolvedDbgValue{ResolvedDbgOps, Value.Properties}; + auto Result = ActiveVLocs.insert(std::make_pair(Var, NewValue)); + if (!Result.second) + Result.first->second = NewValue; + PendingDbgValues.push_back( + MTracker->emitLoc(ResolvedDbgOps, Var, Value.Properties)); + } + + /// Load object with live-in variable values. \p mlocs contains the live-in + /// values in each machine location, while \p vlocs the live-in variable + /// values. This method picks variable locations for the live-in variables, + /// creates DBG_VALUEs and puts them in #Transfers, then prepares the other + /// object fields to track variable locations as we step through the block. + /// FIXME: could just examine mloctracker instead of passing in \p mlocs? + void + loadInlocs(MachineBasicBlock &MBB, ValueTable &MLocs, DbgOpIDMap &DbgOpStore, + const SmallVectorImpl<std::pair<DebugVariable, DbgValue>> &VLocs, + unsigned NumLocs) { + ActiveMLocs.clear(); + ActiveVLocs.clear(); + VarLocs.clear(); + VarLocs.reserve(NumLocs); + UseBeforeDefs.clear(); + UseBeforeDefVariables.clear(); + + // Map of the preferred location for each value. + DenseMap<ValueIDNum, LocationAndQuality> ValueToLoc; + + // Initialized the preferred-location map with illegal locations, to be + // filled in later. + for (const auto &VLoc : VLocs) + if (VLoc.second.Kind == DbgValue::Def) + for (DbgOpID OpID : VLoc.second.getDbgOpIDs()) + if (!OpID.ID.IsConst) + ValueToLoc.insert({DbgOpStore.find(OpID).ID, LocationAndQuality()}); + + ActiveMLocs.reserve(VLocs.size()); + ActiveVLocs.reserve(VLocs.size()); + + // Produce a map of value numbers to the current machine locs they live + // in. When emulating VarLocBasedImpl, there should only be one + // location; when not, we get to pick. + for (auto Location : MTracker->locations()) { + LocIdx Idx = Location.Idx; + ValueIDNum &VNum = MLocs[Idx.asU64()]; + if (VNum == ValueIDNum::EmptyValue) + continue; + VarLocs.push_back(VNum); + + // Is there a variable that wants a location for this value? If not, skip. + auto VIt = ValueToLoc.find(VNum); + if (VIt == ValueToLoc.end()) + continue; + + auto &Previous = VIt->second; + // If this is the first location with that value, pick it. Otherwise, + // consider whether it's a "longer term" location. + std::optional<LocationQuality> ReplacementQuality = + getLocQualityIfBetter(Idx, Previous.getQuality()); + if (ReplacementQuality) + Previous = LocationAndQuality(Idx, *ReplacementQuality); + } + + // Now map variables to their picked LocIdxes. + for (const auto &Var : VLocs) { + loadVarInloc(MBB, DbgOpStore, ValueToLoc, Var.first, Var.second); + } + flushDbgValues(MBB.begin(), &MBB); + } + + /// Record that \p Var has value \p ID, a value that becomes available + /// later in the function. + void addUseBeforeDef(const DebugVariable &Var, + const DbgValueProperties &Properties, + const SmallVectorImpl<DbgOp> &DbgOps, unsigned Inst) { + UseBeforeDefs[Inst].emplace_back(DbgOps, Var, Properties); + UseBeforeDefVariables.insert(Var); + } + + /// After the instruction at index \p Inst and position \p pos has been + /// processed, check whether it defines a variable value in a use-before-def. + /// If so, and the variable value hasn't changed since the start of the + /// block, create a DBG_VALUE. + void checkInstForNewValues(unsigned Inst, MachineBasicBlock::iterator pos) { + auto MIt = UseBeforeDefs.find(Inst); + if (MIt == UseBeforeDefs.end()) + return; + + // Map of values to the locations that store them for every value used by + // the variables that may have become available. + SmallDenseMap<ValueIDNum, LocationAndQuality> ValueToLoc; + + // Populate ValueToLoc with illegal default mappings for every value used by + // any UseBeforeDef variables for this instruction. + for (auto &Use : MIt->second) { + if (!UseBeforeDefVariables.count(Use.Var)) + continue; + + for (DbgOp &Op : Use.Values) { + assert(!Op.isUndef() && "UseBeforeDef erroneously created for a " + "DbgValue with undef values."); + if (Op.IsConst) + continue; + + ValueToLoc.insert({Op.ID, LocationAndQuality()}); + } + } + + // Exit early if we have no DbgValues to produce. + if (ValueToLoc.empty()) + return; + + // Determine the best location for each desired value. + for (auto Location : MTracker->locations()) { + LocIdx Idx = Location.Idx; + ValueIDNum &LocValueID = Location.Value; + + // Is there a variable that wants a location for this value? If not, skip. + auto VIt = ValueToLoc.find(LocValueID); + if (VIt == ValueToLoc.end()) + continue; + + auto &Previous = VIt->second; + // If this is the first location with that value, pick it. Otherwise, + // consider whether it's a "longer term" location. + std::optional<LocationQuality> ReplacementQuality = + getLocQualityIfBetter(Idx, Previous.getQuality()); + if (ReplacementQuality) + Previous = LocationAndQuality(Idx, *ReplacementQuality); + } + + // Using the map of values to locations, produce a final set of values for + // this variable. + for (auto &Use : MIt->second) { + if (!UseBeforeDefVariables.count(Use.Var)) + continue; + + SmallVector<ResolvedDbgOp> DbgOps; + + for (DbgOp &Op : Use.Values) { + if (Op.IsConst) { + DbgOps.push_back(Op.MO); + continue; + } + LocIdx NewLoc = ValueToLoc.find(Op.ID)->second.getLoc(); + if (NewLoc.isIllegal()) + break; + DbgOps.push_back(NewLoc); + } + + // If at least one value used by this debug value is no longer available, + // i.e. one of the values was killed before we finished defining all of + // the values used by this variable, discard. + if (DbgOps.size() != Use.Values.size()) + continue; + + // Otherwise, we're good to go. + PendingDbgValues.push_back( + MTracker->emitLoc(DbgOps, Use.Var, Use.Properties)); + } + flushDbgValues(pos, nullptr); + } + + /// Helper to move created DBG_VALUEs into Transfers collection. + void flushDbgValues(MachineBasicBlock::iterator Pos, MachineBasicBlock *MBB) { + if (PendingDbgValues.size() == 0) + return; + + // Pick out the instruction start position. + MachineBasicBlock::instr_iterator BundleStart; + if (MBB && Pos == MBB->begin()) + BundleStart = MBB->instr_begin(); + else + BundleStart = getBundleStart(Pos->getIterator()); + + Transfers.push_back({BundleStart, MBB, PendingDbgValues}); + PendingDbgValues.clear(); + } + + bool isEntryValueVariable(const DebugVariable &Var, + const DIExpression *Expr) const { + if (!Var.getVariable()->isParameter()) + return false; + + if (Var.getInlinedAt()) + return false; + + if (Expr->getNumElements() > 0) + return false; + + return true; + } + + bool isEntryValueValue(const ValueIDNum &Val) const { + // Must be in entry block (block number zero), and be a PHI / live-in value. + if (Val.getBlock() || !Val.isPHI()) + return false; + + // Entry values must enter in a register. + if (MTracker->isSpill(Val.getLoc())) + return false; + + Register SP = TLI->getStackPointerRegisterToSaveRestore(); + Register FP = TRI.getFrameRegister(MF); + Register Reg = MTracker->LocIdxToLocID[Val.getLoc()]; + return Reg != SP && Reg != FP; + } + + bool recoverAsEntryValue(const DebugVariable &Var, + const DbgValueProperties &Prop, + const ValueIDNum &Num) { + // Is this variable location a candidate to be an entry value. First, + // should we be trying this at all? + if (!ShouldEmitDebugEntryValues) + return false; + + const DIExpression *DIExpr = Prop.DIExpr; + + // We don't currently emit entry values for DBG_VALUE_LISTs. + if (Prop.IsVariadic) { + // If this debug value can be converted to be non-variadic, then do so; + // otherwise give up. + auto NonVariadicExpression = + DIExpression::convertToNonVariadicExpression(DIExpr); + if (!NonVariadicExpression) + return false; + DIExpr = *NonVariadicExpression; + } + + // Is the variable appropriate for entry values (i.e., is a parameter). + if (!isEntryValueVariable(Var, DIExpr)) + return false; + + // Is the value assigned to this variable still the entry value? + if (!isEntryValueValue(Num)) + return false; + + // Emit a variable location using an entry value expression. + DIExpression *NewExpr = + DIExpression::prepend(DIExpr, DIExpression::EntryValue); + Register Reg = MTracker->LocIdxToLocID[Num.getLoc()]; + MachineOperand MO = MachineOperand::CreateReg(Reg, false); + + PendingDbgValues.push_back( + emitMOLoc(MO, Var, {NewExpr, Prop.Indirect, false})); + return true; + } + + /// Change a variable value after encountering a DBG_VALUE inside a block. + void redefVar(const MachineInstr &MI) { + DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(), + MI.getDebugLoc()->getInlinedAt()); + DbgValueProperties Properties(MI); + + // Ignore non-register locations, we don't transfer those. + if (MI.isUndefDebugValue() || + all_of(MI.debug_operands(), + [](const MachineOperand &MO) { return !MO.isReg(); })) { + auto It = ActiveVLocs.find(Var); + if (It != ActiveVLocs.end()) { + for (LocIdx Loc : It->second.loc_indices()) + ActiveMLocs[Loc].erase(Var); + ActiveVLocs.erase(It); + } + // Any use-before-defs no longer apply. + UseBeforeDefVariables.erase(Var); + return; + } + + SmallVector<ResolvedDbgOp> NewLocs; + for (const MachineOperand &MO : MI.debug_operands()) { + if (MO.isReg()) { + // Any undef regs have already been filtered out above. + Register Reg = MO.getReg(); + LocIdx NewLoc = MTracker->getRegMLoc(Reg); + NewLocs.push_back(NewLoc); + } else { + NewLocs.push_back(MO); + } + } + + redefVar(MI, Properties, NewLocs); + } + + /// Handle a change in variable location within a block. Terminate the + /// variables current location, and record the value it now refers to, so + /// that we can detect location transfers later on. + void redefVar(const MachineInstr &MI, const DbgValueProperties &Properties, + SmallVectorImpl<ResolvedDbgOp> &NewLocs) { + DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(), + MI.getDebugLoc()->getInlinedAt()); + // Any use-before-defs no longer apply. + UseBeforeDefVariables.erase(Var); + + // Erase any previous location. + auto It = ActiveVLocs.find(Var); + if (It != ActiveVLocs.end()) { + for (LocIdx Loc : It->second.loc_indices()) + ActiveMLocs[Loc].erase(Var); + } + + // If there _is_ no new location, all we had to do was erase. + if (NewLocs.empty()) { + if (It != ActiveVLocs.end()) + ActiveVLocs.erase(It); + return; + } + + SmallVector<std::pair<LocIdx, DebugVariable>> LostMLocs; + for (ResolvedDbgOp &Op : NewLocs) { + if (Op.IsConst) + continue; + + LocIdx NewLoc = Op.Loc; + + // Check whether our local copy of values-by-location in #VarLocs is out + // of date. Wipe old tracking data for the location if it's been clobbered + // in the meantime. + if (MTracker->readMLoc(NewLoc) != VarLocs[NewLoc.asU64()]) { + for (const auto &P : ActiveMLocs[NewLoc]) { + auto LostVLocIt = ActiveVLocs.find(P); + if (LostVLocIt != ActiveVLocs.end()) { + for (LocIdx Loc : LostVLocIt->second.loc_indices()) { + // Every active variable mapping for NewLoc will be cleared, no + // need to track individual variables. + if (Loc == NewLoc) + continue; + LostMLocs.emplace_back(Loc, P); + } + } + ActiveVLocs.erase(P); + } + for (const auto &LostMLoc : LostMLocs) + ActiveMLocs[LostMLoc.first].erase(LostMLoc.second); + LostMLocs.clear(); + It = ActiveVLocs.find(Var); + ActiveMLocs[NewLoc.asU64()].clear(); + VarLocs[NewLoc.asU64()] = MTracker->readMLoc(NewLoc); + } + + ActiveMLocs[NewLoc].insert(Var); + } + + if (It == ActiveVLocs.end()) { + ActiveVLocs.insert( + std::make_pair(Var, ResolvedDbgValue(NewLocs, Properties))); + } else { + It->second.Ops.assign(NewLocs); + It->second.Properties = Properties; + } + } + + /// Account for a location \p mloc being clobbered. Examine the variable + /// locations that will be terminated: and try to recover them by using + /// another location. Optionally, given \p MakeUndef, emit a DBG_VALUE to + /// explicitly terminate a location if it can't be recovered. + void clobberMloc(LocIdx MLoc, MachineBasicBlock::iterator Pos, + bool MakeUndef = true) { + auto ActiveMLocIt = ActiveMLocs.find(MLoc); + if (ActiveMLocIt == ActiveMLocs.end()) + return; + + // What was the old variable value? + ValueIDNum OldValue = VarLocs[MLoc.asU64()]; + clobberMloc(MLoc, OldValue, Pos, MakeUndef); + } + /// Overload that takes an explicit value \p OldValue for when the value in + /// \p MLoc has changed and the TransferTracker's locations have not been + /// updated yet. + void clobberMloc(LocIdx MLoc, ValueIDNum OldValue, + MachineBasicBlock::iterator Pos, bool MakeUndef = true) { + auto ActiveMLocIt = ActiveMLocs.find(MLoc); + if (ActiveMLocIt == ActiveMLocs.end()) + return; + + VarLocs[MLoc.asU64()] = ValueIDNum::EmptyValue; + + // Examine the remaining variable locations: if we can find the same value + // again, we can recover the location. + std::optional<LocIdx> NewLoc; + for (auto Loc : MTracker->locations()) + if (Loc.Value == OldValue) + NewLoc = Loc.Idx; + + // If there is no location, and we weren't asked to make the variable + // explicitly undef, then stop here. + if (!NewLoc && !MakeUndef) { + // Try and recover a few more locations with entry values. + for (const auto &Var : ActiveMLocIt->second) { + auto &Prop = ActiveVLocs.find(Var)->second.Properties; + recoverAsEntryValue(Var, Prop, OldValue); + } + flushDbgValues(Pos, nullptr); + return; + } + + // Examine all the variables based on this location. + DenseSet<DebugVariable> NewMLocs; + // If no new location has been found, every variable that depends on this + // MLoc is dead, so end their existing MLoc->Var mappings as well. + SmallVector<std::pair<LocIdx, DebugVariable>> LostMLocs; + for (const auto &Var : ActiveMLocIt->second) { + auto ActiveVLocIt = ActiveVLocs.find(Var); + // Re-state the variable location: if there's no replacement then NewLoc + // is std::nullopt and a $noreg DBG_VALUE will be created. Otherwise, a + // DBG_VALUE identifying the alternative location will be emitted. + const DbgValueProperties &Properties = ActiveVLocIt->second.Properties; + + // Produce the new list of debug ops - an empty list if no new location + // was found, or the existing list with the substitution MLoc -> NewLoc + // otherwise. + SmallVector<ResolvedDbgOp> DbgOps; + if (NewLoc) { + ResolvedDbgOp OldOp(MLoc); + ResolvedDbgOp NewOp(*NewLoc); + // Insert illegal ops to overwrite afterwards. + DbgOps.insert(DbgOps.begin(), ActiveVLocIt->second.Ops.size(), + ResolvedDbgOp(LocIdx::MakeIllegalLoc())); + replace_copy(ActiveVLocIt->second.Ops, DbgOps.begin(), OldOp, NewOp); + } + + PendingDbgValues.push_back(MTracker->emitLoc(DbgOps, Var, Properties)); + + // Update machine locations <=> variable locations maps. Defer updating + // ActiveMLocs to avoid invalidating the ActiveMLocIt iterator. + if (!NewLoc) { + for (LocIdx Loc : ActiveVLocIt->second.loc_indices()) { + if (Loc != MLoc) + LostMLocs.emplace_back(Loc, Var); + } + ActiveVLocs.erase(ActiveVLocIt); + } else { + ActiveVLocIt->second.Ops = DbgOps; + NewMLocs.insert(Var); + } + } + + // Remove variables from ActiveMLocs if they no longer use any other MLocs + // due to being killed by this clobber. + for (auto &LocVarIt : LostMLocs) { + auto LostMLocIt = ActiveMLocs.find(LocVarIt.first); + assert(LostMLocIt != ActiveMLocs.end() && + "Variable was using this MLoc, but ActiveMLocs[MLoc] has no " + "entries?"); + LostMLocIt->second.erase(LocVarIt.second); + } + + // We lazily track what locations have which values; if we've found a new + // location for the clobbered value, remember it. + if (NewLoc) + VarLocs[NewLoc->asU64()] = OldValue; + + flushDbgValues(Pos, nullptr); + + // Commit ActiveMLoc changes. + ActiveMLocIt->second.clear(); + if (!NewMLocs.empty()) + for (auto &Var : NewMLocs) + ActiveMLocs[*NewLoc].insert(Var); + } + + /// Transfer variables based on \p Src to be based on \p Dst. This handles + /// both register copies as well as spills and restores. Creates DBG_VALUEs + /// describing the movement. + void transferMlocs(LocIdx Src, LocIdx Dst, MachineBasicBlock::iterator Pos) { + // Does Src still contain the value num we expect? If not, it's been + // clobbered in the meantime, and our variable locations are stale. + if (VarLocs[Src.asU64()] != MTracker->readMLoc(Src)) + return; + + // assert(ActiveMLocs[Dst].size() == 0); + //^^^ Legitimate scenario on account of un-clobbered slot being assigned to? + + // Move set of active variables from one location to another. + auto MovingVars = ActiveMLocs[Src]; + ActiveMLocs[Dst].insert(MovingVars.begin(), MovingVars.end()); + VarLocs[Dst.asU64()] = VarLocs[Src.asU64()]; + + // For each variable based on Src; create a location at Dst. + ResolvedDbgOp SrcOp(Src); + ResolvedDbgOp DstOp(Dst); + for (const auto &Var : MovingVars) { + auto ActiveVLocIt = ActiveVLocs.find(Var); + assert(ActiveVLocIt != ActiveVLocs.end()); + + // Update all instances of Src in the variable's tracked values to Dst. + std::replace(ActiveVLocIt->second.Ops.begin(), + ActiveVLocIt->second.Ops.end(), SrcOp, DstOp); + + MachineInstr *MI = MTracker->emitLoc(ActiveVLocIt->second.Ops, Var, + ActiveVLocIt->second.Properties); + PendingDbgValues.push_back(MI); + } + ActiveMLocs[Src].clear(); + flushDbgValues(Pos, nullptr); + + // XXX XXX XXX "pretend to be old LDV" means dropping all tracking data + // about the old location. + if (EmulateOldLDV) + VarLocs[Src.asU64()] = ValueIDNum::EmptyValue; + } + + MachineInstrBuilder emitMOLoc(const MachineOperand &MO, + const DebugVariable &Var, + const DbgValueProperties &Properties) { + DebugLoc DL = DILocation::get(Var.getVariable()->getContext(), 0, 0, + Var.getVariable()->getScope(), + const_cast<DILocation *>(Var.getInlinedAt())); + auto MIB = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE)); + MIB.add(MO); + if (Properties.Indirect) + MIB.addImm(0); + else + MIB.addReg(0); + MIB.addMetadata(Var.getVariable()); + MIB.addMetadata(Properties.DIExpr); + return MIB; + } +}; + +//===----------------------------------------------------------------------===// +// Implementation +//===----------------------------------------------------------------------===// + +ValueIDNum ValueIDNum::EmptyValue = {UINT_MAX, UINT_MAX, UINT_MAX}; +ValueIDNum ValueIDNum::TombstoneValue = {UINT_MAX, UINT_MAX, UINT_MAX - 1}; + +#ifndef NDEBUG +void ResolvedDbgOp::dump(const MLocTracker *MTrack) const { + if (IsConst) { + dbgs() << MO; + } else { + dbgs() << MTrack->LocIdxToName(Loc); + } +} +void DbgOp::dump(const MLocTracker *MTrack) const { + if (IsConst) { + dbgs() << MO; + } else if (!isUndef()) { + dbgs() << MTrack->IDAsString(ID); + } +} +void DbgOpID::dump(const MLocTracker *MTrack, const DbgOpIDMap *OpStore) const { + if (!OpStore) { + dbgs() << "ID(" << asU32() << ")"; + } else { + OpStore->find(*this).dump(MTrack); + } +} +void DbgValue::dump(const MLocTracker *MTrack, + const DbgOpIDMap *OpStore) const { + if (Kind == NoVal) { + dbgs() << "NoVal(" << BlockNo << ")"; + } else if (Kind == VPHI || Kind == Def) { + if (Kind == VPHI) + dbgs() << "VPHI(" << BlockNo << ","; + else + dbgs() << "Def("; + for (unsigned Idx = 0; Idx < getDbgOpIDs().size(); ++Idx) { + getDbgOpID(Idx).dump(MTrack, OpStore); + if (Idx != 0) + dbgs() << ","; + } + dbgs() << ")"; + } + if (Properties.Indirect) + dbgs() << " indir"; + if (Properties.DIExpr) + dbgs() << " " << *Properties.DIExpr; +} +#endif + +MLocTracker::MLocTracker(MachineFunction &MF, const TargetInstrInfo &TII, + const TargetRegisterInfo &TRI, + const TargetLowering &TLI) + : MF(MF), TII(TII), TRI(TRI), TLI(TLI), + LocIdxToIDNum(ValueIDNum::EmptyValue), LocIdxToLocID(0) { + NumRegs = TRI.getNumRegs(); + reset(); + LocIDToLocIdx.resize(NumRegs, LocIdx::MakeIllegalLoc()); + assert(NumRegs < (1u << NUM_LOC_BITS)); // Detect bit packing failure + + // Always track SP. This avoids the implicit clobbering caused by regmasks + // from affectings its values. (LiveDebugValues disbelieves calls and + // regmasks that claim to clobber SP). + Register SP = TLI.getStackPointerRegisterToSaveRestore(); + if (SP) { + unsigned ID = getLocID(SP); + (void)lookupOrTrackRegister(ID); + + for (MCRegAliasIterator RAI(SP, &TRI, true); RAI.isValid(); ++RAI) + SPAliases.insert(*RAI); + } + + // Build some common stack positions -- full registers being spilt to the + // stack. + StackSlotIdxes.insert({{8, 0}, 0}); + StackSlotIdxes.insert({{16, 0}, 1}); + StackSlotIdxes.insert({{32, 0}, 2}); + StackSlotIdxes.insert({{64, 0}, 3}); + StackSlotIdxes.insert({{128, 0}, 4}); + StackSlotIdxes.insert({{256, 0}, 5}); + StackSlotIdxes.insert({{512, 0}, 6}); + + // Traverse all the subregister idxes, and ensure there's an index for them. + // Duplicates are no problem: we're interested in their position in the + // stack slot, we don't want to type the slot. + for (unsigned int I = 1; I < TRI.getNumSubRegIndices(); ++I) { + unsigned Size = TRI.getSubRegIdxSize(I); + unsigned Offs = TRI.getSubRegIdxOffset(I); + unsigned Idx = StackSlotIdxes.size(); + + // Some subregs have -1, -2 and so forth fed into their fields, to mean + // special backend things. Ignore those. + if (Size > 60000 || Offs > 60000) + continue; + + StackSlotIdxes.insert({{Size, Offs}, Idx}); + } + + // There may also be strange register class sizes (think x86 fp80s). + for (const TargetRegisterClass *RC : TRI.regclasses()) { + unsigned Size = TRI.getRegSizeInBits(*RC); + + // We might see special reserved values as sizes, and classes for other + // stuff the machine tries to model. If it's more than 512 bits, then it + // is very unlikely to be a register than can be spilt. + if (Size > 512) + continue; + + unsigned Idx = StackSlotIdxes.size(); + StackSlotIdxes.insert({{Size, 0}, Idx}); + } + + for (auto &Idx : StackSlotIdxes) + StackIdxesToPos[Idx.second] = Idx.first; + + NumSlotIdxes = StackSlotIdxes.size(); +} + +LocIdx MLocTracker::trackRegister(unsigned ID) { + assert(ID != 0); + LocIdx NewIdx = LocIdx(LocIdxToIDNum.size()); + LocIdxToIDNum.grow(NewIdx); + LocIdxToLocID.grow(NewIdx); + + // Default: it's an mphi. + ValueIDNum ValNum = {CurBB, 0, NewIdx}; + // Was this reg ever touched by a regmask? + for (const auto &MaskPair : reverse(Masks)) { + if (MaskPair.first->clobbersPhysReg(ID)) { + // There was an earlier def we skipped. + ValNum = {CurBB, MaskPair.second, NewIdx}; + break; + } + } + + LocIdxToIDNum[NewIdx] = ValNum; + LocIdxToLocID[NewIdx] = ID; + return NewIdx; +} + +void MLocTracker::writeRegMask(const MachineOperand *MO, unsigned CurBB, + unsigned InstID) { + // Def any register we track have that isn't preserved. The regmask + // terminates the liveness of a register, meaning its value can't be + // relied upon -- we represent this by giving it a new value. + for (auto Location : locations()) { + unsigned ID = LocIdxToLocID[Location.Idx]; + // Don't clobber SP, even if the mask says it's clobbered. + if (ID < NumRegs && !SPAliases.count(ID) && MO->clobbersPhysReg(ID)) + defReg(ID, CurBB, InstID); + } + Masks.push_back(std::make_pair(MO, InstID)); +} + +std::optional<SpillLocationNo> MLocTracker::getOrTrackSpillLoc(SpillLoc L) { + SpillLocationNo SpillID(SpillLocs.idFor(L)); + + if (SpillID.id() == 0) { + // If there is no location, and we have reached the limit of how many stack + // slots to track, then don't track this one. + if (SpillLocs.size() >= StackWorkingSetLimit) + return std::nullopt; + + // Spill location is untracked: create record for this one, and all + // subregister slots too. + SpillID = SpillLocationNo(SpillLocs.insert(L)); + for (unsigned StackIdx = 0; StackIdx < NumSlotIdxes; ++StackIdx) { + unsigned L = getSpillIDWithIdx(SpillID, StackIdx); + LocIdx Idx = LocIdx(LocIdxToIDNum.size()); // New idx + LocIdxToIDNum.grow(Idx); + LocIdxToLocID.grow(Idx); + LocIDToLocIdx.push_back(Idx); + LocIdxToLocID[Idx] = L; + // Initialize to PHI value; corresponds to the location's live-in value + // during transfer function construction. + LocIdxToIDNum[Idx] = ValueIDNum(CurBB, 0, Idx); + } + } + return SpillID; +} + +std::string MLocTracker::LocIdxToName(LocIdx Idx) const { + unsigned ID = LocIdxToLocID[Idx]; + if (ID >= NumRegs) { + StackSlotPos Pos = locIDToSpillIdx(ID); + ID -= NumRegs; + unsigned Slot = ID / NumSlotIdxes; + return Twine("slot ") + .concat(Twine(Slot).concat(Twine(" sz ").concat(Twine(Pos.first) + .concat(Twine(" offs ").concat(Twine(Pos.second)))))) + .str(); + } else { + return TRI.getRegAsmName(ID).str(); + } +} + +std::string MLocTracker::IDAsString(const ValueIDNum &Num) const { + std::string DefName = LocIdxToName(Num.getLoc()); + return Num.asString(DefName); +} + +#ifndef NDEBUG +LLVM_DUMP_METHOD void MLocTracker::dump() { + for (auto Location : locations()) { + std::string MLocName = LocIdxToName(Location.Value.getLoc()); + std::string DefName = Location.Value.asString(MLocName); + dbgs() << LocIdxToName(Location.Idx) << " --> " << DefName << "\n"; + } +} + +LLVM_DUMP_METHOD void MLocTracker::dump_mloc_map() { + for (auto Location : locations()) { + std::string foo = LocIdxToName(Location.Idx); + dbgs() << "Idx " << Location.Idx.asU64() << " " << foo << "\n"; + } +} +#endif + +MachineInstrBuilder +MLocTracker::emitLoc(const SmallVectorImpl<ResolvedDbgOp> &DbgOps, + const DebugVariable &Var, + const DbgValueProperties &Properties) { + DebugLoc DL = DILocation::get(Var.getVariable()->getContext(), 0, 0, + Var.getVariable()->getScope(), + const_cast<DILocation *>(Var.getInlinedAt())); + + const MCInstrDesc &Desc = Properties.IsVariadic + ? TII.get(TargetOpcode::DBG_VALUE_LIST) + : TII.get(TargetOpcode::DBG_VALUE); + +#ifdef EXPENSIVE_CHECKS + assert(all_of(DbgOps, + [](const ResolvedDbgOp &Op) { + return Op.IsConst || !Op.Loc.isIllegal(); + }) && + "Did not expect illegal ops in DbgOps."); + assert((DbgOps.size() == 0 || + DbgOps.size() == Properties.getLocationOpCount()) && + "Expected to have either one DbgOp per MI LocationOp, or none."); +#endif + + auto GetRegOp = [](unsigned Reg) -> MachineOperand { + return MachineOperand::CreateReg( + /* Reg */ Reg, /* isDef */ false, /* isImp */ false, + /* isKill */ false, /* isDead */ false, + /* isUndef */ false, /* isEarlyClobber */ false, + /* SubReg */ 0, /* isDebug */ true); + }; + + SmallVector<MachineOperand> MOs; + + auto EmitUndef = [&]() { + MOs.clear(); + MOs.assign(Properties.getLocationOpCount(), GetRegOp(0)); + return BuildMI(MF, DL, Desc, false, MOs, Var.getVariable(), + Properties.DIExpr); + }; + + // Don't bother passing any real operands to BuildMI if any of them would be + // $noreg. + if (DbgOps.empty()) + return EmitUndef(); + + bool Indirect = Properties.Indirect; + + const DIExpression *Expr = Properties.DIExpr; + + assert(DbgOps.size() == Properties.getLocationOpCount()); + + // If all locations are valid, accumulate them into our list of + // MachineOperands. For any spilled locations, either update the indirectness + // register or apply the appropriate transformations in the DIExpression. + for (size_t Idx = 0; Idx < Properties.getLocationOpCount(); ++Idx) { + const ResolvedDbgOp &Op = DbgOps[Idx]; + + if (Op.IsConst) { + MOs.push_back(Op.MO); + continue; + } + + LocIdx MLoc = Op.Loc; + unsigned LocID = LocIdxToLocID[MLoc]; + if (LocID >= NumRegs) { + SpillLocationNo SpillID = locIDToSpill(LocID); + StackSlotPos StackIdx = locIDToSpillIdx(LocID); + unsigned short Offset = StackIdx.second; + + // TODO: support variables that are located in spill slots, with non-zero + // offsets from the start of the spill slot. It would require some more + // complex DIExpression calculations. This doesn't seem to be produced by + // LLVM right now, so don't try and support it. + // Accept no-subregister slots and subregisters where the offset is zero. + // The consumer should already have type information to work out how large + // the variable is. + if (Offset == 0) { + const SpillLoc &Spill = SpillLocs[SpillID.id()]; + unsigned Base = Spill.SpillBase; + + // There are several ways we can dereference things, and several inputs + // to consider: + // * NRVO variables will appear with IsIndirect set, but should have + // nothing else in their DIExpressions, + // * Variables with DW_OP_stack_value in their expr already need an + // explicit dereference of the stack location, + // * Values that don't match the variable size need DW_OP_deref_size, + // * Everything else can just become a simple location expression. + + // We need to use deref_size whenever there's a mismatch between the + // size of value and the size of variable portion being read. + // Additionally, we should use it whenever dealing with stack_value + // fragments, to avoid the consumer having to determine the deref size + // from DW_OP_piece. + bool UseDerefSize = false; + unsigned ValueSizeInBits = getLocSizeInBits(MLoc); + unsigned DerefSizeInBytes = ValueSizeInBits / 8; + if (auto Fragment = Var.getFragment()) { + unsigned VariableSizeInBits = Fragment->SizeInBits; + if (VariableSizeInBits != ValueSizeInBits || Expr->isComplex()) + UseDerefSize = true; + } else if (auto Size = Var.getVariable()->getSizeInBits()) { + if (*Size != ValueSizeInBits) { + UseDerefSize = true; + } + } + + SmallVector<uint64_t, 5> OffsetOps; + TRI.getOffsetOpcodes(Spill.SpillOffset, OffsetOps); + bool StackValue = false; + + if (Properties.Indirect) { + // This is something like an NRVO variable, where the pointer has been + // spilt to the stack. It should end up being a memory location, with + // the pointer to the variable loaded off the stack with a deref: + assert(!Expr->isImplicit()); + OffsetOps.push_back(dwarf::DW_OP_deref); + } else if (UseDerefSize && Expr->isSingleLocationExpression()) { + // TODO: Figure out how to handle deref size issues for variadic + // values. + // We're loading a value off the stack that's not the same size as the + // variable. Add / subtract stack offset, explicitly deref with a + // size, and add DW_OP_stack_value if not already present. + OffsetOps.push_back(dwarf::DW_OP_deref_size); + OffsetOps.push_back(DerefSizeInBytes); + StackValue = true; + } else if (Expr->isComplex() || Properties.IsVariadic) { + // A variable with no size ambiguity, but with extra elements in it's + // expression. Manually dereference the stack location. + OffsetOps.push_back(dwarf::DW_OP_deref); + } else { + // A plain value that has been spilt to the stack, with no further + // context. Request a location expression, marking the DBG_VALUE as + // IsIndirect. + Indirect = true; + } + + Expr = DIExpression::appendOpsToArg(Expr, OffsetOps, Idx, StackValue); + MOs.push_back(GetRegOp(Base)); + } else { + // This is a stack location with a weird subregister offset: emit an + // undef DBG_VALUE instead. + return EmitUndef(); + } + } else { + // Non-empty, non-stack slot, must be a plain register. + MOs.push_back(GetRegOp(LocID)); + } + } + + return BuildMI(MF, DL, Desc, Indirect, MOs, Var.getVariable(), Expr); +} + +/// Default construct and initialize the pass. +InstrRefBasedLDV::InstrRefBasedLDV() = default; + +bool InstrRefBasedLDV::isCalleeSaved(LocIdx L) const { + unsigned Reg = MTracker->LocIdxToLocID[L]; + return isCalleeSavedReg(Reg); +} +bool InstrRefBasedLDV::isCalleeSavedReg(Register R) const { + for (MCRegAliasIterator RAI(R, TRI, true); RAI.isValid(); ++RAI) + if (CalleeSavedRegs.test(*RAI)) + return true; + return false; +} + +//===----------------------------------------------------------------------===// +// Debug Range Extension Implementation +//===----------------------------------------------------------------------===// + +#ifndef NDEBUG +// Something to restore in the future. +// void InstrRefBasedLDV::printVarLocInMBB(..) +#endif + +std::optional<SpillLocationNo> +InstrRefBasedLDV::extractSpillBaseRegAndOffset(const MachineInstr &MI) { + assert(MI.hasOneMemOperand() && + "Spill instruction does not have exactly one memory operand?"); + auto MMOI = MI.memoperands_begin(); + const PseudoSourceValue *PVal = (*MMOI)->getPseudoValue(); + assert(PVal->kind() == PseudoSourceValue::FixedStack && + "Inconsistent memory operand in spill instruction"); + int FI = cast<FixedStackPseudoSourceValue>(PVal)->getFrameIndex(); + const MachineBasicBlock *MBB = MI.getParent(); + Register Reg; + StackOffset Offset = TFI->getFrameIndexReference(*MBB->getParent(), FI, Reg); + return MTracker->getOrTrackSpillLoc({Reg, Offset}); +} + +std::optional<LocIdx> +InstrRefBasedLDV::findLocationForMemOperand(const MachineInstr &MI) { + std::optional<SpillLocationNo> SpillLoc = extractSpillBaseRegAndOffset(MI); + if (!SpillLoc) + return std::nullopt; + + // Where in the stack slot is this value defined -- i.e., what size of value + // is this? An important question, because it could be loaded into a register + // from the stack at some point. Happily the memory operand will tell us + // the size written to the stack. + auto *MemOperand = *MI.memoperands_begin(); + unsigned SizeInBits = MemOperand->getSizeInBits(); + + // Find that position in the stack indexes we're tracking. + auto IdxIt = MTracker->StackSlotIdxes.find({SizeInBits, 0}); + if (IdxIt == MTracker->StackSlotIdxes.end()) + // That index is not tracked. This is suprising, and unlikely to ever + // occur, but the safe action is to indicate the variable is optimised out. + return std::nullopt; + + unsigned SpillID = MTracker->getSpillIDWithIdx(*SpillLoc, IdxIt->second); + return MTracker->getSpillMLoc(SpillID); +} + +/// End all previous ranges related to @MI and start a new range from @MI +/// if it is a DBG_VALUE instr. +bool InstrRefBasedLDV::transferDebugValue(const MachineInstr &MI) { + if (!MI.isDebugValue()) + return false; + + const DILocalVariable *Var = MI.getDebugVariable(); + const DIExpression *Expr = MI.getDebugExpression(); + const DILocation *DebugLoc = MI.getDebugLoc(); + const DILocation *InlinedAt = DebugLoc->getInlinedAt(); + assert(Var->isValidLocationForIntrinsic(DebugLoc) && + "Expected inlined-at fields to agree"); + + DebugVariable V(Var, Expr, InlinedAt); + DbgValueProperties Properties(MI); + + // If there are no instructions in this lexical scope, do no location tracking + // at all, this variable shouldn't get a legitimate location range. + auto *Scope = LS.findLexicalScope(MI.getDebugLoc().get()); + if (Scope == nullptr) + return true; // handled it; by doing nothing + + // MLocTracker needs to know that this register is read, even if it's only + // read by a debug inst. + for (const MachineOperand &MO : MI.debug_operands()) + if (MO.isReg() && MO.getReg() != 0) + (void)MTracker->readReg(MO.getReg()); + + // If we're preparing for the second analysis (variables), the machine value + // locations are already solved, and we report this DBG_VALUE and the value + // it refers to to VLocTracker. + if (VTracker) { + SmallVector<DbgOpID> DebugOps; + // Feed defVar the new variable location, or if this is a DBG_VALUE $noreg, + // feed defVar None. + if (!MI.isUndefDebugValue()) { + for (const MachineOperand &MO : MI.debug_operands()) { + // There should be no undef registers here, as we've screened for undef + // debug values. + if (MO.isReg()) { + DebugOps.push_back(DbgOpStore.insert(MTracker->readReg(MO.getReg()))); + } else if (MO.isImm() || MO.isFPImm() || MO.isCImm()) { + DebugOps.push_back(DbgOpStore.insert(MO)); + } else { + llvm_unreachable("Unexpected debug operand type."); + } + } + } + VTracker->defVar(MI, Properties, DebugOps); + } + + // If performing final tracking of transfers, report this variable definition + // to the TransferTracker too. + if (TTracker) + TTracker->redefVar(MI); + return true; +} + +std::optional<ValueIDNum> InstrRefBasedLDV::getValueForInstrRef( + unsigned InstNo, unsigned OpNo, MachineInstr &MI, + const ValueTable *MLiveOuts, const ValueTable *MLiveIns) { + // Various optimizations may have happened to the value during codegen, + // recorded in the value substitution table. Apply any substitutions to + // the instruction / operand number in this DBG_INSTR_REF, and collect + // any subregister extractions performed during optimization. + const MachineFunction &MF = *MI.getParent()->getParent(); + + // Create dummy substitution with Src set, for lookup. + auto SoughtSub = + MachineFunction::DebugSubstitution({InstNo, OpNo}, {0, 0}, 0); + + SmallVector<unsigned, 4> SeenSubregs; + auto LowerBoundIt = llvm::lower_bound(MF.DebugValueSubstitutions, SoughtSub); + while (LowerBoundIt != MF.DebugValueSubstitutions.end() && + LowerBoundIt->Src == SoughtSub.Src) { + std::tie(InstNo, OpNo) = LowerBoundIt->Dest; + SoughtSub.Src = LowerBoundIt->Dest; + if (unsigned Subreg = LowerBoundIt->Subreg) + SeenSubregs.push_back(Subreg); + LowerBoundIt = llvm::lower_bound(MF.DebugValueSubstitutions, SoughtSub); + } + + // Default machine value number is <None> -- if no instruction defines + // the corresponding value, it must have been optimized out. + std::optional<ValueIDNum> NewID; + + // Try to lookup the instruction number, and find the machine value number + // that it defines. It could be an instruction, or a PHI. + auto InstrIt = DebugInstrNumToInstr.find(InstNo); + auto PHIIt = llvm::lower_bound(DebugPHINumToValue, InstNo); + if (InstrIt != DebugInstrNumToInstr.end()) { + const MachineInstr &TargetInstr = *InstrIt->second.first; + uint64_t BlockNo = TargetInstr.getParent()->getNumber(); + + // Pick out the designated operand. It might be a memory reference, if + // a register def was folded into a stack store. + if (OpNo == MachineFunction::DebugOperandMemNumber && + TargetInstr.hasOneMemOperand()) { + std::optional<LocIdx> L = findLocationForMemOperand(TargetInstr); + if (L) + NewID = ValueIDNum(BlockNo, InstrIt->second.second, *L); + } else if (OpNo != MachineFunction::DebugOperandMemNumber) { + // Permit the debug-info to be completely wrong: identifying a nonexistant + // operand, or one that is not a register definition, means something + // unexpected happened during optimisation. Broken debug-info, however, + // shouldn't crash the compiler -- instead leave the variable value as + // None, which will make it appear "optimised out". + if (OpNo < TargetInstr.getNumOperands()) { + const MachineOperand &MO = TargetInstr.getOperand(OpNo); + + if (MO.isReg() && MO.isDef() && MO.getReg()) { + unsigned LocID = MTracker->getLocID(MO.getReg()); + LocIdx L = MTracker->LocIDToLocIdx[LocID]; + NewID = ValueIDNum(BlockNo, InstrIt->second.second, L); + } + } + + if (!NewID) { + LLVM_DEBUG( + { dbgs() << "Seen instruction reference to illegal operand\n"; }); + } + } + // else: NewID is left as None. + } else if (PHIIt != DebugPHINumToValue.end() && PHIIt->InstrNum == InstNo) { + // It's actually a PHI value. Which value it is might not be obvious, use + // the resolver helper to find out. + NewID = resolveDbgPHIs(*MI.getParent()->getParent(), MLiveOuts, MLiveIns, + MI, InstNo); + } + + // Apply any subregister extractions, in reverse. We might have seen code + // like this: + // CALL64 @foo, implicit-def $rax + // %0:gr64 = COPY $rax + // %1:gr32 = COPY %0.sub_32bit + // %2:gr16 = COPY %1.sub_16bit + // %3:gr8 = COPY %2.sub_8bit + // In which case each copy would have been recorded as a substitution with + // a subregister qualifier. Apply those qualifiers now. + if (NewID && !SeenSubregs.empty()) { + unsigned Offset = 0; + unsigned Size = 0; + + // Look at each subregister that we passed through, and progressively + // narrow in, accumulating any offsets that occur. Substitutions should + // only ever be the same or narrower width than what they read from; + // iterate in reverse order so that we go from wide to small. + for (unsigned Subreg : reverse(SeenSubregs)) { + unsigned ThisSize = TRI->getSubRegIdxSize(Subreg); + unsigned ThisOffset = TRI->getSubRegIdxOffset(Subreg); + Offset += ThisOffset; + Size = (Size == 0) ? ThisSize : std::min(Size, ThisSize); + } + + // If that worked, look for an appropriate subregister with the register + // where the define happens. Don't look at values that were defined during + // a stack write: we can't currently express register locations within + // spills. + LocIdx L = NewID->getLoc(); + if (NewID && !MTracker->isSpill(L)) { + // Find the register class for the register where this def happened. + // FIXME: no index for this? + Register Reg = MTracker->LocIdxToLocID[L]; + const TargetRegisterClass *TRC = nullptr; + for (const auto *TRCI : TRI->regclasses()) + if (TRCI->contains(Reg)) + TRC = TRCI; + assert(TRC && "Couldn't find target register class?"); + + // If the register we have isn't the right size or in the right place, + // Try to find a subregister inside it. + unsigned MainRegSize = TRI->getRegSizeInBits(*TRC); + if (Size != MainRegSize || Offset) { + // Enumerate all subregisters, searching. + Register NewReg = 0; + for (MCSubRegIterator SRI(Reg, TRI, false); SRI.isValid(); ++SRI) { + unsigned Subreg = TRI->getSubRegIndex(Reg, *SRI); + unsigned SubregSize = TRI->getSubRegIdxSize(Subreg); + unsigned SubregOffset = TRI->getSubRegIdxOffset(Subreg); + if (SubregSize == Size && SubregOffset == Offset) { + NewReg = *SRI; + break; + } + } + + // If we didn't find anything: there's no way to express our value. + if (!NewReg) { + NewID = std::nullopt; + } else { + // Re-state the value as being defined within the subregister + // that we found. + LocIdx NewLoc = MTracker->lookupOrTrackRegister(NewReg); + NewID = ValueIDNum(NewID->getBlock(), NewID->getInst(), NewLoc); + } + } + } else { + // If we can't handle subregisters, unset the new value. + NewID = std::nullopt; + } + } + + return NewID; +} + +bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI, + const ValueTable *MLiveOuts, + const ValueTable *MLiveIns) { + if (!MI.isDebugRef()) + return false; + + // Only handle this instruction when we are building the variable value + // transfer function. + if (!VTracker && !TTracker) + return false; + + const DILocalVariable *Var = MI.getDebugVariable(); + const DIExpression *Expr = MI.getDebugExpression(); + const DILocation *DebugLoc = MI.getDebugLoc(); + const DILocation *InlinedAt = DebugLoc->getInlinedAt(); + assert(Var->isValidLocationForIntrinsic(DebugLoc) && + "Expected inlined-at fields to agree"); + + DebugVariable V(Var, Expr, InlinedAt); + + auto *Scope = LS.findLexicalScope(MI.getDebugLoc().get()); + if (Scope == nullptr) + return true; // Handled by doing nothing. This variable is never in scope. + + SmallVector<DbgOpID> DbgOpIDs; + for (const MachineOperand &MO : MI.debug_operands()) { + if (!MO.isDbgInstrRef()) { + assert(!MO.isReg() && "DBG_INSTR_REF should not contain registers"); + DbgOpID ConstOpID = DbgOpStore.insert(DbgOp(MO)); + DbgOpIDs.push_back(ConstOpID); + continue; + } + + unsigned InstNo = MO.getInstrRefInstrIndex(); + unsigned OpNo = MO.getInstrRefOpIndex(); + + // Default machine value number is <None> -- if no instruction defines + // the corresponding value, it must have been optimized out. + std::optional<ValueIDNum> NewID = + getValueForInstrRef(InstNo, OpNo, MI, MLiveOuts, MLiveIns); + // We have a value number or std::nullopt. If the latter, then kill the + // entire debug value. + if (NewID) { + DbgOpIDs.push_back(DbgOpStore.insert(*NewID)); + } else { + DbgOpIDs.clear(); + break; + } + } + + // We have a DbgOpID for every value or for none. Tell the variable value + // tracker about it. The rest of this LiveDebugValues implementation acts + // exactly the same for DBG_INSTR_REFs as DBG_VALUEs (just, the former can + // refer to values that aren't immediately available). + DbgValueProperties Properties(Expr, false, true); + if (VTracker) + VTracker->defVar(MI, Properties, DbgOpIDs); + + // If we're on the final pass through the function, decompose this INSTR_REF + // into a plain DBG_VALUE. + if (!TTracker) + return true; + + // Fetch the concrete DbgOps now, as we will need them later. + SmallVector<DbgOp> DbgOps; + for (DbgOpID OpID : DbgOpIDs) { + DbgOps.push_back(DbgOpStore.find(OpID)); + } + + // Pick a location for the machine value number, if such a location exists. + // (This information could be stored in TransferTracker to make it faster). + SmallDenseMap<ValueIDNum, TransferTracker::LocationAndQuality> FoundLocs; + SmallVector<ValueIDNum> ValuesToFind; + // Initialized the preferred-location map with illegal locations, to be + // filled in later. + for (const DbgOp &Op : DbgOps) { + if (!Op.IsConst) + if (FoundLocs.insert({Op.ID, TransferTracker::LocationAndQuality()}) + .second) + ValuesToFind.push_back(Op.ID); + } + + for (auto Location : MTracker->locations()) { + LocIdx CurL = Location.Idx; + ValueIDNum ID = MTracker->readMLoc(CurL); + auto ValueToFindIt = find(ValuesToFind, ID); + if (ValueToFindIt == ValuesToFind.end()) + continue; + auto &Previous = FoundLocs.find(ID)->second; + // If this is the first location with that value, pick it. Otherwise, + // consider whether it's a "longer term" location. + std::optional<TransferTracker::LocationQuality> ReplacementQuality = + TTracker->getLocQualityIfBetter(CurL, Previous.getQuality()); + if (ReplacementQuality) { + Previous = TransferTracker::LocationAndQuality(CurL, *ReplacementQuality); + if (Previous.isBest()) { + ValuesToFind.erase(ValueToFindIt); + if (ValuesToFind.empty()) + break; + } + } + } + + SmallVector<ResolvedDbgOp> NewLocs; + for (const DbgOp &DbgOp : DbgOps) { + if (DbgOp.IsConst) { + NewLocs.push_back(DbgOp.MO); + continue; + } + LocIdx FoundLoc = FoundLocs.find(DbgOp.ID)->second.getLoc(); + if (FoundLoc.isIllegal()) { + NewLocs.clear(); + break; + } + NewLocs.push_back(FoundLoc); + } + // Tell transfer tracker that the variable value has changed. + TTracker->redefVar(MI, Properties, NewLocs); + + // If there were values with no location, but all such values are defined in + // later instructions in this block, this is a block-local use-before-def. + if (!DbgOps.empty() && NewLocs.empty()) { + bool IsValidUseBeforeDef = true; + uint64_t LastUseBeforeDef = 0; + for (auto ValueLoc : FoundLocs) { + ValueIDNum NewID = ValueLoc.first; + LocIdx FoundLoc = ValueLoc.second.getLoc(); + if (!FoundLoc.isIllegal()) + continue; + // If we have an value with no location that is not defined in this block, + // then it has no location in this block, leaving this value undefined. + if (NewID.getBlock() != CurBB || NewID.getInst() <= CurInst) { + IsValidUseBeforeDef = false; + break; + } + LastUseBeforeDef = std::max(LastUseBeforeDef, NewID.getInst()); + } + if (IsValidUseBeforeDef) { + TTracker->addUseBeforeDef(V, {MI.getDebugExpression(), false, true}, + DbgOps, LastUseBeforeDef); + } + } + + // Produce a DBG_VALUE representing what this DBG_INSTR_REF meant. + // This DBG_VALUE is potentially a $noreg / undefined location, if + // FoundLoc is illegal. + // (XXX -- could morph the DBG_INSTR_REF in the future). + MachineInstr *DbgMI = MTracker->emitLoc(NewLocs, V, Properties); + + TTracker->PendingDbgValues.push_back(DbgMI); + TTracker->flushDbgValues(MI.getIterator(), nullptr); + return true; +} + +bool InstrRefBasedLDV::transferDebugPHI(MachineInstr &MI) { + if (!MI.isDebugPHI()) + return false; + + // Analyse these only when solving the machine value location problem. + if (VTracker || TTracker) + return true; + + // First operand is the value location, either a stack slot or register. + // Second is the debug instruction number of the original PHI. + const MachineOperand &MO = MI.getOperand(0); + unsigned InstrNum = MI.getOperand(1).getImm(); + + auto EmitBadPHI = [this, &MI, InstrNum]() -> bool { + // Helper lambda to do any accounting when we fail to find a location for + // a DBG_PHI. This can happen if DBG_PHIs are malformed, or refer to a + // dead stack slot, for example. + // Record a DebugPHIRecord with an empty value + location. + DebugPHINumToValue.push_back( + {InstrNum, MI.getParent(), std::nullopt, std::nullopt}); + return true; + }; + + if (MO.isReg() && MO.getReg()) { + // The value is whatever's currently in the register. Read and record it, + // to be analysed later. + Register Reg = MO.getReg(); + ValueIDNum Num = MTracker->readReg(Reg); + auto PHIRec = DebugPHIRecord( + {InstrNum, MI.getParent(), Num, MTracker->lookupOrTrackRegister(Reg)}); + DebugPHINumToValue.push_back(PHIRec); + + // Ensure this register is tracked. + for (MCRegAliasIterator RAI(MO.getReg(), TRI, true); RAI.isValid(); ++RAI) + MTracker->lookupOrTrackRegister(*RAI); + } else if (MO.isFI()) { + // The value is whatever's in this stack slot. + unsigned FI = MO.getIndex(); + + // If the stack slot is dead, then this was optimized away. + // FIXME: stack slot colouring should account for slots that get merged. + if (MFI->isDeadObjectIndex(FI)) + return EmitBadPHI(); + + // Identify this spill slot, ensure it's tracked. + Register Base; + StackOffset Offs = TFI->getFrameIndexReference(*MI.getMF(), FI, Base); + SpillLoc SL = {Base, Offs}; + std::optional<SpillLocationNo> SpillNo = MTracker->getOrTrackSpillLoc(SL); + + // We might be able to find a value, but have chosen not to, to avoid + // tracking too much stack information. + if (!SpillNo) + return EmitBadPHI(); + + // Any stack location DBG_PHI should have an associate bit-size. + assert(MI.getNumOperands() == 3 && "Stack DBG_PHI with no size?"); + unsigned slotBitSize = MI.getOperand(2).getImm(); + + unsigned SpillID = MTracker->getLocID(*SpillNo, {slotBitSize, 0}); + LocIdx SpillLoc = MTracker->getSpillMLoc(SpillID); + ValueIDNum Result = MTracker->readMLoc(SpillLoc); + + // Record this DBG_PHI for later analysis. + auto DbgPHI = DebugPHIRecord({InstrNum, MI.getParent(), Result, SpillLoc}); + DebugPHINumToValue.push_back(DbgPHI); + } else { + // Else: if the operand is neither a legal register or a stack slot, then + // we're being fed illegal debug-info. Record an empty PHI, so that any + // debug users trying to read this number will be put off trying to + // interpret the value. + LLVM_DEBUG( + { dbgs() << "Seen DBG_PHI with unrecognised operand format\n"; }); + return EmitBadPHI(); + } + + return true; +} + +void InstrRefBasedLDV::transferRegisterDef(MachineInstr &MI) { + // Meta Instructions do not affect the debug liveness of any register they + // define. + if (MI.isImplicitDef()) { + // Except when there's an implicit def, and the location it's defining has + // no value number. The whole point of an implicit def is to announce that + // the register is live, without be specific about it's value. So define + // a value if there isn't one already. + ValueIDNum Num = MTracker->readReg(MI.getOperand(0).getReg()); + // Has a legitimate value -> ignore the implicit def. + if (Num.getLoc() != 0) + return; + // Otherwise, def it here. + } else if (MI.isMetaInstruction()) + return; + + // We always ignore SP defines on call instructions, they don't actually + // change the value of the stack pointer... except for win32's _chkstk. This + // is rare: filter quickly for the common case (no stack adjustments, not a + // call, etc). If it is a call that modifies SP, recognise the SP register + // defs. + bool CallChangesSP = false; + if (AdjustsStackInCalls && MI.isCall() && MI.getOperand(0).isSymbol() && + !strcmp(MI.getOperand(0).getSymbolName(), StackProbeSymbolName.data())) + CallChangesSP = true; + + // Test whether we should ignore a def of this register due to it being part + // of the stack pointer. + auto IgnoreSPAlias = [this, &MI, CallChangesSP](Register R) -> bool { + if (CallChangesSP) + return false; + return MI.isCall() && MTracker->SPAliases.count(R); + }; + + // Find the regs killed by MI, and find regmasks of preserved regs. + // Max out the number of statically allocated elements in `DeadRegs`, as this + // prevents fallback to std::set::count() operations. + SmallSet<uint32_t, 32> DeadRegs; + SmallVector<const uint32_t *, 4> RegMasks; + SmallVector<const MachineOperand *, 4> RegMaskPtrs; + for (const MachineOperand &MO : MI.operands()) { + // Determine whether the operand is a register def. + if (MO.isReg() && MO.isDef() && MO.getReg() && MO.getReg().isPhysical() && + !IgnoreSPAlias(MO.getReg())) { + // Remove ranges of all aliased registers. + for (MCRegAliasIterator RAI(MO.getReg(), TRI, true); RAI.isValid(); ++RAI) + // FIXME: Can we break out of this loop early if no insertion occurs? + DeadRegs.insert(*RAI); + } else if (MO.isRegMask()) { + RegMasks.push_back(MO.getRegMask()); + RegMaskPtrs.push_back(&MO); + } + } + + // Tell MLocTracker about all definitions, of regmasks and otherwise. + for (uint32_t DeadReg : DeadRegs) + MTracker->defReg(DeadReg, CurBB, CurInst); + + for (const auto *MO : RegMaskPtrs) + MTracker->writeRegMask(MO, CurBB, CurInst); + + // If this instruction writes to a spill slot, def that slot. + if (hasFoldedStackStore(MI)) { + if (std::optional<SpillLocationNo> SpillNo = + extractSpillBaseRegAndOffset(MI)) { + for (unsigned int I = 0; I < MTracker->NumSlotIdxes; ++I) { + unsigned SpillID = MTracker->getSpillIDWithIdx(*SpillNo, I); + LocIdx L = MTracker->getSpillMLoc(SpillID); + MTracker->setMLoc(L, ValueIDNum(CurBB, CurInst, L)); + } + } + } + + if (!TTracker) + return; + + // When committing variable values to locations: tell transfer tracker that + // we've clobbered things. It may be able to recover the variable from a + // different location. + + // Inform TTracker about any direct clobbers. + for (uint32_t DeadReg : DeadRegs) { + LocIdx Loc = MTracker->lookupOrTrackRegister(DeadReg); + TTracker->clobberMloc(Loc, MI.getIterator(), false); + } + + // Look for any clobbers performed by a register mask. Only test locations + // that are actually being tracked. + if (!RegMaskPtrs.empty()) { + for (auto L : MTracker->locations()) { + // Stack locations can't be clobbered by regmasks. + if (MTracker->isSpill(L.Idx)) + continue; + + Register Reg = MTracker->LocIdxToLocID[L.Idx]; + if (IgnoreSPAlias(Reg)) + continue; + + for (const auto *MO : RegMaskPtrs) + if (MO->clobbersPhysReg(Reg)) + TTracker->clobberMloc(L.Idx, MI.getIterator(), false); + } + } + + // Tell TTracker about any folded stack store. + if (hasFoldedStackStore(MI)) { + if (std::optional<SpillLocationNo> SpillNo = + extractSpillBaseRegAndOffset(MI)) { + for (unsigned int I = 0; I < MTracker->NumSlotIdxes; ++I) { + unsigned SpillID = MTracker->getSpillIDWithIdx(*SpillNo, I); + LocIdx L = MTracker->getSpillMLoc(SpillID); + TTracker->clobberMloc(L, MI.getIterator(), true); + } + } + } +} + +void InstrRefBasedLDV::performCopy(Register SrcRegNum, Register DstRegNum) { + // In all circumstances, re-def all aliases. It's definitely a new value now. + for (MCRegAliasIterator RAI(DstRegNum, TRI, true); RAI.isValid(); ++RAI) + MTracker->defReg(*RAI, CurBB, CurInst); + + ValueIDNum SrcValue = MTracker->readReg(SrcRegNum); + MTracker->setReg(DstRegNum, SrcValue); + + // Copy subregisters from one location to another. + for (MCSubRegIndexIterator SRI(SrcRegNum, TRI); SRI.isValid(); ++SRI) { + unsigned SrcSubReg = SRI.getSubReg(); + unsigned SubRegIdx = SRI.getSubRegIndex(); + unsigned DstSubReg = TRI->getSubReg(DstRegNum, SubRegIdx); + if (!DstSubReg) + continue; + + // Do copy. There are two matching subregisters, the source value should + // have been def'd when the super-reg was, the latter might not be tracked + // yet. + // This will force SrcSubReg to be tracked, if it isn't yet. Will read + // mphi values if it wasn't tracked. + LocIdx SrcL = MTracker->lookupOrTrackRegister(SrcSubReg); + LocIdx DstL = MTracker->lookupOrTrackRegister(DstSubReg); + (void)SrcL; + (void)DstL; + ValueIDNum CpyValue = MTracker->readReg(SrcSubReg); + + MTracker->setReg(DstSubReg, CpyValue); + } +} + +std::optional<SpillLocationNo> +InstrRefBasedLDV::isSpillInstruction(const MachineInstr &MI, + MachineFunction *MF) { + // TODO: Handle multiple stores folded into one. + if (!MI.hasOneMemOperand()) + return std::nullopt; + + // Reject any memory operand that's aliased -- we can't guarantee its value. + auto MMOI = MI.memoperands_begin(); + const PseudoSourceValue *PVal = (*MMOI)->getPseudoValue(); + if (PVal->isAliased(MFI)) + return std::nullopt; + + if (!MI.getSpillSize(TII) && !MI.getFoldedSpillSize(TII)) + return std::nullopt; // This is not a spill instruction, since no valid size + // was returned from either function. + + return extractSpillBaseRegAndOffset(MI); +} + +bool InstrRefBasedLDV::isLocationSpill(const MachineInstr &MI, + MachineFunction *MF, unsigned &Reg) { + if (!isSpillInstruction(MI, MF)) + return false; + + int FI; + Reg = TII->isStoreToStackSlotPostFE(MI, FI); + return Reg != 0; +} + +std::optional<SpillLocationNo> +InstrRefBasedLDV::isRestoreInstruction(const MachineInstr &MI, + MachineFunction *MF, unsigned &Reg) { + if (!MI.hasOneMemOperand()) + return std::nullopt; + + // FIXME: Handle folded restore instructions with more than one memory + // operand. + if (MI.getRestoreSize(TII)) { + Reg = MI.getOperand(0).getReg(); + return extractSpillBaseRegAndOffset(MI); + } + return std::nullopt; +} + +bool InstrRefBasedLDV::transferSpillOrRestoreInst(MachineInstr &MI) { + // XXX -- it's too difficult to implement VarLocBasedImpl's stack location + // limitations under the new model. Therefore, when comparing them, compare + // versions that don't attempt spills or restores at all. + if (EmulateOldLDV) + return false; + + // Strictly limit ourselves to plain loads and stores, not all instructions + // that can access the stack. + int DummyFI = -1; + if (!TII->isStoreToStackSlotPostFE(MI, DummyFI) && + !TII->isLoadFromStackSlotPostFE(MI, DummyFI)) + return false; + + MachineFunction *MF = MI.getMF(); + unsigned Reg; + + LLVM_DEBUG(dbgs() << "Examining instruction: "; MI.dump();); + + // Strictly limit ourselves to plain loads and stores, not all instructions + // that can access the stack. + int FIDummy; + if (!TII->isStoreToStackSlotPostFE(MI, FIDummy) && + !TII->isLoadFromStackSlotPostFE(MI, FIDummy)) + return false; + + // First, if there are any DBG_VALUEs pointing at a spill slot that is + // written to, terminate that variable location. The value in memory + // will have changed. DbgEntityHistoryCalculator doesn't try to detect this. + if (std::optional<SpillLocationNo> Loc = isSpillInstruction(MI, MF)) { + // Un-set this location and clobber, so that earlier locations don't + // continue past this store. + for (unsigned SlotIdx = 0; SlotIdx < MTracker->NumSlotIdxes; ++SlotIdx) { + unsigned SpillID = MTracker->getSpillIDWithIdx(*Loc, SlotIdx); + std::optional<LocIdx> MLoc = MTracker->getSpillMLoc(SpillID); + if (!MLoc) + continue; + + // We need to over-write the stack slot with something (here, a def at + // this instruction) to ensure no values are preserved in this stack slot + // after the spill. It also prevents TTracker from trying to recover the + // location and re-installing it in the same place. + ValueIDNum Def(CurBB, CurInst, *MLoc); + MTracker->setMLoc(*MLoc, Def); + if (TTracker) + TTracker->clobberMloc(*MLoc, MI.getIterator()); + } + } + + // Try to recognise spill and restore instructions that may transfer a value. + if (isLocationSpill(MI, MF, Reg)) { + // isLocationSpill returning true should guarantee we can extract a + // location. + SpillLocationNo Loc = *extractSpillBaseRegAndOffset(MI); + + auto DoTransfer = [&](Register SrcReg, unsigned SpillID) { + auto ReadValue = MTracker->readReg(SrcReg); + LocIdx DstLoc = MTracker->getSpillMLoc(SpillID); + MTracker->setMLoc(DstLoc, ReadValue); + + if (TTracker) { + LocIdx SrcLoc = MTracker->getRegMLoc(SrcReg); + TTracker->transferMlocs(SrcLoc, DstLoc, MI.getIterator()); + } + }; + + // Then, transfer subreg bits. + for (MCSubRegIterator SRI(Reg, TRI, false); SRI.isValid(); ++SRI) { + // Ensure this reg is tracked, + (void)MTracker->lookupOrTrackRegister(*SRI); + unsigned SubregIdx = TRI->getSubRegIndex(Reg, *SRI); + unsigned SpillID = MTracker->getLocID(Loc, SubregIdx); + DoTransfer(*SRI, SpillID); + } + + // Directly lookup size of main source reg, and transfer. + unsigned Size = TRI->getRegSizeInBits(Reg, *MRI); + unsigned SpillID = MTracker->getLocID(Loc, {Size, 0}); + DoTransfer(Reg, SpillID); + } else { + std::optional<SpillLocationNo> Loc = isRestoreInstruction(MI, MF, Reg); + if (!Loc) + return false; + + // Assumption: we're reading from the base of the stack slot, not some + // offset into it. It seems very unlikely LLVM would ever generate + // restores where this wasn't true. This then becomes a question of what + // subregisters in the destination register line up with positions in the + // stack slot. + + // Def all registers that alias the destination. + for (MCRegAliasIterator RAI(Reg, TRI, true); RAI.isValid(); ++RAI) + MTracker->defReg(*RAI, CurBB, CurInst); + + // Now find subregisters within the destination register, and load values + // from stack slot positions. + auto DoTransfer = [&](Register DestReg, unsigned SpillID) { + LocIdx SrcIdx = MTracker->getSpillMLoc(SpillID); + auto ReadValue = MTracker->readMLoc(SrcIdx); + MTracker->setReg(DestReg, ReadValue); + }; + + for (MCSubRegIterator SRI(Reg, TRI, false); SRI.isValid(); ++SRI) { + unsigned Subreg = TRI->getSubRegIndex(Reg, *SRI); + unsigned SpillID = MTracker->getLocID(*Loc, Subreg); + DoTransfer(*SRI, SpillID); + } + + // Directly look up this registers slot idx by size, and transfer. + unsigned Size = TRI->getRegSizeInBits(Reg, *MRI); + unsigned SpillID = MTracker->getLocID(*Loc, {Size, 0}); + DoTransfer(Reg, SpillID); + } + return true; +} + +bool InstrRefBasedLDV::transferRegisterCopy(MachineInstr &MI) { + auto DestSrc = TII->isCopyInstr(MI); + if (!DestSrc) + return false; + + const MachineOperand *DestRegOp = DestSrc->Destination; + const MachineOperand *SrcRegOp = DestSrc->Source; + + Register SrcReg = SrcRegOp->getReg(); + Register DestReg = DestRegOp->getReg(); + + // Ignore identity copies. Yep, these make it as far as LiveDebugValues. + if (SrcReg == DestReg) + return true; + + // For emulating VarLocBasedImpl: + // We want to recognize instructions where destination register is callee + // saved register. If register that could be clobbered by the call is + // included, there would be a great chance that it is going to be clobbered + // soon. It is more likely that previous register, which is callee saved, is + // going to stay unclobbered longer, even if it is killed. + // + // For InstrRefBasedImpl, we can track multiple locations per value, so + // ignore this condition. + if (EmulateOldLDV && !isCalleeSavedReg(DestReg)) + return false; + + // InstrRefBasedImpl only followed killing copies. + if (EmulateOldLDV && !SrcRegOp->isKill()) + return false; + + // Before we update MTracker, remember which values were present in each of + // the locations about to be overwritten, so that we can recover any + // potentially clobbered variables. + DenseMap<LocIdx, ValueIDNum> ClobberedLocs; + if (TTracker) { + for (MCRegAliasIterator RAI(DestReg, TRI, true); RAI.isValid(); ++RAI) { + LocIdx ClobberedLoc = MTracker->getRegMLoc(*RAI); + auto MLocIt = TTracker->ActiveMLocs.find(ClobberedLoc); + // If ActiveMLocs isn't tracking this location or there are no variables + // using it, don't bother remembering. + if (MLocIt == TTracker->ActiveMLocs.end() || MLocIt->second.empty()) + continue; + ValueIDNum Value = MTracker->readReg(*RAI); + ClobberedLocs[ClobberedLoc] = Value; + } + } + + // Copy MTracker info, including subregs if available. + InstrRefBasedLDV::performCopy(SrcReg, DestReg); + + // The copy might have clobbered variables based on the destination register. + // Tell TTracker about it, passing the old ValueIDNum to search for + // alternative locations (or else terminating those variables). + if (TTracker) { + for (auto LocVal : ClobberedLocs) { + TTracker->clobberMloc(LocVal.first, LocVal.second, MI.getIterator(), false); + } + } + + // Only produce a transfer of DBG_VALUE within a block where old LDV + // would have. We might make use of the additional value tracking in some + // other way, later. + if (TTracker && isCalleeSavedReg(DestReg) && SrcRegOp->isKill()) + TTracker->transferMlocs(MTracker->getRegMLoc(SrcReg), + MTracker->getRegMLoc(DestReg), MI.getIterator()); + + // VarLocBasedImpl would quit tracking the old location after copying. + if (EmulateOldLDV && SrcReg != DestReg) + MTracker->defReg(SrcReg, CurBB, CurInst); + + return true; +} + +/// Accumulate a mapping between each DILocalVariable fragment and other +/// fragments of that DILocalVariable which overlap. This reduces work during +/// the data-flow stage from "Find any overlapping fragments" to "Check if the +/// known-to-overlap fragments are present". +/// \param MI A previously unprocessed debug instruction to analyze for +/// fragment usage. +void InstrRefBasedLDV::accumulateFragmentMap(MachineInstr &MI) { + assert(MI.isDebugValueLike()); + DebugVariable MIVar(MI.getDebugVariable(), MI.getDebugExpression(), + MI.getDebugLoc()->getInlinedAt()); + FragmentInfo ThisFragment = MIVar.getFragmentOrDefault(); + + // If this is the first sighting of this variable, then we are guaranteed + // there are currently no overlapping fragments either. Initialize the set + // of seen fragments, record no overlaps for the current one, and return. + auto SeenIt = SeenFragments.find(MIVar.getVariable()); + if (SeenIt == SeenFragments.end()) { + SmallSet<FragmentInfo, 4> OneFragment; + OneFragment.insert(ThisFragment); + SeenFragments.insert({MIVar.getVariable(), OneFragment}); + + OverlapFragments.insert({{MIVar.getVariable(), ThisFragment}, {}}); + return; + } + + // If this particular Variable/Fragment pair already exists in the overlap + // map, it has already been accounted for. + auto IsInOLapMap = + OverlapFragments.insert({{MIVar.getVariable(), ThisFragment}, {}}); + if (!IsInOLapMap.second) + return; + + auto &ThisFragmentsOverlaps = IsInOLapMap.first->second; + auto &AllSeenFragments = SeenIt->second; + + // Otherwise, examine all other seen fragments for this variable, with "this" + // fragment being a previously unseen fragment. Record any pair of + // overlapping fragments. + for (const auto &ASeenFragment : AllSeenFragments) { + // Does this previously seen fragment overlap? + if (DIExpression::fragmentsOverlap(ThisFragment, ASeenFragment)) { + // Yes: Mark the current fragment as being overlapped. + ThisFragmentsOverlaps.push_back(ASeenFragment); + // Mark the previously seen fragment as being overlapped by the current + // one. + auto ASeenFragmentsOverlaps = + OverlapFragments.find({MIVar.getVariable(), ASeenFragment}); + assert(ASeenFragmentsOverlaps != OverlapFragments.end() && + "Previously seen var fragment has no vector of overlaps"); + ASeenFragmentsOverlaps->second.push_back(ThisFragment); + } + } + + AllSeenFragments.insert(ThisFragment); +} + +void InstrRefBasedLDV::process(MachineInstr &MI, const ValueTable *MLiveOuts, + const ValueTable *MLiveIns) { + // Try to interpret an MI as a debug or transfer instruction. Only if it's + // none of these should we interpret it's register defs as new value + // definitions. + if (transferDebugValue(MI)) + return; + if (transferDebugInstrRef(MI, MLiveOuts, MLiveIns)) + return; + if (transferDebugPHI(MI)) + return; + if (transferRegisterCopy(MI)) + return; + if (transferSpillOrRestoreInst(MI)) + return; + transferRegisterDef(MI); +} + +void InstrRefBasedLDV::produceMLocTransferFunction( + MachineFunction &MF, SmallVectorImpl<MLocTransferMap> &MLocTransfer, + unsigned MaxNumBlocks) { + // Because we try to optimize around register mask operands by ignoring regs + // that aren't currently tracked, we set up something ugly for later: RegMask + // operands that are seen earlier than the first use of a register, still need + // to clobber that register in the transfer function. But this information + // isn't actively recorded. Instead, we track each RegMask used in each block, + // and accumulated the clobbered but untracked registers in each block into + // the following bitvector. Later, if new values are tracked, we can add + // appropriate clobbers. + SmallVector<BitVector, 32> BlockMasks; + BlockMasks.resize(MaxNumBlocks); + + // Reserve one bit per register for the masks described above. + unsigned BVWords = MachineOperand::getRegMaskSize(TRI->getNumRegs()); + for (auto &BV : BlockMasks) + BV.resize(TRI->getNumRegs(), true); + + // Step through all instructions and inhale the transfer function. + for (auto &MBB : MF) { + // Object fields that are read by trackers to know where we are in the + // function. + CurBB = MBB.getNumber(); + CurInst = 1; + + // Set all machine locations to a PHI value. For transfer function + // production only, this signifies the live-in value to the block. + MTracker->reset(); + MTracker->setMPhis(CurBB); + + // Step through each instruction in this block. + for (auto &MI : MBB) { + // Pass in an empty unique_ptr for the value tables when accumulating the + // machine transfer function. + process(MI, nullptr, nullptr); + + // Also accumulate fragment map. + if (MI.isDebugValueLike()) + accumulateFragmentMap(MI); + + // Create a map from the instruction number (if present) to the + // MachineInstr and its position. + if (uint64_t InstrNo = MI.peekDebugInstrNum()) { + auto InstrAndPos = std::make_pair(&MI, CurInst); + auto InsertResult = + DebugInstrNumToInstr.insert(std::make_pair(InstrNo, InstrAndPos)); + + // There should never be duplicate instruction numbers. + assert(InsertResult.second); + (void)InsertResult; + } + + ++CurInst; + } + + // Produce the transfer function, a map of machine location to new value. If + // any machine location has the live-in phi value from the start of the + // block, it's live-through and doesn't need recording in the transfer + // function. + for (auto Location : MTracker->locations()) { + LocIdx Idx = Location.Idx; + ValueIDNum &P = Location.Value; + if (P.isPHI() && P.getLoc() == Idx.asU64()) + continue; + + // Insert-or-update. + auto &TransferMap = MLocTransfer[CurBB]; + auto Result = TransferMap.insert(std::make_pair(Idx.asU64(), P)); + if (!Result.second) + Result.first->second = P; + } + + // Accumulate any bitmask operands into the clobbered reg mask for this + // block. + for (auto &P : MTracker->Masks) { + BlockMasks[CurBB].clearBitsNotInMask(P.first->getRegMask(), BVWords); + } + } + + // Compute a bitvector of all the registers that are tracked in this block. + BitVector UsedRegs(TRI->getNumRegs()); + for (auto Location : MTracker->locations()) { + unsigned ID = MTracker->LocIdxToLocID[Location.Idx]; + // Ignore stack slots, and aliases of the stack pointer. + if (ID >= TRI->getNumRegs() || MTracker->SPAliases.count(ID)) + continue; + UsedRegs.set(ID); + } + + // Check that any regmask-clobber of a register that gets tracked, is not + // live-through in the transfer function. It needs to be clobbered at the + // very least. + for (unsigned int I = 0; I < MaxNumBlocks; ++I) { + BitVector &BV = BlockMasks[I]; + BV.flip(); + BV &= UsedRegs; + // This produces all the bits that we clobber, but also use. Check that + // they're all clobbered or at least set in the designated transfer + // elem. + for (unsigned Bit : BV.set_bits()) { + unsigned ID = MTracker->getLocID(Bit); + LocIdx Idx = MTracker->LocIDToLocIdx[ID]; + auto &TransferMap = MLocTransfer[I]; + + // Install a value representing the fact that this location is effectively + // written to in this block. As there's no reserved value, instead use + // a value number that is never generated. Pick the value number for the + // first instruction in the block, def'ing this location, which we know + // this block never used anyway. + ValueIDNum NotGeneratedNum = ValueIDNum(I, 1, Idx); + auto Result = + TransferMap.insert(std::make_pair(Idx.asU64(), NotGeneratedNum)); + if (!Result.second) { + ValueIDNum &ValueID = Result.first->second; + if (ValueID.getBlock() == I && ValueID.isPHI()) + // It was left as live-through. Set it to clobbered. + ValueID = NotGeneratedNum; + } + } + } +} + +bool InstrRefBasedLDV::mlocJoin( + MachineBasicBlock &MBB, SmallPtrSet<const MachineBasicBlock *, 16> &Visited, + FuncValueTable &OutLocs, ValueTable &InLocs) { + LLVM_DEBUG(dbgs() << "join MBB: " << MBB.getNumber() << "\n"); + bool Changed = false; + + // Handle value-propagation when control flow merges on entry to a block. For + // any location without a PHI already placed, the location has the same value + // as its predecessors. If a PHI is placed, test to see whether it's now a + // redundant PHI that we can eliminate. + + SmallVector<const MachineBasicBlock *, 8> BlockOrders; + for (auto *Pred : MBB.predecessors()) + BlockOrders.push_back(Pred); + + // Visit predecessors in RPOT order. + auto Cmp = [&](const MachineBasicBlock *A, const MachineBasicBlock *B) { + return BBToOrder.find(A)->second < BBToOrder.find(B)->second; + }; + llvm::sort(BlockOrders, Cmp); + + // Skip entry block. + if (BlockOrders.size() == 0) + return false; + + // Step through all machine locations, look at each predecessor and test + // whether we can eliminate redundant PHIs. + for (auto Location : MTracker->locations()) { + LocIdx Idx = Location.Idx; + + // Pick out the first predecessors live-out value for this location. It's + // guaranteed to not be a backedge, as we order by RPO. + ValueIDNum FirstVal = OutLocs[BlockOrders[0]->getNumber()][Idx.asU64()]; + + // If we've already eliminated a PHI here, do no further checking, just + // propagate the first live-in value into this block. + if (InLocs[Idx.asU64()] != ValueIDNum(MBB.getNumber(), 0, Idx)) { + if (InLocs[Idx.asU64()] != FirstVal) { + InLocs[Idx.asU64()] = FirstVal; + Changed |= true; + } + continue; + } + + // We're now examining a PHI to see whether it's un-necessary. Loop around + // the other live-in values and test whether they're all the same. + bool Disagree = false; + for (unsigned int I = 1; I < BlockOrders.size(); ++I) { + const MachineBasicBlock *PredMBB = BlockOrders[I]; + const ValueIDNum &PredLiveOut = + OutLocs[PredMBB->getNumber()][Idx.asU64()]; + + // Incoming values agree, continue trying to eliminate this PHI. + if (FirstVal == PredLiveOut) + continue; + + // We can also accept a PHI value that feeds back into itself. + if (PredLiveOut == ValueIDNum(MBB.getNumber(), 0, Idx)) + continue; + + // Live-out of a predecessor disagrees with the first predecessor. + Disagree = true; + } + + // No disagreement? No PHI. Otherwise, leave the PHI in live-ins. + if (!Disagree) { + InLocs[Idx.asU64()] = FirstVal; + Changed |= true; + } + } + + // TODO: Reimplement NumInserted and NumRemoved. + return Changed; +} + +void InstrRefBasedLDV::findStackIndexInterference( + SmallVectorImpl<unsigned> &Slots) { + // We could spend a bit of time finding the exact, minimal, set of stack + // indexes that interfere with each other, much like reg units. Or, we can + // rely on the fact that: + // * The smallest / lowest index will interfere with everything at zero + // offset, which will be the largest set of registers, + // * Most indexes with non-zero offset will end up being interference units + // anyway. + // So just pick those out and return them. + + // We can rely on a single-byte stack index existing already, because we + // initialize them in MLocTracker. + auto It = MTracker->StackSlotIdxes.find({8, 0}); + assert(It != MTracker->StackSlotIdxes.end()); + Slots.push_back(It->second); + + // Find anything that has a non-zero offset and add that too. + for (auto &Pair : MTracker->StackSlotIdxes) { + // Is offset zero? If so, ignore. + if (!Pair.first.second) + continue; + Slots.push_back(Pair.second); + } +} + +void InstrRefBasedLDV::placeMLocPHIs( + MachineFunction &MF, SmallPtrSetImpl<MachineBasicBlock *> &AllBlocks, + FuncValueTable &MInLocs, SmallVectorImpl<MLocTransferMap> &MLocTransfer) { + SmallVector<unsigned, 4> StackUnits; + findStackIndexInterference(StackUnits); + + // To avoid repeatedly running the PHI placement algorithm, leverage the + // fact that a def of register MUST also def its register units. Find the + // units for registers, place PHIs for them, and then replicate them for + // aliasing registers. Some inputs that are never def'd (DBG_PHIs of + // arguments) don't lead to register units being tracked, just place PHIs for + // those registers directly. Stack slots have their own form of "unit", + // store them to one side. + SmallSet<Register, 32> RegUnitsToPHIUp; + SmallSet<LocIdx, 32> NormalLocsToPHI; + SmallSet<SpillLocationNo, 32> StackSlots; + for (auto Location : MTracker->locations()) { + LocIdx L = Location.Idx; + if (MTracker->isSpill(L)) { + StackSlots.insert(MTracker->locIDToSpill(MTracker->LocIdxToLocID[L])); + continue; + } + + Register R = MTracker->LocIdxToLocID[L]; + SmallSet<Register, 8> FoundRegUnits; + bool AnyIllegal = false; + for (MCRegUnitIterator RUI(R.asMCReg(), TRI); RUI.isValid(); ++RUI) { + for (MCRegUnitRootIterator URoot(*RUI, TRI); URoot.isValid(); ++URoot){ + if (!MTracker->isRegisterTracked(*URoot)) { + // Not all roots were loaded into the tracking map: this register + // isn't actually def'd anywhere, we only read from it. Generate PHIs + // for this reg, but don't iterate units. + AnyIllegal = true; + } else { + FoundRegUnits.insert(*URoot); + } + } + } + + if (AnyIllegal) { + NormalLocsToPHI.insert(L); + continue; + } + + RegUnitsToPHIUp.insert(FoundRegUnits.begin(), FoundRegUnits.end()); + } + + // Lambda to fetch PHIs for a given location, and write into the PHIBlocks + // collection. + SmallVector<MachineBasicBlock *, 32> PHIBlocks; + auto CollectPHIsForLoc = [&](LocIdx L) { + // Collect the set of defs. + SmallPtrSet<MachineBasicBlock *, 32> DefBlocks; + for (unsigned int I = 0; I < OrderToBB.size(); ++I) { + MachineBasicBlock *MBB = OrderToBB[I]; + const auto &TransferFunc = MLocTransfer[MBB->getNumber()]; + if (TransferFunc.find(L) != TransferFunc.end()) + DefBlocks.insert(MBB); + } + + // The entry block defs the location too: it's the live-in / argument value. + // Only insert if there are other defs though; everything is trivially live + // through otherwise. + if (!DefBlocks.empty()) + DefBlocks.insert(&*MF.begin()); + + // Ask the SSA construction algorithm where we should put PHIs. Clear + // anything that might have been hanging around from earlier. + PHIBlocks.clear(); + BlockPHIPlacement(AllBlocks, DefBlocks, PHIBlocks); + }; + + auto InstallPHIsAtLoc = [&PHIBlocks, &MInLocs](LocIdx L) { + for (const MachineBasicBlock *MBB : PHIBlocks) + MInLocs[MBB->getNumber()][L.asU64()] = ValueIDNum(MBB->getNumber(), 0, L); + }; + + // For locations with no reg units, just place PHIs. + for (LocIdx L : NormalLocsToPHI) { + CollectPHIsForLoc(L); + // Install those PHI values into the live-in value array. + InstallPHIsAtLoc(L); + } + + // For stack slots, calculate PHIs for the equivalent of the units, then + // install for each index. + for (SpillLocationNo Slot : StackSlots) { + for (unsigned Idx : StackUnits) { + unsigned SpillID = MTracker->getSpillIDWithIdx(Slot, Idx); + LocIdx L = MTracker->getSpillMLoc(SpillID); + CollectPHIsForLoc(L); + InstallPHIsAtLoc(L); + + // Find anything that aliases this stack index, install PHIs for it too. + unsigned Size, Offset; + std::tie(Size, Offset) = MTracker->StackIdxesToPos[Idx]; + for (auto &Pair : MTracker->StackSlotIdxes) { + unsigned ThisSize, ThisOffset; + std::tie(ThisSize, ThisOffset) = Pair.first; + if (ThisSize + ThisOffset <= Offset || Size + Offset <= ThisOffset) + continue; + + unsigned ThisID = MTracker->getSpillIDWithIdx(Slot, Pair.second); + LocIdx ThisL = MTracker->getSpillMLoc(ThisID); + InstallPHIsAtLoc(ThisL); + } + } + } + + // For reg units, place PHIs, and then place them for any aliasing registers. + for (Register R : RegUnitsToPHIUp) { + LocIdx L = MTracker->lookupOrTrackRegister(R); + CollectPHIsForLoc(L); + + // Install those PHI values into the live-in value array. + InstallPHIsAtLoc(L); + + // Now find aliases and install PHIs for those. + for (MCRegAliasIterator RAI(R, TRI, true); RAI.isValid(); ++RAI) { + // Super-registers that are "above" the largest register read/written by + // the function will alias, but will not be tracked. + if (!MTracker->isRegisterTracked(*RAI)) + continue; + + LocIdx AliasLoc = MTracker->lookupOrTrackRegister(*RAI); + InstallPHIsAtLoc(AliasLoc); + } + } +} + +void InstrRefBasedLDV::buildMLocValueMap( + MachineFunction &MF, FuncValueTable &MInLocs, FuncValueTable &MOutLocs, + SmallVectorImpl<MLocTransferMap> &MLocTransfer) { + std::priority_queue<unsigned int, std::vector<unsigned int>, + std::greater<unsigned int>> + Worklist, Pending; + + // We track what is on the current and pending worklist to avoid inserting + // the same thing twice. We could avoid this with a custom priority queue, + // but this is probably not worth it. + SmallPtrSet<MachineBasicBlock *, 16> OnPending, OnWorklist; + + // Initialize worklist with every block to be visited. Also produce list of + // all blocks. + SmallPtrSet<MachineBasicBlock *, 32> AllBlocks; + for (unsigned int I = 0; I < BBToOrder.size(); ++I) { + Worklist.push(I); + OnWorklist.insert(OrderToBB[I]); + AllBlocks.insert(OrderToBB[I]); + } + + // Initialize entry block to PHIs. These represent arguments. + for (auto Location : MTracker->locations()) + MInLocs[0][Location.Idx.asU64()] = ValueIDNum(0, 0, Location.Idx); + + MTracker->reset(); + + // Start by placing PHIs, using the usual SSA constructor algorithm. Consider + // any machine-location that isn't live-through a block to be def'd in that + // block. + placeMLocPHIs(MF, AllBlocks, MInLocs, MLocTransfer); + + // Propagate values to eliminate redundant PHIs. At the same time, this + // produces the table of Block x Location => Value for the entry to each + // block. + // The kind of PHIs we can eliminate are, for example, where one path in a + // conditional spills and restores a register, and the register still has + // the same value once control flow joins, unbeknowns to the PHI placement + // code. Propagating values allows us to identify such un-necessary PHIs and + // remove them. + SmallPtrSet<const MachineBasicBlock *, 16> Visited; + while (!Worklist.empty() || !Pending.empty()) { + // Vector for storing the evaluated block transfer function. + SmallVector<std::pair<LocIdx, ValueIDNum>, 32> ToRemap; + + while (!Worklist.empty()) { + MachineBasicBlock *MBB = OrderToBB[Worklist.top()]; + CurBB = MBB->getNumber(); + Worklist.pop(); + + // Join the values in all predecessor blocks. + bool InLocsChanged; + InLocsChanged = mlocJoin(*MBB, Visited, MOutLocs, MInLocs[CurBB]); + InLocsChanged |= Visited.insert(MBB).second; + + // Don't examine transfer function if we've visited this loc at least + // once, and inlocs haven't changed. + if (!InLocsChanged) + continue; + + // Load the current set of live-ins into MLocTracker. + MTracker->loadFromArray(MInLocs[CurBB], CurBB); + + // Each element of the transfer function can be a new def, or a read of + // a live-in value. Evaluate each element, and store to "ToRemap". + ToRemap.clear(); + for (auto &P : MLocTransfer[CurBB]) { + if (P.second.getBlock() == CurBB && P.second.isPHI()) { + // This is a movement of whatever was live in. Read it. + ValueIDNum NewID = MTracker->readMLoc(P.second.getLoc()); + ToRemap.push_back(std::make_pair(P.first, NewID)); + } else { + // It's a def. Just set it. + assert(P.second.getBlock() == CurBB); + ToRemap.push_back(std::make_pair(P.first, P.second)); + } + } + + // Commit the transfer function changes into mloc tracker, which + // transforms the contents of the MLocTracker into the live-outs. + for (auto &P : ToRemap) + MTracker->setMLoc(P.first, P.second); + + // Now copy out-locs from mloc tracker into out-loc vector, checking + // whether changes have occurred. These changes can have come from both + // the transfer function, and mlocJoin. + bool OLChanged = false; + for (auto Location : MTracker->locations()) { + OLChanged |= MOutLocs[CurBB][Location.Idx.asU64()] != Location.Value; + MOutLocs[CurBB][Location.Idx.asU64()] = Location.Value; + } + + MTracker->reset(); + + // No need to examine successors again if out-locs didn't change. + if (!OLChanged) + continue; + + // All successors should be visited: put any back-edges on the pending + // list for the next pass-through, and any other successors to be + // visited this pass, if they're not going to be already. + for (auto *s : MBB->successors()) { + // Does branching to this successor represent a back-edge? + if (BBToOrder[s] > BBToOrder[MBB]) { + // No: visit it during this dataflow iteration. + if (OnWorklist.insert(s).second) + Worklist.push(BBToOrder[s]); + } else { + // Yes: visit it on the next iteration. + if (OnPending.insert(s).second) + Pending.push(BBToOrder[s]); + } + } + } + + Worklist.swap(Pending); + std::swap(OnPending, OnWorklist); + OnPending.clear(); + // At this point, pending must be empty, since it was just the empty + // worklist + assert(Pending.empty() && "Pending should be empty"); + } + + // Once all the live-ins don't change on mlocJoin(), we've eliminated all + // redundant PHIs. +} + +void InstrRefBasedLDV::BlockPHIPlacement( + const SmallPtrSetImpl<MachineBasicBlock *> &AllBlocks, + const SmallPtrSetImpl<MachineBasicBlock *> &DefBlocks, + SmallVectorImpl<MachineBasicBlock *> &PHIBlocks) { + // Apply IDF calculator to the designated set of location defs, storing + // required PHIs into PHIBlocks. Uses the dominator tree stored in the + // InstrRefBasedLDV object. + IDFCalculatorBase<MachineBasicBlock, false> IDF(DomTree->getBase()); + + IDF.setLiveInBlocks(AllBlocks); + IDF.setDefiningBlocks(DefBlocks); + IDF.calculate(PHIBlocks); +} + +bool InstrRefBasedLDV::pickVPHILoc( + SmallVectorImpl<DbgOpID> &OutValues, const MachineBasicBlock &MBB, + const LiveIdxT &LiveOuts, FuncValueTable &MOutLocs, + const SmallVectorImpl<const MachineBasicBlock *> &BlockOrders) { + + // No predecessors means no PHIs. + if (BlockOrders.empty()) + return false; + + // All the location operands that do not already agree need to be joined, + // track the indices of each such location operand here. + SmallDenseSet<unsigned> LocOpsToJoin; + + auto FirstValueIt = LiveOuts.find(BlockOrders[0]); + if (FirstValueIt == LiveOuts.end()) + return false; + const DbgValue &FirstValue = *FirstValueIt->second; + + for (const auto p : BlockOrders) { + auto OutValIt = LiveOuts.find(p); + if (OutValIt == LiveOuts.end()) + // If we have a predecessor not in scope, we'll never find a PHI position. + return false; + const DbgValue &OutVal = *OutValIt->second; + + // No-values cannot have locations we can join on. + if (OutVal.Kind == DbgValue::NoVal) + return false; + + // For unjoined VPHIs where we don't know the location, we definitely + // can't find a join loc unless the VPHI is a backedge. + if (OutVal.isUnjoinedPHI() && OutVal.BlockNo != MBB.getNumber()) + return false; + + if (!FirstValue.Properties.isJoinable(OutVal.Properties)) + return false; + + for (unsigned Idx = 0; Idx < FirstValue.getLocationOpCount(); ++Idx) { + // An unjoined PHI has no defined locations, and so a shared location must + // be found for every operand. + if (OutVal.isUnjoinedPHI()) { + LocOpsToJoin.insert(Idx); + continue; + } + DbgOpID FirstValOp = FirstValue.getDbgOpID(Idx); + DbgOpID OutValOp = OutVal.getDbgOpID(Idx); + if (FirstValOp != OutValOp) { + // We can never join constant ops - the ops must either both be equal + // constant ops or non-const ops. + if (FirstValOp.isConst() || OutValOp.isConst()) + return false; + else + LocOpsToJoin.insert(Idx); + } + } + } + + SmallVector<DbgOpID> NewDbgOps; + + for (unsigned Idx = 0; Idx < FirstValue.getLocationOpCount(); ++Idx) { + // If this op doesn't need to be joined because the values agree, use that + // already-agreed value. + if (!LocOpsToJoin.contains(Idx)) { + NewDbgOps.push_back(FirstValue.getDbgOpID(Idx)); + continue; + } + + std::optional<ValueIDNum> JoinedOpLoc = + pickOperandPHILoc(Idx, MBB, LiveOuts, MOutLocs, BlockOrders); + + if (!JoinedOpLoc) + return false; + + NewDbgOps.push_back(DbgOpStore.insert(*JoinedOpLoc)); + } + + OutValues.append(NewDbgOps); + return true; +} + +std::optional<ValueIDNum> InstrRefBasedLDV::pickOperandPHILoc( + unsigned DbgOpIdx, const MachineBasicBlock &MBB, const LiveIdxT &LiveOuts, + FuncValueTable &MOutLocs, + const SmallVectorImpl<const MachineBasicBlock *> &BlockOrders) { + + // Collect a set of locations from predecessor where its live-out value can + // be found. + SmallVector<SmallVector<LocIdx, 4>, 8> Locs; + unsigned NumLocs = MTracker->getNumLocs(); + + for (const auto p : BlockOrders) { + unsigned ThisBBNum = p->getNumber(); + auto OutValIt = LiveOuts.find(p); + assert(OutValIt != LiveOuts.end()); + const DbgValue &OutVal = *OutValIt->second; + DbgOpID OutValOpID = OutVal.getDbgOpID(DbgOpIdx); + DbgOp OutValOp = DbgOpStore.find(OutValOpID); + assert(!OutValOp.IsConst); + + // Create new empty vector of locations. + Locs.resize(Locs.size() + 1); + + // If the live-in value is a def, find the locations where that value is + // present. Do the same for VPHIs where we know the VPHI value. + if (OutVal.Kind == DbgValue::Def || + (OutVal.Kind == DbgValue::VPHI && OutVal.BlockNo != MBB.getNumber() && + !OutValOp.isUndef())) { + ValueIDNum ValToLookFor = OutValOp.ID; + // Search the live-outs of the predecessor for the specified value. + for (unsigned int I = 0; I < NumLocs; ++I) { + if (MOutLocs[ThisBBNum][I] == ValToLookFor) + Locs.back().push_back(LocIdx(I)); + } + } else { + assert(OutVal.Kind == DbgValue::VPHI); + // Otherwise: this is a VPHI on a backedge feeding back into itself, i.e. + // a value that's live-through the whole loop. (It has to be a backedge, + // because a block can't dominate itself). We can accept as a PHI location + // any location where the other predecessors agree, _and_ the machine + // locations feed back into themselves. Therefore, add all self-looping + // machine-value PHI locations. + for (unsigned int I = 0; I < NumLocs; ++I) { + ValueIDNum MPHI(MBB.getNumber(), 0, LocIdx(I)); + if (MOutLocs[ThisBBNum][I] == MPHI) + Locs.back().push_back(LocIdx(I)); + } + } + } + // We should have found locations for all predecessors, or returned. + assert(Locs.size() == BlockOrders.size()); + + // Starting with the first set of locations, take the intersection with + // subsequent sets. + SmallVector<LocIdx, 4> CandidateLocs = Locs[0]; + for (unsigned int I = 1; I < Locs.size(); ++I) { + auto &LocVec = Locs[I]; + SmallVector<LocIdx, 4> NewCandidates; + std::set_intersection(CandidateLocs.begin(), CandidateLocs.end(), + LocVec.begin(), LocVec.end(), std::inserter(NewCandidates, NewCandidates.begin())); + CandidateLocs = NewCandidates; + } + if (CandidateLocs.empty()) + return std::nullopt; + + // We now have a set of LocIdxes that contain the right output value in + // each of the predecessors. Pick the lowest; if there's a register loc, + // that'll be it. + LocIdx L = *CandidateLocs.begin(); + + // Return a PHI-value-number for the found location. + ValueIDNum PHIVal = {(unsigned)MBB.getNumber(), 0, L}; + return PHIVal; +} + +bool InstrRefBasedLDV::vlocJoin( + MachineBasicBlock &MBB, LiveIdxT &VLOCOutLocs, + SmallPtrSet<const MachineBasicBlock *, 8> &BlocksToExplore, + DbgValue &LiveIn) { + LLVM_DEBUG(dbgs() << "join MBB: " << MBB.getNumber() << "\n"); + bool Changed = false; + + // Order predecessors by RPOT order, for exploring them in that order. + SmallVector<MachineBasicBlock *, 8> BlockOrders(MBB.predecessors()); + + auto Cmp = [&](MachineBasicBlock *A, MachineBasicBlock *B) { + return BBToOrder[A] < BBToOrder[B]; + }; + + llvm::sort(BlockOrders, Cmp); + + unsigned CurBlockRPONum = BBToOrder[&MBB]; + + // Collect all the incoming DbgValues for this variable, from predecessor + // live-out values. + SmallVector<InValueT, 8> Values; + bool Bail = false; + int BackEdgesStart = 0; + for (auto *p : BlockOrders) { + // If the predecessor isn't in scope / to be explored, we'll never be + // able to join any locations. + if (!BlocksToExplore.contains(p)) { + Bail = true; + break; + } + + // All Live-outs will have been initialized. + DbgValue &OutLoc = *VLOCOutLocs.find(p)->second; + + // Keep track of where back-edges begin in the Values vector. Relies on + // BlockOrders being sorted by RPO. + unsigned ThisBBRPONum = BBToOrder[p]; + if (ThisBBRPONum < CurBlockRPONum) + ++BackEdgesStart; + + Values.push_back(std::make_pair(p, &OutLoc)); + } + + // If there were no values, or one of the predecessors couldn't have a + // value, then give up immediately. It's not safe to produce a live-in + // value. Leave as whatever it was before. + if (Bail || Values.size() == 0) + return false; + + // All (non-entry) blocks have at least one non-backedge predecessor. + // Pick the variable value from the first of these, to compare against + // all others. + const DbgValue &FirstVal = *Values[0].second; + + // If the old live-in value is not a PHI then either a) no PHI is needed + // here, or b) we eliminated the PHI that was here. If so, we can just + // propagate in the first parent's incoming value. + if (LiveIn.Kind != DbgValue::VPHI || LiveIn.BlockNo != MBB.getNumber()) { + Changed = LiveIn != FirstVal; + if (Changed) + LiveIn = FirstVal; + return Changed; + } + + // Scan for variable values that can never be resolved: if they have + // different DIExpressions, different indirectness, or are mixed constants / + // non-constants. + for (const auto &V : Values) { + if (!V.second->Properties.isJoinable(FirstVal.Properties)) + return false; + if (V.second->Kind == DbgValue::NoVal) + return false; + if (!V.second->hasJoinableLocOps(FirstVal)) + return false; + } + + // Try to eliminate this PHI. Do the incoming values all agree? + bool Disagree = false; + for (auto &V : Values) { + if (*V.second == FirstVal) + continue; // No disagreement. + + // If both values are not equal but have equal non-empty IDs then they refer + // to the same value from different sources (e.g. one is VPHI and the other + // is Def), which does not cause disagreement. + if (V.second->hasIdenticalValidLocOps(FirstVal)) + continue; + + // Eliminate if a backedge feeds a VPHI back into itself. + if (V.second->Kind == DbgValue::VPHI && + V.second->BlockNo == MBB.getNumber() && + // Is this a backedge? + std::distance(Values.begin(), &V) >= BackEdgesStart) + continue; + + Disagree = true; + } + + // No disagreement -> live-through value. + if (!Disagree) { + Changed = LiveIn != FirstVal; + if (Changed) + LiveIn = FirstVal; + return Changed; + } else { + // Otherwise use a VPHI. + DbgValue VPHI(MBB.getNumber(), FirstVal.Properties, DbgValue::VPHI); + Changed = LiveIn != VPHI; + if (Changed) + LiveIn = VPHI; + return Changed; + } +} + +void InstrRefBasedLDV::getBlocksForScope( + const DILocation *DILoc, + SmallPtrSetImpl<const MachineBasicBlock *> &BlocksToExplore, + const SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks) { + // Get the set of "normal" in-lexical-scope blocks. + LS.getMachineBasicBlocks(DILoc, BlocksToExplore); + + // VarLoc LiveDebugValues tracks variable locations that are defined in + // blocks not in scope. This is something we could legitimately ignore, but + // lets allow it for now for the sake of coverage. + BlocksToExplore.insert(AssignBlocks.begin(), AssignBlocks.end()); + + // Storage for artificial blocks we intend to add to BlocksToExplore. + DenseSet<const MachineBasicBlock *> ToAdd; + + // To avoid needlessly dropping large volumes of variable locations, propagate + // variables through aritifical blocks, i.e. those that don't have any + // instructions in scope at all. To accurately replicate VarLoc + // LiveDebugValues, this means exploring all artificial successors too. + // Perform a depth-first-search to enumerate those blocks. + for (const auto *MBB : BlocksToExplore) { + // Depth-first-search state: each node is a block and which successor + // we're currently exploring. + SmallVector<std::pair<const MachineBasicBlock *, + MachineBasicBlock::const_succ_iterator>, + 8> + DFS; + + // Find any artificial successors not already tracked. + for (auto *succ : MBB->successors()) { + if (BlocksToExplore.count(succ)) + continue; + if (!ArtificialBlocks.count(succ)) + continue; + ToAdd.insert(succ); + DFS.push_back({succ, succ->succ_begin()}); + } + + // Search all those blocks, depth first. + while (!DFS.empty()) { + const MachineBasicBlock *CurBB = DFS.back().first; + MachineBasicBlock::const_succ_iterator &CurSucc = DFS.back().second; + // Walk back if we've explored this blocks successors to the end. + if (CurSucc == CurBB->succ_end()) { + DFS.pop_back(); + continue; + } + + // If the current successor is artificial and unexplored, descend into + // it. + if (!ToAdd.count(*CurSucc) && ArtificialBlocks.count(*CurSucc)) { + ToAdd.insert(*CurSucc); + DFS.push_back({*CurSucc, (*CurSucc)->succ_begin()}); + continue; + } + + ++CurSucc; + } + }; + + BlocksToExplore.insert(ToAdd.begin(), ToAdd.end()); +} + +void InstrRefBasedLDV::buildVLocValueMap( + const DILocation *DILoc, const SmallSet<DebugVariable, 4> &VarsWeCareAbout, + SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks, LiveInsT &Output, + FuncValueTable &MOutLocs, FuncValueTable &MInLocs, + SmallVectorImpl<VLocTracker> &AllTheVLocs) { + // This method is much like buildMLocValueMap: but focuses on a single + // LexicalScope at a time. Pick out a set of blocks and variables that are + // to have their value assignments solved, then run our dataflow algorithm + // until a fixedpoint is reached. + std::priority_queue<unsigned int, std::vector<unsigned int>, + std::greater<unsigned int>> + Worklist, Pending; + SmallPtrSet<MachineBasicBlock *, 16> OnWorklist, OnPending; + + // The set of blocks we'll be examining. + SmallPtrSet<const MachineBasicBlock *, 8> BlocksToExplore; + + // The order in which to examine them (RPO). + SmallVector<MachineBasicBlock *, 8> BlockOrders; + + // RPO ordering function. + auto Cmp = [&](MachineBasicBlock *A, MachineBasicBlock *B) { + return BBToOrder[A] < BBToOrder[B]; + }; + + getBlocksForScope(DILoc, BlocksToExplore, AssignBlocks); + + // Single block scope: not interesting! No propagation at all. Note that + // this could probably go above ArtificialBlocks without damage, but + // that then produces output differences from original-live-debug-values, + // which propagates from a single block into many artificial ones. + if (BlocksToExplore.size() == 1) + return; + + // Convert a const set to a non-const set. LexicalScopes + // getMachineBasicBlocks returns const MBB pointers, IDF wants mutable ones. + // (Neither of them mutate anything). + SmallPtrSet<MachineBasicBlock *, 8> MutBlocksToExplore; + for (const auto *MBB : BlocksToExplore) + MutBlocksToExplore.insert(const_cast<MachineBasicBlock *>(MBB)); + + // Picks out relevants blocks RPO order and sort them. + for (const auto *MBB : BlocksToExplore) + BlockOrders.push_back(const_cast<MachineBasicBlock *>(MBB)); + + llvm::sort(BlockOrders, Cmp); + unsigned NumBlocks = BlockOrders.size(); + + // Allocate some vectors for storing the live ins and live outs. Large. + SmallVector<DbgValue, 32> LiveIns, LiveOuts; + LiveIns.reserve(NumBlocks); + LiveOuts.reserve(NumBlocks); + + // Initialize all values to start as NoVals. This signifies "it's live + // through, but we don't know what it is". + DbgValueProperties EmptyProperties(EmptyExpr, false, false); + for (unsigned int I = 0; I < NumBlocks; ++I) { + DbgValue EmptyDbgValue(I, EmptyProperties, DbgValue::NoVal); + LiveIns.push_back(EmptyDbgValue); + LiveOuts.push_back(EmptyDbgValue); + } + + // Produce by-MBB indexes of live-in/live-outs, to ease lookup within + // vlocJoin. + LiveIdxT LiveOutIdx, LiveInIdx; + LiveOutIdx.reserve(NumBlocks); + LiveInIdx.reserve(NumBlocks); + for (unsigned I = 0; I < NumBlocks; ++I) { + LiveOutIdx[BlockOrders[I]] = &LiveOuts[I]; + LiveInIdx[BlockOrders[I]] = &LiveIns[I]; + } + + // Loop over each variable and place PHIs for it, then propagate values + // between blocks. This keeps the locality of working on one lexical scope at + // at time, but avoids re-processing variable values because some other + // variable has been assigned. + for (const auto &Var : VarsWeCareAbout) { + // Re-initialize live-ins and live-outs, to clear the remains of previous + // variables live-ins / live-outs. + for (unsigned int I = 0; I < NumBlocks; ++I) { + DbgValue EmptyDbgValue(I, EmptyProperties, DbgValue::NoVal); + LiveIns[I] = EmptyDbgValue; + LiveOuts[I] = EmptyDbgValue; + } + + // Place PHIs for variable values, using the LLVM IDF calculator. + // Collect the set of blocks where variables are def'd. + SmallPtrSet<MachineBasicBlock *, 32> DefBlocks; + for (const MachineBasicBlock *ExpMBB : BlocksToExplore) { + auto &TransferFunc = AllTheVLocs[ExpMBB->getNumber()].Vars; + if (TransferFunc.find(Var) != TransferFunc.end()) + DefBlocks.insert(const_cast<MachineBasicBlock *>(ExpMBB)); + } + + SmallVector<MachineBasicBlock *, 32> PHIBlocks; + + // Request the set of PHIs we should insert for this variable. If there's + // only one value definition, things are very simple. + if (DefBlocks.size() == 1) { + placePHIsForSingleVarDefinition(MutBlocksToExplore, *DefBlocks.begin(), + AllTheVLocs, Var, Output); + continue; + } + + // Otherwise: we need to place PHIs through SSA and propagate values. + BlockPHIPlacement(MutBlocksToExplore, DefBlocks, PHIBlocks); + + // Insert PHIs into the per-block live-in tables for this variable. + for (MachineBasicBlock *PHIMBB : PHIBlocks) { + unsigned BlockNo = PHIMBB->getNumber(); + DbgValue *LiveIn = LiveInIdx[PHIMBB]; + *LiveIn = DbgValue(BlockNo, EmptyProperties, DbgValue::VPHI); + } + + for (auto *MBB : BlockOrders) { + Worklist.push(BBToOrder[MBB]); + OnWorklist.insert(MBB); + } + + // Iterate over all the blocks we selected, propagating the variables value. + // This loop does two things: + // * Eliminates un-necessary VPHIs in vlocJoin, + // * Evaluates the blocks transfer function (i.e. variable assignments) and + // stores the result to the blocks live-outs. + // Always evaluate the transfer function on the first iteration, and when + // the live-ins change thereafter. + bool FirstTrip = true; + while (!Worklist.empty() || !Pending.empty()) { + while (!Worklist.empty()) { + auto *MBB = OrderToBB[Worklist.top()]; + CurBB = MBB->getNumber(); + Worklist.pop(); + + auto LiveInsIt = LiveInIdx.find(MBB); + assert(LiveInsIt != LiveInIdx.end()); + DbgValue *LiveIn = LiveInsIt->second; + + // Join values from predecessors. Updates LiveInIdx, and writes output + // into JoinedInLocs. + bool InLocsChanged = + vlocJoin(*MBB, LiveOutIdx, BlocksToExplore, *LiveIn); + + SmallVector<const MachineBasicBlock *, 8> Preds; + for (const auto *Pred : MBB->predecessors()) + Preds.push_back(Pred); + + // If this block's live-in value is a VPHI, try to pick a machine-value + // for it. This makes the machine-value available and propagated + // through all blocks by the time value propagation finishes. We can't + // do this any earlier as it needs to read the block live-outs. + if (LiveIn->Kind == DbgValue::VPHI && LiveIn->BlockNo == (int)CurBB) { + // There's a small possibility that on a preceeding path, a VPHI is + // eliminated and transitions from VPHI-with-location to + // live-through-value. As a result, the selected location of any VPHI + // might change, so we need to re-compute it on each iteration. + SmallVector<DbgOpID> JoinedOps; + + if (pickVPHILoc(JoinedOps, *MBB, LiveOutIdx, MOutLocs, Preds)) { + bool NewLocPicked = !equal(LiveIn->getDbgOpIDs(), JoinedOps); + InLocsChanged |= NewLocPicked; + if (NewLocPicked) + LiveIn->setDbgOpIDs(JoinedOps); + } + } + + if (!InLocsChanged && !FirstTrip) + continue; + + DbgValue *LiveOut = LiveOutIdx[MBB]; + bool OLChanged = false; + + // Do transfer function. + auto &VTracker = AllTheVLocs[MBB->getNumber()]; + auto TransferIt = VTracker.Vars.find(Var); + if (TransferIt != VTracker.Vars.end()) { + // Erase on empty transfer (DBG_VALUE $noreg). + if (TransferIt->second.Kind == DbgValue::Undef) { + DbgValue NewVal(MBB->getNumber(), EmptyProperties, DbgValue::NoVal); + if (*LiveOut != NewVal) { + *LiveOut = NewVal; + OLChanged = true; + } + } else { + // Insert new variable value; or overwrite. + if (*LiveOut != TransferIt->second) { + *LiveOut = TransferIt->second; + OLChanged = true; + } + } + } else { + // Just copy live-ins to live-outs, for anything not transferred. + if (*LiveOut != *LiveIn) { + *LiveOut = *LiveIn; + OLChanged = true; + } + } + + // If no live-out value changed, there's no need to explore further. + if (!OLChanged) + continue; + + // We should visit all successors. Ensure we'll visit any non-backedge + // successors during this dataflow iteration; book backedge successors + // to be visited next time around. + for (auto *s : MBB->successors()) { + // Ignore out of scope / not-to-be-explored successors. + if (LiveInIdx.find(s) == LiveInIdx.end()) + continue; + + if (BBToOrder[s] > BBToOrder[MBB]) { + if (OnWorklist.insert(s).second) + Worklist.push(BBToOrder[s]); + } else if (OnPending.insert(s).second && (FirstTrip || OLChanged)) { + Pending.push(BBToOrder[s]); + } + } + } + Worklist.swap(Pending); + std::swap(OnWorklist, OnPending); + OnPending.clear(); + assert(Pending.empty()); + FirstTrip = false; + } + + // Save live-ins to output vector. Ignore any that are still marked as being + // VPHIs with no location -- those are variables that we know the value of, + // but are not actually available in the register file. + for (auto *MBB : BlockOrders) { + DbgValue *BlockLiveIn = LiveInIdx[MBB]; + if (BlockLiveIn->Kind == DbgValue::NoVal) + continue; + if (BlockLiveIn->isUnjoinedPHI()) + continue; + if (BlockLiveIn->Kind == DbgValue::VPHI) + BlockLiveIn->Kind = DbgValue::Def; + assert(BlockLiveIn->Properties.DIExpr->getFragmentInfo() == + Var.getFragment() && "Fragment info missing during value prop"); + Output[MBB->getNumber()].push_back(std::make_pair(Var, *BlockLiveIn)); + } + } // Per-variable loop. + + BlockOrders.clear(); + BlocksToExplore.clear(); +} + +void InstrRefBasedLDV::placePHIsForSingleVarDefinition( + const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks, + MachineBasicBlock *AssignMBB, SmallVectorImpl<VLocTracker> &AllTheVLocs, + const DebugVariable &Var, LiveInsT &Output) { + // If there is a single definition of the variable, then working out it's + // value everywhere is very simple: it's every block dominated by the + // definition. At the dominance frontier, the usual algorithm would: + // * Place PHIs, + // * Propagate values into them, + // * Find there's no incoming variable value from the other incoming branches + // of the dominance frontier, + // * Specify there's no variable value in blocks past the frontier. + // This is a common case, hence it's worth special-casing it. + + // Pick out the variables value from the block transfer function. + VLocTracker &VLocs = AllTheVLocs[AssignMBB->getNumber()]; + auto ValueIt = VLocs.Vars.find(Var); + const DbgValue &Value = ValueIt->second; + + // If it's an explicit assignment of "undef", that means there is no location + // anyway, anywhere. + if (Value.Kind == DbgValue::Undef) + return; + + // Assign the variable value to entry to each dominated block that's in scope. + // Skip the definition block -- it's assigned the variable value in the middle + // of the block somewhere. + for (auto *ScopeBlock : InScopeBlocks) { + if (!DomTree->properlyDominates(AssignMBB, ScopeBlock)) + continue; + + Output[ScopeBlock->getNumber()].push_back({Var, Value}); + } + + // All blocks that aren't dominated have no live-in value, thus no variable + // value will be given to them. +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void InstrRefBasedLDV::dump_mloc_transfer( + const MLocTransferMap &mloc_transfer) const { + for (const auto &P : mloc_transfer) { + std::string foo = MTracker->LocIdxToName(P.first); + std::string bar = MTracker->IDAsString(P.second); + dbgs() << "Loc " << foo << " --> " << bar << "\n"; + } +} +#endif + +void InstrRefBasedLDV::initialSetup(MachineFunction &MF) { + // Build some useful data structures. + + LLVMContext &Context = MF.getFunction().getContext(); + EmptyExpr = DIExpression::get(Context, {}); + + auto hasNonArtificialLocation = [](const MachineInstr &MI) -> bool { + if (const DebugLoc &DL = MI.getDebugLoc()) + return DL.getLine() != 0; + return false; + }; + // Collect a set of all the artificial blocks. + for (auto &MBB : MF) + if (none_of(MBB.instrs(), hasNonArtificialLocation)) + ArtificialBlocks.insert(&MBB); + + // Compute mappings of block <=> RPO order. + ReversePostOrderTraversal<MachineFunction *> RPOT(&MF); + unsigned int RPONumber = 0; + auto processMBB = [&](MachineBasicBlock *MBB) { + OrderToBB[RPONumber] = MBB; + BBToOrder[MBB] = RPONumber; + BBNumToRPO[MBB->getNumber()] = RPONumber; + ++RPONumber; + }; + for (MachineBasicBlock *MBB : RPOT) + processMBB(MBB); + for (MachineBasicBlock &MBB : MF) + if (BBToOrder.find(&MBB) == BBToOrder.end()) + processMBB(&MBB); + + // Order value substitutions by their "source" operand pair, for quick lookup. + llvm::sort(MF.DebugValueSubstitutions); + +#ifdef EXPENSIVE_CHECKS + // As an expensive check, test whether there are any duplicate substitution + // sources in the collection. + if (MF.DebugValueSubstitutions.size() > 2) { + for (auto It = MF.DebugValueSubstitutions.begin(); + It != std::prev(MF.DebugValueSubstitutions.end()); ++It) { + assert(It->Src != std::next(It)->Src && "Duplicate variable location " + "substitution seen"); + } + } +#endif +} + +// Produce an "ejection map" for blocks, i.e., what's the highest-numbered +// lexical scope it's used in. When exploring in DFS order and we pass that +// scope, the block can be processed and any tracking information freed. +void InstrRefBasedLDV::makeDepthFirstEjectionMap( + SmallVectorImpl<unsigned> &EjectionMap, + const ScopeToDILocT &ScopeToDILocation, + ScopeToAssignBlocksT &ScopeToAssignBlocks) { + SmallPtrSet<const MachineBasicBlock *, 8> BlocksToExplore; + SmallVector<std::pair<LexicalScope *, ssize_t>, 4> WorkStack; + auto *TopScope = LS.getCurrentFunctionScope(); + + // Unlike lexical scope explorers, we explore in reverse order, to find the + // "last" lexical scope used for each block early. + WorkStack.push_back({TopScope, TopScope->getChildren().size() - 1}); + + while (!WorkStack.empty()) { + auto &ScopePosition = WorkStack.back(); + LexicalScope *WS = ScopePosition.first; + ssize_t ChildNum = ScopePosition.second--; + + const SmallVectorImpl<LexicalScope *> &Children = WS->getChildren(); + if (ChildNum >= 0) { + // If ChildNum is positive, there are remaining children to explore. + // Push the child and its children-count onto the stack. + auto &ChildScope = Children[ChildNum]; + WorkStack.push_back( + std::make_pair(ChildScope, ChildScope->getChildren().size() - 1)); + } else { + WorkStack.pop_back(); + + // We've explored all children and any later blocks: examine all blocks + // in our scope. If they haven't yet had an ejection number set, then + // this scope will be the last to use that block. + auto DILocationIt = ScopeToDILocation.find(WS); + if (DILocationIt != ScopeToDILocation.end()) { + getBlocksForScope(DILocationIt->second, BlocksToExplore, + ScopeToAssignBlocks.find(WS)->second); + for (const auto *MBB : BlocksToExplore) { + unsigned BBNum = MBB->getNumber(); + if (EjectionMap[BBNum] == 0) + EjectionMap[BBNum] = WS->getDFSOut(); + } + + BlocksToExplore.clear(); + } + } + } +} + +bool InstrRefBasedLDV::depthFirstVLocAndEmit( + unsigned MaxNumBlocks, const ScopeToDILocT &ScopeToDILocation, + const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToAssignBlocks, + LiveInsT &Output, FuncValueTable &MOutLocs, FuncValueTable &MInLocs, + SmallVectorImpl<VLocTracker> &AllTheVLocs, MachineFunction &MF, + DenseMap<DebugVariable, unsigned> &AllVarsNumbering, + const TargetPassConfig &TPC) { + TTracker = new TransferTracker(TII, MTracker, MF, *TRI, CalleeSavedRegs, TPC); + unsigned NumLocs = MTracker->getNumLocs(); + VTracker = nullptr; + + // No scopes? No variable locations. + if (!LS.getCurrentFunctionScope()) + return false; + + // Build map from block number to the last scope that uses the block. + SmallVector<unsigned, 16> EjectionMap; + EjectionMap.resize(MaxNumBlocks, 0); + makeDepthFirstEjectionMap(EjectionMap, ScopeToDILocation, + ScopeToAssignBlocks); + + // Helper lambda for ejecting a block -- if nothing is going to use the block, + // we can translate the variable location information into DBG_VALUEs and then + // free all of InstrRefBasedLDV's data structures. + auto EjectBlock = [&](MachineBasicBlock &MBB) -> void { + unsigned BBNum = MBB.getNumber(); + AllTheVLocs[BBNum].clear(); + + // Prime the transfer-tracker, and then step through all the block + // instructions, installing transfers. + MTracker->reset(); + MTracker->loadFromArray(MInLocs[BBNum], BBNum); + TTracker->loadInlocs(MBB, MInLocs[BBNum], DbgOpStore, Output[BBNum], + NumLocs); + + CurBB = BBNum; + CurInst = 1; + for (auto &MI : MBB) { + process(MI, MOutLocs.get(), MInLocs.get()); + TTracker->checkInstForNewValues(CurInst, MI.getIterator()); + ++CurInst; + } + + // Free machine-location tables for this block. + MInLocs[BBNum].reset(); + MOutLocs[BBNum].reset(); + // We don't need live-in variable values for this block either. + Output[BBNum].clear(); + AllTheVLocs[BBNum].clear(); + }; + + SmallPtrSet<const MachineBasicBlock *, 8> BlocksToExplore; + SmallVector<std::pair<LexicalScope *, ssize_t>, 4> WorkStack; + WorkStack.push_back({LS.getCurrentFunctionScope(), 0}); + unsigned HighestDFSIn = 0; + + // Proceed to explore in depth first order. + while (!WorkStack.empty()) { + auto &ScopePosition = WorkStack.back(); + LexicalScope *WS = ScopePosition.first; + ssize_t ChildNum = ScopePosition.second++; + + // We obesrve scopes with children twice here, once descending in, once + // ascending out of the scope nest. Use HighestDFSIn as a ratchet to ensure + // we don't process a scope twice. Additionally, ignore scopes that don't + // have a DILocation -- by proxy, this means we never tracked any variable + // assignments in that scope. + auto DILocIt = ScopeToDILocation.find(WS); + if (HighestDFSIn <= WS->getDFSIn() && DILocIt != ScopeToDILocation.end()) { + const DILocation *DILoc = DILocIt->second; + auto &VarsWeCareAbout = ScopeToVars.find(WS)->second; + auto &BlocksInScope = ScopeToAssignBlocks.find(WS)->second; + + buildVLocValueMap(DILoc, VarsWeCareAbout, BlocksInScope, Output, MOutLocs, + MInLocs, AllTheVLocs); + } + + HighestDFSIn = std::max(HighestDFSIn, WS->getDFSIn()); + + // Descend into any scope nests. + const SmallVectorImpl<LexicalScope *> &Children = WS->getChildren(); + if (ChildNum < (ssize_t)Children.size()) { + // There are children to explore -- push onto stack and continue. + auto &ChildScope = Children[ChildNum]; + WorkStack.push_back(std::make_pair(ChildScope, 0)); + } else { + WorkStack.pop_back(); + + // We've explored a leaf, or have explored all the children of a scope. + // Try to eject any blocks where this is the last scope it's relevant to. + auto DILocationIt = ScopeToDILocation.find(WS); + if (DILocationIt == ScopeToDILocation.end()) + continue; + + getBlocksForScope(DILocationIt->second, BlocksToExplore, + ScopeToAssignBlocks.find(WS)->second); + for (const auto *MBB : BlocksToExplore) + if (WS->getDFSOut() == EjectionMap[MBB->getNumber()]) + EjectBlock(const_cast<MachineBasicBlock &>(*MBB)); + + BlocksToExplore.clear(); + } + } + + // Some artificial blocks may not have been ejected, meaning they're not + // connected to an actual legitimate scope. This can technically happen + // with things like the entry block. In theory, we shouldn't need to do + // anything for such out-of-scope blocks, but for the sake of being similar + // to VarLocBasedLDV, eject these too. + for (auto *MBB : ArtificialBlocks) + if (MOutLocs[MBB->getNumber()]) + EjectBlock(*MBB); + + return emitTransfers(AllVarsNumbering); +} + +bool InstrRefBasedLDV::emitTransfers( + DenseMap<DebugVariable, unsigned> &AllVarsNumbering) { + // Go through all the transfers recorded in the TransferTracker -- this is + // both the live-ins to a block, and any movements of values that happen + // in the middle. + for (const auto &P : TTracker->Transfers) { + // We have to insert DBG_VALUEs in a consistent order, otherwise they + // appear in DWARF in different orders. Use the order that they appear + // when walking through each block / each instruction, stored in + // AllVarsNumbering. + SmallVector<std::pair<unsigned, MachineInstr *>> Insts; + for (MachineInstr *MI : P.Insts) { + DebugVariable Var(MI->getDebugVariable(), MI->getDebugExpression(), + MI->getDebugLoc()->getInlinedAt()); + Insts.emplace_back(AllVarsNumbering.find(Var)->second, MI); + } + llvm::sort(Insts, llvm::less_first()); + + // Insert either before or after the designated point... + if (P.MBB) { + MachineBasicBlock &MBB = *P.MBB; + for (const auto &Pair : Insts) + MBB.insert(P.Pos, Pair.second); + } else { + // Terminators, like tail calls, can clobber things. Don't try and place + // transfers after them. + if (P.Pos->isTerminator()) + continue; + + MachineBasicBlock &MBB = *P.Pos->getParent(); + for (const auto &Pair : Insts) + MBB.insertAfterBundle(P.Pos, Pair.second); + } + } + + return TTracker->Transfers.size() != 0; +} + +/// Calculate the liveness information for the given machine function and +/// extend ranges across basic blocks. +bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF, + MachineDominatorTree *DomTree, + TargetPassConfig *TPC, + unsigned InputBBLimit, + unsigned InputDbgValLimit) { + // No subprogram means this function contains no debuginfo. + if (!MF.getFunction().getSubprogram()) + return false; + + LLVM_DEBUG(dbgs() << "\nDebug Range Extension\n"); + this->TPC = TPC; + + this->DomTree = DomTree; + TRI = MF.getSubtarget().getRegisterInfo(); + MRI = &MF.getRegInfo(); + TII = MF.getSubtarget().getInstrInfo(); + TFI = MF.getSubtarget().getFrameLowering(); + TFI->getCalleeSaves(MF, CalleeSavedRegs); + MFI = &MF.getFrameInfo(); + LS.initialize(MF); + + const auto &STI = MF.getSubtarget(); + AdjustsStackInCalls = MFI->adjustsStack() && + STI.getFrameLowering()->stackProbeFunctionModifiesSP(); + if (AdjustsStackInCalls) + StackProbeSymbolName = STI.getTargetLowering()->getStackProbeSymbolName(MF); + + MTracker = + new MLocTracker(MF, *TII, *TRI, *MF.getSubtarget().getTargetLowering()); + VTracker = nullptr; + TTracker = nullptr; + + SmallVector<MLocTransferMap, 32> MLocTransfer; + SmallVector<VLocTracker, 8> vlocs; + LiveInsT SavedLiveIns; + + int MaxNumBlocks = -1; + for (auto &MBB : MF) + MaxNumBlocks = std::max(MBB.getNumber(), MaxNumBlocks); + assert(MaxNumBlocks >= 0); + ++MaxNumBlocks; + + initialSetup(MF); + + MLocTransfer.resize(MaxNumBlocks); + vlocs.resize(MaxNumBlocks, VLocTracker(OverlapFragments, EmptyExpr)); + SavedLiveIns.resize(MaxNumBlocks); + + produceMLocTransferFunction(MF, MLocTransfer, MaxNumBlocks); + + // Allocate and initialize two array-of-arrays for the live-in and live-out + // machine values. The outer dimension is the block number; while the inner + // dimension is a LocIdx from MLocTracker. + FuncValueTable MOutLocs = std::make_unique<ValueTable[]>(MaxNumBlocks); + FuncValueTable MInLocs = std::make_unique<ValueTable[]>(MaxNumBlocks); + unsigned NumLocs = MTracker->getNumLocs(); + for (int i = 0; i < MaxNumBlocks; ++i) { + // These all auto-initialize to ValueIDNum::EmptyValue + MOutLocs[i] = std::make_unique<ValueIDNum[]>(NumLocs); + MInLocs[i] = std::make_unique<ValueIDNum[]>(NumLocs); + } + + // Solve the machine value dataflow problem using the MLocTransfer function, + // storing the computed live-ins / live-outs into the array-of-arrays. We use + // both live-ins and live-outs for decision making in the variable value + // dataflow problem. + buildMLocValueMap(MF, MInLocs, MOutLocs, MLocTransfer); + + // Patch up debug phi numbers, turning unknown block-live-in values into + // either live-through machine values, or PHIs. + for (auto &DBG_PHI : DebugPHINumToValue) { + // Identify unresolved block-live-ins. + if (!DBG_PHI.ValueRead) + continue; + + ValueIDNum &Num = *DBG_PHI.ValueRead; + if (!Num.isPHI()) + continue; + + unsigned BlockNo = Num.getBlock(); + LocIdx LocNo = Num.getLoc(); + Num = MInLocs[BlockNo][LocNo.asU64()]; + } + // Later, we'll be looking up ranges of instruction numbers. + llvm::sort(DebugPHINumToValue); + + // Walk back through each block / instruction, collecting DBG_VALUE + // instructions and recording what machine value their operands refer to. + for (auto &OrderPair : OrderToBB) { + MachineBasicBlock &MBB = *OrderPair.second; + CurBB = MBB.getNumber(); + VTracker = &vlocs[CurBB]; + VTracker->MBB = &MBB; + MTracker->loadFromArray(MInLocs[CurBB], CurBB); + CurInst = 1; + for (auto &MI : MBB) { + process(MI, MOutLocs.get(), MInLocs.get()); + ++CurInst; + } + MTracker->reset(); + } + + // Number all variables in the order that they appear, to be used as a stable + // insertion order later. + DenseMap<DebugVariable, unsigned> AllVarsNumbering; + + // Map from one LexicalScope to all the variables in that scope. + ScopeToVarsT ScopeToVars; + + // Map from One lexical scope to all blocks where assignments happen for + // that scope. + ScopeToAssignBlocksT ScopeToAssignBlocks; + + // Store map of DILocations that describes scopes. + ScopeToDILocT ScopeToDILocation; + + // To mirror old LiveDebugValues, enumerate variables in RPOT order. Otherwise + // the order is unimportant, it just has to be stable. + unsigned VarAssignCount = 0; + for (unsigned int I = 0; I < OrderToBB.size(); ++I) { + auto *MBB = OrderToBB[I]; + auto *VTracker = &vlocs[MBB->getNumber()]; + // Collect each variable with a DBG_VALUE in this block. + for (auto &idx : VTracker->Vars) { + const auto &Var = idx.first; + const DILocation *ScopeLoc = VTracker->Scopes[Var]; + assert(ScopeLoc != nullptr); + auto *Scope = LS.findLexicalScope(ScopeLoc); + + // No insts in scope -> shouldn't have been recorded. + assert(Scope != nullptr); + + AllVarsNumbering.insert(std::make_pair(Var, AllVarsNumbering.size())); + ScopeToVars[Scope].insert(Var); + ScopeToAssignBlocks[Scope].insert(VTracker->MBB); + ScopeToDILocation[Scope] = ScopeLoc; + ++VarAssignCount; + } + } + + bool Changed = false; + + // If we have an extremely large number of variable assignments and blocks, + // bail out at this point. We've burnt some time doing analysis already, + // however we should cut our losses. + if ((unsigned)MaxNumBlocks > InputBBLimit && + VarAssignCount > InputDbgValLimit) { + LLVM_DEBUG(dbgs() << "Disabling InstrRefBasedLDV: " << MF.getName() + << " has " << MaxNumBlocks << " basic blocks and " + << VarAssignCount + << " variable assignments, exceeding limits.\n"); + } else { + // Optionally, solve the variable value problem and emit to blocks by using + // a lexical-scope-depth search. It should be functionally identical to + // the "else" block of this condition. + Changed = depthFirstVLocAndEmit( + MaxNumBlocks, ScopeToDILocation, ScopeToVars, ScopeToAssignBlocks, + SavedLiveIns, MOutLocs, MInLocs, vlocs, MF, AllVarsNumbering, *TPC); + } + + delete MTracker; + delete TTracker; + MTracker = nullptr; + VTracker = nullptr; + TTracker = nullptr; + + ArtificialBlocks.clear(); + OrderToBB.clear(); + BBToOrder.clear(); + BBNumToRPO.clear(); + DebugInstrNumToInstr.clear(); + DebugPHINumToValue.clear(); + OverlapFragments.clear(); + SeenFragments.clear(); + SeenDbgPHIs.clear(); + DbgOpStore.clear(); + + return Changed; +} + +LDVImpl *llvm::makeInstrRefBasedLiveDebugValues() { + return new InstrRefBasedLDV(); +} + +namespace { +class LDVSSABlock; +class LDVSSAUpdater; + +// Pick a type to identify incoming block values as we construct SSA. We +// can't use anything more robust than an integer unfortunately, as SSAUpdater +// expects to zero-initialize the type. +typedef uint64_t BlockValueNum; + +/// Represents an SSA PHI node for the SSA updater class. Contains the block +/// this PHI is in, the value number it would have, and the expected incoming +/// values from parent blocks. +class LDVSSAPhi { +public: + SmallVector<std::pair<LDVSSABlock *, BlockValueNum>, 4> IncomingValues; + LDVSSABlock *ParentBlock; + BlockValueNum PHIValNum; + LDVSSAPhi(BlockValueNum PHIValNum, LDVSSABlock *ParentBlock) + : ParentBlock(ParentBlock), PHIValNum(PHIValNum) {} + + LDVSSABlock *getParent() { return ParentBlock; } +}; + +/// Thin wrapper around a block predecessor iterator. Only difference from a +/// normal block iterator is that it dereferences to an LDVSSABlock. +class LDVSSABlockIterator { +public: + MachineBasicBlock::pred_iterator PredIt; + LDVSSAUpdater &Updater; + + LDVSSABlockIterator(MachineBasicBlock::pred_iterator PredIt, + LDVSSAUpdater &Updater) + : PredIt(PredIt), Updater(Updater) {} + + bool operator!=(const LDVSSABlockIterator &OtherIt) const { + return OtherIt.PredIt != PredIt; + } + + LDVSSABlockIterator &operator++() { + ++PredIt; + return *this; + } + + LDVSSABlock *operator*(); +}; + +/// Thin wrapper around a block for SSA Updater interface. Necessary because +/// we need to track the PHI value(s) that we may have observed as necessary +/// in this block. +class LDVSSABlock { +public: + MachineBasicBlock &BB; + LDVSSAUpdater &Updater; + using PHIListT = SmallVector<LDVSSAPhi, 1>; + /// List of PHIs in this block. There should only ever be one. + PHIListT PHIList; + + LDVSSABlock(MachineBasicBlock &BB, LDVSSAUpdater &Updater) + : BB(BB), Updater(Updater) {} + + LDVSSABlockIterator succ_begin() { + return LDVSSABlockIterator(BB.succ_begin(), Updater); + } + + LDVSSABlockIterator succ_end() { + return LDVSSABlockIterator(BB.succ_end(), Updater); + } + + /// SSAUpdater has requested a PHI: create that within this block record. + LDVSSAPhi *newPHI(BlockValueNum Value) { + PHIList.emplace_back(Value, this); + return &PHIList.back(); + } + + /// SSAUpdater wishes to know what PHIs already exist in this block. + PHIListT &phis() { return PHIList; } +}; + +/// Utility class for the SSAUpdater interface: tracks blocks, PHIs and values +/// while SSAUpdater is exploring the CFG. It's passed as a handle / baton to +// SSAUpdaterTraits<LDVSSAUpdater>. +class LDVSSAUpdater { +public: + /// Map of value numbers to PHI records. + DenseMap<BlockValueNum, LDVSSAPhi *> PHIs; + /// Map of which blocks generate Undef values -- blocks that are not + /// dominated by any Def. + DenseMap<MachineBasicBlock *, BlockValueNum> UndefMap; + /// Map of machine blocks to our own records of them. + DenseMap<MachineBasicBlock *, LDVSSABlock *> BlockMap; + /// Machine location where any PHI must occur. + LocIdx Loc; + /// Table of live-in machine value numbers for blocks / locations. + const ValueTable *MLiveIns; + + LDVSSAUpdater(LocIdx L, const ValueTable *MLiveIns) + : Loc(L), MLiveIns(MLiveIns) {} + + void reset() { + for (auto &Block : BlockMap) + delete Block.second; + + PHIs.clear(); + UndefMap.clear(); + BlockMap.clear(); + } + + ~LDVSSAUpdater() { reset(); } + + /// For a given MBB, create a wrapper block for it. Stores it in the + /// LDVSSAUpdater block map. + LDVSSABlock *getSSALDVBlock(MachineBasicBlock *BB) { + auto it = BlockMap.find(BB); + if (it == BlockMap.end()) { + BlockMap[BB] = new LDVSSABlock(*BB, *this); + it = BlockMap.find(BB); + } + return it->second; + } + + /// Find the live-in value number for the given block. Looks up the value at + /// the PHI location on entry. + BlockValueNum getValue(LDVSSABlock *LDVBB) { + return MLiveIns[LDVBB->BB.getNumber()][Loc.asU64()].asU64(); + } +}; + +LDVSSABlock *LDVSSABlockIterator::operator*() { + return Updater.getSSALDVBlock(*PredIt); +} + +#ifndef NDEBUG + +raw_ostream &operator<<(raw_ostream &out, const LDVSSAPhi &PHI) { + out << "SSALDVPHI " << PHI.PHIValNum; + return out; +} + +#endif + +} // namespace + +namespace llvm { + +/// Template specialization to give SSAUpdater access to CFG and value +/// information. SSAUpdater calls methods in these traits, passing in the +/// LDVSSAUpdater object, to learn about blocks and the values they define. +/// It also provides methods to create PHI nodes and track them. +template <> class SSAUpdaterTraits<LDVSSAUpdater> { +public: + using BlkT = LDVSSABlock; + using ValT = BlockValueNum; + using PhiT = LDVSSAPhi; + using BlkSucc_iterator = LDVSSABlockIterator; + + // Methods to access block successors -- dereferencing to our wrapper class. + static BlkSucc_iterator BlkSucc_begin(BlkT *BB) { return BB->succ_begin(); } + static BlkSucc_iterator BlkSucc_end(BlkT *BB) { return BB->succ_end(); } + + /// Iterator for PHI operands. + class PHI_iterator { + private: + LDVSSAPhi *PHI; + unsigned Idx; + + public: + explicit PHI_iterator(LDVSSAPhi *P) // begin iterator + : PHI(P), Idx(0) {} + PHI_iterator(LDVSSAPhi *P, bool) // end iterator + : PHI(P), Idx(PHI->IncomingValues.size()) {} + + PHI_iterator &operator++() { + Idx++; + return *this; + } + bool operator==(const PHI_iterator &X) const { return Idx == X.Idx; } + bool operator!=(const PHI_iterator &X) const { return !operator==(X); } + + BlockValueNum getIncomingValue() { return PHI->IncomingValues[Idx].second; } + + LDVSSABlock *getIncomingBlock() { return PHI->IncomingValues[Idx].first; } + }; + + static inline PHI_iterator PHI_begin(PhiT *PHI) { return PHI_iterator(PHI); } + + static inline PHI_iterator PHI_end(PhiT *PHI) { + return PHI_iterator(PHI, true); + } + + /// FindPredecessorBlocks - Put the predecessors of BB into the Preds + /// vector. + static void FindPredecessorBlocks(LDVSSABlock *BB, + SmallVectorImpl<LDVSSABlock *> *Preds) { + for (MachineBasicBlock *Pred : BB->BB.predecessors()) + Preds->push_back(BB->Updater.getSSALDVBlock(Pred)); + } + + /// GetUndefVal - Normally creates an IMPLICIT_DEF instruction with a new + /// register. For LiveDebugValues, represents a block identified as not having + /// any DBG_PHI predecessors. + static BlockValueNum GetUndefVal(LDVSSABlock *BB, LDVSSAUpdater *Updater) { + // Create a value number for this block -- it needs to be unique and in the + // "undef" collection, so that we know it's not real. Use a number + // representing a PHI into this block. + BlockValueNum Num = ValueIDNum(BB->BB.getNumber(), 0, Updater->Loc).asU64(); + Updater->UndefMap[&BB->BB] = Num; + return Num; + } + + /// CreateEmptyPHI - Create a (representation of a) PHI in the given block. + /// SSAUpdater will populate it with information about incoming values. The + /// value number of this PHI is whatever the machine value number problem + /// solution determined it to be. This includes non-phi values if SSAUpdater + /// tries to create a PHI where the incoming values are identical. + static BlockValueNum CreateEmptyPHI(LDVSSABlock *BB, unsigned NumPreds, + LDVSSAUpdater *Updater) { + BlockValueNum PHIValNum = Updater->getValue(BB); + LDVSSAPhi *PHI = BB->newPHI(PHIValNum); + Updater->PHIs[PHIValNum] = PHI; + return PHIValNum; + } + + /// AddPHIOperand - Add the specified value as an operand of the PHI for + /// the specified predecessor block. + static void AddPHIOperand(LDVSSAPhi *PHI, BlockValueNum Val, LDVSSABlock *Pred) { + PHI->IncomingValues.push_back(std::make_pair(Pred, Val)); + } + + /// ValueIsPHI - Check if the instruction that defines the specified value + /// is a PHI instruction. + static LDVSSAPhi *ValueIsPHI(BlockValueNum Val, LDVSSAUpdater *Updater) { + auto PHIIt = Updater->PHIs.find(Val); + if (PHIIt == Updater->PHIs.end()) + return nullptr; + return PHIIt->second; + } + + /// ValueIsNewPHI - Like ValueIsPHI but also check if the PHI has no source + /// operands, i.e., it was just added. + static LDVSSAPhi *ValueIsNewPHI(BlockValueNum Val, LDVSSAUpdater *Updater) { + LDVSSAPhi *PHI = ValueIsPHI(Val, Updater); + if (PHI && PHI->IncomingValues.size() == 0) + return PHI; + return nullptr; + } + + /// GetPHIValue - For the specified PHI instruction, return the value + /// that it defines. + static BlockValueNum GetPHIValue(LDVSSAPhi *PHI) { return PHI->PHIValNum; } +}; + +} // end namespace llvm + +std::optional<ValueIDNum> InstrRefBasedLDV::resolveDbgPHIs( + MachineFunction &MF, const ValueTable *MLiveOuts, + const ValueTable *MLiveIns, MachineInstr &Here, uint64_t InstrNum) { + assert(MLiveOuts && MLiveIns && + "Tried to resolve DBG_PHI before location " + "tables allocated?"); + + // This function will be called twice per DBG_INSTR_REF, and might end up + // computing lots of SSA information: memoize it. + auto SeenDbgPHIIt = SeenDbgPHIs.find(std::make_pair(&Here, InstrNum)); + if (SeenDbgPHIIt != SeenDbgPHIs.end()) + return SeenDbgPHIIt->second; + + std::optional<ValueIDNum> Result = + resolveDbgPHIsImpl(MF, MLiveOuts, MLiveIns, Here, InstrNum); + SeenDbgPHIs.insert({std::make_pair(&Here, InstrNum), Result}); + return Result; +} + +std::optional<ValueIDNum> InstrRefBasedLDV::resolveDbgPHIsImpl( + MachineFunction &MF, const ValueTable *MLiveOuts, + const ValueTable *MLiveIns, MachineInstr &Here, uint64_t InstrNum) { + // Pick out records of DBG_PHI instructions that have been observed. If there + // are none, then we cannot compute a value number. + auto RangePair = std::equal_range(DebugPHINumToValue.begin(), + DebugPHINumToValue.end(), InstrNum); + auto LowerIt = RangePair.first; + auto UpperIt = RangePair.second; + + // No DBG_PHI means there can be no location. + if (LowerIt == UpperIt) + return std::nullopt; + + // If any DBG_PHIs referred to a location we didn't understand, don't try to + // compute a value. There might be scenarios where we could recover a value + // for some range of DBG_INSTR_REFs, but at this point we can have high + // confidence that we've seen a bug. + auto DBGPHIRange = make_range(LowerIt, UpperIt); + for (const DebugPHIRecord &DBG_PHI : DBGPHIRange) + if (!DBG_PHI.ValueRead) + return std::nullopt; + + // If there's only one DBG_PHI, then that is our value number. + if (std::distance(LowerIt, UpperIt) == 1) + return *LowerIt->ValueRead; + + // Pick out the location (physreg, slot) where any PHIs must occur. It's + // technically possible for us to merge values in different registers in each + // block, but highly unlikely that LLVM will generate such code after register + // allocation. + LocIdx Loc = *LowerIt->ReadLoc; + + // We have several DBG_PHIs, and a use position (the Here inst). All each + // DBG_PHI does is identify a value at a program position. We can treat each + // DBG_PHI like it's a Def of a value, and the use position is a Use of a + // value, just like SSA. We use the bulk-standard LLVM SSA updater class to + // determine which Def is used at the Use, and any PHIs that happen along + // the way. + // Adapted LLVM SSA Updater: + LDVSSAUpdater Updater(Loc, MLiveIns); + // Map of which Def or PHI is the current value in each block. + DenseMap<LDVSSABlock *, BlockValueNum> AvailableValues; + // Set of PHIs that we have created along the way. + SmallVector<LDVSSAPhi *, 8> CreatedPHIs; + + // Each existing DBG_PHI is a Def'd value under this model. Record these Defs + // for the SSAUpdater. + for (const auto &DBG_PHI : DBGPHIRange) { + LDVSSABlock *Block = Updater.getSSALDVBlock(DBG_PHI.MBB); + const ValueIDNum &Num = *DBG_PHI.ValueRead; + AvailableValues.insert(std::make_pair(Block, Num.asU64())); + } + + LDVSSABlock *HereBlock = Updater.getSSALDVBlock(Here.getParent()); + const auto &AvailIt = AvailableValues.find(HereBlock); + if (AvailIt != AvailableValues.end()) { + // Actually, we already know what the value is -- the Use is in the same + // block as the Def. + return ValueIDNum::fromU64(AvailIt->second); + } + + // Otherwise, we must use the SSA Updater. It will identify the value number + // that we are to use, and the PHIs that must happen along the way. + SSAUpdaterImpl<LDVSSAUpdater> Impl(&Updater, &AvailableValues, &CreatedPHIs); + BlockValueNum ResultInt = Impl.GetValue(Updater.getSSALDVBlock(Here.getParent())); + ValueIDNum Result = ValueIDNum::fromU64(ResultInt); + + // We have the number for a PHI, or possibly live-through value, to be used + // at this Use. There are a number of things we have to check about it though: + // * Does any PHI use an 'Undef' (like an IMPLICIT_DEF) value? If so, this + // Use was not completely dominated by DBG_PHIs and we should abort. + // * Are the Defs or PHIs clobbered in a block? SSAUpdater isn't aware that + // we've left SSA form. Validate that the inputs to each PHI are the + // expected values. + // * Is a PHI we've created actually a merging of values, or are all the + // predecessor values the same, leading to a non-PHI machine value number? + // (SSAUpdater doesn't know that either). Remap validated PHIs into the + // the ValidatedValues collection below to sort this out. + DenseMap<LDVSSABlock *, ValueIDNum> ValidatedValues; + + // Define all the input DBG_PHI values in ValidatedValues. + for (const auto &DBG_PHI : DBGPHIRange) { + LDVSSABlock *Block = Updater.getSSALDVBlock(DBG_PHI.MBB); + const ValueIDNum &Num = *DBG_PHI.ValueRead; + ValidatedValues.insert(std::make_pair(Block, Num)); + } + + // Sort PHIs to validate into RPO-order. + SmallVector<LDVSSAPhi *, 8> SortedPHIs; + for (auto &PHI : CreatedPHIs) + SortedPHIs.push_back(PHI); + + llvm::sort(SortedPHIs, [&](LDVSSAPhi *A, LDVSSAPhi *B) { + return BBToOrder[&A->getParent()->BB] < BBToOrder[&B->getParent()->BB]; + }); + + for (auto &PHI : SortedPHIs) { + ValueIDNum ThisBlockValueNum = + MLiveIns[PHI->ParentBlock->BB.getNumber()][Loc.asU64()]; + + // Are all these things actually defined? + for (auto &PHIIt : PHI->IncomingValues) { + // Any undef input means DBG_PHIs didn't dominate the use point. + if (Updater.UndefMap.find(&PHIIt.first->BB) != Updater.UndefMap.end()) + return std::nullopt; + + ValueIDNum ValueToCheck; + const ValueTable &BlockLiveOuts = MLiveOuts[PHIIt.first->BB.getNumber()]; + + auto VVal = ValidatedValues.find(PHIIt.first); + if (VVal == ValidatedValues.end()) { + // We cross a loop, and this is a backedge. LLVMs tail duplication + // happens so late that DBG_PHI instructions should not be able to + // migrate into loops -- meaning we can only be live-through this + // loop. + ValueToCheck = ThisBlockValueNum; + } else { + // Does the block have as a live-out, in the location we're examining, + // the value that we expect? If not, it's been moved or clobbered. + ValueToCheck = VVal->second; + } + + if (BlockLiveOuts[Loc.asU64()] != ValueToCheck) + return std::nullopt; + } + + // Record this value as validated. + ValidatedValues.insert({PHI->ParentBlock, ThisBlockValueNum}); + } + + // All the PHIs are valid: we can return what the SSAUpdater said our value + // number was. + return Result; +} diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h new file mode 100644 index 000000000000..2fdc37c6dda2 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h @@ -0,0 +1,1441 @@ +//===- InstrRefBasedImpl.h - Tracking Debug Value MIs ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_CODEGEN_LIVEDEBUGVALUES_INSTRREFBASEDLDV_H +#define LLVM_LIB_CODEGEN_LIVEDEBUGVALUES_INSTRREFBASEDLDV_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/UniqueVector.h" +#include "llvm/CodeGen/LexicalScopes.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include <optional> + +#include "LiveDebugValues.h" + +class TransferTracker; + +// Forward dec of unit test class, so that we can peer into the LDV object. +class InstrRefLDVTest; + +namespace LiveDebugValues { + +class MLocTracker; +class DbgOpIDMap; + +using namespace llvm; + +/// Handle-class for a particular "location". This value-type uniquely +/// symbolises a register or stack location, allowing manipulation of locations +/// without concern for where that location is. Practically, this allows us to +/// treat the state of the machine at a particular point as an array of values, +/// rather than a map of values. +class LocIdx { + unsigned Location; + + // Default constructor is private, initializing to an illegal location number. + // Use only for "not an entry" elements in IndexedMaps. + LocIdx() : Location(UINT_MAX) {} + +public: +#define NUM_LOC_BITS 24 + LocIdx(unsigned L) : Location(L) { + assert(L < (1 << NUM_LOC_BITS) && "Machine locations must fit in 24 bits"); + } + + static LocIdx MakeIllegalLoc() { return LocIdx(); } + static LocIdx MakeTombstoneLoc() { + LocIdx L = LocIdx(); + --L.Location; + return L; + } + + bool isIllegal() const { return Location == UINT_MAX; } + + uint64_t asU64() const { return Location; } + + bool operator==(unsigned L) const { return Location == L; } + + bool operator==(const LocIdx &L) const { return Location == L.Location; } + + bool operator!=(unsigned L) const { return !(*this == L); } + + bool operator!=(const LocIdx &L) const { return !(*this == L); } + + bool operator<(const LocIdx &Other) const { + return Location < Other.Location; + } +}; + +// The location at which a spilled value resides. It consists of a register and +// an offset. +struct SpillLoc { + unsigned SpillBase; + StackOffset SpillOffset; + bool operator==(const SpillLoc &Other) const { + return std::make_pair(SpillBase, SpillOffset) == + std::make_pair(Other.SpillBase, Other.SpillOffset); + } + bool operator<(const SpillLoc &Other) const { + return std::make_tuple(SpillBase, SpillOffset.getFixed(), + SpillOffset.getScalable()) < + std::make_tuple(Other.SpillBase, Other.SpillOffset.getFixed(), + Other.SpillOffset.getScalable()); + } +}; + +/// Unique identifier for a value defined by an instruction, as a value type. +/// Casts back and forth to a uint64_t. Probably replacable with something less +/// bit-constrained. Each value identifies the instruction and machine location +/// where the value is defined, although there may be no corresponding machine +/// operand for it (ex: regmasks clobbering values). The instructions are +/// one-based, and definitions that are PHIs have instruction number zero. +/// +/// The obvious limits of a 1M block function or 1M instruction blocks are +/// problematic; but by that point we should probably have bailed out of +/// trying to analyse the function. +class ValueIDNum { + union { + struct { + uint64_t BlockNo : 20; /// The block where the def happens. + uint64_t InstNo : 20; /// The Instruction where the def happens. + /// One based, is distance from start of block. + uint64_t LocNo + : NUM_LOC_BITS; /// The machine location where the def happens. + } s; + uint64_t Value; + } u; + + static_assert(sizeof(u) == 8, "Badly packed ValueIDNum?"); + +public: + // Default-initialize to EmptyValue. This is necessary to make IndexedMaps + // of values to work. + ValueIDNum() { u.Value = EmptyValue.asU64(); } + + ValueIDNum(uint64_t Block, uint64_t Inst, uint64_t Loc) { + u.s = {Block, Inst, Loc}; + } + + ValueIDNum(uint64_t Block, uint64_t Inst, LocIdx Loc) { + u.s = {Block, Inst, Loc.asU64()}; + } + + uint64_t getBlock() const { return u.s.BlockNo; } + uint64_t getInst() const { return u.s.InstNo; } + uint64_t getLoc() const { return u.s.LocNo; } + bool isPHI() const { return u.s.InstNo == 0; } + + uint64_t asU64() const { return u.Value; } + + static ValueIDNum fromU64(uint64_t v) { + ValueIDNum Val; + Val.u.Value = v; + return Val; + } + + bool operator<(const ValueIDNum &Other) const { + return asU64() < Other.asU64(); + } + + bool operator==(const ValueIDNum &Other) const { + return u.Value == Other.u.Value; + } + + bool operator!=(const ValueIDNum &Other) const { return !(*this == Other); } + + std::string asString(const std::string &mlocname) const { + return Twine("Value{bb: ") + .concat(Twine(u.s.BlockNo) + .concat(Twine(", inst: ") + .concat((u.s.InstNo ? Twine(u.s.InstNo) + : Twine("live-in")) + .concat(Twine(", loc: ").concat( + Twine(mlocname))) + .concat(Twine("}"))))) + .str(); + } + + static ValueIDNum EmptyValue; + static ValueIDNum TombstoneValue; +}; + +} // End namespace LiveDebugValues + +namespace llvm { +using namespace LiveDebugValues; + +template <> struct DenseMapInfo<LocIdx> { + static inline LocIdx getEmptyKey() { return LocIdx::MakeIllegalLoc(); } + static inline LocIdx getTombstoneKey() { return LocIdx::MakeTombstoneLoc(); } + + static unsigned getHashValue(const LocIdx &Loc) { return Loc.asU64(); } + + static bool isEqual(const LocIdx &A, const LocIdx &B) { return A == B; } +}; + +template <> struct DenseMapInfo<ValueIDNum> { + static inline ValueIDNum getEmptyKey() { return ValueIDNum::EmptyValue; } + static inline ValueIDNum getTombstoneKey() { + return ValueIDNum::TombstoneValue; + } + + static unsigned getHashValue(const ValueIDNum &Val) { + return hash_value(Val.asU64()); + } + + static bool isEqual(const ValueIDNum &A, const ValueIDNum &B) { + return A == B; + } +}; + +} // end namespace llvm + +namespace LiveDebugValues { +using namespace llvm; + +/// Type for a table of values in a block. +using ValueTable = std::unique_ptr<ValueIDNum[]>; + +/// Type for a table-of-table-of-values, i.e., the collection of either +/// live-in or live-out values for each block in the function. +using FuncValueTable = std::unique_ptr<ValueTable[]>; + +/// Thin wrapper around an integer -- designed to give more type safety to +/// spill location numbers. +class SpillLocationNo { +public: + explicit SpillLocationNo(unsigned SpillNo) : SpillNo(SpillNo) {} + unsigned SpillNo; + unsigned id() const { return SpillNo; } + + bool operator<(const SpillLocationNo &Other) const { + return SpillNo < Other.SpillNo; + } + + bool operator==(const SpillLocationNo &Other) const { + return SpillNo == Other.SpillNo; + } + bool operator!=(const SpillLocationNo &Other) const { + return !(*this == Other); + } +}; + +/// Meta qualifiers for a value. Pair of whatever expression is used to qualify +/// the value, and Boolean of whether or not it's indirect. +class DbgValueProperties { +public: + DbgValueProperties(const DIExpression *DIExpr, bool Indirect, bool IsVariadic) + : DIExpr(DIExpr), Indirect(Indirect), IsVariadic(IsVariadic) {} + + /// Extract properties from an existing DBG_VALUE instruction. + DbgValueProperties(const MachineInstr &MI) { + assert(MI.isDebugValue()); + assert(MI.getDebugExpression()->getNumLocationOperands() == 0 || + MI.isDebugValueList() || MI.isUndefDebugValue()); + IsVariadic = MI.isDebugValueList(); + DIExpr = MI.getDebugExpression(); + Indirect = MI.isDebugOffsetImm(); + } + + bool isJoinable(const DbgValueProperties &Other) const { + return DIExpression::isEqualExpression(DIExpr, Indirect, Other.DIExpr, + Other.Indirect); + } + + bool operator==(const DbgValueProperties &Other) const { + return std::tie(DIExpr, Indirect, IsVariadic) == + std::tie(Other.DIExpr, Other.Indirect, Other.IsVariadic); + } + + bool operator!=(const DbgValueProperties &Other) const { + return !(*this == Other); + } + + unsigned getLocationOpCount() const { + return IsVariadic ? DIExpr->getNumLocationOperands() : 1; + } + + const DIExpression *DIExpr; + bool Indirect; + bool IsVariadic; +}; + +/// TODO: Might pack better if we changed this to a Struct of Arrays, since +/// MachineOperand is width 32, making this struct width 33. We could also +/// potentially avoid storing the whole MachineOperand (sizeof=32), instead +/// choosing to store just the contents portion (sizeof=8) and a Kind enum, +/// since we already know it is some type of immediate value. +/// Stores a single debug operand, which can either be a MachineOperand for +/// directly storing immediate values, or a ValueIDNum representing some value +/// computed at some point in the program. IsConst is used as a discriminator. +struct DbgOp { + union { + ValueIDNum ID; + MachineOperand MO; + }; + bool IsConst; + + DbgOp() : ID(ValueIDNum::EmptyValue), IsConst(false) {} + DbgOp(ValueIDNum ID) : ID(ID), IsConst(false) {} + DbgOp(MachineOperand MO) : MO(MO), IsConst(true) {} + + bool isUndef() const { return !IsConst && ID == ValueIDNum::EmptyValue; } + +#ifndef NDEBUG + void dump(const MLocTracker *MTrack) const; +#endif +}; + +/// A DbgOp whose ID (if any) has resolved to an actual location, LocIdx. Used +/// when working with concrete debug values, i.e. when joining MLocs and VLocs +/// in the TransferTracker or emitting DBG_VALUE/DBG_VALUE_LIST instructions in +/// the MLocTracker. +struct ResolvedDbgOp { + union { + LocIdx Loc; + MachineOperand MO; + }; + bool IsConst; + + ResolvedDbgOp(LocIdx Loc) : Loc(Loc), IsConst(false) {} + ResolvedDbgOp(MachineOperand MO) : MO(MO), IsConst(true) {} + + bool operator==(const ResolvedDbgOp &Other) const { + if (IsConst != Other.IsConst) + return false; + if (IsConst) + return MO.isIdenticalTo(Other.MO); + return Loc == Other.Loc; + } + +#ifndef NDEBUG + void dump(const MLocTracker *MTrack) const; +#endif +}; + +/// An ID used in the DbgOpIDMap (below) to lookup a stored DbgOp. This is used +/// in place of actual DbgOps inside of a DbgValue to reduce its size, as +/// DbgValue is very frequently used and passed around, and the actual DbgOp is +/// over 8x larger than this class, due to storing a MachineOperand. This ID +/// should be equal for all equal DbgOps, and also encodes whether the mapped +/// DbgOp is a constant, meaning that for simple equality or const-ness checks +/// it is not necessary to lookup this ID. +struct DbgOpID { + struct IsConstIndexPair { + uint32_t IsConst : 1; + uint32_t Index : 31; + }; + + union { + struct IsConstIndexPair ID; + uint32_t RawID; + }; + + DbgOpID() : RawID(UndefID.RawID) { + static_assert(sizeof(DbgOpID) == 4, "DbgOpID should fit within 4 bytes."); + } + DbgOpID(uint32_t RawID) : RawID(RawID) {} + DbgOpID(bool IsConst, uint32_t Index) : ID({IsConst, Index}) {} + + static DbgOpID UndefID; + + bool operator==(const DbgOpID &Other) const { return RawID == Other.RawID; } + bool operator!=(const DbgOpID &Other) const { return !(*this == Other); } + + uint32_t asU32() const { return RawID; } + + bool isUndef() const { return *this == UndefID; } + bool isConst() const { return ID.IsConst && !isUndef(); } + uint32_t getIndex() const { return ID.Index; } + +#ifndef NDEBUG + void dump(const MLocTracker *MTrack, const DbgOpIDMap *OpStore) const; +#endif +}; + +/// Class storing the complete set of values that are observed by DbgValues +/// within the current function. Allows 2-way lookup, with `find` returning the +/// Op for a given ID and `insert` returning the ID for a given Op (creating one +/// if none exists). +class DbgOpIDMap { + + SmallVector<ValueIDNum, 0> ValueOps; + SmallVector<MachineOperand, 0> ConstOps; + + DenseMap<ValueIDNum, DbgOpID> ValueOpToID; + DenseMap<MachineOperand, DbgOpID> ConstOpToID; + +public: + /// If \p Op does not already exist in this map, it is inserted and the + /// corresponding DbgOpID is returned. If Op already exists in this map, then + /// no change is made and the existing ID for Op is returned. + /// Calling this with the undef DbgOp will always return DbgOpID::UndefID. + DbgOpID insert(DbgOp Op) { + if (Op.isUndef()) + return DbgOpID::UndefID; + if (Op.IsConst) + return insertConstOp(Op.MO); + return insertValueOp(Op.ID); + } + /// Returns the DbgOp associated with \p ID. Should only be used for IDs + /// returned from calling `insert` from this map or DbgOpID::UndefID. + DbgOp find(DbgOpID ID) const { + if (ID == DbgOpID::UndefID) + return DbgOp(); + if (ID.isConst()) + return DbgOp(ConstOps[ID.getIndex()]); + return DbgOp(ValueOps[ID.getIndex()]); + } + + void clear() { + ValueOps.clear(); + ConstOps.clear(); + ValueOpToID.clear(); + ConstOpToID.clear(); + } + +private: + DbgOpID insertConstOp(MachineOperand &MO) { + auto ExistingIt = ConstOpToID.find(MO); + if (ExistingIt != ConstOpToID.end()) + return ExistingIt->second; + DbgOpID ID(true, ConstOps.size()); + ConstOpToID.insert(std::make_pair(MO, ID)); + ConstOps.push_back(MO); + return ID; + } + DbgOpID insertValueOp(ValueIDNum VID) { + auto ExistingIt = ValueOpToID.find(VID); + if (ExistingIt != ValueOpToID.end()) + return ExistingIt->second; + DbgOpID ID(false, ValueOps.size()); + ValueOpToID.insert(std::make_pair(VID, ID)); + ValueOps.push_back(VID); + return ID; + } +}; + +// We set the maximum number of operands that we will handle to keep DbgValue +// within a reasonable size (64 bytes), as we store and pass a lot of them +// around. +#define MAX_DBG_OPS 8 + +/// Class recording the (high level) _value_ of a variable. Identifies the value +/// of the variable as a list of ValueIDNums and constant MachineOperands, or as +/// an empty list for undef debug values or VPHI values which we have not found +/// valid locations for. +/// This class also stores meta-information about how the value is qualified. +/// Used to reason about variable values when performing the second +/// (DebugVariable specific) dataflow analysis. +class DbgValue { +private: + /// If Kind is Def or VPHI, the set of IDs corresponding to the DbgOps that + /// are used. VPHIs set every ID to EmptyID when we have not found a valid + /// machine-value for every operand, and sets them to the corresponding + /// machine-values when we have found all of them. + DbgOpID DbgOps[MAX_DBG_OPS]; + unsigned OpCount; + +public: + /// For a NoVal or VPHI DbgValue, which block it was generated in. + int BlockNo; + + /// Qualifiers for the ValueIDNum above. + DbgValueProperties Properties; + + typedef enum { + Undef, // Represents a DBG_VALUE $noreg in the transfer function only. + Def, // This value is defined by some combination of constants, + // instructions, or PHI values. + VPHI, // Incoming values to BlockNo differ, those values must be joined by + // a PHI in this block. + NoVal, // Empty DbgValue indicating an unknown value. Used as initializer, + // before dominating blocks values are propagated in. + } KindT; + /// Discriminator for whether this is a constant or an in-program value. + KindT Kind; + + DbgValue(ArrayRef<DbgOpID> DbgOps, const DbgValueProperties &Prop) + : OpCount(DbgOps.size()), BlockNo(0), Properties(Prop), Kind(Def) { + static_assert(sizeof(DbgValue) <= 64, + "DbgValue should fit within 64 bytes."); + assert(DbgOps.size() == Prop.getLocationOpCount()); + if (DbgOps.size() > MAX_DBG_OPS || + any_of(DbgOps, [](DbgOpID ID) { return ID.isUndef(); })) { + Kind = Undef; + OpCount = 0; +#define DEBUG_TYPE "LiveDebugValues" + if (DbgOps.size() > MAX_DBG_OPS) { + LLVM_DEBUG(dbgs() << "Found DbgValue with more than maximum allowed " + "operands.\n"); + } +#undef DEBUG_TYPE + } else { + for (unsigned Idx = 0; Idx < DbgOps.size(); ++Idx) + this->DbgOps[Idx] = DbgOps[Idx]; + } + } + + DbgValue(unsigned BlockNo, const DbgValueProperties &Prop, KindT Kind) + : OpCount(0), BlockNo(BlockNo), Properties(Prop), Kind(Kind) { + assert(Kind == NoVal || Kind == VPHI); + } + + DbgValue(const DbgValueProperties &Prop, KindT Kind) + : OpCount(0), BlockNo(0), Properties(Prop), Kind(Kind) { + assert(Kind == Undef && + "Empty DbgValue constructor must pass in Undef kind"); + } + +#ifndef NDEBUG + void dump(const MLocTracker *MTrack = nullptr, + const DbgOpIDMap *OpStore = nullptr) const; +#endif + + bool operator==(const DbgValue &Other) const { + if (std::tie(Kind, Properties) != std::tie(Other.Kind, Other.Properties)) + return false; + else if (Kind == Def && !equal(getDbgOpIDs(), Other.getDbgOpIDs())) + return false; + else if (Kind == NoVal && BlockNo != Other.BlockNo) + return false; + else if (Kind == VPHI && BlockNo != Other.BlockNo) + return false; + else if (Kind == VPHI && !equal(getDbgOpIDs(), Other.getDbgOpIDs())) + return false; + + return true; + } + + bool operator!=(const DbgValue &Other) const { return !(*this == Other); } + + // Returns an array of all the machine values used to calculate this variable + // value, or an empty list for an Undef or unjoined VPHI. + ArrayRef<DbgOpID> getDbgOpIDs() const { return {DbgOps, OpCount}; } + + // Returns either DbgOps[Index] if this DbgValue has Debug Operands, or + // the ID for ValueIDNum::EmptyValue otherwise (i.e. if this is an Undef, + // NoVal, or an unjoined VPHI). + DbgOpID getDbgOpID(unsigned Index) const { + if (!OpCount) + return DbgOpID::UndefID; + assert(Index < OpCount); + return DbgOps[Index]; + } + // Replaces this DbgValue's existing DbgOpIDs (if any) with the contents of + // \p NewIDs. The number of DbgOpIDs passed must be equal to the number of + // arguments expected by this DbgValue's properties (the return value of + // `getLocationOpCount()`). + void setDbgOpIDs(ArrayRef<DbgOpID> NewIDs) { + // We can go from no ops to some ops, but not from some ops to no ops. + assert(NewIDs.size() == getLocationOpCount() && + "Incorrect number of Debug Operands for this DbgValue."); + OpCount = NewIDs.size(); + for (unsigned Idx = 0; Idx < NewIDs.size(); ++Idx) + DbgOps[Idx] = NewIDs[Idx]; + } + + // The number of debug operands expected by this DbgValue's expression. + // getDbgOpIDs() should return an array of this length, unless this is an + // Undef or an unjoined VPHI. + unsigned getLocationOpCount() const { + return Properties.getLocationOpCount(); + } + + // Returns true if this or Other are unjoined PHIs, which do not have defined + // Loc Ops, or if the `n`th Loc Op for this has a different constness to the + // `n`th Loc Op for Other. + bool hasJoinableLocOps(const DbgValue &Other) const { + if (isUnjoinedPHI() || Other.isUnjoinedPHI()) + return true; + for (unsigned Idx = 0; Idx < getLocationOpCount(); ++Idx) { + if (getDbgOpID(Idx).isConst() != Other.getDbgOpID(Idx).isConst()) + return false; + } + return true; + } + + bool isUnjoinedPHI() const { return Kind == VPHI && OpCount == 0; } + + bool hasIdenticalValidLocOps(const DbgValue &Other) const { + if (!OpCount) + return false; + return equal(getDbgOpIDs(), Other.getDbgOpIDs()); + } +}; + +class LocIdxToIndexFunctor { +public: + using argument_type = LocIdx; + unsigned operator()(const LocIdx &L) const { return L.asU64(); } +}; + +/// Tracker for what values are in machine locations. Listens to the Things +/// being Done by various instructions, and maintains a table of what machine +/// locations have what values (as defined by a ValueIDNum). +/// +/// There are potentially a much larger number of machine locations on the +/// target machine than the actual working-set size of the function. On x86 for +/// example, we're extremely unlikely to want to track values through control +/// or debug registers. To avoid doing so, MLocTracker has several layers of +/// indirection going on, described below, to avoid unnecessarily tracking +/// any location. +/// +/// Here's a sort of diagram of the indexes, read from the bottom up: +/// +/// Size on stack Offset on stack +/// \ / +/// Stack Idx (Where in slot is this?) +/// / +/// / +/// Slot Num (%stack.0) / +/// FrameIdx => SpillNum / +/// \ / +/// SpillID (int) Register number (int) +/// \ / +/// LocationID => LocIdx +/// | +/// LocIdx => ValueIDNum +/// +/// The aim here is that the LocIdx => ValueIDNum vector is just an array of +/// values in numbered locations, so that later analyses can ignore whether the +/// location is a register or otherwise. To map a register / spill location to +/// a LocIdx, you have to use the (sparse) LocationID => LocIdx map. And to +/// build a LocationID for a stack slot, you need to combine identifiers for +/// which stack slot it is and where within that slot is being described. +/// +/// Register mask operands cause trouble by technically defining every register; +/// various hacks are used to avoid tracking registers that are never read and +/// only written by regmasks. +class MLocTracker { +public: + MachineFunction &MF; + const TargetInstrInfo &TII; + const TargetRegisterInfo &TRI; + const TargetLowering &TLI; + + /// IndexedMap type, mapping from LocIdx to ValueIDNum. + using LocToValueType = IndexedMap<ValueIDNum, LocIdxToIndexFunctor>; + + /// Map of LocIdxes to the ValueIDNums that they store. This is tightly + /// packed, entries only exist for locations that are being tracked. + LocToValueType LocIdxToIDNum; + + /// "Map" of machine location IDs (i.e., raw register or spill number) to the + /// LocIdx key / number for that location. There are always at least as many + /// as the number of registers on the target -- if the value in the register + /// is not being tracked, then the LocIdx value will be zero. New entries are + /// appended if a new spill slot begins being tracked. + /// This, and the corresponding reverse map persist for the analysis of the + /// whole function, and is necessarying for decoding various vectors of + /// values. + std::vector<LocIdx> LocIDToLocIdx; + + /// Inverse map of LocIDToLocIdx. + IndexedMap<unsigned, LocIdxToIndexFunctor> LocIdxToLocID; + + /// When clobbering register masks, we chose to not believe the machine model + /// and don't clobber SP. Do the same for SP aliases, and for efficiency, + /// keep a set of them here. + SmallSet<Register, 8> SPAliases; + + /// Unique-ification of spill. Used to number them -- their LocID number is + /// the index in SpillLocs minus one plus NumRegs. + UniqueVector<SpillLoc> SpillLocs; + + // If we discover a new machine location, assign it an mphi with this + // block number. + unsigned CurBB; + + /// Cached local copy of the number of registers the target has. + unsigned NumRegs; + + /// Number of slot indexes the target has -- distinct segments of a stack + /// slot that can take on the value of a subregister, when a super-register + /// is written to the stack. + unsigned NumSlotIdxes; + + /// Collection of register mask operands that have been observed. Second part + /// of pair indicates the instruction that they happened in. Used to + /// reconstruct where defs happened if we start tracking a location later + /// on. + SmallVector<std::pair<const MachineOperand *, unsigned>, 32> Masks; + + /// Pair for describing a position within a stack slot -- first the size in + /// bits, then the offset. + typedef std::pair<unsigned short, unsigned short> StackSlotPos; + + /// Map from a size/offset pair describing a position in a stack slot, to a + /// numeric identifier for that position. Allows easier identification of + /// individual positions. + DenseMap<StackSlotPos, unsigned> StackSlotIdxes; + + /// Inverse of StackSlotIdxes. + DenseMap<unsigned, StackSlotPos> StackIdxesToPos; + + /// Iterator for locations and the values they contain. Dereferencing + /// produces a struct/pair containing the LocIdx key for this location, + /// and a reference to the value currently stored. Simplifies the process + /// of seeking a particular location. + class MLocIterator { + LocToValueType &ValueMap; + LocIdx Idx; + + public: + class value_type { + public: + value_type(LocIdx Idx, ValueIDNum &Value) : Idx(Idx), Value(Value) {} + const LocIdx Idx; /// Read-only index of this location. + ValueIDNum &Value; /// Reference to the stored value at this location. + }; + + MLocIterator(LocToValueType &ValueMap, LocIdx Idx) + : ValueMap(ValueMap), Idx(Idx) {} + + bool operator==(const MLocIterator &Other) const { + assert(&ValueMap == &Other.ValueMap); + return Idx == Other.Idx; + } + + bool operator!=(const MLocIterator &Other) const { + return !(*this == Other); + } + + void operator++() { Idx = LocIdx(Idx.asU64() + 1); } + + value_type operator*() { return value_type(Idx, ValueMap[LocIdx(Idx)]); } + }; + + MLocTracker(MachineFunction &MF, const TargetInstrInfo &TII, + const TargetRegisterInfo &TRI, const TargetLowering &TLI); + + /// Produce location ID number for a Register. Provides some small amount of + /// type safety. + /// \param Reg The register we're looking up. + unsigned getLocID(Register Reg) { return Reg.id(); } + + /// Produce location ID number for a spill position. + /// \param Spill The number of the spill we're fetching the location for. + /// \param SpillSubReg Subregister within the spill we're addressing. + unsigned getLocID(SpillLocationNo Spill, unsigned SpillSubReg) { + unsigned short Size = TRI.getSubRegIdxSize(SpillSubReg); + unsigned short Offs = TRI.getSubRegIdxOffset(SpillSubReg); + return getLocID(Spill, {Size, Offs}); + } + + /// Produce location ID number for a spill position. + /// \param Spill The number of the spill we're fetching the location for. + /// \apram SpillIdx size/offset within the spill slot to be addressed. + unsigned getLocID(SpillLocationNo Spill, StackSlotPos Idx) { + unsigned SlotNo = Spill.id() - 1; + SlotNo *= NumSlotIdxes; + assert(StackSlotIdxes.find(Idx) != StackSlotIdxes.end()); + SlotNo += StackSlotIdxes[Idx]; + SlotNo += NumRegs; + return SlotNo; + } + + /// Given a spill number, and a slot within the spill, calculate the ID number + /// for that location. + unsigned getSpillIDWithIdx(SpillLocationNo Spill, unsigned Idx) { + unsigned SlotNo = Spill.id() - 1; + SlotNo *= NumSlotIdxes; + SlotNo += Idx; + SlotNo += NumRegs; + return SlotNo; + } + + /// Return the spill number that a location ID corresponds to. + SpillLocationNo locIDToSpill(unsigned ID) const { + assert(ID >= NumRegs); + ID -= NumRegs; + // Truncate away the index part, leaving only the spill number. + ID /= NumSlotIdxes; + return SpillLocationNo(ID + 1); // The UniqueVector is one-based. + } + + /// Returns the spill-slot size/offs that a location ID corresponds to. + StackSlotPos locIDToSpillIdx(unsigned ID) const { + assert(ID >= NumRegs); + ID -= NumRegs; + unsigned Idx = ID % NumSlotIdxes; + return StackIdxesToPos.find(Idx)->second; + } + + unsigned getNumLocs() const { return LocIdxToIDNum.size(); } + + /// Reset all locations to contain a PHI value at the designated block. Used + /// sometimes for actual PHI values, othertimes to indicate the block entry + /// value (before any more information is known). + void setMPhis(unsigned NewCurBB) { + CurBB = NewCurBB; + for (auto Location : locations()) + Location.Value = {CurBB, 0, Location.Idx}; + } + + /// Load values for each location from array of ValueIDNums. Take current + /// bbnum just in case we read a value from a hitherto untouched register. + void loadFromArray(ValueTable &Locs, unsigned NewCurBB) { + CurBB = NewCurBB; + // Iterate over all tracked locations, and load each locations live-in + // value into our local index. + for (auto Location : locations()) + Location.Value = Locs[Location.Idx.asU64()]; + } + + /// Wipe any un-necessary location records after traversing a block. + void reset() { + // We could reset all the location values too; however either loadFromArray + // or setMPhis should be called before this object is re-used. Just + // clear Masks, they're definitely not needed. + Masks.clear(); + } + + /// Clear all data. Destroys the LocID <=> LocIdx map, which makes most of + /// the information in this pass uninterpretable. + void clear() { + reset(); + LocIDToLocIdx.clear(); + LocIdxToLocID.clear(); + LocIdxToIDNum.clear(); + // SpillLocs.reset(); XXX UniqueVector::reset assumes a SpillLoc casts from + // 0 + SpillLocs = decltype(SpillLocs)(); + StackSlotIdxes.clear(); + StackIdxesToPos.clear(); + + LocIDToLocIdx.resize(NumRegs, LocIdx::MakeIllegalLoc()); + } + + /// Set a locaiton to a certain value. + void setMLoc(LocIdx L, ValueIDNum Num) { + assert(L.asU64() < LocIdxToIDNum.size()); + LocIdxToIDNum[L] = Num; + } + + /// Read the value of a particular location + ValueIDNum readMLoc(LocIdx L) { + assert(L.asU64() < LocIdxToIDNum.size()); + return LocIdxToIDNum[L]; + } + + /// Create a LocIdx for an untracked register ID. Initialize it to either an + /// mphi value representing a live-in, or a recent register mask clobber. + LocIdx trackRegister(unsigned ID); + + LocIdx lookupOrTrackRegister(unsigned ID) { + LocIdx &Index = LocIDToLocIdx[ID]; + if (Index.isIllegal()) + Index = trackRegister(ID); + return Index; + } + + /// Is register R currently tracked by MLocTracker? + bool isRegisterTracked(Register R) { + LocIdx &Index = LocIDToLocIdx[R]; + return !Index.isIllegal(); + } + + /// Record a definition of the specified register at the given block / inst. + /// This doesn't take a ValueIDNum, because the definition and its location + /// are synonymous. + void defReg(Register R, unsigned BB, unsigned Inst) { + unsigned ID = getLocID(R); + LocIdx Idx = lookupOrTrackRegister(ID); + ValueIDNum ValueID = {BB, Inst, Idx}; + LocIdxToIDNum[Idx] = ValueID; + } + + /// Set a register to a value number. To be used if the value number is + /// known in advance. + void setReg(Register R, ValueIDNum ValueID) { + unsigned ID = getLocID(R); + LocIdx Idx = lookupOrTrackRegister(ID); + LocIdxToIDNum[Idx] = ValueID; + } + + ValueIDNum readReg(Register R) { + unsigned ID = getLocID(R); + LocIdx Idx = lookupOrTrackRegister(ID); + return LocIdxToIDNum[Idx]; + } + + /// Reset a register value to zero / empty. Needed to replicate the + /// VarLoc implementation where a copy to/from a register effectively + /// clears the contents of the source register. (Values can only have one + /// machine location in VarLocBasedImpl). + void wipeRegister(Register R) { + unsigned ID = getLocID(R); + LocIdx Idx = LocIDToLocIdx[ID]; + LocIdxToIDNum[Idx] = ValueIDNum::EmptyValue; + } + + /// Determine the LocIdx of an existing register. + LocIdx getRegMLoc(Register R) { + unsigned ID = getLocID(R); + assert(ID < LocIDToLocIdx.size()); + assert(LocIDToLocIdx[ID] != UINT_MAX); // Sentinal for IndexedMap. + return LocIDToLocIdx[ID]; + } + + /// Record a RegMask operand being executed. Defs any register we currently + /// track, stores a pointer to the mask in case we have to account for it + /// later. + void writeRegMask(const MachineOperand *MO, unsigned CurBB, unsigned InstID); + + /// Find LocIdx for SpillLoc \p L, creating a new one if it's not tracked. + /// Returns std::nullopt when in scenarios where a spill slot could be + /// tracked, but we would likely run into resource limitations. + std::optional<SpillLocationNo> getOrTrackSpillLoc(SpillLoc L); + + // Get LocIdx of a spill ID. + LocIdx getSpillMLoc(unsigned SpillID) { + assert(LocIDToLocIdx[SpillID] != UINT_MAX); // Sentinal for IndexedMap. + return LocIDToLocIdx[SpillID]; + } + + /// Return true if Idx is a spill machine location. + bool isSpill(LocIdx Idx) const { return LocIdxToLocID[Idx] >= NumRegs; } + + /// How large is this location (aka, how wide is a value defined there?). + unsigned getLocSizeInBits(LocIdx L) const { + unsigned ID = LocIdxToLocID[L]; + if (!isSpill(L)) { + return TRI.getRegSizeInBits(Register(ID), MF.getRegInfo()); + } else { + // The slot location on the stack is uninteresting, we care about the + // position of the value within the slot (which comes with a size). + StackSlotPos Pos = locIDToSpillIdx(ID); + return Pos.first; + } + } + + MLocIterator begin() { return MLocIterator(LocIdxToIDNum, 0); } + + MLocIterator end() { + return MLocIterator(LocIdxToIDNum, LocIdxToIDNum.size()); + } + + /// Return a range over all locations currently tracked. + iterator_range<MLocIterator> locations() { + return llvm::make_range(begin(), end()); + } + + std::string LocIdxToName(LocIdx Idx) const; + + std::string IDAsString(const ValueIDNum &Num) const; + +#ifndef NDEBUG + LLVM_DUMP_METHOD void dump(); + + LLVM_DUMP_METHOD void dump_mloc_map(); +#endif + + /// Create a DBG_VALUE based on debug operands \p DbgOps. Qualify it with the + /// information in \pProperties, for variable Var. Don't insert it anywhere, + /// just return the builder for it. + MachineInstrBuilder emitLoc(const SmallVectorImpl<ResolvedDbgOp> &DbgOps, + const DebugVariable &Var, + const DbgValueProperties &Properties); +}; + +/// Types for recording sets of variable fragments that overlap. For a given +/// local variable, we record all other fragments of that variable that could +/// overlap it, to reduce search time. +using FragmentOfVar = + std::pair<const DILocalVariable *, DIExpression::FragmentInfo>; +using OverlapMap = + DenseMap<FragmentOfVar, SmallVector<DIExpression::FragmentInfo, 1>>; + +/// Collection of DBG_VALUEs observed when traversing a block. Records each +/// variable and the value the DBG_VALUE refers to. Requires the machine value +/// location dataflow algorithm to have run already, so that values can be +/// identified. +class VLocTracker { +public: + /// Map DebugVariable to the latest Value it's defined to have. + /// Needs to be a MapVector because we determine order-in-the-input-MIR from + /// the order in this container. + /// We only retain the last DbgValue in each block for each variable, to + /// determine the blocks live-out variable value. The Vars container forms the + /// transfer function for this block, as part of the dataflow analysis. The + /// movement of values between locations inside of a block is handled at a + /// much later stage, in the TransferTracker class. + MapVector<DebugVariable, DbgValue> Vars; + SmallDenseMap<DebugVariable, const DILocation *, 8> Scopes; + MachineBasicBlock *MBB = nullptr; + const OverlapMap &OverlappingFragments; + DbgValueProperties EmptyProperties; + +public: + VLocTracker(const OverlapMap &O, const DIExpression *EmptyExpr) + : OverlappingFragments(O), EmptyProperties(EmptyExpr, false, false) {} + + void defVar(const MachineInstr &MI, const DbgValueProperties &Properties, + const SmallVectorImpl<DbgOpID> &DebugOps) { + assert(MI.isDebugValueLike()); + DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(), + MI.getDebugLoc()->getInlinedAt()); + DbgValue Rec = (DebugOps.size() > 0) + ? DbgValue(DebugOps, Properties) + : DbgValue(Properties, DbgValue::Undef); + + // Attempt insertion; overwrite if it's already mapped. + auto Result = Vars.insert(std::make_pair(Var, Rec)); + if (!Result.second) + Result.first->second = Rec; + Scopes[Var] = MI.getDebugLoc().get(); + + considerOverlaps(Var, MI.getDebugLoc().get()); + } + + void considerOverlaps(const DebugVariable &Var, const DILocation *Loc) { + auto Overlaps = OverlappingFragments.find( + {Var.getVariable(), Var.getFragmentOrDefault()}); + if (Overlaps == OverlappingFragments.end()) + return; + + // Otherwise: terminate any overlapped variable locations. + for (auto FragmentInfo : Overlaps->second) { + // The "empty" fragment is stored as DebugVariable::DefaultFragment, so + // that it overlaps with everything, however its cannonical representation + // in a DebugVariable is as "None". + std::optional<DIExpression::FragmentInfo> OptFragmentInfo = FragmentInfo; + if (DebugVariable::isDefaultFragment(FragmentInfo)) + OptFragmentInfo = std::nullopt; + + DebugVariable Overlapped(Var.getVariable(), OptFragmentInfo, + Var.getInlinedAt()); + DbgValue Rec = DbgValue(EmptyProperties, DbgValue::Undef); + + // Attempt insertion; overwrite if it's already mapped. + auto Result = Vars.insert(std::make_pair(Overlapped, Rec)); + if (!Result.second) + Result.first->second = Rec; + Scopes[Overlapped] = Loc; + } + } + + void clear() { + Vars.clear(); + Scopes.clear(); + } +}; + +// XXX XXX docs +class InstrRefBasedLDV : public LDVImpl { +public: + friend class ::InstrRefLDVTest; + + using FragmentInfo = DIExpression::FragmentInfo; + using OptFragmentInfo = std::optional<DIExpression::FragmentInfo>; + + // Helper while building OverlapMap, a map of all fragments seen for a given + // DILocalVariable. + using VarToFragments = + DenseMap<const DILocalVariable *, SmallSet<FragmentInfo, 4>>; + + /// Machine location/value transfer function, a mapping of which locations + /// are assigned which new values. + using MLocTransferMap = SmallDenseMap<LocIdx, ValueIDNum>; + + /// Live in/out structure for the variable values: a per-block map of + /// variables to their values. + using LiveIdxT = DenseMap<const MachineBasicBlock *, DbgValue *>; + + using VarAndLoc = std::pair<DebugVariable, DbgValue>; + + /// Type for a live-in value: the predecessor block, and its value. + using InValueT = std::pair<MachineBasicBlock *, DbgValue *>; + + /// Vector (per block) of a collection (inner smallvector) of live-ins. + /// Used as the result type for the variable value dataflow problem. + using LiveInsT = SmallVector<SmallVector<VarAndLoc, 8>, 8>; + + /// Mapping from lexical scopes to a DILocation in that scope. + using ScopeToDILocT = DenseMap<const LexicalScope *, const DILocation *>; + + /// Mapping from lexical scopes to variables in that scope. + using ScopeToVarsT = DenseMap<const LexicalScope *, SmallSet<DebugVariable, 4>>; + + /// Mapping from lexical scopes to blocks where variables in that scope are + /// assigned. Such blocks aren't necessarily "in" the lexical scope, it's + /// just a block where an assignment happens. + using ScopeToAssignBlocksT = DenseMap<const LexicalScope *, SmallPtrSet<MachineBasicBlock *, 4>>; + +private: + MachineDominatorTree *DomTree; + const TargetRegisterInfo *TRI; + const MachineRegisterInfo *MRI; + const TargetInstrInfo *TII; + const TargetFrameLowering *TFI; + const MachineFrameInfo *MFI; + BitVector CalleeSavedRegs; + LexicalScopes LS; + TargetPassConfig *TPC; + + // An empty DIExpression. Used default / placeholder DbgValueProperties + // objects, as we can't have null expressions. + const DIExpression *EmptyExpr; + + /// Object to track machine locations as we step through a block. Could + /// probably be a field rather than a pointer, as it's always used. + MLocTracker *MTracker = nullptr; + + /// Number of the current block LiveDebugValues is stepping through. + unsigned CurBB; + + /// Number of the current instruction LiveDebugValues is evaluating. + unsigned CurInst; + + /// Variable tracker -- listens to DBG_VALUEs occurring as InstrRefBasedImpl + /// steps through a block. Reads the values at each location from the + /// MLocTracker object. + VLocTracker *VTracker = nullptr; + + /// Tracker for transfers, listens to DBG_VALUEs and transfers of values + /// between locations during stepping, creates new DBG_VALUEs when values move + /// location. + TransferTracker *TTracker = nullptr; + + /// Blocks which are artificial, i.e. blocks which exclusively contain + /// instructions without DebugLocs, or with line 0 locations. + SmallPtrSet<MachineBasicBlock *, 16> ArtificialBlocks; + + // Mapping of blocks to and from their RPOT order. + DenseMap<unsigned int, MachineBasicBlock *> OrderToBB; + DenseMap<const MachineBasicBlock *, unsigned int> BBToOrder; + DenseMap<unsigned, unsigned> BBNumToRPO; + + /// Pair of MachineInstr, and its 1-based offset into the containing block. + using InstAndNum = std::pair<const MachineInstr *, unsigned>; + /// Map from debug instruction number to the MachineInstr labelled with that + /// number, and its location within the function. Used to transform + /// instruction numbers in DBG_INSTR_REFs into machine value numbers. + std::map<uint64_t, InstAndNum> DebugInstrNumToInstr; + + /// Record of where we observed a DBG_PHI instruction. + class DebugPHIRecord { + public: + /// Instruction number of this DBG_PHI. + uint64_t InstrNum; + /// Block where DBG_PHI occurred. + MachineBasicBlock *MBB; + /// The value number read by the DBG_PHI -- or std::nullopt if it didn't + /// refer to a value. + std::optional<ValueIDNum> ValueRead; + /// Register/Stack location the DBG_PHI reads -- or std::nullopt if it + /// referred to something unexpected. + std::optional<LocIdx> ReadLoc; + + operator unsigned() const { return InstrNum; } + }; + + /// Map from instruction numbers defined by DBG_PHIs to a record of what that + /// DBG_PHI read and where. Populated and edited during the machine value + /// location problem -- we use LLVMs SSA Updater to fix changes by + /// optimizations that destroy PHI instructions. + SmallVector<DebugPHIRecord, 32> DebugPHINumToValue; + + // Map of overlapping variable fragments. + OverlapMap OverlapFragments; + VarToFragments SeenFragments; + + /// Mapping of DBG_INSTR_REF instructions to their values, for those + /// DBG_INSTR_REFs that call resolveDbgPHIs. These variable references solve + /// a mini SSA problem caused by DBG_PHIs being cloned, this collection caches + /// the result. + DenseMap<std::pair<MachineInstr *, unsigned>, std::optional<ValueIDNum>> + SeenDbgPHIs; + + DbgOpIDMap DbgOpStore; + + /// True if we need to examine call instructions for stack clobbers. We + /// normally assume that they don't clobber SP, but stack probes on Windows + /// do. + bool AdjustsStackInCalls = false; + + /// If AdjustsStackInCalls is true, this holds the name of the target's stack + /// probe function, which is the function we expect will alter the stack + /// pointer. + StringRef StackProbeSymbolName; + + /// Tests whether this instruction is a spill to a stack slot. + std::optional<SpillLocationNo> isSpillInstruction(const MachineInstr &MI, + MachineFunction *MF); + + /// Decide if @MI is a spill instruction and return true if it is. We use 2 + /// criteria to make this decision: + /// - Is this instruction a store to a spill slot? + /// - Is there a register operand that is both used and killed? + /// TODO: Store optimization can fold spills into other stores (including + /// other spills). We do not handle this yet (more than one memory operand). + bool isLocationSpill(const MachineInstr &MI, MachineFunction *MF, + unsigned &Reg); + + /// If a given instruction is identified as a spill, return the spill slot + /// and set \p Reg to the spilled register. + std::optional<SpillLocationNo> isRestoreInstruction(const MachineInstr &MI, + MachineFunction *MF, + unsigned &Reg); + + /// Given a spill instruction, extract the spill slot information, ensure it's + /// tracked, and return the spill number. + std::optional<SpillLocationNo> + extractSpillBaseRegAndOffset(const MachineInstr &MI); + + /// For an instruction reference given by \p InstNo and \p OpNo in instruction + /// \p MI returns the Value pointed to by that instruction reference if any + /// exists, otherwise returns None. + std::optional<ValueIDNum> getValueForInstrRef(unsigned InstNo, unsigned OpNo, + MachineInstr &MI, + const ValueTable *MLiveOuts, + const ValueTable *MLiveIns); + + /// Observe a single instruction while stepping through a block. + void process(MachineInstr &MI, const ValueTable *MLiveOuts, + const ValueTable *MLiveIns); + + /// Examines whether \p MI is a DBG_VALUE and notifies trackers. + /// \returns true if MI was recognized and processed. + bool transferDebugValue(const MachineInstr &MI); + + /// Examines whether \p MI is a DBG_INSTR_REF and notifies trackers. + /// \returns true if MI was recognized and processed. + bool transferDebugInstrRef(MachineInstr &MI, const ValueTable *MLiveOuts, + const ValueTable *MLiveIns); + + /// Stores value-information about where this PHI occurred, and what + /// instruction number is associated with it. + /// \returns true if MI was recognized and processed. + bool transferDebugPHI(MachineInstr &MI); + + /// Examines whether \p MI is copy instruction, and notifies trackers. + /// \returns true if MI was recognized and processed. + bool transferRegisterCopy(MachineInstr &MI); + + /// Examines whether \p MI is stack spill or restore instruction, and + /// notifies trackers. \returns true if MI was recognized and processed. + bool transferSpillOrRestoreInst(MachineInstr &MI); + + /// Examines \p MI for any registers that it defines, and notifies trackers. + void transferRegisterDef(MachineInstr &MI); + + /// Copy one location to the other, accounting for movement of subregisters + /// too. + void performCopy(Register Src, Register Dst); + + void accumulateFragmentMap(MachineInstr &MI); + + /// Determine the machine value number referred to by (potentially several) + /// DBG_PHI instructions. Block duplication and tail folding can duplicate + /// DBG_PHIs, shifting the position where values in registers merge, and + /// forming another mini-ssa problem to solve. + /// \p Here the position of a DBG_INSTR_REF seeking a machine value number + /// \p InstrNum Debug instruction number defined by DBG_PHI instructions. + /// \returns The machine value number at position Here, or std::nullopt. + std::optional<ValueIDNum> resolveDbgPHIs(MachineFunction &MF, + const ValueTable *MLiveOuts, + const ValueTable *MLiveIns, + MachineInstr &Here, + uint64_t InstrNum); + + std::optional<ValueIDNum> resolveDbgPHIsImpl(MachineFunction &MF, + const ValueTable *MLiveOuts, + const ValueTable *MLiveIns, + MachineInstr &Here, + uint64_t InstrNum); + + /// Step through the function, recording register definitions and movements + /// in an MLocTracker. Convert the observations into a per-block transfer + /// function in \p MLocTransfer, suitable for using with the machine value + /// location dataflow problem. + void + produceMLocTransferFunction(MachineFunction &MF, + SmallVectorImpl<MLocTransferMap> &MLocTransfer, + unsigned MaxNumBlocks); + + /// Solve the machine value location dataflow problem. Takes as input the + /// transfer functions in \p MLocTransfer. Writes the output live-in and + /// live-out arrays to the (initialized to zero) multidimensional arrays in + /// \p MInLocs and \p MOutLocs. The outer dimension is indexed by block + /// number, the inner by LocIdx. + void buildMLocValueMap(MachineFunction &MF, FuncValueTable &MInLocs, + FuncValueTable &MOutLocs, + SmallVectorImpl<MLocTransferMap> &MLocTransfer); + + /// Examine the stack indexes (i.e. offsets within the stack) to find the + /// basic units of interference -- like reg units, but for the stack. + void findStackIndexInterference(SmallVectorImpl<unsigned> &Slots); + + /// Install PHI values into the live-in array for each block, according to + /// the IDF of each register. + void placeMLocPHIs(MachineFunction &MF, + SmallPtrSetImpl<MachineBasicBlock *> &AllBlocks, + FuncValueTable &MInLocs, + SmallVectorImpl<MLocTransferMap> &MLocTransfer); + + /// Propagate variable values to blocks in the common case where there's + /// only one value assigned to the variable. This function has better + /// performance as it doesn't have to find the dominance frontier between + /// different assignments. + void placePHIsForSingleVarDefinition( + const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks, + MachineBasicBlock *MBB, SmallVectorImpl<VLocTracker> &AllTheVLocs, + const DebugVariable &Var, LiveInsT &Output); + + /// Calculate the iterated-dominance-frontier for a set of defs, using the + /// existing LLVM facilities for this. Works for a single "value" or + /// machine/variable location. + /// \p AllBlocks Set of blocks where we might consume the value. + /// \p DefBlocks Set of blocks where the value/location is defined. + /// \p PHIBlocks Output set of blocks where PHIs must be placed. + void BlockPHIPlacement(const SmallPtrSetImpl<MachineBasicBlock *> &AllBlocks, + const SmallPtrSetImpl<MachineBasicBlock *> &DefBlocks, + SmallVectorImpl<MachineBasicBlock *> &PHIBlocks); + + /// Perform a control flow join (lattice value meet) of the values in machine + /// locations at \p MBB. Follows the algorithm described in the file-comment, + /// reading live-outs of predecessors from \p OutLocs, the current live ins + /// from \p InLocs, and assigning the newly computed live ins back into + /// \p InLocs. \returns two bools -- the first indicates whether a change + /// was made, the second whether a lattice downgrade occurred. If the latter + /// is true, revisiting this block is necessary. + bool mlocJoin(MachineBasicBlock &MBB, + SmallPtrSet<const MachineBasicBlock *, 16> &Visited, + FuncValueTable &OutLocs, ValueTable &InLocs); + + /// Produce a set of blocks that are in the current lexical scope. This means + /// those blocks that contain instructions "in" the scope, blocks where + /// assignments to variables in scope occur, and artificial blocks that are + /// successors to any of the earlier blocks. See https://llvm.org/PR48091 for + /// more commentry on what "in scope" means. + /// \p DILoc A location in the scope that we're fetching blocks for. + /// \p Output Set to put in-scope-blocks into. + /// \p AssignBlocks Blocks known to contain assignments of variables in scope. + void + getBlocksForScope(const DILocation *DILoc, + SmallPtrSetImpl<const MachineBasicBlock *> &Output, + const SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks); + + /// Solve the variable value dataflow problem, for a single lexical scope. + /// Uses the algorithm from the file comment to resolve control flow joins + /// using PHI placement and value propagation. Reads the locations of machine + /// values from the \p MInLocs and \p MOutLocs arrays (see buildMLocValueMap) + /// and reads the variable values transfer function from \p AllTheVlocs. + /// Live-in and Live-out variable values are stored locally, with the live-ins + /// permanently stored to \p Output once a fixedpoint is reached. + /// \p VarsWeCareAbout contains a collection of the variables in \p Scope + /// that we should be tracking. + /// \p AssignBlocks contains the set of blocks that aren't in \p DILoc's + /// scope, but which do contain DBG_VALUEs, which VarLocBasedImpl tracks + /// locations through. + void buildVLocValueMap(const DILocation *DILoc, + const SmallSet<DebugVariable, 4> &VarsWeCareAbout, + SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks, + LiveInsT &Output, FuncValueTable &MOutLocs, + FuncValueTable &MInLocs, + SmallVectorImpl<VLocTracker> &AllTheVLocs); + + /// Attempt to eliminate un-necessary PHIs on entry to a block. Examines the + /// live-in values coming from predecessors live-outs, and replaces any PHIs + /// already present in this blocks live-ins with a live-through value if the + /// PHI isn't needed. + /// \p LiveIn Old live-in value, overwritten with new one if live-in changes. + /// \returns true if any live-ins change value, either from value propagation + /// or PHI elimination. + bool vlocJoin(MachineBasicBlock &MBB, LiveIdxT &VLOCOutLocs, + SmallPtrSet<const MachineBasicBlock *, 8> &BlocksToExplore, + DbgValue &LiveIn); + + /// For the given block and live-outs feeding into it, try to find + /// machine locations for each debug operand where all the values feeding + /// into that operand join together. + /// \returns true if a joined location was found for every value that needed + /// to be joined. + bool + pickVPHILoc(SmallVectorImpl<DbgOpID> &OutValues, const MachineBasicBlock &MBB, + const LiveIdxT &LiveOuts, FuncValueTable &MOutLocs, + const SmallVectorImpl<const MachineBasicBlock *> &BlockOrders); + + std::optional<ValueIDNum> pickOperandPHILoc( + unsigned DbgOpIdx, const MachineBasicBlock &MBB, const LiveIdxT &LiveOuts, + FuncValueTable &MOutLocs, + const SmallVectorImpl<const MachineBasicBlock *> &BlockOrders); + + /// Take collections of DBG_VALUE instructions stored in TTracker, and + /// install them into their output blocks. Preserves a stable order of + /// DBG_VALUEs produced (which would otherwise cause nondeterminism) through + /// the AllVarsNumbering order. + bool emitTransfers(DenseMap<DebugVariable, unsigned> &AllVarsNumbering); + + /// Boilerplate computation of some initial sets, artifical blocks and + /// RPOT block ordering. + void initialSetup(MachineFunction &MF); + + /// Produce a map of the last lexical scope that uses a block, using the + /// scopes DFSOut number. Mapping is block-number to DFSOut. + /// \p EjectionMap Pre-allocated vector in which to install the built ma. + /// \p ScopeToDILocation Mapping of LexicalScopes to their DILocations. + /// \p AssignBlocks Map of blocks where assignments happen for a scope. + void makeDepthFirstEjectionMap(SmallVectorImpl<unsigned> &EjectionMap, + const ScopeToDILocT &ScopeToDILocation, + ScopeToAssignBlocksT &AssignBlocks); + + /// When determining per-block variable values and emitting to DBG_VALUEs, + /// this function explores by lexical scope depth. Doing so means that per + /// block information can be fully computed before exploration finishes, + /// allowing us to emit it and free data structures earlier than otherwise. + /// It's also good for locality. + bool depthFirstVLocAndEmit( + unsigned MaxNumBlocks, const ScopeToDILocT &ScopeToDILocation, + const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToBlocks, + LiveInsT &Output, FuncValueTable &MOutLocs, FuncValueTable &MInLocs, + SmallVectorImpl<VLocTracker> &AllTheVLocs, MachineFunction &MF, + DenseMap<DebugVariable, unsigned> &AllVarsNumbering, + const TargetPassConfig &TPC); + + bool ExtendRanges(MachineFunction &MF, MachineDominatorTree *DomTree, + TargetPassConfig *TPC, unsigned InputBBLimit, + unsigned InputDbgValLimit) override; + +public: + /// Default construct and initialize the pass. + InstrRefBasedLDV(); + + LLVM_DUMP_METHOD + void dump_mloc_transfer(const MLocTransferMap &mloc_transfer) const; + + bool isCalleeSaved(LocIdx L) const; + bool isCalleeSavedReg(Register R) const; + + bool hasFoldedStackStore(const MachineInstr &MI) { + // Instruction must have a memory operand that's a stack slot, and isn't + // aliased, meaning it's a spill from regalloc instead of a variable. + // If it's aliased, we can't guarantee its value. + if (!MI.hasOneMemOperand()) + return false; + auto *MemOperand = *MI.memoperands_begin(); + return MemOperand->isStore() && + MemOperand->getPseudoValue() && + MemOperand->getPseudoValue()->kind() == PseudoSourceValue::FixedStack + && !MemOperand->getPseudoValue()->isAliased(MFI); + } + + std::optional<LocIdx> findLocationForMemOperand(const MachineInstr &MI); +}; + +} // namespace LiveDebugValues + +#endif /* LLVM_LIB_CODEGEN_LIVEDEBUGVALUES_INSTRREFBASEDLDV_H */ diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp new file mode 100644 index 000000000000..9dba9a88f703 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp @@ -0,0 +1,139 @@ +//===- LiveDebugValues.cpp - Tracking Debug Value MIs ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "LiveDebugValues.h" + +#include "llvm/ADT/Triple.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/PassRegistry.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Target/TargetMachine.h" + +/// \file LiveDebugValues.cpp +/// +/// The LiveDebugValues pass extends the range of variable locations +/// (specified by DBG_VALUE instructions) from single blocks to successors +/// and any other code locations where the variable location is valid. +/// There are currently two implementations: the "VarLoc" implementation +/// explicitly tracks the location of a variable, while the "InstrRef" +/// implementation tracks the values defined by instructions through locations. +/// +/// This file implements neither; it merely registers the pass, allows the +/// user to pick which implementation will be used to propagate variable +/// locations. + +#define DEBUG_TYPE "livedebugvalues" + +using namespace llvm; + +static cl::opt<bool> + ForceInstrRefLDV("force-instr-ref-livedebugvalues", cl::Hidden, + cl::desc("Use instruction-ref based LiveDebugValues with " + "normal DBG_VALUE inputs"), + cl::init(false)); + +static cl::opt<cl::boolOrDefault> ValueTrackingVariableLocations( + "experimental-debug-variable-locations", + cl::desc("Use experimental new value-tracking variable locations")); + +// Options to prevent pathological compile-time behavior. If InputBBLimit and +// InputDbgValueLimit are both exceeded, range extension is disabled. +static cl::opt<unsigned> InputBBLimit( + "livedebugvalues-input-bb-limit", + cl::desc("Maximum input basic blocks before DBG_VALUE limit applies"), + cl::init(10000), cl::Hidden); +static cl::opt<unsigned> InputDbgValueLimit( + "livedebugvalues-input-dbg-value-limit", + cl::desc( + "Maximum input DBG_VALUE insts supported by debug range extension"), + cl::init(50000), cl::Hidden); + +namespace { +/// Generic LiveDebugValues pass. Calls through to VarLocBasedLDV or +/// InstrRefBasedLDV to perform location propagation, via the LDVImpl +/// base class. +class LiveDebugValues : public MachineFunctionPass { +public: + static char ID; + + LiveDebugValues(); + ~LiveDebugValues() = default; + + /// Calculate the liveness information for the given machine function. + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } + +private: + std::unique_ptr<LDVImpl> InstrRefImpl; + std::unique_ptr<LDVImpl> VarLocImpl; + TargetPassConfig *TPC; + MachineDominatorTree MDT; +}; +} // namespace + +char LiveDebugValues::ID = 0; + +char &llvm::LiveDebugValuesID = LiveDebugValues::ID; + +INITIALIZE_PASS(LiveDebugValues, DEBUG_TYPE, "Live DEBUG_VALUE analysis", false, + false) + +/// Default construct and initialize the pass. +LiveDebugValues::LiveDebugValues() : MachineFunctionPass(ID) { + initializeLiveDebugValuesPass(*PassRegistry::getPassRegistry()); + InstrRefImpl = + std::unique_ptr<LDVImpl>(llvm::makeInstrRefBasedLiveDebugValues()); + VarLocImpl = std::unique_ptr<LDVImpl>(llvm::makeVarLocBasedLiveDebugValues()); +} + +bool LiveDebugValues::runOnMachineFunction(MachineFunction &MF) { + // Except for Wasm, all targets should be only using physical register at this + // point. Wasm only use virtual registers throught its pipeline, but its + // virtual registers don't participate in this LiveDebugValues analysis; only + // its target indices do. + assert(MF.getTarget().getTargetTriple().isWasm() || + MF.getProperties().hasProperty( + MachineFunctionProperties::Property::NoVRegs)); + + bool InstrRefBased = MF.useDebugInstrRef(); + // Allow the user to force selection of InstrRef LDV. + InstrRefBased |= ForceInstrRefLDV; + + TPC = getAnalysisIfAvailable<TargetPassConfig>(); + LDVImpl *TheImpl = &*VarLocImpl; + + MachineDominatorTree *DomTree = nullptr; + if (InstrRefBased) { + DomTree = &MDT; + MDT.calculate(MF); + TheImpl = &*InstrRefImpl; + } + + return TheImpl->ExtendRanges(MF, DomTree, TPC, InputBBLimit, + InputDbgValueLimit); +} + +bool llvm::debuginfoShouldUseDebugInstrRef(const Triple &T) { + // Enable by default on x86_64, disable if explicitly turned off on cmdline. + if (T.getArch() == llvm::Triple::x86_64 && + ValueTrackingVariableLocations != cl::boolOrDefault::BOU_FALSE) + return true; + + // Enable if explicitly requested on command line. + return ValueTrackingVariableLocations == cl::boolOrDefault::BOU_TRUE; +} diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.h b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.h new file mode 100644 index 000000000000..6cc1685c0022 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.h @@ -0,0 +1,43 @@ +//===- LiveDebugValues.cpp - Tracking Debug Value MIs ---------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_CODEGEN_LIVEDEBUGVALUES_LIVEDEBUGVALUES_H +#define LLVM_LIB_CODEGEN_LIVEDEBUGVALUES_LIVEDEBUGVALUES_H + +namespace llvm { +class MachineDominatorTree; +class MachineFunction; +class TargetPassConfig; +class Triple; + +// Inline namespace for types / symbols shared between different +// LiveDebugValues implementations. +inline namespace SharedLiveDebugValues { + +// Expose a base class for LiveDebugValues interfaces to inherit from. This +// allows the generic LiveDebugValues pass handles to call into the +// implementation. +class LDVImpl { +public: + virtual bool ExtendRanges(MachineFunction &MF, MachineDominatorTree *DomTree, + TargetPassConfig *TPC, unsigned InputBBLimit, + unsigned InputDbgValLimit) = 0; + virtual ~LDVImpl() = default; +}; + +} // namespace SharedLiveDebugValues + +// Factory functions for LiveDebugValues implementations. +extern LDVImpl *makeVarLocBasedLiveDebugValues(); +extern LDVImpl *makeInstrRefBasedLiveDebugValues(); + +extern bool debuginfoShouldUseDebugInstrRef(const Triple &T); + +} // namespace llvm + +#endif // LLVM_LIB_CODEGEN_LIVEDEBUGVALUES_LIVEDEBUGVALUES_H diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp new file mode 100644 index 000000000000..b78757b855f4 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp @@ -0,0 +1,2403 @@ +//===- VarLocBasedImpl.cpp - Tracking Debug Value MIs with VarLoc class----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file VarLocBasedImpl.cpp +/// +/// LiveDebugValues is an optimistic "available expressions" dataflow +/// algorithm. The set of expressions is the set of machine locations +/// (registers, spill slots, constants, and target indices) that a variable +/// fragment might be located, qualified by a DIExpression and indirect-ness +/// flag, while each variable is identified by a DebugVariable object. The +/// availability of an expression begins when a DBG_VALUE instruction specifies +/// the location of a DebugVariable, and continues until that location is +/// clobbered or re-specified by a different DBG_VALUE for the same +/// DebugVariable. +/// +/// The output of LiveDebugValues is additional DBG_VALUE instructions, +/// placed to extend variable locations as far they're available. This file +/// and the VarLocBasedLDV class is an implementation that explicitly tracks +/// locations, using the VarLoc class. +/// +/// The canonical "available expressions" problem doesn't have expression +/// clobbering, instead when a variable is re-assigned, any expressions using +/// that variable get invalidated. LiveDebugValues can map onto "available +/// expressions" by having every register represented by a variable, which is +/// used in an expression that becomes available at a DBG_VALUE instruction. +/// When the register is clobbered, its variable is effectively reassigned, and +/// expressions computed from it become unavailable. A similar construct is +/// needed when a DebugVariable has its location re-specified, to invalidate +/// all other locations for that DebugVariable. +/// +/// Using the dataflow analysis to compute the available expressions, we create +/// a DBG_VALUE at the beginning of each block where the expression is +/// live-in. This propagates variable locations into every basic block where +/// the location can be determined, rather than only having DBG_VALUEs in blocks +/// where locations are specified due to an assignment or some optimization. +/// Movements of values between registers and spill slots are annotated with +/// DBG_VALUEs too to track variable values bewteen locations. All this allows +/// DbgEntityHistoryCalculator to focus on only the locations within individual +/// blocks, facilitating testing and improving modularity. +/// +/// We follow an optimisic dataflow approach, with this lattice: +/// +/// \verbatim +/// ┬ "Unknown" +/// | +/// v +/// True +/// | +/// v +/// ⊥ False +/// \endverbatim With "True" signifying that the expression is available (and +/// thus a DebugVariable's location is the corresponding register), while +/// "False" signifies that the expression is unavailable. "Unknown"s never +/// survive to the end of the analysis (see below). +/// +/// Formally, all DebugVariable locations that are live-out of a block are +/// initialized to \top. A blocks live-in values take the meet of the lattice +/// value for every predecessors live-outs, except for the entry block, where +/// all live-ins are \bot. The usual dataflow propagation occurs: the transfer +/// function for a block assigns an expression for a DebugVariable to be "True" +/// if a DBG_VALUE in the block specifies it; "False" if the location is +/// clobbered; or the live-in value if it is unaffected by the block. We +/// visit each block in reverse post order until a fixedpoint is reached. The +/// solution produced is maximal. +/// +/// Intuitively, we start by assuming that every expression / variable location +/// is at least "True", and then propagate "False" from the entry block and any +/// clobbers until there are no more changes to make. This gives us an accurate +/// solution because all incorrect locations will have a "False" propagated into +/// them. It also gives us a solution that copes well with loops by assuming +/// that variable locations are live-through every loop, and then removing those +/// that are not through dataflow. +/// +/// Within LiveDebugValues: each variable location is represented by a +/// VarLoc object that identifies the source variable, the set of +/// machine-locations that currently describe it (a single location for +/// DBG_VALUE or multiple for DBG_VALUE_LIST), and the DBG_VALUE inst that +/// specifies the location. Each VarLoc is indexed in the (function-scope) \p +/// VarLocMap, giving each VarLoc a set of unique indexes, each of which +/// corresponds to one of the VarLoc's machine-locations and can be used to +/// lookup the VarLoc in the VarLocMap. Rather than operate directly on machine +/// locations, the dataflow analysis in this pass identifies locations by their +/// indices in the VarLocMap, meaning all the variable locations in a block can +/// be described by a sparse vector of VarLocMap indicies. +/// +/// All the storage for the dataflow analysis is local to the ExtendRanges +/// method and passed down to helper methods. "OutLocs" and "InLocs" record the +/// in and out lattice values for each block. "OpenRanges" maintains a list of +/// variable locations and, with the "process" method, evaluates the transfer +/// function of each block. "flushPendingLocs" installs debug value instructions +/// for each live-in location at the start of blocks, while "Transfers" records +/// transfers of values between machine-locations. +/// +/// We avoid explicitly representing the "Unknown" (\top) lattice value in the +/// implementation. Instead, unvisited blocks implicitly have all lattice +/// values set as "Unknown". After being visited, there will be path back to +/// the entry block where the lattice value is "False", and as the transfer +/// function cannot make new "Unknown" locations, there are no scenarios where +/// a block can have an "Unknown" location after being visited. Similarly, we +/// don't enumerate all possible variable locations before exploring the +/// function: when a new location is discovered, all blocks previously explored +/// were implicitly "False" but unrecorded, and become explicitly "False" when +/// a new VarLoc is created with its bit not set in predecessor InLocs or +/// OutLocs. +/// +//===----------------------------------------------------------------------===// + +#include "LiveDebugValues.h" + +#include "llvm/ADT/CoalescingBitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/LexicalScopes.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/CodeGen/TargetFrameLowering.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Function.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/TypeSize.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <functional> +#include <map> +#include <optional> +#include <queue> +#include <tuple> +#include <utility> +#include <vector> + +using namespace llvm; + +#define DEBUG_TYPE "livedebugvalues" + +STATISTIC(NumInserted, "Number of DBG_VALUE instructions inserted"); + +/// If \p Op is a stack or frame register return true, otherwise return false. +/// This is used to avoid basing the debug entry values on the registers, since +/// we do not support it at the moment. +static bool isRegOtherThanSPAndFP(const MachineOperand &Op, + const MachineInstr &MI, + const TargetRegisterInfo *TRI) { + if (!Op.isReg()) + return false; + + const MachineFunction *MF = MI.getParent()->getParent(); + const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); + Register SP = TLI->getStackPointerRegisterToSaveRestore(); + Register FP = TRI->getFrameRegister(*MF); + Register Reg = Op.getReg(); + + return Reg && Reg != SP && Reg != FP; +} + +namespace { + +// Max out the number of statically allocated elements in DefinedRegsSet, as +// this prevents fallback to std::set::count() operations. +using DefinedRegsSet = SmallSet<Register, 32>; + +// The IDs in this set correspond to MachineLocs in VarLocs, as well as VarLocs +// that represent Entry Values; every VarLoc in the set will also appear +// exactly once at Location=0. +// As a result, each VarLoc may appear more than once in this "set", but each +// range corresponding to a Reg, SpillLoc, or EntryValue type will still be a +// "true" set (i.e. each VarLoc may appear only once), and the range Location=0 +// is the set of all VarLocs. +using VarLocSet = CoalescingBitVector<uint64_t>; + +/// A type-checked pair of {Register Location (or 0), Index}, used to index +/// into a \ref VarLocMap. This can be efficiently converted to a 64-bit int +/// for insertion into a \ref VarLocSet, and efficiently converted back. The +/// type-checker helps ensure that the conversions aren't lossy. +/// +/// Why encode a location /into/ the VarLocMap index? This makes it possible +/// to find the open VarLocs killed by a register def very quickly. This is a +/// performance-critical operation for LiveDebugValues. +struct LocIndex { + using u32_location_t = uint32_t; + using u32_index_t = uint32_t; + + u32_location_t Location; // Physical registers live in the range [1;2^30) (see + // \ref MCRegister), so we have plenty of range left + // here to encode non-register locations. + u32_index_t Index; + + /// The location that has an entry for every VarLoc in the map. + static constexpr u32_location_t kUniversalLocation = 0; + + /// The first location that is reserved for VarLocs with locations of kind + /// RegisterKind. + static constexpr u32_location_t kFirstRegLocation = 1; + + /// The first location greater than 0 that is not reserved for VarLocs with + /// locations of kind RegisterKind. + static constexpr u32_location_t kFirstInvalidRegLocation = 1 << 30; + + /// A special location reserved for VarLocs with locations of kind + /// SpillLocKind. + static constexpr u32_location_t kSpillLocation = kFirstInvalidRegLocation; + + /// A special location reserved for VarLocs of kind EntryValueBackupKind and + /// EntryValueCopyBackupKind. + static constexpr u32_location_t kEntryValueBackupLocation = + kFirstInvalidRegLocation + 1; + + /// A special location reserved for VarLocs with locations of kind + /// WasmLocKind. + /// TODO Placing all Wasm target index locations in this single kWasmLocation + /// may cause slowdown in compilation time in very large functions. Consider + /// giving a each target index/offset pair its own u32_location_t if this + /// becomes a problem. + static constexpr u32_location_t kWasmLocation = kFirstInvalidRegLocation + 2; + + LocIndex(u32_location_t Location, u32_index_t Index) + : Location(Location), Index(Index) {} + + uint64_t getAsRawInteger() const { + return (static_cast<uint64_t>(Location) << 32) | Index; + } + + template<typename IntT> static LocIndex fromRawInteger(IntT ID) { + static_assert(std::is_unsigned_v<IntT> && sizeof(ID) == sizeof(uint64_t), + "Cannot convert raw integer to LocIndex"); + return {static_cast<u32_location_t>(ID >> 32), + static_cast<u32_index_t>(ID)}; + } + + /// Get the start of the interval reserved for VarLocs of kind RegisterKind + /// which reside in \p Reg. The end is at rawIndexForReg(Reg+1)-1. + static uint64_t rawIndexForReg(Register Reg) { + return LocIndex(Reg, 0).getAsRawInteger(); + } + + /// Return a range covering all set indices in the interval reserved for + /// \p Location in \p Set. + static auto indexRangeForLocation(const VarLocSet &Set, + u32_location_t Location) { + uint64_t Start = LocIndex(Location, 0).getAsRawInteger(); + uint64_t End = LocIndex(Location + 1, 0).getAsRawInteger(); + return Set.half_open_range(Start, End); + } +}; + +// Simple Set for storing all the VarLoc Indices at a Location bucket. +using VarLocsInRange = SmallSet<LocIndex::u32_index_t, 32>; +// Vector of all `LocIndex`s for a given VarLoc; the same Location should not +// appear in any two of these, as each VarLoc appears at most once in any +// Location bucket. +using LocIndices = SmallVector<LocIndex, 2>; + +class VarLocBasedLDV : public LDVImpl { +private: + const TargetRegisterInfo *TRI; + const TargetInstrInfo *TII; + const TargetFrameLowering *TFI; + TargetPassConfig *TPC; + BitVector CalleeSavedRegs; + LexicalScopes LS; + VarLocSet::Allocator Alloc; + + const MachineInstr *LastNonDbgMI; + + enum struct TransferKind { TransferCopy, TransferSpill, TransferRestore }; + + using FragmentInfo = DIExpression::FragmentInfo; + using OptFragmentInfo = std::optional<DIExpression::FragmentInfo>; + + /// A pair of debug variable and value location. + struct VarLoc { + // The location at which a spilled variable resides. It consists of a + // register and an offset. + struct SpillLoc { + unsigned SpillBase; + StackOffset SpillOffset; + bool operator==(const SpillLoc &Other) const { + return SpillBase == Other.SpillBase && SpillOffset == Other.SpillOffset; + } + bool operator!=(const SpillLoc &Other) const { + return !(*this == Other); + } + }; + + // Target indices used for wasm-specific locations. + struct WasmLoc { + // One of TargetIndex values defined in WebAssembly.h. We deal with + // local-related TargetIndex in this analysis (TI_LOCAL and + // TI_LOCAL_INDIRECT). Stack operands (TI_OPERAND_STACK) will be handled + // separately WebAssemblyDebugFixup pass, and we don't associate debug + // info with values in global operands (TI_GLOBAL_RELOC) at the moment. + int Index; + int64_t Offset; + bool operator==(const WasmLoc &Other) const { + return Index == Other.Index && Offset == Other.Offset; + } + bool operator!=(const WasmLoc &Other) const { return !(*this == Other); } + }; + + /// Identity of the variable at this location. + const DebugVariable Var; + + /// The expression applied to this location. + const DIExpression *Expr; + + /// DBG_VALUE to clone var/expr information from if this location + /// is moved. + const MachineInstr &MI; + + enum class MachineLocKind { + InvalidKind = 0, + RegisterKind, + SpillLocKind, + ImmediateKind, + WasmLocKind + }; + + enum class EntryValueLocKind { + NonEntryValueKind = 0, + EntryValueKind, + EntryValueBackupKind, + EntryValueCopyBackupKind + } EVKind = EntryValueLocKind::NonEntryValueKind; + + /// The value location. Stored separately to avoid repeatedly + /// extracting it from MI. + union MachineLocValue { + uint64_t RegNo; + SpillLoc SpillLocation; + uint64_t Hash; + int64_t Immediate; + const ConstantFP *FPImm; + const ConstantInt *CImm; + WasmLoc WasmLocation; + MachineLocValue() : Hash(0) {} + }; + + /// A single machine location; its Kind is either a register, spill + /// location, or immediate value. + /// If the VarLoc is not a NonEntryValueKind, then it will use only a + /// single MachineLoc of RegisterKind. + struct MachineLoc { + MachineLocKind Kind; + MachineLocValue Value; + bool operator==(const MachineLoc &Other) const { + if (Kind != Other.Kind) + return false; + switch (Kind) { + case MachineLocKind::SpillLocKind: + return Value.SpillLocation == Other.Value.SpillLocation; + case MachineLocKind::WasmLocKind: + return Value.WasmLocation == Other.Value.WasmLocation; + case MachineLocKind::RegisterKind: + case MachineLocKind::ImmediateKind: + return Value.Hash == Other.Value.Hash; + default: + llvm_unreachable("Invalid kind"); + } + } + bool operator<(const MachineLoc &Other) const { + switch (Kind) { + case MachineLocKind::SpillLocKind: + return std::make_tuple( + Kind, Value.SpillLocation.SpillBase, + Value.SpillLocation.SpillOffset.getFixed(), + Value.SpillLocation.SpillOffset.getScalable()) < + std::make_tuple( + Other.Kind, Other.Value.SpillLocation.SpillBase, + Other.Value.SpillLocation.SpillOffset.getFixed(), + Other.Value.SpillLocation.SpillOffset.getScalable()); + case MachineLocKind::WasmLocKind: + return std::make_tuple(Kind, Value.WasmLocation.Index, + Value.WasmLocation.Offset) < + std::make_tuple(Other.Kind, Other.Value.WasmLocation.Index, + Other.Value.WasmLocation.Offset); + case MachineLocKind::RegisterKind: + case MachineLocKind::ImmediateKind: + return std::tie(Kind, Value.Hash) < + std::tie(Other.Kind, Other.Value.Hash); + default: + llvm_unreachable("Invalid kind"); + } + } + }; + + /// The set of machine locations used to determine the variable's value, in + /// conjunction with Expr. Initially populated with MI's debug operands, + /// but may be transformed independently afterwards. + SmallVector<MachineLoc, 8> Locs; + /// Used to map the index of each location in Locs back to the index of its + /// original debug operand in MI. Used when multiple location operands are + /// coalesced and the original MI's operands need to be accessed while + /// emitting a debug value. + SmallVector<unsigned, 8> OrigLocMap; + + VarLoc(const MachineInstr &MI) + : Var(MI.getDebugVariable(), MI.getDebugExpression(), + MI.getDebugLoc()->getInlinedAt()), + Expr(MI.getDebugExpression()), MI(MI) { + assert(MI.isDebugValue() && "not a DBG_VALUE"); + assert((MI.isDebugValueList() || MI.getNumOperands() == 4) && + "malformed DBG_VALUE"); + for (const MachineOperand &Op : MI.debug_operands()) { + MachineLoc ML = GetLocForOp(Op); + auto It = find(Locs, ML); + if (It == Locs.end()) { + Locs.push_back(ML); + OrigLocMap.push_back(MI.getDebugOperandIndex(&Op)); + } else { + // ML duplicates an element in Locs; replace references to Op + // with references to the duplicating element. + unsigned OpIdx = Locs.size(); + unsigned DuplicatingIdx = std::distance(Locs.begin(), It); + Expr = DIExpression::replaceArg(Expr, OpIdx, DuplicatingIdx); + } + } + + // We create the debug entry values from the factory functions rather + // than from this ctor. + assert(EVKind != EntryValueLocKind::EntryValueKind && + !isEntryBackupLoc()); + } + + static MachineLoc GetLocForOp(const MachineOperand &Op) { + MachineLocKind Kind; + MachineLocValue Loc; + if (Op.isReg()) { + Kind = MachineLocKind::RegisterKind; + Loc.RegNo = Op.getReg(); + } else if (Op.isImm()) { + Kind = MachineLocKind::ImmediateKind; + Loc.Immediate = Op.getImm(); + } else if (Op.isFPImm()) { + Kind = MachineLocKind::ImmediateKind; + Loc.FPImm = Op.getFPImm(); + } else if (Op.isCImm()) { + Kind = MachineLocKind::ImmediateKind; + Loc.CImm = Op.getCImm(); + } else if (Op.isTargetIndex()) { + Kind = MachineLocKind::WasmLocKind; + Loc.WasmLocation = {Op.getIndex(), Op.getOffset()}; + } else + llvm_unreachable("Invalid Op kind for MachineLoc."); + return {Kind, Loc}; + } + + /// Take the variable and machine-location in DBG_VALUE MI, and build an + /// entry location using the given expression. + static VarLoc CreateEntryLoc(const MachineInstr &MI, + const DIExpression *EntryExpr, Register Reg) { + VarLoc VL(MI); + assert(VL.Locs.size() == 1 && + VL.Locs[0].Kind == MachineLocKind::RegisterKind); + VL.EVKind = EntryValueLocKind::EntryValueKind; + VL.Expr = EntryExpr; + VL.Locs[0].Value.RegNo = Reg; + return VL; + } + + /// Take the variable and machine-location from the DBG_VALUE (from the + /// function entry), and build an entry value backup location. The backup + /// location will turn into the normal location if the backup is valid at + /// the time of the primary location clobbering. + static VarLoc CreateEntryBackupLoc(const MachineInstr &MI, + const DIExpression *EntryExpr) { + VarLoc VL(MI); + assert(VL.Locs.size() == 1 && + VL.Locs[0].Kind == MachineLocKind::RegisterKind); + VL.EVKind = EntryValueLocKind::EntryValueBackupKind; + VL.Expr = EntryExpr; + return VL; + } + + /// Take the variable and machine-location from the DBG_VALUE (from the + /// function entry), and build a copy of an entry value backup location by + /// setting the register location to NewReg. + static VarLoc CreateEntryCopyBackupLoc(const MachineInstr &MI, + const DIExpression *EntryExpr, + Register NewReg) { + VarLoc VL(MI); + assert(VL.Locs.size() == 1 && + VL.Locs[0].Kind == MachineLocKind::RegisterKind); + VL.EVKind = EntryValueLocKind::EntryValueCopyBackupKind; + VL.Expr = EntryExpr; + VL.Locs[0].Value.RegNo = NewReg; + return VL; + } + + /// Copy the register location in DBG_VALUE MI, updating the register to + /// be NewReg. + static VarLoc CreateCopyLoc(const VarLoc &OldVL, const MachineLoc &OldML, + Register NewReg) { + VarLoc VL = OldVL; + for (MachineLoc &ML : VL.Locs) + if (ML == OldML) { + ML.Kind = MachineLocKind::RegisterKind; + ML.Value.RegNo = NewReg; + return VL; + } + llvm_unreachable("Should have found OldML in new VarLoc."); + } + + /// Take the variable described by DBG_VALUE* MI, and create a VarLoc + /// locating it in the specified spill location. + static VarLoc CreateSpillLoc(const VarLoc &OldVL, const MachineLoc &OldML, + unsigned SpillBase, StackOffset SpillOffset) { + VarLoc VL = OldVL; + for (MachineLoc &ML : VL.Locs) + if (ML == OldML) { + ML.Kind = MachineLocKind::SpillLocKind; + ML.Value.SpillLocation = {SpillBase, SpillOffset}; + return VL; + } + llvm_unreachable("Should have found OldML in new VarLoc."); + } + + /// Create a DBG_VALUE representing this VarLoc in the given function. + /// Copies variable-specific information such as DILocalVariable and + /// inlining information from the original DBG_VALUE instruction, which may + /// have been several transfers ago. + MachineInstr *BuildDbgValue(MachineFunction &MF) const { + assert(!isEntryBackupLoc() && + "Tried to produce DBG_VALUE for backup VarLoc"); + const DebugLoc &DbgLoc = MI.getDebugLoc(); + bool Indirect = MI.isIndirectDebugValue(); + const auto &IID = MI.getDesc(); + const DILocalVariable *Var = MI.getDebugVariable(); + NumInserted++; + + const DIExpression *DIExpr = Expr; + SmallVector<MachineOperand, 8> MOs; + for (unsigned I = 0, E = Locs.size(); I < E; ++I) { + MachineLocKind LocKind = Locs[I].Kind; + MachineLocValue Loc = Locs[I].Value; + const MachineOperand &Orig = MI.getDebugOperand(OrigLocMap[I]); + switch (LocKind) { + case MachineLocKind::RegisterKind: + // An entry value is a register location -- but with an updated + // expression. The register location of such DBG_VALUE is always the + // one from the entry DBG_VALUE, it does not matter if the entry value + // was copied in to another register due to some optimizations. + // Non-entry value register locations are like the source + // DBG_VALUE, but with the register number from this VarLoc. + MOs.push_back(MachineOperand::CreateReg( + EVKind == EntryValueLocKind::EntryValueKind ? Orig.getReg() + : Register(Loc.RegNo), + false)); + break; + case MachineLocKind::SpillLocKind: { + // Spills are indirect DBG_VALUEs, with a base register and offset. + // Use the original DBG_VALUEs expression to build the spilt location + // on top of. FIXME: spill locations created before this pass runs + // are not recognized, and not handled here. + unsigned Base = Loc.SpillLocation.SpillBase; + auto *TRI = MF.getSubtarget().getRegisterInfo(); + if (MI.isNonListDebugValue()) { + auto Deref = Indirect ? DIExpression::DerefAfter : 0; + DIExpr = TRI->prependOffsetExpression( + DIExpr, DIExpression::ApplyOffset | Deref, + Loc.SpillLocation.SpillOffset); + Indirect = true; + } else { + SmallVector<uint64_t, 4> Ops; + TRI->getOffsetOpcodes(Loc.SpillLocation.SpillOffset, Ops); + Ops.push_back(dwarf::DW_OP_deref); + DIExpr = DIExpression::appendOpsToArg(DIExpr, Ops, I); + } + MOs.push_back(MachineOperand::CreateReg(Base, false)); + break; + } + case MachineLocKind::ImmediateKind: { + MOs.push_back(Orig); + break; + } + case MachineLocKind::WasmLocKind: { + MOs.push_back(Orig); + break; + } + case MachineLocKind::InvalidKind: + llvm_unreachable("Tried to produce DBG_VALUE for invalid VarLoc"); + } + } + return BuildMI(MF, DbgLoc, IID, Indirect, MOs, Var, DIExpr); + } + + /// Is the Loc field a constant or constant object? + bool isConstant(MachineLocKind Kind) const { + return Kind == MachineLocKind::ImmediateKind; + } + + /// Check if the Loc field is an entry backup location. + bool isEntryBackupLoc() const { + return EVKind == EntryValueLocKind::EntryValueBackupKind || + EVKind == EntryValueLocKind::EntryValueCopyBackupKind; + } + + /// If this variable is described by register \p Reg holding the entry + /// value, return true. + bool isEntryValueBackupReg(Register Reg) const { + return EVKind == EntryValueLocKind::EntryValueBackupKind && usesReg(Reg); + } + + /// If this variable is described by register \p Reg holding a copy of the + /// entry value, return true. + bool isEntryValueCopyBackupReg(Register Reg) const { + return EVKind == EntryValueLocKind::EntryValueCopyBackupKind && + usesReg(Reg); + } + + /// If this variable is described in whole or part by \p Reg, return true. + bool usesReg(Register Reg) const { + MachineLoc RegML; + RegML.Kind = MachineLocKind::RegisterKind; + RegML.Value.RegNo = Reg; + return is_contained(Locs, RegML); + } + + /// If this variable is described in whole or part by \p Reg, return true. + unsigned getRegIdx(Register Reg) const { + for (unsigned Idx = 0; Idx < Locs.size(); ++Idx) + if (Locs[Idx].Kind == MachineLocKind::RegisterKind && + Register{static_cast<unsigned>(Locs[Idx].Value.RegNo)} == Reg) + return Idx; + llvm_unreachable("Could not find given Reg in Locs"); + } + + /// If this variable is described in whole or part by 1 or more registers, + /// add each of them to \p Regs and return true. + bool getDescribingRegs(SmallVectorImpl<uint32_t> &Regs) const { + bool AnyRegs = false; + for (const auto &Loc : Locs) + if (Loc.Kind == MachineLocKind::RegisterKind) { + Regs.push_back(Loc.Value.RegNo); + AnyRegs = true; + } + return AnyRegs; + } + + bool containsSpillLocs() const { + return any_of(Locs, [](VarLoc::MachineLoc ML) { + return ML.Kind == VarLoc::MachineLocKind::SpillLocKind; + }); + } + + /// If this variable is described in whole or part by \p SpillLocation, + /// return true. + bool usesSpillLoc(SpillLoc SpillLocation) const { + MachineLoc SpillML; + SpillML.Kind = MachineLocKind::SpillLocKind; + SpillML.Value.SpillLocation = SpillLocation; + return is_contained(Locs, SpillML); + } + + /// If this variable is described in whole or part by \p SpillLocation, + /// return the index . + unsigned getSpillLocIdx(SpillLoc SpillLocation) const { + for (unsigned Idx = 0; Idx < Locs.size(); ++Idx) + if (Locs[Idx].Kind == MachineLocKind::SpillLocKind && + Locs[Idx].Value.SpillLocation == SpillLocation) + return Idx; + llvm_unreachable("Could not find given SpillLoc in Locs"); + } + + bool containsWasmLocs() const { + return any_of(Locs, [](VarLoc::MachineLoc ML) { + return ML.Kind == VarLoc::MachineLocKind::WasmLocKind; + }); + } + + /// If this variable is described in whole or part by \p WasmLocation, + /// return true. + bool usesWasmLoc(WasmLoc WasmLocation) const { + MachineLoc WasmML; + WasmML.Kind = MachineLocKind::WasmLocKind; + WasmML.Value.WasmLocation = WasmLocation; + return is_contained(Locs, WasmML); + } + + /// Determine whether the lexical scope of this value's debug location + /// dominates MBB. + bool dominates(LexicalScopes &LS, MachineBasicBlock &MBB) const { + return LS.dominates(MI.getDebugLoc().get(), &MBB); + } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + // TRI and TII can be null. + void dump(const TargetRegisterInfo *TRI, const TargetInstrInfo *TII, + raw_ostream &Out = dbgs()) const { + Out << "VarLoc("; + for (const MachineLoc &MLoc : Locs) { + if (Locs.begin() != &MLoc) + Out << ", "; + switch (MLoc.Kind) { + case MachineLocKind::RegisterKind: + Out << printReg(MLoc.Value.RegNo, TRI); + break; + case MachineLocKind::SpillLocKind: + Out << printReg(MLoc.Value.SpillLocation.SpillBase, TRI); + Out << "[" << MLoc.Value.SpillLocation.SpillOffset.getFixed() << " + " + << MLoc.Value.SpillLocation.SpillOffset.getScalable() + << "x vscale" + << "]"; + break; + case MachineLocKind::ImmediateKind: + Out << MLoc.Value.Immediate; + break; + case MachineLocKind::WasmLocKind: { + if (TII) { + auto Indices = TII->getSerializableTargetIndices(); + auto Found = + find_if(Indices, [&](const std::pair<int, const char *> &I) { + return I.first == MLoc.Value.WasmLocation.Index; + }); + assert(Found != Indices.end()); + Out << Found->second; + if (MLoc.Value.WasmLocation.Offset > 0) + Out << " + " << MLoc.Value.WasmLocation.Offset; + } else { + Out << "WasmLoc"; + } + break; + } + case MachineLocKind::InvalidKind: + llvm_unreachable("Invalid VarLoc in dump method"); + } + } + + Out << ", \"" << Var.getVariable()->getName() << "\", " << *Expr << ", "; + if (Var.getInlinedAt()) + Out << "!" << Var.getInlinedAt()->getMetadataID() << ")\n"; + else + Out << "(null))"; + + if (isEntryBackupLoc()) + Out << " (backup loc)\n"; + else + Out << "\n"; + } +#endif + + bool operator==(const VarLoc &Other) const { + return std::tie(EVKind, Var, Expr, Locs) == + std::tie(Other.EVKind, Other.Var, Other.Expr, Other.Locs); + } + + /// This operator guarantees that VarLocs are sorted by Variable first. + bool operator<(const VarLoc &Other) const { + return std::tie(Var, EVKind, Locs, Expr) < + std::tie(Other.Var, Other.EVKind, Other.Locs, Other.Expr); + } + }; + +#ifndef NDEBUG + using VarVec = SmallVector<VarLoc, 32>; +#endif + + /// VarLocMap is used for two things: + /// 1) Assigning LocIndices to a VarLoc. The LocIndices can be used to + /// virtually insert a VarLoc into a VarLocSet. + /// 2) Given a LocIndex, look up the unique associated VarLoc. + class VarLocMap { + /// Map a VarLoc to an index within the vector reserved for its location + /// within Loc2Vars. + std::map<VarLoc, LocIndices> Var2Indices; + + /// Map a location to a vector which holds VarLocs which live in that + /// location. + SmallDenseMap<LocIndex::u32_location_t, std::vector<VarLoc>> Loc2Vars; + + public: + /// Retrieve LocIndices for \p VL. + LocIndices insert(const VarLoc &VL) { + LocIndices &Indices = Var2Indices[VL]; + // If Indices is not empty, VL is already in the map. + if (!Indices.empty()) + return Indices; + SmallVector<LocIndex::u32_location_t, 4> Locations; + // LocIndices are determined by EVKind and MLs; each Register has a + // unique location, while all SpillLocs use a single bucket, and any EV + // VarLocs use only the Backup bucket or none at all (except the + // compulsory entry at the universal location index). LocIndices will + // always have an index at the universal location index as the last index. + if (VL.EVKind == VarLoc::EntryValueLocKind::NonEntryValueKind) { + VL.getDescribingRegs(Locations); + assert(all_of(Locations, + [](auto RegNo) { + return RegNo < LocIndex::kFirstInvalidRegLocation; + }) && + "Physreg out of range?"); + if (VL.containsSpillLocs()) + Locations.push_back(LocIndex::kSpillLocation); + if (VL.containsWasmLocs()) + Locations.push_back(LocIndex::kWasmLocation); + } else if (VL.EVKind != VarLoc::EntryValueLocKind::EntryValueKind) { + LocIndex::u32_location_t Loc = LocIndex::kEntryValueBackupLocation; + Locations.push_back(Loc); + } + Locations.push_back(LocIndex::kUniversalLocation); + for (LocIndex::u32_location_t Location : Locations) { + auto &Vars = Loc2Vars[Location]; + Indices.push_back( + {Location, static_cast<LocIndex::u32_index_t>(Vars.size())}); + Vars.push_back(VL); + } + return Indices; + } + + LocIndices getAllIndices(const VarLoc &VL) const { + auto IndIt = Var2Indices.find(VL); + assert(IndIt != Var2Indices.end() && "VarLoc not tracked"); + return IndIt->second; + } + + /// Retrieve the unique VarLoc associated with \p ID. + const VarLoc &operator[](LocIndex ID) const { + auto LocIt = Loc2Vars.find(ID.Location); + assert(LocIt != Loc2Vars.end() && "Location not tracked"); + return LocIt->second[ID.Index]; + } + }; + + using VarLocInMBB = + SmallDenseMap<const MachineBasicBlock *, std::unique_ptr<VarLocSet>>; + struct TransferDebugPair { + MachineInstr *TransferInst; ///< Instruction where this transfer occurs. + LocIndex LocationID; ///< Location number for the transfer dest. + }; + using TransferMap = SmallVector<TransferDebugPair, 4>; + // Types for recording Entry Var Locations emitted by a single MachineInstr, + // as well as recording MachineInstr which last defined a register. + using InstToEntryLocMap = std::multimap<const MachineInstr *, LocIndex>; + using RegDefToInstMap = DenseMap<Register, MachineInstr *>; + + // Types for recording sets of variable fragments that overlap. For a given + // local variable, we record all other fragments of that variable that could + // overlap it, to reduce search time. + using FragmentOfVar = + std::pair<const DILocalVariable *, DIExpression::FragmentInfo>; + using OverlapMap = + DenseMap<FragmentOfVar, SmallVector<DIExpression::FragmentInfo, 1>>; + + // Helper while building OverlapMap, a map of all fragments seen for a given + // DILocalVariable. + using VarToFragments = + DenseMap<const DILocalVariable *, SmallSet<FragmentInfo, 4>>; + + /// Collects all VarLocs from \p CollectFrom. Each unique VarLoc is added + /// to \p Collected once, in order of insertion into \p VarLocIDs. + static void collectAllVarLocs(SmallVectorImpl<VarLoc> &Collected, + const VarLocSet &CollectFrom, + const VarLocMap &VarLocIDs); + + /// Get the registers which are used by VarLocs of kind RegisterKind tracked + /// by \p CollectFrom. + void getUsedRegs(const VarLocSet &CollectFrom, + SmallVectorImpl<Register> &UsedRegs) const; + + /// This holds the working set of currently open ranges. For fast + /// access, this is done both as a set of VarLocIDs, and a map of + /// DebugVariable to recent VarLocID. Note that a DBG_VALUE ends all + /// previous open ranges for the same variable. In addition, we keep + /// two different maps (Vars/EntryValuesBackupVars), so erase/insert + /// methods act differently depending on whether a VarLoc is primary + /// location or backup one. In the case the VarLoc is backup location + /// we will erase/insert from the EntryValuesBackupVars map, otherwise + /// we perform the operation on the Vars. + class OpenRangesSet { + VarLocSet::Allocator &Alloc; + VarLocSet VarLocs; + // Map the DebugVariable to recent primary location ID. + SmallDenseMap<DebugVariable, LocIndices, 8> Vars; + // Map the DebugVariable to recent backup location ID. + SmallDenseMap<DebugVariable, LocIndices, 8> EntryValuesBackupVars; + OverlapMap &OverlappingFragments; + + public: + OpenRangesSet(VarLocSet::Allocator &Alloc, OverlapMap &_OLapMap) + : Alloc(Alloc), VarLocs(Alloc), OverlappingFragments(_OLapMap) {} + + const VarLocSet &getVarLocs() const { return VarLocs; } + + // Fetches all VarLocs in \p VarLocIDs and inserts them into \p Collected. + // This method is needed to get every VarLoc once, as each VarLoc may have + // multiple indices in a VarLocMap (corresponding to each applicable + // location), but all VarLocs appear exactly once at the universal location + // index. + void getUniqueVarLocs(SmallVectorImpl<VarLoc> &Collected, + const VarLocMap &VarLocIDs) const { + collectAllVarLocs(Collected, VarLocs, VarLocIDs); + } + + /// Terminate all open ranges for VL.Var by removing it from the set. + void erase(const VarLoc &VL); + + /// Terminate all open ranges listed as indices in \c KillSet with + /// \c Location by removing them from the set. + void erase(const VarLocsInRange &KillSet, const VarLocMap &VarLocIDs, + LocIndex::u32_location_t Location); + + /// Insert a new range into the set. + void insert(LocIndices VarLocIDs, const VarLoc &VL); + + /// Insert a set of ranges. + void insertFromLocSet(const VarLocSet &ToLoad, const VarLocMap &Map); + + std::optional<LocIndices> getEntryValueBackup(DebugVariable Var); + + /// Empty the set. + void clear() { + VarLocs.clear(); + Vars.clear(); + EntryValuesBackupVars.clear(); + } + + /// Return whether the set is empty or not. + bool empty() const { + assert(Vars.empty() == EntryValuesBackupVars.empty() && + Vars.empty() == VarLocs.empty() && + "open ranges are inconsistent"); + return VarLocs.empty(); + } + + /// Get an empty range of VarLoc IDs. + auto getEmptyVarLocRange() const { + return iterator_range<VarLocSet::const_iterator>(getVarLocs().end(), + getVarLocs().end()); + } + + /// Get all set IDs for VarLocs with MLs of kind RegisterKind in \p Reg. + auto getRegisterVarLocs(Register Reg) const { + return LocIndex::indexRangeForLocation(getVarLocs(), Reg); + } + + /// Get all set IDs for VarLocs with MLs of kind SpillLocKind. + auto getSpillVarLocs() const { + return LocIndex::indexRangeForLocation(getVarLocs(), + LocIndex::kSpillLocation); + } + + /// Get all set IDs for VarLocs of EVKind EntryValueBackupKind or + /// EntryValueCopyBackupKind. + auto getEntryValueBackupVarLocs() const { + return LocIndex::indexRangeForLocation( + getVarLocs(), LocIndex::kEntryValueBackupLocation); + } + + /// Get all set IDs for VarLocs with MLs of kind WasmLocKind. + auto getWasmVarLocs() const { + return LocIndex::indexRangeForLocation(getVarLocs(), + LocIndex::kWasmLocation); + } + }; + + /// Collect all VarLoc IDs from \p CollectFrom for VarLocs with MLs of kind + /// RegisterKind which are located in any reg in \p Regs. The IDs for each + /// VarLoc correspond to entries in the universal location bucket, which every + /// VarLoc has exactly 1 entry for. Insert collected IDs into \p Collected. + static void collectIDsForRegs(VarLocsInRange &Collected, + const DefinedRegsSet &Regs, + const VarLocSet &CollectFrom, + const VarLocMap &VarLocIDs); + + VarLocSet &getVarLocsInMBB(const MachineBasicBlock *MBB, VarLocInMBB &Locs) { + std::unique_ptr<VarLocSet> &VLS = Locs[MBB]; + if (!VLS) + VLS = std::make_unique<VarLocSet>(Alloc); + return *VLS; + } + + const VarLocSet &getVarLocsInMBB(const MachineBasicBlock *MBB, + const VarLocInMBB &Locs) const { + auto It = Locs.find(MBB); + assert(It != Locs.end() && "MBB not in map"); + return *It->second; + } + + /// Tests whether this instruction is a spill to a stack location. + bool isSpillInstruction(const MachineInstr &MI, MachineFunction *MF); + + /// Decide if @MI is a spill instruction and return true if it is. We use 2 + /// criteria to make this decision: + /// - Is this instruction a store to a spill slot? + /// - Is there a register operand that is both used and killed? + /// TODO: Store optimization can fold spills into other stores (including + /// other spills). We do not handle this yet (more than one memory operand). + bool isLocationSpill(const MachineInstr &MI, MachineFunction *MF, + Register &Reg); + + /// Returns true if the given machine instruction is a debug value which we + /// can emit entry values for. + /// + /// Currently, we generate debug entry values only for parameters that are + /// unmodified throughout the function and located in a register. + bool isEntryValueCandidate(const MachineInstr &MI, + const DefinedRegsSet &Regs) const; + + /// If a given instruction is identified as a spill, return the spill location + /// and set \p Reg to the spilled register. + std::optional<VarLoc::SpillLoc> isRestoreInstruction(const MachineInstr &MI, + MachineFunction *MF, + Register &Reg); + /// Given a spill instruction, extract the register and offset used to + /// address the spill location in a target independent way. + VarLoc::SpillLoc extractSpillBaseRegAndOffset(const MachineInstr &MI); + void insertTransferDebugPair(MachineInstr &MI, OpenRangesSet &OpenRanges, + TransferMap &Transfers, VarLocMap &VarLocIDs, + LocIndex OldVarID, TransferKind Kind, + const VarLoc::MachineLoc &OldLoc, + Register NewReg = Register()); + + void transferDebugValue(const MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + InstToEntryLocMap &EntryValTransfers, + RegDefToInstMap &RegSetInstrs); + void transferSpillOrRestoreInst(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, TransferMap &Transfers); + void cleanupEntryValueTransfers(const MachineInstr *MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, const VarLoc &EntryVL, + InstToEntryLocMap &EntryValTransfers); + void removeEntryValue(const MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, const VarLoc &EntryVL, + InstToEntryLocMap &EntryValTransfers, + RegDefToInstMap &RegSetInstrs); + void emitEntryValues(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + InstToEntryLocMap &EntryValTransfers, + VarLocsInRange &KillSet); + void recordEntryValue(const MachineInstr &MI, + const DefinedRegsSet &DefinedRegs, + OpenRangesSet &OpenRanges, VarLocMap &VarLocIDs); + void transferRegisterCopy(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, TransferMap &Transfers); + void transferRegisterDef(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + InstToEntryLocMap &EntryValTransfers, + RegDefToInstMap &RegSetInstrs); + void transferWasmDef(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs); + bool transferTerminator(MachineBasicBlock *MBB, OpenRangesSet &OpenRanges, + VarLocInMBB &OutLocs, const VarLocMap &VarLocIDs); + + void process(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, TransferMap &Transfers, + InstToEntryLocMap &EntryValTransfers, + RegDefToInstMap &RegSetInstrs); + + void accumulateFragmentMap(MachineInstr &MI, VarToFragments &SeenFragments, + OverlapMap &OLapMap); + + bool join(MachineBasicBlock &MBB, VarLocInMBB &OutLocs, VarLocInMBB &InLocs, + const VarLocMap &VarLocIDs, + SmallPtrSet<const MachineBasicBlock *, 16> &Visited, + SmallPtrSetImpl<const MachineBasicBlock *> &ArtificialBlocks); + + /// Create DBG_VALUE insts for inlocs that have been propagated but + /// had their instruction creation deferred. + void flushPendingLocs(VarLocInMBB &PendingInLocs, VarLocMap &VarLocIDs); + + bool ExtendRanges(MachineFunction &MF, MachineDominatorTree *DomTree, + TargetPassConfig *TPC, unsigned InputBBLimit, + unsigned InputDbgValLimit) override; + +public: + /// Default construct and initialize the pass. + VarLocBasedLDV(); + + ~VarLocBasedLDV(); + + /// Print to ostream with a message. + void printVarLocInMBB(const MachineFunction &MF, const VarLocInMBB &V, + const VarLocMap &VarLocIDs, const char *msg, + raw_ostream &Out) const; +}; + +} // end anonymous namespace + +//===----------------------------------------------------------------------===// +// Implementation +//===----------------------------------------------------------------------===// + +VarLocBasedLDV::VarLocBasedLDV() = default; + +VarLocBasedLDV::~VarLocBasedLDV() = default; + +/// Erase a variable from the set of open ranges, and additionally erase any +/// fragments that may overlap it. If the VarLoc is a backup location, erase +/// the variable from the EntryValuesBackupVars set, indicating we should stop +/// tracking its backup entry location. Otherwise, if the VarLoc is primary +/// location, erase the variable from the Vars set. +void VarLocBasedLDV::OpenRangesSet::erase(const VarLoc &VL) { + // Erasure helper. + auto DoErase = [VL, this](DebugVariable VarToErase) { + auto *EraseFrom = VL.isEntryBackupLoc() ? &EntryValuesBackupVars : &Vars; + auto It = EraseFrom->find(VarToErase); + if (It != EraseFrom->end()) { + LocIndices IDs = It->second; + for (LocIndex ID : IDs) + VarLocs.reset(ID.getAsRawInteger()); + EraseFrom->erase(It); + } + }; + + DebugVariable Var = VL.Var; + + // Erase the variable/fragment that ends here. + DoErase(Var); + + // Extract the fragment. Interpret an empty fragment as one that covers all + // possible bits. + FragmentInfo ThisFragment = Var.getFragmentOrDefault(); + + // There may be fragments that overlap the designated fragment. Look them up + // in the pre-computed overlap map, and erase them too. + auto MapIt = OverlappingFragments.find({Var.getVariable(), ThisFragment}); + if (MapIt != OverlappingFragments.end()) { + for (auto Fragment : MapIt->second) { + VarLocBasedLDV::OptFragmentInfo FragmentHolder; + if (!DebugVariable::isDefaultFragment(Fragment)) + FragmentHolder = VarLocBasedLDV::OptFragmentInfo(Fragment); + DoErase({Var.getVariable(), FragmentHolder, Var.getInlinedAt()}); + } + } +} + +void VarLocBasedLDV::OpenRangesSet::erase(const VarLocsInRange &KillSet, + const VarLocMap &VarLocIDs, + LocIndex::u32_location_t Location) { + VarLocSet RemoveSet(Alloc); + for (LocIndex::u32_index_t ID : KillSet) { + const VarLoc &VL = VarLocIDs[LocIndex(Location, ID)]; + auto *EraseFrom = VL.isEntryBackupLoc() ? &EntryValuesBackupVars : &Vars; + EraseFrom->erase(VL.Var); + LocIndices VLI = VarLocIDs.getAllIndices(VL); + for (LocIndex ID : VLI) + RemoveSet.set(ID.getAsRawInteger()); + } + VarLocs.intersectWithComplement(RemoveSet); +} + +void VarLocBasedLDV::OpenRangesSet::insertFromLocSet(const VarLocSet &ToLoad, + const VarLocMap &Map) { + VarLocsInRange UniqueVarLocIDs; + DefinedRegsSet Regs; + Regs.insert(LocIndex::kUniversalLocation); + collectIDsForRegs(UniqueVarLocIDs, Regs, ToLoad, Map); + for (uint64_t ID : UniqueVarLocIDs) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VarL = Map[Idx]; + const LocIndices Indices = Map.getAllIndices(VarL); + insert(Indices, VarL); + } +} + +void VarLocBasedLDV::OpenRangesSet::insert(LocIndices VarLocIDs, + const VarLoc &VL) { + auto *InsertInto = VL.isEntryBackupLoc() ? &EntryValuesBackupVars : &Vars; + for (LocIndex ID : VarLocIDs) + VarLocs.set(ID.getAsRawInteger()); + InsertInto->insert({VL.Var, VarLocIDs}); +} + +/// Return the Loc ID of an entry value backup location, if it exists for the +/// variable. +std::optional<LocIndices> +VarLocBasedLDV::OpenRangesSet::getEntryValueBackup(DebugVariable Var) { + auto It = EntryValuesBackupVars.find(Var); + if (It != EntryValuesBackupVars.end()) + return It->second; + + return std::nullopt; +} + +void VarLocBasedLDV::collectIDsForRegs(VarLocsInRange &Collected, + const DefinedRegsSet &Regs, + const VarLocSet &CollectFrom, + const VarLocMap &VarLocIDs) { + assert(!Regs.empty() && "Nothing to collect"); + SmallVector<Register, 32> SortedRegs; + append_range(SortedRegs, Regs); + array_pod_sort(SortedRegs.begin(), SortedRegs.end()); + auto It = CollectFrom.find(LocIndex::rawIndexForReg(SortedRegs.front())); + auto End = CollectFrom.end(); + for (Register Reg : SortedRegs) { + // The half-open interval [FirstIndexForReg, FirstInvalidIndex) contains + // all possible VarLoc IDs for VarLocs with MLs of kind RegisterKind which + // live in Reg. + uint64_t FirstIndexForReg = LocIndex::rawIndexForReg(Reg); + uint64_t FirstInvalidIndex = LocIndex::rawIndexForReg(Reg + 1); + It.advanceToLowerBound(FirstIndexForReg); + + // Iterate through that half-open interval and collect all the set IDs. + for (; It != End && *It < FirstInvalidIndex; ++It) { + LocIndex ItIdx = LocIndex::fromRawInteger(*It); + const VarLoc &VL = VarLocIDs[ItIdx]; + LocIndices LI = VarLocIDs.getAllIndices(VL); + // For now, the back index is always the universal location index. + assert(LI.back().Location == LocIndex::kUniversalLocation && + "Unexpected order of LocIndices for VarLoc; was it inserted into " + "the VarLocMap correctly?"); + Collected.insert(LI.back().Index); + } + + if (It == End) + return; + } +} + +void VarLocBasedLDV::getUsedRegs(const VarLocSet &CollectFrom, + SmallVectorImpl<Register> &UsedRegs) const { + // All register-based VarLocs are assigned indices greater than or equal to + // FirstRegIndex. + uint64_t FirstRegIndex = + LocIndex::rawIndexForReg(LocIndex::kFirstRegLocation); + uint64_t FirstInvalidIndex = + LocIndex::rawIndexForReg(LocIndex::kFirstInvalidRegLocation); + for (auto It = CollectFrom.find(FirstRegIndex), + End = CollectFrom.find(FirstInvalidIndex); + It != End;) { + // We found a VarLoc ID for a VarLoc that lives in a register. Figure out + // which register and add it to UsedRegs. + uint32_t FoundReg = LocIndex::fromRawInteger(*It).Location; + assert((UsedRegs.empty() || FoundReg != UsedRegs.back()) && + "Duplicate used reg"); + UsedRegs.push_back(FoundReg); + + // Skip to the next /set/ register. Note that this finds a lower bound, so + // even if there aren't any VarLocs living in `FoundReg+1`, we're still + // guaranteed to move on to the next register (or to end()). + uint64_t NextRegIndex = LocIndex::rawIndexForReg(FoundReg + 1); + It.advanceToLowerBound(NextRegIndex); + } +} + +//===----------------------------------------------------------------------===// +// Debug Range Extension Implementation +//===----------------------------------------------------------------------===// + +#ifndef NDEBUG +void VarLocBasedLDV::printVarLocInMBB(const MachineFunction &MF, + const VarLocInMBB &V, + const VarLocMap &VarLocIDs, + const char *msg, + raw_ostream &Out) const { + Out << '\n' << msg << '\n'; + for (const MachineBasicBlock &BB : MF) { + if (!V.count(&BB)) + continue; + const VarLocSet &L = getVarLocsInMBB(&BB, V); + if (L.empty()) + continue; + SmallVector<VarLoc, 32> VarLocs; + collectAllVarLocs(VarLocs, L, VarLocIDs); + Out << "MBB: " << BB.getNumber() << ":\n"; + for (const VarLoc &VL : VarLocs) { + Out << " Var: " << VL.Var.getVariable()->getName(); + Out << " MI: "; + VL.dump(TRI, TII, Out); + } + } + Out << "\n"; +} +#endif + +VarLocBasedLDV::VarLoc::SpillLoc +VarLocBasedLDV::extractSpillBaseRegAndOffset(const MachineInstr &MI) { + assert(MI.hasOneMemOperand() && + "Spill instruction does not have exactly one memory operand?"); + auto MMOI = MI.memoperands_begin(); + const PseudoSourceValue *PVal = (*MMOI)->getPseudoValue(); + assert(PVal->kind() == PseudoSourceValue::FixedStack && + "Inconsistent memory operand in spill instruction"); + int FI = cast<FixedStackPseudoSourceValue>(PVal)->getFrameIndex(); + const MachineBasicBlock *MBB = MI.getParent(); + Register Reg; + StackOffset Offset = TFI->getFrameIndexReference(*MBB->getParent(), FI, Reg); + return {Reg, Offset}; +} + +/// Do cleanup of \p EntryValTransfers created by \p TRInst, by removing the +/// Transfer, which uses the to-be-deleted \p EntryVL. +void VarLocBasedLDV::cleanupEntryValueTransfers( + const MachineInstr *TRInst, OpenRangesSet &OpenRanges, VarLocMap &VarLocIDs, + const VarLoc &EntryVL, InstToEntryLocMap &EntryValTransfers) { + if (EntryValTransfers.empty() || TRInst == nullptr) + return; + + auto TransRange = EntryValTransfers.equal_range(TRInst); + for (auto TDPair : llvm::make_range(TransRange.first, TransRange.second)) { + const VarLoc &EmittedEV = VarLocIDs[TDPair.second]; + if (std::tie(EntryVL.Var, EntryVL.Locs[0].Value.RegNo, EntryVL.Expr) == + std::tie(EmittedEV.Var, EmittedEV.Locs[0].Value.RegNo, + EmittedEV.Expr)) { + OpenRanges.erase(EmittedEV); + EntryValTransfers.erase(TRInst); + break; + } + } +} + +/// Try to salvage the debug entry value if we encounter a new debug value +/// describing the same parameter, otherwise stop tracking the value. Return +/// true if we should stop tracking the entry value and do the cleanup of +/// emitted Entry Value Transfers, otherwise return false. +void VarLocBasedLDV::removeEntryValue(const MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + const VarLoc &EntryVL, + InstToEntryLocMap &EntryValTransfers, + RegDefToInstMap &RegSetInstrs) { + // Skip the DBG_VALUE which is the debug entry value itself. + if (&MI == &EntryVL.MI) + return; + + // If the parameter's location is not register location, we can not track + // the entry value any more. It doesn't have the TransferInst which defines + // register, so no Entry Value Transfers have been emitted already. + if (!MI.getDebugOperand(0).isReg()) + return; + + // Try to get non-debug instruction responsible for the DBG_VALUE. + const MachineInstr *TransferInst = nullptr; + Register Reg = MI.getDebugOperand(0).getReg(); + if (Reg.isValid() && RegSetInstrs.find(Reg) != RegSetInstrs.end()) + TransferInst = RegSetInstrs.find(Reg)->second; + + // Case of the parameter's DBG_VALUE at the start of entry MBB. + if (!TransferInst && !LastNonDbgMI && MI.getParent()->isEntryBlock()) + return; + + // If the debug expression from the DBG_VALUE is not empty, we can assume the + // parameter's value has changed indicating that we should stop tracking its + // entry value as well. + if (MI.getDebugExpression()->getNumElements() == 0 && TransferInst) { + // If the DBG_VALUE comes from a copy instruction that copies the entry + // value, it means the parameter's value has not changed and we should be + // able to use its entry value. + // TODO: Try to keep tracking of an entry value if we encounter a propagated + // DBG_VALUE describing the copy of the entry value. (Propagated entry value + // does not indicate the parameter modification.) + auto DestSrc = TII->isCopyInstr(*TransferInst); + if (DestSrc) { + const MachineOperand *SrcRegOp, *DestRegOp; + SrcRegOp = DestSrc->Source; + DestRegOp = DestSrc->Destination; + if (Reg == DestRegOp->getReg()) { + for (uint64_t ID : OpenRanges.getEntryValueBackupVarLocs()) { + const VarLoc &VL = VarLocIDs[LocIndex::fromRawInteger(ID)]; + if (VL.isEntryValueCopyBackupReg(Reg) && + // Entry Values should not be variadic. + VL.MI.getDebugOperand(0).getReg() == SrcRegOp->getReg()) + return; + } + } + } + } + + LLVM_DEBUG(dbgs() << "Deleting a DBG entry value because of: "; + MI.print(dbgs(), /*IsStandalone*/ false, + /*SkipOpers*/ false, /*SkipDebugLoc*/ false, + /*AddNewLine*/ true, TII)); + cleanupEntryValueTransfers(TransferInst, OpenRanges, VarLocIDs, EntryVL, + EntryValTransfers); + OpenRanges.erase(EntryVL); +} + +/// End all previous ranges related to @MI and start a new range from @MI +/// if it is a DBG_VALUE instr. +void VarLocBasedLDV::transferDebugValue(const MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + InstToEntryLocMap &EntryValTransfers, + RegDefToInstMap &RegSetInstrs) { + if (!MI.isDebugValue()) + return; + const DILocalVariable *Var = MI.getDebugVariable(); + const DIExpression *Expr = MI.getDebugExpression(); + const DILocation *DebugLoc = MI.getDebugLoc(); + const DILocation *InlinedAt = DebugLoc->getInlinedAt(); + assert(Var->isValidLocationForIntrinsic(DebugLoc) && + "Expected inlined-at fields to agree"); + + DebugVariable V(Var, Expr, InlinedAt); + + // Check if this DBG_VALUE indicates a parameter's value changing. + // If that is the case, we should stop tracking its entry value. + auto EntryValBackupID = OpenRanges.getEntryValueBackup(V); + if (Var->isParameter() && EntryValBackupID) { + const VarLoc &EntryVL = VarLocIDs[EntryValBackupID->back()]; + removeEntryValue(MI, OpenRanges, VarLocIDs, EntryVL, EntryValTransfers, + RegSetInstrs); + } + + if (all_of(MI.debug_operands(), [](const MachineOperand &MO) { + return (MO.isReg() && MO.getReg()) || MO.isImm() || MO.isFPImm() || + MO.isCImm() || MO.isTargetIndex(); + })) { + // Use normal VarLoc constructor for registers and immediates. + VarLoc VL(MI); + // End all previous ranges of VL.Var. + OpenRanges.erase(VL); + + LocIndices IDs = VarLocIDs.insert(VL); + // Add the VarLoc to OpenRanges from this DBG_VALUE. + OpenRanges.insert(IDs, VL); + } else if (MI.memoperands().size() > 0) { + llvm_unreachable("DBG_VALUE with mem operand encountered after regalloc?"); + } else { + // This must be an undefined location. If it has an open range, erase it. + assert(MI.isUndefDebugValue() && + "Unexpected non-undef DBG_VALUE encountered"); + VarLoc VL(MI); + OpenRanges.erase(VL); + } +} + +// This should be removed later, doesn't fit the new design. +void VarLocBasedLDV::collectAllVarLocs(SmallVectorImpl<VarLoc> &Collected, + const VarLocSet &CollectFrom, + const VarLocMap &VarLocIDs) { + // The half-open interval [FirstIndexForReg, FirstInvalidIndex) contains all + // possible VarLoc IDs for VarLocs with MLs of kind RegisterKind which live + // in Reg. + uint64_t FirstIndex = LocIndex::rawIndexForReg(LocIndex::kUniversalLocation); + uint64_t FirstInvalidIndex = + LocIndex::rawIndexForReg(LocIndex::kUniversalLocation + 1); + // Iterate through that half-open interval and collect all the set IDs. + for (auto It = CollectFrom.find(FirstIndex), End = CollectFrom.end(); + It != End && *It < FirstInvalidIndex; ++It) { + LocIndex RegIdx = LocIndex::fromRawInteger(*It); + Collected.push_back(VarLocIDs[RegIdx]); + } +} + +/// Turn the entry value backup locations into primary locations. +void VarLocBasedLDV::emitEntryValues(MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + InstToEntryLocMap &EntryValTransfers, + VarLocsInRange &KillSet) { + // Do not insert entry value locations after a terminator. + if (MI.isTerminator()) + return; + + for (uint32_t ID : KillSet) { + // The KillSet IDs are indices for the universal location bucket. + LocIndex Idx = LocIndex(LocIndex::kUniversalLocation, ID); + const VarLoc &VL = VarLocIDs[Idx]; + if (!VL.Var.getVariable()->isParameter()) + continue; + + auto DebugVar = VL.Var; + std::optional<LocIndices> EntryValBackupIDs = + OpenRanges.getEntryValueBackup(DebugVar); + + // If the parameter has the entry value backup, it means we should + // be able to use its entry value. + if (!EntryValBackupIDs) + continue; + + const VarLoc &EntryVL = VarLocIDs[EntryValBackupIDs->back()]; + VarLoc EntryLoc = VarLoc::CreateEntryLoc(EntryVL.MI, EntryVL.Expr, + EntryVL.Locs[0].Value.RegNo); + LocIndices EntryValueIDs = VarLocIDs.insert(EntryLoc); + assert(EntryValueIDs.size() == 1 && + "EntryValue loc should not be variadic"); + EntryValTransfers.insert({&MI, EntryValueIDs.back()}); + OpenRanges.insert(EntryValueIDs, EntryLoc); + } +} + +/// Create new TransferDebugPair and insert it in \p Transfers. The VarLoc +/// with \p OldVarID should be deleted form \p OpenRanges and replaced with +/// new VarLoc. If \p NewReg is different than default zero value then the +/// new location will be register location created by the copy like instruction, +/// otherwise it is variable's location on the stack. +void VarLocBasedLDV::insertTransferDebugPair( + MachineInstr &MI, OpenRangesSet &OpenRanges, TransferMap &Transfers, + VarLocMap &VarLocIDs, LocIndex OldVarID, TransferKind Kind, + const VarLoc::MachineLoc &OldLoc, Register NewReg) { + const VarLoc &OldVarLoc = VarLocIDs[OldVarID]; + + auto ProcessVarLoc = [&MI, &OpenRanges, &Transfers, &VarLocIDs](VarLoc &VL) { + LocIndices LocIds = VarLocIDs.insert(VL); + + // Close this variable's previous location range. + OpenRanges.erase(VL); + + // Record the new location as an open range, and a postponed transfer + // inserting a DBG_VALUE for this location. + OpenRanges.insert(LocIds, VL); + assert(!MI.isTerminator() && "Cannot insert DBG_VALUE after terminator"); + TransferDebugPair MIP = {&MI, LocIds.back()}; + Transfers.push_back(MIP); + }; + + // End all previous ranges of VL.Var. + OpenRanges.erase(VarLocIDs[OldVarID]); + switch (Kind) { + case TransferKind::TransferCopy: { + assert(NewReg && + "No register supplied when handling a copy of a debug value"); + // Create a DBG_VALUE instruction to describe the Var in its new + // register location. + VarLoc VL = VarLoc::CreateCopyLoc(OldVarLoc, OldLoc, NewReg); + ProcessVarLoc(VL); + LLVM_DEBUG({ + dbgs() << "Creating VarLoc for register copy:"; + VL.dump(TRI, TII); + }); + return; + } + case TransferKind::TransferSpill: { + // Create a DBG_VALUE instruction to describe the Var in its spilled + // location. + VarLoc::SpillLoc SpillLocation = extractSpillBaseRegAndOffset(MI); + VarLoc VL = VarLoc::CreateSpillLoc( + OldVarLoc, OldLoc, SpillLocation.SpillBase, SpillLocation.SpillOffset); + ProcessVarLoc(VL); + LLVM_DEBUG({ + dbgs() << "Creating VarLoc for spill:"; + VL.dump(TRI, TII); + }); + return; + } + case TransferKind::TransferRestore: { + assert(NewReg && + "No register supplied when handling a restore of a debug value"); + // DebugInstr refers to the pre-spill location, therefore we can reuse + // its expression. + VarLoc VL = VarLoc::CreateCopyLoc(OldVarLoc, OldLoc, NewReg); + ProcessVarLoc(VL); + LLVM_DEBUG({ + dbgs() << "Creating VarLoc for restore:"; + VL.dump(TRI, TII); + }); + return; + } + } + llvm_unreachable("Invalid transfer kind"); +} + +/// A definition of a register may mark the end of a range. +void VarLocBasedLDV::transferRegisterDef(MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + InstToEntryLocMap &EntryValTransfers, + RegDefToInstMap &RegSetInstrs) { + + // Meta Instructions do not affect the debug liveness of any register they + // define. + if (MI.isMetaInstruction()) + return; + + MachineFunction *MF = MI.getMF(); + const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); + Register SP = TLI->getStackPointerRegisterToSaveRestore(); + + // Find the regs killed by MI, and find regmasks of preserved regs. + DefinedRegsSet DeadRegs; + SmallVector<const uint32_t *, 4> RegMasks; + for (const MachineOperand &MO : MI.operands()) { + // Determine whether the operand is a register def. + if (MO.isReg() && MO.isDef() && MO.getReg() && MO.getReg().isPhysical() && + !(MI.isCall() && MO.getReg() == SP)) { + // Remove ranges of all aliased registers. + for (MCRegAliasIterator RAI(MO.getReg(), TRI, true); RAI.isValid(); ++RAI) + // FIXME: Can we break out of this loop early if no insertion occurs? + DeadRegs.insert(*RAI); + RegSetInstrs.erase(MO.getReg()); + RegSetInstrs.insert({MO.getReg(), &MI}); + } else if (MO.isRegMask()) { + RegMasks.push_back(MO.getRegMask()); + } + } + + // Erase VarLocs which reside in one of the dead registers. For performance + // reasons, it's critical to not iterate over the full set of open VarLocs. + // Iterate over the set of dying/used regs instead. + if (!RegMasks.empty()) { + SmallVector<Register, 32> UsedRegs; + getUsedRegs(OpenRanges.getVarLocs(), UsedRegs); + for (Register Reg : UsedRegs) { + // Remove ranges of all clobbered registers. Register masks don't usually + // list SP as preserved. Assume that call instructions never clobber SP, + // because some backends (e.g., AArch64) never list SP in the regmask. + // While the debug info may be off for an instruction or two around + // callee-cleanup calls, transferring the DEBUG_VALUE across the call is + // still a better user experience. + if (Reg == SP) + continue; + bool AnyRegMaskKillsReg = + any_of(RegMasks, [Reg](const uint32_t *RegMask) { + return MachineOperand::clobbersPhysReg(RegMask, Reg); + }); + if (AnyRegMaskKillsReg) + DeadRegs.insert(Reg); + if (AnyRegMaskKillsReg) { + RegSetInstrs.erase(Reg); + RegSetInstrs.insert({Reg, &MI}); + } + } + } + + if (DeadRegs.empty()) + return; + + VarLocsInRange KillSet; + collectIDsForRegs(KillSet, DeadRegs, OpenRanges.getVarLocs(), VarLocIDs); + OpenRanges.erase(KillSet, VarLocIDs, LocIndex::kUniversalLocation); + + if (TPC) { + auto &TM = TPC->getTM<TargetMachine>(); + if (TM.Options.ShouldEmitDebugEntryValues()) + emitEntryValues(MI, OpenRanges, VarLocIDs, EntryValTransfers, KillSet); + } +} + +void VarLocBasedLDV::transferWasmDef(MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs) { + // If this is not a Wasm local.set or local.tee, which sets local values, + // return. + int Index; + int64_t Offset; + if (!TII->isExplicitTargetIndexDef(MI, Index, Offset)) + return; + + // Find the target indices killed by MI, and delete those variable locations + // from the open range. + VarLocsInRange KillSet; + VarLoc::WasmLoc Loc{Index, Offset}; + for (uint64_t ID : OpenRanges.getWasmVarLocs()) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VL = VarLocIDs[Idx]; + assert(VL.containsWasmLocs() && "Broken VarLocSet?"); + if (VL.usesWasmLoc(Loc)) + KillSet.insert(ID); + } + OpenRanges.erase(KillSet, VarLocIDs, LocIndex::kWasmLocation); +} + +bool VarLocBasedLDV::isSpillInstruction(const MachineInstr &MI, + MachineFunction *MF) { + // TODO: Handle multiple stores folded into one. + if (!MI.hasOneMemOperand()) + return false; + + if (!MI.getSpillSize(TII) && !MI.getFoldedSpillSize(TII)) + return false; // This is not a spill instruction, since no valid size was + // returned from either function. + + return true; +} + +bool VarLocBasedLDV::isLocationSpill(const MachineInstr &MI, + MachineFunction *MF, Register &Reg) { + if (!isSpillInstruction(MI, MF)) + return false; + + auto isKilledReg = [&](const MachineOperand MO, Register &Reg) { + if (!MO.isReg() || !MO.isUse()) { + Reg = 0; + return false; + } + Reg = MO.getReg(); + return MO.isKill(); + }; + + for (const MachineOperand &MO : MI.operands()) { + // In a spill instruction generated by the InlineSpiller the spilled + // register has its kill flag set. + if (isKilledReg(MO, Reg)) + return true; + if (Reg != 0) { + // Check whether next instruction kills the spilled register. + // FIXME: Current solution does not cover search for killed register in + // bundles and instructions further down the chain. + auto NextI = std::next(MI.getIterator()); + // Skip next instruction that points to basic block end iterator. + if (MI.getParent()->end() == NextI) + continue; + Register RegNext; + for (const MachineOperand &MONext : NextI->operands()) { + // Return true if we came across the register from the + // previous spill instruction that is killed in NextI. + if (isKilledReg(MONext, RegNext) && RegNext == Reg) + return true; + } + } + } + // Return false if we didn't find spilled register. + return false; +} + +std::optional<VarLocBasedLDV::VarLoc::SpillLoc> +VarLocBasedLDV::isRestoreInstruction(const MachineInstr &MI, + MachineFunction *MF, Register &Reg) { + if (!MI.hasOneMemOperand()) + return std::nullopt; + + // FIXME: Handle folded restore instructions with more than one memory + // operand. + if (MI.getRestoreSize(TII)) { + Reg = MI.getOperand(0).getReg(); + return extractSpillBaseRegAndOffset(MI); + } + return std::nullopt; +} + +/// A spilled register may indicate that we have to end the current range of +/// a variable and create a new one for the spill location. +/// A restored register may indicate the reverse situation. +/// We don't want to insert any instructions in process(), so we just create +/// the DBG_VALUE without inserting it and keep track of it in \p Transfers. +/// It will be inserted into the BB when we're done iterating over the +/// instructions. +void VarLocBasedLDV::transferSpillOrRestoreInst(MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + TransferMap &Transfers) { + MachineFunction *MF = MI.getMF(); + TransferKind TKind; + Register Reg; + std::optional<VarLoc::SpillLoc> Loc; + + LLVM_DEBUG(dbgs() << "Examining instruction: "; MI.dump();); + + // First, if there are any DBG_VALUEs pointing at a spill slot that is + // written to, then close the variable location. The value in memory + // will have changed. + VarLocsInRange KillSet; + if (isSpillInstruction(MI, MF)) { + Loc = extractSpillBaseRegAndOffset(MI); + for (uint64_t ID : OpenRanges.getSpillVarLocs()) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VL = VarLocIDs[Idx]; + assert(VL.containsSpillLocs() && "Broken VarLocSet?"); + if (VL.usesSpillLoc(*Loc)) { + // This location is overwritten by the current instruction -- terminate + // the open range, and insert an explicit DBG_VALUE $noreg. + // + // Doing this at a later stage would require re-interpreting all + // DBG_VALUes and DIExpressions to identify whether they point at + // memory, and then analysing all memory writes to see if they + // overwrite that memory, which is expensive. + // + // At this stage, we already know which DBG_VALUEs are for spills and + // where they are located; it's best to fix handle overwrites now. + KillSet.insert(ID); + unsigned SpillLocIdx = VL.getSpillLocIdx(*Loc); + VarLoc::MachineLoc OldLoc = VL.Locs[SpillLocIdx]; + VarLoc UndefVL = VarLoc::CreateCopyLoc(VL, OldLoc, 0); + LocIndices UndefLocIDs = VarLocIDs.insert(UndefVL); + Transfers.push_back({&MI, UndefLocIDs.back()}); + } + } + OpenRanges.erase(KillSet, VarLocIDs, LocIndex::kSpillLocation); + } + + // Try to recognise spill and restore instructions that may create a new + // variable location. + if (isLocationSpill(MI, MF, Reg)) { + TKind = TransferKind::TransferSpill; + LLVM_DEBUG(dbgs() << "Recognized as spill: "; MI.dump();); + LLVM_DEBUG(dbgs() << "Register: " << Reg << " " << printReg(Reg, TRI) + << "\n"); + } else { + if (!(Loc = isRestoreInstruction(MI, MF, Reg))) + return; + TKind = TransferKind::TransferRestore; + LLVM_DEBUG(dbgs() << "Recognized as restore: "; MI.dump();); + LLVM_DEBUG(dbgs() << "Register: " << Reg << " " << printReg(Reg, TRI) + << "\n"); + } + // Check if the register or spill location is the location of a debug value. + auto TransferCandidates = OpenRanges.getEmptyVarLocRange(); + if (TKind == TransferKind::TransferSpill) + TransferCandidates = OpenRanges.getRegisterVarLocs(Reg); + else if (TKind == TransferKind::TransferRestore) + TransferCandidates = OpenRanges.getSpillVarLocs(); + for (uint64_t ID : TransferCandidates) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VL = VarLocIDs[Idx]; + unsigned LocIdx; + if (TKind == TransferKind::TransferSpill) { + assert(VL.usesReg(Reg) && "Broken VarLocSet?"); + LLVM_DEBUG(dbgs() << "Spilling Register " << printReg(Reg, TRI) << '(' + << VL.Var.getVariable()->getName() << ")\n"); + LocIdx = VL.getRegIdx(Reg); + } else { + assert(TKind == TransferKind::TransferRestore && VL.containsSpillLocs() && + "Broken VarLocSet?"); + if (!VL.usesSpillLoc(*Loc)) + // The spill location is not the location of a debug value. + continue; + LLVM_DEBUG(dbgs() << "Restoring Register " << printReg(Reg, TRI) << '(' + << VL.Var.getVariable()->getName() << ")\n"); + LocIdx = VL.getSpillLocIdx(*Loc); + } + VarLoc::MachineLoc MLoc = VL.Locs[LocIdx]; + insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, Idx, TKind, + MLoc, Reg); + // FIXME: A comment should explain why it's correct to return early here, + // if that is in fact correct. + return; + } +} + +/// If \p MI is a register copy instruction, that copies a previously tracked +/// value from one register to another register that is callee saved, we +/// create new DBG_VALUE instruction described with copy destination register. +void VarLocBasedLDV::transferRegisterCopy(MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + TransferMap &Transfers) { + auto DestSrc = TII->isCopyInstr(MI); + if (!DestSrc) + return; + + const MachineOperand *DestRegOp = DestSrc->Destination; + const MachineOperand *SrcRegOp = DestSrc->Source; + + if (!DestRegOp->isDef()) + return; + + auto isCalleeSavedReg = [&](Register Reg) { + for (MCRegAliasIterator RAI(Reg, TRI, true); RAI.isValid(); ++RAI) + if (CalleeSavedRegs.test(*RAI)) + return true; + return false; + }; + + Register SrcReg = SrcRegOp->getReg(); + Register DestReg = DestRegOp->getReg(); + + // We want to recognize instructions where destination register is callee + // saved register. If register that could be clobbered by the call is + // included, there would be a great chance that it is going to be clobbered + // soon. It is more likely that previous register location, which is callee + // saved, is going to stay unclobbered longer, even if it is killed. + if (!isCalleeSavedReg(DestReg)) + return; + + // Remember an entry value movement. If we encounter a new debug value of + // a parameter describing only a moving of the value around, rather then + // modifying it, we are still able to use the entry value if needed. + if (isRegOtherThanSPAndFP(*DestRegOp, MI, TRI)) { + for (uint64_t ID : OpenRanges.getEntryValueBackupVarLocs()) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VL = VarLocIDs[Idx]; + if (VL.isEntryValueBackupReg(SrcReg)) { + LLVM_DEBUG(dbgs() << "Copy of the entry value: "; MI.dump();); + VarLoc EntryValLocCopyBackup = + VarLoc::CreateEntryCopyBackupLoc(VL.MI, VL.Expr, DestReg); + // Stop tracking the original entry value. + OpenRanges.erase(VL); + + // Start tracking the entry value copy. + LocIndices EntryValCopyLocIDs = VarLocIDs.insert(EntryValLocCopyBackup); + OpenRanges.insert(EntryValCopyLocIDs, EntryValLocCopyBackup); + break; + } + } + } + + if (!SrcRegOp->isKill()) + return; + + for (uint64_t ID : OpenRanges.getRegisterVarLocs(SrcReg)) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + assert(VarLocIDs[Idx].usesReg(SrcReg) && "Broken VarLocSet?"); + VarLoc::MachineLocValue Loc; + Loc.RegNo = SrcReg; + VarLoc::MachineLoc MLoc{VarLoc::MachineLocKind::RegisterKind, Loc}; + insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, Idx, + TransferKind::TransferCopy, MLoc, DestReg); + // FIXME: A comment should explain why it's correct to return early here, + // if that is in fact correct. + return; + } +} + +/// Terminate all open ranges at the end of the current basic block. +bool VarLocBasedLDV::transferTerminator(MachineBasicBlock *CurMBB, + OpenRangesSet &OpenRanges, + VarLocInMBB &OutLocs, + const VarLocMap &VarLocIDs) { + bool Changed = false; + LLVM_DEBUG({ + VarVec VarLocs; + OpenRanges.getUniqueVarLocs(VarLocs, VarLocIDs); + for (VarLoc &VL : VarLocs) { + // Copy OpenRanges to OutLocs, if not already present. + dbgs() << "Add to OutLocs in MBB #" << CurMBB->getNumber() << ": "; + VL.dump(TRI, TII); + } + }); + VarLocSet &VLS = getVarLocsInMBB(CurMBB, OutLocs); + Changed = VLS != OpenRanges.getVarLocs(); + // New OutLocs set may be different due to spill, restore or register + // copy instruction processing. + if (Changed) + VLS = OpenRanges.getVarLocs(); + OpenRanges.clear(); + return Changed; +} + +/// Accumulate a mapping between each DILocalVariable fragment and other +/// fragments of that DILocalVariable which overlap. This reduces work during +/// the data-flow stage from "Find any overlapping fragments" to "Check if the +/// known-to-overlap fragments are present". +/// \param MI A previously unprocessed DEBUG_VALUE instruction to analyze for +/// fragment usage. +/// \param SeenFragments Map from DILocalVariable to all fragments of that +/// Variable which are known to exist. +/// \param OverlappingFragments The overlap map being constructed, from one +/// Var/Fragment pair to a vector of fragments known to overlap. +void VarLocBasedLDV::accumulateFragmentMap(MachineInstr &MI, + VarToFragments &SeenFragments, + OverlapMap &OverlappingFragments) { + DebugVariable MIVar(MI.getDebugVariable(), MI.getDebugExpression(), + MI.getDebugLoc()->getInlinedAt()); + FragmentInfo ThisFragment = MIVar.getFragmentOrDefault(); + + // If this is the first sighting of this variable, then we are guaranteed + // there are currently no overlapping fragments either. Initialize the set + // of seen fragments, record no overlaps for the current one, and return. + auto SeenIt = SeenFragments.find(MIVar.getVariable()); + if (SeenIt == SeenFragments.end()) { + SmallSet<FragmentInfo, 4> OneFragment; + OneFragment.insert(ThisFragment); + SeenFragments.insert({MIVar.getVariable(), OneFragment}); + + OverlappingFragments.insert({{MIVar.getVariable(), ThisFragment}, {}}); + return; + } + + // If this particular Variable/Fragment pair already exists in the overlap + // map, it has already been accounted for. + auto IsInOLapMap = + OverlappingFragments.insert({{MIVar.getVariable(), ThisFragment}, {}}); + if (!IsInOLapMap.second) + return; + + auto &ThisFragmentsOverlaps = IsInOLapMap.first->second; + auto &AllSeenFragments = SeenIt->second; + + // Otherwise, examine all other seen fragments for this variable, with "this" + // fragment being a previously unseen fragment. Record any pair of + // overlapping fragments. + for (const auto &ASeenFragment : AllSeenFragments) { + // Does this previously seen fragment overlap? + if (DIExpression::fragmentsOverlap(ThisFragment, ASeenFragment)) { + // Yes: Mark the current fragment as being overlapped. + ThisFragmentsOverlaps.push_back(ASeenFragment); + // Mark the previously seen fragment as being overlapped by the current + // one. + auto ASeenFragmentsOverlaps = + OverlappingFragments.find({MIVar.getVariable(), ASeenFragment}); + assert(ASeenFragmentsOverlaps != OverlappingFragments.end() && + "Previously seen var fragment has no vector of overlaps"); + ASeenFragmentsOverlaps->second.push_back(ThisFragment); + } + } + + AllSeenFragments.insert(ThisFragment); +} + +/// This routine creates OpenRanges. +void VarLocBasedLDV::process(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, TransferMap &Transfers, + InstToEntryLocMap &EntryValTransfers, + RegDefToInstMap &RegSetInstrs) { + if (!MI.isDebugInstr()) + LastNonDbgMI = &MI; + transferDebugValue(MI, OpenRanges, VarLocIDs, EntryValTransfers, + RegSetInstrs); + transferRegisterDef(MI, OpenRanges, VarLocIDs, EntryValTransfers, + RegSetInstrs); + transferWasmDef(MI, OpenRanges, VarLocIDs); + transferRegisterCopy(MI, OpenRanges, VarLocIDs, Transfers); + transferSpillOrRestoreInst(MI, OpenRanges, VarLocIDs, Transfers); +} + +/// This routine joins the analysis results of all incoming edges in @MBB by +/// inserting a new DBG_VALUE instruction at the start of the @MBB - if the same +/// source variable in all the predecessors of @MBB reside in the same location. +bool VarLocBasedLDV::join( + MachineBasicBlock &MBB, VarLocInMBB &OutLocs, VarLocInMBB &InLocs, + const VarLocMap &VarLocIDs, + SmallPtrSet<const MachineBasicBlock *, 16> &Visited, + SmallPtrSetImpl<const MachineBasicBlock *> &ArtificialBlocks) { + LLVM_DEBUG(dbgs() << "join MBB: " << MBB.getNumber() << "\n"); + + VarLocSet InLocsT(Alloc); // Temporary incoming locations. + + // For all predecessors of this MBB, find the set of VarLocs that + // can be joined. + int NumVisited = 0; + for (auto *p : MBB.predecessors()) { + // Ignore backedges if we have not visited the predecessor yet. As the + // predecessor hasn't yet had locations propagated into it, most locations + // will not yet be valid, so treat them as all being uninitialized and + // potentially valid. If a location guessed to be correct here is + // invalidated later, we will remove it when we revisit this block. + if (!Visited.count(p)) { + LLVM_DEBUG(dbgs() << " ignoring unvisited pred MBB: " << p->getNumber() + << "\n"); + continue; + } + auto OL = OutLocs.find(p); + // Join is null in case of empty OutLocs from any of the pred. + if (OL == OutLocs.end()) + return false; + + // Just copy over the Out locs to incoming locs for the first visited + // predecessor, and for all other predecessors join the Out locs. + VarLocSet &OutLocVLS = *OL->second; + if (!NumVisited) + InLocsT = OutLocVLS; + else + InLocsT &= OutLocVLS; + + LLVM_DEBUG({ + if (!InLocsT.empty()) { + VarVec VarLocs; + collectAllVarLocs(VarLocs, InLocsT, VarLocIDs); + for (const VarLoc &VL : VarLocs) + dbgs() << " gathered candidate incoming var: " + << VL.Var.getVariable()->getName() << "\n"; + } + }); + + NumVisited++; + } + + // Filter out DBG_VALUES that are out of scope. + VarLocSet KillSet(Alloc); + bool IsArtificial = ArtificialBlocks.count(&MBB); + if (!IsArtificial) { + for (uint64_t ID : InLocsT) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + if (!VarLocIDs[Idx].dominates(LS, MBB)) { + KillSet.set(ID); + LLVM_DEBUG({ + auto Name = VarLocIDs[Idx].Var.getVariable()->getName(); + dbgs() << " killing " << Name << ", it doesn't dominate MBB\n"; + }); + } + } + } + InLocsT.intersectWithComplement(KillSet); + + // As we are processing blocks in reverse post-order we + // should have processed at least one predecessor, unless it + // is the entry block which has no predecessor. + assert((NumVisited || MBB.pred_empty()) && + "Should have processed at least one predecessor"); + + VarLocSet &ILS = getVarLocsInMBB(&MBB, InLocs); + bool Changed = false; + if (ILS != InLocsT) { + ILS = InLocsT; + Changed = true; + } + + return Changed; +} + +void VarLocBasedLDV::flushPendingLocs(VarLocInMBB &PendingInLocs, + VarLocMap &VarLocIDs) { + // PendingInLocs records all locations propagated into blocks, which have + // not had DBG_VALUE insts created. Go through and create those insts now. + for (auto &Iter : PendingInLocs) { + // Map is keyed on a constant pointer, unwrap it so we can insert insts. + auto &MBB = const_cast<MachineBasicBlock &>(*Iter.first); + VarLocSet &Pending = *Iter.second; + + SmallVector<VarLoc, 32> VarLocs; + collectAllVarLocs(VarLocs, Pending, VarLocIDs); + + for (VarLoc DiffIt : VarLocs) { + // The ID location is live-in to MBB -- work out what kind of machine + // location it is and create a DBG_VALUE. + if (DiffIt.isEntryBackupLoc()) + continue; + MachineInstr *MI = DiffIt.BuildDbgValue(*MBB.getParent()); + MBB.insert(MBB.instr_begin(), MI); + + (void)MI; + LLVM_DEBUG(dbgs() << "Inserted: "; MI->dump();); + } + } +} + +bool VarLocBasedLDV::isEntryValueCandidate( + const MachineInstr &MI, const DefinedRegsSet &DefinedRegs) const { + assert(MI.isDebugValue() && "This must be DBG_VALUE."); + + // TODO: Add support for local variables that are expressed in terms of + // parameters entry values. + // TODO: Add support for modified arguments that can be expressed + // by using its entry value. + auto *DIVar = MI.getDebugVariable(); + if (!DIVar->isParameter()) + return false; + + // Do not consider parameters that belong to an inlined function. + if (MI.getDebugLoc()->getInlinedAt()) + return false; + + // Only consider parameters that are described using registers. Parameters + // that are passed on the stack are not yet supported, so ignore debug + // values that are described by the frame or stack pointer. + if (!isRegOtherThanSPAndFP(MI.getDebugOperand(0), MI, TRI)) + return false; + + // If a parameter's value has been propagated from the caller, then the + // parameter's DBG_VALUE may be described using a register defined by some + // instruction in the entry block, in which case we shouldn't create an + // entry value. + if (DefinedRegs.count(MI.getDebugOperand(0).getReg())) + return false; + + // TODO: Add support for parameters that have a pre-existing debug expressions + // (e.g. fragments). + if (MI.getDebugExpression()->getNumElements() > 0) + return false; + + return true; +} + +/// Collect all register defines (including aliases) for the given instruction. +static void collectRegDefs(const MachineInstr &MI, DefinedRegsSet &Regs, + const TargetRegisterInfo *TRI) { + for (const MachineOperand &MO : MI.operands()) { + if (MO.isReg() && MO.isDef() && MO.getReg() && MO.getReg().isPhysical()) { + Regs.insert(MO.getReg()); + for (MCRegAliasIterator AI(MO.getReg(), TRI, true); AI.isValid(); ++AI) + Regs.insert(*AI); + } + } +} + +/// This routine records the entry values of function parameters. The values +/// could be used as backup values. If we loose the track of some unmodified +/// parameters, the backup values will be used as a primary locations. +void VarLocBasedLDV::recordEntryValue(const MachineInstr &MI, + const DefinedRegsSet &DefinedRegs, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs) { + if (TPC) { + auto &TM = TPC->getTM<TargetMachine>(); + if (!TM.Options.ShouldEmitDebugEntryValues()) + return; + } + + DebugVariable V(MI.getDebugVariable(), MI.getDebugExpression(), + MI.getDebugLoc()->getInlinedAt()); + + if (!isEntryValueCandidate(MI, DefinedRegs) || + OpenRanges.getEntryValueBackup(V)) + return; + + LLVM_DEBUG(dbgs() << "Creating the backup entry location: "; MI.dump();); + + // Create the entry value and use it as a backup location until it is + // valid. It is valid until a parameter is not changed. + DIExpression *NewExpr = + DIExpression::prepend(MI.getDebugExpression(), DIExpression::EntryValue); + VarLoc EntryValLocAsBackup = VarLoc::CreateEntryBackupLoc(MI, NewExpr); + LocIndices EntryValLocIDs = VarLocIDs.insert(EntryValLocAsBackup); + OpenRanges.insert(EntryValLocIDs, EntryValLocAsBackup); +} + +/// Calculate the liveness information for the given machine function and +/// extend ranges across basic blocks. +bool VarLocBasedLDV::ExtendRanges(MachineFunction &MF, + MachineDominatorTree *DomTree, + TargetPassConfig *TPC, unsigned InputBBLimit, + unsigned InputDbgValLimit) { + (void)DomTree; + LLVM_DEBUG(dbgs() << "\nDebug Range Extension: " << MF.getName() << "\n"); + + if (!MF.getFunction().getSubprogram()) + // VarLocBaseLDV will already have removed all DBG_VALUEs. + return false; + + // Skip functions from NoDebug compilation units. + if (MF.getFunction().getSubprogram()->getUnit()->getEmissionKind() == + DICompileUnit::NoDebug) + return false; + + TRI = MF.getSubtarget().getRegisterInfo(); + TII = MF.getSubtarget().getInstrInfo(); + TFI = MF.getSubtarget().getFrameLowering(); + TFI->getCalleeSaves(MF, CalleeSavedRegs); + this->TPC = TPC; + LS.initialize(MF); + + bool Changed = false; + bool OLChanged = false; + bool MBBJoined = false; + + VarLocMap VarLocIDs; // Map VarLoc<>unique ID for use in bitvectors. + OverlapMap OverlapFragments; // Map of overlapping variable fragments. + OpenRangesSet OpenRanges(Alloc, OverlapFragments); + // Ranges that are open until end of bb. + VarLocInMBB OutLocs; // Ranges that exist beyond bb. + VarLocInMBB InLocs; // Ranges that are incoming after joining. + TransferMap Transfers; // DBG_VALUEs associated with transfers (such as + // spills, copies and restores). + // Map responsible MI to attached Transfer emitted from Backup Entry Value. + InstToEntryLocMap EntryValTransfers; + // Map a Register to the last MI which clobbered it. + RegDefToInstMap RegSetInstrs; + + VarToFragments SeenFragments; + + // Blocks which are artificial, i.e. blocks which exclusively contain + // instructions without locations, or with line 0 locations. + SmallPtrSet<const MachineBasicBlock *, 16> ArtificialBlocks; + + DenseMap<unsigned int, MachineBasicBlock *> OrderToBB; + DenseMap<MachineBasicBlock *, unsigned int> BBToOrder; + std::priority_queue<unsigned int, std::vector<unsigned int>, + std::greater<unsigned int>> + Worklist; + std::priority_queue<unsigned int, std::vector<unsigned int>, + std::greater<unsigned int>> + Pending; + + // Set of register defines that are seen when traversing the entry block + // looking for debug entry value candidates. + DefinedRegsSet DefinedRegs; + + // Only in the case of entry MBB collect DBG_VALUEs representing + // function parameters in order to generate debug entry values for them. + MachineBasicBlock &First_MBB = *(MF.begin()); + for (auto &MI : First_MBB) { + collectRegDefs(MI, DefinedRegs, TRI); + if (MI.isDebugValue()) + recordEntryValue(MI, DefinedRegs, OpenRanges, VarLocIDs); + } + + // Initialize per-block structures and scan for fragment overlaps. + for (auto &MBB : MF) + for (auto &MI : MBB) + if (MI.isDebugValue()) + accumulateFragmentMap(MI, SeenFragments, OverlapFragments); + + auto hasNonArtificialLocation = [](const MachineInstr &MI) -> bool { + if (const DebugLoc &DL = MI.getDebugLoc()) + return DL.getLine() != 0; + return false; + }; + for (auto &MBB : MF) + if (none_of(MBB.instrs(), hasNonArtificialLocation)) + ArtificialBlocks.insert(&MBB); + + LLVM_DEBUG(printVarLocInMBB(MF, OutLocs, VarLocIDs, + "OutLocs after initialization", dbgs())); + + ReversePostOrderTraversal<MachineFunction *> RPOT(&MF); + unsigned int RPONumber = 0; + for (MachineBasicBlock *MBB : RPOT) { + OrderToBB[RPONumber] = MBB; + BBToOrder[MBB] = RPONumber; + Worklist.push(RPONumber); + ++RPONumber; + } + + if (RPONumber > InputBBLimit) { + unsigned NumInputDbgValues = 0; + for (auto &MBB : MF) + for (auto &MI : MBB) + if (MI.isDebugValue()) + ++NumInputDbgValues; + if (NumInputDbgValues > InputDbgValLimit) { + LLVM_DEBUG(dbgs() << "Disabling VarLocBasedLDV: " << MF.getName() + << " has " << RPONumber << " basic blocks and " + << NumInputDbgValues + << " input DBG_VALUEs, exceeding limits.\n"); + return false; + } + } + + // This is a standard "union of predecessor outs" dataflow problem. + // To solve it, we perform join() and process() using the two worklist method + // until the ranges converge. + // Ranges have converged when both worklists are empty. + SmallPtrSet<const MachineBasicBlock *, 16> Visited; + while (!Worklist.empty() || !Pending.empty()) { + // We track what is on the pending worklist to avoid inserting the same + // thing twice. We could avoid this with a custom priority queue, but this + // is probably not worth it. + SmallPtrSet<MachineBasicBlock *, 16> OnPending; + LLVM_DEBUG(dbgs() << "Processing Worklist\n"); + while (!Worklist.empty()) { + MachineBasicBlock *MBB = OrderToBB[Worklist.top()]; + Worklist.pop(); + MBBJoined = join(*MBB, OutLocs, InLocs, VarLocIDs, Visited, + ArtificialBlocks); + MBBJoined |= Visited.insert(MBB).second; + if (MBBJoined) { + MBBJoined = false; + Changed = true; + // Now that we have started to extend ranges across BBs we need to + // examine spill, copy and restore instructions to see whether they + // operate with registers that correspond to user variables. + // First load any pending inlocs. + OpenRanges.insertFromLocSet(getVarLocsInMBB(MBB, InLocs), VarLocIDs); + LastNonDbgMI = nullptr; + RegSetInstrs.clear(); + for (auto &MI : *MBB) + process(MI, OpenRanges, VarLocIDs, Transfers, EntryValTransfers, + RegSetInstrs); + OLChanged |= transferTerminator(MBB, OpenRanges, OutLocs, VarLocIDs); + + LLVM_DEBUG(printVarLocInMBB(MF, OutLocs, VarLocIDs, + "OutLocs after propagating", dbgs())); + LLVM_DEBUG(printVarLocInMBB(MF, InLocs, VarLocIDs, + "InLocs after propagating", dbgs())); + + if (OLChanged) { + OLChanged = false; + for (auto *s : MBB->successors()) + if (OnPending.insert(s).second) { + Pending.push(BBToOrder[s]); + } + } + } + } + Worklist.swap(Pending); + // At this point, pending must be empty, since it was just the empty + // worklist + assert(Pending.empty() && "Pending should be empty"); + } + + // Add any DBG_VALUE instructions created by location transfers. + for (auto &TR : Transfers) { + assert(!TR.TransferInst->isTerminator() && + "Cannot insert DBG_VALUE after terminator"); + MachineBasicBlock *MBB = TR.TransferInst->getParent(); + const VarLoc &VL = VarLocIDs[TR.LocationID]; + MachineInstr *MI = VL.BuildDbgValue(MF); + MBB->insertAfterBundle(TR.TransferInst->getIterator(), MI); + } + Transfers.clear(); + + // Add DBG_VALUEs created using Backup Entry Value location. + for (auto &TR : EntryValTransfers) { + MachineInstr *TRInst = const_cast<MachineInstr *>(TR.first); + assert(!TRInst->isTerminator() && + "Cannot insert DBG_VALUE after terminator"); + MachineBasicBlock *MBB = TRInst->getParent(); + const VarLoc &VL = VarLocIDs[TR.second]; + MachineInstr *MI = VL.BuildDbgValue(MF); + MBB->insertAfterBundle(TRInst->getIterator(), MI); + } + EntryValTransfers.clear(); + + // Deferred inlocs will not have had any DBG_VALUE insts created; do + // that now. + flushPendingLocs(InLocs, VarLocIDs); + + LLVM_DEBUG(printVarLocInMBB(MF, OutLocs, VarLocIDs, "Final OutLocs", dbgs())); + LLVM_DEBUG(printVarLocInMBB(MF, InLocs, VarLocIDs, "Final InLocs", dbgs())); + return Changed; +} + +LDVImpl * +llvm::makeVarLocBasedLiveDebugValues() +{ + return new VarLocBasedLDV(); +} |