diff options
Diffstat (limited to 'lib/XRay')
| -rw-r--r-- | lib/XRay/BlockIndexer.cpp | 98 | ||||
| -rw-r--r-- | lib/XRay/BlockPrinter.cpp | 114 | ||||
| -rw-r--r-- | lib/XRay/BlockVerifier.cpp | 205 | ||||
| -rw-r--r-- | lib/XRay/CMakeLists.txt | 12 | ||||
| -rw-r--r-- | lib/XRay/FDRRecordProducer.cpp | 198 | ||||
| -rw-r--r-- | lib/XRay/FDRRecords.cpp | 67 | ||||
| -rw-r--r-- | lib/XRay/FDRTraceExpander.cpp | 132 | ||||
| -rw-r--r-- | lib/XRay/FDRTraceWriter.cpp | 154 | ||||
| -rw-r--r-- | lib/XRay/FileHeaderReader.cpp | 70 | ||||
| -rw-r--r-- | lib/XRay/InstrumentationMap.cpp | 53 | ||||
| -rw-r--r-- | lib/XRay/LLVMBuild.txt | 23 | ||||
| -rw-r--r-- | lib/XRay/LogBuilderConsumer.cpp | 38 | ||||
| -rw-r--r-- | lib/XRay/Profile.cpp | 403 | ||||
| -rw-r--r-- | lib/XRay/RecordInitializer.cpp | 418 | ||||
| -rw-r--r-- | lib/XRay/RecordPrinter.cpp | 109 | ||||
| -rw-r--r-- | lib/XRay/Trace.cpp | 736 |
16 files changed, 2309 insertions, 521 deletions
diff --git a/lib/XRay/BlockIndexer.cpp b/lib/XRay/BlockIndexer.cpp new file mode 100644 index 000000000000..4dbe2d2717ad --- /dev/null +++ b/lib/XRay/BlockIndexer.cpp @@ -0,0 +1,98 @@ +//===- BlockIndexer.cpp - FDR Block Indexing VIsitor ----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// An implementation of the RecordVisitor which generates a mapping between a +// thread and a range of records representing a block. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/BlockIndexer.h" + +namespace llvm { +namespace xray { + +Error BlockIndexer::visit(BufferExtents &) { return Error::success(); } + +Error BlockIndexer::visit(WallclockRecord &R) { + CurrentBlock.Records.push_back(&R); + CurrentBlock.WallclockTime = &R; + return Error::success(); +} + +Error BlockIndexer::visit(NewCPUIDRecord &R) { + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(TSCWrapRecord &R) { + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(CustomEventRecord &R) { + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(CustomEventRecordV5 &R) { + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(TypedEventRecord &R) { + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(CallArgRecord &R) { + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(PIDRecord &R) { + CurrentBlock.ProcessID = R.pid(); + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(NewBufferRecord &R) { + if (!CurrentBlock.Records.empty()) + if (auto E = flush()) + return E; + + CurrentBlock.ThreadID = R.tid(); + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(EndBufferRecord &R) { + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::visit(FunctionRecord &R) { + CurrentBlock.Records.push_back(&R); + return Error::success(); +} + +Error BlockIndexer::flush() { + Index::iterator It; + std::tie(It, std::ignore) = + Indices.insert({{CurrentBlock.ProcessID, CurrentBlock.ThreadID}, {}}); + It->second.push_back({CurrentBlock.ProcessID, CurrentBlock.ThreadID, + CurrentBlock.WallclockTime, + std::move(CurrentBlock.Records)}); + CurrentBlock.ProcessID = 0; + CurrentBlock.ThreadID = 0; + CurrentBlock.Records = {}; + CurrentBlock.WallclockTime = nullptr; + return Error::success(); +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/BlockPrinter.cpp b/lib/XRay/BlockPrinter.cpp new file mode 100644 index 000000000000..0acebee0cbdd --- /dev/null +++ b/lib/XRay/BlockPrinter.cpp @@ -0,0 +1,114 @@ +//===- BlockPrinter.cpp - FDR Block Pretty Printer Implementation --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/BlockPrinter.h" + +namespace llvm { +namespace xray { + +Error BlockPrinter::visit(BufferExtents &R) { + OS << "\n[New Block]\n"; + CurrentState = State::Preamble; + return RP.visit(R); +} + +// Preamble printing. +Error BlockPrinter::visit(NewBufferRecord &R) { + if (CurrentState == State::Start) + OS << "\n[New Block]\n"; + + OS << "Preamble: \n"; + CurrentState = State::Preamble; + return RP.visit(R); +} + +Error BlockPrinter::visit(WallclockRecord &R) { + CurrentState = State::Preamble; + return RP.visit(R); +} + +Error BlockPrinter::visit(PIDRecord &R) { + CurrentState = State::Preamble; + return RP.visit(R); +} + +// Metadata printing. +Error BlockPrinter::visit(NewCPUIDRecord &R) { + if (CurrentState == State::Preamble) + OS << "\nBody:\n"; + if (CurrentState == State::Function) + OS << "\nMetadata: "; + CurrentState = State::Metadata; + OS << " "; + auto E = RP.visit(R); + return E; +} + +Error BlockPrinter::visit(TSCWrapRecord &R) { + if (CurrentState == State::Function) + OS << "\nMetadata:"; + CurrentState = State::Metadata; + OS << " "; + auto E = RP.visit(R); + return E; +} + +// Custom events will be rendered like "function" events. +Error BlockPrinter::visit(CustomEventRecord &R) { + if (CurrentState == State::Metadata) + OS << "\n"; + CurrentState = State::CustomEvent; + OS << "* "; + auto E = RP.visit(R); + return E; +} + +Error BlockPrinter::visit(CustomEventRecordV5 &R) { + if (CurrentState == State::Metadata) + OS << "\n"; + CurrentState = State::CustomEvent; + OS << "* "; + auto E = RP.visit(R); + return E; +} + +Error BlockPrinter::visit(TypedEventRecord &R) { + if (CurrentState == State::Metadata) + OS << "\n"; + CurrentState = State::CustomEvent; + OS << "* "; + auto E = RP.visit(R); + return E; +} + +// Function call printing. +Error BlockPrinter::visit(FunctionRecord &R) { + if (CurrentState == State::Metadata) + OS << "\n"; + CurrentState = State::Function; + OS << "- "; + auto E = RP.visit(R); + return E; +} + +Error BlockPrinter::visit(CallArgRecord &R) { + CurrentState = State::Arg; + OS << " : "; + auto E = RP.visit(R); + return E; +} + +Error BlockPrinter::visit(EndBufferRecord &R) { + CurrentState = State::End; + OS << " *** "; + auto E = RP.visit(R); + return E; +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/BlockVerifier.cpp b/lib/XRay/BlockVerifier.cpp new file mode 100644 index 000000000000..5e949ec4e46a --- /dev/null +++ b/lib/XRay/BlockVerifier.cpp @@ -0,0 +1,205 @@ +//===- BlockVerifier.cpp - FDR Block Verifier -----------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/BlockVerifier.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace xray { +namespace { + +constexpr unsigned long long mask(BlockVerifier::State S) { + return 1uLL << static_cast<std::size_t>(S); +} + +constexpr std::size_t number(BlockVerifier::State S) { + return static_cast<std::size_t>(S); +} + +StringRef recordToString(BlockVerifier::State R) { + switch (R) { + case BlockVerifier::State::BufferExtents: + return "BufferExtents"; + case BlockVerifier::State::NewBuffer: + return "NewBuffer"; + case BlockVerifier::State::WallClockTime: + return "WallClockTime"; + case BlockVerifier::State::PIDEntry: + return "PIDEntry"; + case BlockVerifier::State::NewCPUId: + return "NewCPUId"; + case BlockVerifier::State::TSCWrap: + return "TSCWrap"; + case BlockVerifier::State::CustomEvent: + return "CustomEvent"; + case BlockVerifier::State::Function: + return "Function"; + case BlockVerifier::State::CallArg: + return "CallArg"; + case BlockVerifier::State::EndOfBuffer: + return "EndOfBuffer"; + case BlockVerifier::State::TypedEvent: + return "TypedEvent"; + case BlockVerifier::State::StateMax: + case BlockVerifier::State::Unknown: + return "Unknown"; + } + llvm_unreachable("Unkown state!"); +} + +struct Transition { + BlockVerifier::State From; + std::bitset<number(BlockVerifier::State::StateMax)> ToStates; +}; + +} // namespace + +Error BlockVerifier::transition(State To) { + using ToSet = std::bitset<number(State::StateMax)>; + static constexpr std::array<const Transition, number(State::StateMax)> + TransitionTable{{{State::Unknown, + {mask(State::BufferExtents) | mask(State::NewBuffer)}}, + + {State::BufferExtents, {mask(State::NewBuffer)}}, + + {State::NewBuffer, {mask(State::WallClockTime)}}, + + {State::WallClockTime, + {mask(State::PIDEntry) | mask(State::NewCPUId)}}, + + {State::PIDEntry, {mask(State::NewCPUId)}}, + + {State::NewCPUId, + {mask(State::NewCPUId) | mask(State::TSCWrap) | + mask(State::CustomEvent) | mask(State::Function) | + mask(State::EndOfBuffer) | mask(State::TypedEvent)}}, + + {State::TSCWrap, + {mask(State::TSCWrap) | mask(State::NewCPUId) | + mask(State::CustomEvent) | mask(State::Function) | + mask(State::EndOfBuffer) | mask(State::TypedEvent)}}, + + {State::CustomEvent, + {mask(State::CustomEvent) | mask(State::TSCWrap) | + mask(State::NewCPUId) | mask(State::Function) | + mask(State::EndOfBuffer) | mask(State::TypedEvent)}}, + + {State::TypedEvent, + {mask(State::TypedEvent) | mask(State::TSCWrap) | + mask(State::NewCPUId) | mask(State::Function) | + mask(State::EndOfBuffer) | mask(State::CustomEvent)}}, + + {State::Function, + {mask(State::Function) | mask(State::TSCWrap) | + mask(State::NewCPUId) | mask(State::CustomEvent) | + mask(State::CallArg) | mask(State::EndOfBuffer) | + mask(State::TypedEvent)}}, + + {State::CallArg, + {mask(State::CallArg) | mask(State::Function) | + mask(State::TSCWrap) | mask(State::NewCPUId) | + mask(State::CustomEvent) | mask(State::EndOfBuffer) | + mask(State::TypedEvent)}}, + + {State::EndOfBuffer, {}}}}; + + if (CurrentRecord >= State::StateMax) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "BUG (BlockVerifier): Cannot find transition table entry for %s, " + "transitioning to %s.", + recordToString(CurrentRecord).data(), recordToString(To).data()); + + // If we're at an EndOfBuffer record, we ignore anything that follows that + // isn't a NewBuffer record. + if (CurrentRecord == State::EndOfBuffer && To != State::NewBuffer) + return Error::success(); + + auto &Mapping = TransitionTable[number(CurrentRecord)]; + auto &Destinations = Mapping.ToStates; + assert(Mapping.From == CurrentRecord && + "BUG: Wrong index for record mapping."); + if ((Destinations & ToSet(mask(To))) == 0) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "BlockVerifier: Invalid transition from %s to %s.", + recordToString(CurrentRecord).data(), recordToString(To).data()); + + CurrentRecord = To; + return Error::success(); +} // namespace xray + +Error BlockVerifier::visit(BufferExtents &) { + return transition(State::BufferExtents); +} + +Error BlockVerifier::visit(WallclockRecord &) { + return transition(State::WallClockTime); +} + +Error BlockVerifier::visit(NewCPUIDRecord &) { + return transition(State::NewCPUId); +} + +Error BlockVerifier::visit(TSCWrapRecord &) { + return transition(State::TSCWrap); +} + +Error BlockVerifier::visit(CustomEventRecord &) { + return transition(State::CustomEvent); +} + +Error BlockVerifier::visit(CustomEventRecordV5 &) { + return transition(State::CustomEvent); +} + +Error BlockVerifier::visit(TypedEventRecord &) { + return transition(State::TypedEvent); +} + +Error BlockVerifier::visit(CallArgRecord &) { + return transition(State::CallArg); +} + +Error BlockVerifier::visit(PIDRecord &) { return transition(State::PIDEntry); } + +Error BlockVerifier::visit(NewBufferRecord &) { + return transition(State::NewBuffer); +} + +Error BlockVerifier::visit(EndBufferRecord &) { + return transition(State::EndOfBuffer); +} + +Error BlockVerifier::visit(FunctionRecord &) { + return transition(State::Function); +} + +Error BlockVerifier::verify() { + // The known terminal conditions are the following: + switch (CurrentRecord) { + case State::EndOfBuffer: + case State::NewCPUId: + case State::CustomEvent: + case State::TypedEvent: + case State::Function: + case State::CallArg: + case State::TSCWrap: + return Error::success(); + default: + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "BlockVerifier: Invalid terminal condition %s, malformed block.", + recordToString(CurrentRecord).data()); + } +} + +void BlockVerifier::reset() { CurrentRecord = State::Unknown; } + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/CMakeLists.txt b/lib/XRay/CMakeLists.txt index 8d558209d8ee..c667083ed13a 100644 --- a/lib/XRay/CMakeLists.txt +++ b/lib/XRay/CMakeLists.txt @@ -1,5 +1,17 @@ add_llvm_library(LLVMXRay + BlockIndexer.cpp + BlockPrinter.cpp + BlockVerifier.cpp + FDRRecordProducer.cpp + FDRRecords.cpp + FDRTraceExpander.cpp + FDRTraceWriter.cpp + FileHeaderReader.cpp InstrumentationMap.cpp + LogBuilderConsumer.cpp + Profile.cpp + RecordInitializer.cpp + RecordPrinter.cpp Trace.cpp ADDITIONAL_HEADER_DIRS diff --git a/lib/XRay/FDRRecordProducer.cpp b/lib/XRay/FDRRecordProducer.cpp new file mode 100644 index 000000000000..25b3ee8af219 --- /dev/null +++ b/lib/XRay/FDRRecordProducer.cpp @@ -0,0 +1,198 @@ +//===- FDRRecordProducer.cpp - XRay FDR Mode Record Producer --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FDRRecordProducer.h" +#include "llvm/Support/DataExtractor.h" + +#include <cstdint> + +namespace llvm { +namespace xray { + +namespace { + +// Keep this in sync with the values written in the XRay FDR mode runtime in +// compiler-rt. +enum MetadataRecordKinds : uint8_t { + NewBufferKind, + EndOfBufferKind, + NewCPUIdKind, + TSCWrapKind, + WalltimeMarkerKind, + CustomEventMarkerKind, + CallArgumentKind, + BufferExtentsKind, + TypedEventMarkerKind, + PidKind, + // This is an end marker, used to identify the upper bound for this enum. + EnumEndMarker, +}; + +Expected<std::unique_ptr<Record>> +metadataRecordType(const XRayFileHeader &Header, uint8_t T) { + + if (T >= static_cast<uint8_t>(MetadataRecordKinds::EnumEndMarker)) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Invalid metadata record type: %d", T); + switch (T) { + case MetadataRecordKinds::NewBufferKind: + return make_unique<NewBufferRecord>(); + case MetadataRecordKinds::EndOfBufferKind: + if (Header.Version >= 2) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "End of buffer records are no longer supported starting version " + "2 of the log."); + return make_unique<EndBufferRecord>(); + case MetadataRecordKinds::NewCPUIdKind: + return make_unique<NewCPUIDRecord>(); + case MetadataRecordKinds::TSCWrapKind: + return make_unique<TSCWrapRecord>(); + case MetadataRecordKinds::WalltimeMarkerKind: + return make_unique<WallclockRecord>(); + case MetadataRecordKinds::CustomEventMarkerKind: + if (Header.Version >= 5) + return make_unique<CustomEventRecordV5>(); + return make_unique<CustomEventRecord>(); + case MetadataRecordKinds::CallArgumentKind: + return make_unique<CallArgRecord>(); + case MetadataRecordKinds::BufferExtentsKind: + return make_unique<BufferExtents>(); + case MetadataRecordKinds::TypedEventMarkerKind: + return make_unique<TypedEventRecord>(); + case MetadataRecordKinds::PidKind: + return make_unique<PIDRecord>(); + case MetadataRecordKinds::EnumEndMarker: + llvm_unreachable("Invalid MetadataRecordKind"); + } + llvm_unreachable("Unhandled MetadataRecordKinds enum value"); +} + +constexpr bool isMetadataIntroducer(uint8_t FirstByte) { + return FirstByte & 0x01u; +} + +} // namespace + +Expected<std::unique_ptr<Record>> +FileBasedRecordProducer::findNextBufferExtent() { + // We seek one byte at a time until we find a suitable buffer extents metadata + // record introducer. + std::unique_ptr<Record> R; + while (!R) { + auto PreReadOffset = OffsetPtr; + uint8_t FirstByte = E.getU8(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading one byte from offset %d.", OffsetPtr); + + if (isMetadataIntroducer(FirstByte)) { + auto LoadedType = FirstByte >> 1; + if (LoadedType == MetadataRecordKinds::BufferExtentsKind) { + auto MetadataRecordOrErr = metadataRecordType(Header, LoadedType); + if (!MetadataRecordOrErr) + return MetadataRecordOrErr.takeError(); + + R = std::move(MetadataRecordOrErr.get()); + RecordInitializer RI(E, OffsetPtr); + if (auto Err = R->apply(RI)) + return std::move(Err); + return std::move(R); + } + } + } + llvm_unreachable("Must always terminate with either an error or a record."); +} + +Expected<std::unique_ptr<Record>> FileBasedRecordProducer::produce() { + // First, we set up our result record. + std::unique_ptr<Record> R; + + // Before we do any further reading, we should check whether we're at the end + // of the current buffer we're been consuming. In FDR logs version >= 3, we + // rely on the buffer extents record to determine how many bytes we should be + // considering as valid records. + if (Header.Version >= 3 && CurrentBufferBytes == 0) { + // Find the next buffer extents record. + auto BufferExtentsOrError = findNextBufferExtent(); + if (!BufferExtentsOrError) + return joinErrors( + BufferExtentsOrError.takeError(), + createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed to find the next BufferExtents record.")); + + R = std::move(BufferExtentsOrError.get()); + assert(R != nullptr); + assert(isa<BufferExtents>(R.get())); + auto BE = dyn_cast<BufferExtents>(R.get()); + CurrentBufferBytes = BE->size(); + return std::move(R); + } + + // + // At the top level, we read one byte to determine the type of the record to + // create. This byte will comprise of the following bits: + // + // - offset 0: A '1' indicates a metadata record, a '0' indicates a function + // record. + // - offsets 1-7: For metadata records, this will indicate the kind of + // metadata record should be loaded. + // + // We read first byte, then create the appropriate type of record to consume + // the rest of the bytes. + auto PreReadOffset = OffsetPtr; + uint8_t FirstByte = E.getU8(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading one byte from offset %d.", OffsetPtr); + + // For metadata records, handle especially here. + if (isMetadataIntroducer(FirstByte)) { + auto LoadedType = FirstByte >> 1; + auto MetadataRecordOrErr = metadataRecordType(Header, LoadedType); + if (!MetadataRecordOrErr) + return joinErrors( + MetadataRecordOrErr.takeError(), + createStringError( + std::make_error_code(std::errc::executable_format_error), + "Encountered an unsupported metadata record (%d) at offset %d.", + LoadedType, PreReadOffset)); + R = std::move(MetadataRecordOrErr.get()); + } else { + R = llvm::make_unique<FunctionRecord>(); + } + RecordInitializer RI(E, OffsetPtr); + + if (auto Err = R->apply(RI)) + return std::move(Err); + + // If we encountered a BufferExtents record, we should record the remaining + // bytes for the current buffer, to determine when we should start ignoring + // potentially malformed data and looking for buffer extents records. + if (auto BE = dyn_cast<BufferExtents>(R.get())) { + CurrentBufferBytes = BE->size(); + } else if (Header.Version >= 3) { + if (OffsetPtr - PreReadOffset > CurrentBufferBytes) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Buffer over-read at offset %d (over-read by %d bytes); Record Type " + "= %s.", + OffsetPtr, (OffsetPtr - PreReadOffset) - CurrentBufferBytes, + Record::kindToString(R->getRecordType()).data()); + + CurrentBufferBytes -= OffsetPtr - PreReadOffset; + } + assert(R != nullptr); + return std::move(R); +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/FDRRecords.cpp b/lib/XRay/FDRRecords.cpp new file mode 100644 index 000000000000..2a40d5e06229 --- /dev/null +++ b/lib/XRay/FDRRecords.cpp @@ -0,0 +1,67 @@ +//===- FDRRecords.cpp - XRay Flight Data Recorder Mode Records -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Define types and operations on these types that represent the different kinds +// of records we encounter in XRay flight data recorder mode traces. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FDRRecords.h" + +namespace llvm { +namespace xray { + +Error BufferExtents::apply(RecordVisitor &V) { return V.visit(*this); } +Error WallclockRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error NewCPUIDRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error TSCWrapRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error CustomEventRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error CallArgRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error PIDRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error NewBufferRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error EndBufferRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error FunctionRecord::apply(RecordVisitor &V) { return V.visit(*this); } +Error CustomEventRecordV5::apply(RecordVisitor &V) { return V.visit(*this); } +Error TypedEventRecord::apply(RecordVisitor &V) { return V.visit(*this); } + +StringRef Record::kindToString(RecordKind K) { + switch (K) { + case RecordKind::RK_Metadata: + return "Metadata"; + case RecordKind::RK_Metadata_BufferExtents: + return "Metadata:BufferExtents"; + case RecordKind::RK_Metadata_WallClockTime: + return "Metadata:WallClockTime"; + case RecordKind::RK_Metadata_NewCPUId: + return "Metadata:NewCPUId"; + case RecordKind::RK_Metadata_TSCWrap: + return "Metadata:TSCWrap"; + case RecordKind::RK_Metadata_CustomEvent: + return "Metadata:CustomEvent"; + case RecordKind::RK_Metadata_CustomEventV5: + return "Metadata:CustomEventV5"; + case RecordKind::RK_Metadata_CallArg: + return "Metadata:CallArg"; + case RecordKind::RK_Metadata_PIDEntry: + return "Metadata:PIDEntry"; + case RecordKind::RK_Metadata_NewBuffer: + return "Metadata:NewBuffer"; + case RecordKind::RK_Metadata_EndOfBuffer: + return "Metadata:EndOfBuffer"; + case RecordKind::RK_Metadata_TypedEvent: + return "Metadata:TypedEvent"; + case RecordKind::RK_Metadata_LastMetadata: + return "Metadata:LastMetadata"; + case RecordKind::RK_Function: + return "Function"; + } + return "Unknown"; +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/FDRTraceExpander.cpp b/lib/XRay/FDRTraceExpander.cpp new file mode 100644 index 000000000000..a6e1521da87f --- /dev/null +++ b/lib/XRay/FDRTraceExpander.cpp @@ -0,0 +1,132 @@ +//===- FDRTraceExpander.cpp -----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FDRTraceExpander.h" + +namespace llvm { +namespace xray { + +void TraceExpander::resetCurrentRecord() { + if (BuildingRecord) + C(CurrentRecord); + BuildingRecord = false; + CurrentRecord.CallArgs.clear(); + CurrentRecord.Data.clear(); +} + +Error TraceExpander::visit(BufferExtents &) { + resetCurrentRecord(); + return Error::success(); +} + +Error TraceExpander::visit(WallclockRecord &) { return Error::success(); } + +Error TraceExpander::visit(NewCPUIDRecord &R) { + CPUId = R.cpuid(); + BaseTSC = R.tsc(); + return Error::success(); +} + +Error TraceExpander::visit(TSCWrapRecord &R) { + BaseTSC = R.tsc(); + return Error::success(); +} + +Error TraceExpander::visit(CustomEventRecord &R) { + resetCurrentRecord(); + if (!IgnoringRecords) { + CurrentRecord.TSC = R.tsc(); + CurrentRecord.CPU = R.cpu(); + CurrentRecord.PId = PID; + CurrentRecord.TId = TID; + CurrentRecord.Type = RecordTypes::CUSTOM_EVENT; + CurrentRecord.Data = R.data(); + BuildingRecord = true; + } + return Error::success(); +} + +Error TraceExpander::visit(CustomEventRecordV5 &R) { + resetCurrentRecord(); + if (!IgnoringRecords) { + BaseTSC += R.delta(); + CurrentRecord.TSC = BaseTSC; + CurrentRecord.CPU = CPUId; + CurrentRecord.PId = PID; + CurrentRecord.TId = TID; + CurrentRecord.Type = RecordTypes::CUSTOM_EVENT; + CurrentRecord.Data = R.data(); + BuildingRecord = true; + } + return Error::success(); +} + +Error TraceExpander::visit(TypedEventRecord &R) { + resetCurrentRecord(); + if (!IgnoringRecords) { + BaseTSC += R.delta(); + CurrentRecord.TSC = BaseTSC; + CurrentRecord.CPU = CPUId; + CurrentRecord.PId = PID; + CurrentRecord.TId = TID; + CurrentRecord.RecordType = R.eventType(); + CurrentRecord.Type = RecordTypes::TYPED_EVENT; + CurrentRecord.Data = R.data(); + BuildingRecord = true; + } + return Error::success(); +} + +Error TraceExpander::visit(CallArgRecord &R) { + CurrentRecord.CallArgs.push_back(R.arg()); + CurrentRecord.Type = RecordTypes::ENTER_ARG; + return Error::success(); +} + +Error TraceExpander::visit(PIDRecord &R) { + PID = R.pid(); + return Error::success(); +} + +Error TraceExpander::visit(NewBufferRecord &R) { + if (IgnoringRecords) + IgnoringRecords = false; + TID = R.tid(); + if (LogVersion == 2) + PID = R.tid(); + return Error::success(); +} + +Error TraceExpander::visit(EndBufferRecord &) { + IgnoringRecords = true; + resetCurrentRecord(); + return Error::success(); +} + +Error TraceExpander::visit(FunctionRecord &R) { + resetCurrentRecord(); + if (!IgnoringRecords) { + BaseTSC += R.delta(); + CurrentRecord.Type = R.recordType(); + CurrentRecord.FuncId = R.functionId(); + CurrentRecord.TSC = BaseTSC; + CurrentRecord.PId = PID; + CurrentRecord.TId = TID; + CurrentRecord.CPU = CPUId; + BuildingRecord = true; + } + return Error::success(); +} + +Error TraceExpander::flush() { + resetCurrentRecord(); + return Error::success(); +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/FDRTraceWriter.cpp b/lib/XRay/FDRTraceWriter.cpp new file mode 100644 index 000000000000..c5224f4be094 --- /dev/null +++ b/lib/XRay/FDRTraceWriter.cpp @@ -0,0 +1,154 @@ +//===- FDRTraceWriter.cpp - XRay FDR Trace Writer ---------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Test a utility that can write out XRay FDR Mode formatted trace files. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FDRTraceWriter.h" +#include <tuple> + +namespace llvm { +namespace xray { + +namespace { + +template <size_t Index> struct IndexedWriter { + template < + class Tuple, + typename std::enable_if< + (Index < + std::tuple_size<typename std::remove_reference<Tuple>::type>::value), + int>::type = 0> + static size_t write(support::endian::Writer &OS, Tuple &&T) { + OS.write(std::get<Index>(T)); + return sizeof(std::get<Index>(T)) + IndexedWriter<Index + 1>::write(OS, T); + } + + template < + class Tuple, + typename std::enable_if< + (Index >= + std::tuple_size<typename std::remove_reference<Tuple>::type>::value), + int>::type = 0> + static size_t write(support::endian::Writer &OS, Tuple &&) { + return 0; + } +}; + +template <uint8_t Kind, class... Values> +Error writeMetadata(support::endian::Writer &OS, Values &&... Ds) { + // The first bit in the first byte of metadata records is always set to 1, so + // we ensure this is the case when we write out the first byte of the record. + uint8_t FirstByte = (static_cast<uint8_t>(Kind) << 1) | uint8_t{0x01u}; + auto T = std::make_tuple(std::forward<Values>(std::move(Ds))...); + // Write in field order. + OS.write(FirstByte); + auto Bytes = IndexedWriter<0>::write(OS, T); + assert(Bytes <= 15 && "Must only ever write at most 16 byte metadata!"); + // Pad out with appropriate numbers of zero's. + for (; Bytes < 15; ++Bytes) + OS.write('\0'); + return Error::success(); +} + +} // namespace + +FDRTraceWriter::FDRTraceWriter(raw_ostream &O, const XRayFileHeader &H) + : OS(O, support::endianness::native) { + // We need to re-construct a header, by writing the fields we care about for + // traces, in the format that the runtime would have written. + uint32_t BitField = + (H.ConstantTSC ? 0x01 : 0x0) | (H.NonstopTSC ? 0x02 : 0x0); + + // For endian-correctness, we need to write these fields in the order they + // appear and that we expect, instead of blasting bytes of the struct through. + OS.write(H.Version); + OS.write(H.Type); + OS.write(BitField); + OS.write(H.CycleFrequency); + ArrayRef<char> FreeFormBytes(H.FreeFormData, + sizeof(XRayFileHeader::FreeFormData)); + OS.write(FreeFormBytes); +} + +FDRTraceWriter::~FDRTraceWriter() {} + +Error FDRTraceWriter::visit(BufferExtents &R) { + return writeMetadata<7u>(OS, R.size()); +} + +Error FDRTraceWriter::visit(WallclockRecord &R) { + return writeMetadata<4u>(OS, R.seconds(), R.nanos()); +} + +Error FDRTraceWriter::visit(NewCPUIDRecord &R) { + return writeMetadata<2u>(OS, R.cpuid(), R.tsc()); +} + +Error FDRTraceWriter::visit(TSCWrapRecord &R) { + return writeMetadata<3u>(OS, R.tsc()); +} + +Error FDRTraceWriter::visit(CustomEventRecord &R) { + if (auto E = writeMetadata<5u>(OS, R.size(), R.tsc(), R.cpu())) + return E; + auto D = R.data(); + ArrayRef<char> Bytes(D.data(), D.size()); + OS.write(Bytes); + return Error::success(); +} + +Error FDRTraceWriter::visit(CustomEventRecordV5 &R) { + if (auto E = writeMetadata<5u>(OS, R.size(), R.delta())) + return E; + auto D = R.data(); + ArrayRef<char> Bytes(D.data(), D.size()); + OS.write(Bytes); + return Error::success(); +} + +Error FDRTraceWriter::visit(TypedEventRecord &R) { + if (auto E = writeMetadata<8u>(OS, R.size(), R.delta(), R.eventType())) + return E; + auto D = R.data(); + ArrayRef<char> Bytes(D.data(), D.size()); + OS.write(Bytes); + return Error::success(); +} + +Error FDRTraceWriter::visit(CallArgRecord &R) { + return writeMetadata<6u>(OS, R.arg()); +} + +Error FDRTraceWriter::visit(PIDRecord &R) { + return writeMetadata<9u>(OS, R.pid()); +} + +Error FDRTraceWriter::visit(NewBufferRecord &R) { + return writeMetadata<0u>(OS, R.tid()); +} + +Error FDRTraceWriter::visit(EndBufferRecord &R) { + return writeMetadata<1u>(OS, 0); +} + +Error FDRTraceWriter::visit(FunctionRecord &R) { + // Write out the data in "field" order, to be endian-aware. + uint32_t TypeRecordFuncId = uint32_t{R.functionId() & ~uint32_t{0x0Fu << 28}}; + TypeRecordFuncId <<= 3; + TypeRecordFuncId |= static_cast<uint32_t>(R.recordType()); + TypeRecordFuncId <<= 1; + TypeRecordFuncId &= ~uint32_t{0x01}; + OS.write(TypeRecordFuncId); + OS.write(R.delta()); + return Error::success(); +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/FileHeaderReader.cpp b/lib/XRay/FileHeaderReader.cpp new file mode 100644 index 000000000000..0b3fb8b6f692 --- /dev/null +++ b/lib/XRay/FileHeaderReader.cpp @@ -0,0 +1,70 @@ +//===- FileHeaderReader.cpp - XRay File Header Reader --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FileHeaderReader.h" + +namespace llvm { +namespace xray { + +// Populates the FileHeader reference by reading the first 32 bytes of the file. +Expected<XRayFileHeader> readBinaryFormatHeader(DataExtractor &HeaderExtractor, + uint32_t &OffsetPtr) { + // FIXME: Maybe deduce whether the data is little or big-endian using some + // magic bytes in the beginning of the file? + + // First 32 bytes of the file will always be the header. We assume a certain + // format here: + // + // (2) uint16 : version + // (2) uint16 : type + // (4) uint32 : bitfield + // (8) uint64 : cycle frequency + // (16) - : padding + XRayFileHeader FileHeader; + auto PreReadOffset = OffsetPtr; + FileHeader.Version = HeaderExtractor.getU16(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading version from file header at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + FileHeader.Type = HeaderExtractor.getU16(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading file type from file header at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + uint32_t Bitfield = HeaderExtractor.getU32(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading flag bits from file header at offset %d.", OffsetPtr); + + FileHeader.ConstantTSC = Bitfield & 1uL; + FileHeader.NonstopTSC = Bitfield & 1uL << 1; + PreReadOffset = OffsetPtr; + FileHeader.CycleFrequency = HeaderExtractor.getU64(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading cycle frequency from file header at offset %d.", + OffsetPtr); + + std::memcpy(&FileHeader.FreeFormData, + HeaderExtractor.getData().bytes_begin() + OffsetPtr, 16); + + // Manually advance the offset pointer 16 bytes, after getting a raw memcpy + // from the underlying data. + OffsetPtr += 16; + return std::move(FileHeader); +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/InstrumentationMap.cpp b/lib/XRay/InstrumentationMap.cpp index a7d6600b0d8a..9f2b179486f0 100644 --- a/lib/XRay/InstrumentationMap.cpp +++ b/lib/XRay/InstrumentationMap.cpp @@ -12,12 +12,14 @@ //===----------------------------------------------------------------------===// #include "llvm/XRay/InstrumentationMap.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" #include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/DataExtractor.h" #include "llvm/Support/Error.h" @@ -46,19 +48,21 @@ Optional<uint64_t> InstrumentationMap::getFunctionAddr(int32_t FuncId) const { return None; } +using RelocMap = DenseMap<uint64_t, uint64_t>; + static Error -loadELF64(StringRef Filename, object::OwningBinary<object::ObjectFile> &ObjFile, +loadObj(StringRef Filename, object::OwningBinary<object::ObjectFile> &ObjFile, InstrumentationMap::SledContainer &Sleds, InstrumentationMap::FunctionAddressMap &FunctionAddresses, InstrumentationMap::FunctionAddressReverseMap &FunctionIds) { InstrumentationMap Map; // Find the section named "xray_instr_map". - if (!ObjFile.getBinary()->isELF() || + if ((!ObjFile.getBinary()->isELF() && !ObjFile.getBinary()->isMachO()) || !(ObjFile.getBinary()->getArch() == Triple::x86_64 || ObjFile.getBinary()->getArch() == Triple::ppc64le)) return make_error<StringError>( - "File format not supported (only does ELF little endian 64-bit).", + "File format not supported (only does ELF and Mach-O little endian 64-bit).", std::make_error_code(std::errc::not_supported)); StringRef Contents = ""; @@ -79,6 +83,31 @@ loadELF64(StringRef Filename, object::OwningBinary<object::ObjectFile> &ObjFile, return errorCodeToError( std::make_error_code(std::errc::executable_format_error)); + RelocMap Relocs; + if (ObjFile.getBinary()->isELF()) { + uint32_t RelativeRelocation = [](object::ObjectFile *ObjFile) { + if (const auto *ELFObj = dyn_cast<object::ELF32LEObjectFile>(ObjFile)) + return ELFObj->getELFFile()->getRelativeRelocationType(); + else if (const auto *ELFObj = dyn_cast<object::ELF32BEObjectFile>(ObjFile)) + return ELFObj->getELFFile()->getRelativeRelocationType(); + else if (const auto *ELFObj = dyn_cast<object::ELF64LEObjectFile>(ObjFile)) + return ELFObj->getELFFile()->getRelativeRelocationType(); + else if (const auto *ELFObj = dyn_cast<object::ELF64BEObjectFile>(ObjFile)) + return ELFObj->getELFFile()->getRelativeRelocationType(); + else + return static_cast<uint32_t>(0); + }(ObjFile.getBinary()); + + for (const object::SectionRef &Section : Sections) { + for (const object::RelocationRef &Reloc : Section.relocations()) { + if (Reloc.getType() != RelativeRelocation) + continue; + if (auto AddendOrErr = object::ELFRelocationRef(Reloc).getAddend()) + Relocs.insert({Reloc.getOffset(), *AddendOrErr}); + } + } + } + // Copy the instrumentation map data into the Sleds data structure. auto C = Contents.bytes_begin(); static constexpr size_t ELF64SledEntrySize = 32; @@ -89,6 +118,16 @@ loadELF64(StringRef Filename, object::OwningBinary<object::ObjectFile> &ObjFile, "an XRay sled entry in ELF64."), std::make_error_code(std::errc::executable_format_error)); + auto RelocateOrElse = [&](uint32_t Offset, uint64_t Address) { + if (!Address) { + uint64_t A = I->getAddress() + C - Contents.bytes_begin() + Offset; + RelocMap::const_iterator R = Relocs.find(A); + if (R != Relocs.end()) + return R->second; + } + return Address; + }; + int32_t FuncId = 1; uint64_t CurFn = 0; for (; C != Contents.bytes_end(); C += ELF64SledEntrySize) { @@ -98,8 +137,10 @@ loadELF64(StringRef Filename, object::OwningBinary<object::ObjectFile> &ObjFile, Sleds.push_back({}); auto &Entry = Sleds.back(); uint32_t OffsetPtr = 0; - Entry.Address = Extractor.getU64(&OffsetPtr); - Entry.Function = Extractor.getU64(&OffsetPtr); + uint32_t AddrOff = OffsetPtr; + Entry.Address = RelocateOrElse(AddrOff, Extractor.getU64(&OffsetPtr)); + uint32_t FuncOff = OffsetPtr; + Entry.Function = RelocateOrElse(FuncOff, Extractor.getU64(&OffsetPtr)); auto Kind = Extractor.getU8(&OffsetPtr); static constexpr SledEntry::FunctionKinds Kinds[] = { SledEntry::FunctionKinds::ENTRY, SledEntry::FunctionKinds::EXIT, @@ -191,7 +232,7 @@ llvm::xray::loadInstrumentationMap(StringRef Filename) { if (auto E = loadYAML(Fd, FileSize, Filename, Map.Sleds, Map.FunctionAddresses, Map.FunctionIds)) return std::move(E); - } else if (auto E = loadELF64(Filename, *ObjectFileOrError, Map.Sleds, + } else if (auto E = loadObj(Filename, *ObjectFileOrError, Map.Sleds, Map.FunctionAddresses, Map.FunctionIds)) { return std::move(E); } diff --git a/lib/XRay/LLVMBuild.txt b/lib/XRay/LLVMBuild.txt new file mode 100644 index 000000000000..904168dad939 --- /dev/null +++ b/lib/XRay/LLVMBuild.txt @@ -0,0 +1,23 @@ +;===- ./lib/XRay/LLVMBuild.txt ---------------------------------*- Conf -*--===; +; +; The LLVM Compiler Infrastructure +; +; This file is distributed under the University of Illinois Open Source +; License. See LICENSE.TXT for details. +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Library +name = XRay +parent = Libraries +required_libraries = Support Object +installed = 1 diff --git a/lib/XRay/LogBuilderConsumer.cpp b/lib/XRay/LogBuilderConsumer.cpp new file mode 100644 index 000000000000..88b7d2d728b1 --- /dev/null +++ b/lib/XRay/LogBuilderConsumer.cpp @@ -0,0 +1,38 @@ +//===- FDRRecordConsumer.h - XRay Flight Data Recorder Mode Records -------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FDRRecordConsumer.h" + +namespace llvm { +namespace xray { + +Error LogBuilderConsumer::consume(std::unique_ptr<Record> R) { + if (!R) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Must not call RecordConsumer::consume() with a null pointer."); + Records.push_back(std::move(R)); + return Error::success(); +} + +Error PipelineConsumer::consume(std::unique_ptr<Record> R) { + if (!R) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Must not call RecordConsumer::consume() with a null pointer."); + + // We apply all of the visitors in order, and concatenate errors + // appropriately. + Error Result = Error::success(); + for (auto *V : Visitors) + Result = joinErrors(std::move(Result), R->apply(*V)); + return Result; +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/Profile.cpp b/lib/XRay/Profile.cpp new file mode 100644 index 000000000000..e8a082884d69 --- /dev/null +++ b/lib/XRay/Profile.cpp @@ -0,0 +1,403 @@ +//===- Profile.cpp - XRay Profile Abstraction -----------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Defines the XRay Profile class representing the latency profile generated by +// XRay's profiling mode. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/Profile.h" + +#include "llvm/Support/DataExtractor.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/XRay/Trace.h" +#include <deque> +#include <memory> + +namespace llvm { +namespace xray { + +Profile::Profile(const Profile &O) { + // We need to re-create all the tries from the original (O), into the current + // Profile being initialized, through the Block instances we see. + for (const auto &Block : O) { + Blocks.push_back({Block.Thread, {}}); + auto &B = Blocks.back(); + for (const auto &PathData : Block.PathData) + B.PathData.push_back({internPath(cantFail(O.expandPath(PathData.first))), + PathData.second}); + } +} + +Profile &Profile::operator=(const Profile &O) { + Profile P = O; + *this = std::move(P); + return *this; +} + +namespace { + +struct BlockHeader { + uint32_t Size; + uint32_t Number; + uint64_t Thread; +}; + +static Expected<BlockHeader> readBlockHeader(DataExtractor &Extractor, + uint32_t &Offset) { + BlockHeader H; + uint32_t CurrentOffset = Offset; + H.Size = Extractor.getU32(&Offset); + if (Offset == CurrentOffset) + return make_error<StringError>( + Twine("Error parsing block header size at offset '") + + Twine(CurrentOffset) + "'", + std::make_error_code(std::errc::invalid_argument)); + CurrentOffset = Offset; + H.Number = Extractor.getU32(&Offset); + if (Offset == CurrentOffset) + return make_error<StringError>( + Twine("Error parsing block header number at offset '") + + Twine(CurrentOffset) + "'", + std::make_error_code(std::errc::invalid_argument)); + CurrentOffset = Offset; + H.Thread = Extractor.getU64(&Offset); + if (Offset == CurrentOffset) + return make_error<StringError>( + Twine("Error parsing block header thread id at offset '") + + Twine(CurrentOffset) + "'", + std::make_error_code(std::errc::invalid_argument)); + return H; +} + +static Expected<std::vector<Profile::FuncID>> readPath(DataExtractor &Extractor, + uint32_t &Offset) { + // We're reading a sequence of int32_t's until we find a 0. + std::vector<Profile::FuncID> Path; + auto CurrentOffset = Offset; + int32_t FuncId; + do { + FuncId = Extractor.getSigned(&Offset, 4); + if (CurrentOffset == Offset) + return make_error<StringError>( + Twine("Error parsing path at offset '") + Twine(CurrentOffset) + "'", + std::make_error_code(std::errc::invalid_argument)); + CurrentOffset = Offset; + Path.push_back(FuncId); + } while (FuncId != 0); + return std::move(Path); +} + +static Expected<Profile::Data> readData(DataExtractor &Extractor, + uint32_t &Offset) { + // We expect a certain number of elements for Data: + // - A 64-bit CallCount + // - A 64-bit CumulativeLocalTime counter + Profile::Data D; + auto CurrentOffset = Offset; + D.CallCount = Extractor.getU64(&Offset); + if (CurrentOffset == Offset) + return make_error<StringError>( + Twine("Error parsing call counts at offset '") + Twine(CurrentOffset) + + "'", + std::make_error_code(std::errc::invalid_argument)); + CurrentOffset = Offset; + D.CumulativeLocalTime = Extractor.getU64(&Offset); + if (CurrentOffset == Offset) + return make_error<StringError>( + Twine("Error parsing cumulative local time at offset '") + + Twine(CurrentOffset) + "'", + std::make_error_code(std::errc::invalid_argument)); + return D; +} + +} // namespace + +Error Profile::addBlock(Block &&B) { + if (B.PathData.empty()) + return make_error<StringError>( + "Block may not have empty path data.", + std::make_error_code(std::errc::invalid_argument)); + + Blocks.emplace_back(std::move(B)); + return Error::success(); +} + +Expected<std::vector<Profile::FuncID>> Profile::expandPath(PathID P) const { + auto It = PathIDMap.find(P); + if (It == PathIDMap.end()) + return make_error<StringError>( + Twine("PathID not found: ") + Twine(P), + std::make_error_code(std::errc::invalid_argument)); + std::vector<Profile::FuncID> Path; + for (auto Node = It->second; Node; Node = Node->Caller) + Path.push_back(Node->Func); + return std::move(Path); +} + +Profile::PathID Profile::internPath(ArrayRef<FuncID> P) { + if (P.empty()) + return 0; + + auto RootToLeafPath = reverse(P); + + // Find the root. + auto It = RootToLeafPath.begin(); + auto PathRoot = *It++; + auto RootIt = + find_if(Roots, [PathRoot](TrieNode *N) { return N->Func == PathRoot; }); + + // If we've not seen this root before, remember it. + TrieNode *Node = nullptr; + if (RootIt == Roots.end()) { + NodeStorage.emplace_back(); + Node = &NodeStorage.back(); + Node->Func = PathRoot; + Roots.push_back(Node); + } else { + Node = *RootIt; + } + + // Now traverse the path, re-creating if necessary. + while (It != RootToLeafPath.end()) { + auto NodeFuncID = *It++; + auto CalleeIt = find_if(Node->Callees, [NodeFuncID](TrieNode *N) { + return N->Func == NodeFuncID; + }); + if (CalleeIt == Node->Callees.end()) { + NodeStorage.emplace_back(); + auto NewNode = &NodeStorage.back(); + NewNode->Func = NodeFuncID; + NewNode->Caller = Node; + Node->Callees.push_back(NewNode); + Node = NewNode; + } else { + Node = *CalleeIt; + } + } + + // At this point, Node *must* be pointing at the leaf. + assert(Node->Func == P.front()); + if (Node->ID == 0) { + Node->ID = NextID++; + PathIDMap.insert({Node->ID, Node}); + } + return Node->ID; +} + +Profile mergeProfilesByThread(const Profile &L, const Profile &R) { + Profile Merged; + using PathDataMap = DenseMap<Profile::PathID, Profile::Data>; + using PathDataMapPtr = std::unique_ptr<PathDataMap>; + using PathDataVector = decltype(Profile::Block::PathData); + using ThreadProfileIndexMap = DenseMap<Profile::ThreadID, PathDataMapPtr>; + ThreadProfileIndexMap ThreadProfileIndex; + + for (const auto &P : {std::ref(L), std::ref(R)}) + for (const auto &Block : P.get()) { + ThreadProfileIndexMap::iterator It; + std::tie(It, std::ignore) = ThreadProfileIndex.insert( + {Block.Thread, PathDataMapPtr{new PathDataMap()}}); + for (const auto &PathAndData : Block.PathData) { + auto &PathID = PathAndData.first; + auto &Data = PathAndData.second; + auto NewPathID = + Merged.internPath(cantFail(P.get().expandPath(PathID))); + PathDataMap::iterator PathDataIt; + bool Inserted; + std::tie(PathDataIt, Inserted) = It->second->insert({NewPathID, Data}); + if (!Inserted) { + auto &ExistingData = PathDataIt->second; + ExistingData.CallCount += Data.CallCount; + ExistingData.CumulativeLocalTime += Data.CumulativeLocalTime; + } + } + } + + for (const auto &IndexedThreadBlock : ThreadProfileIndex) { + PathDataVector PathAndData; + PathAndData.reserve(IndexedThreadBlock.second->size()); + copy(*IndexedThreadBlock.second, std::back_inserter(PathAndData)); + cantFail( + Merged.addBlock({IndexedThreadBlock.first, std::move(PathAndData)})); + } + return Merged; +} + +Profile mergeProfilesByStack(const Profile &L, const Profile &R) { + Profile Merged; + using PathDataMap = DenseMap<Profile::PathID, Profile::Data>; + PathDataMap PathData; + using PathDataVector = decltype(Profile::Block::PathData); + for (const auto &P : {std::ref(L), std::ref(R)}) + for (const auto &Block : P.get()) + for (const auto &PathAndData : Block.PathData) { + auto &PathId = PathAndData.first; + auto &Data = PathAndData.second; + auto NewPathID = + Merged.internPath(cantFail(P.get().expandPath(PathId))); + PathDataMap::iterator PathDataIt; + bool Inserted; + std::tie(PathDataIt, Inserted) = PathData.insert({NewPathID, Data}); + if (!Inserted) { + auto &ExistingData = PathDataIt->second; + ExistingData.CallCount += Data.CallCount; + ExistingData.CumulativeLocalTime += Data.CumulativeLocalTime; + } + } + + // In the end there's a single Block, for thread 0. + PathDataVector Block; + Block.reserve(PathData.size()); + copy(PathData, std::back_inserter(Block)); + cantFail(Merged.addBlock({0, std::move(Block)})); + return Merged; +} + +Expected<Profile> loadProfile(StringRef Filename) { + int Fd; + if (auto EC = sys::fs::openFileForRead(Filename, Fd)) + return make_error<StringError>( + Twine("Cannot read profile from '") + Filename + "'", EC); + + uint64_t FileSize; + if (auto EC = sys::fs::file_size(Filename, FileSize)) + return make_error<StringError>( + Twine("Cannot get filesize of '") + Filename + "'", EC); + + std::error_code EC; + sys::fs::mapped_file_region MappedFile( + Fd, sys::fs::mapped_file_region::mapmode::readonly, FileSize, 0, EC); + if (EC) + return make_error<StringError>( + Twine("Cannot mmap profile '") + Filename + "'", EC); + StringRef Data(MappedFile.data(), MappedFile.size()); + + Profile P; + uint32_t Offset = 0; + DataExtractor Extractor(Data, true, 8); + + // For each block we get from the file: + while (Offset != MappedFile.size()) { + auto HeaderOrError = readBlockHeader(Extractor, Offset); + if (!HeaderOrError) + return HeaderOrError.takeError(); + + // TODO: Maybe store this header information for each block, even just for + // debugging? + const auto &Header = HeaderOrError.get(); + + // Read in the path data. + auto PathOrError = readPath(Extractor, Offset); + if (!PathOrError) + return PathOrError.takeError(); + const auto &Path = PathOrError.get(); + + // For each path we encounter, we should intern it to get a PathID. + auto DataOrError = readData(Extractor, Offset); + if (!DataOrError) + return DataOrError.takeError(); + auto &Data = DataOrError.get(); + + if (auto E = + P.addBlock(Profile::Block{Profile::ThreadID{Header.Thread}, + {{P.internPath(Path), std::move(Data)}}})) + return std::move(E); + } + + return P; +} + +namespace { + +struct StackEntry { + uint64_t Timestamp; + Profile::FuncID FuncId; +}; + +} // namespace + +Expected<Profile> profileFromTrace(const Trace &T) { + Profile P; + + // The implementation of the algorithm re-creates the execution of + // the functions based on the trace data. To do this, we set up a number of + // data structures to track the execution context of every thread in the + // Trace. + DenseMap<Profile::ThreadID, std::vector<StackEntry>> ThreadStacks; + DenseMap<Profile::ThreadID, DenseMap<Profile::PathID, Profile::Data>> + ThreadPathData; + + // We then do a pass through the Trace to account data on a per-thread-basis. + for (const auto &E : T) { + auto &TSD = ThreadStacks[E.TId]; + switch (E.Type) { + case RecordTypes::ENTER: + case RecordTypes::ENTER_ARG: + + // Push entries into the function call stack. + TSD.push_back({E.TSC, E.FuncId}); + break; + + case RecordTypes::EXIT: + case RecordTypes::TAIL_EXIT: + + // Exits cause some accounting to happen, based on the state of the stack. + // For each function we pop off the stack, we take note of the path and + // record the cumulative state for this path. As we're doing this, we + // intern the path into the Profile. + while (!TSD.empty()) { + auto Top = TSD.back(); + auto FunctionLocalTime = AbsoluteDifference(Top.Timestamp, E.TSC); + SmallVector<Profile::FuncID, 16> Path; + transform(reverse(TSD), std::back_inserter(Path), + std::mem_fn(&StackEntry::FuncId)); + auto InternedPath = P.internPath(Path); + auto &TPD = ThreadPathData[E.TId][InternedPath]; + ++TPD.CallCount; + TPD.CumulativeLocalTime += FunctionLocalTime; + TSD.pop_back(); + + // If we've matched the corresponding entry event for this function, + // then we exit the loop. + if (Top.FuncId == E.FuncId) + break; + + // FIXME: Consider the intermediate times and the cumulative tree time + // as well. + } + + break; + + case RecordTypes::CUSTOM_EVENT: + case RecordTypes::TYPED_EVENT: + // TODO: Support an extension point to allow handling of custom and typed + // events in profiles. + break; + } + } + + // Once we've gone through the Trace, we now create one Block per thread in + // the Profile. + for (const auto &ThreadPaths : ThreadPathData) { + const auto &TID = ThreadPaths.first; + const auto &PathsData = ThreadPaths.second; + if (auto E = P.addBlock({ + TID, + std::vector<std::pair<Profile::PathID, Profile::Data>>( + PathsData.begin(), PathsData.end()), + })) + return std::move(E); + } + + return P; +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/RecordInitializer.cpp b/lib/XRay/RecordInitializer.cpp new file mode 100644 index 000000000000..f136a1e456b7 --- /dev/null +++ b/lib/XRay/RecordInitializer.cpp @@ -0,0 +1,418 @@ +//===- FDRRecordProducer.cpp - XRay FDR Mode Record Producer --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FDRRecords.h" + +namespace llvm { +namespace xray { + +Error RecordInitializer::visit(BufferExtents &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, sizeof(uint64_t))) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a buffer extent (%d).", + OffsetPtr); + + auto PreReadOffset = OffsetPtr; + R.Size = E.getU64(&OffsetPtr); + if (PreReadOffset == OffsetPtr) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Cannot read buffer extent at offset %d.", + OffsetPtr); + + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - PreReadOffset); + return Error::success(); +} + +Error RecordInitializer::visit(WallclockRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a wallclock record (%d).", + OffsetPtr); + auto BeginOffset = OffsetPtr; + auto PreReadOffset = OffsetPtr; + R.Seconds = E.getU64(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read wall clock 'seconds' field at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + R.Nanos = E.getU32(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read wall clock 'nanos' field at offset %d.", OffsetPtr); + + // Align to metadata record size boundary. + assert(OffsetPtr - BeginOffset <= MetadataRecord::kMetadataBodySize); + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - BeginOffset); + return Error::success(); +} + +Error RecordInitializer::visit(NewCPUIDRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a new cpu id record (%d).", + OffsetPtr); + auto BeginOffset = OffsetPtr; + auto PreReadOffset = OffsetPtr; + R.CPUId = E.getU16(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Cannot read CPU id at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + R.TSC = E.getU64(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Cannot read CPU TSC at offset %d.", OffsetPtr); + + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - BeginOffset); + return Error::success(); +} + +Error RecordInitializer::visit(TSCWrapRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a new TSC wrap record (%d).", + OffsetPtr); + + auto PreReadOffset = OffsetPtr; + R.BaseTSC = E.getU64(&OffsetPtr); + if (PreReadOffset == OffsetPtr) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Cannot read TSC wrap record at offset %d.", + OffsetPtr); + + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - PreReadOffset); + return Error::success(); +} + +Error RecordInitializer::visit(CustomEventRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a custom event record (%d).", + OffsetPtr); + + auto BeginOffset = OffsetPtr; + auto PreReadOffset = OffsetPtr; + R.Size = E.getSigned(&OffsetPtr, sizeof(int32_t)); + if (PreReadOffset == OffsetPtr) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read a custom event record size field offset %d.", OffsetPtr); + + if (R.Size <= 0) + return createStringError( + std::make_error_code(std::errc::bad_address), + "Invalid size for custom event (size = %d) at offset %d.", R.Size, + OffsetPtr); + + PreReadOffset = OffsetPtr; + R.TSC = E.getU64(&OffsetPtr); + if (PreReadOffset == OffsetPtr) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read a custom event TSC field at offset %d.", OffsetPtr); + + // For version 4 onwards, of the FDR log, we want to also capture the CPU ID + // of the custom event. + if (Version >= 4) { + PreReadOffset = OffsetPtr; + R.CPU = E.getU16(&OffsetPtr); + if (PreReadOffset == OffsetPtr) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Missing CPU field at offset %d", OffsetPtr); + } + + assert(OffsetPtr > BeginOffset && + OffsetPtr - BeginOffset <= MetadataRecord::kMetadataBodySize); + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - BeginOffset); + + // Next we read in a fixed chunk of data from the given offset. + if (!E.isValidOffsetForDataOfSize(OffsetPtr, R.Size)) + return createStringError( + std::make_error_code(std::errc::bad_address), + "Cannot read %d bytes of custom event data from offset %d.", R.Size, + OffsetPtr); + + std::vector<uint8_t> Buffer; + Buffer.resize(R.Size); + PreReadOffset = OffsetPtr; + if (E.getU8(&OffsetPtr, Buffer.data(), R.Size) != Buffer.data()) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading data into buffer of size %d at offset %d.", R.Size, + OffsetPtr); + + assert(OffsetPtr >= PreReadOffset); + if (OffsetPtr - PreReadOffset != static_cast<uint32_t>(R.Size)) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading enough bytes for the custom event payload -- read %d " + "expecting %d bytes at offset %d.", + OffsetPtr - PreReadOffset, R.Size, PreReadOffset); + + R.Data.assign(Buffer.begin(), Buffer.end()); + return Error::success(); +} + +Error RecordInitializer::visit(CustomEventRecordV5 &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a custom event record (%d).", + OffsetPtr); + + auto BeginOffset = OffsetPtr; + auto PreReadOffset = OffsetPtr; + + R.Size = E.getSigned(&OffsetPtr, sizeof(int32_t)); + if (PreReadOffset == OffsetPtr) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read a custom event record size field offset %d.", OffsetPtr); + + if (R.Size <= 0) + return createStringError( + std::make_error_code(std::errc::bad_address), + "Invalid size for custom event (size = %d) at offset %d.", R.Size, + OffsetPtr); + + PreReadOffset = OffsetPtr; + R.Delta = E.getSigned(&OffsetPtr, sizeof(int32_t)); + if (PreReadOffset == OffsetPtr) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read a custom event record TSC delta field at offset %d.", + OffsetPtr); + + assert(OffsetPtr > BeginOffset && + OffsetPtr - BeginOffset <= MetadataRecord::kMetadataBodySize); + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - BeginOffset); + + // Next we read in a fixed chunk of data from the given offset. + if (!E.isValidOffsetForDataOfSize(OffsetPtr, R.Size)) + return createStringError( + std::make_error_code(std::errc::bad_address), + "Cannot read %d bytes of custom event data from offset %d.", R.Size, + OffsetPtr); + + std::vector<uint8_t> Buffer; + Buffer.resize(R.Size); + PreReadOffset = OffsetPtr; + if (E.getU8(&OffsetPtr, Buffer.data(), R.Size) != Buffer.data()) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading data into buffer of size %d at offset %d.", R.Size, + OffsetPtr); + + assert(OffsetPtr >= PreReadOffset); + if (OffsetPtr - PreReadOffset != static_cast<uint32_t>(R.Size)) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading enough bytes for the custom event payload -- read %d " + "expecting %d bytes at offset %d.", + OffsetPtr - PreReadOffset, R.Size, PreReadOffset); + + R.Data.assign(Buffer.begin(), Buffer.end()); + return Error::success(); +} + +Error RecordInitializer::visit(TypedEventRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a typed event record (%d).", + OffsetPtr); + + auto BeginOffset = OffsetPtr; + auto PreReadOffset = OffsetPtr; + + R.Size = E.getSigned(&OffsetPtr, sizeof(int32_t)); + if (PreReadOffset == OffsetPtr) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read a typed event record size field offset %d.", OffsetPtr); + + if (R.Size <= 0) + return createStringError( + std::make_error_code(std::errc::bad_address), + "Invalid size for typed event (size = %d) at offset %d.", R.Size, + OffsetPtr); + + PreReadOffset = OffsetPtr; + R.Delta = E.getSigned(&OffsetPtr, sizeof(int32_t)); + if (PreReadOffset == OffsetPtr) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read a typed event record TSC delta field at offset %d.", + OffsetPtr); + + PreReadOffset = OffsetPtr; + R.EventType = E.getU16(&OffsetPtr); + if (PreReadOffset == OffsetPtr) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Cannot read a typed event record type field at offset %d.", OffsetPtr); + + assert(OffsetPtr > BeginOffset && + OffsetPtr - BeginOffset <= MetadataRecord::kMetadataBodySize); + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - BeginOffset); + + // Next we read in a fixed chunk of data from the given offset. + if (!E.isValidOffsetForDataOfSize(OffsetPtr, R.Size)) + return createStringError( + std::make_error_code(std::errc::bad_address), + "Cannot read %d bytes of custom event data from offset %d.", R.Size, + OffsetPtr); + + std::vector<uint8_t> Buffer; + Buffer.resize(R.Size); + PreReadOffset = OffsetPtr; + if (E.getU8(&OffsetPtr, Buffer.data(), R.Size) != Buffer.data()) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading data into buffer of size %d at offset %d.", R.Size, + OffsetPtr); + + assert(OffsetPtr >= PreReadOffset); + if (OffsetPtr - PreReadOffset != static_cast<uint32_t>(R.Size)) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Failed reading enough bytes for the typed event payload -- read %d " + "expecting %d bytes at offset %d.", + OffsetPtr - PreReadOffset, R.Size, PreReadOffset); + + R.Data.assign(Buffer.begin(), Buffer.end()); + return Error::success(); +} + +Error RecordInitializer::visit(CallArgRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a call argument record (%d).", + OffsetPtr); + + auto PreReadOffset = OffsetPtr; + R.Arg = E.getU64(&OffsetPtr); + if (PreReadOffset == OffsetPtr) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Cannot read a call arg record at offset %d.", + OffsetPtr); + + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - PreReadOffset); + return Error::success(); +} + +Error RecordInitializer::visit(PIDRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a process ID record (%d).", + OffsetPtr); + + auto PreReadOffset = OffsetPtr; + R.PID = E.getSigned(&OffsetPtr, 4); + if (PreReadOffset == OffsetPtr) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Cannot read a process ID record at offset %d.", + OffsetPtr); + + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - PreReadOffset); + return Error::success(); +} + +Error RecordInitializer::visit(NewBufferRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a new buffer record (%d).", + OffsetPtr); + + auto PreReadOffset = OffsetPtr; + R.TID = E.getSigned(&OffsetPtr, sizeof(int32_t)); + if (PreReadOffset == OffsetPtr) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Cannot read a new buffer record at offset %d.", + OffsetPtr); + + OffsetPtr += MetadataRecord::kMetadataBodySize - (OffsetPtr - PreReadOffset); + return Error::success(); +} + +Error RecordInitializer::visit(EndBufferRecord &R) { + if (!E.isValidOffsetForDataOfSize(OffsetPtr, + MetadataRecord::kMetadataBodySize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for an end-of-buffer record (%d).", + OffsetPtr); + + OffsetPtr += MetadataRecord::kMetadataBodySize; + return Error::success(); +} + +Error RecordInitializer::visit(FunctionRecord &R) { + // For function records, we need to retreat one byte back to read a full + // unsigned 32-bit value. The first four bytes will have the following + // layout: + // + // bit 0 : function record indicator (must be 0) + // bits 1..3 : function record type + // bits 4..32 : function id + // + if (OffsetPtr == 0 || !E.isValidOffsetForDataOfSize( + --OffsetPtr, FunctionRecord::kFunctionRecordSize)) + return createStringError(std::make_error_code(std::errc::bad_address), + "Invalid offset for a function record (%d).", + OffsetPtr); + + auto BeginOffset = OffsetPtr; + auto PreReadOffset = BeginOffset; + uint32_t Buffer = E.getU32(&OffsetPtr); + if (PreReadOffset == OffsetPtr) + return createStringError(std::make_error_code(std::errc::bad_address), + "Cannot read function id field from offset %d.", + OffsetPtr); + + // To get the function record type, we shift the buffer one to the right + // (truncating the function record indicator) then take the three bits + // (0b0111) to get the record type as an unsigned value. + unsigned FunctionType = (Buffer >> 1) & 0x07u; + switch (FunctionType) { + case static_cast<unsigned>(RecordTypes::ENTER): + case static_cast<unsigned>(RecordTypes::ENTER_ARG): + case static_cast<unsigned>(RecordTypes::EXIT): + case static_cast<unsigned>(RecordTypes::TAIL_EXIT): + R.Kind = static_cast<RecordTypes>(FunctionType); + break; + default: + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Unknown function record type '%d' at offset %d.", + FunctionType, BeginOffset); + } + + R.FuncId = Buffer >> 4; + PreReadOffset = OffsetPtr; + R.Delta = E.getU32(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Failed reading TSC delta from offset %d.", + OffsetPtr); + assert(FunctionRecord::kFunctionRecordSize == (OffsetPtr - BeginOffset)); + return Error::success(); +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/RecordPrinter.cpp b/lib/XRay/RecordPrinter.cpp new file mode 100644 index 000000000000..71ea7d0e969f --- /dev/null +++ b/lib/XRay/RecordPrinter.cpp @@ -0,0 +1,109 @@ +//===- RecordPrinter.cpp - FDR Record Printer -----------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/RecordPrinter.h" + +#include "llvm/Support/FormatVariadic.h" + +namespace llvm { +namespace xray { + +Error RecordPrinter::visit(BufferExtents &R) { + OS << formatv("<Buffer: size = {0} bytes>", R.size()) << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(WallclockRecord &R) { + OS << formatv("<Wall Time: seconds = {0}.{1,0+6}>", R.seconds(), R.nanos()) + << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(NewCPUIDRecord &R) { + OS << formatv("<CPU: id = {0}, tsc = {1}>", R.cpuid(), R.tsc()) << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(TSCWrapRecord &R) { + OS << formatv("<TSC Wrap: base = {0}>", R.tsc()) << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(CustomEventRecord &R) { + OS << formatv( + "<Custom Event: tsc = {0}, cpu = {1}, size = {2}, data = '{3}'>", + R.tsc(), R.cpu(), R.size(), R.data()) + << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(CustomEventRecordV5 &R) { + OS << formatv("<Custom Event: delta = +{0}, size = {1}, data = '{2}'>", + R.delta(), R.size(), R.data()) + << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(TypedEventRecord &R) { + OS << formatv( + "<Typed Event: delta = +{0}, type = {1}, size = {2}, data = '{3}'", + R.delta(), R.eventType(), R.size(), R.data()) + << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(CallArgRecord &R) { + OS << formatv("<Call Argument: data = {0} (hex = {0:x})>", R.arg()) << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(PIDRecord &R) { + OS << formatv("<PID: {0}>", R.pid()) << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(NewBufferRecord &R) { + OS << formatv("<Thread ID: {0}>", R.tid()) << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(EndBufferRecord &R) { + OS << "<End of Buffer>" << Delim; + return Error::success(); +} + +Error RecordPrinter::visit(FunctionRecord &R) { + // FIXME: Support symbolization here? + switch (R.recordType()) { + case RecordTypes::ENTER: + OS << formatv("<Function Enter: #{0} delta = +{1}>", R.functionId(), + R.delta()); + break; + case RecordTypes::ENTER_ARG: + OS << formatv("<Function Enter With Arg: #{0} delta = +{1}>", + R.functionId(), R.delta()); + break; + case RecordTypes::EXIT: + OS << formatv("<Function Exit: #{0} delta = +{1}>", R.functionId(), + R.delta()); + break; + case RecordTypes::TAIL_EXIT: + OS << formatv("<Function Tail Exit: #{0} delta = +{1}>", R.functionId(), + R.delta()); + break; + case RecordTypes::CUSTOM_EVENT: + case RecordTypes::TYPED_EVENT: + // TODO: Flag as a bug? + break; + } + OS << Delim; + return Error::success(); +} + +} // namespace xray +} // namespace llvm diff --git a/lib/XRay/Trace.cpp b/lib/XRay/Trace.cpp index a8764b25483c..4f28f3f754c1 100644 --- a/lib/XRay/Trace.cpp +++ b/lib/XRay/Trace.cpp @@ -15,7 +15,16 @@ #include "llvm/Support/DataExtractor.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" +#include "llvm/XRay/BlockIndexer.h" +#include "llvm/XRay/BlockVerifier.h" +#include "llvm/XRay/FDRRecordConsumer.h" +#include "llvm/XRay/FDRRecordProducer.h" +#include "llvm/XRay/FDRRecords.h" +#include "llvm/XRay/FDRTraceExpander.h" +#include "llvm/XRay/FileHeaderReader.h" #include "llvm/XRay/YAMLXRayRecord.h" +#include <memory> +#include <vector> using namespace llvm; using namespace llvm::xray; @@ -25,38 +34,8 @@ namespace { using XRayRecordStorage = std::aligned_storage<sizeof(XRayRecord), alignof(XRayRecord)>::type; -// Populates the FileHeader reference by reading the first 32 bytes of the file. -Error readBinaryFormatHeader(StringRef Data, XRayFileHeader &FileHeader) { - // FIXME: Maybe deduce whether the data is little or big-endian using some - // magic bytes in the beginning of the file? - - // First 32 bytes of the file will always be the header. We assume a certain - // format here: - // - // (2) uint16 : version - // (2) uint16 : type - // (4) uint32 : bitfield - // (8) uint64 : cycle frequency - // (16) - : padding - - DataExtractor HeaderExtractor(Data, true, 8); - uint32_t OffsetPtr = 0; - FileHeader.Version = HeaderExtractor.getU16(&OffsetPtr); - FileHeader.Type = HeaderExtractor.getU16(&OffsetPtr); - uint32_t Bitfield = HeaderExtractor.getU32(&OffsetPtr); - FileHeader.ConstantTSC = Bitfield & 1uL; - FileHeader.NonstopTSC = Bitfield & 1uL << 1; - FileHeader.CycleFrequency = HeaderExtractor.getU64(&OffsetPtr); - std::memcpy(&FileHeader.FreeFormData, Data.bytes_begin() + OffsetPtr, 16); - if (FileHeader.Version != 1 && FileHeader.Version != 2 && - FileHeader.Version != 3) - return make_error<StringError>( - Twine("Unsupported XRay file version: ") + Twine(FileHeader.Version), - std::make_error_code(std::errc::invalid_argument)); - return Error::success(); -} - -Error loadNaiveFormatLog(StringRef Data, XRayFileHeader &FileHeader, +Error loadNaiveFormatLog(StringRef Data, bool IsLittleEndian, + XRayFileHeader &FileHeader, std::vector<XRayRecord> &Records) { if (Data.size() < 32) return make_error<StringError>( @@ -68,8 +47,12 @@ Error loadNaiveFormatLog(StringRef Data, XRayFileHeader &FileHeader, "Invalid-sized XRay data.", std::make_error_code(std::errc::invalid_argument)); - if (auto E = readBinaryFormatHeader(Data, FileHeader)) - return E; + DataExtractor Reader(Data, IsLittleEndian, 8); + uint32_t OffsetPtr = 0; + auto FileHeaderOrError = readBinaryFormatHeader(Reader, OffsetPtr); + if (!FileHeaderOrError) + return FileHeaderOrError.takeError(); + FileHeader = std::move(FileHeaderOrError.get()); // Each record after the header will be 32 bytes, in the following format: // @@ -81,16 +64,38 @@ Error loadNaiveFormatLog(StringRef Data, XRayFileHeader &FileHeader, // (4) uint32 : thread id // (4) uint32 : process id // (8) - : padding - for (auto S = Data.drop_front(32); !S.empty(); S = S.drop_front(32)) { - DataExtractor RecordExtractor(S, true, 8); - uint32_t OffsetPtr = 0; - switch (auto RecordType = RecordExtractor.getU16(&OffsetPtr)) { + while (Reader.isValidOffset(OffsetPtr)) { + if (!Reader.isValidOffsetForDataOfSize(OffsetPtr, 32)) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Not enough bytes to read a full record at offset %d.", OffsetPtr); + auto PreReadOffset = OffsetPtr; + auto RecordType = Reader.getU16(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading record type at offset %d.", OffsetPtr); + + switch (RecordType) { case 0: { // Normal records. Records.emplace_back(); auto &Record = Records.back(); Record.RecordType = RecordType; - Record.CPU = RecordExtractor.getU8(&OffsetPtr); - auto Type = RecordExtractor.getU8(&OffsetPtr); + + PreReadOffset = OffsetPtr; + Record.CPU = Reader.getU8(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading CPU field at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + auto Type = Reader.getU8(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading record type field at offset %d.", OffsetPtr); + switch (Type) { case 0: Record.Type = RecordTypes::ENTER; @@ -105,393 +110,96 @@ Error loadNaiveFormatLog(StringRef Data, XRayFileHeader &FileHeader, Record.Type = RecordTypes::ENTER_ARG; break; default: - return make_error<StringError>( - Twine("Unknown record type '") + Twine(int{Type}) + "'", - std::make_error_code(std::errc::executable_format_error)); + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Unknown record type '%d' at offset %d.", Type, OffsetPtr); } - Record.FuncId = RecordExtractor.getSigned(&OffsetPtr, sizeof(int32_t)); - Record.TSC = RecordExtractor.getU64(&OffsetPtr); - Record.TId = RecordExtractor.getU32(&OffsetPtr); - Record.PId = RecordExtractor.getU32(&OffsetPtr); + + PreReadOffset = OffsetPtr; + Record.FuncId = Reader.getSigned(&OffsetPtr, sizeof(int32_t)); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading function id field at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + Record.TSC = Reader.getU64(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading TSC field at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + Record.TId = Reader.getU32(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading thread id field at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + Record.PId = Reader.getU32(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading process id at offset %d.", OffsetPtr); + break; } case 1: { // Arg payload record. auto &Record = Records.back(); - // Advance two bytes to avoid padding. + + // We skip the next two bytes of the record, because we don't need the + // type and the CPU record for arg payloads. OffsetPtr += 2; - int32_t FuncId = RecordExtractor.getSigned(&OffsetPtr, sizeof(int32_t)); - auto TId = RecordExtractor.getU32(&OffsetPtr); - auto PId = RecordExtractor.getU32(&OffsetPtr); + PreReadOffset = OffsetPtr; + int32_t FuncId = Reader.getSigned(&OffsetPtr, sizeof(int32_t)); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading function id field at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + auto TId = Reader.getU32(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading thread id field at offset %d.", OffsetPtr); + + PreReadOffset = OffsetPtr; + auto PId = Reader.getU32(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading process id field at offset %d.", OffsetPtr); // Make a check for versions above 3 for the Pid field if (Record.FuncId != FuncId || Record.TId != TId || (FileHeader.Version >= 3 ? Record.PId != PId : false)) - return make_error<StringError>( - Twine("Corrupted log, found arg payload following non-matching " - "function + thread record. Record for function ") + - Twine(Record.FuncId) + " != " + Twine(FuncId) + "; offset: " + - Twine(S.data() - Data.data()), - std::make_error_code(std::errc::executable_format_error)); - - auto Arg = RecordExtractor.getU64(&OffsetPtr); + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Corrupted log, found arg payload following non-matching " + "function+thread record. Record for function %d != %d at offset " + "%d", + Record.FuncId, FuncId, OffsetPtr); + + PreReadOffset = OffsetPtr; + auto Arg = Reader.getU64(&OffsetPtr); + if (OffsetPtr == PreReadOffset) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Failed reading argument payload at offset %d.", OffsetPtr); + Record.CallArgs.push_back(Arg); break; } default: - return make_error<StringError>( - Twine("Unknown record type == ") + Twine(RecordType), - std::make_error_code(std::errc::executable_format_error)); + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "Unknown record type '%d' at offset %d.", RecordType, OffsetPtr); } - } - return Error::success(); -} - -/// When reading from a Flight Data Recorder mode log, metadata records are -/// sparse compared to packed function records, so we must maintain state as we -/// read through the sequence of entries. This allows the reader to denormalize -/// the CPUId and Thread Id onto each Function Record and transform delta -/// encoded TSC values into absolute encodings on each record. -struct FDRState { - uint16_t CPUId; - uint16_t ThreadId; - int32_t ProcessId; - uint64_t BaseTSC; - - /// Encode some of the state transitions for the FDR log reader as explicit - /// checks. These are expectations for the next Record in the stream. - enum class Token { - NEW_BUFFER_RECORD_OR_EOF, - WALLCLOCK_RECORD, - NEW_CPU_ID_RECORD, - FUNCTION_SEQUENCE, - SCAN_TO_END_OF_THREAD_BUF, - CUSTOM_EVENT_DATA, - CALL_ARGUMENT, - BUFFER_EXTENTS, - PID_RECORD, - }; - Token Expects; - - // Each threads buffer may have trailing garbage to scan over, so we track our - // progress. - uint64_t CurrentBufferSize; - uint64_t CurrentBufferConsumed; -}; - -const char *fdrStateToTwine(const FDRState::Token &state) { - switch (state) { - case FDRState::Token::NEW_BUFFER_RECORD_OR_EOF: - return "NEW_BUFFER_RECORD_OR_EOF"; - case FDRState::Token::WALLCLOCK_RECORD: - return "WALLCLOCK_RECORD"; - case FDRState::Token::NEW_CPU_ID_RECORD: - return "NEW_CPU_ID_RECORD"; - case FDRState::Token::FUNCTION_SEQUENCE: - return "FUNCTION_SEQUENCE"; - case FDRState::Token::SCAN_TO_END_OF_THREAD_BUF: - return "SCAN_TO_END_OF_THREAD_BUF"; - case FDRState::Token::CUSTOM_EVENT_DATA: - return "CUSTOM_EVENT_DATA"; - case FDRState::Token::CALL_ARGUMENT: - return "CALL_ARGUMENT"; - case FDRState::Token::BUFFER_EXTENTS: - return "BUFFER_EXTENTS"; - case FDRState::Token::PID_RECORD: - return "PID_RECORD"; - } - return "UNKNOWN"; -} - -/// State transition when a NewBufferRecord is encountered. -Error processFDRNewBufferRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor) { - - if (State.Expects != FDRState::Token::NEW_BUFFER_RECORD_OR_EOF) - return make_error<StringError>( - Twine("Malformed log. Read New Buffer record kind out of sequence; " - "expected: ") + - fdrStateToTwine(State.Expects), - std::make_error_code(std::errc::executable_format_error)); - uint32_t OffsetPtr = 1; // 1 byte into record. - State.ThreadId = RecordExtractor.getU16(&OffsetPtr); - State.Expects = FDRState::Token::WALLCLOCK_RECORD; - return Error::success(); -} - -/// State transition when an EndOfBufferRecord is encountered. -Error processFDREndOfBufferRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor) { - if (State.Expects == FDRState::Token::NEW_BUFFER_RECORD_OR_EOF) - return make_error<StringError>( - Twine("Malformed log. Received EOB message without current buffer; " - "expected: ") + - fdrStateToTwine(State.Expects), - std::make_error_code(std::errc::executable_format_error)); - State.Expects = FDRState::Token::SCAN_TO_END_OF_THREAD_BUF; - return Error::success(); -} - -/// State transition when a NewCPUIdRecord is encountered. -Error processFDRNewCPUIdRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor) { - if (State.Expects != FDRState::Token::FUNCTION_SEQUENCE && - State.Expects != FDRState::Token::NEW_CPU_ID_RECORD) - return make_error<StringError>( - Twine("Malformed log. Read NewCPUId record kind out of sequence; " - "expected: ") + - fdrStateToTwine(State.Expects), - std::make_error_code(std::errc::executable_format_error)); - uint32_t OffsetPtr = 1; // Read starting after the first byte. - State.CPUId = RecordExtractor.getU16(&OffsetPtr); - State.BaseTSC = RecordExtractor.getU64(&OffsetPtr); - State.Expects = FDRState::Token::FUNCTION_SEQUENCE; - return Error::success(); -} - -/// State transition when a TSCWrapRecord (overflow detection) is encountered. -Error processFDRTSCWrapRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor) { - if (State.Expects != FDRState::Token::FUNCTION_SEQUENCE) - return make_error<StringError>( - Twine("Malformed log. Read TSCWrap record kind out of sequence; " - "expecting: ") + - fdrStateToTwine(State.Expects), - std::make_error_code(std::errc::executable_format_error)); - uint32_t OffsetPtr = 1; // Read starting after the first byte. - State.BaseTSC = RecordExtractor.getU64(&OffsetPtr); - return Error::success(); -} - -/// State transition when a WallTimeMarkerRecord is encountered. -Error processFDRWallTimeRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor) { - if (State.Expects != FDRState::Token::WALLCLOCK_RECORD) - return make_error<StringError>( - Twine("Malformed log. Read Wallclock record kind out of sequence; " - "expecting: ") + - fdrStateToTwine(State.Expects), - std::make_error_code(std::errc::executable_format_error)); - - // TODO: Someday, reconcile the TSC ticks to wall clock time for presentation - // purposes. For now, we're ignoring these records. - State.Expects = FDRState::Token::NEW_CPU_ID_RECORD; - return Error::success(); -} - -/// State transition when a PidRecord is encountered. -Error processFDRPidRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor) { - - if (State.Expects != FDRState::Token::PID_RECORD) - return make_error<StringError>( - Twine("Malformed log. Read Pid record kind out of sequence; " - "expected: ") + - fdrStateToTwine(State.Expects), - std::make_error_code(std::errc::executable_format_error)); - - uint32_t OffsetPtr = 1; // Read starting after the first byte. - State.ProcessId = RecordExtractor.getU32(&OffsetPtr); - State.Expects = FDRState::Token::NEW_CPU_ID_RECORD; - return Error::success(); -} - -/// State transition when a CustomEventMarker is encountered. -Error processCustomEventMarker(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor, - size_t &RecordSize) { - // We can encounter a CustomEventMarker anywhere in the log, so we can handle - // it regardless of the expectation. However, we do set the expectation to - // read a set number of fixed bytes, as described in the metadata. - uint32_t OffsetPtr = 1; // Read after the first byte. - uint32_t DataSize = RecordExtractor.getU32(&OffsetPtr); - uint64_t TSC = RecordExtractor.getU64(&OffsetPtr); - - // FIXME: Actually represent the record through the API. For now we only - // skip through the data. - (void)TSC; - RecordSize = 16 + DataSize; - return Error::success(); -} - -/// State transition when an BufferExtents record is encountered. -Error processBufferExtents(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor) { - if (State.Expects != FDRState::Token::BUFFER_EXTENTS) - return make_error<StringError>( - Twine("Malformed log. Buffer Extents unexpected; expected: ") + - fdrStateToTwine(State.Expects), - std::make_error_code(std::errc::executable_format_error)); - uint32_t OffsetPtr = 1; // Read after the first byte. - State.CurrentBufferSize = RecordExtractor.getU64(&OffsetPtr); - State.Expects = FDRState::Token::NEW_BUFFER_RECORD_OR_EOF; - return Error::success(); -} - -/// State transition when a CallArgumentRecord is encountered. -Error processFDRCallArgumentRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor, - std::vector<XRayRecord> &Records) { - uint32_t OffsetPtr = 1; // Read starting after the first byte. - auto &Enter = Records.back(); - - if (Enter.Type != RecordTypes::ENTER) - return make_error<StringError>( - "CallArgument needs to be right after a function entry", - std::make_error_code(std::errc::executable_format_error)); - Enter.Type = RecordTypes::ENTER_ARG; - Enter.CallArgs.emplace_back(RecordExtractor.getU64(&OffsetPtr)); - return Error::success(); -} - -/// Advances the state machine for reading the FDR record type by reading one -/// Metadata Record and updating the State appropriately based on the kind of -/// record encountered. The RecordKind is encoded in the first byte of the -/// Record, which the caller should pass in because they have already read it -/// to determine that this is a metadata record as opposed to a function record. -/// -/// Beginning with Version 2 of the FDR log, we do not depend on the size of the -/// buffer, but rather use the extents to determine how far to read in the log -/// for this particular buffer. -/// -/// In Version 3, FDR log now includes a pid metadata record after -/// WallTimeMarker -Error processFDRMetadataRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor, - size_t &RecordSize, - std::vector<XRayRecord> &Records, - uint16_t Version) { - // The remaining 7 bits are the RecordKind enum. - uint8_t RecordKind = RecordFirstByte >> 1; - switch (RecordKind) { - case 0: // NewBuffer - if (auto E = - processFDRNewBufferRecord(State, RecordFirstByte, RecordExtractor)) - return E; - break; - case 1: // EndOfBuffer - if (Version >= 2) - return make_error<StringError>( - "Since Version 2 of FDR logging, we no longer support EOB records.", - std::make_error_code(std::errc::executable_format_error)); - if (auto E = processFDREndOfBufferRecord(State, RecordFirstByte, - RecordExtractor)) - return E; - break; - case 2: // NewCPUId - if (auto E = - processFDRNewCPUIdRecord(State, RecordFirstByte, RecordExtractor)) - return E; - break; - case 3: // TSCWrap - if (auto E = - processFDRTSCWrapRecord(State, RecordFirstByte, RecordExtractor)) - return E; - break; - case 4: // WallTimeMarker - if (auto E = - processFDRWallTimeRecord(State, RecordFirstByte, RecordExtractor)) - return E; - // In Version 3 and and above, a PidRecord is expected after WallTimeRecord - if (Version >= 3) - State.Expects = FDRState::Token::PID_RECORD; - break; - case 5: // CustomEventMarker - if (auto E = processCustomEventMarker(State, RecordFirstByte, - RecordExtractor, RecordSize)) - return E; - break; - case 6: // CallArgument - if (auto E = processFDRCallArgumentRecord(State, RecordFirstByte, - RecordExtractor, Records)) - return E; - break; - case 7: // BufferExtents - if (auto E = processBufferExtents(State, RecordFirstByte, RecordExtractor)) - return E; - break; - case 9: // Pid - if (auto E = processFDRPidRecord(State, RecordFirstByte, RecordExtractor)) - return E; - break; - default: - // Widen the record type to uint16_t to prevent conversion to char. - return make_error<StringError>( - Twine("Illegal metadata record type: ") - .concat(Twine(static_cast<unsigned>(RecordKind))), - std::make_error_code(std::errc::executable_format_error)); - } - return Error::success(); -} - -/// Reads a function record from an FDR format log, appending a new XRayRecord -/// to the vector being populated and updating the State with a new value -/// reference value to interpret TSC deltas. -/// -/// The XRayRecord constructed includes information from the function record -/// processed here as well as Thread ID and CPU ID formerly extracted into -/// State. -Error processFDRFunctionRecord(FDRState &State, uint8_t RecordFirstByte, - DataExtractor &RecordExtractor, - std::vector<XRayRecord> &Records) { - switch (State.Expects) { - case FDRState::Token::NEW_BUFFER_RECORD_OR_EOF: - return make_error<StringError>( - "Malformed log. Received Function Record before new buffer setup.", - std::make_error_code(std::errc::executable_format_error)); - case FDRState::Token::WALLCLOCK_RECORD: - return make_error<StringError>( - "Malformed log. Received Function Record when expecting wallclock.", - std::make_error_code(std::errc::executable_format_error)); - case FDRState::Token::PID_RECORD: - return make_error<StringError>( - "Malformed log. Received Function Record when expecting pid.", - std::make_error_code(std::errc::executable_format_error)); - case FDRState::Token::NEW_CPU_ID_RECORD: - return make_error<StringError>( - "Malformed log. Received Function Record before first CPU record.", - std::make_error_code(std::errc::executable_format_error)); - default: - Records.emplace_back(); - auto &Record = Records.back(); - Record.RecordType = 0; // Record is type NORMAL. - // Strip off record type bit and use the next three bits. - uint8_t RecordType = (RecordFirstByte >> 1) & 0x07; - switch (RecordType) { - case static_cast<uint8_t>(RecordTypes::ENTER): - Record.Type = RecordTypes::ENTER; - break; - case static_cast<uint8_t>(RecordTypes::EXIT): - Record.Type = RecordTypes::EXIT; - break; - case static_cast<uint8_t>(RecordTypes::TAIL_EXIT): - Record.Type = RecordTypes::TAIL_EXIT; - break; - default: - // Cast to an unsigned integer to not interpret the record type as a char. - return make_error<StringError>( - Twine("Illegal function record type: ") - .concat(Twine(static_cast<unsigned>(RecordType))), - std::make_error_code(std::errc::executable_format_error)); - } - Record.CPU = State.CPUId; - Record.TId = State.ThreadId; - Record.PId = State.ProcessId; - // Back up to read first 32 bits, including the 4 we pulled RecordType - // and RecordKind out of. The remaining 28 are FunctionId. - uint32_t OffsetPtr = 0; - // Despite function Id being a signed int on XRayRecord, - // when it is written to an FDR format, the top bits are truncated, - // so it is effectively an unsigned value. When we shift off the - // top four bits, we want the shift to be logical, so we read as - // uint32_t. - uint32_t FuncIdBitField = RecordExtractor.getU32(&OffsetPtr); - Record.FuncId = FuncIdBitField >> 4; - // FunctionRecords have a 32 bit delta from the previous absolute TSC - // or TSC delta. If this would overflow, we should read a TSCWrap record - // with an absolute TSC reading. - uint64_t NewTSC = State.BaseTSC + RecordExtractor.getU32(&OffsetPtr); - State.BaseTSC = NewTSC; - Record.TSC = NewTSC; + // Advance the offset pointer enough bytes to align to 32-byte records for + // basic mode logs. + OffsetPtr += 8; } return Error::success(); } @@ -539,112 +247,97 @@ Error processFDRFunctionRecord(FDRState &State, uint8_t RecordFirstByte, /// ThreadBuffer: BufferExtents NewBuffer WallClockTime Pid NewCPUId /// FunctionSequence /// EOB: *deprecated* -Error loadFDRLog(StringRef Data, XRayFileHeader &FileHeader, - std::vector<XRayRecord> &Records) { +/// +/// In Version 4, we make the following changes: +/// +/// CustomEventRecord now includes the CPU data. +/// +/// In Version 5, we make the following changes: +/// +/// CustomEventRecord and TypedEventRecord now use TSC delta encoding similar to +/// what FunctionRecord instances use, and we no longer need to include the CPU +/// id in the CustomEventRecord. +/// +Error loadFDRLog(StringRef Data, bool IsLittleEndian, + XRayFileHeader &FileHeader, std::vector<XRayRecord> &Records) { + if (Data.size() < 32) - return make_error<StringError>( - "Not enough bytes for an XRay log.", - std::make_error_code(std::errc::invalid_argument)); + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Not enough bytes for an XRay FDR log."); + DataExtractor DE(Data, IsLittleEndian, 8); - // For an FDR log, there are records sized 16 and 8 bytes. - // There actually may be no records if no non-trivial functions are - // instrumented. - if (Data.size() % 8 != 0) - return make_error<StringError>( - "Invalid-sized XRay data.", - std::make_error_code(std::errc::invalid_argument)); + uint32_t OffsetPtr = 0; + auto FileHeaderOrError = readBinaryFormatHeader(DE, OffsetPtr); + if (!FileHeaderOrError) + return FileHeaderOrError.takeError(); + FileHeader = std::move(FileHeaderOrError.get()); - if (auto E = readBinaryFormatHeader(Data, FileHeader)) - return E; + // First we load the records into memory. + std::vector<std::unique_ptr<Record>> FDRRecords; - uint64_t BufferSize = 0; { - StringRef ExtraDataRef(FileHeader.FreeFormData, 16); - DataExtractor ExtraDataExtractor(ExtraDataRef, true, 8); - uint32_t ExtraDataOffset = 0; - BufferSize = ExtraDataExtractor.getU64(&ExtraDataOffset); + FileBasedRecordProducer P(FileHeader, DE, OffsetPtr); + LogBuilderConsumer C(FDRRecords); + while (DE.isValidOffsetForDataOfSize(OffsetPtr, 1)) { + auto R = P.produce(); + if (!R) + return R.takeError(); + if (auto E = C.consume(std::move(R.get()))) + return E; + } } - FDRState::Token InitialExpectation; - switch (FileHeader.Version) { - case 1: - InitialExpectation = FDRState::Token::NEW_BUFFER_RECORD_OR_EOF; - break; - case 2: - case 3: - InitialExpectation = FDRState::Token::BUFFER_EXTENTS; - break; - default: - return make_error<StringError>( - Twine("Unsupported version '") + Twine(FileHeader.Version) + "'", - std::make_error_code(std::errc::executable_format_error)); + // Next we index the records into blocks. + BlockIndexer::Index Index; + { + BlockIndexer Indexer(Index); + for (auto &R : FDRRecords) + if (auto E = R->apply(Indexer)) + return E; + if (auto E = Indexer.flush()) + return E; } - FDRState State{0, 0, 0, 0, InitialExpectation, BufferSize, 0}; - - // RecordSize will tell the loop how far to seek ahead based on the record - // type that we have just read. - size_t RecordSize = 0; - for (auto S = Data.drop_front(32); !S.empty(); S = S.drop_front(RecordSize)) { - DataExtractor RecordExtractor(S, true, 8); - uint32_t OffsetPtr = 0; - if (State.Expects == FDRState::Token::SCAN_TO_END_OF_THREAD_BUF) { - RecordSize = State.CurrentBufferSize - State.CurrentBufferConsumed; - if (S.size() < RecordSize) { - return make_error<StringError>( - Twine("Incomplete thread buffer. Expected at least ") + - Twine(RecordSize) + " bytes but found " + Twine(S.size()), - make_error_code(std::errc::invalid_argument)); + + // Then we verify the consistency of the blocks. + { + for (auto &PTB : Index) { + auto &Blocks = PTB.second; + for (auto &B : Blocks) { + BlockVerifier Verifier; + for (auto *R : B.Records) + if (auto E = R->apply(Verifier)) + return E; + if (auto E = Verifier.verify()) + return E; } - State.CurrentBufferConsumed = 0; - State.Expects = FDRState::Token::NEW_BUFFER_RECORD_OR_EOF; - continue; - } - uint8_t BitField = RecordExtractor.getU8(&OffsetPtr); - bool isMetadataRecord = BitField & 0x01uL; - bool isBufferExtents = - (BitField >> 1) == 7; // BufferExtents record kind == 7 - if (isMetadataRecord) { - RecordSize = 16; - if (auto E = - processFDRMetadataRecord(State, BitField, RecordExtractor, - RecordSize, Records, FileHeader.Version)) - return E; - } else { // Process Function Record - RecordSize = 8; - if (auto E = processFDRFunctionRecord(State, BitField, RecordExtractor, - Records)) - return E; } + } - // The BufferExtents record is technically not part of the buffer, so we - // don't count the size of that record against the buffer's actual size. - if (!isBufferExtents) - State.CurrentBufferConsumed += RecordSize; - assert(State.CurrentBufferConsumed <= State.CurrentBufferSize); - if ((FileHeader.Version == 2 || FileHeader.Version == 3) && - State.CurrentBufferSize == State.CurrentBufferConsumed) { - // In Version 2 of the log, we don't need to scan to the end of the thread - // buffer if we've already consumed all the bytes we need to. - State.Expects = FDRState::Token::BUFFER_EXTENTS; - State.CurrentBufferSize = BufferSize; - State.CurrentBufferConsumed = 0; + // This is now the meat of the algorithm. Here we sort the blocks according to + // the Walltime record in each of the blocks for the same thread. This allows + // us to more consistently recreate the execution trace in temporal order. + // After the sort, we then reconstitute `Trace` records using a stateful + // visitor associated with a single process+thread pair. + { + for (auto &PTB : Index) { + auto &Blocks = PTB.second; + llvm::sort(Blocks, [](const BlockIndexer::Block &L, + const BlockIndexer::Block &R) { + return (L.WallclockTime->seconds() < R.WallclockTime->seconds() && + L.WallclockTime->nanos() < R.WallclockTime->nanos()); + }); + auto Adder = [&](const XRayRecord &R) { Records.push_back(R); }; + TraceExpander Expander(Adder, FileHeader.Version); + for (auto &B : Blocks) { + for (auto *R : B.Records) + if (auto E = R->apply(Expander)) + return E; + } + if (auto E = Expander.flush()) + return E; } } - // Having iterated over everything we've been given, we've either consumed - // everything and ended up in the end state, or were told to skip the rest. - bool Finished = State.Expects == FDRState::Token::SCAN_TO_END_OF_THREAD_BUF && - State.CurrentBufferSize == State.CurrentBufferConsumed; - if ((State.Expects != FDRState::Token::NEW_BUFFER_RECORD_OR_EOF && - State.Expects != FDRState::Token::BUFFER_EXTENTS) && - !Finished) - return make_error<StringError>( - Twine("Encountered EOF with unexpected state expectation ") + - fdrStateToTwine(State.Expects) + - ". Remaining expected bytes in thread buffer total " + - Twine(State.CurrentBufferSize - State.CurrentBufferConsumed), - std::make_error_code(std::errc::executable_format_error)); - return Error::success(); } @@ -670,8 +363,9 @@ Error loadYAMLLog(StringRef Data, XRayFileHeader &FileHeader, Records.clear(); std::transform(Trace.Records.begin(), Trace.Records.end(), std::back_inserter(Records), [&](const YAMLXRayRecord &R) { - return XRayRecord{R.RecordType, R.CPU, R.Type, R.FuncId, - R.TSC, R.TId, R.PId, R.CallArgs}; + return XRayRecord{R.RecordType, R.CPU, R.Type, + R.FuncId, R.TSC, R.TId, + R.PId, R.CallArgs, R.Data}; }); return Error::success(); } @@ -705,6 +399,17 @@ Expected<Trace> llvm::xray::loadTraceFile(StringRef Filename, bool Sort) { } auto Data = StringRef(MappedFile.data(), MappedFile.size()); + // TODO: Lift the endianness and implementation selection here. + DataExtractor LittleEndianDE(Data, true, 8); + auto TraceOrError = loadTrace(LittleEndianDE, Sort); + if (!TraceOrError) { + DataExtractor BigEndianDE(Data, false, 8); + TraceOrError = loadTrace(BigEndianDE, Sort); + } + return TraceOrError; +} + +Expected<Trace> llvm::xray::loadTrace(const DataExtractor &DE, bool Sort) { // Attempt to detect the file type using file magic. We have a slight bias // towards the binary format, and we do this by making sure that the first 4 // bytes of the binary file is some combination of the following byte @@ -719,8 +424,7 @@ Expected<Trace> llvm::xray::loadTraceFile(StringRef Filename, bool Sort) { // // Only if we can't load either the binary or the YAML format will we yield an // error. - StringRef Magic(MappedFile.data(), 4); - DataExtractor HeaderExtractor(Magic, true, 8); + DataExtractor HeaderExtractor(DE.getData(), DE.isLittleEndian(), 8); uint32_t OffsetPtr = 0; uint16_t Version = HeaderExtractor.getU16(&OffsetPtr); uint16_t Type = HeaderExtractor.getU16(&OffsetPtr); @@ -731,7 +435,8 @@ Expected<Trace> llvm::xray::loadTraceFile(StringRef Filename, bool Sort) { switch (Type) { case NAIVE_FORMAT: if (Version == 1 || Version == 2 || Version == 3) { - if (auto E = loadNaiveFormatLog(Data, T.FileHeader, T.Records)) + if (auto E = loadNaiveFormatLog(DE.getData(), DE.isLittleEndian(), + T.FileHeader, T.Records)) return std::move(E); } else { return make_error<StringError>( @@ -741,8 +446,9 @@ Expected<Trace> llvm::xray::loadTraceFile(StringRef Filename, bool Sort) { } break; case FLIGHT_DATA_RECORDER_FORMAT: - if (Version == 1 || Version == 2 || Version == 3) { - if (auto E = loadFDRLog(Data, T.FileHeader, T.Records)) + if (Version >= 1 && Version <= 5) { + if (auto E = loadFDRLog(DE.getData(), DE.isLittleEndian(), T.FileHeader, + T.Records)) return std::move(E); } else { return make_error<StringError>( @@ -751,15 +457,15 @@ Expected<Trace> llvm::xray::loadTraceFile(StringRef Filename, bool Sort) { } break; default: - if (auto E = loadYAMLLog(Data, T.FileHeader, T.Records)) + if (auto E = loadYAMLLog(DE.getData(), T.FileHeader, T.Records)) return std::move(E); } if (Sort) std::stable_sort(T.Records.begin(), T.Records.end(), - [&](const XRayRecord &L, const XRayRecord &R) { - return L.TSC < R.TSC; - }); + [&](const XRayRecord &L, const XRayRecord &R) { + return L.TSC < R.TSC; + }); return std::move(T); } |