From 1d5ae1026e831016fc29fd927877c86af904481f Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Wed, 23 Oct 2019 17:51:42 +0000 Subject: Vendor import of stripped llvm trunk r375505, the last commit before the upstream Subversion repository was made read-only, and the LLVM project migrated to GitHub: https://llvm.org/svn/llvm-project/llvm/trunk@375505 --- lib/Remarks/BitstreamRemarkParser.cpp | 597 ++++++++++++++++++++++++++++++ lib/Remarks/BitstreamRemarkParser.h | 83 +++++ lib/Remarks/BitstreamRemarkSerializer.cpp | 386 +++++++++++++++++++ lib/Remarks/RemarkFormat.cpp | 4 +- lib/Remarks/RemarkParser.cpp | 72 +++- lib/Remarks/RemarkSerializer.cpp | 54 +++ lib/Remarks/RemarkStringTable.cpp | 28 +- lib/Remarks/YAMLRemarkParser.cpp | 165 ++++++++- lib/Remarks/YAMLRemarkParser.h | 38 +- lib/Remarks/YAMLRemarkSerializer.cpp | 134 +++++-- 10 files changed, 1501 insertions(+), 60 deletions(-) create mode 100644 lib/Remarks/BitstreamRemarkParser.cpp create mode 100644 lib/Remarks/BitstreamRemarkParser.h create mode 100644 lib/Remarks/BitstreamRemarkSerializer.cpp create mode 100644 lib/Remarks/RemarkSerializer.cpp (limited to 'lib/Remarks') diff --git a/lib/Remarks/BitstreamRemarkParser.cpp b/lib/Remarks/BitstreamRemarkParser.cpp new file mode 100644 index 000000000000..99a82e1ee3af --- /dev/null +++ b/lib/Remarks/BitstreamRemarkParser.cpp @@ -0,0 +1,597 @@ +//===- BitstreamRemarkParser.cpp ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides utility methods used by clients that want to use the +// parser for remark diagnostics in LLVM. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Remarks/BitstreamRemarkParser.h" +#include "BitstreamRemarkParser.h" +#include "llvm/Remarks/BitstreamRemarkContainer.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" + +using namespace llvm; +using namespace llvm::remarks; + +static Error unknownRecord(const char *BlockName, unsigned RecordID) { + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing %s: unknown record entry (%lu).", BlockName, + RecordID); +} + +static Error malformedRecord(const char *BlockName, const char *RecordName) { + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing %s: malformed record entry (%s).", BlockName, + RecordName); +} + +BitstreamMetaParserHelper::BitstreamMetaParserHelper( + BitstreamCursor &Stream, BitstreamBlockInfo &BlockInfo) + : Stream(Stream), BlockInfo(BlockInfo) {} + +/// Parse a record and fill in the fields in the parser. +static Error parseRecord(BitstreamMetaParserHelper &Parser, unsigned Code) { + BitstreamCursor &Stream = Parser.Stream; + // Note: 2 is used here because it's the max number of fields we have per + // record. + SmallVector Record; + StringRef Blob; + Expected RecordID = Stream.readRecord(Code, Record, &Blob); + if (!RecordID) + return RecordID.takeError(); + + switch (*RecordID) { + case RECORD_META_CONTAINER_INFO: { + if (Record.size() != 2) + return malformedRecord("BLOCK_META", "RECORD_META_CONTAINER_INFO"); + Parser.ContainerVersion = Record[0]; + Parser.ContainerType = Record[1]; + break; + } + case RECORD_META_REMARK_VERSION: { + if (Record.size() != 1) + return malformedRecord("BLOCK_META", "RECORD_META_REMARK_VERSION"); + Parser.RemarkVersion = Record[0]; + break; + } + case RECORD_META_STRTAB: { + if (Record.size() != 0) + return malformedRecord("BLOCK_META", "RECORD_META_STRTAB"); + Parser.StrTabBuf = Blob; + break; + } + case RECORD_META_EXTERNAL_FILE: { + if (Record.size() != 0) + return malformedRecord("BLOCK_META", "RECORD_META_EXTERNAL_FILE"); + Parser.ExternalFilePath = Blob; + break; + } + default: + return unknownRecord("BLOCK_META", *RecordID); + } + return Error::success(); +} + +BitstreamRemarkParserHelper::BitstreamRemarkParserHelper( + BitstreamCursor &Stream) + : Stream(Stream) {} + +/// Parse a record and fill in the fields in the parser. +static Error parseRecord(BitstreamRemarkParserHelper &Parser, unsigned Code) { + BitstreamCursor &Stream = Parser.Stream; + // Note: 5 is used here because it's the max number of fields we have per + // record. + SmallVector Record; + StringRef Blob; + Expected RecordID = Stream.readRecord(Code, Record, &Blob); + if (!RecordID) + return RecordID.takeError(); + + switch (*RecordID) { + case RECORD_REMARK_HEADER: { + if (Record.size() != 4) + return malformedRecord("BLOCK_REMARK", "RECORD_REMARK_HEADER"); + Parser.Type = Record[0]; + Parser.RemarkNameIdx = Record[1]; + Parser.PassNameIdx = Record[2]; + Parser.FunctionNameIdx = Record[3]; + break; + } + case RECORD_REMARK_DEBUG_LOC: { + if (Record.size() != 3) + return malformedRecord("BLOCK_REMARK", "RECORD_REMARK_DEBUG_LOC"); + Parser.SourceFileNameIdx = Record[0]; + Parser.SourceLine = Record[1]; + Parser.SourceColumn = Record[2]; + break; + } + case RECORD_REMARK_HOTNESS: { + if (Record.size() != 1) + return malformedRecord("BLOCK_REMARK", "RECORD_REMARK_HOTNESS"); + Parser.Hotness = Record[0]; + break; + } + case RECORD_REMARK_ARG_WITH_DEBUGLOC: { + if (Record.size() != 5) + return malformedRecord("BLOCK_REMARK", "RECORD_REMARK_ARG_WITH_DEBUGLOC"); + // Create a temporary argument. Use that as a valid memory location for this + // argument entry. + Parser.TmpArgs.emplace_back(); + Parser.TmpArgs.back().KeyIdx = Record[0]; + Parser.TmpArgs.back().ValueIdx = Record[1]; + Parser.TmpArgs.back().SourceFileNameIdx = Record[2]; + Parser.TmpArgs.back().SourceLine = Record[3]; + Parser.TmpArgs.back().SourceColumn = Record[4]; + Parser.Args = + ArrayRef(Parser.TmpArgs); + break; + } + case RECORD_REMARK_ARG_WITHOUT_DEBUGLOC: { + if (Record.size() != 2) + return malformedRecord("BLOCK_REMARK", + "RECORD_REMARK_ARG_WITHOUT_DEBUGLOC"); + // Create a temporary argument. Use that as a valid memory location for this + // argument entry. + Parser.TmpArgs.emplace_back(); + Parser.TmpArgs.back().KeyIdx = Record[0]; + Parser.TmpArgs.back().ValueIdx = Record[1]; + Parser.Args = + ArrayRef(Parser.TmpArgs); + break; + } + default: + return unknownRecord("BLOCK_REMARK", *RecordID); + } + return Error::success(); +} + +template +static Error parseBlock(T &ParserHelper, unsigned BlockID, + const char *BlockName) { + BitstreamCursor &Stream = ParserHelper.Stream; + Expected Next = Stream.advance(); + if (!Next) + return Next.takeError(); + if (Next->Kind != BitstreamEntry::SubBlock || Next->ID != BlockID) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing %s: expecting [ENTER_SUBBLOCK, %s, ...].", + BlockName, BlockName); + if (Stream.EnterSubBlock(BlockID)) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while entering %s.", BlockName); + + // Stop when there is nothing to read anymore or when we encounter an + // END_BLOCK. + while (!Stream.AtEndOfStream()) { + Expected Next = Stream.advance(); + if (!Next) + return Next.takeError(); + switch (Next->Kind) { + case BitstreamEntry::EndBlock: + return Error::success(); + case BitstreamEntry::Error: + case BitstreamEntry::SubBlock: + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing %s: expecting records.", BlockName); + case BitstreamEntry::Record: + if (Error E = parseRecord(ParserHelper, Next->ID)) + return E; + continue; + } + } + // If we're here, it means we didn't get an END_BLOCK yet, but we're at the + // end of the stream. In this case, error. + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing %s: unterminated block.", BlockName); +} + +Error BitstreamMetaParserHelper::parse() { + return parseBlock(*this, META_BLOCK_ID, "META_BLOCK"); +} + +Error BitstreamRemarkParserHelper::parse() { + return parseBlock(*this, REMARK_BLOCK_ID, "REMARK_BLOCK"); +} + +BitstreamParserHelper::BitstreamParserHelper(StringRef Buffer) + : Stream(Buffer) {} + +Expected> BitstreamParserHelper::parseMagic() { + std::array Result; + for (unsigned i = 0; i < 4; ++i) + if (Expected R = Stream.Read(8)) + Result[i] = *R; + else + return R.takeError(); + return Result; +} + +Error BitstreamParserHelper::parseBlockInfoBlock() { + Expected Next = Stream.advance(); + if (!Next) + return Next.takeError(); + if (Next->Kind != BitstreamEntry::SubBlock || + Next->ID != llvm::bitc::BLOCKINFO_BLOCK_ID) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCKINFO_BLOCK: expecting [ENTER_SUBBLOCK, " + "BLOCKINFO_BLOCK, ...]."); + + Expected> MaybeBlockInfo = + Stream.ReadBlockInfoBlock(); + if (!MaybeBlockInfo) + return MaybeBlockInfo.takeError(); + + if (!*MaybeBlockInfo) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCKINFO_BLOCK."); + + BlockInfo = **MaybeBlockInfo; + + Stream.setBlockInfo(&BlockInfo); + return Error::success(); +} + +static Expected isBlock(BitstreamCursor &Stream, unsigned BlockID) { + bool Result = false; + uint64_t PreviousBitNo = Stream.GetCurrentBitNo(); + Expected Next = Stream.advance(); + if (!Next) + return Next.takeError(); + switch (Next->Kind) { + case BitstreamEntry::SubBlock: + // Check for the block id. + Result = Next->ID == BlockID; + break; + case BitstreamEntry::Error: + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Unexpected error while parsing bitstream."); + default: + Result = false; + break; + } + if (Error E = Stream.JumpToBit(PreviousBitNo)) + return std::move(E); + return Result; +} + +Expected BitstreamParserHelper::isMetaBlock() { + return isBlock(Stream, META_BLOCK_ID); +} + +Expected BitstreamParserHelper::isRemarkBlock() { + return isBlock(Stream, META_BLOCK_ID); +} + +static Error validateMagicNumber(StringRef Magic) { + if (Magic != remarks::ContainerMagic) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Unknown magic number: expecting %s, got %.4s.", + remarks::ContainerMagic.data(), Magic.data()); + return Error::success(); +} + +static Error advanceToMetaBlock(BitstreamParserHelper &Helper) { + Expected> Magic = Helper.parseMagic(); + if (!Magic) + return Magic.takeError(); + if (Error E = validateMagicNumber(StringRef(Magic->data(), Magic->size()))) + return E; + if (Error E = Helper.parseBlockInfoBlock()) + return E; + Expected isMetaBlock = Helper.isMetaBlock(); + if (!isMetaBlock) + return isMetaBlock.takeError(); + if (!*isMetaBlock) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Expecting META_BLOCK after the BLOCKINFO_BLOCK."); + return Error::success(); +} + +Expected> +remarks::createBitstreamParserFromMeta( + StringRef Buf, Optional StrTab, + Optional ExternalFilePrependPath) { + BitstreamParserHelper Helper(Buf); + Expected> Magic = Helper.parseMagic(); + if (!Magic) + return Magic.takeError(); + + if (Error E = validateMagicNumber(StringRef(Magic->data(), Magic->size()))) + return std::move(E); + + auto Parser = + StrTab ? std::make_unique(Buf, std::move(*StrTab)) + : std::make_unique(Buf); + + if (ExternalFilePrependPath) + Parser->ExternalFilePrependPath = *ExternalFilePrependPath; + + return std::move(Parser); +} + +Expected> BitstreamRemarkParser::next() { + if (ParserHelper.atEndOfStream()) + return make_error(); + + if (!ReadyToParseRemarks) { + if (Error E = parseMeta()) + return std::move(E); + ReadyToParseRemarks = true; + } + + return parseRemark(); +} + +Error BitstreamRemarkParser::parseMeta() { + // Advance and to the meta block. + if (Error E = advanceToMetaBlock(ParserHelper)) + return E; + + BitstreamMetaParserHelper MetaHelper(ParserHelper.Stream, + ParserHelper.BlockInfo); + if (Error E = MetaHelper.parse()) + return E; + + if (Error E = processCommonMeta(MetaHelper)) + return E; + + switch (ContainerType) { + case BitstreamRemarkContainerType::Standalone: + return processStandaloneMeta(MetaHelper); + case BitstreamRemarkContainerType::SeparateRemarksFile: + return processSeparateRemarksFileMeta(MetaHelper); + case BitstreamRemarkContainerType::SeparateRemarksMeta: + return processSeparateRemarksMetaMeta(MetaHelper); + } + llvm_unreachable("Unknown BitstreamRemarkContainerType enum"); +} + +Error BitstreamRemarkParser::processCommonMeta( + BitstreamMetaParserHelper &MetaHelper) { + if (Optional Version = MetaHelper.ContainerVersion) + ContainerVersion = *Version; + else + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_META: missing container version."); + + if (Optional Type = MetaHelper.ContainerType) { + // Always >= BitstreamRemarkContainerType::First since it's unsigned. + if (*Type > static_cast(BitstreamRemarkContainerType::Last)) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_META: invalid container type."); + + ContainerType = static_cast(*Type); + } else + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_META: missing container type."); + + return Error::success(); +} + +static Error processStrTab(BitstreamRemarkParser &P, + Optional StrTabBuf) { + if (!StrTabBuf) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_META: missing string table."); + // Parse and assign the string table. + P.StrTab.emplace(*StrTabBuf); + return Error::success(); +} + +static Error processRemarkVersion(BitstreamRemarkParser &P, + Optional RemarkVersion) { + if (!RemarkVersion) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_META: missing remark version."); + P.RemarkVersion = *RemarkVersion; + return Error::success(); +} + +Error BitstreamRemarkParser::processExternalFilePath( + Optional ExternalFilePath) { + if (!ExternalFilePath) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_META: missing external file path."); + + SmallString<80> FullPath(ExternalFilePrependPath); + sys::path::append(FullPath, *ExternalFilePath); + + // External file: open the external file, parse it, check if its metadata + // matches the one from the separate metadata, then replace the current parser + // with the one parsing the remarks. + ErrorOr> BufferOrErr = + MemoryBuffer::getFile(FullPath); + if (std::error_code EC = BufferOrErr.getError()) + return createFileError(FullPath, EC); + TmpRemarkBuffer = std::move(*BufferOrErr); + + // Create a separate parser used for parsing the separate file. + ParserHelper = BitstreamParserHelper(TmpRemarkBuffer->getBuffer()); + // Advance and check until we can parse the meta block. + if (Error E = advanceToMetaBlock(ParserHelper)) + return E; + // Parse the meta from the separate file. + // Note: here we overwrite the BlockInfo with the one from the file. This will + // be used to parse the rest of the file. + BitstreamMetaParserHelper SeparateMetaHelper(ParserHelper.Stream, + ParserHelper.BlockInfo); + if (Error E = SeparateMetaHelper.parse()) + return E; + + uint64_t PreviousContainerVersion = ContainerVersion; + if (Error E = processCommonMeta(SeparateMetaHelper)) + return E; + + if (ContainerType != BitstreamRemarkContainerType::SeparateRemarksFile) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing external file's BLOCK_META: wrong container " + "type."); + + if (PreviousContainerVersion != ContainerVersion) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing external file's BLOCK_META: mismatching versions: " + "original meta: %lu, external file meta: %lu.", + PreviousContainerVersion, ContainerVersion); + + // Process the meta from the separate file. + return processSeparateRemarksFileMeta(SeparateMetaHelper); +} + +Error BitstreamRemarkParser::processStandaloneMeta( + BitstreamMetaParserHelper &Helper) { + if (Error E = processStrTab(*this, Helper.StrTabBuf)) + return E; + return processRemarkVersion(*this, Helper.RemarkVersion); +} + +Error BitstreamRemarkParser::processSeparateRemarksFileMeta( + BitstreamMetaParserHelper &Helper) { + return processRemarkVersion(*this, Helper.RemarkVersion); +} + +Error BitstreamRemarkParser::processSeparateRemarksMetaMeta( + BitstreamMetaParserHelper &Helper) { + if (Error E = processStrTab(*this, Helper.StrTabBuf)) + return E; + return processExternalFilePath(Helper.ExternalFilePath); +} + +Expected> BitstreamRemarkParser::parseRemark() { + BitstreamRemarkParserHelper RemarkHelper(ParserHelper.Stream); + if (Error E = RemarkHelper.parse()) + return std::move(E); + + return processRemark(RemarkHelper); +} + +Expected> +BitstreamRemarkParser::processRemark(BitstreamRemarkParserHelper &Helper) { + std::unique_ptr Result = std::make_unique(); + Remark &R = *Result; + + if (StrTab == None) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Error while parsing BLOCK_REMARK: missing string table."); + + if (!Helper.Type) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_REMARK: missing remark type."); + + // Always >= Type::First since it's unsigned. + if (*Helper.Type > static_cast(Type::Last)) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_REMARK: unknown remark type."); + + R.RemarkType = static_cast(*Helper.Type); + + if (!Helper.RemarkNameIdx) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_REMARK: missing remark name."); + + if (Expected RemarkName = (*StrTab)[*Helper.RemarkNameIdx]) + R.RemarkName = *RemarkName; + else + return RemarkName.takeError(); + + if (!Helper.PassNameIdx) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_REMARK: missing remark pass."); + + if (Expected PassName = (*StrTab)[*Helper.PassNameIdx]) + R.PassName = *PassName; + else + return PassName.takeError(); + + if (!Helper.FunctionNameIdx) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_REMARK: missing remark function name."); + if (Expected FunctionName = (*StrTab)[*Helper.FunctionNameIdx]) + R.FunctionName = *FunctionName; + else + return FunctionName.takeError(); + + if (Helper.SourceFileNameIdx && Helper.SourceLine && Helper.SourceColumn) { + Expected SourceFileName = (*StrTab)[*Helper.SourceFileNameIdx]; + if (!SourceFileName) + return SourceFileName.takeError(); + R.Loc.emplace(); + R.Loc->SourceFilePath = *SourceFileName; + R.Loc->SourceLine = *Helper.SourceLine; + R.Loc->SourceColumn = *Helper.SourceColumn; + } + + if (Helper.Hotness) + R.Hotness = *Helper.Hotness; + + if (!Helper.Args) + return std::move(Result); + + for (const BitstreamRemarkParserHelper::Argument &Arg : *Helper.Args) { + if (!Arg.KeyIdx) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_REMARK: missing key in remark argument."); + if (!Arg.ValueIdx) + return createStringError( + std::make_error_code(std::errc::illegal_byte_sequence), + "Error while parsing BLOCK_REMARK: missing value in remark " + "argument."); + + // We have at least a key and a value, create an entry. + R.Args.emplace_back(); + + if (Expected Key = (*StrTab)[*Arg.KeyIdx]) + R.Args.back().Key = *Key; + else + return Key.takeError(); + + if (Expected Value = (*StrTab)[*Arg.ValueIdx]) + R.Args.back().Val = *Value; + else + return Value.takeError(); + + if (Arg.SourceFileNameIdx && Arg.SourceLine && Arg.SourceColumn) { + if (Expected SourceFileName = + (*StrTab)[*Arg.SourceFileNameIdx]) { + R.Args.back().Loc.emplace(); + R.Args.back().Loc->SourceFilePath = *SourceFileName; + R.Args.back().Loc->SourceLine = *Arg.SourceLine; + R.Args.back().Loc->SourceColumn = *Arg.SourceColumn; + } else + return SourceFileName.takeError(); + } + } + + return std::move(Result); +} diff --git a/lib/Remarks/BitstreamRemarkParser.h b/lib/Remarks/BitstreamRemarkParser.h new file mode 100644 index 000000000000..7c9cc2f1e7db --- /dev/null +++ b/lib/Remarks/BitstreamRemarkParser.h @@ -0,0 +1,83 @@ +//===-- BitstreamRemarkParser.h - Parser for Bitstream remarks --*- C++/-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides the impementation of the Bitstream remark parser. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_REMARKS_BITSTREAM_REMARK_PARSER_H +#define LLVM_LIB_REMARKS_BITSTREAM_REMARK_PARSER_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Remarks/BitstreamRemarkParser.h" +#include "llvm/Remarks/RemarkFormat.h" +#include "llvm/Remarks/RemarkParser.h" +#include "llvm/Support/raw_ostream.h" +#include +#include + +namespace llvm { +namespace remarks { +/// Parses and holds the state of the latest parsed remark. +struct BitstreamRemarkParser : public RemarkParser { + /// The buffer to parse. + BitstreamParserHelper ParserHelper; + /// The string table used for parsing strings. + Optional StrTab; + /// Temporary remark buffer used when the remarks are stored separately. + std::unique_ptr TmpRemarkBuffer; + /// The common metadata used to decide how to parse the buffer. + /// This is filled when parsing the metadata block. + uint64_t ContainerVersion; + uint64_t RemarkVersion; + BitstreamRemarkContainerType ContainerType; + /// Wether the parser is ready to parse remarks. + bool ReadyToParseRemarks = false; + + /// Create a parser that expects to find a string table embedded in the + /// stream. + BitstreamRemarkParser(StringRef Buf) + : RemarkParser(Format::Bitstream), ParserHelper(Buf) {} + + /// Create a parser that uses a pre-parsed string table. + BitstreamRemarkParser(StringRef Buf, ParsedStringTable StrTab) + : RemarkParser(Format::Bitstream), ParserHelper(Buf), + StrTab(std::move(StrTab)) {} + + Expected> next() override; + + static bool classof(const RemarkParser *P) { + return P->ParserFormat == Format::Bitstream; + } + + /// Parse and process the metadata of the buffer. + Error parseMeta(); + + /// Parse a Bitstream remark. + Expected> parseRemark(); + +private: + /// Helper functions. + Error processCommonMeta(BitstreamMetaParserHelper &Helper); + Error processStandaloneMeta(BitstreamMetaParserHelper &Helper); + Error processSeparateRemarksFileMeta(BitstreamMetaParserHelper &Helper); + Error processSeparateRemarksMetaMeta(BitstreamMetaParserHelper &Helper); + Expected> + processRemark(BitstreamRemarkParserHelper &Helper); + Error processExternalFilePath(Optional ExternalFilePath); +}; + +Expected> createBitstreamParserFromMeta( + StringRef Buf, Optional StrTab = None, + Optional ExternalFilePrependPath = None); + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_LIB_REMARKS_BITSTREAM_REMARK_PARSER_H */ diff --git a/lib/Remarks/BitstreamRemarkSerializer.cpp b/lib/Remarks/BitstreamRemarkSerializer.cpp new file mode 100644 index 000000000000..d02782c7954d --- /dev/null +++ b/lib/Remarks/BitstreamRemarkSerializer.cpp @@ -0,0 +1,386 @@ +//===- BitstreamRemarkSerializer.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides the implementation of the LLVM bitstream remark serializer +// using LLVM's bitstream writer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Remarks/BitstreamRemarkSerializer.h" + +using namespace llvm; +using namespace llvm::remarks; + +BitstreamRemarkSerializerHelper::BitstreamRemarkSerializerHelper( + BitstreamRemarkContainerType ContainerType) + : Encoded(), R(), Bitstream(Encoded), ContainerType(ContainerType) {} + +static void push(SmallVectorImpl &R, StringRef Str) { + for (const char C : Str) + R.push_back(C); +} + +static void setRecordName(unsigned RecordID, BitstreamWriter &Bitstream, + SmallVectorImpl &R, StringRef Str) { + R.clear(); + R.push_back(RecordID); + push(R, Str); + Bitstream.EmitRecord(bitc::BLOCKINFO_CODE_SETRECORDNAME, R); +} + +static void initBlock(unsigned BlockID, BitstreamWriter &Bitstream, + SmallVectorImpl &R, StringRef Str) { + R.clear(); + R.push_back(BlockID); + Bitstream.EmitRecord(bitc::BLOCKINFO_CODE_SETBID, R); + + R.clear(); + push(R, Str); + Bitstream.EmitRecord(bitc::BLOCKINFO_CODE_BLOCKNAME, R); +} + +void BitstreamRemarkSerializerHelper::setupMetaBlockInfo() { + // Setup the metadata block. + initBlock(META_BLOCK_ID, Bitstream, R, MetaBlockName); + + // The container information. + setRecordName(RECORD_META_CONTAINER_INFO, Bitstream, R, + MetaContainerInfoName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_META_CONTAINER_INFO)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Version. + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // Type. + RecordMetaContainerInfoAbbrevID = + Bitstream.EmitBlockInfoAbbrev(META_BLOCK_ID, Abbrev); +} + +void BitstreamRemarkSerializerHelper::setupMetaRemarkVersion() { + setRecordName(RECORD_META_REMARK_VERSION, Bitstream, R, + MetaRemarkVersionName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_META_REMARK_VERSION)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Version. + RecordMetaRemarkVersionAbbrevID = + Bitstream.EmitBlockInfoAbbrev(META_BLOCK_ID, Abbrev); +} + +void BitstreamRemarkSerializerHelper::emitMetaRemarkVersion( + uint64_t RemarkVersion) { + // The remark version is emitted only if we emit remarks. + R.clear(); + R.push_back(RECORD_META_REMARK_VERSION); + R.push_back(RemarkVersion); + Bitstream.EmitRecordWithAbbrev(RecordMetaRemarkVersionAbbrevID, R); +} + +void BitstreamRemarkSerializerHelper::setupMetaStrTab() { + setRecordName(RECORD_META_STRTAB, Bitstream, R, MetaStrTabName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_META_STRTAB)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Raw table. + RecordMetaStrTabAbbrevID = + Bitstream.EmitBlockInfoAbbrev(META_BLOCK_ID, Abbrev); +} + +void BitstreamRemarkSerializerHelper::emitMetaStrTab( + const StringTable &StrTab) { + // The string table is not emitted if we emit remarks separately. + R.clear(); + R.push_back(RECORD_META_STRTAB); + + // Serialize to a blob. + std::string Buf; + raw_string_ostream OS(Buf); + StrTab.serialize(OS); + StringRef Blob = OS.str(); + Bitstream.EmitRecordWithBlob(RecordMetaStrTabAbbrevID, R, Blob); +} + +void BitstreamRemarkSerializerHelper::setupMetaExternalFile() { + setRecordName(RECORD_META_EXTERNAL_FILE, Bitstream, R, MetaExternalFileName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_META_EXTERNAL_FILE)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Filename. + RecordMetaExternalFileAbbrevID = + Bitstream.EmitBlockInfoAbbrev(META_BLOCK_ID, Abbrev); +} + +void BitstreamRemarkSerializerHelper::emitMetaExternalFile(StringRef Filename) { + // The external file is emitted only if we emit the separate metadata. + R.clear(); + R.push_back(RECORD_META_EXTERNAL_FILE); + Bitstream.EmitRecordWithBlob(RecordMetaExternalFileAbbrevID, R, Filename); +} + +void BitstreamRemarkSerializerHelper::setupRemarkBlockInfo() { + // Setup the remark block. + initBlock(REMARK_BLOCK_ID, Bitstream, R, RemarkBlockName); + + // The header of a remark. + { + setRecordName(RECORD_REMARK_HEADER, Bitstream, R, RemarkHeaderName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_REMARK_HEADER)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // Type + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Remark Name + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Pass name + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Function name + RecordRemarkHeaderAbbrevID = + Bitstream.EmitBlockInfoAbbrev(REMARK_BLOCK_ID, Abbrev); + } + + // The location of a remark. + { + setRecordName(RECORD_REMARK_DEBUG_LOC, Bitstream, R, RemarkDebugLocName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_REMARK_DEBUG_LOC)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // File + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Line + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Column + RecordRemarkDebugLocAbbrevID = + Bitstream.EmitBlockInfoAbbrev(REMARK_BLOCK_ID, Abbrev); + } + + // The hotness of a remark. + { + setRecordName(RECORD_REMARK_HOTNESS, Bitstream, R, RemarkHotnessName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_REMARK_HOTNESS)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Hotness + RecordRemarkHotnessAbbrevID = + Bitstream.EmitBlockInfoAbbrev(REMARK_BLOCK_ID, Abbrev); + } + + // An argument entry with a debug location attached. + { + setRecordName(RECORD_REMARK_ARG_WITH_DEBUGLOC, Bitstream, R, + RemarkArgWithDebugLocName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_REMARK_ARG_WITH_DEBUGLOC)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // Key + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // Value + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // File + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Line + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Column + RecordRemarkArgWithDebugLocAbbrevID = + Bitstream.EmitBlockInfoAbbrev(REMARK_BLOCK_ID, Abbrev); + } + + // An argument entry with no debug location attached. + { + setRecordName(RECORD_REMARK_ARG_WITHOUT_DEBUGLOC, Bitstream, R, + RemarkArgWithoutDebugLocName); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(RECORD_REMARK_ARG_WITHOUT_DEBUGLOC)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // Key + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // Value + RecordRemarkArgWithoutDebugLocAbbrevID = + Bitstream.EmitBlockInfoAbbrev(REMARK_BLOCK_ID, Abbrev); + } +} + +void BitstreamRemarkSerializerHelper::setupBlockInfo() { + // Emit magic number. + for (const char C : ContainerMagic) + Bitstream.Emit(static_cast(C), 8); + + Bitstream.EnterBlockInfoBlock(); + + // Setup the main metadata. Depending on the container type, we'll setup the + // required records next. + setupMetaBlockInfo(); + + switch (ContainerType) { + case BitstreamRemarkContainerType::SeparateRemarksMeta: + // Needs a string table that the separate remark file is using. + setupMetaStrTab(); + // Needs to know where the external remarks file is. + setupMetaExternalFile(); + break; + case BitstreamRemarkContainerType::SeparateRemarksFile: + // Contains remarks: emit the version. + setupMetaRemarkVersion(); + // Contains remarks: emit the remark abbrevs. + setupRemarkBlockInfo(); + break; + case BitstreamRemarkContainerType::Standalone: + // Contains remarks: emit the version. + setupMetaRemarkVersion(); + // Needs a string table. + setupMetaStrTab(); + // Contains remarks: emit the remark abbrevs. + setupRemarkBlockInfo(); + break; + } + + Bitstream.ExitBlock(); +} + +void BitstreamRemarkSerializerHelper::emitMetaBlock( + uint64_t ContainerVersion, Optional RemarkVersion, + Optional StrTab, Optional Filename) { + // Emit the meta block + Bitstream.EnterSubblock(META_BLOCK_ID, 3); + + // The container version and type. + R.clear(); + R.push_back(RECORD_META_CONTAINER_INFO); + R.push_back(ContainerVersion); + R.push_back(static_cast(ContainerType)); + Bitstream.EmitRecordWithAbbrev(RecordMetaContainerInfoAbbrevID, R); + + switch (ContainerType) { + case BitstreamRemarkContainerType::SeparateRemarksMeta: + assert(StrTab != None && *StrTab != nullptr); + emitMetaStrTab(**StrTab); + assert(Filename != None); + emitMetaExternalFile(*Filename); + break; + case BitstreamRemarkContainerType::SeparateRemarksFile: + assert(RemarkVersion != None); + emitMetaRemarkVersion(*RemarkVersion); + break; + case BitstreamRemarkContainerType::Standalone: + assert(RemarkVersion != None); + emitMetaRemarkVersion(*RemarkVersion); + assert(StrTab != None && *StrTab != nullptr); + emitMetaStrTab(**StrTab); + break; + } + + Bitstream.ExitBlock(); +} + +void BitstreamRemarkSerializerHelper::emitRemarkBlock(const Remark &Remark, + StringTable &StrTab) { + Bitstream.EnterSubblock(REMARK_BLOCK_ID, 4); + + R.clear(); + R.push_back(RECORD_REMARK_HEADER); + R.push_back(static_cast(Remark.RemarkType)); + R.push_back(StrTab.add(Remark.RemarkName).first); + R.push_back(StrTab.add(Remark.PassName).first); + R.push_back(StrTab.add(Remark.FunctionName).first); + Bitstream.EmitRecordWithAbbrev(RecordRemarkHeaderAbbrevID, R); + + if (const Optional &Loc = Remark.Loc) { + R.clear(); + R.push_back(RECORD_REMARK_DEBUG_LOC); + R.push_back(StrTab.add(Loc->SourceFilePath).first); + R.push_back(Loc->SourceLine); + R.push_back(Loc->SourceColumn); + Bitstream.EmitRecordWithAbbrev(RecordRemarkDebugLocAbbrevID, R); + } + + if (Optional Hotness = Remark.Hotness) { + R.clear(); + R.push_back(RECORD_REMARK_HOTNESS); + R.push_back(*Hotness); + Bitstream.EmitRecordWithAbbrev(RecordRemarkHotnessAbbrevID, R); + } + + for (const Argument &Arg : Remark.Args) { + R.clear(); + unsigned Key = StrTab.add(Arg.Key).first; + unsigned Val = StrTab.add(Arg.Val).first; + bool HasDebugLoc = Arg.Loc != None; + R.push_back(HasDebugLoc ? RECORD_REMARK_ARG_WITH_DEBUGLOC + : RECORD_REMARK_ARG_WITHOUT_DEBUGLOC); + R.push_back(Key); + R.push_back(Val); + if (HasDebugLoc) { + R.push_back(StrTab.add(Arg.Loc->SourceFilePath).first); + R.push_back(Arg.Loc->SourceLine); + R.push_back(Arg.Loc->SourceColumn); + } + Bitstream.EmitRecordWithAbbrev(HasDebugLoc + ? RecordRemarkArgWithDebugLocAbbrevID + : RecordRemarkArgWithoutDebugLocAbbrevID, + R); + } + Bitstream.ExitBlock(); +} + +void BitstreamRemarkSerializerHelper::flushToStream(raw_ostream &OS) { + OS.write(Encoded.data(), Encoded.size()); + Encoded.clear(); +} + +StringRef BitstreamRemarkSerializerHelper::getBuffer() { + return StringRef(Encoded.data(), Encoded.size()); +} + +BitstreamRemarkSerializer::BitstreamRemarkSerializer(raw_ostream &OS, + SerializerMode Mode) + : RemarkSerializer(Format::Bitstream, OS, Mode), + Helper(BitstreamRemarkContainerType::SeparateRemarksFile) { + assert(Mode == SerializerMode::Separate && + "For SerializerMode::Standalone, a pre-filled string table needs to " + "be provided."); + // We always use a string table with bitstream. + StrTab.emplace(); +} + +BitstreamRemarkSerializer::BitstreamRemarkSerializer(raw_ostream &OS, + SerializerMode Mode, + StringTable StrTabIn) + : RemarkSerializer(Format::Bitstream, OS, Mode), + Helper(Mode == SerializerMode::Separate + ? BitstreamRemarkContainerType::SeparateRemarksFile + : BitstreamRemarkContainerType::Standalone) { + StrTab = std::move(StrTabIn); +} + +void BitstreamRemarkSerializer::emit(const Remark &Remark) { + if (!DidSetUp) { + // Emit the metadata that is embedded in the remark file. + // If we're in standalone mode, serialize the string table as well. + bool IsStandalone = + Helper.ContainerType == BitstreamRemarkContainerType::Standalone; + BitstreamMetaSerializer MetaSerializer( + OS, Helper, + IsStandalone ? &*StrTab : Optional(None)); + MetaSerializer.emit(); + DidSetUp = true; + } + + assert(DidSetUp && + "The Block info block and the meta block were not emitted yet."); + Helper.emitRemarkBlock(Remark, *StrTab); + + Helper.flushToStream(OS); +} + +std::unique_ptr BitstreamRemarkSerializer::metaSerializer( + raw_ostream &OS, Optional ExternalFilename) { + assert(Helper.ContainerType != + BitstreamRemarkContainerType::SeparateRemarksMeta); + bool IsStandalone = + Helper.ContainerType == BitstreamRemarkContainerType::Standalone; + return std::make_unique( + OS, + IsStandalone ? BitstreamRemarkContainerType::Standalone + : BitstreamRemarkContainerType::SeparateRemarksMeta, + &*StrTab, ExternalFilename); +} + +void BitstreamMetaSerializer::emit() { + Helper->setupBlockInfo(); + Helper->emitMetaBlock(CurrentContainerVersion, CurrentRemarkVersion, StrTab, + ExternalFilename); + Helper->flushToStream(OS); +} diff --git a/lib/Remarks/RemarkFormat.cpp b/lib/Remarks/RemarkFormat.cpp index bcd0f753ff64..f2d0331ec6a8 100644 --- a/lib/Remarks/RemarkFormat.cpp +++ b/lib/Remarks/RemarkFormat.cpp @@ -19,11 +19,13 @@ using namespace llvm::remarks; Expected llvm::remarks::parseFormat(StringRef FormatStr) { auto Result = StringSwitch(FormatStr) .Cases("", "yaml", Format::YAML) + .Case("yaml-strtab", Format::YAMLStrTab) + .Case("bitstream", Format::Bitstream) .Default(Format::Unknown); if (Result == Format::Unknown) return createStringError(std::make_error_code(std::errc::invalid_argument), - "Unknown remark serializer format: '%s'", + "Unknown remark format: '%s'", FormatStr.data()); return Result; diff --git a/lib/Remarks/RemarkParser.cpp b/lib/Remarks/RemarkParser.cpp index f67464073bd1..c5c3d0badd3e 100644 --- a/lib/Remarks/RemarkParser.cpp +++ b/lib/Remarks/RemarkParser.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Remarks/RemarkParser.h" +#include "BitstreamRemarkParser.h" #include "YAMLRemarkParser.h" #include "llvm-c/Remarks.h" #include "llvm/ADT/STLExtras.h" @@ -47,32 +48,81 @@ Expected ParsedStringTable::operator[](size_t Index) const { return StringRef(Buffer.data() + Offset, NextOffset - Offset - 1); } -Expected> +Expected> +llvm::remarks::createRemarkParser(Format ParserFormat, StringRef Buf) { + switch (ParserFormat) { + case Format::YAML: + return std::make_unique(Buf); + case Format::YAMLStrTab: + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "The YAML with string table format requires a parsed string table."); + case Format::Bitstream: + return std::make_unique(Buf); + case Format::Unknown: + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Unknown remark parser format."); + } + llvm_unreachable("unhandled ParseFormat"); +} + +Expected> llvm::remarks::createRemarkParser(Format ParserFormat, StringRef Buf, - Optional StrTab) { + ParsedStringTable StrTab) { + switch (ParserFormat) { + case Format::YAML: + return createStringError(std::make_error_code(std::errc::invalid_argument), + "The YAML format can't be used with a string " + "table. Use yaml-strtab instead."); + case Format::YAMLStrTab: + return std::make_unique(Buf, std::move(StrTab)); + case Format::Bitstream: + return std::make_unique(Buf, std::move(StrTab)); + case Format::Unknown: + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Unknown remark parser format."); + } + llvm_unreachable("unhandled ParseFormat"); +} + +Expected> +llvm::remarks::createRemarkParserFromMeta( + Format ParserFormat, StringRef Buf, Optional StrTab, + Optional ExternalFilePrependPath) { switch (ParserFormat) { + // Depending on the metadata, the format can be either yaml or yaml-strtab, + // regardless of the input argument. case Format::YAML: - return llvm::make_unique(Buf, StrTab); + case Format::YAMLStrTab: + return createYAMLParserFromMeta(Buf, std::move(StrTab), + std::move(ExternalFilePrependPath)); + case Format::Bitstream: + return createBitstreamParserFromMeta(Buf, std::move(StrTab), + std::move(ExternalFilePrependPath)); case Format::Unknown: return createStringError(std::make_error_code(std::errc::invalid_argument), "Unknown remark parser format."); } - llvm_unreachable("unknown format"); + llvm_unreachable("unhandled ParseFormat"); } +namespace { // Wrapper that holds the state needed to interact with the C API. struct CParser { - std::unique_ptr TheParser; + std::unique_ptr TheParser; Optional Err; CParser(Format ParserFormat, StringRef Buf, - Optional StrTab = None) - : TheParser(cantFail(createRemarkParser(ParserFormat, Buf, StrTab))) {} + Optional StrTab = None) + : TheParser(cantFail( + StrTab ? createRemarkParser(ParserFormat, Buf, std::move(*StrTab)) + : createRemarkParser(ParserFormat, Buf))) {} void handleError(Error E) { Err.emplace(toString(std::move(E))); } bool hasError() const { return Err.hasValue(); } const char *getMessage() const { return Err ? Err->c_str() : nullptr; }; }; +} // namespace // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_SIMPLE_CONVERSION_FUNCTIONS(CParser, LLVMRemarkParserRef) @@ -83,10 +133,16 @@ extern "C" LLVMRemarkParserRef LLVMRemarkParserCreateYAML(const void *Buf, StringRef(static_cast(Buf), Size))); } +extern "C" LLVMRemarkParserRef LLVMRemarkParserCreateBitstream(const void *Buf, + uint64_t Size) { + return wrap(new CParser(Format::Bitstream, + StringRef(static_cast(Buf), Size))); +} + extern "C" LLVMRemarkEntryRef LLVMRemarkParserGetNext(LLVMRemarkParserRef Parser) { CParser &TheCParser = *unwrap(Parser); - remarks::Parser &TheParser = *TheCParser.TheParser; + remarks::RemarkParser &TheParser = *TheCParser.TheParser; Expected> MaybeRemark = TheParser.next(); if (Error E = MaybeRemark.takeError()) { diff --git a/lib/Remarks/RemarkSerializer.cpp b/lib/Remarks/RemarkSerializer.cpp new file mode 100644 index 000000000000..ab19c84bbadb --- /dev/null +++ b/lib/Remarks/RemarkSerializer.cpp @@ -0,0 +1,54 @@ +//===- RemarkSerializer.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides tools for serializing remarks. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Remarks/RemarkSerializer.h" +#include "llvm/Remarks/BitstreamRemarkSerializer.h" +#include "llvm/Remarks/YAMLRemarkSerializer.h" + +using namespace llvm; +using namespace llvm::remarks; + +Expected> +remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS) { + switch (RemarksFormat) { + case Format::Unknown: + return createStringError(std::errc::invalid_argument, + "Unknown remark serializer format."); + case Format::YAML: + return std::make_unique(OS, Mode); + case Format::YAMLStrTab: + return std::make_unique(OS, Mode); + case Format::Bitstream: + return std::make_unique(OS, Mode); + } + llvm_unreachable("Unknown remarks::Format enum"); +} + +Expected> +remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS, remarks::StringTable StrTab) { + switch (RemarksFormat) { + case Format::Unknown: + return createStringError(std::errc::invalid_argument, + "Unknown remark serializer format."); + case Format::YAML: + return std::make_unique(OS, Mode, std::move(StrTab)); + case Format::YAMLStrTab: + return std::make_unique(OS, Mode, + std::move(StrTab)); + case Format::Bitstream: + return std::make_unique(OS, Mode, + std::move(StrTab)); + } + llvm_unreachable("Unknown remarks::Format enum"); +} diff --git a/lib/Remarks/RemarkStringTable.cpp b/lib/Remarks/RemarkStringTable.cpp index 984aa5b33b48..51156465be51 100644 --- a/lib/Remarks/RemarkStringTable.cpp +++ b/lib/Remarks/RemarkStringTable.cpp @@ -11,6 +11,8 @@ //===----------------------------------------------------------------------===// #include "llvm/Remarks/RemarkStringTable.h" +#include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkParser.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/Error.h" #include @@ -18,6 +20,14 @@ using namespace llvm; using namespace llvm::remarks; +StringTable::StringTable(const ParsedStringTable &Other) : StrTab() { + for (unsigned i = 0, e = Other.size(); i < e; ++i) + if (Expected MaybeStr = Other[i]) + add(*MaybeStr); + else + llvm_unreachable("Unexpected error while building remarks string table."); +} + std::pair StringTable::add(StringRef Str) { size_t NextID = StrTab.size(); auto KV = StrTab.insert({Str, NextID}); @@ -28,10 +38,22 @@ std::pair StringTable::add(StringRef Str) { return {KV.first->second, KV.first->first()}; } +void StringTable::internalize(Remark &R) { + auto Impl = [&](StringRef &S) { S = add(S).second; }; + Impl(R.PassName); + Impl(R.RemarkName); + Impl(R.FunctionName); + if (R.Loc) + Impl(R.Loc->SourceFilePath); + for (Argument &Arg : R.Args) { + Impl(Arg.Key); + Impl(Arg.Val); + if (Arg.Loc) + Impl(Arg.Loc->SourceFilePath); + } +} + void StringTable::serialize(raw_ostream &OS) const { - // Emit the number of strings. - uint64_t StrTabSize = SerializedSize; - support::endian::write(OS, StrTabSize, support::little); // Emit the sequence of strings. for (StringRef Str : serialize()) { OS << Str; diff --git a/lib/Remarks/YAMLRemarkParser.cpp b/lib/Remarks/YAMLRemarkParser.cpp index ed78b7ba5d95..dd834d85676e 100644 --- a/lib/Remarks/YAMLRemarkParser.cpp +++ b/lib/Remarks/YAMLRemarkParser.cpp @@ -14,6 +14,8 @@ #include "YAMLRemarkParser.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Remarks/RemarkParser.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Path.h" using namespace llvm; using namespace llvm::remarks; @@ -54,9 +56,123 @@ static SourceMgr setupSM(std::string &LastErrorMessage) { return SM; } +// Parse the magic number. This function returns true if this represents remark +// metadata, false otherwise. +static Expected parseMagic(StringRef &Buf) { + if (!Buf.consume_front(remarks::Magic)) + return false; + + if (Buf.size() < 1 || !Buf.consume_front(StringRef("\0", 1))) + return createStringError(std::errc::illegal_byte_sequence, + "Expecting \\0 after magic number."); + return true; +} + +static Expected parseVersion(StringRef &Buf) { + if (Buf.size() < sizeof(uint64_t)) + return createStringError(std::errc::illegal_byte_sequence, + "Expecting version number."); + + uint64_t Version = + support::endian::read( + Buf.data()); + if (Version != remarks::CurrentRemarkVersion) + return createStringError(std::errc::illegal_byte_sequence, + "Mismatching remark version. Got %" PRId64 + ", expected %" PRId64 ".", + Version, remarks::CurrentRemarkVersion); + Buf = Buf.drop_front(sizeof(uint64_t)); + return Version; +} + +static Expected parseStrTabSize(StringRef &Buf) { + if (Buf.size() < sizeof(uint64_t)) + return createStringError(std::errc::illegal_byte_sequence, + "Expecting string table size."); + uint64_t StrTabSize = + support::endian::read( + Buf.data()); + Buf = Buf.drop_front(sizeof(uint64_t)); + return StrTabSize; +} + +static Expected parseStrTab(StringRef &Buf, + uint64_t StrTabSize) { + if (Buf.size() < StrTabSize) + return createStringError(std::errc::illegal_byte_sequence, + "Expecting string table."); + + // Attach the string table to the parser. + ParsedStringTable Result(StringRef(Buf.data(), StrTabSize)); + Buf = Buf.drop_front(StrTabSize); + return Expected(std::move(Result)); +} + +Expected> +remarks::createYAMLParserFromMeta(StringRef Buf, + Optional StrTab, + Optional ExternalFilePrependPath) { + // We now have a magic number. The metadata has to be correct. + Expected isMeta = parseMagic(Buf); + if (!isMeta) + return isMeta.takeError(); + // If it's not recognized as metadata, roll back. + std::unique_ptr SeparateBuf; + if (*isMeta) { + Expected Version = parseVersion(Buf); + if (!Version) + return Version.takeError(); + + Expected StrTabSize = parseStrTabSize(Buf); + if (!StrTabSize) + return StrTabSize.takeError(); + + // If the size of string table is not 0, try to build one. + if (*StrTabSize != 0) { + if (StrTab) + return createStringError(std::errc::illegal_byte_sequence, + "String table already provided."); + Expected MaybeStrTab = parseStrTab(Buf, *StrTabSize); + if (!MaybeStrTab) + return MaybeStrTab.takeError(); + StrTab = std::move(*MaybeStrTab); + } + // If it starts with "---", there is no external file. + if (!Buf.startswith("---")) { + // At this point, we expect Buf to contain the external file path. + StringRef ExternalFilePath = Buf; + SmallString<80> FullPath; + if (ExternalFilePrependPath) + FullPath = *ExternalFilePrependPath; + sys::path::append(FullPath, ExternalFilePath); + + // Try to open the file and start parsing from there. + ErrorOr> BufferOrErr = + MemoryBuffer::getFile(FullPath); + if (std::error_code EC = BufferOrErr.getError()) + return createFileError(FullPath, EC); + + // Keep the buffer alive. + SeparateBuf = std::move(*BufferOrErr); + Buf = SeparateBuf->getBuffer(); + } + } + + std::unique_ptr Result = + StrTab + ? std::make_unique(Buf, std::move(*StrTab)) + : std::make_unique(Buf); + if (SeparateBuf) + Result->SeparateBuf = std::move(SeparateBuf); + return std::move(Result); +} + +YAMLRemarkParser::YAMLRemarkParser(StringRef Buf) + : YAMLRemarkParser(Buf, None) {} + YAMLRemarkParser::YAMLRemarkParser(StringRef Buf, - Optional StrTab) - : Parser{Format::YAML}, StrTab(StrTab), LastErrorMessage(), + Optional StrTab) + : RemarkParser{Format::YAML}, StrTab(std::move(StrTab)), LastErrorMessage(), SM(setupSM(LastErrorMessage)), Stream(Buf, SM), YAMLIt(Stream.begin()) {} Error YAMLRemarkParser::error(StringRef Message, yaml::Node &Node) { @@ -86,7 +202,7 @@ YAMLRemarkParser::parseRemark(yaml::Document &RemarkEntry) { if (!Root) return error("document root is not of mapping type.", *YAMLRoot); - std::unique_ptr Result = llvm::make_unique(); + std::unique_ptr Result = std::make_unique(); Remark &TheRemark = *Result; // First, the type. It needs special handling since is not part of the @@ -179,22 +295,7 @@ Expected YAMLRemarkParser::parseStr(yaml::KeyValueNode &Node) { auto *Value = dyn_cast(Node.getValue()); if (!Value) return error("expected a value of scalar type.", Node); - StringRef Result; - if (!StrTab) { - Result = Value->getRawValue(); - } else { - // If we have a string table, parse it as an unsigned. - unsigned StrID = 0; - if (Expected MaybeStrID = parseUnsigned(Node)) - StrID = *MaybeStrID; - else - return MaybeStrID.takeError(); - - if (Expected Str = (**StrTab)[StrID]) - Result = *Str; - else - return Str.takeError(); - } + StringRef Result = Value->getRawValue(); if (Result.front() == '\'') Result = Result.drop_front(); @@ -325,3 +426,29 @@ Expected> YAMLRemarkParser::next() { return std::move(*MaybeResult); } + +Expected YAMLStrTabRemarkParser::parseStr(yaml::KeyValueNode &Node) { + auto *Value = dyn_cast(Node.getValue()); + if (!Value) + return error("expected a value of scalar type.", Node); + StringRef Result; + // If we have a string table, parse it as an unsigned. + unsigned StrID = 0; + if (Expected MaybeStrID = parseUnsigned(Node)) + StrID = *MaybeStrID; + else + return MaybeStrID.takeError(); + + if (Expected Str = (*StrTab)[StrID]) + Result = *Str; + else + return Str.takeError(); + + if (Result.front() == '\'') + Result = Result.drop_front(); + + if (Result.back() == '\'') + Result = Result.drop_back(); + + return Result; +} diff --git a/lib/Remarks/YAMLRemarkParser.h b/lib/Remarks/YAMLRemarkParser.h index cea76e63e75c..03707433bc03 100644 --- a/lib/Remarks/YAMLRemarkParser.h +++ b/lib/Remarks/YAMLRemarkParser.h @@ -18,6 +18,7 @@ #include "llvm/Remarks/Remark.h" #include "llvm/Remarks/RemarkParser.h" #include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" @@ -46,9 +47,9 @@ private: }; /// Regular YAML to Remark parser. -struct YAMLRemarkParser : public Parser { +struct YAMLRemarkParser : public RemarkParser { /// The string table used for parsing strings. - Optional StrTab; + Optional StrTab; /// Last error message that can come from the YAML parser diagnostics. /// We need this for catching errors in the constructor. std::string LastErrorMessage; @@ -58,17 +59,20 @@ struct YAMLRemarkParser : public Parser { yaml::Stream Stream; /// Iterator in the YAML stream. yaml::document_iterator YAMLIt; + /// If we parse remark metadata in separate mode, we need to open a new file + /// and parse that. + std::unique_ptr SeparateBuf; - YAMLRemarkParser(StringRef Buf, - Optional StrTab = None); + YAMLRemarkParser(StringRef Buf); Expected> next() override; - static bool classof(const Parser *P) { + static bool classof(const RemarkParser *P) { return P->ParserFormat == Format::YAML; } -private: +protected: + YAMLRemarkParser(StringRef Buf, Optional StrTab); /// Create a YAMLParseError error from an existing error generated by the YAML /// parser. /// If there is no error, this returns Success. @@ -82,7 +86,7 @@ private: /// Parse one key to a string. Expected parseKey(yaml::KeyValueNode &Node); /// Parse one value to a string. - Expected parseStr(yaml::KeyValueNode &Node); + virtual Expected parseStr(yaml::KeyValueNode &Node); /// Parse one value to an unsigned. Expected parseUnsigned(yaml::KeyValueNode &Node); /// Parse a debug location. @@ -90,6 +94,26 @@ private: /// Parse an argument. Expected parseArg(yaml::Node &Node); }; + +/// YAML with a string table to Remark parser. +struct YAMLStrTabRemarkParser : public YAMLRemarkParser { + YAMLStrTabRemarkParser(StringRef Buf, ParsedStringTable StrTab) + : YAMLRemarkParser(Buf, std::move(StrTab)) {} + + static bool classof(const RemarkParser *P) { + return P->ParserFormat == Format::YAMLStrTab; + } + +protected: + /// Parse one value to a string. + Expected parseStr(yaml::KeyValueNode &Node) override; +}; + +Expected> +createYAMLParserFromMeta(StringRef Buf, + Optional StrTab = None, + Optional ExternalFilePrependPath = None); + } // end namespace remarks } // end namespace llvm diff --git a/lib/Remarks/YAMLRemarkSerializer.cpp b/lib/Remarks/YAMLRemarkSerializer.cpp index d64ae8e12ab0..3a42fe0678eb 100644 --- a/lib/Remarks/YAMLRemarkSerializer.cpp +++ b/lib/Remarks/YAMLRemarkSerializer.cpp @@ -11,16 +11,12 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Remarks/RemarkSerializer.h" +#include "llvm/Remarks/YAMLRemarkSerializer.h" #include "llvm/Support/CommandLine.h" using namespace llvm; using namespace llvm::remarks; -cl::opt RemarksYAMLStringTable( - "remarks-yaml-string-table", cl::init(false), cl::Hidden, - cl::desc("Enable the usage of a string table with YAML remarks.")); - // Use the same keys whether we use a string table or not (respectively, T is an // unsigned or a StringRef). template @@ -60,11 +56,14 @@ template <> struct MappingTraits { else llvm_unreachable("Unknown remark type"); - if (Optional &StrTab = - reinterpret_cast(io.getContext())->StrTab) { - unsigned PassID = StrTab->add(Remark->PassName).first; - unsigned NameID = StrTab->add(Remark->RemarkName).first; - unsigned FunctionID = StrTab->add(Remark->FunctionName).first; + if (auto *Serializer = dyn_cast( + reinterpret_cast(io.getContext()))) { + assert(Serializer->StrTab.hasValue() && + "YAMLStrTabSerializer with no StrTab."); + StringTable &StrTab = *Serializer->StrTab; + unsigned PassID = StrTab.add(Remark->PassName).first; + unsigned NameID = StrTab.add(Remark->RemarkName).first; + unsigned FunctionID = StrTab.add(Remark->FunctionName).first; mapRemarkHeader(io, PassID, NameID, Remark->Loc, FunctionID, Remark->Hotness, Remark->Args); } else { @@ -82,9 +81,12 @@ template <> struct MappingTraits { unsigned Line = RL.SourceLine; unsigned Col = RL.SourceColumn; - if (Optional &StrTab = - reinterpret_cast(io.getContext())->StrTab) { - unsigned FileID = StrTab->add(File).first; + if (auto *Serializer = dyn_cast( + reinterpret_cast(io.getContext()))) { + assert(Serializer->StrTab.hasValue() && + "YAMLStrTabSerializer with no StrTab."); + StringTable &StrTab = *Serializer->StrTab; + unsigned FileID = StrTab.add(File).first; io.mapRequired("File", FileID); } else { io.mapRequired("File", File); @@ -101,7 +103,7 @@ template <> struct MappingTraits { /// newlines in strings. struct StringBlockVal { StringRef Value; - StringBlockVal(const std::string &Value) : Value(Value) {} + StringBlockVal(StringRef R) : Value(R) {} }; template <> struct BlockScalarTraits { @@ -134,9 +136,12 @@ template <> struct MappingTraits { static void mapping(IO &io, Argument &A) { assert(io.outputting() && "input not yet implemented"); - if (Optional &StrTab = - reinterpret_cast(io.getContext())->StrTab) { - auto ValueID = StrTab->add(A.Val).first; + if (auto *Serializer = dyn_cast( + reinterpret_cast(io.getContext()))) { + assert(Serializer->StrTab.hasValue() && + "YAMLStrTabSerializer with no StrTab."); + StringTable &StrTab = *Serializer->StrTab; + auto ValueID = StrTab.add(A.Val).first; io.mapRequired(A.Key.data(), ValueID); } else if (StringRef(A.Val).count('\n') > 1) { StringBlockVal S(A.Val); @@ -153,15 +158,100 @@ template <> struct MappingTraits { LLVM_YAML_IS_SEQUENCE_VECTOR(Argument) -YAMLSerializer::YAMLSerializer(raw_ostream &OS, UseStringTable UseStringTable) - : Serializer(OS), YAMLOutput(OS, reinterpret_cast(this)) { - if (UseStringTable == remarks::UseStringTable::Yes || RemarksYAMLStringTable) - StrTab.emplace(); +YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode, + Optional StrTabIn) + : YAMLRemarkSerializer(Format::YAML, OS, Mode, std::move(StrTabIn)) {} + +YAMLRemarkSerializer::YAMLRemarkSerializer(Format SerializerFormat, + raw_ostream &OS, SerializerMode Mode, + Optional StrTabIn) + : RemarkSerializer(SerializerFormat, OS, Mode), + YAMLOutput(OS, reinterpret_cast(this)) { + StrTab = std::move(StrTabIn); } -void YAMLSerializer::emit(const Remark &Remark) { +void YAMLRemarkSerializer::emit(const Remark &Remark) { // Again, YAMLTraits expect a non-const object for inputting, but we're not // using that here. auto R = const_cast(&Remark); YAMLOutput << R; } + +std::unique_ptr +YAMLRemarkSerializer::metaSerializer(raw_ostream &OS, + Optional ExternalFilename) { + return std::make_unique(OS, ExternalFilename); +} + +void YAMLStrTabRemarkSerializer::emit(const Remark &Remark) { + // In standalone mode, for the serializer with a string table, emit the + // metadata first and set DidEmitMeta to avoid emitting it again. + if (Mode == SerializerMode::Standalone && !DidEmitMeta) { + std::unique_ptr MetaSerializer = + metaSerializer(OS, /*ExternalFilename=*/None); + MetaSerializer->emit(); + DidEmitMeta = true; + } + + // Then do the usual remark emission. + YAMLRemarkSerializer::emit(Remark); +} + +std::unique_ptr YAMLStrTabRemarkSerializer::metaSerializer( + raw_ostream &OS, Optional ExternalFilename) { + assert(StrTab); + return std::make_unique(OS, ExternalFilename, + *StrTab); +} + +static void emitMagic(raw_ostream &OS) { + // Emit the magic number. + OS << remarks::Magic; + // Explicitly emit a '\0'. + OS.write('\0'); +} + +static void emitVersion(raw_ostream &OS) { + // Emit the version number: little-endian uint64_t. + std::array Version; + support::endian::write64le(Version.data(), remarks::CurrentRemarkVersion); + OS.write(Version.data(), Version.size()); +} + +static void emitStrTab(raw_ostream &OS, Optional StrTab) { + // Emit the string table in the section. + uint64_t StrTabSize = StrTab ? (*StrTab)->SerializedSize : 0; + // Emit the total size of the string table (the size itself excluded): + // little-endian uint64_t. + // Note: even if no string table is used, emit 0. + std::array StrTabSizeBuf; + support::endian::write64le(StrTabSizeBuf.data(), StrTabSize); + OS.write(StrTabSizeBuf.data(), StrTabSizeBuf.size()); + if (StrTab) + (*StrTab)->serialize(OS); +} + +static void emitExternalFile(raw_ostream &OS, StringRef Filename) { + // Emit the null-terminated absolute path to the remark file. + SmallString<128> FilenameBuf = Filename; + sys::fs::make_absolute(FilenameBuf); + assert(!FilenameBuf.empty() && "The filename can't be empty."); + OS.write(FilenameBuf.data(), FilenameBuf.size()); + OS.write('\0'); +} + +void YAMLMetaSerializer::emit() { + emitMagic(OS); + emitVersion(OS); + emitStrTab(OS, None); + if (ExternalFilename) + emitExternalFile(OS, *ExternalFilename); +} + +void YAMLStrTabMetaSerializer::emit() { + emitMagic(OS); + emitVersion(OS); + emitStrTab(OS, &StrTab); + if (ExternalFilename) + emitExternalFile(OS, *ExternalFilename); +} -- cgit v1.2.3