diff options
Diffstat (limited to 'tools')
221 files changed, 17046 insertions, 3310 deletions
diff --git a/tools/LLVMBuild.txt b/tools/LLVMBuild.txt index bcf58842eac3..63caea64cf34 100644 --- a/tools/LLVMBuild.txt +++ b/tools/LLVMBuild.txt @@ -25,6 +25,7 @@ subdirectories = llvm-as llvm-bcanalyzer llvm-cat + llvm-cfi-verify llvm-cov llvm-cvtres llvm-diff @@ -38,10 +39,13 @@ subdirectories = llvm-mc llvm-mcmarkup llvm-modextract + llvm-mt llvm-nm + llvm-objcopy llvm-objdump llvm-pdbutil llvm-profdata + llvm-rc llvm-rtdyld llvm-size llvm-split diff --git a/tools/bugpoint-passes/TestPasses.cpp b/tools/bugpoint-passes/TestPasses.cpp index c1eb0fbb67c1..22ded6261a1a 100644 --- a/tools/bugpoint-passes/TestPasses.cpp +++ b/tools/bugpoint-passes/TestPasses.cpp @@ -99,7 +99,6 @@ static RegisterPass<CrashOnDeclFunc> Z("bugpoint-crash-decl-funcs", "BugPoint Test Pass - Intentionally crash on declared functions"); -#include <iostream> namespace { /// CrashOnOneCU - This pass is used to test bugpoint. It intentionally /// crashes if the Module has two or more compile units diff --git a/tools/bugpoint/BugDriver.cpp b/tools/bugpoint/BugDriver.cpp index 78f35293d2a5..37bdb7bc96b6 100644 --- a/tools/bugpoint/BugDriver.cpp +++ b/tools/bugpoint/BugDriver.cpp @@ -32,6 +32,16 @@ namespace llvm { Triple TargetTriple; } +DiscardTemp::~DiscardTemp() { + if (SaveTemps) { + if (Error E = File.keep()) + errs() << "Failed to keep temp file " << toString(std::move(E)) << '\n'; + return; + } + if (Error E = File.discard()) + errs() << "Failed to delete temp file " << toString(std::move(E)) << '\n'; +} + // Anonymous namespace to define command line options for debugging. // namespace { diff --git a/tools/bugpoint/BugDriver.h b/tools/bugpoint/BugDriver.h index af80e8b2b77d..0e6a9b4f2f38 100644 --- a/tools/bugpoint/BugDriver.h +++ b/tools/bugpoint/BugDriver.h @@ -18,6 +18,7 @@ #include "llvm/IR/ValueMap.h" #include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" #include "llvm/Transforms/Utils/ValueMapper.h" #include <memory> #include <string> @@ -270,6 +271,7 @@ public: bool writeProgramToFile(const std::string &Filename, const Module *M) const; bool writeProgramToFile(const std::string &Filename, int FD, const Module *M) const; + bool writeProgramToFile(int FD, const Module *M) const; private: /// initializeExecutionEnvironment - This method is used to set up the @@ -278,6 +280,11 @@ private: Error initializeExecutionEnvironment(); }; +struct DiscardTemp { + sys::fs::TempFile &File; + ~DiscardTemp(); +}; + /// Given a bitcode or assembly input filename, parse and return it, or return /// null if not possible. /// diff --git a/tools/bugpoint/CMakeLists.txt b/tools/bugpoint/CMakeLists.txt index 8975e6763434..72c597379c8b 100644 --- a/tools/bugpoint/CMakeLists.txt +++ b/tools/bugpoint/CMakeLists.txt @@ -37,7 +37,7 @@ add_llvm_tool(bugpoint export_executable_symbols(bugpoint) if(WITH_POLLY AND LINK_POLLY_INTO_TOOLS) - target_link_libraries(bugpoint Polly) + target_link_libraries(bugpoint PRIVATE Polly) # Ensure LLVMTarget can resolve dependences in Polly. - target_link_libraries(bugpoint LLVMTarget) + target_link_libraries(bugpoint PRIVATE LLVMTarget) endif(WITH_POLLY AND LINK_POLLY_INTO_TOOLS) diff --git a/tools/bugpoint/CrashDebugger.cpp b/tools/bugpoint/CrashDebugger.cpp index 2fd8699c5fc8..9097917d5fef 100644 --- a/tools/bugpoint/CrashDebugger.cpp +++ b/tools/bugpoint/CrashDebugger.cpp @@ -648,7 +648,7 @@ bool ReduceSimplifyCFG::TestBlocks(std::vector<const BasicBlock *> &BBs) { ++BBIt; continue; } - SimplifyCFG(&*BBIt++, TTI, 1); + simplifyCFG(&*BBIt++, TTI); } // Verify we didn't break anything std::vector<std::string> Passes; diff --git a/tools/bugpoint/ExecutionDriver.cpp b/tools/bugpoint/ExecutionDriver.cpp index b26ba93cd616..7562aa603bbb 100644 --- a/tools/bugpoint/ExecutionDriver.cpp +++ b/tools/bugpoint/ExecutionDriver.cpp @@ -271,26 +271,23 @@ Error BugDriver::initializeExecutionEnvironment() { /// Error BugDriver::compileProgram(Module *M) const { // Emit the program to a bitcode file... - SmallString<128> BitcodeFile; - int BitcodeFD; - std::error_code EC = sys::fs::createUniqueFile( - OutputPrefix + "-test-program-%%%%%%%.bc", BitcodeFD, BitcodeFile); - if (EC) { - errs() << ToolName << ": Error making unique filename: " << EC.message() + auto Temp = + sys::fs::TempFile::create(OutputPrefix + "-test-program-%%%%%%%.bc"); + if (!Temp) { + errs() << ToolName + << ": Error making unique filename: " << toString(Temp.takeError()) << "\n"; exit(1); } - if (writeProgramToFile(BitcodeFile.str(), BitcodeFD, M)) { - errs() << ToolName << ": Error emitting bitcode to file '" << BitcodeFile + DiscardTemp Discard{*Temp}; + if (writeProgramToFile(Temp->FD, M)) { + errs() << ToolName << ": Error emitting bitcode to file '" << Temp->TmpName << "'!\n"; exit(1); } - // Remove the temporary bitcode file when we are done. - FileRemover BitcodeFileRemover(BitcodeFile.str(), !SaveTemps); - // Actually compile the program! - return Interpreter->compileProgram(BitcodeFile.str(), Timeout, MemoryLimit); + return Interpreter->compileProgram(Temp->TmpName, Timeout, MemoryLimit); } /// executeProgram - This method runs "Program", capturing the output of the @@ -305,32 +302,26 @@ Expected<std::string> BugDriver::executeProgram(const Module *Program, if (!AI) AI = Interpreter; assert(AI && "Interpreter should have been created already!"); - bool CreatedBitcode = false; if (BitcodeFile.empty()) { // Emit the program to a bitcode file... - SmallString<128> UniqueFilename; - int UniqueFD; - std::error_code EC = sys::fs::createUniqueFile( - OutputPrefix + "-test-program-%%%%%%%.bc", UniqueFD, UniqueFilename); - if (EC) { - errs() << ToolName << ": Error making unique filename: " << EC.message() + auto File = + sys::fs::TempFile::create(OutputPrefix + "-test-program-%%%%%%%.bc"); + if (!File) { + errs() << ToolName + << ": Error making unique filename: " << toString(File.takeError()) << "!\n"; exit(1); } - BitcodeFile = UniqueFilename.str(); + DiscardTemp Discard{*File}; + BitcodeFile = File->TmpName; - if (writeProgramToFile(BitcodeFile, UniqueFD, Program)) { + if (writeProgramToFile(File->FD, Program)) { errs() << ToolName << ": Error emitting bitcode to file '" << BitcodeFile << "'!\n"; exit(1); } - CreatedBitcode = true; } - // Remove the temporary bitcode file when we are done. - std::string BitcodePath(BitcodeFile); - FileRemover BitcodeFileRemover(BitcodePath, CreatedBitcode && !SaveTemps); - if (OutputFile.empty()) OutputFile = OutputPrefix + "-execution-output-%%%%%%%"; diff --git a/tools/bugpoint/ExtractFunction.cpp b/tools/bugpoint/ExtractFunction.cpp index 72872e83f792..431dcedfe203 100644 --- a/tools/bugpoint/ExtractFunction.cpp +++ b/tools/bugpoint/ExtractFunction.cpp @@ -373,19 +373,17 @@ llvm::SplitFunctionsOutOfModule(Module *M, const std::vector<Function *> &F, std::unique_ptr<Module> BugDriver::extractMappedBlocksFromModule(const std::vector<BasicBlock *> &BBs, Module *M) { - SmallString<128> Filename; - int FD; - std::error_code EC = sys::fs::createUniqueFile( - OutputPrefix + "-extractblocks%%%%%%%", FD, Filename); - if (EC) { + auto Temp = sys::fs::TempFile::create(OutputPrefix + "-extractblocks%%%%%%%"); + if (!Temp) { outs() << "*** Basic Block extraction failed!\n"; - errs() << "Error creating temporary file: " << EC.message() << "\n"; + errs() << "Error creating temporary file: " << toString(Temp.takeError()) + << "\n"; EmitProgressBitcode(M, "basicblockextractfail", true); return nullptr; } - sys::RemoveFileOnSignal(Filename); + DiscardTemp Discard{*Temp}; - tool_output_file BlocksToNotExtractFile(Filename.c_str(), FD); + raw_fd_ostream OS(Temp->FD, /*shouldClose*/ false); for (std::vector<BasicBlock *>::const_iterator I = BBs.begin(), E = BBs.end(); I != E; ++I) { BasicBlock *BB = *I; @@ -393,28 +391,24 @@ BugDriver::extractMappedBlocksFromModule(const std::vector<BasicBlock *> &BBs, // off of. if (!BB->hasName()) BB->setName("tmpbb"); - BlocksToNotExtractFile.os() << BB->getParent()->getName() << " " - << BB->getName() << "\n"; + OS << BB->getParent()->getName() << " " << BB->getName() << "\n"; } - BlocksToNotExtractFile.os().close(); - if (BlocksToNotExtractFile.os().has_error()) { + OS.flush(); + if (OS.has_error()) { errs() << "Error writing list of blocks to not extract\n"; EmitProgressBitcode(M, "basicblockextractfail", true); - BlocksToNotExtractFile.os().clear_error(); + OS.clear_error(); return nullptr; } - BlocksToNotExtractFile.keep(); std::string uniqueFN = "--extract-blocks-file="; - uniqueFN += Filename.str(); + uniqueFN += Temp->TmpName; const char *ExtraArg = uniqueFN.c_str(); std::vector<std::string> PI; PI.push_back("extract-blocks"); std::unique_ptr<Module> Ret = runPassesOn(M, PI, 1, &ExtraArg); - sys::fs::remove(Filename.c_str()); - if (!Ret) { outs() << "*** Basic Block extraction failed, please report a bug!\n"; EmitProgressBitcode(M, "basicblockextractfail", true); diff --git a/tools/bugpoint/FindBugs.cpp b/tools/bugpoint/FindBugs.cpp index 3093169ba8b0..40502cbf9495 100644 --- a/tools/bugpoint/FindBugs.cpp +++ b/tools/bugpoint/FindBugs.cpp @@ -15,12 +15,8 @@ //===----------------------------------------------------------------------===// #include "BugDriver.h" -#include "ToolRunner.h" -#include "llvm/Pass.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <ctime> #include <random> using namespace llvm; diff --git a/tools/bugpoint/OptimizerDriver.cpp b/tools/bugpoint/OptimizerDriver.cpp index 489e50b88101..ee3f2f0174d2 100644 --- a/tools/bugpoint/OptimizerDriver.cpp +++ b/tools/bugpoint/OptimizerDriver.cpp @@ -18,21 +18,17 @@ #include "BugDriver.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" -#include "llvm/IR/Verifier.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FileUtilities.h" #include "llvm/Support/Path.h" #include "llvm/Support/Program.h" -#include "llvm/Support/SystemUtils.h" #include "llvm/Support/ToolOutputFile.h" #define DONT_GET_PLUGIN_LOADER_OPTION #include "llvm/Support/PluginLoader.h" -#include <fstream> using namespace llvm; @@ -58,7 +54,7 @@ static cl::opt<std::string> /// writeProgramToFile - This writes the current "Program" to the named bitcode /// file. If an error occurs, true is returned. /// -static bool writeProgramToFileAux(tool_output_file &Out, const Module *M) { +static bool writeProgramToFileAux(ToolOutputFile &Out, const Module *M) { WriteBitcodeToFile(M, Out.os(), PreserveBitcodeUseListOrder); Out.os().close(); if (!Out.os().has_error()) { @@ -70,14 +66,24 @@ static bool writeProgramToFileAux(tool_output_file &Out, const Module *M) { bool BugDriver::writeProgramToFile(const std::string &Filename, int FD, const Module *M) const { - tool_output_file Out(Filename, FD); + ToolOutputFile Out(Filename, FD); return writeProgramToFileAux(Out, M); } +bool BugDriver::writeProgramToFile(int FD, const Module *M) const { + raw_fd_ostream OS(FD, /*shouldClose*/ false); + WriteBitcodeToFile(M, OS, PreserveBitcodeUseListOrder); + OS.flush(); + if (!OS.has_error()) + return false; + OS.clear_error(); + return true; +} + bool BugDriver::writeProgramToFile(const std::string &Filename, const Module *M) const { std::error_code EC; - tool_output_file Out(Filename, EC, sys::fs::F_None); + ToolOutputFile Out(Filename, EC, sys::fs::F_None); if (!EC) return writeProgramToFileAux(Out, M); return true; @@ -144,23 +150,22 @@ bool BugDriver::runPasses(Module *Program, OutputFilename = UniqueFilename.str(); // set up the input file name - SmallString<128> InputFilename; - int InputFD; - EC = sys::fs::createUniqueFile(OutputPrefix + "-input-%%%%%%%.bc", InputFD, - InputFilename); - if (EC) { + Expected<sys::fs::TempFile> Temp = + sys::fs::TempFile::create(OutputPrefix + "-input-%%%%%%%.bc"); + if (!Temp) { errs() << getToolName() - << ": Error making unique filename: " << EC.message() << "\n"; + << ": Error making unique filename: " << toString(Temp.takeError()) + << "\n"; return 1; } - - tool_output_file InFile(InputFilename, InputFD); - - WriteBitcodeToFile(Program, InFile.os(), PreserveBitcodeUseListOrder); - InFile.os().close(); - if (InFile.os().has_error()) { - errs() << "Error writing bitcode file: " << InputFilename << "\n"; - InFile.os().clear_error(); + DiscardTemp Discard{*Temp}; + raw_fd_ostream OS(Temp->FD, /*shouldClose*/ false); + + WriteBitcodeToFile(Program, OS, PreserveBitcodeUseListOrder); + OS.flush(); + if (OS.has_error()) { + errs() << "Error writing bitcode file: " << Temp->TmpName << "\n"; + OS.clear_error(); return 1; } @@ -189,9 +194,6 @@ bool BugDriver::runPasses(Module *Program, return 1; } - // Ok, everything that could go wrong before running opt is done. - InFile.keep(); - // setup the child process' arguments SmallVector<const char *, 8> Args; if (UseValgrind) { @@ -220,7 +222,7 @@ bool BugDriver::runPasses(Module *Program, E = pass_args.end(); I != E; ++I) Args.push_back(I->c_str()); - Args.push_back(InputFilename.c_str()); + Args.push_back(Temp->TmpName.c_str()); for (unsigned i = 0; i < NumExtraArgs; ++i) Args.push_back(*ExtraArgs); Args.push_back(nullptr); @@ -230,13 +232,15 @@ bool BugDriver::runPasses(Module *Program, << " " << Args[i]; errs() << "\n";); - // Redirect stdout and stderr to nowhere if SilencePasses is given - StringRef Nowhere; - const StringRef *Redirects[3] = {nullptr, &Nowhere, &Nowhere}; + Optional<StringRef> Redirects[3] = {None, None, None}; + // Redirect stdout and stderr to nowhere if SilencePasses is given. + if (SilencePasses) { + Redirects[1] = ""; + Redirects[2] = ""; + } std::string ErrMsg; - int result = sys::ExecuteAndWait(Prog, Args.data(), nullptr, - (SilencePasses ? Redirects : nullptr), + int result = sys::ExecuteAndWait(Prog, Args.data(), nullptr, Redirects, Timeout, MemoryLimit, &ErrMsg); // If we are supposed to delete the bitcode file or if the passes crashed, @@ -244,9 +248,6 @@ bool BugDriver::runPasses(Module *Program, if (DeleteOutput || result != 0) sys::fs::remove(OutputFilename); - // Remove the temporary input file as well - sys::fs::remove(InputFilename.c_str()); - if (!Quiet) { if (result == 0) outs() << "Success!\n"; diff --git a/tools/bugpoint/ToolRunner.cpp b/tools/bugpoint/ToolRunner.cpp index 70b18e3dbbf9..8094dfdd78fa 100644 --- a/tools/bugpoint/ToolRunner.cpp +++ b/tools/bugpoint/ToolRunner.cpp @@ -58,7 +58,7 @@ static int RunProgramWithTimeout(StringRef ProgramPath, const char **Args, StringRef StdErrFile, unsigned NumSeconds = 0, unsigned MemoryLimit = 0, std::string *ErrMsg = nullptr) { - const StringRef *Redirects[3] = {&StdInFile, &StdOutFile, &StdErrFile}; + Optional<StringRef> Redirects[3] = {StdInFile, StdOutFile, StdErrFile}; return sys::ExecuteAndWait(ProgramPath, Args, nullptr, Redirects, NumSeconds, MemoryLimit, ErrMsg); } @@ -75,7 +75,7 @@ static int RunProgramRemotelyWithTimeout(StringRef RemoteClientPath, StringRef StdErrFile, unsigned NumSeconds = 0, unsigned MemoryLimit = 0) { - const StringRef *Redirects[3] = {&StdInFile, &StdOutFile, &StdErrFile}; + Optional<StringRef> Redirects[3] = {StdInFile, StdOutFile, StdErrFile}; // Run the program remotely with the remote client int ReturnCode = sys::ExecuteAndWait(RemoteClientPath, Args, nullptr, diff --git a/tools/bugpoint/bugpoint.cpp b/tools/bugpoint/bugpoint.cpp index 4ddea8dbec19..ec1ca2e54968 100644 --- a/tools/bugpoint/bugpoint.cpp +++ b/tools/bugpoint/bugpoint.cpp @@ -50,10 +50,10 @@ static cl::opt<unsigned> TimeoutValue( cl::desc("Number of seconds program is allowed to run before it " "is killed (default is 300s), 0 disables timeout")); -static cl::opt<int> - MemoryLimit("mlimit", cl::init(-1), cl::value_desc("MBytes"), - cl::desc("Maximum amount of memory to use. 0 disables check." - " Defaults to 400MB (800MB under valgrind).")); +static cl::opt<int> MemoryLimit( + "mlimit", cl::init(-1), cl::value_desc("MBytes"), + cl::desc("Maximum amount of memory to use. 0 disables check. Defaults to " + "400MB (800MB under valgrind, 0 with sanitizers).")); static cl::opt<bool> UseValgrind("enable-valgrind", @@ -169,6 +169,12 @@ int main(int argc, char **argv) { MemoryLimit = 800; else MemoryLimit = 400; +#if (LLVM_ADDRESS_SANITIZER_BUILD || LLVM_MEMORY_SANITIZER_BUILD || \ + LLVM_THREAD_SANITIZER_BUILD) + // Starting from kernel 4.9 memory allocated with mmap is counted against + // RLIMIT_DATA. Sanitizers need to allocate tens of terabytes for shadow. + MemoryLimit = 0; +#endif } BugDriver D(argv[0], FindBugs, TimeoutValue, MemoryLimit, UseValgrind, diff --git a/tools/dsymutil/CFBundle.cpp b/tools/dsymutil/CFBundle.cpp new file mode 100644 index 000000000000..304838f7ee2c --- /dev/null +++ b/tools/dsymutil/CFBundle.cpp @@ -0,0 +1,207 @@ +//===- tools/dsymutil/CFBundle.cpp - CFBundle helper ------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "CFBundle.h" + +#ifdef __APPLE__ +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" +#include <CoreFoundation/CoreFoundation.h> +#include <assert.h> +#include <glob.h> +#include <memory> +#endif + +namespace llvm { +namespace dsymutil { + +#ifdef __APPLE__ +/// Deleter that calls CFRelease rather than deleting the pointer. +template <typename T> struct CFDeleter { + void operator()(T *P) { + if (P) + ::CFRelease(P); + } +}; + +/// This helper owns any CoreFoundation pointer and will call CFRelease() on +/// any valid pointer it owns unless that pointer is explicitly released using +/// the release() member function. +template <typename T> +using CFReleaser = + std::unique_ptr<typename std::remove_pointer<T>::type, + CFDeleter<typename std::remove_pointer<T>::type>>; + +/// RAII wrapper around CFBundleRef. +class CFString : public CFReleaser<CFStringRef> { +public: + CFString(CFStringRef CFStr = nullptr) : CFReleaser<CFStringRef>(CFStr) {} + + const char *UTF8(std::string &Str) const { + return CFString::UTF8(get(), Str); + } + + CFIndex GetLength() const { + if (CFStringRef Str = get()) + return CFStringGetLength(Str); + return 0; + } + + static const char *UTF8(CFStringRef CFStr, std::string &Str); +}; + +/// Static function that puts a copy of the UTF8 contents of CFStringRef into +/// std::string and returns the C string pointer that is contained in the +/// std::string when successful, nullptr otherwise. +/// +/// This allows the std::string parameter to own the extracted string, and also +/// allows that string to be returned as a C string pointer that can be used. +const char *CFString::UTF8(CFStringRef CFStr, std::string &Str) { + if (!CFStr) + return nullptr; + + const CFStringEncoding Encoding = kCFStringEncodingUTF8; + CFIndex MaxUTF8StrLength = CFStringGetLength(CFStr); + MaxUTF8StrLength = + CFStringGetMaximumSizeForEncoding(MaxUTF8StrLength, Encoding); + if (MaxUTF8StrLength > 0) { + Str.resize(MaxUTF8StrLength); + if (!Str.empty() && + CFStringGetCString(CFStr, &Str[0], Str.size(), Encoding)) { + Str.resize(strlen(Str.c_str())); + return Str.c_str(); + } + } + + return nullptr; +} + +/// RAII wrapper around CFBundleRef. +class CFBundle : public CFReleaser<CFBundleRef> { +public: + CFBundle(const char *Path = nullptr) : CFReleaser<CFBundleRef>() { + if (Path && Path[0]) + SetFromPath(Path); + } + + CFBundle(CFURLRef url) + : CFReleaser<CFBundleRef>(url ? ::CFBundleCreate(nullptr, url) + : nullptr) {} + + /// Return the bundle identifier. + CFStringRef GetIdentifier() const { + if (CFBundleRef bundle = get()) + return ::CFBundleGetIdentifier(bundle); + return nullptr; + } + + /// Return value for key. + CFTypeRef GetValueForInfoDictionaryKey(CFStringRef key) const { + if (CFBundleRef bundle = get()) + return ::CFBundleGetValueForInfoDictionaryKey(bundle, key); + return nullptr; + } + +private: + /// Update this instance with a new bundle created from the given path. + bool SetFromPath(const char *Path); +}; + +bool CFBundle::SetFromPath(const char *InPath) { + // Release our old bundle and URL. + reset(); + + if (InPath && InPath[0]) { + char ResolvedPath[PATH_MAX]; + const char *Path = ::realpath(InPath, ResolvedPath); + if (Path == nullptr) + Path = InPath; + + CFAllocatorRef Allocator = kCFAllocatorDefault; + // Make our Bundle URL. + CFReleaser<CFURLRef> BundleURL(::CFURLCreateFromFileSystemRepresentation( + Allocator, (const UInt8 *)Path, strlen(Path), false)); + if (BundleURL.get()) { + CFIndex LastLength = LONG_MAX; + + while (BundleURL.get() != nullptr) { + // Check the Path range and make sure we didn't make it to just "/", + // ".", or "..". + CFRange rangeIncludingSeparators; + CFRange range = ::CFURLGetByteRangeForComponent( + BundleURL.get(), kCFURLComponentPath, &rangeIncludingSeparators); + if (range.length > LastLength) + break; + + reset(::CFBundleCreate(Allocator, BundleURL.get())); + if (get() != nullptr) { + if (GetIdentifier() != nullptr) + break; + reset(); + } + BundleURL.reset(::CFURLCreateCopyDeletingLastPathComponent( + Allocator, BundleURL.get())); + + LastLength = range.length; + } + } + } + + return get() != nullptr; +} + +#endif + +/// On Darwin, try and find the original executable's Info.plist information +/// using CoreFoundation calls by creating a URL for the executable and +/// chopping off the last Path component. The CFBundle can then get the +/// identifier and grab any needed information from it directly. Return default +/// CFBundleInfo on other platforms. +CFBundleInfo getBundleInfo(StringRef ExePath) { + CFBundleInfo BundleInfo; + +#ifdef __APPLE__ + if (ExePath.empty() || !sys::fs::exists(ExePath)) + return BundleInfo; + + auto PrintError = [&](CFTypeID TypeID) { + CFString TypeIDCFStr(::CFCopyTypeIDDescription(TypeID)); + std::string TypeIDStr; + errs() << "The Info.plist key \"CFBundleShortVersionString\" is" + << "a " << TypeIDCFStr.UTF8(TypeIDStr) + << ", but it should be a string in: " << ExePath << ".\n"; + }; + + CFBundle Bundle(ExePath.data()); + if (CFStringRef BundleID = Bundle.GetIdentifier()) { + CFString::UTF8(BundleID, BundleInfo.IDStr); + if (CFTypeRef TypeRef = + Bundle.GetValueForInfoDictionaryKey(CFSTR("CFBundleVersion"))) { + CFTypeID TypeID = ::CFGetTypeID(TypeRef); + if (TypeID == ::CFStringGetTypeID()) + CFString::UTF8((CFStringRef)TypeRef, BundleInfo.VersionStr); + else + PrintError(TypeID); + } + if (CFTypeRef TypeRef = Bundle.GetValueForInfoDictionaryKey( + CFSTR("CFBundleShortVersionString"))) { + CFTypeID TypeID = ::CFGetTypeID(TypeRef); + if (TypeID == ::CFStringGetTypeID()) + CFString::UTF8((CFStringRef)TypeRef, BundleInfo.ShortVersionStr); + else + PrintError(TypeID); + } + } +#endif + + return BundleInfo; +} + +} // end namespace dsymutil +} // end namespace llvm diff --git a/tools/dsymutil/CFBundle.h b/tools/dsymutil/CFBundle.h new file mode 100644 index 000000000000..bdbecb4785c0 --- /dev/null +++ b/tools/dsymutil/CFBundle.h @@ -0,0 +1,26 @@ +//===- tools/dsymutil/CFBundle.h - CFBundle helper --------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringRef.h" +#include <string> + +namespace llvm { +namespace dsymutil { + +struct CFBundleInfo { + std::string VersionStr = "1"; + std::string ShortVersionStr = "1.0"; + std::string IDStr; + bool OmitShortVersion() const { return ShortVersionStr.empty(); } +}; + +CFBundleInfo getBundleInfo(llvm::StringRef ExePath); + +} // end namespace dsymutil +} // end namespace llvm diff --git a/tools/dsymutil/CMakeLists.txt b/tools/dsymutil/CMakeLists.txt index 3a9b29326b3f..1dcb2116f34b 100644 --- a/tools/dsymutil/CMakeLists.txt +++ b/tools/dsymutil/CMakeLists.txt @@ -11,6 +11,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_tool(llvm-dsymutil dsymutil.cpp BinaryHolder.cpp + CFBundle.cpp DebugMap.cpp DwarfLinker.cpp MachODebugMapParser.cpp @@ -20,3 +21,6 @@ add_llvm_tool(llvm-dsymutil intrinsics_gen ) +IF(APPLE) + target_link_libraries(llvm-dsymutil PRIVATE "-framework CoreFoundation") +ENDIF(APPLE) diff --git a/tools/dsymutil/DebugMap.cpp b/tools/dsymutil/DebugMap.cpp index 636d65836c6d..b543c212fe80 100644 --- a/tools/dsymutil/DebugMap.cpp +++ b/tools/dsymutil/DebugMap.cpp @@ -6,23 +6,42 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// + #include "DebugMap.h" #include "BinaryHolder.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/Support/DataTypes.h" +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Chrono.h" +#include "llvm/Support/Error.h" #include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> +#include <cinttypes> +#include <cstdint> +#include <memory> +#include <string> +#include <utility> +#include <vector> namespace llvm { + namespace dsymutil { using namespace llvm::object; DebugMapObject::DebugMapObject(StringRef ObjectFilename, - sys::TimePoint<std::chrono::seconds> Timestamp) - : Filename(ObjectFilename), Timestamp(Timestamp) {} + sys::TimePoint<std::chrono::seconds> Timestamp, + uint8_t Type) + : Filename(ObjectFilename), Timestamp(Timestamp), Type(Type) {} bool DebugMapObject::addSymbol(StringRef Name, Optional<uint64_t> ObjectAddress, uint64_t LinkedAddress, uint32_t Size) { @@ -38,7 +57,7 @@ void DebugMapObject::print(raw_ostream &OS) const { OS << getObjectFilename() << ":\n"; // Sort the symbols in alphabetical order, like llvm-nm (and to get // deterministic output for testing). - typedef std::pair<StringRef, SymbolMapping> Entry; + using Entry = std::pair<StringRef, SymbolMapping>; std::vector<Entry> Entries; Entries.reserve(Symbols.getNumItems()); for (const auto &Sym : make_range(Symbols.begin(), Symbols.end())) @@ -64,8 +83,9 @@ void DebugMapObject::dump() const { print(errs()); } DebugMapObject & DebugMap::addDebugMapObject(StringRef ObjectFilePath, - sys::TimePoint<std::chrono::seconds> Timestamp) { - Objects.emplace_back(new DebugMapObject(ObjectFilePath, Timestamp)); + sys::TimePoint<std::chrono::seconds> Timestamp, + uint8_t Type) { + Objects.emplace_back(new DebugMapObject(ObjectFilePath, Timestamp, Type)); return *Objects.back(); } @@ -95,11 +115,13 @@ void DebugMap::dump() const { print(errs()); } #endif namespace { + struct YAMLContext { StringRef PrependPath; Triple BinaryTriple; }; -} + +} // end anonymous namespace ErrorOr<std::vector<std::unique_ptr<DebugMap>>> DebugMap::parseYAMLDebugMap(StringRef InputFile, StringRef PrependPath, @@ -122,7 +144,8 @@ DebugMap::parseYAMLDebugMap(StringRef InputFile, StringRef PrependPath, Result.push_back(std::move(Res)); return std::move(Result); } -} + +} // end namespace dsymutil namespace yaml { @@ -153,8 +176,7 @@ void MappingTraits<dsymutil::DebugMapObject>::mapping( io.mapRequired("symbols", Norm->Entries); } -void ScalarTraits<Triple>::output(const Triple &val, void *, - llvm::raw_ostream &out) { +void ScalarTraits<Triple>::output(const Triple &val, void *, raw_ostream &out) { out << val.str(); } @@ -219,8 +241,7 @@ MappingTraits<dsymutil::DebugMapObject>::YamlDMO::denormalize(IO &IO) { sys::path::append(Path, Filename); auto ErrOrObjectFiles = BinHolder.GetObjectFiles(Path); if (auto EC = ErrOrObjectFiles.getError()) { - llvm::errs() << "warning: Unable to open " << Path << " " << EC.message() - << '\n'; + errs() << "warning: Unable to open " << Path << " " << EC.message() << '\n'; } else if (auto ErrOrObjectFile = BinHolder.Get(Ctxt.BinaryTriple)) { // Rewrite the object file symbol addresses in the debug map. The // YAML input is mainly used to test llvm-dsymutil without @@ -241,7 +262,7 @@ MappingTraits<dsymutil::DebugMapObject>::YamlDMO::denormalize(IO &IO) { } } - dsymutil::DebugMapObject Res(Path, sys::toTimePoint(Timestamp)); + dsymutil::DebugMapObject Res(Path, sys::toTimePoint(Timestamp), MachO::N_OSO); for (auto &Entry : Entries) { auto &Mapping = Entry.second; Optional<uint64_t> ObjAddress; @@ -254,5 +275,7 @@ MappingTraits<dsymutil::DebugMapObject>::YamlDMO::denormalize(IO &IO) { } return Res; } -} -} + +} // end namespace yaml + +} // end namespace llvm diff --git a/tools/dsymutil/DebugMap.h b/tools/dsymutil/DebugMap.h index eab0cb0a8009..3b5b437ccff9 100644 --- a/tools/dsymutil/DebugMap.h +++ b/tools/dsymutil/DebugMap.h @@ -1,4 +1,4 @@ -//=== tools/dsymutil/DebugMap.h - Generic debug map representation -*- C++ -*-// +//=- tools/dsymutil/DebugMap.h - Generic debug map representation -*- C++ -*-=// // // The LLVM Linker // @@ -6,7 +6,7 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -/// +// /// \file /// /// This file contains the class declaration of the DebugMap @@ -16,27 +16,35 @@ /// The DebugMap is an input to the DwarfLinker class that will /// extract the Dwarf debug information from the referenced object /// files and link their usefull debug info together. -/// +// //===----------------------------------------------------------------------===// + #ifndef LLVM_TOOLS_DSYMUTIL_DEBUGMAP_H #define LLVM_TOOLS_DSYMUTIL_DEBUGMAP_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/Object/ObjectFile.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/ErrorOr.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/Path.h" #include "llvm/Support/YAMLTraits.h" +#include <chrono> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <string> +#include <utility> #include <vector> namespace llvm { + class raw_ostream; namespace dsymutil { + class DebugMapObject; /// \brief The DebugMap object stores the list of object files to @@ -67,20 +75,24 @@ class DebugMapObject; class DebugMap { Triple BinaryTriple; std::string BinaryPath; - typedef std::vector<std::unique_ptr<DebugMapObject>> ObjectContainer; + + using ObjectContainer = std::vector<std::unique_ptr<DebugMapObject>>; + ObjectContainer Objects; /// For YAML IO support. ///@{ friend yaml::MappingTraits<std::unique_ptr<DebugMap>>; friend yaml::MappingTraits<DebugMap>; + DebugMap() = default; ///@} + public: DebugMap(const Triple &BinaryTriple, StringRef BinaryPath) : BinaryTriple(BinaryTriple), BinaryPath(BinaryPath) {} - typedef ObjectContainer::const_iterator const_iterator; + using const_iterator = ObjectContainer::const_iterator; iterator_range<const_iterator> objects() const { return make_range(begin(), end()); @@ -94,7 +106,8 @@ public: /// debug map. DebugMapObject & addDebugMapObject(StringRef ObjectFilePath, - sys::TimePoint<std::chrono::seconds> Timestamp); + sys::TimePoint<std::chrono::seconds> Timestamp, + uint8_t Type); const Triple &getTriple() const { return BinaryTriple; } @@ -121,23 +134,25 @@ public: Optional<yaml::Hex64> ObjectAddress; yaml::Hex64 BinaryAddress; yaml::Hex32 Size; + SymbolMapping(Optional<uint64_t> ObjectAddr, uint64_t BinaryAddress, uint32_t Size) : BinaryAddress(BinaryAddress), Size(Size) { if (ObjectAddr) ObjectAddress = *ObjectAddr; } + /// For YAML IO support SymbolMapping() = default; }; - typedef std::pair<std::string, SymbolMapping> YAMLSymbolMapping; - typedef StringMapEntry<SymbolMapping> DebugMapEntry; + using YAMLSymbolMapping = std::pair<std::string, SymbolMapping>; + using DebugMapEntry = StringMapEntry<SymbolMapping>; /// \brief Adds a symbol mapping to this DebugMapObject. /// \returns false if the symbol was already registered. The request /// is discarded in this case. - bool addSymbol(llvm::StringRef SymName, Optional<uint64_t> ObjectAddress, + bool addSymbol(StringRef SymName, Optional<uint64_t> ObjectAddress, uint64_t LinkedAddress, uint32_t Size); /// \brief Lookup a symbol mapping. @@ -148,12 +163,14 @@ public: /// \returns null if the address isn't found. const DebugMapEntry *lookupObjectAddress(uint64_t Address) const; - llvm::StringRef getObjectFilename() const { return Filename; } + StringRef getObjectFilename() const { return Filename; } sys::TimePoint<std::chrono::seconds> getTimestamp() const { return Timestamp; } + uint8_t getType() const { return Type; } + iterator_range<StringMap<SymbolMapping>::const_iterator> symbols() const { return make_range(Symbols.begin(), Symbols.end()); } @@ -162,21 +179,25 @@ public: #ifndef NDEBUG void dump() const; #endif + private: friend class DebugMap; + /// DebugMapObjects can only be constructed by the owning DebugMap. DebugMapObject(StringRef ObjectFilename, - sys::TimePoint<std::chrono::seconds> Timestamp); + sys::TimePoint<std::chrono::seconds> Timestamp, uint8_t Type); std::string Filename; sys::TimePoint<std::chrono::seconds> Timestamp; StringMap<SymbolMapping> Symbols; DenseMap<uint64_t, DebugMapEntry *> AddressToMapping; + uint8_t Type; /// For YAMLIO support. ///@{ friend yaml::MappingTraits<dsymutil::DebugMapObject>; friend yaml::SequenceTraits<std::vector<std::unique_ptr<DebugMapObject>>>; + DebugMapObject() = default; public: @@ -184,8 +205,10 @@ public: DebugMapObject &operator=(DebugMapObject &&) = default; ///@} }; -} -} + +} // end namespace dsymutil + +} // end namespace llvm LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::dsymutil::DebugMapObject::YAMLSymbolMapping) @@ -207,9 +230,9 @@ template <> struct MappingTraits<dsymutil::DebugMapObject> { }; template <> struct ScalarTraits<Triple> { - static void output(const Triple &val, void *, llvm::raw_ostream &out); + static void output(const Triple &val, void *, raw_ostream &out); static StringRef input(StringRef scalar, void *, Triple &value); - static bool mustQuote(StringRef) { return true; } + static QuotingType mustQuote(StringRef) { return QuotingType::Single; } }; template <> @@ -228,7 +251,8 @@ template <> struct MappingTraits<dsymutil::DebugMap> { template <> struct MappingTraits<std::unique_ptr<dsymutil::DebugMap>> { static void mapping(IO &io, std::unique_ptr<dsymutil::DebugMap> &DM); }; -} -} + +} // end namespace yaml +} // end namespace llvm #endif // LLVM_TOOLS_DSYMUTIL_DEBUGMAP_H diff --git a/tools/dsymutil/DwarfLinker.cpp b/tools/dsymutil/DwarfLinker.cpp index 86621e3260f0..50ffc69dfaa0 100644 --- a/tools/dsymutil/DwarfLinker.cpp +++ b/tools/dsymutil/DwarfLinker.cpp @@ -6,21 +6,43 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// + #include "BinaryHolder.h" #include "DebugMap.h" #include "MachOUtils.h" #include "NonRelocatableStringpool.h" #include "dsymutil.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/Hashing.h" #include "llvm/ADT/IntervalMap.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/BinaryFormat/MachO.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/DIE.h" #include "llvm/Config/config.h" +#include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h" +#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h" +#include "llvm/DebugInfo/DWARF/DWARFDie.h" #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" +#include "llvm/DebugInfo/DWARF/DWARFSection.h" +#include "llvm/DebugInfo/DWARF/DWARFUnit.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCCodeEmitter.h" @@ -29,17 +51,47 @@ #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptionsCommandFlags.h" +#include "llvm/MC/MCTargetOptions.h" +#include "llvm/MC/MCTargetOptionsCommandFlags.def" #include "llvm/Object/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/DataExtractor.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Format.h" #include "llvm/Support/LEB128.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" #include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" +#include <algorithm> +#include <cassert> +#include <cinttypes> +#include <climits> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <limits> +#include <map> #include <memory> #include <string> +#include <system_error> #include <tuple> +#include <utility> +#include <vector> namespace llvm { namespace dsymutil { @@ -51,7 +103,7 @@ using HalfOpenIntervalMap = IntervalMap<KeyT, ValT, IntervalMapImpl::NodeSizer<KeyT, ValT>::LeafSize, IntervalMapHalfOpenInfo<KeyT>>; -typedef HalfOpenIntervalMap<uint64_t, int64_t> FunctionIntervals; +using FunctionIntervals = HalfOpenIntervalMap<uint64_t, int64_t>; // FIXME: Delete this structure. struct PatchLocation { @@ -94,33 +146,31 @@ struct DeclMapInfo; /// specific DeclContext using a separate DenseMap keyed on the hash /// of the fully qualified name of the context. class DeclContext { - unsigned QualifiedNameHash; - uint32_t Line; - uint32_t ByteSize; - uint16_t Tag; + friend DeclMapInfo; + + unsigned QualifiedNameHash = 0; + uint32_t Line = 0; + uint32_t ByteSize = 0; + uint16_t Tag = dwarf::DW_TAG_compile_unit; + unsigned DefinedInClangModule : 1; StringRef Name; StringRef File; const DeclContext &Parent; DWARFDie LastSeenDIE; - uint32_t LastSeenCompileUnitID; - uint32_t CanonicalDIEOffset; - - friend DeclMapInfo; + uint32_t LastSeenCompileUnitID = 0; + uint32_t CanonicalDIEOffset = 0; public: - typedef DenseSet<DeclContext *, DeclMapInfo> Map; + using Map = DenseSet<DeclContext *, DeclMapInfo>; - DeclContext() - : QualifiedNameHash(0), Line(0), ByteSize(0), - Tag(dwarf::DW_TAG_compile_unit), Name(), File(), Parent(*this), - LastSeenDIE(), LastSeenCompileUnitID(0), CanonicalDIEOffset(0) {} + DeclContext() : DefinedInClangModule(0), Parent(*this) {} DeclContext(unsigned Hash, uint32_t Line, uint32_t ByteSize, uint16_t Tag, StringRef Name, StringRef File, const DeclContext &Parent, DWARFDie LastSeenDIE = DWARFDie(), unsigned CUId = 0) : QualifiedNameHash(Hash), Line(Line), ByteSize(ByteSize), Tag(Tag), - Name(Name), File(File), Parent(Parent), LastSeenDIE(LastSeenDIE), - LastSeenCompileUnitID(CUId), CanonicalDIEOffset(0) {} + DefinedInClangModule(0), Name(Name), File(File), Parent(Parent), + LastSeenDIE(LastSeenDIE), LastSeenCompileUnitID(CUId) {} uint32_t getQualifiedNameHash() const { return QualifiedNameHash; } @@ -129,6 +179,9 @@ public: uint32_t getCanonicalDIEOffset() const { return CanonicalDIEOffset; } void setCanonicalDIEOffset(uint32_t Offset) { CanonicalDIEOffset = Offset; } + bool isDefinedInClangModule() const { return DefinedInClangModule; } + void setDefinedInClangModule(bool Val) { DefinedInClangModule = Val; } + uint16_t getTag() const { return Tag; } StringRef getName() const { return Name; } }; @@ -178,26 +231,41 @@ public: DeclContext &getRoot() { return Root; } }; -/// \brief Stores all information relating to a compile unit, be it in -/// its original instance in the object file to its brand new cloned -/// and linked DIE tree. +/// Stores all information relating to a compile unit, be it in its original +/// instance in the object file to its brand new cloned and linked DIE tree. class CompileUnit { public: - /// \brief Information gathered about a DIE in the object file. + /// Information gathered about a DIE in the object file. struct DIEInfo { - int64_t AddrAdjust; ///< Address offset to apply to the described entity. - DeclContext *Ctxt; ///< ODR Declaration context. - DIE *Clone; ///< Cloned version of that DIE. - uint32_t ParentIdx; ///< The index of this DIE's parent. - bool Keep : 1; ///< Is the DIE part of the linked output? - bool InDebugMap : 1;///< Was this DIE's entity found in the map? - bool Prune : 1; ///< Is this a pure forward declaration we can strip? + /// Address offset to apply to the described entity. + int64_t AddrAdjust; + + /// ODR Declaration context. + DeclContext *Ctxt; + + /// Cloned version of that DIE. + DIE *Clone; + + /// The index of this DIE's parent. + uint32_t ParentIdx; + + /// Is the DIE part of the linked output? + bool Keep : 1; + + /// Was this DIE's entity found in the map? + bool InDebugMap : 1; + + /// Is this a pure forward declaration we can strip? + bool Prune : 1; + + /// Does DIE transitively refer an incomplete decl? + bool Incomplete : 1; }; CompileUnit(DWARFUnit &OrigUnit, unsigned ID, bool CanUseODR, StringRef ClangModuleName) - : OrigUnit(OrigUnit), ID(ID), LowPc(UINT64_MAX), HighPc(0), RangeAlloc(), - Ranges(RangeAlloc), ClangModuleName(ClangModuleName) { + : OrigUnit(OrigUnit), ID(ID), Ranges(RangeAlloc), + ClangModuleName(ClangModuleName) { Info.resize(OrigUnit.getNumDIEs()); auto CUDie = OrigUnit.getUnitDIE(false); @@ -243,7 +311,9 @@ public: Optional<PatchLocation> getUnitRangesAttribute() const { return UnitRangeAttribute; } + const FunctionIntervals &getFunctionRanges() const { return Ranges; } + const std::vector<PatchLocation> &getRangesAttributes() const { return RangeAttributes; } @@ -261,41 +331,39 @@ public: /// reconstructed accelerator tables. void markEverythingAsKept(); - /// \brief Compute the end offset for this unit. Must be - /// called after the CU's DIEs have been cloned. - /// \returns the next unit offset (which is also the current - /// debug_info section size). + /// Compute the end offset for this unit. Must be called after the CU's DIEs + /// have been cloned. \returns the next unit offset (which is also the + /// current debug_info section size). uint64_t computeNextUnitOffset(); - /// \brief Keep track of a forward reference to DIE \p Die in \p - /// RefUnit by \p Attr. The attribute should be fixed up later to - /// point to the absolute offset of \p Die in the debug_info section - /// or to the canonical offset of \p Ctxt if it is non-null. + /// Keep track of a forward reference to DIE \p Die in \p RefUnit by \p + /// Attr. The attribute should be fixed up later to point to the absolute + /// offset of \p Die in the debug_info section or to the canonical offset of + /// \p Ctxt if it is non-null. void noteForwardReference(DIE *Die, const CompileUnit *RefUnit, DeclContext *Ctxt, PatchLocation Attr); - /// \brief Apply all fixups recored by noteForwardReference(). + /// Apply all fixups recored by noteForwardReference(). void fixupForwardReferences(); - /// \brief Add a function range [\p LowPC, \p HighPC) that is - /// relocatad by applying offset \p PCOffset. + /// Add a function range [\p LowPC, \p HighPC) that is relocatad by applying + /// offset \p PCOffset. void addFunctionRange(uint64_t LowPC, uint64_t HighPC, int64_t PCOffset); - /// \brief Keep track of a DW_AT_range attribute that we will need to - /// patch up later. + /// Keep track of a DW_AT_range attribute that we will need to patch up later. void noteRangeAttribute(const DIE &Die, PatchLocation Attr); - /// \brief Keep track of a location attribute pointing to a location - /// list in the debug_loc section. + /// Keep track of a location attribute pointing to a location list in the + /// debug_loc section. void noteLocationAttribute(PatchLocation Attr, int64_t PcOffset); - /// \brief Add a name accelerator entry for \p Die with \p Name - /// which is stored in the string table at \p Offset. + /// Add a name accelerator entry for \p Die with \p Name which is stored in + /// the string table at \p Offset. void addNameAccelerator(const DIE *Die, const char *Name, uint32_t Offset, bool SkipPubnamesSection = false); - /// \brief Add a type accelerator entry for \p Die with \p Name - /// which is stored in the string table at \p Offset. + /// Add a type accelerator entry for \p Die with \p Name which is stored in + /// the string table at \p Offset. void addTypeAccelerator(const DIE *Die, const char *Name, uint32_t Offset); struct AccelInfo { @@ -337,10 +405,10 @@ private: uint64_t StartOffset; uint64_t NextUnitOffset; - uint64_t LowPc; - uint64_t HighPc; + uint64_t LowPc = std::numeric_limits<uint64_t>::max(); + uint64_t HighPc = 0; - /// \brief A list of attributes to fixup with the absolute offset of + /// A list of attributes to fixup with the absolute offset of /// a DIE in the debug_info section. /// /// The offsets for the attributes in this array couldn't be set while @@ -350,25 +418,26 @@ private: PatchLocation>> ForwardDIEReferences; FunctionIntervals::Allocator RangeAlloc; - /// \brief The ranges in that interval map are the PC ranges for + + /// The ranges in that interval map are the PC ranges for /// functions in this unit, associated with the PC offset to apply /// to the addresses to get the linked address. FunctionIntervals Ranges; - /// \brief DW_AT_ranges attributes to patch after we have gathered + /// DW_AT_ranges attributes to patch after we have gathered /// all the unit's function addresses. /// @{ std::vector<PatchLocation> RangeAttributes; Optional<PatchLocation> UnitRangeAttribute; /// @} - /// \brief Location attributes that need to be transferred from the + /// Location attributes that need to be transferred from the /// original debug_loc section to the liked one. They are stored /// along with the PC offset that is to be applied to their /// function's address. std::vector<std::pair<PatchLocation, int64_t>> LocationAttributes; - /// \brief Accelerator entries for the unit, both for the pub* + /// Accelerator entries for the unit, both for the pub* /// sections and the apple* ones. /// @{ std::vector<AccelInfo> Pubnames; @@ -383,12 +452,16 @@ private: /// Is this unit subject to the ODR rule? bool HasODR; + /// Did a DIE actually contain a valid reloc? bool HasInterestingContent; + /// If this is a Clang module, this holds the module's name. std::string ClangModuleName; }; +} // end anonymous namespace + void CompileUnit::markEverythingAsKept() { for (auto &I : Info) // Mark everything that wasn't explicity marked for pruning. @@ -405,14 +478,14 @@ uint64_t CompileUnit::computeNextUnitOffset() { return NextUnitOffset; } -/// \brief Keep track of a forward cross-cu reference from this unit +/// Keep track of a forward cross-cu reference from this unit /// to \p Die that lives in \p RefUnit. void CompileUnit::noteForwardReference(DIE *Die, const CompileUnit *RefUnit, DeclContext *Ctxt, PatchLocation Attr) { ForwardDIEReferences.emplace_back(Die, RefUnit, Ctxt, Attr); } -/// \brief Apply all fixups recorded by noteForwardReference(). +/// Apply all fixups recorded by noteForwardReference(). void CompileUnit::fixupForwardReferences() { for (const auto &Ref : ForwardDIEReferences) { DIE *RefDie; @@ -445,21 +518,23 @@ void CompileUnit::noteLocationAttribute(PatchLocation Attr, int64_t PcOffset) { LocationAttributes.emplace_back(Attr, PcOffset); } -/// \brief Add a name accelerator entry for \p Die with \p Name +/// Add a name accelerator entry for \p Die with \p Name /// which is stored in the string table at \p Offset. void CompileUnit::addNameAccelerator(const DIE *Die, const char *Name, uint32_t Offset, bool SkipPubSection) { Pubnames.emplace_back(Name, Die, Offset, SkipPubSection); } -/// \brief Add a type accelerator entry for \p Die with \p Name +/// Add a type accelerator entry for \p Die with \p Name /// which is stored in the string table at \p Offset. void CompileUnit::addTypeAccelerator(const DIE *Die, const char *Name, uint32_t Offset) { Pubtypes.emplace_back(Name, Die, Offset, false); } -/// \brief The Dwarf streaming logic +namespace { + +/// The Dwarf streaming logic /// /// All interactions with the MC layer that is used to build the debug /// information binary representation are handled in this class. @@ -479,55 +554,54 @@ class DwarfStreamer { std::unique_ptr<AsmPrinter> Asm; /// @} - /// \brief the file we stream the linked Dwarf to. - std::unique_ptr<raw_fd_ostream> OutFile; + /// The file we stream the linked Dwarf to. + raw_fd_ostream &OutFile; uint32_t RangesSectionSize; uint32_t LocSectionSize; uint32_t LineSectionSize; uint32_t FrameSectionSize; - /// \brief Emit the pubnames or pubtypes section contribution for \p + /// Emit the pubnames or pubtypes section contribution for \p /// Unit into \p Sec. The data is provided in \p Names. void emitPubSectionForUnit(MCSection *Sec, StringRef Name, const CompileUnit &Unit, const std::vector<CompileUnit::AccelInfo> &Names); public: - /// \brief Actually create the streamer and the ouptut file. - /// - /// This could be done directly in the constructor, but it feels - /// more natural to handle errors through return value. - bool init(Triple TheTriple, StringRef OutputFilename); + DwarfStreamer(raw_fd_ostream &OutFile) : OutFile(OutFile) {} + bool init(Triple TheTriple); - /// \brief Dump the file to the disk. + /// Dump the file to the disk. bool finish(const DebugMap &); AsmPrinter &getAsmPrinter() const { return *Asm; } - /// \brief Set the current output section to debug_info and change + /// Set the current output section to debug_info and change /// the MC Dwarf version to \p DwarfVersion. void switchToDebugInfoSection(unsigned DwarfVersion); - /// \brief Emit the compilation unit header for \p Unit in the + /// Emit the compilation unit header for \p Unit in the /// debug_info section. /// /// As a side effect, this also switches the current Dwarf version /// of the MC layer to the one of U.getOrigUnit(). void emitCompileUnitHeader(CompileUnit &Unit); - /// \brief Recursively emit the DIE tree rooted at \p Die. + /// Recursively emit the DIE tree rooted at \p Die. void emitDIE(DIE &Die); - /// \brief Emit the abbreviation table \p Abbrevs to the - /// debug_abbrev section. + /// Emit the abbreviation table \p Abbrevs to the debug_abbrev section. void emitAbbrevs(const std::vector<std::unique_ptr<DIEAbbrev>> &Abbrevs, unsigned DwarfVersion); - /// \brief Emit the string table described by \p Pool. + /// Emit the string table described by \p Pool. void emitStrings(const NonRelocatableStringpool &Pool); - /// \brief Emit debug_ranges for \p FuncRange by translating the + /// Emit the swift_ast section stored in \p Buffer. + void emitSwiftAST(StringRef Buffer); + + /// Emit debug_ranges for \p FuncRange by translating the /// original \p Entries. void emitRangesEntries( int64_t UnitPcOffset, uint64_t OrigLowPc, @@ -535,20 +609,19 @@ public: const std::vector<DWARFDebugRangeList::RangeListEntry> &Entries, unsigned AddressSize); - /// \brief Emit debug_aranges entries for \p Unit and if \p - /// DoRangesSection is true, also emit the debug_ranges entries for - /// the DW_TAG_compile_unit's DW_AT_ranges attribute. + /// Emit debug_aranges entries for \p Unit and if \p DoRangesSection is true, + /// also emit the debug_ranges entries for the DW_TAG_compile_unit's + /// DW_AT_ranges attribute. void emitUnitRangesEntries(CompileUnit &Unit, bool DoRangesSection); uint32_t getRangesSectionSize() const { return RangesSectionSize; } - /// \brief Emit the debug_loc contribution for \p Unit by copying - /// the entries from \p Dwarf and offseting them. Update the - /// location attributes to point to the new entries. + /// Emit the debug_loc contribution for \p Unit by copying the entries from \p + /// Dwarf and offseting them. Update the location attributes to point to the + /// new entries. void emitLocationsForUnit(const CompileUnit &Unit, DWARFContext &Dwarf); - /// \brief Emit the line table described in \p Rows into the - /// debug_line section. + /// Emit the line table described in \p Rows into the debug_line section. void emitLineTableForUnit(MCDwarfLineTableParams Params, StringRef PrologueBytes, unsigned MinInstLength, std::vector<DWARFDebugLine::Row> &Rows, @@ -556,23 +629,25 @@ public: uint32_t getLineSectionSize() const { return LineSectionSize; } - /// \brief Emit the .debug_pubnames contribution for \p Unit. + /// Emit the .debug_pubnames contribution for \p Unit. void emitPubNamesForUnit(const CompileUnit &Unit); - /// \brief Emit the .debug_pubtypes contribution for \p Unit. + /// Emit the .debug_pubtypes contribution for \p Unit. void emitPubTypesForUnit(const CompileUnit &Unit); - /// \brief Emit a CIE. + /// Emit a CIE. void emitCIE(StringRef CIEBytes); - /// \brief Emit an FDE with data \p Bytes. + /// Emit an FDE with data \p Bytes. void emitFDE(uint32_t CIEOffset, uint32_t AddreSize, uint32_t Address, StringRef Bytes); uint32_t getFrameSectionSize() const { return FrameSectionSize; } }; -bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) { +} // end anonymous namespace + +bool DwarfStreamer::init(Triple TheTriple) { std::string ErrorStr; std::string TripleName; StringRef Context = "dwarf streamer init"; @@ -595,7 +670,7 @@ bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) { MOFI.reset(new MCObjectFileInfo); MC.reset(new MCContext(MAI.get(), MRI.get(), MOFI.get())); - MOFI->InitMCObjectFileInfo(TheTriple, /*PIC*/ false, CodeModel::Default, *MC); + MOFI->InitMCObjectFileInfo(TheTriple, /*PIC*/ false, *MC); MCTargetOptions Options; MAB = TheTarget->createMCAsmBackend(*MRI, TripleName, "", Options); @@ -614,16 +689,10 @@ bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) { if (!MCE) return error("no code emitter for target " + TripleName, Context); - // Create the output file. - std::error_code EC; - OutFile = - llvm::make_unique<raw_fd_ostream>(OutputFilename, EC, sys::fs::F_None); - if (EC) - return error(Twine(OutputFilename) + ": " + EC.message(), Context); - MCTargetOptions MCOptions = InitMCTargetOptionsFromFlags(); MS = TheTarget->createMCObjectStreamer( - TheTriple, *MC, *MAB, *OutFile, MCE, *MSTI, MCOptions.MCRelaxAll, + TheTriple, *MC, std::unique_ptr<MCAsmBackend>(MAB), OutFile, + std::unique_ptr<MCCodeEmitter>(MCE), *MSTI, MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible, /*DWARFMustBeAtTheEnd*/ false); if (!MS) @@ -648,22 +717,22 @@ bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) { } bool DwarfStreamer::finish(const DebugMap &DM) { + bool Result = true; if (DM.getTriple().isOSDarwin() && !DM.getBinaryPath().empty()) - return MachOUtils::generateDsymCompanion(DM, *MS, *OutFile); - - MS->Finish(); - return true; + Result = MachOUtils::generateDsymCompanion(DM, *MS, OutFile); + else + MS->Finish(); + return Result; } -/// \brief Set the current output section to debug_info and change +/// Set the current output section to debug_info and change /// the MC Dwarf version to \p DwarfVersion. void DwarfStreamer::switchToDebugInfoSection(unsigned DwarfVersion) { MS->SwitchSection(MOFI->getDwarfInfoSection()); MC->setDwarfVersion(DwarfVersion); } -/// \brief Emit the compilation unit header for \p Unit in the -/// debug_info section. +/// Emit the compilation unit header for \p Unit in the debug_info section. /// /// A Dwarf scetion header is encoded as: /// uint32_t Unit length (omiting this field) @@ -687,7 +756,7 @@ void DwarfStreamer::emitCompileUnitHeader(CompileUnit &Unit) { Asm->EmitInt8(Unit.getOrigUnit().getAddressByteSize()); } -/// \brief Emit the \p Abbrevs array as the shared abbreviation table +/// Emit the \p Abbrevs array as the shared abbreviation table /// for the linked Dwarf file. void DwarfStreamer::emitAbbrevs( const std::vector<std::unique_ptr<DIEAbbrev>> &Abbrevs, @@ -697,13 +766,13 @@ void DwarfStreamer::emitAbbrevs( Asm->emitDwarfAbbrevs(Abbrevs); } -/// \brief Recursively emit the DIE tree rooted at \p Die. +/// Recursively emit the DIE tree rooted at \p Die. void DwarfStreamer::emitDIE(DIE &Die) { MS->SwitchSection(MOFI->getDwarfInfoSection()); Asm->emitDwarfDIE(Die); } -/// \brief Emit the debug_str section stored in \p Pool. +/// Emit the debug_str section stored in \p Pool. void DwarfStreamer::emitStrings(const NonRelocatableStringpool &Pool) { Asm->OutStreamer->SwitchSection(MOFI->getDwarfStrSection()); for (auto *Entry = Pool.getFirstEntry(); Entry; @@ -712,7 +781,15 @@ void DwarfStreamer::emitStrings(const NonRelocatableStringpool &Pool) { StringRef(Entry->getKey().data(), Entry->getKey().size() + 1)); } -/// \brief Emit the debug_range section contents for \p FuncRange by +/// Emit the swift_ast section stored in \p Buffers. +void DwarfStreamer::emitSwiftAST(StringRef Buffer) { + MCSection *SwiftASTSection = MOFI->getDwarfSwiftASTSection(); + SwiftASTSection->setAlignment(1 << 5); + MS->SwitchSection(SwiftASTSection); + MS->EmitBytes(Buffer); +} + +/// Emit the debug_range section contents for \p FuncRange by /// translating the original \p Entries. The debug_range section /// format is totally trivial, consisting just of pairs of address /// sized addresses describing the ranges. @@ -750,7 +827,7 @@ void DwarfStreamer::emitRangesEntries( RangesSectionSize += 2 * AddressSize; } -/// \brief Emit the debug_aranges contribution of a unit and +/// Emit the debug_aranges contribution of a unit and /// if \p DoDebugRanges is true the debug_range contents for a /// compile_unit level DW_AT_ranges attribute (Which are basically the /// same thing with a different base address). @@ -833,7 +910,7 @@ void DwarfStreamer::emitUnitRangesEntries(CompileUnit &Unit, RangesSectionSize += 2 * AddressSize; } -/// \brief Emit location lists for \p Unit and update attribtues to +/// Emit location lists for \p Unit and update attribtues to /// point to the new entries. void DwarfStreamer::emitLocationsForUnit(const CompileUnit &Unit, DWARFContext &Dwarf) { @@ -845,7 +922,7 @@ void DwarfStreamer::emitLocationsForUnit(const CompileUnit &Unit, MS->SwitchSection(MC->getObjectFileInfo()->getDwarfLocSection()); unsigned AddressSize = Unit.getOrigUnit().getAddressByteSize(); - const DWARFSection &InputSec = Dwarf.getLocSection(); + const DWARFSection &InputSec = Dwarf.getDWARFObj().getLocSection(); DataExtractor Data(InputSec.Data, Dwarf.isLittleEndian(), AddressSize); DWARFUnit &OrigUnit = Unit.getOrigUnit(); auto OrigUnitDie = OrigUnit.getUnitDIE(false); @@ -905,7 +982,8 @@ void DwarfStreamer::emitLineTableForUnit(MCDwarfLineTableParams Params, if (Rows.empty()) { // We only have the dummy entry, dsymutil emits an entry with a 0 // address in that case. - MCDwarfLineAddr::Encode(*MC, Params, INT64_MAX, 0, EncodingOS); + MCDwarfLineAddr::Encode(*MC, Params, std::numeric_limits<int64_t>::max(), 0, + EncodingOS); MS->EmitBytes(EncodingOS.str()); LineSectionSize += EncodingBuffer.size(); MS->EmitLabel(LineEndSym); @@ -1005,7 +1083,8 @@ void DwarfStreamer::emitLineTableForUnit(MCDwarfLineTableParams Params, MS->EmitULEB128IntValue(AddressDelta); LineSectionSize += 1 + getULEB128Size(AddressDelta); } - MCDwarfLineAddr::Encode(*MC, Params, INT64_MAX, 0, EncodingOS); + MCDwarfLineAddr::Encode(*MC, Params, std::numeric_limits<int64_t>::max(), + 0, EncodingOS); MS->EmitBytes(EncodingOS.str()); LineSectionSize += EncodingBuffer.size(); EncodingBuffer.resize(0); @@ -1016,7 +1095,8 @@ void DwarfStreamer::emitLineTableForUnit(MCDwarfLineTableParams Params, } if (RowsSinceLastSequence) { - MCDwarfLineAddr::Encode(*MC, Params, INT64_MAX, 0, EncodingOS); + MCDwarfLineAddr::Encode(*MC, Params, std::numeric_limits<int64_t>::max(), 0, + EncodingOS); MS->EmitBytes(EncodingOS.str()); LineSectionSize += EncodingBuffer.size(); EncodingBuffer.resize(0); @@ -1025,7 +1105,7 @@ void DwarfStreamer::emitLineTableForUnit(MCDwarfLineTableParams Params, MS->EmitLabel(LineEndSym); } -/// \brief Emit the pubnames or pubtypes section contribution for \p +/// Emit the pubnames or pubtypes section contribution for \p /// Unit into \p Sec. The data is provided in \p Names. void DwarfStreamer::emitPubSectionForUnit( MCSection *Sec, StringRef SecName, const CompileUnit &Unit, @@ -1064,19 +1144,19 @@ void DwarfStreamer::emitPubSectionForUnit( Asm->OutStreamer->EmitLabel(EndLabel); } -/// \brief Emit .debug_pubnames for \p Unit. +/// Emit .debug_pubnames for \p Unit. void DwarfStreamer::emitPubNamesForUnit(const CompileUnit &Unit) { emitPubSectionForUnit(MC->getObjectFileInfo()->getDwarfPubNamesSection(), "names", Unit, Unit.getPubnames()); } -/// \brief Emit .debug_pubtypes for \p Unit. +/// Emit .debug_pubtypes for \p Unit. void DwarfStreamer::emitPubTypesForUnit(const CompileUnit &Unit) { emitPubSectionForUnit(MC->getObjectFileInfo()->getDwarfPubTypesSection(), "types", Unit, Unit.getPubtypes()); } -/// \brief Emit a CIE into the debug_frame section. +/// Emit a CIE into the debug_frame section. void DwarfStreamer::emitCIE(StringRef CIEBytes) { MS->SwitchSection(MC->getObjectFileInfo()->getDwarfFrameSection()); @@ -1084,7 +1164,7 @@ void DwarfStreamer::emitCIE(StringRef CIEBytes) { FrameSectionSize += CIEBytes.size(); } -/// \brief Emit a FDE into the debug_frame section. \p FDEBytes +/// Emit a FDE into the debug_frame section. \p FDEBytes /// contains the FDE data without the length, CIE offset and address /// which will be replaced with the parameter values. void DwarfStreamer::emitFDE(uint32_t CIEOffset, uint32_t AddrSize, @@ -1098,7 +1178,9 @@ void DwarfStreamer::emitFDE(uint32_t CIEOffset, uint32_t AddrSize, FrameSectionSize += FDEBytes.size() + 8 + AddrSize; } -/// \brief The core of the Dwarf linking logic. +namespace { + +/// The core of the Dwarf linking logic. /// /// The link of the dwarf information from the object files will be /// driven by the selection of 'root DIEs', which are DIEs that @@ -1114,21 +1196,20 @@ void DwarfStreamer::emitFDE(uint32_t CIEOffset, uint32_t AddrSize, /// first step when we start processing a DebugMapObject. class DwarfLinker { public: - DwarfLinker(StringRef OutputFilename, const LinkOptions &Options) - : OutputFilename(OutputFilename), Options(Options), - BinHolder(Options.Verbose), LastCIEOffset(0) {} + DwarfLinker(raw_fd_ostream &OutFile, const LinkOptions &Options) + : OutFile(OutFile), Options(Options), BinHolder(Options.Verbose) {} - /// \brief Link the contents of the DebugMap. + /// Link the contents of the DebugMap. bool link(const DebugMap &); void reportWarning(const Twine &Warning, const DWARFDie *DIE = nullptr) const; private: - /// \brief Called at the start of a debug object link. + /// Called at the start of a debug object link. void startDebugObject(DWARFContext &, DebugMapObject &); - /// \brief Called at the end of a debug object link. + /// Called at the end of a debug object link. void endDebugObject(); /// Remembers the newest DWARF version we've seen in a unit. @@ -1156,23 +1237,23 @@ private: DwarfLinker &Linker; - /// \brief The valid relocations for the current DebugMapObject. + /// The valid relocations for the current DebugMapObject. /// This vector is sorted by relocation offset. std::vector<ValidReloc> ValidRelocs; - /// \brief Index into ValidRelocs of the next relocation to + /// Index into ValidRelocs of the next relocation to /// consider. As we walk the DIEs in acsending file offset and as /// ValidRelocs is sorted by file offset, keeping this index /// uptodate is all we have to do to have a cheap lookup during the /// root DIE selection and during DIE cloning. - unsigned NextValidReloc; + unsigned NextValidReloc = 0; public: - RelocationManager(DwarfLinker &Linker) - : Linker(Linker), NextValidReloc(0) {} + RelocationManager(DwarfLinker &Linker) : Linker(Linker) {} bool hasValidRelocs() const { return !ValidRelocs.empty(); } - /// \brief Reset the NextValidReloc counter. + + /// Reset the NextValidReloc counter. void resetValidRelocs() { NextValidReloc = 0; } /// \defgroup FindValidRelocations Translate debug map into a list @@ -1201,10 +1282,11 @@ private: /// \defgroup FindRootDIEs Find DIEs corresponding to debug map entries. /// /// @{ - /// \brief Recursively walk the \p DIE tree and look for DIEs to + /// Recursively walk the \p DIE tree and look for DIEs to /// keep. Store that information in \p CU's DIEInfo. - void lookForDIEsToKeep(RelocationManager &RelocMgr, - const DWARFDie &DIE, + /// + /// The return value indicates whether the DIE is incomplete. + bool lookForDIEsToKeep(RelocationManager &RelocMgr, const DWARFDie &DIE, const DebugMapObject &DMO, CompileUnit &CU, unsigned Flags); @@ -1221,11 +1303,11 @@ private: /// Recursively add the debug info in this clang module .pcm /// file (and all the modules imported by it in a bottom-up fashion) /// to Units. - void loadClangModule(StringRef Filename, StringRef ModulePath, - StringRef ModuleName, uint64_t DwoId, - DebugMap &ModuleMap, unsigned Indent = 0); + Error loadClangModule(StringRef Filename, StringRef ModulePath, + StringRef ModuleName, uint64_t DwoId, + DebugMap &ModuleMap, unsigned Indent = 0); - /// \brief Flags passed to DwarfLinker::lookForDIEsToKeep + /// Flags passed to DwarfLinker::lookForDIEsToKeep enum TravesalFlags { TF_Keep = 1 << 0, ///< Mark the traversed DIEs as kept. TF_InFunctionScope = 1 << 1, ///< Current scope is a fucntion scope. @@ -1235,8 +1317,7 @@ private: TF_SkipPC = 1 << 5, ///< Skip all location attributes. }; - /// \brief Mark the passed DIE as well as all the ones it depends on - /// as kept. + /// Mark the passed DIE as well as all the ones it depends on as kept. void keepDIEAndDependencies(RelocationManager &RelocMgr, const DWARFDie &DIE, CompileUnit::DIEInfo &MyInfo, @@ -1270,8 +1351,10 @@ private: class DIECloner { DwarfLinker &Linker; RelocationManager &RelocMgr; + /// Allocator used for all the DIEValue objects. BumpPtrAllocator &DIEAlloc; + std::vector<std::unique_ptr<CompileUnit>> &CompileUnits; LinkOptions Options; @@ -1301,28 +1384,38 @@ private: /// Construct the output DIE tree by cloning the DIEs we /// chose to keep above. If there are no valid relocs, then there's /// nothing to clone/emit. - void cloneAllCompileUnits(DWARFContextInMemory &DwarfContext); + void cloneAllCompileUnits(DWARFContext &DwarfContext); private: - typedef DWARFAbbreviationDeclaration::AttributeSpec AttributeSpec; + using AttributeSpec = DWARFAbbreviationDeclaration::AttributeSpec; /// Information gathered and exchanged between the various /// clone*Attributes helpers about the attributes of a particular DIE. struct AttributesInfo { - const char *Name, *MangledName; ///< Names. - uint32_t NameOffset, MangledNameOffset; ///< Offsets in the string pool. + /// Names. + const char *Name = nullptr; + const char *MangledName = nullptr; + + /// Offsets in the string pool. + uint32_t NameOffset = 0; + uint32_t MangledNameOffset = 0; - uint64_t OrigLowPc; ///< Value of AT_low_pc in the input DIE - uint64_t OrigHighPc; ///< Value of AT_high_pc in the input DIE - int64_t PCOffset; ///< Offset to apply to PC addresses inside a function. + /// Value of AT_low_pc in the input DIE + uint64_t OrigLowPc = std::numeric_limits<uint64_t>::max(); - bool HasLowPc; ///< Does the DIE have a low_pc attribute? - bool IsDeclaration; ///< Is this DIE only a declaration? + /// Value of AT_high_pc in the input DIE + uint64_t OrigHighPc = 0; - AttributesInfo() - : Name(nullptr), MangledName(nullptr), NameOffset(0), - MangledNameOffset(0), OrigLowPc(UINT64_MAX), OrigHighPc(0), - PCOffset(0), HasLowPc(false), IsDeclaration(false) {} + /// Offset to apply to PC addresses inside a function. + int64_t PCOffset = 0; + + /// Does the DIE have a low_pc attribute? + bool HasLowPc = false; + + /// Is this DIE only a declaration? + bool IsDeclaration = false; + + AttributesInfo() = default; }; /// Helper for cloneDIE. @@ -1380,41 +1473,44 @@ private: void copyAbbrev(const DWARFAbbreviationDeclaration &Abbrev, bool hasODR); }; - /// \brief Assign an abbreviation number to \p Abbrev + /// Assign an abbreviation number to \p Abbrev void AssignAbbrev(DIEAbbrev &Abbrev); - /// \brief FoldingSet that uniques the abbreviations. - FoldingSet<DIEAbbrev> AbbreviationsSet; - /// \brief Storage for the unique Abbreviations. - /// This is passed to AsmPrinter::emitDwarfAbbrevs(), thus it cannot - /// be changed to a vecot of unique_ptrs. - std::vector<std::unique_ptr<DIEAbbrev>> Abbreviations; - - /// \brief Compute and emit debug_ranges section for \p Unit, and + /// Compute and emit debug_ranges section for \p Unit, and /// patch the attributes referencing it. void patchRangesForUnit(const CompileUnit &Unit, DWARFContext &Dwarf) const; - /// \brief Generate and emit the DW_AT_ranges attribute for a + /// Generate and emit the DW_AT_ranges attribute for a /// compile_unit if it had one. void generateUnitRanges(CompileUnit &Unit) const; - /// \brief Extract the line tables fromt he original dwarf, extract + /// Extract the line tables fromt he original dwarf, extract /// the relevant parts according to the linked function ranges and /// emit the result in the debug_line section. void patchLineTableForUnit(CompileUnit &Unit, DWARFContext &OrigDwarf); - /// \brief Emit the accelerator entries for \p Unit. + /// Emit the accelerator entries for \p Unit. void emitAcceleratorEntriesForUnit(CompileUnit &Unit); - /// \brief Patch the frame info for an object file and emit it. + /// Patch the frame info for an object file and emit it. void patchFrameInfoForObject(const DebugMapObject &, DWARFContext &, unsigned AddressSize); - /// \brief DIELoc objects that need to be destructed (but not freed!). + /// FoldingSet that uniques the abbreviations. + FoldingSet<DIEAbbrev> AbbreviationsSet; + + /// Storage for the unique Abbreviations. + /// This is passed to AsmPrinter::emitDwarfAbbrevs(), thus it cannot + /// be changed to a vecot of unique_ptrs. + std::vector<std::unique_ptr<DIEAbbrev>> Abbreviations; + + /// DIELoc objects that need to be destructed (but not freed!). std::vector<DIELoc *> DIELocs; - /// \brief DIEBlock objects that need to be destructed (but not freed!). + + /// DIEBlock objects that need to be destructed (but not freed!). std::vector<DIEBlock *> DIEBlocks; - /// \brief Allocator used for all the DIEValue objects. + + /// Allocator used for all the DIEValue objects. BumpPtrAllocator DIEAlloc; /// @} @@ -1424,33 +1520,35 @@ private: /// \defgroup Helpers Various helper methods. /// /// @{ - bool createStreamer(const Triple &TheTriple, StringRef OutputFilename); + bool createStreamer(const Triple &TheTriple, raw_fd_ostream &OutFile); - /// \brief Attempt to load a debug object from disk. + /// Attempt to load a debug object from disk. ErrorOr<const object::ObjectFile &> loadObject(BinaryHolder &BinaryHolder, DebugMapObject &Obj, const DebugMap &Map); /// @} - std::string OutputFilename; + raw_fd_ostream &OutFile; LinkOptions Options; BinaryHolder BinHolder; std::unique_ptr<DwarfStreamer> Streamer; uint64_t OutputDebugInfoSize; - unsigned UnitID; ///< A unique ID that identifies each compile unit. + + /// A unique ID that identifies each compile unit. + unsigned UnitID; + unsigned MaxDwarfVersion = 0; /// The units of the current debug map object. std::vector<std::unique_ptr<CompileUnit>> Units; - /// The debug map object currently under consideration. DebugMapObject *CurrentDebugObject; - /// \brief The Dwarf string pool + /// The Dwarf string pool. NonRelocatableStringpool StringPool; - /// \brief This map is keyed by the entry PC of functions in that + /// This map is keyed by the entry PC of functions in that /// debug object and the associated value is a pair storing the /// corresponding end PC and the offset to apply to get the linked /// address. @@ -1458,7 +1556,7 @@ private: /// See startDebugObject() for a more complete description of its use. std::map<uint64_t, std::pair<uint64_t, int64_t>> Ranges; - /// \brief The CIEs that have been emitted in the output + /// The CIEs that have been emitted in the output /// section. The actual CIE data serves a the key to this StringMap, /// this takes care of comparing the semantics of CIEs defined in /// different object files. @@ -1466,7 +1564,7 @@ private: /// Offset of the last CIE that has been emitted in the output /// debug_frame section. - uint32_t LastCIEOffset; + uint32_t LastCIEOffset = 0; /// Mapping the PCM filename to the DwoId. StringMap<uint64_t> ClangModules; @@ -1475,6 +1573,8 @@ private: bool ArchiveHintDisplayed = false; }; +} // end anonymous namespace + /// Similar to DWARFUnitSection::getUnitForOffset(), but returning our /// CompileUnit object instead. static CompileUnit *getUnitForOffset( @@ -1499,8 +1599,12 @@ static DWARFDie resolveDIEReference( uint64_t RefOffset = *RefValue.getAsReference(); if ((RefCU = getUnitForOffset(Units, RefOffset))) - if (const auto RefDie = RefCU->getOrigUnit().getDIEForOffset(RefOffset)) - return RefDie; + if (const auto RefDie = RefCU->getOrigUnit().getDIEForOffset(RefOffset)) { + // In a file with broken references, an attribute might point to a NULL + // DIE. + if(!RefDie.isNULL()) + return RefDie; + } Linker.reportWarning("could not find referenced DIE", &DIE); return DWARFDie(); @@ -1615,7 +1719,7 @@ PointerIntPair<DeclContext *, 1> DeclContextTree::getChildDeclContext( return PointerIntPair<DeclContext *, 1>(nullptr); unsigned Line = 0; - unsigned ByteSize = UINT32_MAX; + unsigned ByteSize = std::numeric_limits<uint32_t>::max(); if (!InClangModule) { // Gather some discriminating data about the DeclContext we will be @@ -1625,7 +1729,8 @@ PointerIntPair<DeclContext *, 1> DeclContextTree::getChildDeclContext( // namespaces, use these additional data points to make the process // safer. This is disabled for clang modules, because forward // declarations of module-defined types do not have a file and line. - ByteSize = dwarf::toUnsigned(DIE.find(dwarf::DW_AT_byte_size), UINT64_MAX); + ByteSize = dwarf::toUnsigned(DIE.find(dwarf::DW_AT_byte_size), + std::numeric_limits<uint64_t>::max()); if (Tag != dwarf::DW_TAG_namespace || !Name) { if (unsigned FileNum = dwarf::toUnsigned(DIE.find(dwarf::DW_AT_decl_file), 0)) { if (const auto *LT = U.getOrigUnit().getContext().getLineTableForUnit( @@ -1735,7 +1840,7 @@ bool DwarfLinker::DIECloner::getDIENames(const DWARFDie &Die, return Info.Name || Info.MangledName; } -/// \brief Report a warning to the user, optionaly including +/// Report a warning to the user, optionaly including /// information about a specific \p DIE related to the warning. void DwarfLinker::reportWarning(const Twine &Warning, const DWARFDie *DIE) const { @@ -1747,17 +1852,21 @@ void DwarfLinker::reportWarning(const Twine &Warning, if (!Options.Verbose || !DIE) return; + DIDumpOptions DumpOpts; + DumpOpts.RecurseDepth = 0; + DumpOpts.Verbose = Options.Verbose; + errs() << " in DIE:\n"; - DIE->dump(errs(), 0 /* RecurseDepth */, 6 /* Indent */); + DIE->dump(errs(), 6 /* Indent */, DumpOpts); } bool DwarfLinker::createStreamer(const Triple &TheTriple, - StringRef OutputFilename) { + raw_fd_ostream &OutFile) { if (Options.NoOutput) return true; - Streamer = llvm::make_unique<DwarfStreamer>(); - return Streamer->init(TheTriple, OutputFilename); + Streamer = llvm::make_unique<DwarfStreamer>(OutFile); + return Streamer->init(TheTriple); } /// Recursive helper to build the global DeclContext information and @@ -1788,7 +1897,8 @@ static bool analyzeContextInfo(const DWARFDie &DIE, // // We treat non-C++ modules like namespaces for this reason. if (DIE.getTag() == dwarf::DW_TAG_module && ParentIdx == 0 && - dwarf::toString(DIE.find(dwarf::DW_AT_name), "") != CU.getClangModuleName()) { + dwarf::toString(DIE.find(dwarf::DW_AT_name), "") != + CU.getClangModuleName()) { InImportedModule = true; } @@ -1801,6 +1911,8 @@ static bool analyzeContextInfo(const DWARFDie &DIE, CurrentDeclContext = PtrInvalidPair.getPointer(); Info.Ctxt = PtrInvalidPair.getInt() ? nullptr : PtrInvalidPair.getPointer(); + if (Info.Ctxt) + Info.Ctxt->setDefinedInClangModule(InClangModule); } else Info.Ctxt = CurrentDeclContext = nullptr; } @@ -1897,7 +2009,7 @@ static bool isMachOPairedReloc(uint64_t RelocType, uint64_t Arch) { } } -/// \brief Iterate over the relocations of the given \p Section and +/// Iterate over the relocations of the given \p Section and /// store the ones that correspond to debug map entries into the /// ValidRelocs array. void DwarfLinker::RelocationManager:: @@ -1967,7 +2079,7 @@ findValidRelocsMachO(const object::SectionRef &Section, } } -/// \brief Dispatch the valid relocation finding logic to the +/// Dispatch the valid relocation finding logic to the /// appropriate handler depending on the object file format. bool DwarfLinker::RelocationManager::findValidRelocs( const object::SectionRef &Section, const object::ObjectFile &Obj, @@ -1990,7 +2102,7 @@ bool DwarfLinker::RelocationManager::findValidRelocs( return true; } -/// \brief Look for relocations in the debug_info section that match +/// Look for relocations in the debug_info section that match /// entries in the debug map. These relocations will drive the Dwarf /// link by indicating which DIEs refer to symbols present in the /// linked binary. @@ -2010,7 +2122,7 @@ findValidRelocsInDebugInfo(const object::ObjectFile &Obj, return false; } -/// \brief Checks that there is a relocation against an actual debug +/// Checks that there is a relocation against an actual debug /// map entry between \p StartOffset and \p NextOffset. /// /// This function must be called with offsets in strictly ascending @@ -2038,8 +2150,9 @@ hasValidRelocation(uint32_t StartOffset, uint32_t EndOffset, const auto &ValidReloc = ValidRelocs[NextValidReloc++]; const auto &Mapping = ValidReloc.Mapping->getValue(); - uint64_t ObjectAddress = - Mapping.ObjectAddress ? uint64_t(*Mapping.ObjectAddress) : UINT64_MAX; + uint64_t ObjectAddress = Mapping.ObjectAddress + ? uint64_t(*Mapping.ObjectAddress) + : std::numeric_limits<uint64_t>::max(); if (Linker.Options.Verbose) outs() << "Found valid debug map entry: " << ValidReloc.Mapping->getKey() << " " << format("\t%016" PRIx64 " => %016" PRIx64, ObjectAddress, @@ -2052,7 +2165,7 @@ hasValidRelocation(uint32_t StartOffset, uint32_t EndOffset, return true; } -/// \brief Get the starting and ending (exclusive) offset for the +/// Get the starting and ending (exclusive) offset for the /// attribute with index \p Idx descibed by \p Abbrev. \p Offset is /// supposed to point to the position of the first attribute described /// by \p Abbrev. @@ -2073,7 +2186,7 @@ getAttributeOffsets(const DWARFAbbreviationDeclaration *Abbrev, unsigned Idx, return std::make_pair(Offset, End); } -/// \brief Check if a variable describing DIE should be kept. +/// Check if a variable describing DIE should be kept. /// \returns updated TraversalFlags. unsigned DwarfLinker::shouldKeepVariableDIE(RelocationManager &RelocMgr, const DWARFDie &DIE, @@ -2109,13 +2222,17 @@ unsigned DwarfLinker::shouldKeepVariableDIE(RelocationManager &RelocMgr, (Flags & TF_InFunctionScope)) return Flags; - if (Options.Verbose) - DIE.dump(outs(), 0, 8 /* Indent */); + if (Options.Verbose) { + DIDumpOptions DumpOpts; + DumpOpts.RecurseDepth = 0; + DumpOpts.Verbose = Options.Verbose; + DIE.dump(outs(), 8 /* Indent */, DumpOpts); + } return Flags | TF_Keep; } -/// \brief Check if a function describing DIE should be kept. +/// Check if a function describing DIE should be kept. /// \returns updated TraversalFlags. unsigned DwarfLinker::shouldKeepSubprogramDIE( RelocationManager &RelocMgr, @@ -2141,8 +2258,12 @@ unsigned DwarfLinker::shouldKeepSubprogramDIE( !RelocMgr.hasValidRelocation(LowPcOffset, LowPcEndOffset, MyInfo)) return Flags; - if (Options.Verbose) - DIE.dump(outs(), 0, 8 /* Indent */); + if (Options.Verbose) { + DIDumpOptions DumpOpts; + DumpOpts.RecurseDepth = 0; + DumpOpts.Verbose = Options.Verbose; + DIE.dump(outs(), 8 /* Indent */, DumpOpts); + } Flags |= TF_Keep; @@ -2159,7 +2280,7 @@ unsigned DwarfLinker::shouldKeepSubprogramDIE( return Flags; } -/// \brief Check if a DIE should be kept. +/// Check if a DIE should be kept. /// \returns updated TraversalFlags. unsigned DwarfLinker::shouldKeepDIE(RelocationManager &RelocMgr, const DWARFDie &DIE, @@ -2172,7 +2293,6 @@ unsigned DwarfLinker::shouldKeepDIE(RelocationManager &RelocMgr, return shouldKeepVariableDIE(RelocMgr, DIE, Unit, MyInfo, Flags); case dwarf::DW_TAG_subprogram: return shouldKeepSubprogramDIE(RelocMgr, DIE, Unit, MyInfo, Flags); - case dwarf::DW_TAG_module: case dwarf::DW_TAG_imported_module: case dwarf::DW_TAG_imported_declaration: case dwarf::DW_TAG_imported_unit: @@ -2185,7 +2305,7 @@ unsigned DwarfLinker::shouldKeepDIE(RelocationManager &RelocMgr, return Flags; } -/// \brief Mark the passed DIE as well as all the ones it depends on +/// Mark the passed DIE as well as all the ones it depends on /// as kept. /// /// This function is called by lookForDIEsToKeep on DIEs that are @@ -2201,6 +2321,11 @@ void DwarfLinker::keepDIEAndDependencies(RelocationManager &RelocMgr, DWARFUnit &Unit = CU.getOrigUnit(); MyInfo.Keep = true; + // We're looking for incomplete types. + MyInfo.Incomplete = Die.getTag() != dwarf::DW_TAG_subprogram && + Die.getTag() != dwarf::DW_TAG_member && + dwarf::toUnsigned(Die.find(dwarf::DW_AT_declaration), 0); + // First mark all the parent chain as kept. unsigned AncestorIdx = MyInfo.ParentIdx; while (!CU.getInfo(AncestorIdx).Keep) { @@ -2216,7 +2341,7 @@ void DwarfLinker::keepDIEAndDependencies(RelocationManager &RelocMgr, const auto *Abbrev = Die.getAbbreviationDeclarationPtr(); uint32_t Offset = Die.getOffset() + getULEB128Size(Abbrev->getCode()); - // Mark all DIEs referenced through atttributes as kept. + // Mark all DIEs referenced through attributes as kept. for (const auto &AttrSpec : Abbrev->attributes()) { DWARFFormValue Val(AttrSpec.Form); @@ -2226,12 +2351,14 @@ void DwarfLinker::keepDIEAndDependencies(RelocationManager &RelocMgr, continue; } - Val.extractValue(Data, &Offset, &Unit); + Val.extractValue(Data, &Offset, Unit.getFormParams(), &Unit); CompileUnit *ReferencedCU; - if (auto RefDIE = + if (auto RefDie = resolveDIEReference(*this, Units, Val, Unit, Die, ReferencedCU)) { - uint32_t RefIdx = ReferencedCU->getOrigUnit().getDIEIndex(RefDIE); + uint32_t RefIdx = ReferencedCU->getOrigUnit().getDIEIndex(RefDie); CompileUnit::DIEInfo &Info = ReferencedCU->getInfo(RefIdx); + bool IsModuleRef = Info.Ctxt && Info.Ctxt->getCanonicalDIEOffset() && + Info.Ctxt->isDefinedInClangModule(); // If the referenced DIE has a DeclContext that has already been // emitted, then do not keep the one in this CU. We'll link to // the canonical DIE in cloneDieReferenceAttribute. @@ -2240,7 +2367,8 @@ void DwarfLinker::keepDIEAndDependencies(RelocationManager &RelocMgr, // ReferencedCU->hasODR() && CU.hasODR(). // FIXME: compatibility with dsymutil-classic. There is no // reason not to unique ref_addr references. - if (AttrSpec.Form != dwarf::DW_FORM_ref_addr && UseODR && Info.Ctxt && + if (AttrSpec.Form != dwarf::DW_FORM_ref_addr && (UseODR || IsModuleRef) && + Info.Ctxt && Info.Ctxt != ReferencedCU->getInfo(Info.ParentIdx).Ctxt && Info.Ctxt->getCanonicalDIEOffset() && isODRAttribute(AttrSpec.Attr)) continue; @@ -2251,13 +2379,23 @@ void DwarfLinker::keepDIEAndDependencies(RelocationManager &RelocMgr, Info.Prune = false; unsigned ODRFlag = UseODR ? TF_ODR : 0; - lookForDIEsToKeep(RelocMgr, RefDIE, DMO, *ReferencedCU, + lookForDIEsToKeep(RelocMgr, RefDie, DMO, *ReferencedCU, TF_Keep | TF_DependencyWalk | ODRFlag); + + // The incomplete property is propagated if the current DIE is complete + // but references an incomplete DIE. + if (Info.Incomplete && !MyInfo.Incomplete && + (Die.getTag() == dwarf::DW_TAG_typedef || + Die.getTag() == dwarf::DW_TAG_member || + Die.getTag() == dwarf::DW_TAG_reference_type || + Die.getTag() == dwarf::DW_TAG_ptr_to_member_type || + Die.getTag() == dwarf::DW_TAG_pointer_type)) + MyInfo.Incomplete = true; } } } -/// \brief Recursively walk the \p DIE tree and look for DIEs to +/// Recursively walk the \p DIE tree and look for DIEs to /// keep. Store that information in \p CU's DIEInfo. /// /// This function is the entry point of the DIE selection @@ -2269,7 +2407,9 @@ void DwarfLinker::keepDIEAndDependencies(RelocationManager &RelocMgr, /// also called, but during these dependency walks the file order is /// not respected. The TF_DependencyWalk flag tells us which kind of /// traversal we are currently doing. -void DwarfLinker::lookForDIEsToKeep(RelocationManager &RelocMgr, +/// +/// The return value indicates whether the DIE is incomplete. +bool DwarfLinker::lookForDIEsToKeep(RelocationManager &RelocMgr, const DWARFDie &Die, const DebugMapObject &DMO, CompileUnit &CU, unsigned Flags) { @@ -2277,13 +2417,13 @@ void DwarfLinker::lookForDIEsToKeep(RelocationManager &RelocMgr, CompileUnit::DIEInfo &MyInfo = CU.getInfo(Idx); bool AlreadyKept = MyInfo.Keep; if (MyInfo.Prune) - return; + return true; // If the Keep flag is set, we are marking a required DIE's // dependencies. If our target is already marked as kept, we're all // set. if ((Flags & TF_DependencyWalk) && AlreadyKept) - return; + return MyInfo.Incomplete; // We must not call shouldKeepDIE while called from keepDIEAndDependencies, // because it would screw up the relocation finding logic. @@ -2305,13 +2445,22 @@ void DwarfLinker::lookForDIEsToKeep(RelocationManager &RelocMgr, Flags &= ~TF_ParentWalk; if (!Die.hasChildren() || (Flags & TF_ParentWalk)) - return; + return MyInfo.Incomplete; + + bool Incomplete = false; + for (auto Child : Die.children()) { + Incomplete |= lookForDIEsToKeep(RelocMgr, Child, DMO, CU, Flags); - for (auto Child: Die.children()) - lookForDIEsToKeep(RelocMgr, Child, DMO, CU, Flags); + // If any of the members are incomplete we propagate the incompleteness. + if (!MyInfo.Incomplete && Incomplete && + (Die.getTag() == dwarf::DW_TAG_structure_type || + Die.getTag() == dwarf::DW_TAG_class_type)) + MyInfo.Incomplete = true; + } + return MyInfo.Incomplete; } -/// \brief Assign an abbreviation numer to \p Abbrev. +/// Assign an abbreviation numer to \p Abbrev. /// /// Our DIEs get freed after every DebugMapObject has been processed, /// thus the FoldingSet we use to unique DIEAbbrevs cannot refer to @@ -2478,11 +2627,13 @@ unsigned DwarfLinker::DIECloner::cloneAddressAttribute( // relocated because it happens to match the low_pc of the // enclosing subprogram. To prevent issues with that, always use // the low_pc from the input DIE if relocations have been applied. - Addr = (Info.OrigLowPc != UINT64_MAX ? Info.OrigLowPc : Addr) + + Addr = (Info.OrigLowPc != std::numeric_limits<uint64_t>::max() + ? Info.OrigLowPc + : Addr) + Info.PCOffset; else if (Die.getTag() == dwarf::DW_TAG_compile_unit) { Addr = Unit.getLowPc(); - if (Addr == UINT64_MAX) + if (Addr == std::numeric_limits<uint64_t>::max()) return 0; } Info.HasLowPc = true; @@ -2545,7 +2696,7 @@ unsigned DwarfLinker::DIECloner::cloneScalarAttribute( return AttrSize; } -/// \brief Clone \p InputDIE's attribute described by \p AttrSpec with +/// Clone \p InputDIE's attribute described by \p AttrSpec with /// value \p Val, and add it to \p Die. /// \returns the size of the cloned attribute. unsigned DwarfLinker::DIECloner::cloneAttribute( @@ -2592,7 +2743,7 @@ unsigned DwarfLinker::DIECloner::cloneAttribute( return 0; } -/// \brief Apply the valid relocations found by findValidRelocs() to +/// Apply the valid relocations found by findValidRelocs() to /// the buffer \p Data, taking into account that Data is at \p BaseOffset /// in the debug_info section. /// @@ -2718,7 +2869,7 @@ DIE *DwarfLinker::DIECloner::cloneDIE( assert(Die->getTag() == InputDIE.getTag()); Die->setOffset(OutOffset); - if ((Unit.hasODR() || Unit.isClangModule()) && + if ((Unit.hasODR() || Unit.isClangModule()) && !Info.Incomplete && Die->getTag() != dwarf::DW_TAG_namespace && Info.Ctxt && Info.Ctxt != Unit.getInfo(Info.ParentIdx).Ctxt && !Info.Ctxt->getCanonicalDIEOffset()) { @@ -2754,11 +2905,13 @@ DIE *DwarfLinker::DIECloner::cloneDIE( // file might be start address of another function which got moved // independantly by the linker). The computation of the actual // high_pc value is done in cloneAddressAttribute(). - AttrInfo.OrigHighPc = dwarf::toAddress(InputDIE.find(dwarf::DW_AT_high_pc), 0); + AttrInfo.OrigHighPc = + dwarf::toAddress(InputDIE.find(dwarf::DW_AT_high_pc), 0); // Also store the low_pc. It might get relocated in an // inline_subprogram that happens at the beginning of its // inlining function. - AttrInfo.OrigLowPc = dwarf::toAddress(InputDIE.find(dwarf::DW_AT_low_pc), UINT64_MAX); + AttrInfo.OrigLowPc = dwarf::toAddress(InputDIE.find(dwarf::DW_AT_low_pc), + std::numeric_limits<uint64_t>::max()); } // Reset the Offset to 0 as we will be working on the local copy of @@ -2797,7 +2950,7 @@ DIE *DwarfLinker::DIECloner::cloneDIE( DWARFFormValue Val(AttrSpec.Form); uint32_t AttrSize = Offset; - Val.extractValue(Data, &Offset, &U); + Val.extractValue(Data, &Offset, U.getFormParams(), &U); AttrSize = Offset - AttrSize; OutOffset += @@ -2821,7 +2974,8 @@ DIE *DwarfLinker::DIECloner::cloneDIE( Tag == dwarf::DW_TAG_inlined_subroutine); } else if (isTypeTag(Tag) && !AttrInfo.IsDeclaration && getDIENames(InputDIE, AttrInfo)) { - Unit.addTypeAccelerator(Die, AttrInfo.Name, AttrInfo.NameOffset); + if (AttrInfo.Name) + Unit.addTypeAccelerator(Die, AttrInfo.Name, AttrInfo.NameOffset); } // Determine whether there are any children that we want to keep. @@ -2865,7 +3019,7 @@ DIE *DwarfLinker::DIECloner::cloneDIE( return Die; } -/// \brief Patch the input object file relevant debug_ranges entries +/// Patch the input object file relevant debug_ranges entries /// and emit them in the output file. Update the relevant attributes /// to point at the new entries. void DwarfLinker::patchRangesForUnit(const CompileUnit &Unit, @@ -2873,12 +3027,14 @@ void DwarfLinker::patchRangesForUnit(const CompileUnit &Unit, DWARFDebugRangeList RangeList; const auto &FunctionRanges = Unit.getFunctionRanges(); unsigned AddressSize = Unit.getOrigUnit().getAddressByteSize(); - DWARFDataExtractor RangeExtractor(OrigDwarf.getRangeSection(), + DWARFDataExtractor RangeExtractor(OrigDwarf.getDWARFObj(), + OrigDwarf.getDWARFObj().getRangeSection(), OrigDwarf.isLittleEndian(), AddressSize); auto InvalidRange = FunctionRanges.end(), CurrRange = InvalidRange; DWARFUnit &OrigUnit = Unit.getOrigUnit(); auto OrigUnitDie = OrigUnit.getUnitDIE(false); - uint64_t OrigLowPc = dwarf::toAddress(OrigUnitDie.find(dwarf::DW_AT_low_pc), -1ULL); + uint64_t OrigLowPc = + dwarf::toAddress(OrigUnitDie.find(dwarf::DW_AT_low_pc), -1ULL); // Ranges addresses are based on the unit's low_pc. Compute the // offset we need to apply to adapt to the new unit's low_pc. int64_t UnitPcOffset = 0; @@ -2910,7 +3066,7 @@ void DwarfLinker::patchRangesForUnit(const CompileUnit &Unit, } } -/// \brief Generate the debug_aranges entries for \p Unit and if the +/// Generate the debug_aranges entries for \p Unit and if the /// unit has a DW_AT_ranges attribute, also emit the debug_ranges /// contribution for this attribute. /// FIXME: this could actually be done right in patchRangesForUnit, @@ -2923,7 +3079,7 @@ void DwarfLinker::generateUnitRanges(CompileUnit &Unit) const { Streamer->emitUnitRangesEntries(Unit, static_cast<bool>(Attr)); } -/// \brief Insert the new line info sequence \p Seq into the current +/// Insert the new line info sequence \p Seq into the current /// set of already linked line info \p Rows. static void insertLineSequence(std::vector<DWARFDebugLine::Row> &Seq, std::vector<DWARFDebugLine::Row> &Rows) { @@ -2967,7 +3123,7 @@ static void patchStmtList(DIE &Die, DIEInteger Offset) { llvm_unreachable("Didn't find DW_AT_stmt_list in cloned DIE!"); } -/// \brief Extract the line table for \p Unit from \p OrigDwarf, and +/// Extract the line table for \p Unit from \p OrigDwarf, and /// recreate a relocated version of these for the address ranges that /// are present in the binary. void DwarfLinker::patchLineTableForUnit(CompileUnit &Unit, @@ -2984,10 +3140,10 @@ void DwarfLinker::patchLineTableForUnit(CompileUnit &Unit, // Parse the original line info for the unit. DWARFDebugLine::LineTable LineTable; uint32_t StmtOffset = *StmtList; - DWARFDataExtractor LineExtractor(OrigDwarf.getLineSection(), - OrigDwarf.isLittleEndian(), - Unit.getOrigUnit().getAddressByteSize()); - LineTable.parse(LineExtractor, &StmtOffset); + DWARFDataExtractor LineExtractor( + OrigDwarf.getDWARFObj(), OrigDwarf.getDWARFObj().getLineSection(), + OrigDwarf.isLittleEndian(), Unit.getOrigUnit().getAddressByteSize()); + LineTable.parse(LineExtractor, &StmtOffset, &Unit.getOrigUnit()); // This vector is the output line table. std::vector<DWARFDebugLine::Row> NewRows; @@ -3076,17 +3232,22 @@ void DwarfLinker::patchLineTableForUnit(CompileUnit &Unit, } // Finished extracting, now emit the line tables. - uint32_t PrologueEnd = *StmtList + 10 + LineTable.Prologue.PrologueLength; - // FIXME: LLVM hardcodes it's prologue values. We just copy the + // FIXME: LLVM hardcodes its prologue values. We just copy the // prologue over and that works because we act as both producer and // consumer. It would be nicer to have a real configurable line // table emitter. - if (LineTable.Prologue.getVersion() != 2 || + if (LineTable.Prologue.getVersion() < 2 || + LineTable.Prologue.getVersion() > 5 || LineTable.Prologue.DefaultIsStmt != DWARF2_LINE_DEFAULT_IS_STMT || LineTable.Prologue.OpcodeBase > 13) reportWarning("line table parameters mismatch. Cannot emit."); else { - StringRef LineData = OrigDwarf.getLineSection().Data; + uint32_t PrologueEnd = *StmtList + 10 + LineTable.Prologue.PrologueLength; + // DWARFv5 has an extra 2 bytes of information before the header_length + // field. + if (LineTable.Prologue.getVersion() == 5) + PrologueEnd += 2; + StringRef LineData = OrigDwarf.getDWARFObj().getLineSection().Data; MCDwarfLineTableParams Params; Params.DWARF2LineOpcodeBase = LineTable.Prologue.OpcodeBase; Params.DWARF2LineBase = LineTable.Prologue.LineBase; @@ -3103,7 +3264,7 @@ void DwarfLinker::emitAcceleratorEntriesForUnit(CompileUnit &Unit) { Streamer->emitPubTypesForUnit(Unit); } -/// \brief Read the frame info stored in the object, and emit the +/// Read the frame info stored in the object, and emit the /// patched frame descriptions for the linked binary. /// /// This is actually pretty easy as the data of the CIEs and FDEs can @@ -3112,7 +3273,7 @@ void DwarfLinker::emitAcceleratorEntriesForUnit(CompileUnit &Unit) { void DwarfLinker::patchFrameInfoForObject(const DebugMapObject &DMO, DWARFContext &OrigDwarf, unsigned AddrSize) { - StringRef FrameData = OrigDwarf.getDebugFrameSection(); + StringRef FrameData = OrigDwarf.getDWARFObj().getDebugFrameSection(); if (FrameData.empty()) return; @@ -3254,7 +3415,11 @@ bool DwarfLinker::registerModuleReference( // Cyclic dependencies are disallowed by Clang, but we still // shouldn't run into an infinite loop, so mark it as processed now. ClangModules.insert({PCMfile, DwoId}); - loadClangModule(PCMfile, PCMpath, Name, DwoId, ModuleMap, Indent + 2); + if (Error E = loadClangModule(PCMfile, PCMpath, Name, DwoId, ModuleMap, + Indent + 2)) { + consumeError(std::move(E)); + return false; + } return true; } @@ -3273,17 +3438,17 @@ DwarfLinker::loadObject(BinaryHolder &BinaryHolder, DebugMapObject &Obj, return ErrOrObj; } -void DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath, - StringRef ModuleName, uint64_t DwoId, - DebugMap &ModuleMap, unsigned Indent) { +Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath, + StringRef ModuleName, uint64_t DwoId, + DebugMap &ModuleMap, unsigned Indent) { SmallString<80> Path(Options.PrependPath); if (sys::path::is_relative(Filename)) sys::path::append(Path, ModulePath, Filename); else sys::path::append(Path, Filename); BinaryHolder ObjHolder(Options.Verbose); - auto &Obj = - ModuleMap.addDebugMapObject(Path, sys::TimePoint<std::chrono::seconds>()); + auto &Obj = ModuleMap.addDebugMapObject( + Path, sys::TimePoint<std::chrono::seconds>(), MachO::N_OSO); auto ErrOrObj = loadObject(ObjHolder, Obj, ModuleMap); if (!ErrOrObj) { // Try and emit more helpful warnings by applying some heuristics. @@ -3317,22 +3482,27 @@ void DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath, } } } - return; + return Error::success(); } std::unique_ptr<CompileUnit> Unit; // Setup access to the debug info. - DWARFContextInMemory DwarfContext(*ErrOrObj); + auto DwarfContext = DWARFContext::create(*ErrOrObj); RelocationManager RelocMgr(*this); - for (const auto &CU : DwarfContext.compile_units()) { - auto CUDie = CU->getUnitDIE(false); + for (const auto &CU : DwarfContext->compile_units()) { + maybeUpdateMaxDwarfVersion(CU->getVersion()); + // Recursively get all modules imported by this one. + auto CUDie = CU->getUnitDIE(false); if (!registerModuleReference(CUDie, *CU, ModuleMap, Indent)) { if (Unit) { - errs() << Filename << ": Clang modules are expected to have exactly" - << " 1 compile unit.\n"; - exitDsymutil(1); + std::string Err = + (Filename + + ": Clang modules are expected to have exactly 1 compile unit.\n") + .str(); + errs() << Err; + return make_error<StringError>(Err, inconvertibleErrorCode()); } // FIXME: Until PR27449 (https://llvm.org/bugs/show_bug.cgi?id=27449) is // fixed in clang, only warn about DWO_id mismatches in verbose mode. @@ -3357,6 +3527,8 @@ void DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath, Unit->markEverythingAsKept(); } } + if (!Unit->getOrigUnit().getUnitDIE().hasChildren()) + return Error::success(); if (Options.Verbose) { outs().indent(Indent); outs() << "cloning .debug_info from " << Filename << "\n"; @@ -3365,11 +3537,11 @@ void DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath, std::vector<std::unique_ptr<CompileUnit>> CompileUnits; CompileUnits.push_back(std::move(Unit)); DIECloner(*this, RelocMgr, DIEAlloc, CompileUnits, Options) - .cloneAllCompileUnits(DwarfContext); + .cloneAllCompileUnits(*DwarfContext); + return Error::success(); } -void DwarfLinker::DIECloner::cloneAllCompileUnits( - DWARFContextInMemory &DwarfContext) { +void DwarfLinker::DIECloner::cloneAllCompileUnits(DWARFContext &DwarfContext) { if (!Linker.Streamer) return; @@ -3410,8 +3582,7 @@ void DwarfLinker::DIECloner::cloneAllCompileUnits( } bool DwarfLinker::link(const DebugMap &Map) { - - if (!createStreamer(Map.getTriple(), OutputFilename)) + if (!createStreamer(Map.getTriple(), OutFile)) return false; // Size of the DIEs (and headers) generated for the linked output. @@ -3425,6 +3596,35 @@ bool DwarfLinker::link(const DebugMap &Map) { if (Options.Verbose) outs() << "DEBUG MAP OBJECT: " << Obj->getObjectFilename() << "\n"; + + // N_AST objects (swiftmodule files) should get dumped directly into the + // appropriate DWARF section. + if (Obj->getType() == MachO::N_AST) { + StringRef File = Obj->getObjectFilename(); + auto ErrorOrMem = MemoryBuffer::getFile(File); + if (!ErrorOrMem) { + errs() << "Warning: Could not open " << File << "\n"; + continue; + } + sys::fs::file_status Stat; + if (auto errc = sys::fs::status(File, Stat)) { + errs() << "Warning: " << errc.message() << "\n"; + continue; + } + if (!Options.NoTimestamp && Stat.getLastModificationTime() != + sys::TimePoint<>(Obj->getTimestamp())) { + errs() << "Warning: Timestamp mismatch for " << File << ": " + << Stat.getLastModificationTime() << " and " + << sys::TimePoint<>(Obj->getTimestamp()) << "\n"; + continue; + } + + // Copy the module into the .swift_ast section. + if (!Options.NoOutput) + Streamer->emitSwiftAST((*ErrorOrMem)->getBuffer()); + continue; + } + auto ErrOrObj = loadObject(BinHolder, *Obj, Map); if (!ErrOrObj) continue; @@ -3438,15 +3638,18 @@ bool DwarfLinker::link(const DebugMap &Map) { } // Setup access to the debug info. - DWARFContextInMemory DwarfContext(*ErrOrObj); - startDebugObject(DwarfContext, *Obj); + auto DwarfContext = DWARFContext::create(*ErrOrObj); + startDebugObject(*DwarfContext, *Obj); // In a first phase, just read in the debug info and load all clang modules. - for (const auto &CU : DwarfContext.compile_units()) { + for (const auto &CU : DwarfContext->compile_units()) { auto CUDie = CU->getUnitDIE(false); if (Options.Verbose) { outs() << "Input compilation unit:"; - CUDie.dump(outs(), 0); + DIDumpOptions DumpOpts; + DumpOpts.RecurseDepth = 0; + DumpOpts.Verbose = Options.Verbose; + CUDie.dump(outs(), 0, DumpOpts); } if (!registerModuleReference(CUDie, *CU, ModuleMap)) { @@ -3476,9 +3679,9 @@ bool DwarfLinker::link(const DebugMap &Map) { RelocMgr.resetValidRelocs(); if (RelocMgr.hasValidRelocs()) DIECloner(*this, RelocMgr, DIEAlloc, Units, Options) - .cloneAllCompileUnits(DwarfContext); + .cloneAllCompileUnits(*DwarfContext); if (!Options.NoOutput && !Units.empty()) - patchFrameInfoForObject(*Obj, DwarfContext, + patchFrameInfoForObject(*Obj, *DwarfContext, Units[0]->getOrigUnit().getAddressByteSize()); // Clean-up before starting working on the next object. @@ -3493,9 +3696,8 @@ bool DwarfLinker::link(const DebugMap &Map) { return Options.NoOutput ? true : Streamer->finish(Map); } -} -/// \brief Get the offset of string \p S in the string table. This +/// Get the offset of string \p S in the string table. This /// can insert a new element or return the offset of a preexisitng /// one. uint32_t NonRelocatableStringpool::getStringOffset(StringRef S) { @@ -3519,7 +3721,7 @@ uint32_t NonRelocatableStringpool::getStringOffset(StringRef S) { return It->getValue().first; } -/// \brief Put \p S into the StringMap so that it gets permanent +/// Put \p S into the StringMap so that it gets permanent /// storage, but do not actually link it in the chain of elements /// that go into the output section. A latter call to /// getStringOffset() with the same string will chain it though. @@ -3540,10 +3742,11 @@ bool error(const Twine &Error, const Twine &Context) { return false; } -bool linkDwarf(StringRef OutputFilename, const DebugMap &DM, +bool linkDwarf(raw_fd_ostream &OutFile, const DebugMap &DM, const LinkOptions &Options) { - DwarfLinker Linker(OutputFilename, Options); + DwarfLinker Linker(OutFile, Options); return Linker.link(DM); } -} -} + +} // end namespace dsymutil +} // end namespace llvm diff --git a/tools/dsymutil/MachODebugMapParser.cpp b/tools/dsymutil/MachODebugMapParser.cpp index 75ae67cd53fe..2142baa72ec1 100644 --- a/tools/dsymutil/MachODebugMapParser.cpp +++ b/tools/dsymutil/MachODebugMapParser.cpp @@ -9,7 +9,6 @@ #include "BinaryHolder.h" #include "DebugMap.h" -#include "dsymutil.h" #include "llvm/ADT/Optional.h" #include "llvm/Object/MachO.h" #include "llvm/Support/Path.h" @@ -70,6 +69,7 @@ private: sys::TimePoint<std::chrono::seconds> Timestamp); void resetParserState(); uint64_t getMainBinarySymbolAddress(StringRef Name); + std::vector<StringRef> getMainBinarySymbolNames(uint64_t Value); void loadMainBinarySymbols(const MachOObjectFile &MainBinary); void loadCurrentObjectFileSymbols(const object::MachOObjectFile &Obj); void handleStabSymbolTableEntry(uint32_t StringIndex, uint8_t Type, @@ -134,7 +134,8 @@ void MachODebugMapParser::switchToNewDebugMapObject( Err.message() + "\n"); } - CurrentDebugMapObject = &Result->addDebugMapObject(Path, Timestamp); + CurrentDebugMapObject = + &Result->addDebugMapObject(Path, Timestamp, MachO::N_OSO); loadCurrentObjectFileSymbols(*ErrOrAchObj); } @@ -348,6 +349,13 @@ void MachODebugMapParser::handleStabSymbolTableEntry(uint32_t StringIndex, if (Type == MachO::N_OSO) return switchToNewDebugMapObject(Name, sys::toTimePoint(Value)); + if (Type == MachO::N_AST) { + SmallString<80> Path(PathPrefix); + sys::path::append(Path, Name); + Result->addDebugMapObject(Path, sys::toTimePoint(Value), Type); + return; + } + // If the last N_OSO object file wasn't found, // CurrentDebugMapObject will be null. Do not update anything // until we find the next valid N_OSO entry. @@ -382,9 +390,21 @@ void MachODebugMapParser::handleStabSymbolTableEntry(uint32_t StringIndex, } auto ObjectSymIt = CurrentObjectAddresses.find(Name); + + // If the name of a (non-static) symbol is not in the current object, we + // check all its aliases from the main binary. + if (ObjectSymIt == CurrentObjectAddresses.end() && Type != MachO::N_STSYM) { + for (const auto &Alias : getMainBinarySymbolNames(Value)) { + ObjectSymIt = CurrentObjectAddresses.find(Alias); + if (ObjectSymIt != CurrentObjectAddresses.end()) + break; + } + } + if (ObjectSymIt == CurrentObjectAddresses.end()) return Warning("could not find object file symbol for symbol " + Twine(Name)); + if (!CurrentDebugMapObject->addSymbol(Name, ObjectSymIt->getValue(), Value, Size)) return Warning(Twine("failed to insert symbol '") + Name + @@ -429,6 +449,17 @@ uint64_t MachODebugMapParser::getMainBinarySymbolAddress(StringRef Name) { return Sym->second; } +/// Get all symbol names in the main binary for the given value. +std::vector<StringRef> +MachODebugMapParser::getMainBinarySymbolNames(uint64_t Value) { + std::vector<StringRef> Names; + for (const auto &Entry : MainBinarySymbolAddresses) { + if (Entry.second == Value) + Names.push_back(Entry.first()); + } + return Names; +} + /// Load the interesting main binary symbols' addresses into /// MainBinarySymbolAddresses. void MachODebugMapParser::loadMainBinarySymbols( @@ -450,7 +481,9 @@ void MachODebugMapParser::loadMainBinarySymbols( // are the only ones that need to be queried because the address // of common data won't be described in the debug map. All other // addresses should be fetched for the debug map. - if (!(Sym.getFlags() & SymbolRef::SF_Global)) + uint8_t SymType = + MainBinary.getSymbolTableEntry(Sym.getRawDataRefImpl()).n_type; + if (!(SymType & (MachO::N_EXT | MachO::N_PEXT))) continue; Expected<section_iterator> SectionOrErr = Sym.getSection(); if (!SectionOrErr) { diff --git a/tools/dsymutil/MachOUtils.cpp b/tools/dsymutil/MachOUtils.cpp index ea6f113e4fae..06d0f72a61cc 100644 --- a/tools/dsymutil/MachOUtils.cpp +++ b/tools/dsymutil/MachOUtils.cpp @@ -45,7 +45,7 @@ static bool runLipo(StringRef SDKPath, SmallVectorImpl<const char *> &Args) { std::string ErrMsg; int result = - sys::ExecuteAndWait(*Path, Args.data(), nullptr, nullptr, 0, 0, &ErrMsg); + sys::ExecuteAndWait(*Path, Args.data(), nullptr, {}, 0, 0, &ErrMsg); if (result) { errs() << "error: lipo: " << ErrMsg << "\n"; return false; diff --git a/tools/dsymutil/NonRelocatableStringpool.h b/tools/dsymutil/NonRelocatableStringpool.h index fffd02084e34..aa21d77ad438 100644 --- a/tools/dsymutil/NonRelocatableStringpool.h +++ b/tools/dsymutil/NonRelocatableStringpool.h @@ -1,4 +1,4 @@ -//===-- NonRelocatableStringpool.h - A simple stringpool -----------------===// +//===- NonRelocatableStringpool.h - A simple stringpool --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,10 +6,15 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// + #ifndef LLVM_TOOLS_DSYMUTIL_NONRELOCATABLESTRINGPOOL_H #define LLVM_TOOLS_DSYMUTIL_NONRELOCATABLESTRINGPOOL_H #include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Allocator.h" +#include <cstdint> +#include <utility> namespace llvm { namespace dsymutil { @@ -25,11 +30,10 @@ public: /// \brief Entries are stored into the StringMap and simply linked /// together through the second element of this pair in order to /// keep track of insertion order. - typedef StringMap<std::pair<uint32_t, StringMapEntryBase *>, BumpPtrAllocator> - MapTy; + using MapTy = + StringMap<std::pair<uint32_t, StringMapEntryBase *>, BumpPtrAllocator>; - NonRelocatableStringpool() - : CurrentEndOffset(0), Sentinel(0), Last(&Sentinel) { + NonRelocatableStringpool() : Sentinel(0), Last(&Sentinel) { // Legacy dsymutil puts an empty string at the start of the line // table. getStringOffset(""); @@ -61,10 +65,11 @@ public: private: MapTy Strings; - uint32_t CurrentEndOffset; + uint32_t CurrentEndOffset = 0; MapTy::MapEntryTy Sentinel, *Last; }; -} -} -#endif +} // end namespace dsymutil +} // end namespace llvm + +#endif // LLVM_TOOLS_DSYMUTIL_NONRELOCATABLESTRINGPOOL_H diff --git a/tools/dsymutil/dsymutil.cpp b/tools/dsymutil/dsymutil.cpp index 1ce0aefeec2a..1f882abd1811 100644 --- a/tools/dsymutil/dsymutil.cpp +++ b/tools/dsymutil/dsymutil.cpp @@ -1,4 +1,4 @@ -//===-- dsymutil.cpp - Debug info dumping utility for llvm ----------------===// +//===- dsymutil.cpp - Debug info dumping utility for llvm -----------------===// // // The LLVM Compiler Infrastructure // @@ -12,27 +12,41 @@ // //===----------------------------------------------------------------------===// +#include "dsymutil.h" +#include "CFBundle.h" #include "DebugMap.h" #include "MachOUtils.h" -#include "dsymutil.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" +#include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/DebugInfo/DWARF/DWARFVerifier.h" +#include "llvm/Object/Binary.h" #include "llvm/Object/MachO.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/FileUtilities.h" #include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Options.h" +#include "llvm/Support/Path.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/Signals.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/thread.h" +#include <algorithm> #include <cstdint> +#include <cstdlib> #include <string> +#include <system_error> -using namespace llvm::dsymutil; - -namespace { +using namespace llvm; using namespace llvm::cl; +using namespace llvm::dsymutil; +using namespace object; -OptionCategory DsymCategory("Specific Options"); +static OptionCategory DsymCategory("Specific Options"); static opt<bool> Help("h", desc("Alias for -help"), Hidden); static opt<bool> Version("v", desc("Alias for -version"), Hidden); @@ -61,6 +75,14 @@ static opt<bool> FlatOut("flat", init(false), cat(DsymCategory)); static alias FlatOutA("f", desc("Alias for --flat"), aliasopt(FlatOut)); +static opt<unsigned> NumThreads( + "num-threads", + desc("Specifies the maximum number (n) of simultaneous threads to use\n" + "when linking multiple architectures."), + value_desc("n"), init(0), cat(DsymCategory)); +static alias NumThreadsA("j", desc("Alias for --num-threads"), + aliasopt(NumThreads)); + static opt<bool> Verbose("verbose", desc("Verbosity level"), init(false), cat(DsymCategory)); @@ -69,13 +91,18 @@ static opt<bool> desc("Do the link in memory, but do not emit the result file."), init(false), cat(DsymCategory)); +static opt<bool> + NoTimestamp("no-swiftmodule-timestamp", + desc("Don't check timestamp for swiftmodule files."), + init(false), cat(DsymCategory)); + static list<std::string> ArchFlags( "arch", desc("Link DWARF debug information only for specified CPU architecture\n" "types. This option can be specified multiple times, once for each\n" - "desired architecture. All cpu architectures will be linked by\n" + "desired architecture. All CPU architectures will be linked by\n" "default."), - ZeroOrMore, cat(DsymCategory)); + value_desc("arch"), ZeroOrMore, cat(DsymCategory)); static opt<bool> NoODR("no-odr", @@ -91,9 +118,11 @@ static opt<bool> DumpDebugMap( static opt<bool> InputIsYAMLDebugMap( "y", desc("Treat the input file is a YAML debug map rather than a binary."), init(false), cat(DsymCategory)); -} -static bool createPlistFile(llvm::StringRef BundleRoot) { +static opt<bool> Verify("verify", desc("Verify the linked DWARF debug info."), + cat(DsymCategory)); + +static bool createPlistFile(llvm::StringRef Bin, llvm::StringRef BundleRoot) { if (NoOutput) return true; @@ -108,16 +137,15 @@ static bool createPlistFile(llvm::StringRef BundleRoot) { return false; } - // FIXME: Use CoreFoundation to get executable bundle info. Use - // dummy values for now. - std::string bundleVersionStr = "1", bundleShortVersionStr = "1.0", - bundleIDStr; + CFBundleInfo BI = getBundleInfo(Bin); - llvm::StringRef BundleID = *llvm::sys::path::rbegin(BundleRoot); - if (llvm::sys::path::extension(BundleRoot) == ".dSYM") - bundleIDStr = llvm::sys::path::stem(BundleID); - else - bundleIDStr = BundleID; + if (BI.IDStr.empty()) { + llvm::StringRef BundleID = *llvm::sys::path::rbegin(BundleRoot); + if (llvm::sys::path::extension(BundleRoot) == ".dSYM") + BI.IDStr = llvm::sys::path::stem(BundleID); + else + BI.IDStr = BundleID; + } // Print out information to the plist file. PL << "<?xml version=\"1.0\" encoding=\"UTF-8\"\?>\n" @@ -128,17 +156,20 @@ static bool createPlistFile(llvm::StringRef BundleRoot) { << "\t\t<key>CFBundleDevelopmentRegion</key>\n" << "\t\t<string>English</string>\n" << "\t\t<key>CFBundleIdentifier</key>\n" - << "\t\t<string>com.apple.xcode.dsym." << bundleIDStr << "</string>\n" + << "\t\t<string>com.apple.xcode.dsym." << BI.IDStr << "</string>\n" << "\t\t<key>CFBundleInfoDictionaryVersion</key>\n" << "\t\t<string>6.0</string>\n" << "\t\t<key>CFBundlePackageType</key>\n" << "\t\t<string>dSYM</string>\n" << "\t\t<key>CFBundleSignature</key>\n" - << "\t\t<string>\?\?\?\?</string>\n" - << "\t\t<key>CFBundleShortVersionString</key>\n" - << "\t\t<string>" << bundleShortVersionStr << "</string>\n" - << "\t\t<key>CFBundleVersion</key>\n" - << "\t\t<string>" << bundleVersionStr << "</string>\n" + << "\t\t<string>\?\?\?\?</string>\n"; + + if (!BI.OmitShortVersion()) + PL << "\t\t<key>CFBundleShortVersionString</key>\n" + << "\t\t<string>" << BI.ShortVersionStr << "</string>\n"; + + PL << "\t\t<key>CFBundleVersion</key>\n" + << "\t\t<string>" << BI.VersionStr << "</string>\n" << "\t</dict>\n" << "</plist>\n"; @@ -161,43 +192,35 @@ static bool createBundleDir(llvm::StringRef BundleBase) { return true; } -static std::error_code getUniqueFile(const llvm::Twine &Model, int &ResultFD, - llvm::SmallVectorImpl<char> &ResultPath) { - // If in NoOutput mode, use the createUniqueFile variant that - // doesn't open the file but still generates a somewhat unique - // name. In the real usage scenario, we'll want to ensure that the - // file is trully unique, and creating it is the only way to achieve - // that. - if (NoOutput) - return llvm::sys::fs::createUniqueFile(Model, ResultPath); - return llvm::sys::fs::createUniqueFile(Model, ResultFD, ResultPath); -} +static bool verify(llvm::StringRef OutputFile, llvm::StringRef Arch) { + if (OutputFile == "-") { + llvm::errs() << "warning: verification skipped for " << Arch + << "because writing to stdout.\n"; + return true; + } -static std::string getOutputFileName(llvm::StringRef InputFile, - bool TempFile = false) { - if (TempFile) { - llvm::SmallString<128> TmpFile; - llvm::sys::path::system_temp_directory(true, TmpFile); - llvm::StringRef Basename = - OutputFileOpt.empty() ? InputFile : llvm::StringRef(OutputFileOpt); - llvm::sys::path::append(TmpFile, llvm::sys::path::filename(Basename)); - - int FD; - llvm::SmallString<128> UniqueFile; - if (auto EC = getUniqueFile(TmpFile + ".tmp%%%%%.dwarf", FD, UniqueFile)) { - llvm::errs() << "error: failed to create temporary outfile '" - << TmpFile << "': " << EC.message() << '\n'; - return ""; - } - llvm::sys::RemoveFileOnSignal(UniqueFile); - if (!NoOutput) { - // Close the file immediately. We know it is unique. It will be - // reopened and written to later. - llvm::raw_fd_ostream CloseImmediately(FD, true /* shouldClose */, true); - } - return UniqueFile.str(); + Expected<OwningBinary<Binary>> BinOrErr = createBinary(OutputFile); + if (!BinOrErr) { + errs() << OutputFile << ": " << toString(BinOrErr.takeError()); + return false; + } + + Binary &Binary = *BinOrErr.get().getBinary(); + if (auto *Obj = dyn_cast<MachOObjectFile>(&Binary)) { + raw_ostream &os = Verbose ? errs() : nulls(); + os << "Verifying DWARF for architecture: " << Arch << "\n"; + std::unique_ptr<DWARFContext> DICtx = DWARFContext::create(*Obj); + DIDumpOptions DumpOpts; + bool success = DICtx->verify(os, DumpOpts.noImplicitRecursion()); + if (!success) + errs() << "error: verification failed for " << Arch << '\n'; + return success; } + return false; +} + +static std::string getOutputFileName(llvm::StringRef InputFile) { if (FlatOut) { // If a flat dSYM has been requested, things are pretty simple. if (OutputFileOpt.empty()) { @@ -222,7 +245,7 @@ static std::string getOutputFileName(llvm::StringRef InputFile, llvm::SmallString<128> BundleDir(OutputFileOpt); if (BundleDir.empty()) BundleDir = DwarfFile + ".dSYM"; - if (!createBundleDir(BundleDir) || !createPlistFile(BundleDir)) + if (!createBundleDir(BundleDir) || !createPlistFile(DwarfFile, BundleDir)) return ""; llvm::sys::path::append(BundleDir, "Contents", "Resources", "DWARF", @@ -230,19 +253,32 @@ static std::string getOutputFileName(llvm::StringRef InputFile, return BundleDir.str(); } -void llvm::dsymutil::exitDsymutil(int ExitStatus) { - // Cleanup temporary files. - llvm::sys::RunInterruptHandlers(); - exit(ExitStatus); +static Expected<sys::fs::TempFile> createTempFile() { + llvm::SmallString<128> TmpModel; + llvm::sys::path::system_temp_directory(true, TmpModel); + llvm::sys::path::append(TmpModel, "dsym.tmp%%%%%.dwarf"); + return sys::fs::TempFile::create(TmpModel); } +namespace { +struct TempFileVector { + std::vector<sys::fs::TempFile> Files; + ~TempFileVector() { + for (sys::fs::TempFile &Tmp : Files) { + if (Error E = Tmp.discard()) + errs() << toString(std::move(E)); + } + } +}; +} // namespace + int main(int argc, char **argv) { llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); llvm::PrettyStackTraceProgram StackPrinter(argc, argv); llvm::llvm_shutdown_obj Shutdown; LinkOptions Options; - void *MainAddr = (void *)(intptr_t)&exitDsymutil; - std::string SDKPath = llvm::sys::fs::getMainExecutable(argv[0], MainAddr); + void *P = (void *)(intptr_t)getOutputFileName; + std::string SDKPath = llvm::sys::fs::getMainExecutable(argv[0], P); SDKPath = llvm::sys::path::parent_path(SDKPath); HideUnrelatedOptions(DsymCategory); @@ -253,8 +289,10 @@ int main(int argc, char **argv) { "for the executable <input file> by using debug symbols information\n" "contained in its symbol table.\n"); - if (Help) + if (Help) { PrintHelpMessage(); + return 0; + } if (Version) { llvm::cl::PrintVersionMessage(); @@ -264,6 +302,7 @@ int main(int argc, char **argv) { Options.Verbose = Verbose; Options.NoOutput = NoOutput; Options.NoODR = NoODR; + Options.NoTimestamp = NoTimestamp; Options.PrependPath = OsoPrependPath; llvm::InitializeAllTargetInfos(); @@ -285,14 +324,14 @@ int main(int argc, char **argv) { if (Arch != "*" && Arch != "all" && !llvm::object::MachOObjectFile::isValidArch(Arch)) { llvm::errs() << "error: Unsupported cpu architecture: '" << Arch << "'\n"; - exitDsymutil(1); + return 1; } for (auto &InputFile : InputFiles) { // Dump the symbol table for each input file and requested arch if (DumpStab) { if (!dumpStab(InputFile, ArchFlags, OsoPrependPath)) - exitDsymutil(1); + return 1; continue; } @@ -302,18 +341,28 @@ int main(int argc, char **argv) { if (auto EC = DebugMapPtrsOrErr.getError()) { llvm::errs() << "error: cannot parse the debug map for \"" << InputFile << "\": " << EC.message() << '\n'; - exitDsymutil(1); + return 1; } if (DebugMapPtrsOrErr->empty()) { llvm::errs() << "error: no architecture to link\n"; - exitDsymutil(1); + return 1; } + if (NumThreads == 0) + NumThreads = llvm::thread::hardware_concurrency(); + if (DumpDebugMap || Verbose) + NumThreads = 1; + NumThreads = std::min<unsigned>(NumThreads, DebugMapPtrsOrErr->size()); + + llvm::ThreadPool Threads(NumThreads); + // If there is more than one link to execute, we need to generate // temporary files. bool NeedsTempFiles = !DumpDebugMap && (*DebugMapPtrsOrErr).size() != 1; llvm::SmallVector<MachOUtils::ArchAndFilename, 4> TempFiles; + TempFileVector TempFileStore; + std::atomic_char AllOK(1); for (auto &Map : *DebugMapPtrsOrErr) { if (Verbose || DumpDebugMap) Map->print(llvm::outs()); @@ -326,20 +375,58 @@ int main(int argc, char **argv) { << MachOUtils::getArchName(Map->getTriple().getArchName()) << ")\n"; - std::string OutputFile = getOutputFileName(InputFile, NeedsTempFiles); - if (OutputFile.empty() || !linkDwarf(OutputFile, *Map, Options)) - exitDsymutil(1); - - if (NeedsTempFiles) + // Using a std::shared_ptr rather than std::unique_ptr because move-only + // types don't work with std::bind in the ThreadPool implementation. + std::shared_ptr<raw_fd_ostream> OS; + std::string OutputFile = getOutputFileName(InputFile); + if (NeedsTempFiles) { + Expected<sys::fs::TempFile> T = createTempFile(); + if (!T) { + errs() << toString(T.takeError()); + return 1; + } + OS = std::make_shared<raw_fd_ostream>(T->FD, /*shouldClose*/ false); + OutputFile = T->TmpName; + TempFileStore.Files.push_back(std::move(*T)); TempFiles.emplace_back(Map->getTriple().getArchName().str(), OutputFile); + } else { + std::error_code EC; + OS = std::make_shared<raw_fd_ostream>(NoOutput ? "-" : OutputFile, EC, + sys::fs::F_None); + if (EC) { + errs() << OutputFile << ": " << EC.message(); + return 1; + } + } + + auto LinkLambda = [&, + OutputFile](std::shared_ptr<raw_fd_ostream> Stream) { + AllOK.fetch_and(linkDwarf(*Stream, *Map, Options)); + Stream->flush(); + if (Verify && !NoOutput) + AllOK.fetch_and(verify(OutputFile, Map->getTriple().getArchName())); + }; + + // FIXME: The DwarfLinker can have some very deep recursion that can max + // out the (significantly smaller) stack when using threads. We don't + // want this limitation when we only have a single thread. + if (NumThreads == 1) + LinkLambda(OS); + else + Threads.async(LinkLambda, OS); } + Threads.wait(); + + if (!AllOK) + return 1; + if (NeedsTempFiles && !MachOUtils::generateUniversalBinary( TempFiles, getOutputFileName(InputFile), Options, SDKPath)) - exitDsymutil(1); + return 1; } - exitDsymutil(0); + return 0; } diff --git a/tools/dsymutil/dsymutil.h b/tools/dsymutil/dsymutil.h index 91cb32766129..09053d1b1cc8 100644 --- a/tools/dsymutil/dsymutil.h +++ b/tools/dsymutil/dsymutil.h @@ -6,37 +6,52 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -/// +// /// \file /// /// This file contains the class declaration for the code that parses STABS /// debug maps that are embedded in the binaries symbol tables. -/// +// //===----------------------------------------------------------------------===// + #ifndef LLVM_TOOLS_DSYMUTIL_DSYMUTIL_H #define LLVM_TOOLS_DSYMUTIL_DSYMUTIL_H #include "DebugMap.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorOr.h" #include <memory> +#include <string> +#include <vector> namespace llvm { namespace dsymutil { struct LinkOptions { - bool Verbose; ///< Verbosity - bool NoOutput; ///< Skip emitting output - bool NoODR; ///< Do not unique types according to ODR - std::string PrependPath; ///< -oso-prepend-path + /// Verbosity + bool Verbose = false; + + /// Skip emitting output + bool NoOutput = false; - LinkOptions() : Verbose(false), NoOutput(false) {} + /// Do not unique types according to ODR + bool NoODR; + + /// Do not check swiftmodule timestamp + bool NoTimestamp = false; + + /// -oso-prepend-path + std::string PrependPath; + + LinkOptions() = default; }; /// \brief Extract the DebugMaps from the given file. /// The file has to be a MachO object file. Multiple debug maps can be /// returned when the file is universal (aka fat) binary. -llvm::ErrorOr<std::vector<std::unique_ptr<DebugMap>>> +ErrorOr<std::vector<std::unique_ptr<DebugMap>>> parseDebugMap(StringRef InputFile, ArrayRef<std::string> Archs, StringRef PrependPath, bool Verbose, bool InputIsYAML); @@ -47,15 +62,13 @@ bool dumpStab(StringRef InputFile, ArrayRef<std::string> Archs, /// \brief Link the Dwarf debuginfo as directed by the passed DebugMap /// \p DM into a DwarfFile named \p OutputFilename. /// \returns false if the link failed. -bool linkDwarf(StringRef OutputFilename, const DebugMap &DM, +bool linkDwarf(raw_fd_ostream &OutFile, const DebugMap &DM, const LinkOptions &Options); -/// \brief Exit the dsymutil process, cleaning up every temporary -/// files that we created. -LLVM_ATTRIBUTE_NORETURN void exitDsymutil(int ExitStatus); - void warn(const Twine &Warning, const Twine &Context); bool error(const Twine &Error, const Twine &Context); -} -} + +} // end namespace dsymutil +} // end namespace llvm + #endif // LLVM_TOOLS_DSYMUTIL_DSYMUTIL_H diff --git a/tools/gold/CMakeLists.txt b/tools/gold/CMakeLists.txt index e9029e1e7c10..d2580329acab 100644 --- a/tools/gold/CMakeLists.txt +++ b/tools/gold/CMakeLists.txt @@ -3,10 +3,6 @@ set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/gold.exports) if( LLVM_ENABLE_PIC AND LLVM_BINUTILS_INCDIR ) include_directories( ${LLVM_BINUTILS_INCDIR} ) - # Because off_t is used in the public API, the largefile parts are required for - # ABI compatibility. - add_definitions( -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 ) - set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD} Linker @@ -20,4 +16,3 @@ if( LLVM_ENABLE_PIC AND LLVM_BINUTILS_INCDIR ) ) endif() - diff --git a/tools/gold/gold-plugin.cpp b/tools/gold/gold-plugin.cpp index 6d011bab079d..856d8172fc95 100644 --- a/tools/gold/gold-plugin.cpp +++ b/tools/gold/gold-plugin.cpp @@ -15,13 +15,14 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" -#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/CommandFlags.def" #include "llvm/Config/config.h" // plugin-api.h requires HAVE_STDINT_H #include "llvm/IR/Constants.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/LTO/Caching.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/Error.h" +#include "llvm/Support/CachePruning.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/ManagedStatic.h" @@ -174,12 +175,16 @@ namespace options { static std::string thinlto_object_suffix_replace; // Optional path to a directory for caching ThinLTO objects. static std::string cache_dir; + // Optional pruning policy for ThinLTO caches. + static std::string cache_policy; // Additional options to pass into the code generator. // Note: This array will contain all plugin options which are not claimed // as plugin exclusive to pass to the code generator. static std::vector<const char *> extra; // Sample profile file path static std::string sample_profile; + // New pass manager + static bool new_pass_manager = false; static void process_plugin_option(const char *opt_) { @@ -222,6 +227,8 @@ namespace options { "thinlto-object-suffix-replace expects 'old;new' format"); } else if (opt.startswith("cache-dir=")) { cache_dir = opt.substr(strlen("cache-dir=")); + } else if (opt.startswith("cache-policy=")) { + cache_policy = opt.substr(strlen("cache-policy=")); } else if (opt.size() == 2 && opt[0] == 'O') { if (opt[1] < '0' || opt[1] > '3') message(LDPL_FATAL, "Optimization level must be between 0 and 3"); @@ -237,6 +244,8 @@ namespace options { DisableVerify = true; } else if (opt.startswith("sample-profile=")) { sample_profile= opt.substr(strlen("sample-profile=")); + } else if (opt == "new-pass-manager") { + new_pass_manager = true; } else { // Save this option to pass to the code generator. // ParseCommandLineOptions() expects argv[0] to be program name. Lazily @@ -587,22 +596,31 @@ static void getThinLTOOldAndNewSuffix(std::string &OldSuffix, assert(options::thinlto_object_suffix_replace.empty() || options::thinlto_object_suffix_replace.find(";") != StringRef::npos); StringRef SuffixReplace = options::thinlto_object_suffix_replace; - std::pair<StringRef, StringRef> Split = SuffixReplace.split(";"); - OldSuffix = Split.first.str(); - NewSuffix = Split.second.str(); + std::tie(OldSuffix, NewSuffix) = SuffixReplace.split(';'); } /// Given the original \p Path to an output file, replace any filename /// suffix matching \p OldSuffix with \p NewSuffix. -static std::string getThinLTOObjectFileName(const std::string Path, - const std::string &OldSuffix, - const std::string &NewSuffix) { +static std::string getThinLTOObjectFileName(StringRef Path, StringRef OldSuffix, + StringRef NewSuffix) { if (OldSuffix.empty() && NewSuffix.empty()) return Path; StringRef NewPath = Path; NewPath.consume_back(OldSuffix); - std::string NewNewPath = NewPath.str() + NewSuffix; - return NewPath.str() + NewSuffix; + std::string NewNewPath = NewPath; + NewNewPath += NewSuffix; + return NewNewPath; +} + +// Returns true if S is valid as a C language identifier. +static bool isValidCIdentifier(StringRef S) { + return !S.empty() && (isAlpha(S[0]) || S[0] == '_') && + std::all_of(S.begin() + 1, S.end(), + [](char C) { return C == '_' || isAlnum(C); }); +} + +static bool isUndefined(ld_plugin_symbol &Sym) { + return Sym.def == LDPK_UNDEF || Sym.def == LDPK_WEAKUNDEF; } static void addModule(LTO &Lto, claimed_file &F, const void *View, @@ -616,8 +634,12 @@ static void addModule(LTO &Lto, claimed_file &F, const void *View, toString(ObjOrErr.takeError()).c_str()); unsigned SymNum = 0; + std::unique_ptr<InputFile> Input = std::move(ObjOrErr.get()); + auto InputFileSyms = Input->symbols(); + assert(InputFileSyms.size() == F.syms.size()); std::vector<SymbolResolution> Resols(F.syms.size()); for (ld_plugin_symbol &Sym : F.syms) { + const InputFile::Symbol &InpSym = InputFileSyms[SymNum]; SymbolResolution &R = Resols[SymNum++]; ld_plugin_symbol_resolution Resolution = @@ -638,21 +660,28 @@ static void addModule(LTO &Lto, claimed_file &F, const void *View, break; case LDPR_PREVAILING_DEF_IRONLY: - R.Prevailing = true; + R.Prevailing = !isUndefined(Sym); break; case LDPR_PREVAILING_DEF: - R.Prevailing = true; + R.Prevailing = !isUndefined(Sym); R.VisibleToRegularObj = true; break; case LDPR_PREVAILING_DEF_IRONLY_EXP: - R.Prevailing = true; + R.Prevailing = !isUndefined(Sym); if (!Res.CanOmitFromDynSym) R.VisibleToRegularObj = true; break; } + // If the symbol has a C identifier section name, we need to mark + // it as visible to a regular object so that LTO will keep it around + // to ensure the linker generates special __start_<secname> and + // __stop_<secname> symbols which may be used elsewhere. + if (isValidCIdentifier(InpSym.getSectionName())) + R.VisibleToRegularObj = true; + if (Resolution != LDPR_RESOLVED_DYN && Resolution != LDPR_UNDEF && (IsExecutable || !Res.DefaultVisibility)) R.FinalDefinitionInLinkageUnit = true; @@ -660,27 +689,28 @@ static void addModule(LTO &Lto, claimed_file &F, const void *View, freeSymName(Sym); } - check(Lto.add(std::move(*ObjOrErr), Resols), + check(Lto.add(std::move(Input), Resols), std::string("Failed to link module ") + F.name); } -static void recordFile(std::string Filename, bool TempOutFile) { +static void recordFile(const std::string &Filename, bool TempOutFile) { if (add_input_file(Filename.c_str()) != LDPS_OK) message(LDPL_FATAL, "Unable to add .o file to the link. File left behind in: %s", Filename.c_str()); if (TempOutFile) - Cleanup.push_back(Filename.c_str()); + Cleanup.push_back(Filename); } /// Return the desired output filename given a base input name, a flag /// indicating whether a temp file should be generated, and an optional task id. /// The new filename generated is returned in \p NewFilename. -static void getOutputFileName(SmallString<128> InFilename, bool TempOutFile, - SmallString<128> &NewFilename, int TaskID) { +static int getOutputFileName(StringRef InFilename, bool TempOutFile, + SmallString<128> &NewFilename, int TaskID) { + int FD = -1; if (TempOutFile) { std::error_code EC = - sys::fs::createTemporaryFile("lto-llvm", "o", NewFilename); + sys::fs::createTemporaryFile("lto-llvm", "o", FD, NewFilename); if (EC) message(LDPL_FATAL, "Could not create temporary file: %s", EC.message().c_str()); @@ -688,7 +718,13 @@ static void getOutputFileName(SmallString<128> InFilename, bool TempOutFile, NewFilename = InFilename; if (TaskID > 0) NewFilename += utostr(TaskID); + std::error_code EC = + sys::fs::openFileForWrite(NewFilename, FD, sys::fs::F_None); + if (EC) + message(LDPL_FATAL, "Could not open file %s: %s", NewFilename.c_str(), + EC.message().c_str()); } + return FD; } static CodeGenOpt::Level getCGOptLevel() { @@ -711,9 +747,7 @@ static void getThinLTOOldAndNewPrefix(std::string &OldPrefix, std::string &NewPrefix) { StringRef PrefixReplace = options::thinlto_prefix_replace; assert(PrefixReplace.empty() || PrefixReplace.find(";") != StringRef::npos); - std::pair<StringRef, StringRef> Split = PrefixReplace.split(";"); - OldPrefix = Split.first.str(); - NewPrefix = Split.second.str(); + std::tie(OldPrefix, NewPrefix) = PrefixReplace.split(';'); } static std::unique_ptr<LTO> createLTO() { @@ -727,6 +761,10 @@ static std::unique_ptr<LTO> createLTO() { // FIXME: Check the gold version or add a new option to enable them. Conf.Options.RelaxELFRelocations = false; + // Enable function/data sections by default. + Conf.Options.FunctionSections = true; + Conf.Options.DataSections = true; + Conf.MAttrs = MAttrs; Conf.RelocModel = RelocationModel; Conf.CGOptLevel = getCGOptLevel(); @@ -775,6 +813,9 @@ static std::unique_ptr<LTO> createLTO() { if (!options::sample_profile.empty()) Conf.SampleProfile = options::sample_profile; + // Use new pass manager if set in driver + Conf.UseNewPM = options::new_pass_manager; + return llvm::make_unique<LTO>(std::move(Conf), Backend, options::ParallelCodeGenParallelismLevel); } @@ -872,22 +913,17 @@ static ld_plugin_status allSymbolsReadHook() { auto AddStream = [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> { IsTemporary[Task] = !SaveTemps; - getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps, Filenames[Task], - Task); - int FD; - std::error_code EC = - sys::fs::openFileForWrite(Filenames[Task], FD, sys::fs::F_None); - if (EC) - message(LDPL_FATAL, "Could not open file %s: %s", Filenames[Task].c_str(), - EC.message().c_str()); + int FD = getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps, + Filenames[Task], Task); return llvm::make_unique<lto::NativeObjectStream>( llvm::make_unique<llvm::raw_fd_ostream>(FD, true)); }; - auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) { + auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB, + StringRef Path) { // Note that this requires that the memory buffers provided to AddBuffer are // backed by a file. - Filenames[Task] = MB->getBufferIdentifier(); + Filenames[Task] = Path; }; NativeObjectCache Cache; @@ -946,5 +982,11 @@ static ld_plugin_status cleanup_hook(void) { EC.message().c_str()); } + // Prune cache + if (!options::cache_policy.empty()) { + CachePruningPolicy policy = check(parseCachePruningPolicy(options::cache_policy)); + pruneCache(options::cache_dir, policy); + } + return LDPS_OK; } diff --git a/tools/llc/llc.cpp b/tools/llc/llc.cpp index f13a19213c69..a4810890f9b4 100644 --- a/tools/llc/llc.cpp +++ b/tools/llc/llc.cpp @@ -13,17 +13,17 @@ // //===----------------------------------------------------------------------===// - #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/CommandFlags.def" #include "llvm/CodeGen/LinkAllAsmWriterComponents.h" #include "llvm/CodeGen/LinkAllCodegenComponents.h" #include "llvm/CodeGen/MIRParser/MIRParser.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" @@ -49,7 +49,6 @@ #include "llvm/Support/TargetSelect.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetSubtargetInfo.h" #include "llvm/Transforms/Utils/Cloning.h" #include <memory> using namespace llvm; @@ -126,22 +125,6 @@ static cl::opt<bool> DiscardValueNames( cl::desc("Discard names from Value (other than GlobalValue)."), cl::init(false), cl::Hidden); -static cl::opt<std::string> StopBefore("stop-before", - cl::desc("Stop compilation before a specific pass"), - cl::value_desc("pass-name"), cl::init("")); - -static cl::opt<std::string> StopAfter("stop-after", - cl::desc("Stop compilation after a specific pass"), - cl::value_desc("pass-name"), cl::init("")); - -static cl::opt<std::string> StartBefore("start-before", - cl::desc("Resume compilation before a specific pass"), - cl::value_desc("pass-name"), cl::init("")); - -static cl::opt<std::string> StartAfter("start-after", - cl::desc("Resume compilation after a specific pass"), - cl::value_desc("pass-name"), cl::init("")); - static cl::list<std::string> IncludeDirs("I", cl::desc("include search path")); static cl::opt<bool> PassRemarksWithHotness( @@ -183,9 +166,9 @@ static cl::opt<RunPassOption, true, cl::parser<std::string>> RunPass( static int compileModule(char **, LLVMContext &); -static std::unique_ptr<tool_output_file> -GetOutputStream(const char *TargetName, Triple::OSType OS, - const char *ProgName) { +static std::unique_ptr<ToolOutputFile> GetOutputStream(const char *TargetName, + Triple::OSType OS, + const char *ProgName) { // If we don't yet have an output filename, make one. if (OutputFilename.empty()) { if (InputFilename == "-") @@ -241,8 +224,7 @@ GetOutputStream(const char *TargetName, Triple::OSType OS, sys::fs::OpenFlags OpenFlags = sys::fs::F_None; if (!Binary) OpenFlags |= sys::fs::F_Text; - auto FDOut = llvm::make_unique<tool_output_file>(OutputFilename, EC, - OpenFlags); + auto FDOut = llvm::make_unique<ToolOutputFile>(OutputFilename, EC, OpenFlags); if (EC) { errs() << EC.message() << '\n'; return nullptr; @@ -251,20 +233,24 @@ GetOutputStream(const char *TargetName, Triple::OSType OS, return FDOut; } -static void DiagnosticHandler(const DiagnosticInfo &DI, void *Context) { - bool *HasError = static_cast<bool *>(Context); - if (DI.getSeverity() == DS_Error) - *HasError = true; - - if (auto *Remark = dyn_cast<DiagnosticInfoOptimizationBase>(&DI)) - if (!Remark->isEnabled()) - return; - - DiagnosticPrinterRawOStream DP(errs()); - errs() << LLVMContext::getDiagnosticMessagePrefix(DI.getSeverity()) << ": "; - DI.print(DP); - errs() << "\n"; -} +struct LLCDiagnosticHandler : public DiagnosticHandler { + bool *HasError; + LLCDiagnosticHandler(bool *HasErrorPtr) : HasError(HasErrorPtr) {} + bool handleDiagnostics(const DiagnosticInfo &DI) override { + if (DI.getSeverity() == DS_Error) + *HasError = true; + + if (auto *Remark = dyn_cast<DiagnosticInfoOptimizationBase>(&DI)) + if (!Remark->isEnabled()) + return true; + + DiagnosticPrinterRawOStream DP(errs()); + errs() << LLVMContext::getDiagnosticMessagePrefix(DI.getSeverity()) << ": "; + DI.print(DP); + errs() << "\n"; + return true; + } +}; static void InlineAsmDiagHandler(const SMDiagnostic &SMD, void *Context, unsigned LocCookie) { @@ -304,7 +290,8 @@ int main(int argc, char **argv) { initializeCodeGen(*Registry); initializeLoopStrengthReducePass(*Registry); initializeLowerIntrinsicsPass(*Registry); - initializeCountingFunctionInserterPass(*Registry); + initializeEntryExitInstrumenterPass(*Registry); + initializePostInlineEntryExitInstrumenterPass(*Registry); initializeUnreachableBlockElimLegacyPassPass(*Registry); initializeConstantHoistingLegacyPassPass(*Registry); initializeScalarOpts(*Registry); @@ -324,7 +311,8 @@ int main(int argc, char **argv) { // Set a diagnostic handler that doesn't exit on the first error bool HasError = false; - Context.setDiagnosticHandler(DiagnosticHandler, &HasError); + Context.setDiagnosticHandler( + llvm::make_unique<LLCDiagnosticHandler>(&HasError)); Context.setInlineAsmDiagnosticHandler(InlineAsmDiagHandler, &HasError); if (PassRemarksWithHotness) @@ -333,11 +321,11 @@ int main(int argc, char **argv) { if (PassRemarksHotnessThreshold) Context.setDiagnosticsHotnessThreshold(PassRemarksHotnessThreshold); - std::unique_ptr<tool_output_file> YamlFile; + std::unique_ptr<ToolOutputFile> YamlFile; if (RemarksFilename != "") { std::error_code EC; - YamlFile = llvm::make_unique<tool_output_file>(RemarksFilename, EC, - sys::fs::F_None); + YamlFile = + llvm::make_unique<ToolOutputFile>(RemarksFilename, EC, sys::fs::F_None); if (EC) { errs() << EC.message() << '\n'; return 1; @@ -389,20 +377,6 @@ static bool addPass(PassManagerBase &PM, const char *argv0, return false; } -static AnalysisID getPassID(const char *argv0, const char *OptionName, - StringRef PassName) { - if (PassName.empty()) - return nullptr; - - const PassRegistry &PR = *PassRegistry::getPassRegistry(); - const PassInfo *PI = PR.getPassInfo(PassName); - if (!PI) { - errs() << argv0 << ": " << OptionName << " pass is not registered.\n"; - exit(1); - } - return PI->getTypeInfo(); -} - static int compileModule(char **argv, LLVMContext &Context) { // Load the module to be compiled... SMDiagnostic Err; @@ -478,9 +452,9 @@ static int compileModule(char **argv, LLVMContext &Context) { Options.MCOptions.IASSearchPaths = IncludeDirs; Options.MCOptions.SplitDwarfFile = SplitDwarfFile; - std::unique_ptr<TargetMachine> Target( - TheTarget->createTargetMachine(TheTriple.getTriple(), CPUStr, FeaturesStr, - Options, getRelocModel(), CMModel, OLvl)); + std::unique_ptr<TargetMachine> Target(TheTarget->createTargetMachine( + TheTriple.getTriple(), CPUStr, FeaturesStr, Options, getRelocModel(), + getCodeModel(), OLvl)); assert(Target && "Could not allocate target machine!"); @@ -495,7 +469,7 @@ static int compileModule(char **argv, LLVMContext &Context) { Options.FloatABIType = FloatABIForCalls; // Figure out where we are going to send the output. - std::unique_ptr<tool_output_file> Out = + std::unique_ptr<ToolOutputFile> Out = GetOutputStream(TheTarget->getName(), TheTriple.getOS(), argv[0]); if (!Out) return 1; @@ -537,68 +511,46 @@ static int compileModule(char **argv, LLVMContext &Context) { } const char *argv0 = argv[0]; - AnalysisID StartBeforeID = getPassID(argv0, "start-before", StartBefore); - AnalysisID StartAfterID = getPassID(argv0, "start-after", StartAfter); - AnalysisID StopAfterID = getPassID(argv0, "stop-after", StopAfter); - AnalysisID StopBeforeID = getPassID(argv0, "stop-before", StopBefore); - if (StartBeforeID && StartAfterID) { - errs() << argv0 << ": -start-before and -start-after specified!\n"; - return 1; - } - if (StopBeforeID && StopAfterID) { - errs() << argv0 << ": -stop-before and -stop-after specified!\n"; - return 1; - } - - if (MIR) { - // Construct a custom pass pipeline that starts after instruction - // selection. - LLVMTargetMachine &LLVMTM = static_cast<LLVMTargetMachine&>(*Target); + LLVMTargetMachine &LLVMTM = static_cast<LLVMTargetMachine&>(*Target); + MachineModuleInfo *MMI = new MachineModuleInfo(&LLVMTM); + + // Construct a custom pass pipeline that starts after instruction + // selection. + if (!RunPassNames->empty()) { + if (!MIR) { + errs() << argv0 << ": run-pass is for .mir file only.\n"; + return 1; + } TargetPassConfig &TPC = *LLVMTM.createPassConfig(PM); + if (TPC.hasLimitedCodeGenPipeline()) { + errs() << argv0 << ": run-pass cannot be used with " + << TPC.getLimitedCodeGenPipelineReason(" and ") << ".\n"; + return 1; + } + TPC.setDisableVerify(NoVerify); PM.add(&TPC); - MachineModuleInfo *MMI = new MachineModuleInfo(&LLVMTM); - if (MIR->parseMachineFunctions(*M, *MMI)) - return 1; PM.add(MMI); TPC.printAndVerify(""); - - if (!RunPassNames->empty()) { - if (!StartAfter.empty() || !StopAfter.empty() || !StartBefore.empty() || - !StopBefore.empty()) { - errs() << argv0 << ": start-after and/or stop-after passes are " - "redundant when run-pass is specified.\n"; + for (const std::string &RunPassName : *RunPassNames) { + if (addPass(PM, argv0, RunPassName, TPC)) return 1; - } - - for (const std::string &RunPassName : *RunPassNames) { - if (addPass(PM, argv0, RunPassName, TPC)) - return 1; - } - } else { - TPC.setStartStopPasses(StartBeforeID, StartAfterID, StopBeforeID, - StopAfterID); - TPC.addISelPasses(); - TPC.addMachinePasses(); } TPC.setInitialized(); - - if (!StopBefore.empty() || !StopAfter.empty() || !RunPassNames->empty()) { - PM.add(createPrintMIRPass(*OS)); - } else if (LLVMTM.addAsmPrinter(PM, *OS, FileType, MMI->getContext())) { - errs() << argv0 << ": target does not support generation of this" - << " file type!\n"; - return 1; - } + PM.add(createPrintMIRPass(*OS)); PM.add(createFreeMachineFunctionPass()); - } else if (Target->addPassesToEmitFile(PM, *OS, FileType, NoVerify, - StartBeforeID, StartAfterID, - StopBeforeID, StopAfterID)) { + } else if (Target->addPassesToEmitFile(PM, *OS, FileType, NoVerify, MMI)) { errs() << argv0 << ": target does not support generation of this" - << " file type!\n"; + << " file type!\n"; return 1; } + if (MIR) { + assert(MMI && "Forgot to create MMI?"); + if (MIR->parseMachineFunctions(*M, *MMI)) + return 1; + } + // Before executing passes, print the final values of the LLVM options. cl::PrintOptionValues(); @@ -616,8 +568,9 @@ static int compileModule(char **argv, LLVMContext &Context) { PM.run(*M); - auto HasError = *static_cast<bool *>(Context.getDiagnosticContext()); - if (HasError) + auto HasError = + ((const LLCDiagnosticHandler *)(Context.getDiagHandlerPtr()))->HasError; + if (*HasError) return 1; // Compare the two outputs and make sure they're the same diff --git a/tools/lli/OrcLazyJIT.h b/tools/lli/OrcLazyJIT.h index 47a2acc4d7e6..a5cc804bb045 100644 --- a/tools/lli/OrcLazyJIT.h +++ b/tools/lli/OrcLazyJIT.h @@ -15,15 +15,16 @@ #ifndef LLVM_TOOLS_LLI_ORCLAZYJIT_H #define LLVM_TOOLS_LLI_ORCLAZYJIT_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Twine.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" #include "llvm/ExecutionEngine/Orc/CompileUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" -#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ExecutionEngine/Orc/LambdaResolver.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" @@ -61,8 +62,8 @@ public: IndirectStubsManagerBuilder IndirectStubsMgrBuilder, bool InlineStubs) : TM(std::move(TM)), DL(this->TM->createDataLayout()), - CCMgr(std::move(CCMgr)), - ObjectLayer([]() { return std::make_shared<SectionMemoryManager>(); }), + CCMgr(std::move(CCMgr)), + ObjectLayer([]() { return std::make_shared<SectionMemoryManager>(); }), CompileLayer(ObjectLayer, orc::SimpleCompiler(*this->TM)), IRDumpLayer(CompileLayer, createDebugDumper()), CODLayer(IRDumpLayer, extractSingleFunction, *this->CCMgr, @@ -112,7 +113,7 @@ public: // 1) Search the JIT symbols. // 2) Check for C++ runtime overrides. // 3) Search the host process (LLI)'s symbol table. - if (ModulesHandle == CODLayerT::ModuleHandleT()) { + if (!ModulesHandle) { auto Resolver = orc::createLambdaResolver( [this](const std::string &Name) -> JITSymbol { @@ -135,19 +136,18 @@ public: else return ModulesHandleOrErr.takeError(); - } else - if (auto Err = CODLayer.addExtraModule(ModulesHandle, std::move(M))) - return Err; + } else if (auto Err = CODLayer.addExtraModule(*ModulesHandle, std::move(M))) + return Err; // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. orc::CtorDtorRunner<CODLayerT> CtorRunner(std::move(CtorNames), - ModulesHandle); + *ModulesHandle); if (auto Err = CtorRunner.runViaLayer(CODLayer)) return Err; IRStaticDestructorRunners.emplace_back(std::move(DtorNames), - ModulesHandle); + *ModulesHandle); return Error::success(); } @@ -190,7 +190,7 @@ private: orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; std::vector<orc::CtorDtorRunner<CODLayerT>> IRStaticDestructorRunners; - CODLayerT::ModuleHandleT ModulesHandle; + llvm::Optional<CODLayerT::ModuleHandleT> ModulesHandle; }; int runOrcLazyJIT(std::vector<std::unique_ptr<Module>> Ms, diff --git a/tools/lli/lli.cpp b/tools/lli/lli.cpp index 091ca22b4e82..a33c51d77877 100644 --- a/tools/lli/lli.cpp +++ b/tools/lli/lli.cpp @@ -15,20 +15,21 @@ #include "OrcLazyJIT.h" #include "RemoteJITUtils.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/CodeGen/CommandFlags.def" #include "llvm/CodeGen/LinkAllCodegenComponents.h" #include "llvm/ExecutionEngine/GenericValue.h" #include "llvm/ExecutionEngine/Interpreter.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/MCJIT.h" #include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h" #include "llvm/ExecutionEngine/OrcMCJITReplacement.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" -#include "llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/TypeBuilder.h" @@ -124,22 +125,6 @@ namespace { TargetTriple("mtriple", cl::desc("Override target triple for module")); cl::opt<std::string> - MArch("march", - cl::desc("Architecture to generate assembly for (see --version)")); - - cl::opt<std::string> - MCPU("mcpu", - cl::desc("Target a specific cpu type (-mcpu=help for details)"), - cl::value_desc("cpu-name"), - cl::init("")); - - cl::list<std::string> - MAttrs("mattr", - cl::CommaSeparated, - cl::desc("Target specific attributes (-mattr=help for details)"), - cl::value_desc("a1,+a2,-a3,...")); - - cl::opt<std::string> EntryFunc("entry-function", cl::desc("Specify the entry function (default = 'main') " "of the executable"), @@ -186,47 +171,11 @@ namespace { cl::desc("Disable JIT lazy compilation"), cl::init(false)); - cl::opt<Reloc::Model> RelocModel( - "relocation-model", cl::desc("Choose relocation model"), - cl::values( - clEnumValN(Reloc::Static, "static", "Non-relocatable code"), - clEnumValN(Reloc::PIC_, "pic", - "Fully relocatable, position independent code"), - clEnumValN(Reloc::DynamicNoPIC, "dynamic-no-pic", - "Relocatable external references, non-relocatable code"))); - - cl::opt<llvm::CodeModel::Model> - CMModel("code-model", - cl::desc("Choose code model"), - cl::init(CodeModel::JITDefault), - cl::values(clEnumValN(CodeModel::JITDefault, "default", - "Target default JIT code model"), - clEnumValN(CodeModel::Small, "small", - "Small code model"), - clEnumValN(CodeModel::Kernel, "kernel", - "Kernel code model"), - clEnumValN(CodeModel::Medium, "medium", - "Medium code model"), - clEnumValN(CodeModel::Large, "large", - "Large code model"))); - cl::opt<bool> GenerateSoftFloatCalls("soft-float", cl::desc("Generate software floating point library calls"), cl::init(false)); - cl::opt<llvm::FloatABI::ABIType> - FloatABIForCalls("float-abi", - cl::desc("Choose float ABI type"), - cl::init(FloatABI::Default), - cl::values( - clEnumValN(FloatABI::Default, "default", - "Target default float ABI type"), - clEnumValN(FloatABI::Soft, "soft", - "Soft float ABI (implied by -soft-float)"), - clEnumValN(FloatABI::Hard, "hard", - "Hard float ABI (uses FP registers)"))); - ExitOnError ExitOnErr; } @@ -433,7 +382,8 @@ int main(int argc, char **argv, char * const *envp) { builder.setMAttrs(MAttrs); if (RelocModel.getNumOccurrences()) builder.setRelocationModel(RelocModel); - builder.setCodeModel(CMModel); + if (CMModel.getNumOccurrences()) + builder.setCodeModel(CMModel); builder.setErrorStr(&ErrorMsg); builder.setEngineKind(ForceInterpreter ? EngineKind::Interpreter @@ -464,7 +414,7 @@ int main(int argc, char **argv, char * const *envp) { builder.setOptLevel(getOptLevel()); - TargetOptions Options; + TargetOptions Options = InitTargetOptionsFromCodeGenFlags(); if (FloatABIForCalls != FloatABI::Default) Options.FloatABIType = FloatABIForCalls; @@ -657,28 +607,25 @@ int main(int argc, char **argv, char * const *envp) { } // Create a remote target client running over the channel. - typedef orc::remote::OrcRemoteTargetClient<orc::rpc::RawByteChannel> - MyRemote; - auto R = ExitOnErr(MyRemote::Create(*C)); + typedef orc::remote::OrcRemoteTargetClient MyRemote; + auto R = ExitOnErr(MyRemote::Create(*C, ExitOnErr)); // Create a remote memory manager. - std::unique_ptr<MyRemote::RCMemoryManager> RemoteMM; - ExitOnErr(R->createRemoteMemoryManager(RemoteMM)); + auto RemoteMM = ExitOnErr(R->createRemoteMemoryManager()); // Forward MCJIT's memory manager calls to the remote memory manager. static_cast<ForwardingMemoryManager*>(RTDyldMM)->setMemMgr( std::move(RemoteMM)); // Forward MCJIT's symbol resolution calls to the remote. - static_cast<ForwardingMemoryManager*>(RTDyldMM)->setResolver( - orc::createLambdaResolver( - [](const std::string &Name) { return nullptr; }, - [&](const std::string &Name) { - if (auto Addr = ExitOnErr(R->getSymbolAddress(Name))) - return JITSymbol(Addr, JITSymbolFlags::Exported); - return JITSymbol(nullptr); - } - )); + static_cast<ForwardingMemoryManager *>(RTDyldMM)->setResolver( + orc::createLambdaResolver( + [](const std::string &Name) { return nullptr; }, + [&](const std::string &Name) { + if (auto Addr = ExitOnErr(R->getSymbolAddress(Name))) + return JITSymbol(Addr, JITSymbolFlags::Exported); + return JITSymbol(nullptr); + })); // Grab the target address of the JIT'd main function on the remote and call // it. diff --git a/tools/llvm-ar/CMakeLists.txt b/tools/llvm-ar/CMakeLists.txt index 731bcbd8ac9d..2970a59beee2 100644 --- a/tools/llvm-ar/CMakeLists.txt +++ b/tools/llvm-ar/CMakeLists.txt @@ -17,3 +17,9 @@ add_llvm_tool(llvm-ar add_llvm_tool_symlink(llvm-ranlib llvm-ar) add_llvm_tool_symlink(llvm-lib llvm-ar) add_llvm_tool_symlink(llvm-dlltool llvm-ar) + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(ar llvm-ar) + add_llvm_tool_symlink(dlltool llvm-ar) + add_llvm_tool_symlink(ranlib llvm-ar) +endif() diff --git a/tools/llvm-ar/llvm-ar.cpp b/tools/llvm-ar/llvm-ar.cpp index af4d3efa52f7..ae7d1a7f1b7a 100644 --- a/tools/llvm-ar/llvm-ar.cpp +++ b/tools/llvm-ar/llvm-ar.cpp @@ -15,7 +15,6 @@ #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" #include "llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h" #include "llvm/ToolDrivers/llvm-lib/LibDriver.h" #include "llvm/Object/Archive.h" @@ -36,9 +35,6 @@ #include "llvm/Support/TargetSelect.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <cstdlib> -#include <memory> #if !defined(_MSC_VER) && !defined(__MINGW32__) #include <unistd.h> @@ -54,6 +50,7 @@ static StringRef ToolName; // Show the error message and exit. LLVM_ATTRIBUTE_NORETURN static void fail(Twine Error) { errs() << ToolName << ": " << Error << ".\n"; + cl::PrintHelpMessage(); exit(1); } @@ -126,6 +123,8 @@ static cl::extrahelp MoreHelp( " [v] - be verbose about actions taken\n" ); +static const char OptionChars[] = "dmpqrtxabiosSTucv"; + // This enumeration delineates the kinds of operations on an archive // that are permitted. enum ArchiveOperation { @@ -164,26 +163,18 @@ static std::string ArchiveName; // on the command line. static std::vector<StringRef> Members; -// Show the error message, the help message and exit. -LLVM_ATTRIBUTE_NORETURN static void -show_help(const std::string &msg) { - errs() << ToolName << ": " << msg << "\n\n"; - cl::PrintHelpMessage(); - exit(1); -} - // Extract the member filename from the command line for the [relpos] argument // associated with a, b, and i modifiers static void getRelPos() { if(RestOfArgs.size() == 0) - show_help("Expected [relpos] for a, b, or i modifier"); + fail("Expected [relpos] for a, b, or i modifier"); RelPos = RestOfArgs[0]; RestOfArgs.erase(RestOfArgs.begin()); } static void getOptions() { if(RestOfArgs.size() == 0) - show_help("Expected options"); + fail("Expected options"); Options = RestOfArgs[0]; RestOfArgs.erase(RestOfArgs.begin()); } @@ -191,7 +182,7 @@ static void getOptions() { // Get the archive file name from the command line static void getArchive() { if(RestOfArgs.size() == 0) - show_help("An archive name must be specified"); + fail("An archive name must be specified"); ArchiveName = RestOfArgs[0]; RestOfArgs.erase(RestOfArgs.begin()); } @@ -275,7 +266,7 @@ static ArchiveOperation parseCommandLine() { Thin = true; break; default: - cl::PrintHelpMessage(); + fail(std::string("unknown option ") + Options[i]); } } @@ -290,26 +281,26 @@ static ArchiveOperation parseCommandLine() { NumOperations = 1; Operation = CreateSymTab; if (!Members.empty()) - show_help("The s operation takes only an archive as argument"); + fail("The s operation takes only an archive as argument"); } // Perform various checks on the operation/modifier specification // to make sure we are dealing with a legal request. if (NumOperations == 0) - show_help("You must specify at least one of the operations"); + fail("You must specify at least one of the operations"); if (NumOperations > 1) - show_help("Only one operation may be specified"); + fail("Only one operation may be specified"); if (NumPositional > 1) - show_help("You may only specify one of a, b, and i modifiers"); + fail("You may only specify one of a, b, and i modifiers"); if (AddAfter || AddBefore) { if (Operation != Move && Operation != ReplaceOrInsert) - show_help("The 'a', 'b' and 'i' modifiers can only be specified with " - "the 'm' or 'r' operations"); + fail("The 'a', 'b' and 'i' modifiers can only be specified with " + "the 'm' or 'r' operations"); } if (OriginalDates && Operation != Extract) - show_help("The 'o' modifier is only applicable to the 'x' operation"); + fail("The 'o' modifier is only applicable to the 'x' operation"); if (OnlyUpdate && Operation != ReplaceOrInsert) - show_help("The 'u' modifier is only applicable to the 'r' operation"); + fail("The 'u' modifier is only applicable to the 'r' operation"); // Return the parsed operation to the caller return Operation; @@ -688,10 +679,10 @@ performWriteOperation(ArchiveOperation Operation, break; } - std::pair<StringRef, std::error_code> Result = + Error E = writeArchive(ArchiveName, NewMembersP ? *NewMembersP : NewMembers, Symtab, Kind, Deterministic, Thin, std::move(OldArchiveBuf)); - failIfError(Result.second, Result.first); + failIfError(std::move(E), ArchiveName); } static void createSymbolTable(object::Archive *OldArchive) { @@ -871,6 +862,24 @@ int main(int argc, char **argv) { Stem.find("lib") != StringRef::npos) return libDriverMain(makeArrayRef(argv, argc)); + for (int i = 1; i < argc; i++) { + // If an argument starts with a dash and only contains chars + // that belong to the options chars set, remove the dash. + // We can't handle it after the command line options parsing + // is done, since it will error out on an unrecognized string + // starting with a dash. + // Make sure this doesn't match the actual llvm-ar specific options + // that start with a dash. + StringRef S = argv[i]; + if (S.startswith("-") && + S.find_first_not_of(OptionChars, 1) == StringRef::npos) { + argv[i]++; + break; + } + if (S == "--") + break; + } + // Have the command line options parsed and handle things // like --help and --version. cl::ParseCommandLineOptions(argc, argv, diff --git a/tools/llvm-as-fuzzer/CMakeLists.txt b/tools/llvm-as-fuzzer/CMakeLists.txt index ff9bdaaf4c43..4d75ad4825a8 100644 --- a/tools/llvm-as-fuzzer/CMakeLists.txt +++ b/tools/llvm-as-fuzzer/CMakeLists.txt @@ -1,13 +1,7 @@ -if( LLVM_USE_SANITIZE_COVERAGE ) - set(LLVM_LINK_COMPONENTS - AsmParser - BitWriter - Core - Support - ) - add_llvm_tool(llvm-as-fuzzer - llvm-as-fuzzer.cpp) - target_link_libraries(llvm-as-fuzzer - LLVMFuzzer - ) -endif() +set(LLVM_LINK_COMPONENTS + AsmParser + BitWriter + Core + Support +) +add_llvm_fuzzer(llvm-as-fuzzer llvm-as-fuzzer.cpp) diff --git a/tools/llvm-as/llvm-as.cpp b/tools/llvm-as/llvm-as.cpp index 06bc2e990105..9f0f162b74f8 100644 --- a/tools/llvm-as/llvm-as.cpp +++ b/tools/llvm-as/llvm-as.cpp @@ -72,8 +72,8 @@ static void WriteOutputFile(const Module *M) { } std::error_code EC; - std::unique_ptr<tool_output_file> Out( - new tool_output_file(OutputFilename, EC, sys::fs::F_None)); + std::unique_ptr<ToolOutputFile> Out( + new ToolOutputFile(OutputFilename, EC, sys::fs::F_None)); if (EC) { errs() << EC.message() << '\n'; exit(1); @@ -97,7 +97,8 @@ int main(int argc, char **argv) { // Parse the file now... SMDiagnostic Err; - std::unique_ptr<Module> M = parseAssemblyFile(InputFilename, Err, Context); + std::unique_ptr<Module> M = + parseAssemblyFile(InputFilename, Err, Context, nullptr, !DisableVerify); if (!M.get()) { Err.print(argv[0], errs()); return 1; diff --git a/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp b/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp index 529bdf5b7d93..7f20e136eefd 100644 --- a/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp +++ b/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp @@ -31,7 +31,6 @@ #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitstreamReader.h" #include "llvm/Bitcode/LLVMBitCodes.h" -#include "llvm/IR/Verifier.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Format.h" #include "llvm/Support/ManagedStatic.h" @@ -40,10 +39,6 @@ #include "llvm/Support/SHA1.h" #include "llvm/Support/Signals.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <cctype> -#include <map> -#include <system_error> using namespace llvm; static cl::opt<std::string> diff --git a/tools/llvm-c-test/CMakeLists.txt b/tools/llvm-c-test/CMakeLists.txt index a2bde0d9714f..bce0f4a5a420 100644 --- a/tools/llvm-c-test/CMakeLists.txt +++ b/tools/llvm-c-test/CMakeLists.txt @@ -38,6 +38,7 @@ endif () add_llvm_tool(llvm-c-test attributes.c calc.c + debuginfo.c diagnostic.c disassemble.c echo.cpp diff --git a/tools/llvm-c-test/debuginfo.c b/tools/llvm-c-test/debuginfo.c new file mode 100644 index 000000000000..c88c74167b63 --- /dev/null +++ b/tools/llvm-c-test/debuginfo.c @@ -0,0 +1,36 @@ +/*===-- debuginfo.c - tool for testing libLLVM and llvm-c API -------------===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* Tests for the LLVM C DebugInfo API *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#include "llvm-c/DebugInfo.h" +#include <stdio.h> + +int llvm_test_dibuilder(void) { + LLVMModuleRef M = LLVMModuleCreateWithName("debuginfo.c"); + LLVMDIBuilderRef DIB = LLVMCreateDIBuilder(M); + + LLVMMetadataRef File = LLVMDIBuilderCreateFile(DIB, "debuginfo.c", 12, + ".", 1); + + LLVMDIBuilderCreateCompileUnit(DIB, + LLVMDWARFSourceLanguageC, File,"llvm-c-test", 11, 0, NULL, 0, 0, + NULL, 0, LLVMDWARFEmissionFull, 0, 0, 0); + + char *MStr = LLVMPrintModuleToString(M); + puts(MStr); + LLVMDisposeMessage(MStr); + + LLVMDisposeDIBuilder(DIB); + LLVMDisposeModule(M); + + return 0; +} diff --git a/tools/llvm-c-test/echo.cpp b/tools/llvm-c-test/echo.cpp index 966c0083bf87..885f9c0d899d 100644 --- a/tools/llvm-c-test/echo.cpp +++ b/tools/llvm-c-test/echo.cpp @@ -142,7 +142,7 @@ struct TypeCloner { LLVMGetVectorSize(Src) ); case LLVMMetadataTypeKind: - break; + return LLVMMetadataTypeInContext(Ctx); case LLVMX86_MMXTypeKind: return LLVMX86MMXTypeInContext(Ctx); default: diff --git a/tools/llvm-c-test/helpers.c b/tools/llvm-c-test/helpers.c index 97fbaab6d6c3..9af88bd8be90 100644 --- a/tools/llvm-c-test/helpers.c +++ b/tools/llvm-c-test/helpers.c @@ -11,7 +11,6 @@ |* *| \*===----------------------------------------------------------------------===*/ -#include "llvm-c-test.h" #include <stdio.h> #include <string.h> diff --git a/tools/llvm-c-test/llvm-c-test.h b/tools/llvm-c-test/llvm-c-test.h index 2a7090484b0e..cf9a0f99de64 100644 --- a/tools/llvm-c-test/llvm-c-test.h +++ b/tools/llvm-c-test/llvm-c-test.h @@ -35,6 +35,9 @@ int llvm_calc(void); // disassemble.c int llvm_disassemble(void); +// debuginfo.c +int llvm_test_dibuilder(void); + // metadata.c int llvm_add_named_metadata_operand(void); int llvm_set_metadata(void); diff --git a/tools/llvm-c-test/main.c b/tools/llvm-c-test/main.c index 9bc0c96c3d65..60ab7f0f9009 100644 --- a/tools/llvm-c-test/main.c +++ b/tools/llvm-c-test/main.c @@ -12,9 +12,7 @@ \*===----------------------------------------------------------------------===*/ #include "llvm-c-test.h" -#include "llvm-c/BitReader.h" #include <stdio.h> -#include <stdlib.h> #include <string.h> static void print_usage(void) { @@ -55,6 +53,9 @@ static void print_usage(void) { fprintf(stderr, " * --test-diagnostic-handler\n"); fprintf(stderr, " Read bitcode file form stdin with a diagnostic handler set\n\n"); + fprintf(stderr, " * --test-dibuilder\n"); + fprintf(stderr, + " Run tests for the DIBuilder C API - print generated module\n\n"); } int main(int argc, char **argv) { @@ -96,6 +97,8 @@ int main(int argc, char **argv) { return llvm_echo(); } else if (argc == 2 && !strcmp(argv[1], "--test-diagnostic-handler")) { return llvm_test_diagnostic_handler(); + } else if (argc == 2 && !strcmp(argv[1], "--test-dibuilder")) { + return llvm_test_dibuilder(); } else { print_usage(); } diff --git a/tools/llvm-c-test/module.c b/tools/llvm-c-test/module.c index c47b55d50294..cbb44d0bd15e 100644 --- a/tools/llvm-c-test/module.c +++ b/tools/llvm-c-test/module.c @@ -16,7 +16,6 @@ #include "llvm-c/BitReader.h" #include <stdio.h> #include <stdlib.h> -#include <string.h> static void diagnosticHandler(LLVMDiagnosticInfoRef DI, void *C) { char *CErr = LLVMGetDiagInfoDescription(DI); diff --git a/tools/llvm-cat/llvm-cat.cpp b/tools/llvm-cat/llvm-cat.cpp index 8a21a6d07caa..26e0a0d6c61e 100644 --- a/tools/llvm-cat/llvm-cat.cpp +++ b/tools/llvm-cat/llvm-cat.cpp @@ -1,4 +1,4 @@ -//===-- llvm-cat.cpp - LLVM module concatenation utility ------------------===// +//===- llvm-cat.cpp - LLVM module concatenation utility -------------------===// // // The LLVM Compiler Infrastructure // @@ -13,11 +13,23 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/SmallVector.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/IRReader/IRReader.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <memory> +#include <string> +#include <system_error> +#include <vector> using namespace llvm; @@ -70,8 +82,8 @@ int main(int argc, char **argv) { std::error_code EC; raw_fd_ostream OS(OutputFilename, EC, sys::fs::OpenFlags::F_None); if (EC) { - llvm::errs() << argv[0] << ": cannot open " << OutputFilename - << " for writing: " << EC.message(); + errs() << argv[0] << ": cannot open " << OutputFilename << " for writing: " + << EC.message(); return 1; } diff --git a/tools/llvm-cfi-verify/CMakeLists.txt b/tools/llvm-cfi-verify/CMakeLists.txt new file mode 100644 index 000000000000..7a008a66770c --- /dev/null +++ b/tools/llvm-cfi-verify/CMakeLists.txt @@ -0,0 +1,18 @@ +set(LLVM_LINK_COMPONENTS + AllTargetsAsmPrinters + AllTargetsAsmParsers + AllTargetsDescs + AllTargetsDisassemblers + AllTargetsInfos + MC + MCParser + Object + Support + Symbolize + ) + +add_llvm_tool(llvm-cfi-verify + llvm-cfi-verify.cpp) + +add_subdirectory(lib) +target_link_libraries(llvm-cfi-verify PRIVATE LLVMCFIVerify) diff --git a/tools/llvm-cfi-verify/LLVMBuild.txt b/tools/llvm-cfi-verify/LLVMBuild.txt new file mode 100644 index 000000000000..d5e932302728 --- /dev/null +++ b/tools/llvm-cfi-verify/LLVMBuild.txt @@ -0,0 +1,22 @@ +;===- ./tools/llvm-cfi-verify/LLVMBuild.txt --------------------*- Conf -*--===; +; +; The LLVM Compiler Infrastructure +; +; This file is distributed under the University of Illinois Open Source +; License. See LICENSE.TXT for details. +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Tool +name = llvm-cfi-verify +parent = Tools +required_libraries = all-targets MC MCDisassembler MCParser Support Symbolize diff --git a/tools/llvm-cfi-verify/lib/CMakeLists.txt b/tools/llvm-cfi-verify/lib/CMakeLists.txt new file mode 100644 index 000000000000..cd728e004b26 --- /dev/null +++ b/tools/llvm-cfi-verify/lib/CMakeLists.txt @@ -0,0 +1,17 @@ +add_library(LLVMCFIVerify + STATIC + FileAnalysis.cpp + FileAnalysis.h + GraphBuilder.cpp + GraphBuilder.h) + +llvm_update_compile_flags(LLVMCFIVerify) +llvm_map_components_to_libnames(libs + DebugInfoDWARF + MC + MCParser + Object + Support + Symbolize) +target_link_libraries(LLVMCFIVerify ${libs}) +set_target_properties(LLVMCFIVerify PROPERTIES FOLDER "Libraries")
\ No newline at end of file diff --git a/tools/llvm-cfi-verify/lib/FileAnalysis.cpp b/tools/llvm-cfi-verify/lib/FileAnalysis.cpp new file mode 100644 index 000000000000..754825447183 --- /dev/null +++ b/tools/llvm-cfi-verify/lib/FileAnalysis.cpp @@ -0,0 +1,501 @@ +//===- FileAnalysis.cpp -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FileAnalysis.h" +#include "GraphBuilder.h" + +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstPrinter.h" +#include "llvm/MC/MCInstrAnalysis.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/raw_ostream.h" + + +using Instr = llvm::cfi_verify::FileAnalysis::Instr; +using LLVMSymbolizer = llvm::symbolize::LLVMSymbolizer; + +namespace llvm { +namespace cfi_verify { + +bool IgnoreDWARFFlag; + +static cl::opt<bool, true> IgnoreDWARFArg( + "ignore-dwarf", + cl::desc( + "Ignore all DWARF data. This relaxes the requirements for all " + "statically linked libraries to have been compiled with '-g', but " + "will result in false positives for 'CFI unprotected' instructions."), + cl::location(IgnoreDWARFFlag), cl::init(false)); + +StringRef stringCFIProtectionStatus(CFIProtectionStatus Status) { + switch (Status) { + case CFIProtectionStatus::PROTECTED: + return "PROTECTED"; + case CFIProtectionStatus::FAIL_NOT_INDIRECT_CF: + return "FAIL_NOT_INDIRECT_CF"; + case CFIProtectionStatus::FAIL_ORPHANS: + return "FAIL_ORPHANS"; + case CFIProtectionStatus::FAIL_BAD_CONDITIONAL_BRANCH: + return "FAIL_BAD_CONDITIONAL_BRANCH"; + case CFIProtectionStatus::FAIL_REGISTER_CLOBBERED: + return "FAIL_REGISTER_CLOBBERED"; + case CFIProtectionStatus::FAIL_INVALID_INSTRUCTION: + return "FAIL_INVALID_INSTRUCTION"; + } + llvm_unreachable("Attempted to stringify an unknown enum value."); +} + +Expected<FileAnalysis> FileAnalysis::Create(StringRef Filename) { + // Open the filename provided. + Expected<object::OwningBinary<object::Binary>> BinaryOrErr = + object::createBinary(Filename); + if (!BinaryOrErr) + return BinaryOrErr.takeError(); + + // Construct the object and allow it to take ownership of the binary. + object::OwningBinary<object::Binary> Binary = std::move(BinaryOrErr.get()); + FileAnalysis Analysis(std::move(Binary)); + + Analysis.Object = dyn_cast<object::ObjectFile>(Analysis.Binary.getBinary()); + if (!Analysis.Object) + return make_error<UnsupportedDisassembly>("Failed to cast object"); + + Analysis.ObjectTriple = Analysis.Object->makeTriple(); + Analysis.Features = Analysis.Object->getFeatures(); + + // Init the rest of the object. + if (auto InitResponse = Analysis.initialiseDisassemblyMembers()) + return std::move(InitResponse); + + if (auto SectionParseResponse = Analysis.parseCodeSections()) + return std::move(SectionParseResponse); + + return std::move(Analysis); +} + +FileAnalysis::FileAnalysis(object::OwningBinary<object::Binary> Binary) + : Binary(std::move(Binary)) {} + +FileAnalysis::FileAnalysis(const Triple &ObjectTriple, + const SubtargetFeatures &Features) + : ObjectTriple(ObjectTriple), Features(Features) {} + +const Instr * +FileAnalysis::getPrevInstructionSequential(const Instr &InstrMeta) const { + std::map<uint64_t, Instr>::const_iterator KV = + Instructions.find(InstrMeta.VMAddress); + if (KV == Instructions.end() || KV == Instructions.begin()) + return nullptr; + + if (!(--KV)->second.Valid) + return nullptr; + + return &KV->second; +} + +const Instr * +FileAnalysis::getNextInstructionSequential(const Instr &InstrMeta) const { + std::map<uint64_t, Instr>::const_iterator KV = + Instructions.find(InstrMeta.VMAddress); + if (KV == Instructions.end() || ++KV == Instructions.end()) + return nullptr; + + if (!KV->second.Valid) + return nullptr; + + return &KV->second; +} + +bool FileAnalysis::usesRegisterOperand(const Instr &InstrMeta) const { + for (const auto &Operand : InstrMeta.Instruction) { + if (Operand.isReg()) + return true; + } + return false; +} + +const Instr *FileAnalysis::getInstruction(uint64_t Address) const { + const auto &InstrKV = Instructions.find(Address); + if (InstrKV == Instructions.end()) + return nullptr; + + return &InstrKV->second; +} + +const Instr &FileAnalysis::getInstructionOrDie(uint64_t Address) const { + const auto &InstrKV = Instructions.find(Address); + assert(InstrKV != Instructions.end() && "Address doesn't exist."); + return InstrKV->second; +} + +bool FileAnalysis::isCFITrap(const Instr &InstrMeta) const { + return MII->getName(InstrMeta.Instruction.getOpcode()) == "TRAP"; +} + +bool FileAnalysis::canFallThrough(const Instr &InstrMeta) const { + if (!InstrMeta.Valid) + return false; + + if (isCFITrap(InstrMeta)) + return false; + + const auto &InstrDesc = MII->get(InstrMeta.Instruction.getOpcode()); + if (InstrDesc.mayAffectControlFlow(InstrMeta.Instruction, *RegisterInfo)) + return InstrDesc.isConditionalBranch(); + + return true; +} + +const Instr * +FileAnalysis::getDefiniteNextInstruction(const Instr &InstrMeta) const { + if (!InstrMeta.Valid) + return nullptr; + + if (isCFITrap(InstrMeta)) + return nullptr; + + const auto &InstrDesc = MII->get(InstrMeta.Instruction.getOpcode()); + const Instr *NextMetaPtr; + if (InstrDesc.mayAffectControlFlow(InstrMeta.Instruction, *RegisterInfo)) { + if (InstrDesc.isConditionalBranch()) + return nullptr; + + uint64_t Target; + if (!MIA->evaluateBranch(InstrMeta.Instruction, InstrMeta.VMAddress, + InstrMeta.InstructionSize, Target)) + return nullptr; + + NextMetaPtr = getInstruction(Target); + } else { + NextMetaPtr = + getInstruction(InstrMeta.VMAddress + InstrMeta.InstructionSize); + } + + if (!NextMetaPtr || !NextMetaPtr->Valid) + return nullptr; + + return NextMetaPtr; +} + +std::set<const Instr *> +FileAnalysis::getDirectControlFlowXRefs(const Instr &InstrMeta) const { + std::set<const Instr *> CFCrossReferences; + const Instr *PrevInstruction = getPrevInstructionSequential(InstrMeta); + + if (PrevInstruction && canFallThrough(*PrevInstruction)) + CFCrossReferences.insert(PrevInstruction); + + const auto &TargetRefsKV = StaticBranchTargetings.find(InstrMeta.VMAddress); + if (TargetRefsKV == StaticBranchTargetings.end()) + return CFCrossReferences; + + for (uint64_t SourceInstrAddress : TargetRefsKV->second) { + const auto &SourceInstrKV = Instructions.find(SourceInstrAddress); + if (SourceInstrKV == Instructions.end()) { + errs() << "Failed to find source instruction at address " + << format_hex(SourceInstrAddress, 2) + << " for the cross-reference to instruction at address " + << format_hex(InstrMeta.VMAddress, 2) << ".\n"; + continue; + } + + CFCrossReferences.insert(&SourceInstrKV->second); + } + + return CFCrossReferences; +} + +const std::set<uint64_t> &FileAnalysis::getIndirectInstructions() const { + return IndirectInstructions; +} + +const MCRegisterInfo *FileAnalysis::getRegisterInfo() const { + return RegisterInfo.get(); +} + +const MCInstrInfo *FileAnalysis::getMCInstrInfo() const { return MII.get(); } + +const MCInstrAnalysis *FileAnalysis::getMCInstrAnalysis() const { + return MIA.get(); +} + +Expected<DIInliningInfo> FileAnalysis::symbolizeInlinedCode(uint64_t Address) { + assert(Symbolizer != nullptr && "Symbolizer is invalid."); + return Symbolizer->symbolizeInlinedCode(Object->getFileName(), Address); +} + +CFIProtectionStatus +FileAnalysis::validateCFIProtection(const GraphResult &Graph) const { + const Instr *InstrMetaPtr = getInstruction(Graph.BaseAddress); + if (!InstrMetaPtr) + return CFIProtectionStatus::FAIL_INVALID_INSTRUCTION; + + const auto &InstrDesc = MII->get(InstrMetaPtr->Instruction.getOpcode()); + if (!InstrDesc.mayAffectControlFlow(InstrMetaPtr->Instruction, *RegisterInfo)) + return CFIProtectionStatus::FAIL_NOT_INDIRECT_CF; + + if (!usesRegisterOperand(*InstrMetaPtr)) + return CFIProtectionStatus::FAIL_NOT_INDIRECT_CF; + + if (!Graph.OrphanedNodes.empty()) + return CFIProtectionStatus::FAIL_ORPHANS; + + for (const auto &BranchNode : Graph.ConditionalBranchNodes) { + if (!BranchNode.CFIProtection) + return CFIProtectionStatus::FAIL_BAD_CONDITIONAL_BRANCH; + } + + if (indirectCFOperandClobber(Graph) != Graph.BaseAddress) + return CFIProtectionStatus::FAIL_REGISTER_CLOBBERED; + + return CFIProtectionStatus::PROTECTED; +} + +uint64_t FileAnalysis::indirectCFOperandClobber(const GraphResult &Graph) const { + assert(Graph.OrphanedNodes.empty() && "Orphaned nodes should be empty."); + + // Get the set of registers we must check to ensure they're not clobbered. + const Instr &IndirectCF = getInstructionOrDie(Graph.BaseAddress); + DenseSet<unsigned> RegisterNumbers; + for (const auto &Operand : IndirectCF.Instruction) { + if (Operand.isReg()) + RegisterNumbers.insert(Operand.getReg()); + } + assert(RegisterNumbers.size() && "Zero register operands on indirect CF."); + + // Now check all branches to indirect CFs and ensure no clobbering happens. + for (const auto &Branch : Graph.ConditionalBranchNodes) { + uint64_t Node; + if (Branch.IndirectCFIsOnTargetPath) + Node = Branch.Target; + else + Node = Branch.Fallthrough; + + while (Node != Graph.BaseAddress) { + const Instr &NodeInstr = getInstructionOrDie(Node); + const auto &InstrDesc = MII->get(NodeInstr.Instruction.getOpcode()); + + for (unsigned RegNum : RegisterNumbers) { + if (InstrDesc.hasDefOfPhysReg(NodeInstr.Instruction, RegNum, + *RegisterInfo)) + return Node; + } + + const auto &KV = Graph.IntermediateNodes.find(Node); + assert((KV != Graph.IntermediateNodes.end()) && + "Could not get next node."); + Node = KV->second; + } + } + + return Graph.BaseAddress; +} + +void FileAnalysis::printInstruction(const Instr &InstrMeta, + raw_ostream &OS) const { + Printer->printInst(&InstrMeta.Instruction, OS, "", *SubtargetInfo.get()); +} + +Error FileAnalysis::initialiseDisassemblyMembers() { + std::string TripleName = ObjectTriple.getTriple(); + ArchName = ""; + MCPU = ""; + std::string ErrorString; + + Symbolizer.reset(new LLVMSymbolizer()); + + ObjectTarget = + TargetRegistry::lookupTarget(ArchName, ObjectTriple, ErrorString); + if (!ObjectTarget) + return make_error<UnsupportedDisassembly>( + (Twine("Couldn't find target \"") + ObjectTriple.getTriple() + + "\", failed with error: " + ErrorString) + .str()); + + RegisterInfo.reset(ObjectTarget->createMCRegInfo(TripleName)); + if (!RegisterInfo) + return make_error<UnsupportedDisassembly>( + "Failed to initialise RegisterInfo."); + + AsmInfo.reset(ObjectTarget->createMCAsmInfo(*RegisterInfo, TripleName)); + if (!AsmInfo) + return make_error<UnsupportedDisassembly>("Failed to initialise AsmInfo."); + + SubtargetInfo.reset(ObjectTarget->createMCSubtargetInfo( + TripleName, MCPU, Features.getString())); + if (!SubtargetInfo) + return make_error<UnsupportedDisassembly>( + "Failed to initialise SubtargetInfo."); + + MII.reset(ObjectTarget->createMCInstrInfo()); + if (!MII) + return make_error<UnsupportedDisassembly>("Failed to initialise MII."); + + Context.reset(new MCContext(AsmInfo.get(), RegisterInfo.get(), &MOFI)); + + Disassembler.reset( + ObjectTarget->createMCDisassembler(*SubtargetInfo, *Context)); + + if (!Disassembler) + return make_error<UnsupportedDisassembly>( + "No disassembler available for target"); + + MIA.reset(ObjectTarget->createMCInstrAnalysis(MII.get())); + + Printer.reset(ObjectTarget->createMCInstPrinter( + ObjectTriple, AsmInfo->getAssemblerDialect(), *AsmInfo, *MII, + *RegisterInfo)); + + return Error::success(); +} + +Error FileAnalysis::parseCodeSections() { + if (!IgnoreDWARFFlag) { + std::unique_ptr<DWARFContext> DWARF = DWARFContext::create(*Object); + if (!DWARF) + return make_error<StringError>("Could not create DWARF information.", + inconvertibleErrorCode()); + + bool LineInfoValid = false; + + for (auto &Unit : DWARF->compile_units()) { + const auto &LineTable = DWARF->getLineTableForUnit(Unit.get()); + if (LineTable && !LineTable->Rows.empty()) { + LineInfoValid = true; + break; + } + } + + if (!LineInfoValid) + return make_error<StringError>( + "DWARF line information missing. Did you compile with '-g'?", + inconvertibleErrorCode()); + } + + for (const object::SectionRef &Section : Object->sections()) { + // Ensure only executable sections get analysed. + if (!(object::ELFSectionRef(Section).getFlags() & ELF::SHF_EXECINSTR)) + continue; + + StringRef SectionContents; + if (Section.getContents(SectionContents)) + return make_error<StringError>("Failed to retrieve section contents", + inconvertibleErrorCode()); + + ArrayRef<uint8_t> SectionBytes((const uint8_t *)SectionContents.data(), + Section.getSize()); + parseSectionContents(SectionBytes, Section.getAddress()); + } + return Error::success(); +} + +void FileAnalysis::parseSectionContents(ArrayRef<uint8_t> SectionBytes, + uint64_t SectionAddress) { + assert(Symbolizer && "Symbolizer is uninitialised."); + MCInst Instruction; + Instr InstrMeta; + uint64_t InstructionSize; + + for (uint64_t Byte = 0; Byte < SectionBytes.size();) { + bool ValidInstruction = + Disassembler->getInstruction(Instruction, InstructionSize, + SectionBytes.drop_front(Byte), 0, nulls(), + outs()) == MCDisassembler::Success; + + Byte += InstructionSize; + + uint64_t VMAddress = SectionAddress + Byte - InstructionSize; + InstrMeta.Instruction = Instruction; + InstrMeta.VMAddress = VMAddress; + InstrMeta.InstructionSize = InstructionSize; + InstrMeta.Valid = ValidInstruction; + + addInstruction(InstrMeta); + + if (!ValidInstruction) + continue; + + // Skip additional parsing for instructions that do not affect the control + // flow. + const auto &InstrDesc = MII->get(Instruction.getOpcode()); + if (!InstrDesc.mayAffectControlFlow(Instruction, *RegisterInfo)) + continue; + + uint64_t Target; + if (MIA->evaluateBranch(Instruction, VMAddress, InstructionSize, Target)) { + // If the target can be evaluated, it's not indirect. + StaticBranchTargetings[Target].push_back(VMAddress); + continue; + } + + if (!usesRegisterOperand(InstrMeta)) + continue; + + // Check if this instruction exists in the range of the DWARF metadata. + if (!IgnoreDWARFFlag) { + auto LineInfo = + Symbolizer->symbolizeCode(Object->getFileName(), VMAddress); + if (!LineInfo) { + handleAllErrors(LineInfo.takeError(), [](const ErrorInfoBase &E) { + errs() << "Symbolizer failed to get line: " << E.message() << "\n"; + }); + continue; + } + + if (LineInfo->FileName == "<invalid>") + continue; + } + + IndirectInstructions.insert(VMAddress); + } +} + +void FileAnalysis::addInstruction(const Instr &Instruction) { + const auto &KV = + Instructions.insert(std::make_pair(Instruction.VMAddress, Instruction)); + if (!KV.second) { + errs() << "Failed to add instruction at address " + << format_hex(Instruction.VMAddress, 2) + << ": Instruction at this address already exists.\n"; + exit(EXIT_FAILURE); + } +} + +UnsupportedDisassembly::UnsupportedDisassembly(StringRef Text) : Text(Text) {} + +char UnsupportedDisassembly::ID; +void UnsupportedDisassembly::log(raw_ostream &OS) const { + OS << "Could not initialise disassembler: " << Text; +} + +std::error_code UnsupportedDisassembly::convertToErrorCode() const { + return std::error_code(); +} + +} // namespace cfi_verify +} // namespace llvm diff --git a/tools/llvm-cfi-verify/lib/FileAnalysis.h b/tools/llvm-cfi-verify/lib/FileAnalysis.h new file mode 100644 index 000000000000..ce81f8bfbe3b --- /dev/null +++ b/tools/llvm-cfi-verify/lib/FileAnalysis.h @@ -0,0 +1,234 @@ +//===- FileAnalysis.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CFI_VERIFY_FILE_ANALYSIS_H +#define LLVM_CFI_VERIFY_FILE_ANALYSIS_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/DebugInfo/Symbolize/Symbolize.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstPrinter.h" +#include "llvm/MC/MCInstrAnalysis.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/raw_ostream.h" + +#include <functional> +#include <set> +#include <string> +#include <unordered_map> + +namespace llvm { +namespace cfi_verify { + +struct GraphResult; + +extern bool IgnoreDWARFFlag; + +enum class CFIProtectionStatus { + // This instruction is protected by CFI. + PROTECTED, + // The instruction is not an indirect control flow instruction, and thus + // shouldn't be protected. + FAIL_NOT_INDIRECT_CF, + // There is a path to the instruction that was unexpected. + FAIL_ORPHANS, + // There is a path to the instruction from a conditional branch that does not + // properly check the destination for this vcall/icall. + FAIL_BAD_CONDITIONAL_BRANCH, + // One of the operands of the indirect CF instruction is modified between the + // CFI-check and execution. + FAIL_REGISTER_CLOBBERED, + // The instruction referenced does not exist. This normally indicates an + // error in the program, where you try and validate a graph that was created + // in a different FileAnalysis object. + FAIL_INVALID_INSTRUCTION, +}; + +StringRef stringCFIProtectionStatus(CFIProtectionStatus Status); + +// Disassembler and analysis tool for machine code files. Keeps track of non- +// sequential control flows, including indirect control flow instructions. +class FileAnalysis { +public: + // A metadata struct for an instruction. + struct Instr { + uint64_t VMAddress; // Virtual memory address of this instruction. + MCInst Instruction; // Instruction. + uint64_t InstructionSize; // Size of this instruction. + bool Valid; // Is this a valid instruction? If false, Instr::Instruction is + // undefined. + }; + + // Construct a FileAnalysis from a file path. + static Expected<FileAnalysis> Create(StringRef Filename); + + // Construct and take ownership of the supplied object. Do not use this + // constructor, prefer to use FileAnalysis::Create instead. + FileAnalysis(object::OwningBinary<object::Binary> Binary); + FileAnalysis() = delete; + FileAnalysis(const FileAnalysis &) = delete; + FileAnalysis(FileAnalysis &&Other) = default; + + // Returns the instruction at the provided address. Returns nullptr if there + // is no instruction at the provided address. + const Instr *getInstruction(uint64_t Address) const; + + // Returns the instruction at the provided adress, dying if the instruction is + // not found. + const Instr &getInstructionOrDie(uint64_t Address) const; + + // Returns a pointer to the previous/next instruction in sequence, + // respectively. Returns nullptr if the next/prev instruction doesn't exist, + // or if the provided instruction doesn't exist. + const Instr *getPrevInstructionSequential(const Instr &InstrMeta) const; + const Instr *getNextInstructionSequential(const Instr &InstrMeta) const; + + // Returns whether this instruction is used by CFI to trap the program. + bool isCFITrap(const Instr &InstrMeta) const; + + // Returns whether this function can fall through to the next instruction. + // Undefined (and bad) instructions cannot fall through, and instruction that + // modify the control flow can only fall through if they are conditional + // branches or calls. + bool canFallThrough(const Instr &InstrMeta) const; + + // Returns the definitive next instruction. This is different from the next + // instruction sequentially as it will follow unconditional branches (assuming + // they can be resolved at compile time, i.e. not indirect). This method + // returns nullptr if the provided instruction does not transfer control flow + // to exactly one instruction that is known deterministically at compile time. + // Also returns nullptr if the deterministic target does not exist in this + // file. + const Instr *getDefiniteNextInstruction(const Instr &InstrMeta) const; + + // Get a list of deterministic control flows that lead to the provided + // instruction. This list includes all static control flow cross-references as + // well as the previous instruction if it can fall through. + std::set<const Instr *> + getDirectControlFlowXRefs(const Instr &InstrMeta) const; + + // Returns whether this instruction uses a register operand. + bool usesRegisterOperand(const Instr &InstrMeta) const; + + // Returns the list of indirect instructions. + const std::set<uint64_t> &getIndirectInstructions() const; + + const MCRegisterInfo *getRegisterInfo() const; + const MCInstrInfo *getMCInstrInfo() const; + const MCInstrAnalysis *getMCInstrAnalysis() const; + + // Returns the inlining information for the provided address. + Expected<DIInliningInfo> symbolizeInlinedCode(uint64_t Address); + + // Returns whether the provided Graph represents a protected indirect control + // flow instruction in this file. + CFIProtectionStatus validateCFIProtection(const GraphResult &Graph) const; + + // Returns the first place the operand register is clobbered between the CFI- + // check and the indirect CF instruction execution. If the register is not + // modified, returns the address of the indirect CF instruction. The result is + // undefined if the provided graph does not fall under either the + // FAIL_REGISTER_CLOBBERED or PROTECTED status (see CFIProtectionStatus). + uint64_t indirectCFOperandClobber(const GraphResult& Graph) const; + + // Prints an instruction to the provided stream using this object's pretty- + // printers. + void printInstruction(const Instr &InstrMeta, raw_ostream &OS) const; + +protected: + // Construct a blank object with the provided triple and features. Used in + // testing, where a sub class will dependency inject protected methods to + // allow analysis of raw binary, without requiring a fully valid ELF file. + FileAnalysis(const Triple &ObjectTriple, const SubtargetFeatures &Features); + + // Add an instruction to this object. + void addInstruction(const Instr &Instruction); + + // Disassemble and parse the provided bytes into this object. Instruction + // address calculation is done relative to the provided SectionAddress. + void parseSectionContents(ArrayRef<uint8_t> SectionBytes, + uint64_t SectionAddress); + + // Constructs and initialises members required for disassembly. + Error initialiseDisassemblyMembers(); + + // Parses code sections from the internal object file. Saves them into the + // internal members. Should only be called once by Create(). + Error parseCodeSections(); + +private: + // Members that describe the input file. + object::OwningBinary<object::Binary> Binary; + const object::ObjectFile *Object = nullptr; + Triple ObjectTriple; + std::string ArchName; + std::string MCPU; + const Target *ObjectTarget = nullptr; + SubtargetFeatures Features; + + // Members required for disassembly. + std::unique_ptr<const MCRegisterInfo> RegisterInfo; + std::unique_ptr<const MCAsmInfo> AsmInfo; + std::unique_ptr<MCSubtargetInfo> SubtargetInfo; + std::unique_ptr<const MCInstrInfo> MII; + MCObjectFileInfo MOFI; + std::unique_ptr<MCContext> Context; + std::unique_ptr<const MCDisassembler> Disassembler; + std::unique_ptr<const MCInstrAnalysis> MIA; + std::unique_ptr<MCInstPrinter> Printer; + + // Symbolizer used for debug information parsing. + std::unique_ptr<symbolize::LLVMSymbolizer> Symbolizer; + + // A mapping between the virtual memory address to the instruction metadata + // struct. TODO(hctim): Reimplement this as a sorted vector to avoid per- + // insertion allocation. + std::map<uint64_t, Instr> Instructions; + + // Contains a mapping between a specific address, and a list of instructions + // that use this address as a branch target (including call instructions). + DenseMap<uint64_t, std::vector<uint64_t>> StaticBranchTargetings; + + // A list of addresses of indirect control flow instructions. + std::set<uint64_t> IndirectInstructions; +}; + +class UnsupportedDisassembly : public ErrorInfo<UnsupportedDisassembly> { +public: + static char ID; + std::string Text; + + UnsupportedDisassembly(StringRef Text); + + void log(raw_ostream &OS) const override; + std::error_code convertToErrorCode() const override; +}; + +} // namespace cfi_verify +} // namespace llvm + +#endif // LLVM_CFI_VERIFY_FILE_ANALYSIS_H diff --git a/tools/llvm-cfi-verify/lib/GraphBuilder.cpp b/tools/llvm-cfi-verify/lib/GraphBuilder.cpp new file mode 100644 index 000000000000..4153b5f6844a --- /dev/null +++ b/tools/llvm-cfi-verify/lib/GraphBuilder.cpp @@ -0,0 +1,321 @@ +//===- GraphBuilder.cpp -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "GraphBuilder.h" + +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstPrinter.h" +#include "llvm/MC/MCInstrAnalysis.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/raw_ostream.h" + + +using Instr = llvm::cfi_verify::FileAnalysis::Instr; + +namespace llvm { +namespace cfi_verify { + +unsigned long long SearchLengthForUndef; +unsigned long long SearchLengthForConditionalBranch; + +static cl::opt<unsigned long long, true> SearchLengthForUndefArg( + "search-length-undef", + cl::desc("Specify the maximum amount of instructions " + "to inspect when searching for an undefined " + "instruction from a conditional branch."), + cl::location(SearchLengthForUndef), cl::init(2)); + +static cl::opt<unsigned long long, true> SearchLengthForConditionalBranchArg( + "search-length-cb", + cl::desc("Specify the maximum amount of instructions " + "to inspect when searching for a conditional " + "branch from an indirect control flow."), + cl::location(SearchLengthForConditionalBranch), cl::init(20)); + +std::vector<uint64_t> GraphResult::flattenAddress(uint64_t Address) const { + std::vector<uint64_t> Addresses; + + auto It = IntermediateNodes.find(Address); + Addresses.push_back(Address); + + while (It != IntermediateNodes.end()) { + Addresses.push_back(It->second); + It = IntermediateNodes.find(It->second); + } + return Addresses; +} + +void printPairToDOT(const FileAnalysis &Analysis, raw_ostream &OS, + uint64_t From, uint64_t To) { + OS << " \"" << format_hex(From, 2) << ": "; + Analysis.printInstruction(Analysis.getInstructionOrDie(From), OS); + OS << "\" -> \"" << format_hex(To, 2) << ": "; + Analysis.printInstruction(Analysis.getInstructionOrDie(To), OS); + OS << "\"\n"; +} + +void GraphResult::printToDOT(const FileAnalysis &Analysis, + raw_ostream &OS) const { + std::map<uint64_t, uint64_t> SortedIntermediateNodes( + IntermediateNodes.begin(), IntermediateNodes.end()); + OS << "digraph graph_" << format_hex(BaseAddress, 2) << " {\n"; + for (const auto &KV : SortedIntermediateNodes) + printPairToDOT(Analysis, OS, KV.first, KV.second); + + for (auto &BranchNode : ConditionalBranchNodes) { + for (auto &V : {BranchNode.Target, BranchNode.Fallthrough}) + printPairToDOT(Analysis, OS, BranchNode.Address, V); + } + OS << "}\n"; +} + +GraphResult GraphBuilder::buildFlowGraph(const FileAnalysis &Analysis, + uint64_t Address) { + GraphResult Result; + Result.BaseAddress = Address; + DenseSet<uint64_t> OpenedNodes; + + const auto &IndirectInstructions = Analysis.getIndirectInstructions(); + + if (IndirectInstructions.find(Address) == IndirectInstructions.end()) + return Result; + + buildFlowGraphImpl(Analysis, OpenedNodes, Result, Address, 0); + return Result; +} + +void GraphBuilder::buildFlowsToUndefined(const FileAnalysis &Analysis, + GraphResult &Result, + ConditionalBranchNode &BranchNode, + const Instr &BranchInstrMeta) { + assert(SearchLengthForUndef > 0 && + "Search length for undefined flow must be greater than zero."); + + // Start setting up the next node in the block. + uint64_t NextAddress = 0; + const Instr *NextMetaPtr; + + // Find out the next instruction in the block and add it to the new + // node. + if (BranchNode.Target && !BranchNode.Fallthrough) { + // We know the target of the branch, find the fallthrough. + NextMetaPtr = Analysis.getNextInstructionSequential(BranchInstrMeta); + if (!NextMetaPtr) { + errs() << "Failed to get next instruction from " + << format_hex(BranchNode.Address, 2) << ".\n"; + return; + } + + NextAddress = NextMetaPtr->VMAddress; + BranchNode.Fallthrough = + NextMetaPtr->VMAddress; // Add the new node to the branch head. + } else if (BranchNode.Fallthrough && !BranchNode.Target) { + // We already know the fallthrough, evaluate the target. + uint64_t Target; + if (!Analysis.getMCInstrAnalysis()->evaluateBranch( + BranchInstrMeta.Instruction, BranchInstrMeta.VMAddress, + BranchInstrMeta.InstructionSize, Target)) { + errs() << "Failed to get branch target for conditional branch at address " + << format_hex(BranchInstrMeta.VMAddress, 2) << ".\n"; + return; + } + + // Resolve the meta pointer for the target of this branch. + NextMetaPtr = Analysis.getInstruction(Target); + if (!NextMetaPtr) { + errs() << "Failed to find instruction at address " + << format_hex(Target, 2) << ".\n"; + return; + } + + NextAddress = Target; + BranchNode.Target = + NextMetaPtr->VMAddress; // Add the new node to the branch head. + } else { + errs() << "ControlBranchNode supplied to buildFlowsToUndefined should " + "provide Target xor Fallthrough.\n"; + return; + } + + uint64_t CurrentAddress = NextAddress; + const Instr *CurrentMetaPtr = NextMetaPtr; + + // Now the branch head has been set properly, complete the rest of the block. + for (uint64_t i = 1; i < SearchLengthForUndef; ++i) { + // Check to see whether the block should die. + if (Analysis.isCFITrap(*CurrentMetaPtr)) { + BranchNode.CFIProtection = true; + return; + } + + // Find the metadata of the next instruction. + NextMetaPtr = Analysis.getDefiniteNextInstruction(*CurrentMetaPtr); + if (!NextMetaPtr) + return; + + // Setup the next node. + NextAddress = NextMetaPtr->VMAddress; + + // Add this as an intermediate. + Result.IntermediateNodes[CurrentAddress] = NextAddress; + + // Move the 'current' pointers to the new tail of the block. + CurrentMetaPtr = NextMetaPtr; + CurrentAddress = NextAddress; + } + + // Final check of the last thing we added to the block. + if (Analysis.isCFITrap(*CurrentMetaPtr)) + BranchNode.CFIProtection = true; +} + +void GraphBuilder::buildFlowGraphImpl(const FileAnalysis &Analysis, + DenseSet<uint64_t> &OpenedNodes, + GraphResult &Result, uint64_t Address, + uint64_t Depth) { + // If we've exceeded the flow length, terminate. + if (Depth >= SearchLengthForConditionalBranch) { + Result.OrphanedNodes.push_back(Address); + return; + } + + // Ensure this flow is acyclic. + if (OpenedNodes.count(Address)) + Result.OrphanedNodes.push_back(Address); + + // If this flow is already explored, stop here. + if (Result.IntermediateNodes.count(Address)) + return; + + // Get the metadata for the node instruction. + const auto &InstrMetaPtr = Analysis.getInstruction(Address); + if (!InstrMetaPtr) { + errs() << "Failed to build flow graph for instruction at address " + << format_hex(Address, 2) << ".\n"; + Result.OrphanedNodes.push_back(Address); + return; + } + const auto &ChildMeta = *InstrMetaPtr; + + OpenedNodes.insert(Address); + std::set<const Instr *> CFCrossRefs = + Analysis.getDirectControlFlowXRefs(ChildMeta); + + bool HasValidCrossRef = false; + + for (const auto *ParentMetaPtr : CFCrossRefs) { + assert(ParentMetaPtr && "CFCrossRefs returned nullptr."); + const auto &ParentMeta = *ParentMetaPtr; + const auto &ParentDesc = + Analysis.getMCInstrInfo()->get(ParentMeta.Instruction.getOpcode()); + + if (!ParentDesc.mayAffectControlFlow(ParentMeta.Instruction, + *Analysis.getRegisterInfo())) { + // If this cross reference doesn't affect CF, continue the graph. + buildFlowGraphImpl(Analysis, OpenedNodes, Result, ParentMeta.VMAddress, + Depth + 1); + Result.IntermediateNodes[ParentMeta.VMAddress] = Address; + HasValidCrossRef = true; + continue; + } + + // Call instructions are not valid in the upwards traversal. + if (ParentDesc.isCall()) { + Result.IntermediateNodes[ParentMeta.VMAddress] = Address; + Result.OrphanedNodes.push_back(ParentMeta.VMAddress); + continue; + } + + // Evaluate the branch target to ascertain whether this XRef is the result + // of a fallthrough or the target of a branch. + uint64_t BranchTarget; + if (!Analysis.getMCInstrAnalysis()->evaluateBranch( + ParentMeta.Instruction, ParentMeta.VMAddress, + ParentMeta.InstructionSize, BranchTarget)) { + errs() << "Failed to evaluate branch target for instruction at address " + << format_hex(ParentMeta.VMAddress, 2) << ".\n"; + Result.IntermediateNodes[ParentMeta.VMAddress] = Address; + Result.OrphanedNodes.push_back(ParentMeta.VMAddress); + continue; + } + + // Allow unconditional branches to be part of the upwards traversal. + if (ParentDesc.isUnconditionalBranch()) { + // Ensures that the unconditional branch is actually an XRef to the child. + if (BranchTarget != Address) { + errs() << "Control flow to " << format_hex(Address, 2) + << ", but target resolution of " + << format_hex(ParentMeta.VMAddress, 2) + << " is not this address?\n"; + Result.IntermediateNodes[ParentMeta.VMAddress] = Address; + Result.OrphanedNodes.push_back(ParentMeta.VMAddress); + continue; + } + + buildFlowGraphImpl(Analysis, OpenedNodes, Result, ParentMeta.VMAddress, + Depth + 1); + Result.IntermediateNodes[ParentMeta.VMAddress] = Address; + HasValidCrossRef = true; + continue; + } + + // Ensure that any unknown CFs are caught. + if (!ParentDesc.isConditionalBranch()) { + errs() << "Unknown control flow encountered when building graph at " + << format_hex(Address, 2) << "\n."; + Result.IntermediateNodes[ParentMeta.VMAddress] = Address; + Result.OrphanedNodes.push_back(ParentMeta.VMAddress); + continue; + } + + // Only direct conditional branches should be present at this point. Setup + // a conditional branch node and build flows to the ud2. + ConditionalBranchNode BranchNode; + BranchNode.Address = ParentMeta.VMAddress; + BranchNode.Target = 0; + BranchNode.Fallthrough = 0; + BranchNode.CFIProtection = false; + BranchNode.IndirectCFIsOnTargetPath = (BranchTarget == Address); + + if (BranchTarget == Address) + BranchNode.Target = Address; + else + BranchNode.Fallthrough = Address; + + HasValidCrossRef = true; + buildFlowsToUndefined(Analysis, Result, BranchNode, ParentMeta); + Result.ConditionalBranchNodes.push_back(BranchNode); + } + + if (!HasValidCrossRef) + Result.OrphanedNodes.push_back(Address); + + OpenedNodes.erase(Address); +} + +} // namespace cfi_verify +} // namespace llvm diff --git a/tools/llvm-cfi-verify/lib/GraphBuilder.h b/tools/llvm-cfi-verify/lib/GraphBuilder.h new file mode 100644 index 000000000000..2bf07b53bf6d --- /dev/null +++ b/tools/llvm-cfi-verify/lib/GraphBuilder.h @@ -0,0 +1,137 @@ +//===- GraphBuilder.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CFI_VERIFY_GRAPH_BUILDER_H +#define LLVM_CFI_VERIFY_GRAPH_BUILDER_H + +#include "FileAnalysis.h" + +#include "llvm/ADT/DenseMap.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstPrinter.h" +#include "llvm/MC/MCInstrAnalysis.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/raw_ostream.h" + +#include <functional> +#include <set> +#include <string> +#include <unordered_map> + +using Instr = llvm::cfi_verify::FileAnalysis::Instr; + +namespace llvm { +namespace cfi_verify { + +extern unsigned long long SearchLengthForUndef; +extern unsigned long long SearchLengthForConditionalBranch; + +struct ConditionalBranchNode { + uint64_t Address; + uint64_t Target; + uint64_t Fallthrough; + // Does this conditional branch look like it's used for CFI protection? i.e. + // - The exit point of a basic block whos entry point is {target|fallthrough} + // is a CFI trap, and... + // - The exit point of the other basic block is an undirect CF instruction. + bool CFIProtection; + bool IndirectCFIsOnTargetPath; +}; + +// The canonical graph result structure returned by GraphBuilder. The members +// in this structure encapsulate all possible code paths to the instruction +// located at `BaseAddress`. +struct GraphResult { + uint64_t BaseAddress; + + // Map between an instruction address, and the address of the next instruction + // that will be executed. This map will contain all keys in the range: + // - [orphaned node, base address) + // - [conditional branch node {target|fallthrough}, base address) + DenseMap<uint64_t, uint64_t> IntermediateNodes; + + // A list of orphaned nodes. A node is an 'orphan' if it meets any of the + // following criteria: + // - The length of the path from the base to this node has exceeded + // `SearchLengthForConditionalBranch`. + // - The node has no cross references to it. + // - The path from the base to this node is cyclic. + std::vector<uint64_t> OrphanedNodes; + + // A list of top-level conditional branches that exist at the top of any + // non-orphan paths from the base. + std::vector<ConditionalBranchNode> ConditionalBranchNodes; + + // Returns an in-order list of the path between the address provided and the + // base. The provided address must be part of this graph, and must not be a + // conditional branch. + std::vector<uint64_t> flattenAddress(uint64_t Address) const; + + // Print the DOT representation of this result. + void printToDOT(const FileAnalysis &Analysis, raw_ostream &OS) const; +}; + +class GraphBuilder { +public: + // Build the control flow graph for a provided control flow node. This method + // will enumerate all branch nodes that can lead to this node, and place them + // into GraphResult::ConditionalBranchNodes. It will also provide any orphaned + // (i.e. the upwards traversal did not make it to a branch node) flows to the + // provided node in GraphResult::OrphanedNodes. + static GraphResult buildFlowGraph(const FileAnalysis &Analysis, + uint64_t Address); + +private: + // Implementation function that actually builds the flow graph. Retrieves a + // list of cross references to instruction referenced in `Address`. If any of + // these XRefs are conditional branches, it will build the other potential + // path (fallthrough or target) using `buildFlowsToUndefined`. Otherwise, this + // function will recursively call itself where `Address` in the recursive call + // is now the XRef. If any XRef is an orphan, it is added to + // `Result.OrphanedNodes`. `OpenedNodes` keeps track of the list of nodes + // in the current path and is used for cycle-checking. If the path is found + // to be cyclic, it will be added to `Result.OrphanedNodes`. + static void buildFlowGraphImpl(const FileAnalysis &Analysis, + DenseSet<uint64_t> &OpenedNodes, + GraphResult &Result, uint64_t Address, + uint64_t Depth); + + // Utilised by buildFlowGraphImpl to build the tree out from the provided + // conditional branch node to an undefined instruction. The provided + // conditional branch node must have exactly one of its subtrees set, and will + // update the node's CFIProtection field if a deterministic flow can be found + // to an undefined instruction. + static void buildFlowsToUndefined(const FileAnalysis &Analysis, + GraphResult &Result, + ConditionalBranchNode &BranchNode, + const Instr &BranchInstrMeta); +}; + +} // end namespace cfi_verify +} // end namespace llvm + +#endif // LLVM_CFI_VERIFY_GRAPH_BUILDER_H diff --git a/tools/llvm-cfi-verify/lib/LLVMBuild.txt b/tools/llvm-cfi-verify/lib/LLVMBuild.txt new file mode 100644 index 000000000000..c0ae1905521a --- /dev/null +++ b/tools/llvm-cfi-verify/lib/LLVMBuild.txt @@ -0,0 +1,22 @@ +;===- ./tools/llvm-cfi-verify/lib/LLVMBuild.txt ----------------*- Conf -*--===; +; +; The LLVM Compiler Infrastructure +; +; This file is distributed under the University of Illinois Open Source +; License. See LICENSE.TXT for details. +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Library +name = CFIVerify +parent = Libraries +required_libraries = DebugInfoDWARF MC MCDisassembler MCParser Support Symbolize diff --git a/tools/llvm-cfi-verify/llvm-cfi-verify.cpp b/tools/llvm-cfi-verify/llvm-cfi-verify.cpp new file mode 100644 index 000000000000..245ce05a254e --- /dev/null +++ b/tools/llvm-cfi-verify/llvm-cfi-verify.cpp @@ -0,0 +1,200 @@ +//===-- llvm-cfi-verify.cpp - CFI Verification tool for LLVM --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This tool verifies Control Flow Integrity (CFI) instrumentation by static +// binary anaylsis. See the design document in /docs/CFIVerify.rst for more +// information. +// +// This tool is currently incomplete. It currently only does disassembly for +// object files, and searches through the code for indirect control flow +// instructions, printing them once found. +// +//===----------------------------------------------------------------------===// + +#include "lib/FileAnalysis.h" +#include "lib/GraphBuilder.h" + +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/SpecialCaseList.h" + +#include <cstdlib> + +using namespace llvm; +using namespace llvm::object; +using namespace llvm::cfi_verify; + +cl::opt<std::string> InputFilename(cl::Positional, cl::desc("<input file>"), + cl::Required); +cl::opt<std::string> BlacklistFilename(cl::Positional, + cl::desc("[blacklist file]"), + cl::init("-")); +cl::opt<bool> PrintGraphs( + "print-graphs", + cl::desc("Print graphs around indirect CF instructions in DOT format."), + cl::init(false)); + +ExitOnError ExitOnErr; + +void printIndirectCFInstructions(FileAnalysis &Analysis, + const SpecialCaseList *SpecialCaseList) { + uint64_t ExpectedProtected = 0; + uint64_t UnexpectedProtected = 0; + uint64_t ExpectedUnprotected = 0; + uint64_t UnexpectedUnprotected = 0; + + std::map<unsigned, uint64_t> BlameCounter; + + for (uint64_t Address : Analysis.getIndirectInstructions()) { + const auto &InstrMeta = Analysis.getInstructionOrDie(Address); + GraphResult Graph = GraphBuilder::buildFlowGraph(Analysis, Address); + + CFIProtectionStatus ProtectionStatus = + Analysis.validateCFIProtection(Graph); + bool CFIProtected = (ProtectionStatus == CFIProtectionStatus::PROTECTED); + + if (CFIProtected) + outs() << "P "; + else + outs() << "U "; + + outs() << format_hex(Address, 2) << " | "; + Analysis.printInstruction(InstrMeta, outs()); + outs() << " \n"; + + if (PrintGraphs) + Graph.printToDOT(Analysis, outs()); + + if (IgnoreDWARFFlag) { + if (CFIProtected) + ExpectedProtected++; + else + UnexpectedUnprotected++; + continue; + } + + auto InliningInfo = Analysis.symbolizeInlinedCode(Address); + if (!InliningInfo || InliningInfo->getNumberOfFrames() == 0) { + errs() << "Failed to symbolise " << format_hex(Address, 2) + << " with line tables from " << InputFilename << "\n"; + exit(EXIT_FAILURE); + } + + const auto &LineInfo = + InliningInfo->getFrame(InliningInfo->getNumberOfFrames() - 1); + + // Print the inlining symbolisation of this instruction. + for (uint32_t i = 0; i < InliningInfo->getNumberOfFrames(); ++i) { + const auto &Line = InliningInfo->getFrame(i); + outs() << " " << format_hex(Address, 2) << " = " << Line.FileName << ":" + << Line.Line << ":" << Line.Column << " (" << Line.FunctionName + << ")\n"; + } + + if (!SpecialCaseList) { + if (CFIProtected) + ExpectedProtected++; + else + UnexpectedUnprotected++; + continue; + } + + unsigned BlameLine = 0; + for (auto &K : {"cfi-icall", "cfi-vcall"}) { + if (!BlameLine) + BlameLine = + SpecialCaseList->inSectionBlame(K, "src", LineInfo.FileName); + if (!BlameLine) + BlameLine = + SpecialCaseList->inSectionBlame(K, "fun", LineInfo.FunctionName); + } + + if (BlameLine) { + outs() << "Blacklist Match: " << BlacklistFilename << ":" << BlameLine + << "\n"; + BlameCounter[BlameLine]++; + if (CFIProtected) { + UnexpectedProtected++; + outs() << "====> Unexpected Protected\n"; + } else { + ExpectedUnprotected++; + outs() << "====> Expected Unprotected\n"; + } + } else { + if (CFIProtected) { + ExpectedProtected++; + outs() << "====> Expected Protected\n"; + } else { + UnexpectedUnprotected++; + outs() << "====> Unexpected Unprotected\n"; + } + } + } + + uint64_t IndirectCFInstructions = ExpectedProtected + UnexpectedProtected + + ExpectedUnprotected + UnexpectedUnprotected; + + if (IndirectCFInstructions == 0) { + outs() << "No indirect CF instructions found.\n"; + return; + } + + outs() << formatv("Expected Protected: {0} ({1:P})\n" + "Unexpected Protected: {2} ({3:P})\n" + "Expected Unprotected: {4} ({5:P})\n" + "Unexpected Unprotected (BAD): {6} ({7:P})\n", + ExpectedProtected, + ((double)ExpectedProtected) / IndirectCFInstructions, + UnexpectedProtected, + ((double)UnexpectedProtected) / IndirectCFInstructions, + ExpectedUnprotected, + ((double)ExpectedUnprotected) / IndirectCFInstructions, + UnexpectedUnprotected, + ((double)UnexpectedUnprotected) / IndirectCFInstructions); + + if (!SpecialCaseList) + return; + + outs() << "Blacklist Results:\n"; + for (const auto &KV : BlameCounter) { + outs() << " " << BlacklistFilename << ":" << KV.first << " affects " + << KV.second << " indirect CF instructions.\n"; + } +} + +int main(int argc, char **argv) { + cl::ParseCommandLineOptions( + argc, argv, + "Identifies whether Control Flow Integrity protects all indirect control " + "flow instructions in the provided object file, DSO or binary.\nNote: " + "Anything statically linked into the provided file *must* be compiled " + "with '-g'. This can be relaxed through the '--ignore-dwarf' flag."); + + InitializeAllTargetInfos(); + InitializeAllTargetMCs(); + InitializeAllAsmParsers(); + InitializeAllDisassemblers(); + + std::unique_ptr<SpecialCaseList> SpecialCaseList; + if (BlacklistFilename != "-") { + std::string Error; + SpecialCaseList = SpecialCaseList::create({BlacklistFilename}, Error); + if (!SpecialCaseList) { + errs() << "Failed to get blacklist: " << Error << "\n"; + exit(EXIT_FAILURE); + } + } + + FileAnalysis Analysis = ExitOnErr(FileAnalysis::Create(InputFilename)); + printIndirectCFInstructions(Analysis, SpecialCaseList.get()); + + return EXIT_SUCCESS; +} diff --git a/tools/llvm-config/CMakeLists.txt b/tools/llvm-config/CMakeLists.txt index 5112648ea731..25f99cec9788 100644 --- a/tools/llvm-config/CMakeLists.txt +++ b/tools/llvm-config/CMakeLists.txt @@ -37,11 +37,7 @@ set(LLVM_CXXFLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${uppercase_CMAKE_BUILD_ set(LLVM_BUILD_SYSTEM cmake) set(LLVM_HAS_RTTI ${LLVM_CONFIG_HAS_RTTI}) set(LLVM_DYLIB_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}${LLVM_VERSION_SUFFIX}") -if(LLVM_BUILD_GLOBAL_ISEL) - set(LLVM_HAS_GLOBAL_ISEL "ON") -else() - set(LLVM_HAS_GLOBAL_ISEL "OFF") -endif() +set(LLVM_HAS_GLOBAL_ISEL "ON") # Use the C++ link flags, since they should be a superset of C link flags. set(LLVM_LDFLAGS "${CMAKE_CXX_LINK_FLAGS}") @@ -59,6 +55,12 @@ configure_file(${BUILDVARIABLES_SRCPATH} ${BUILDVARIABLES_OBJPATH} @ONLY) # Set build-time environment(s). add_definitions(-DCMAKE_CFG_INTDIR="${CMAKE_CFG_INTDIR}") +if(LLVM_ENABLE_MODULES) + target_compile_options(llvm-config PUBLIC + "-fmodules-ignore-macro=CMAKE_CFG_INTDIR" + ) +endif() + # Add the dependency on the generation step. add_file_dependencies(${CMAKE_CURRENT_SOURCE_DIR}/llvm-config.cpp ${BUILDVARIABLES_OBJPATH}) diff --git a/tools/llvm-cov/CodeCoverage.cpp b/tools/llvm-cov/CodeCoverage.cpp index 3cbd6591134b..00258f2a1b33 100644 --- a/tools/llvm-cov/CodeCoverage.cpp +++ b/tools/llvm-cov/CodeCoverage.cpp @@ -35,13 +35,16 @@ #include "llvm/Support/Threading.h" #include "llvm/Support/ThreadPool.h" #include "llvm/Support/ToolOutputFile.h" + #include <functional> +#include <map> #include <system_error> using namespace llvm; using namespace coverage; void exportCoverageDataToJson(const coverage::CoverageMapping &CoverageMapping, + const CoverageViewOptions &Options, raw_ostream &OS); namespace { @@ -94,6 +97,10 @@ private: /// \brief Load the coverage mapping data. Return nullptr if an error occurred. std::unique_ptr<CoverageMapping> load(); + /// \brief Create a mapping from files in the Coverage data to local copies + /// (path-equivalence). + void remapPathNames(const CoverageMapping &Coverage); + /// \brief Remove input source files which aren't mapped by \p Coverage. void removeUnmappedInputs(const CoverageMapping &Coverage); @@ -125,15 +132,16 @@ private: /// A list of input source files. std::vector<std::string> SourceFiles; - /// Whether or not we're in -filename-equivalence mode. - bool CompareFilenamesOnly; - - /// In -filename-equivalence mode, this maps absolute paths from the - /// coverage mapping data to input source files. + /// In -path-equivalence mode, this maps the absolute paths from the coverage + /// mapping data to the input source files. StringMap<std::string> RemappedFilenames; + /// The coverage data path to be remapped from, and the source path to be + /// remapped to, when using -path-equivalence. + Optional<std::pair<std::string, std::string>> PathRemapping; + /// The architecture the coverage mapping data targets. - std::string CoverageArch; + std::vector<StringRef> CoverageArches; /// A cache for demangled symbols. DemangleCache DC; @@ -145,6 +153,9 @@ private: std::mutex LoadedSourceFilesLock; std::vector<std::pair<std::string, std::unique_ptr<MemoryBuffer>>> LoadedSourceFiles; + + /// Whitelist from -name-whitelist to be used for filtering. + std::unique_ptr<SpecialCaseList> NameWhitelist; }; } @@ -171,24 +182,20 @@ void CodeCoverageTool::warning(const Twine &Message, StringRef Whence) { } void CodeCoverageTool::addCollectedPath(const std::string &Path) { - if (CompareFilenamesOnly) { - SourceFiles.emplace_back(Path); - } else { - SmallString<128> EffectivePath(Path); - if (std::error_code EC = sys::fs::make_absolute(EffectivePath)) { - error(EC.message(), Path); - return; - } - sys::path::remove_dots(EffectivePath, /*remove_dot_dots=*/true); - SourceFiles.emplace_back(EffectivePath.str()); + SmallString<128> EffectivePath(Path); + if (std::error_code EC = sys::fs::make_absolute(EffectivePath)) { + error(EC.message(), Path); + return; } + sys::path::remove_dots(EffectivePath, /*remove_dot_dots=*/true); + SourceFiles.emplace_back(EffectivePath.str()); } void CodeCoverageTool::collectPaths(const std::string &Path) { llvm::sys::fs::file_status Status; llvm::sys::fs::status(Path, Status); if (!llvm::sys::fs::exists(Status)) { - if (CompareFilenamesOnly) + if (PathRemapping) addCollectedPath(Path); else error("Missing source file", Path); @@ -288,26 +295,34 @@ CodeCoverageTool::createSourceFileView(StringRef SourceFile, auto View = SourceCoverageView::create(SourceFile, SourceBuffer.get(), ViewOpts, std::move(FileCoverage)); attachExpansionSubViews(*View, Expansions, Coverage); + if (!ViewOpts.ShowFunctionInstantiations) + return View; - for (const auto *Function : Coverage.getInstantiations(SourceFile)) { - std::unique_ptr<SourceCoverageView> SubView{nullptr}; + for (const auto &Group : Coverage.getInstantiationGroups(SourceFile)) { + // Skip functions which have a single instantiation. + if (Group.size() < 2) + continue; - StringRef Funcname = DC.demangle(Function->Name); + for (const FunctionRecord *Function : Group.getInstantiations()) { + std::unique_ptr<SourceCoverageView> SubView{nullptr}; - if (Function->ExecutionCount > 0) { - auto SubViewCoverage = Coverage.getCoverageForFunction(*Function); - auto SubViewExpansions = SubViewCoverage.getExpansions(); - SubView = SourceCoverageView::create( - Funcname, SourceBuffer.get(), ViewOpts, std::move(SubViewCoverage)); - attachExpansionSubViews(*SubView, SubViewExpansions, Coverage); - } + StringRef Funcname = DC.demangle(Function->Name); - unsigned FileID = Function->CountedRegions.front().FileID; - unsigned Line = 0; - for (const auto &CR : Function->CountedRegions) - if (CR.FileID == FileID) - Line = std::max(CR.LineEnd, Line); - View->addInstantiation(Funcname, Line, std::move(SubView)); + if (Function->ExecutionCount > 0) { + auto SubViewCoverage = Coverage.getCoverageForFunction(*Function); + auto SubViewExpansions = SubViewCoverage.getExpansions(); + SubView = SourceCoverageView::create( + Funcname, SourceBuffer.get(), ViewOpts, std::move(SubViewCoverage)); + attachExpansionSubViews(*SubView, SubViewExpansions, Coverage); + } + + unsigned FileID = Function->CountedRegions.front().FileID; + unsigned Line = 0; + for (const auto &CR : Function->CountedRegions) + if (CR.FileID == FileID) + Line = std::max(CR.LineEnd, Line); + View->addInstantiation(Funcname, Line, std::move(SubView)); + } } return View; } @@ -329,7 +344,7 @@ std::unique_ptr<CoverageMapping> CodeCoverageTool::load() { warning("profile data may be out of date - object is newer", ObjectFilename); auto CoverageOrErr = - CoverageMapping::load(ObjectFilenames, PGOFilename, CoverageArch); + CoverageMapping::load(ObjectFilenames, PGOFilename, CoverageArches); if (Error E = CoverageOrErr.takeError()) { error("Failed to load coverage: " + toString(std::move(E)), join(ObjectFilenames.begin(), ObjectFilenames.end(), ", ")); @@ -337,9 +352,25 @@ std::unique_ptr<CoverageMapping> CodeCoverageTool::load() { } auto Coverage = std::move(CoverageOrErr.get()); unsigned Mismatched = Coverage->getMismatchedCount(); - if (Mismatched) + if (Mismatched) { warning(utostr(Mismatched) + " functions have mismatched data"); + if (ViewOpts.Debug) { + for (const auto &HashMismatch : Coverage->getHashMismatches()) + errs() << "hash-mismatch: " + << "No profile record found for '" << HashMismatch.first << "'" + << " with hash = 0x" << utohexstr(HashMismatch.second) << "\n"; + + for (const auto &CounterMismatch : Coverage->getCounterMismatches()) + errs() << "counter-mismatch: " + << "Coverage mapping for " << CounterMismatch.first + << " only has " << CounterMismatch.second + << " valid counter expressions\n"; + } + } + + remapPathNames(*Coverage); + if (!SourceFiles.empty()) removeUnmappedInputs(*Coverage); @@ -348,33 +379,58 @@ std::unique_ptr<CoverageMapping> CodeCoverageTool::load() { return Coverage; } +void CodeCoverageTool::remapPathNames(const CoverageMapping &Coverage) { + if (!PathRemapping) + return; + + // Convert remapping paths to native paths with trailing seperators. + auto nativeWithTrailing = [](StringRef Path) -> std::string { + if (Path.empty()) + return ""; + SmallString<128> NativePath; + sys::path::native(Path, NativePath); + if (!sys::path::is_separator(NativePath.back())) + NativePath += sys::path::get_separator(); + return NativePath.c_str(); + }; + std::string RemapFrom = nativeWithTrailing(PathRemapping->first); + std::string RemapTo = nativeWithTrailing(PathRemapping->second); + + // Create a mapping from coverage data file paths to local paths. + for (StringRef Filename : Coverage.getUniqueSourceFiles()) { + SmallString<128> NativeFilename; + sys::path::native(Filename, NativeFilename); + if (NativeFilename.startswith(RemapFrom)) { + RemappedFilenames[Filename] = + RemapTo + NativeFilename.substr(RemapFrom.size()).str(); + } + } + + // Convert input files from local paths to coverage data file paths. + StringMap<std::string> InvRemappedFilenames; + for (const auto &RemappedFilename : RemappedFilenames) + InvRemappedFilenames[RemappedFilename.getValue()] = RemappedFilename.getKey(); + + for (std::string &Filename : SourceFiles) { + SmallString<128> NativeFilename; + sys::path::native(Filename, NativeFilename); + auto CovFileName = InvRemappedFilenames.find(NativeFilename); + if (CovFileName != InvRemappedFilenames.end()) + Filename = CovFileName->second; + } +} + void CodeCoverageTool::removeUnmappedInputs(const CoverageMapping &Coverage) { std::vector<StringRef> CoveredFiles = Coverage.getUniqueSourceFiles(); auto UncoveredFilesIt = SourceFiles.end(); - if (!CompareFilenamesOnly) { - // The user may have specified source files which aren't in the coverage - // mapping. Filter these files away. - UncoveredFilesIt = std::remove_if( - SourceFiles.begin(), SourceFiles.end(), [&](const std::string &SF) { - return !std::binary_search(CoveredFiles.begin(), CoveredFiles.end(), - SF); - }); - } else { - for (auto &SF : SourceFiles) { - StringRef SFBase = sys::path::filename(SF); - for (const auto &CF : CoveredFiles) { - if (SFBase == sys::path::filename(CF)) { - RemappedFilenames[CF] = SF; - SF = CF; - break; - } - } - } - UncoveredFilesIt = std::remove_if( - SourceFiles.begin(), SourceFiles.end(), - [&](const std::string &SF) { return !RemappedFilenames.count(SF); }); - } + // The user may have specified source files which aren't in the coverage + // mapping. Filter these files away. + UncoveredFilesIt = std::remove_if( + SourceFiles.begin(), SourceFiles.end(), [&](const std::string &SF) { + return !std::binary_search(CoveredFiles.begin(), CoveredFiles.end(), + SF); + }); SourceFiles.erase(UncoveredFilesIt, SourceFiles.end()); } @@ -392,7 +448,7 @@ void CodeCoverageTool::demangleSymbols(const CoverageMapping &Coverage) { error(InputPath, EC.message()); return; } - tool_output_file InputTOF{InputPath, InputFD}; + ToolOutputFile InputTOF{InputPath, InputFD}; unsigned NumSymbols = 0; for (const auto &Function : Coverage.getCoveredFunctions()) { @@ -410,7 +466,7 @@ void CodeCoverageTool::demangleSymbols(const CoverageMapping &Coverage) { error(OutputPath, EC.message()); return; } - tool_output_file OutputTOF{OutputPath, OutputFD}; + ToolOutputFile OutputTOF{OutputPath, OutputFD}; OutputTOF.os().close(); // Invoke the demangler. @@ -418,10 +474,7 @@ void CodeCoverageTool::demangleSymbols(const CoverageMapping &Coverage) { for (const std::string &Arg : ViewOpts.DemanglerOpts) ArgsV.push_back(Arg.c_str()); ArgsV.push_back(nullptr); - StringRef InputPathRef = InputPath.str(); - StringRef OutputPathRef = OutputPath.str(); - StringRef StderrRef; - const StringRef *Redirects[] = {&InputPathRef, &OutputPathRef, &StderrRef}; + Optional<StringRef> Redirects[] = {InputPath.str(), OutputPath.str(), {""}}; std::string ErrMsg; int RC = sys::ExecuteAndWait(ViewOpts.DemanglerOpts[0], ArgsV.data(), /*env=*/nullptr, Redirects, /*secondsToWait=*/0, @@ -475,7 +528,8 @@ void CodeCoverageTool::writeSourceFileView(StringRef SourceFile, auto OS = std::move(OSOrErr.get()); View->print(*OS.get(), /*Wholefile=*/true, - /*ShowSourceName=*/ShowFilenames); + /*ShowSourceName=*/ShowFilenames, + /*ShowTitle=*/ViewOpts.hasOutputDirectory()); Printer->closeViewFile(std::move(OS)); } @@ -499,8 +553,8 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { cl::desc( "File with the profile data obtained after an instrumented run")); - cl::opt<std::string> Arch( - "arch", cl::desc("architecture of the coverage mapping binary")); + cl::list<std::string> Arches( + "arch", cl::desc("architectures of the coverage mapping binaries")); cl::opt<bool> DebugDump("dump", cl::Optional, cl::desc("Show internal debug dump")); @@ -513,10 +567,10 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { "HTML output")), cl::init(CoverageViewOptions::OutputFormat::Text)); - cl::opt<bool> FilenameEquivalence( - "filename-equivalence", cl::Optional, - cl::desc("Treat source files as equivalent to paths in the coverage data " - "when the file names match, even if the full paths do not")); + cl::opt<std::string> PathRemap( + "path-equivalence", cl::Optional, + cl::desc("<from>,<to> Map coverage data paths to local source file " + "paths")); cl::OptionCategory FilteringCategory("Function filtering options"); @@ -525,6 +579,12 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { cl::desc("Show code coverage only for functions with the given name"), cl::ZeroOrMore, cl::cat(FilteringCategory)); + cl::list<std::string> NameFilterFiles( + "name-whitelist", cl::Optional, + cl::desc("Show code coverage only for functions listed in the given " + "file"), + cl::ZeroOrMore, cl::cat(FilteringCategory)); + cl::list<std::string> NameRegexFilters( "name-regex", cl::Optional, cl::desc("Show code coverage only for functions that match the given " @@ -562,10 +622,22 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { cl::list<std::string> DemanglerOpts( "Xdemangler", cl::desc("<demangler-path>|<demangler-option>")); + cl::opt<bool> RegionSummary( + "show-region-summary", cl::Optional, + cl::desc("Show region statistics in summary table"), + cl::init(true)); + + cl::opt<bool> InstantiationSummary( + "show-instantiation-summary", cl::Optional, + cl::desc("Show instantiation statistics in summary table")); + + cl::opt<bool> SummaryOnly( + "summary-only", cl::Optional, + cl::desc("Export only summary information for each source file")); + auto commandLineParser = [&, this](int argc, const char **argv) -> int { cl::ParseCommandLineOptions(argc, argv, "LLVM code coverage tool\n"); ViewOpts.Debug = DebugDump; - CompareFilenamesOnly = FilenameEquivalence; if (!CovFilename.empty()) ObjectFilenames.emplace_back(CovFilename); @@ -590,6 +662,12 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { break; } + // If path-equivalence was given and is a comma seperated pair then set + // PathRemapping. + auto EquivPair = StringRef(PathRemap).split(','); + if (!(EquivPair.first.empty() && EquivPair.second.empty())) + PathRemapping = EquivPair; + // If a demangler is supplied, check if it exists and register it. if (DemanglerOpts.size()) { auto DemanglerPathOrErr = sys::findProgramByName(DemanglerOpts[0]); @@ -602,21 +680,33 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { ViewOpts.DemanglerOpts.swap(DemanglerOpts); } + // Read in -name-whitelist files. + if (!NameFilterFiles.empty()) { + std::string SpecialCaseListErr; + NameWhitelist = + SpecialCaseList::create(NameFilterFiles, SpecialCaseListErr); + if (!NameWhitelist) + error(SpecialCaseListErr); + } + // Create the function filters - if (!NameFilters.empty() || !NameRegexFilters.empty()) { - auto NameFilterer = new CoverageFilters; + if (!NameFilters.empty() || NameWhitelist || !NameRegexFilters.empty()) { + auto NameFilterer = llvm::make_unique<CoverageFilters>(); for (const auto &Name : NameFilters) NameFilterer->push_back(llvm::make_unique<NameCoverageFilter>(Name)); + if (NameWhitelist) + NameFilterer->push_back( + llvm::make_unique<NameWhitelistCoverageFilter>(*NameWhitelist)); for (const auto &Regex : NameRegexFilters) NameFilterer->push_back( llvm::make_unique<NameRegexCoverageFilter>(Regex)); - Filters.push_back(std::unique_ptr<CoverageFilter>(NameFilterer)); + Filters.push_back(std::move(NameFilterer)); } if (RegionCoverageLtFilter.getNumOccurrences() || RegionCoverageGtFilter.getNumOccurrences() || LineCoverageLtFilter.getNumOccurrences() || LineCoverageGtFilter.getNumOccurrences()) { - auto StatFilterer = new CoverageFilters; + auto StatFilterer = llvm::make_unique<CoverageFilters>(); if (RegionCoverageLtFilter.getNumOccurrences()) StatFilterer->push_back(llvm::make_unique<RegionCoverageFilter>( RegionCoverageFilter::LessThan, RegionCoverageLtFilter)); @@ -629,15 +719,22 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { if (LineCoverageGtFilter.getNumOccurrences()) StatFilterer->push_back(llvm::make_unique<LineCoverageFilter>( RegionCoverageFilter::GreaterThan, LineCoverageGtFilter)); - Filters.push_back(std::unique_ptr<CoverageFilter>(StatFilterer)); + Filters.push_back(std::move(StatFilterer)); } - if (!Arch.empty() && - Triple(Arch).getArch() == llvm::Triple::ArchType::UnknownArch) { - error("Unknown architecture: " + Arch); - return 1; + if (!Arches.empty()) { + for (const std::string &Arch : Arches) { + if (Triple(Arch).getArch() == llvm::Triple::ArchType::UnknownArch) { + error("Unknown architecture: " + Arch); + return 1; + } + CoverageArches.emplace_back(Arch); + } + if (CoverageArches.size() != ObjectFilenames.size()) { + error("Number of architectures doesn't match the number of objects"); + return 1; + } } - CoverageArch = Arch; for (const std::string &File : InputSourceFiles) collectPaths(File); @@ -648,6 +745,10 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) { ::exit(0); } + ViewOpts.ShowRegionSummary = RegionSummary; + ViewOpts.ShowInstantiationSummary = InstantiationSummary; + ViewOpts.ExportSummaryOnly = SummaryOnly; + return 0; }; @@ -689,7 +790,7 @@ int CodeCoverageTool::show(int argc, const char **argv, cl::opt<bool> ShowInstantiations("show-instantiations", cl::Optional, cl::desc("Show function instantiations"), - cl::cat(ViewCategory)); + cl::init(true), cl::cat(ViewCategory)); cl::opt<std::string> ShowOutputDirectory( "output-dir", cl::init(""), @@ -720,7 +821,6 @@ int CodeCoverageTool::show(int argc, const char **argv, ViewOpts.ShowLineStats = ShowLineExecutionCounts.getNumOccurrences() != 0 || !ShowRegions || ShowBestLineRegionsCounts; ViewOpts.ShowRegionMarkers = ShowRegions || ShowBestLineRegionsCounts; - ViewOpts.ShowLineStatsOrRegionMarkers = ShowBestLineRegionsCounts; ViewOpts.ShowExpandedRegions = ShowExpansions; ViewOpts.ShowFunctionInstantiations = ShowInstantiations; ViewOpts.ShowOutputDirectory = ShowOutputDirectory; @@ -753,29 +853,54 @@ int CodeCoverageTool::show(int argc, const char **argv, auto Printer = CoveragePrinter::create(ViewOpts); - if (!Filters.empty()) { - auto OSOrErr = Printer->createViewFile("functions", /*InToplevel=*/true); - if (Error E = OSOrErr.takeError()) { - error("Could not create view file!", toString(std::move(E))); + if (SourceFiles.empty()) + // Get the source files from the function coverage mapping. + for (StringRef Filename : Coverage->getUniqueSourceFiles()) + SourceFiles.push_back(Filename); + + // Create an index out of the source files. + if (ViewOpts.hasOutputDirectory()) { + if (Error E = Printer->createIndexFile(SourceFiles, *Coverage, Filters)) { + error("Could not create index file!", toString(std::move(E))); return 1; } - auto OS = std::move(OSOrErr.get()); - - // Show functions. - for (const auto &Function : Coverage->getCoveredFunctions()) { - if (!Filters.matches(Function)) - continue; + } - auto mainView = createFunctionView(Function, *Coverage); - if (!mainView) { - warning("Could not read coverage for '" + Function.Name + "'."); - continue; + if (!Filters.empty()) { + // Build the map of filenames to functions. + std::map<llvm::StringRef, std::vector<const FunctionRecord *>> + FilenameFunctionMap; + for (const auto &SourceFile : SourceFiles) + for (const auto &Function : Coverage->getCoveredFunctions(SourceFile)) + if (Filters.matches(*Coverage.get(), Function)) + FilenameFunctionMap[SourceFile].push_back(&Function); + + // Only print filter matching functions for each file. + for (const auto &FileFunc : FilenameFunctionMap) { + StringRef File = FileFunc.first; + const auto &Functions = FileFunc.second; + + auto OSOrErr = Printer->createViewFile(File, /*InToplevel=*/false); + if (Error E = OSOrErr.takeError()) { + error("Could not create view file!", toString(std::move(E))); + return 1; + } + auto OS = std::move(OSOrErr.get()); + + bool ShowTitle = ViewOpts.hasOutputDirectory(); + for (const auto *Function : Functions) { + auto FunctionView = createFunctionView(*Function, *Coverage); + if (!FunctionView) { + warning("Could not read coverage for '" + Function->Name + "'."); + continue; + } + FunctionView->print(*OS.get(), /*WholeFile=*/false, + /*ShowSourceName=*/true, ShowTitle); + ShowTitle = false; } - mainView->print(*OS.get(), /*WholeFile=*/false, /*ShowSourceName=*/true); + Printer->closeViewFile(std::move(OS)); } - - Printer->closeViewFile(std::move(OS)); return 0; } @@ -784,19 +909,6 @@ int CodeCoverageTool::show(int argc, const char **argv, (SourceFiles.size() != 1) || ViewOpts.hasOutputDirectory() || (ViewOpts.Format == CoverageViewOptions::OutputFormat::HTML); - if (SourceFiles.empty()) - // Get the source files from the function coverage mapping. - for (StringRef Filename : Coverage->getUniqueSourceFiles()) - SourceFiles.push_back(Filename); - - // Create an index out of the source files. - if (ViewOpts.hasOutputDirectory()) { - if (Error E = Printer->createIndexFile(SourceFiles, *Coverage)) { - error("Could not create index file!", toString(std::move(E))); - return 1; - } - } - // If NumThreads is not specified, auto-detect a good default. if (NumThreads == 0) NumThreads = @@ -839,10 +951,20 @@ int CodeCoverageTool::report(int argc, const char **argv, return 1; CoverageReport Report(ViewOpts, *Coverage.get()); - if (!ShowFunctionSummaries) - Report.renderFileReports(llvm::outs()); - else + if (!ShowFunctionSummaries) { + if (SourceFiles.empty()) + Report.renderFileReports(llvm::outs()); + else + Report.renderFileReports(llvm::outs(), SourceFiles); + } else { + if (SourceFiles.empty()) { + error("Source files must be specified when -show-functions=true is " + "specified"); + return 1; + } + Report.renderFunctionReports(SourceFiles, DC, llvm::outs()); + } return 0; } @@ -864,7 +986,7 @@ int CodeCoverageTool::export_(int argc, const char **argv, return 1; } - exportCoverageDataToJson(*Coverage.get(), outs()); + exportCoverageDataToJson(*Coverage.get(), ViewOpts, outs()); return 0; } diff --git a/tools/llvm-cov/CoverageExporterJson.cpp b/tools/llvm-cov/CoverageExporterJson.cpp index ef50bba21238..7b700908968d 100644 --- a/tools/llvm-cov/CoverageExporterJson.cpp +++ b/tools/llvm-cov/CoverageExporterJson.cpp @@ -57,6 +57,8 @@ using namespace llvm; using namespace coverage; class CoverageExporterJson { + const CoverageViewOptions &Options; + /// \brief Output stream to print JSON to. raw_ostream &OS; @@ -171,12 +173,15 @@ class CoverageExporterJson { std::vector<std::string> SourceFiles; for (StringRef SF : Coverage.getUniqueSourceFiles()) SourceFiles.emplace_back(SF); - auto FileReports = - CoverageReport::prepareFileReports(Coverage, Totals, SourceFiles); + auto FileReports = CoverageReport::prepareFileReports(Coverage, Totals, + SourceFiles, Options); renderFiles(SourceFiles, FileReports); - emitDictKey("functions"); - renderFunctions(Coverage.getCoveredFunctions()); + // Skip functions-level information for summary-only export mode. + if (!Options.ExportSummaryOnly) { + emitDictKey("functions"); + renderFunctions(Coverage.getCoveredFunctions()); + } emitDictKey("totals"); renderSummary(Totals); @@ -252,27 +257,31 @@ class CoverageExporterJson { emitDictStart(); emitDictElement("filename", FileCoverage.getFilename()); - emitDictKey("segments"); - // Start List of Segments. - emitArrayStart(); + // Skip segments and expansions for summary-only export mode. + if (!Options.ExportSummaryOnly) { + emitDictKey("segments"); - for (const auto &Segment : FileCoverage) - renderSegment(Segment); + // Start List of Segments. + emitArrayStart(); - // End List of Segments. - emitArrayEnd(); + for (const auto &Segment : FileCoverage) + renderSegment(Segment); - emitDictKey("expansions"); + // End List of Segments. + emitArrayEnd(); - // Start List of Expansions. - emitArrayStart(); + emitDictKey("expansions"); - for (const auto &Expansion : FileCoverage.getExpansions()) - renderExpansion(Expansion); + // Start List of Expansions. + emitArrayStart(); - // End List of Expansions. - emitArrayEnd(); + for (const auto &Expansion : FileCoverage.getExpansions()) + renderExpansion(Expansion); + + // End List of Expansions. + emitArrayEnd(); + } emitDictKey("summary"); renderSummary(FileReport); @@ -360,8 +369,8 @@ class CoverageExporterJson { // Start Line Coverage Summary. emitDictStart(); - emitDictElement("count", Summary.LineCoverage.NumLines); - emitDictElement("covered", Summary.LineCoverage.Covered); + emitDictElement("count", Summary.LineCoverage.getNumLines()); + emitDictElement("covered", Summary.LineCoverage.getCovered()); emitDictElement("percent", Summary.LineCoverage.getPercentCovered()); // End Line Coverage Summary. emitDictEnd(); @@ -370,8 +379,8 @@ class CoverageExporterJson { // Start Function Coverage Summary. emitDictStart(); - emitDictElement("count", Summary.FunctionCoverage.NumFunctions); - emitDictElement("covered", Summary.FunctionCoverage.Executed); + emitDictElement("count", Summary.FunctionCoverage.getNumFunctions()); + emitDictElement("covered", Summary.FunctionCoverage.getExecuted()); emitDictElement("percent", Summary.FunctionCoverage.getPercentCovered()); // End Function Coverage Summary. emitDictEnd(); @@ -380,8 +389,8 @@ class CoverageExporterJson { // Start Instantiation Coverage Summary. emitDictStart(); - emitDictElement("count", Summary.InstantiationCoverage.NumFunctions); - emitDictElement("covered", Summary.InstantiationCoverage.Executed); + emitDictElement("count", Summary.InstantiationCoverage.getNumFunctions()); + emitDictElement("covered", Summary.InstantiationCoverage.getExecuted()); emitDictElement("percent", Summary.InstantiationCoverage.getPercentCovered()); // End Function Coverage Summary. @@ -391,9 +400,11 @@ class CoverageExporterJson { // Start Region Coverage Summary. emitDictStart(); - emitDictElement("count", Summary.RegionCoverage.NumRegions); - emitDictElement("covered", Summary.RegionCoverage.Covered); - emitDictElement("notcovered", Summary.RegionCoverage.NotCovered); + emitDictElement("count", Summary.RegionCoverage.getNumRegions()); + emitDictElement("covered", Summary.RegionCoverage.getCovered()); + emitDictElement("notcovered", + Summary.RegionCoverage.getNumRegions() - + Summary.RegionCoverage.getCovered()); emitDictElement("percent", Summary.RegionCoverage.getPercentCovered()); // End Region Coverage Summary. emitDictEnd(); @@ -403,8 +414,9 @@ class CoverageExporterJson { } public: - CoverageExporterJson(const CoverageMapping &CoverageMapping, raw_ostream &OS) - : OS(OS), Coverage(CoverageMapping) { + CoverageExporterJson(const CoverageMapping &CoverageMapping, + const CoverageViewOptions &Options, raw_ostream &OS) + : Options(Options), OS(OS), Coverage(CoverageMapping) { State.push(JsonState::None); } @@ -414,8 +426,9 @@ public: /// \brief Export the given CoverageMapping to a JSON Format. void exportCoverageDataToJson(const CoverageMapping &CoverageMapping, + const CoverageViewOptions &Options, raw_ostream &OS) { - auto Exporter = CoverageExporterJson(CoverageMapping, OS); + auto Exporter = CoverageExporterJson(CoverageMapping, Options, OS); Exporter.print(); } diff --git a/tools/llvm-cov/CoverageFilters.cpp b/tools/llvm-cov/CoverageFilters.cpp index 325dd7235789..441179601dcc 100644 --- a/tools/llvm-cov/CoverageFilters.cpp +++ b/tools/llvm-cov/CoverageFilters.cpp @@ -17,42 +17,57 @@ using namespace llvm; -bool NameCoverageFilter::matches(const coverage::FunctionRecord &Function) { +bool NameCoverageFilter::matches( + const coverage::CoverageMapping &, + const coverage::FunctionRecord &Function) const { StringRef FuncName = Function.Name; return FuncName.find(Name) != StringRef::npos; } -bool -NameRegexCoverageFilter::matches(const coverage::FunctionRecord &Function) { +bool NameRegexCoverageFilter::matches( + const coverage::CoverageMapping &, + const coverage::FunctionRecord &Function) const { return llvm::Regex(Regex).match(Function.Name); } -bool RegionCoverageFilter::matches(const coverage::FunctionRecord &Function) { - return PassesThreshold(FunctionCoverageSummary::get(Function) +bool NameWhitelistCoverageFilter::matches( + const coverage::CoverageMapping &, + const coverage::FunctionRecord &Function) const { + return Whitelist.inSection("llvmcov", "whitelist_fun", Function.Name); +} + +bool RegionCoverageFilter::matches( + const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const { + return PassesThreshold(FunctionCoverageSummary::get(CM, Function) .RegionCoverage.getPercentCovered()); } -bool LineCoverageFilter::matches(const coverage::FunctionRecord &Function) { - return PassesThreshold( - FunctionCoverageSummary::get(Function).LineCoverage.getPercentCovered()); +bool LineCoverageFilter::matches( + const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const { + return PassesThreshold(FunctionCoverageSummary::get(CM, Function) + .LineCoverage.getPercentCovered()); } void CoverageFilters::push_back(std::unique_ptr<CoverageFilter> Filter) { Filters.push_back(std::move(Filter)); } -bool CoverageFilters::matches(const coverage::FunctionRecord &Function) { +bool CoverageFilters::matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const { for (const auto &Filter : Filters) { - if (Filter->matches(Function)) + if (Filter->matches(CM, Function)) return true; } return false; } -bool -CoverageFiltersMatchAll::matches(const coverage::FunctionRecord &Function) { +bool CoverageFiltersMatchAll::matches( + const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const { for (const auto &Filter : Filters) { - if (!Filter->matches(Function)) + if (!Filter->matches(CM, Function)) return false; } return true; diff --git a/tools/llvm-cov/CoverageFilters.h b/tools/llvm-cov/CoverageFilters.h index 756c4b47872c..aeaf61de1730 100644 --- a/tools/llvm-cov/CoverageFilters.h +++ b/tools/llvm-cov/CoverageFilters.h @@ -14,7 +14,9 @@ #ifndef LLVM_COV_COVERAGEFILTERS_H #define LLVM_COV_COVERAGEFILTERS_H +#include "CoverageSummaryInfo.h" #include "llvm/ProfileData/Coverage/CoverageMapping.h" +#include "llvm/Support/SpecialCaseList.h" #include <memory> #include <vector> @@ -26,7 +28,8 @@ public: virtual ~CoverageFilter() {} /// \brief Return true if the function passes the requirements of this filter. - virtual bool matches(const coverage::FunctionRecord &Function) { + virtual bool matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const { return true; } }; @@ -38,7 +41,8 @@ class NameCoverageFilter : public CoverageFilter { public: NameCoverageFilter(StringRef Name) : Name(Name) {} - bool matches(const coverage::FunctionRecord &Function) override; + bool matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const override; }; /// \brief Matches functions whose name matches a certain regular expression. @@ -48,7 +52,21 @@ class NameRegexCoverageFilter : public CoverageFilter { public: NameRegexCoverageFilter(StringRef Regex) : Regex(Regex) {} - bool matches(const coverage::FunctionRecord &Function) override; + bool matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const override; +}; + +/// \brief Matches functions whose name appears in a SpecialCaseList in the +/// whitelist_fun section. +class NameWhitelistCoverageFilter : public CoverageFilter { + const SpecialCaseList &Whitelist; + +public: + NameWhitelistCoverageFilter(const SpecialCaseList &Whitelist) + : Whitelist(Whitelist) {} + + bool matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const override; }; /// \brief Matches numbers that pass a certain threshold. @@ -84,7 +102,8 @@ public: RegionCoverageFilter(Operation Op, double Threshold) : StatisticThresholdFilter(Op, Threshold) {} - bool matches(const coverage::FunctionRecord &Function) override; + bool matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const override; }; /// \brief Matches functions whose line coverage percentage @@ -95,7 +114,8 @@ public: LineCoverageFilter(Operation Op, double Threshold) : StatisticThresholdFilter(Op, Threshold) {} - bool matches(const coverage::FunctionRecord &Function) override; + bool matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const override; }; /// \brief A collection of filters. @@ -111,7 +131,8 @@ public: bool empty() const { return Filters.empty(); } - bool matches(const coverage::FunctionRecord &Function) override; + bool matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const override; }; /// \brief A collection of filters. @@ -119,7 +140,8 @@ public: /// in an instance of this class. class CoverageFiltersMatchAll : public CoverageFilters { public: - bool matches(const coverage::FunctionRecord &Function) override; + bool matches(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function) const override; }; } // namespace llvm diff --git a/tools/llvm-cov/CoverageReport.cpp b/tools/llvm-cov/CoverageReport.cpp index c68bb9048df1..9c553a7f64c7 100644 --- a/tools/llvm-cov/CoverageReport.cpp +++ b/tools/llvm-cov/CoverageReport.cpp @@ -14,7 +14,6 @@ #include "CoverageReport.h" #include "RenderingSupport.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/Support/FileSystem.h" #include "llvm/Support/Format.h" #include "llvm/Support/Path.h" #include <numeric> @@ -181,47 +180,58 @@ void CoverageReport::render(const FileCoverageSummary &File, SmallString<256> FileName = File.Name; sys::path::remove_dots(FileName, /*remove_dot_dots=*/true); sys::path::native(FileName); - OS << column(FileName, FileReportColumns[0], Column::NoTrim) - << format("%*u", FileReportColumns[1], - (unsigned)File.RegionCoverage.NumRegions); - Options.colored_ostream(OS, FileCoverageColor) << format( - "%*u", FileReportColumns[2], (unsigned)File.RegionCoverage.NotCovered); - if (File.RegionCoverage.NumRegions) + OS << column(FileName, FileReportColumns[0], Column::NoTrim); + + if (Options.ShowRegionSummary) { + OS << format("%*u", FileReportColumns[1], + (unsigned)File.RegionCoverage.getNumRegions()); Options.colored_ostream(OS, FileCoverageColor) - << format("%*.2f", FileReportColumns[3] - 1, - File.RegionCoverage.getPercentCovered()) - << '%'; - else - OS << column("-", FileReportColumns[3], Column::RightAlignment); + << format("%*u", FileReportColumns[2], + (unsigned)(File.RegionCoverage.getNumRegions() - + File.RegionCoverage.getCovered())); + if (File.RegionCoverage.getNumRegions()) + Options.colored_ostream(OS, FileCoverageColor) + << format("%*.2f", FileReportColumns[3] - 1, + File.RegionCoverage.getPercentCovered()) + << '%'; + else + OS << column("-", FileReportColumns[3], Column::RightAlignment); + } + OS << format("%*u", FileReportColumns[4], - (unsigned)File.FunctionCoverage.NumFunctions); + (unsigned)File.FunctionCoverage.getNumFunctions()); OS << format("%*u", FileReportColumns[5], - (unsigned)(File.FunctionCoverage.NumFunctions - - File.FunctionCoverage.Executed)); - if (File.FunctionCoverage.NumFunctions) + (unsigned)(File.FunctionCoverage.getNumFunctions() - + File.FunctionCoverage.getExecuted())); + if (File.FunctionCoverage.getNumFunctions()) Options.colored_ostream(OS, FuncCoverageColor) << format("%*.2f", FileReportColumns[6] - 1, File.FunctionCoverage.getPercentCovered()) << '%'; else OS << column("-", FileReportColumns[6], Column::RightAlignment); - OS << format("%*u", FileReportColumns[7], - (unsigned)File.InstantiationCoverage.NumFunctions); - OS << format("%*u", FileReportColumns[8], - (unsigned)(File.InstantiationCoverage.NumFunctions - - File.InstantiationCoverage.Executed)); - if (File.InstantiationCoverage.NumFunctions) - Options.colored_ostream(OS, InstantiationCoverageColor) - << format("%*.2f", FileReportColumns[9] - 1, - File.InstantiationCoverage.getPercentCovered()) - << '%'; - else - OS << column("-", FileReportColumns[9], Column::RightAlignment); + + if (Options.ShowInstantiationSummary) { + OS << format("%*u", FileReportColumns[7], + (unsigned)File.InstantiationCoverage.getNumFunctions()); + OS << format("%*u", FileReportColumns[8], + (unsigned)(File.InstantiationCoverage.getNumFunctions() - + File.InstantiationCoverage.getExecuted())); + if (File.InstantiationCoverage.getNumFunctions()) + Options.colored_ostream(OS, InstantiationCoverageColor) + << format("%*.2f", FileReportColumns[9] - 1, + File.InstantiationCoverage.getPercentCovered()) + << '%'; + else + OS << column("-", FileReportColumns[9], Column::RightAlignment); + } + OS << format("%*u", FileReportColumns[10], - (unsigned)File.LineCoverage.NumLines); + (unsigned)File.LineCoverage.getNumLines()); Options.colored_ostream(OS, LineCoverageColor) << format( - "%*u", FileReportColumns[11], (unsigned)File.LineCoverage.NotCovered); - if (File.LineCoverage.NumLines) + "%*u", FileReportColumns[11], (unsigned)(File.LineCoverage.getNumLines() - + File.LineCoverage.getCovered())); + if (File.LineCoverage.getNumLines()) Options.colored_ostream(OS, LineCoverageColor) << format("%*.2f", FileReportColumns[12] - 1, File.LineCoverage.getPercentCovered()) @@ -241,20 +251,22 @@ void CoverageReport::render(const FunctionCoverageSummary &Function, OS << column(DC.demangle(Function.Name), FunctionReportColumns[0], Column::RightTrim) << format("%*u", FunctionReportColumns[1], - (unsigned)Function.RegionCoverage.NumRegions); + (unsigned)Function.RegionCoverage.getNumRegions()); Options.colored_ostream(OS, FuncCoverageColor) << format("%*u", FunctionReportColumns[2], - (unsigned)Function.RegionCoverage.NotCovered); + (unsigned)(Function.RegionCoverage.getNumRegions() - + Function.RegionCoverage.getCovered())); Options.colored_ostream( OS, determineCoveragePercentageColor(Function.RegionCoverage)) << format("%*.2f", FunctionReportColumns[3] - 1, Function.RegionCoverage.getPercentCovered()) << '%'; OS << format("%*u", FunctionReportColumns[4], - (unsigned)Function.LineCoverage.NumLines); + (unsigned)Function.LineCoverage.getNumLines()); Options.colored_ostream(OS, LineCoverageColor) << format("%*u", FunctionReportColumns[5], - (unsigned)Function.LineCoverage.NotCovered); + (unsigned)(Function.LineCoverage.getNumLines() - + Function.LineCoverage.getCovered())); Options.colored_ostream( OS, determineCoveragePercentageColor(Function.LineCoverage)) << format("%*.2f", FunctionReportColumns[6] - 1, @@ -293,7 +305,7 @@ void CoverageReport::renderFunctionReports(ArrayRef<std::string> Files, OS << "\n"; FunctionCoverageSummary Totals("TOTAL"); for (const auto &F : Functions) { - FunctionCoverageSummary Function = FunctionCoverageSummary::get(F); + auto Function = FunctionCoverageSummary::get(Coverage, F); ++Totals.ExecutionCount; Totals.RegionCoverage += Function.RegionCoverage; Totals.LineCoverage += Function.LineCoverage; @@ -307,35 +319,38 @@ void CoverageReport::renderFunctionReports(ArrayRef<std::string> Files, } } -std::vector<FileCoverageSummary> -CoverageReport::prepareFileReports(const coverage::CoverageMapping &Coverage, - FileCoverageSummary &Totals, - ArrayRef<std::string> Files) { +std::vector<FileCoverageSummary> CoverageReport::prepareFileReports( + const coverage::CoverageMapping &Coverage, FileCoverageSummary &Totals, + ArrayRef<std::string> Files, const CoverageViewOptions &Options, + const CoverageFilter &Filters) { std::vector<FileCoverageSummary> FileReports; unsigned LCP = getRedundantPrefixLen(Files); for (StringRef Filename : Files) { FileCoverageSummary Summary(Filename.drop_front(LCP)); - // Map source locations to aggregate function coverage summaries. - DenseMap<std::pair<unsigned, unsigned>, FunctionCoverageSummary> Summaries; - - for (const auto &F : Coverage.getCoveredFunctions(Filename)) { - FunctionCoverageSummary Function = FunctionCoverageSummary::get(F); - auto StartLoc = F.CountedRegions[0].startLoc(); + for (const auto &Group : Coverage.getInstantiationGroups(Filename)) { + std::vector<FunctionCoverageSummary> InstantiationSummaries; + for (const coverage::FunctionRecord *F : Group.getInstantiations()) { + if (!Filters.matches(Coverage, *F)) + continue; + auto InstantiationSummary = FunctionCoverageSummary::get(Coverage, *F); + Summary.addInstantiation(InstantiationSummary); + Totals.addInstantiation(InstantiationSummary); + InstantiationSummaries.push_back(InstantiationSummary); + } + if (InstantiationSummaries.empty()) + continue; - auto UniquedSummary = Summaries.insert({StartLoc, Function}); - if (!UniquedSummary.second) - UniquedSummary.first->second.update(Function); + auto GroupSummary = + FunctionCoverageSummary::get(Group, InstantiationSummaries); - Summary.addInstantiation(Function); - Totals.addInstantiation(Function); - } + if (Options.Debug) + outs() << "InstantiationGroup: " << GroupSummary.Name << " with " + << "size = " << Group.size() << "\n"; - for (const auto &UniquedSummary : Summaries) { - const FunctionCoverageSummary &FCS = UniquedSummary.second; - Summary.addFunction(FCS); - Totals.addFunction(FCS); + Summary.addFunction(GroupSummary); + Totals.addFunction(GroupSummary); } FileReports.push_back(Summary); @@ -351,34 +366,57 @@ void CoverageReport::renderFileReports(raw_ostream &OS) const { renderFileReports(OS, UniqueSourceFiles); } -void CoverageReport::renderFileReports(raw_ostream &OS, - ArrayRef<std::string> Files) const { +void CoverageReport::renderFileReports( + raw_ostream &OS, ArrayRef<std::string> Files) const { + renderFileReports(OS, Files, CoverageFiltersMatchAll()); +} + +void CoverageReport::renderFileReports( + raw_ostream &OS, ArrayRef<std::string> Files, + const CoverageFiltersMatchAll &Filters) const { FileCoverageSummary Totals("TOTAL"); - auto FileReports = prepareFileReports(Coverage, Totals, Files); + auto FileReports = + prepareFileReports(Coverage, Totals, Files, Options, Filters); std::vector<StringRef> Filenames; for (const FileCoverageSummary &FCS : FileReports) Filenames.emplace_back(FCS.Name); adjustColumnWidths(Filenames, {}); - OS << column("Filename", FileReportColumns[0]) - << column("Regions", FileReportColumns[1], Column::RightAlignment) - << column("Missed Regions", FileReportColumns[2], Column::RightAlignment) - << column("Cover", FileReportColumns[3], Column::RightAlignment) - << column("Functions", FileReportColumns[4], Column::RightAlignment) + OS << column("Filename", FileReportColumns[0]); + if (Options.ShowRegionSummary) + OS << column("Regions", FileReportColumns[1], Column::RightAlignment) + << column("Missed Regions", FileReportColumns[2], Column::RightAlignment) + << column("Cover", FileReportColumns[3], Column::RightAlignment); + OS << column("Functions", FileReportColumns[4], Column::RightAlignment) << column("Missed Functions", FileReportColumns[5], Column::RightAlignment) - << column("Executed", FileReportColumns[6], Column::RightAlignment) - << column("Instantiations", FileReportColumns[7], Column::RightAlignment) - << column("Missed Insts.", FileReportColumns[8], Column::RightAlignment) - << column("Executed", FileReportColumns[9], Column::RightAlignment) - << column("Lines", FileReportColumns[10], Column::RightAlignment) + << column("Executed", FileReportColumns[6], Column::RightAlignment); + if (Options.ShowInstantiationSummary) + OS << column("Instantiations", FileReportColumns[7], Column::RightAlignment) + << column("Missed Insts.", FileReportColumns[8], Column::RightAlignment) + << column("Executed", FileReportColumns[9], Column::RightAlignment); + OS << column("Lines", FileReportColumns[10], Column::RightAlignment) << column("Missed Lines", FileReportColumns[11], Column::RightAlignment) << column("Cover", FileReportColumns[12], Column::RightAlignment) << "\n"; renderDivider(FileReportColumns, OS); OS << "\n"; - for (const FileCoverageSummary &FCS : FileReports) - render(FCS, OS); + bool EmptyFiles = false; + for (const FileCoverageSummary &FCS : FileReports) { + if (FCS.FunctionCoverage.getNumFunctions()) + render(FCS, OS); + else + EmptyFiles = true; + } + + if (EmptyFiles && Filters.empty()) { + OS << "\n" + << "Files which contain no functions:\n"; + + for (const FileCoverageSummary &FCS : FileReports) + if (!FCS.FunctionCoverage.getNumFunctions()) + render(FCS, OS); + } renderDivider(FileReportColumns, OS); OS << "\n"; diff --git a/tools/llvm-cov/CoverageReport.h b/tools/llvm-cov/CoverageReport.h index 071be2e21594..1c9e68e832f3 100644 --- a/tools/llvm-cov/CoverageReport.h +++ b/tools/llvm-cov/CoverageReport.h @@ -14,6 +14,7 @@ #ifndef LLVM_COV_COVERAGEREPORT_H #define LLVM_COV_COVERAGEREPORT_H +#include "CoverageFilters.h" #include "CoverageSummaryInfo.h" #include "CoverageViewOptions.h" @@ -39,13 +40,20 @@ public: /// Prepare file reports for the files specified in \p Files. static std::vector<FileCoverageSummary> prepareFileReports(const coverage::CoverageMapping &Coverage, - FileCoverageSummary &Totals, ArrayRef<std::string> Files); + FileCoverageSummary &Totals, ArrayRef<std::string> Files, + const CoverageViewOptions &Options, + const CoverageFilter &Filters = CoverageFiltersMatchAll()); /// Render file reports for every unique file in the coverage mapping. void renderFileReports(raw_ostream &OS) const; /// Render file reports for the files specified in \p Files. void renderFileReports(raw_ostream &OS, ArrayRef<std::string> Files) const; + + /// Render file reports for the files specified in \p Files and the functions + /// in \p Filters. + void renderFileReports(raw_ostream &OS, ArrayRef<std::string> Files, + const CoverageFiltersMatchAll &Filters) const; }; } // end namespace llvm diff --git a/tools/llvm-cov/CoverageSummaryInfo.cpp b/tools/llvm-cov/CoverageSummaryInfo.cpp index 21aa7ff73a05..7847a2abf48c 100644 --- a/tools/llvm-cov/CoverageSummaryInfo.cpp +++ b/tools/llvm-cov/CoverageSummaryInfo.cpp @@ -18,7 +18,8 @@ using namespace llvm; using namespace coverage; FunctionCoverageSummary -FunctionCoverageSummary::get(const coverage::FunctionRecord &Function) { +FunctionCoverageSummary::get(const CoverageMapping &CM, + const coverage::FunctionRecord &Function) { // Compute the region coverage. size_t NumCodeRegions = 0, CoveredRegions = 0; for (auto &CR : Function.CountedRegions) { @@ -31,53 +32,40 @@ FunctionCoverageSummary::get(const coverage::FunctionRecord &Function) { // Compute the line coverage size_t NumLines = 0, CoveredLines = 0; - for (unsigned FileID = 0, E = Function.Filenames.size(); FileID < E; - ++FileID) { - // Find the line start and end of the function's source code - // in that particular file - unsigned LineStart = std::numeric_limits<unsigned>::max(); - unsigned LineEnd = 0; - for (auto &CR : Function.CountedRegions) { - if (CR.FileID != FileID) - continue; - LineStart = std::min(LineStart, CR.LineStart); - LineEnd = std::max(LineEnd, CR.LineEnd); - } - unsigned LineCount = LineEnd - LineStart + 1; - - // Get counters - llvm::SmallVector<uint64_t, 16> ExecutionCounts; - ExecutionCounts.resize(LineCount, 0); - for (auto &CR : Function.CountedRegions) { - if (CR.FileID != FileID) - continue; - // Ignore the lines that were skipped by the preprocessor. - auto ExecutionCount = CR.ExecutionCount; - if (CR.Kind == CounterMappingRegion::SkippedRegion) { - LineCount -= CR.LineEnd - CR.LineStart + 1; - ExecutionCount = 1; - } - for (unsigned I = CR.LineStart; I <= CR.LineEnd; ++I) - ExecutionCounts[I - LineStart] = ExecutionCount; - } - CoveredLines += LineCount - std::count(ExecutionCounts.begin(), - ExecutionCounts.end(), 0); - NumLines += LineCount; + CoverageData CD = CM.getCoverageForFunction(Function); + for (const auto &LCS : getLineCoverageStats(CD)) { + if (!LCS.isMapped()) + continue; + ++NumLines; + if (LCS.getExecutionCount()) + ++CoveredLines; } + return FunctionCoverageSummary( Function.Name, Function.ExecutionCount, RegionCoverageInfo(CoveredRegions, NumCodeRegions), LineCoverageInfo(CoveredLines, NumLines)); } -void FunctionCoverageSummary::update(const FunctionCoverageSummary &Summary) { - ExecutionCount += Summary.ExecutionCount; - RegionCoverage.Covered = - std::max(RegionCoverage.Covered, Summary.RegionCoverage.Covered); - RegionCoverage.NotCovered = - std::min(RegionCoverage.NotCovered, Summary.RegionCoverage.NotCovered); - LineCoverage.Covered = - std::max(LineCoverage.Covered, Summary.LineCoverage.Covered); - LineCoverage.NotCovered = - std::min(LineCoverage.NotCovered, Summary.LineCoverage.NotCovered); +FunctionCoverageSummary +FunctionCoverageSummary::get(const InstantiationGroup &Group, + ArrayRef<FunctionCoverageSummary> Summaries) { + std::string Name; + if (Group.hasName()) { + Name = Group.getName(); + } else { + llvm::raw_string_ostream OS(Name); + OS << "Definition at line " << Group.getLine() << ", column " + << Group.getColumn(); + } + + FunctionCoverageSummary Summary(Name); + Summary.ExecutionCount = Group.getTotalExecutionCount(); + Summary.RegionCoverage = Summaries[0].RegionCoverage; + Summary.LineCoverage = Summaries[0].LineCoverage; + for (const auto &FCS : Summaries.drop_front()) { + Summary.RegionCoverage.merge(FCS.RegionCoverage); + Summary.LineCoverage.merge(FCS.LineCoverage); + } + return Summary; } diff --git a/tools/llvm-cov/CoverageSummaryInfo.h b/tools/llvm-cov/CoverageSummaryInfo.h index 680fc3757686..8eae0b7fec97 100644 --- a/tools/llvm-cov/CoverageSummaryInfo.h +++ b/tools/llvm-cov/CoverageSummaryInfo.h @@ -21,32 +21,40 @@ namespace llvm { /// \brief Provides information about region coverage for a function/file. -struct RegionCoverageInfo { +class RegionCoverageInfo { /// \brief The number of regions that were executed at least once. size_t Covered; - /// \brief The number of regions that weren't executed. - size_t NotCovered; - /// \brief The total number of regions in a function/file. size_t NumRegions; - RegionCoverageInfo() : Covered(0), NotCovered(0), NumRegions(0) {} +public: + RegionCoverageInfo() : Covered(0), NumRegions(0) {} RegionCoverageInfo(size_t Covered, size_t NumRegions) - : Covered(Covered), NotCovered(NumRegions - Covered), - NumRegions(NumRegions) {} + : Covered(Covered), NumRegions(NumRegions) { + assert(Covered <= NumRegions && "Covered regions over-counted"); + } RegionCoverageInfo &operator+=(const RegionCoverageInfo &RHS) { Covered += RHS.Covered; - NotCovered += RHS.NotCovered; NumRegions += RHS.NumRegions; return *this; } + void merge(const RegionCoverageInfo &RHS) { + Covered = std::max(Covered, RHS.Covered); + NumRegions = std::max(NumRegions, RHS.NumRegions); + } + + size_t getCovered() const { return Covered; } + + size_t getNumRegions() const { return NumRegions; } + bool isFullyCovered() const { return Covered == NumRegions; } double getPercentCovered() const { + assert(Covered <= NumRegions && "Covered regions over-counted"); if (NumRegions == 0) return 0.0; return double(Covered) / double(NumRegions) * 100.0; @@ -54,31 +62,40 @@ struct RegionCoverageInfo { }; /// \brief Provides information about line coverage for a function/file. -struct LineCoverageInfo { +class LineCoverageInfo { /// \brief The number of lines that were executed at least once. size_t Covered; - /// \brief The number of lines that weren't executed. - size_t NotCovered; - /// \brief The total number of lines in a function/file. size_t NumLines; - LineCoverageInfo() : Covered(0), NotCovered(0), NumLines(0) {} +public: + LineCoverageInfo() : Covered(0), NumLines(0) {} LineCoverageInfo(size_t Covered, size_t NumLines) - : Covered(Covered), NotCovered(NumLines - Covered), NumLines(NumLines) {} + : Covered(Covered), NumLines(NumLines) { + assert(Covered <= NumLines && "Covered lines over-counted"); + } LineCoverageInfo &operator+=(const LineCoverageInfo &RHS) { Covered += RHS.Covered; - NotCovered += RHS.NotCovered; NumLines += RHS.NumLines; return *this; } + void merge(const LineCoverageInfo &RHS) { + Covered = std::max(Covered, RHS.Covered); + NumLines = std::max(NumLines, RHS.NumLines); + } + + size_t getCovered() const { return Covered; } + + size_t getNumLines() const { return NumLines; } + bool isFullyCovered() const { return Covered == NumLines; } double getPercentCovered() const { + assert(Covered <= NumLines && "Covered lines over-counted"); if (NumLines == 0) return 0.0; return double(Covered) / double(NumLines) * 100.0; @@ -86,13 +103,14 @@ struct LineCoverageInfo { }; /// \brief Provides information about function coverage for a file. -struct FunctionCoverageInfo { +class FunctionCoverageInfo { /// \brief The number of functions that were executed. size_t Executed; /// \brief The total number of functions in this file. size_t NumFunctions; +public: FunctionCoverageInfo() : Executed(0), NumFunctions(0) {} FunctionCoverageInfo(size_t Executed, size_t NumFunctions) @@ -104,9 +122,14 @@ struct FunctionCoverageInfo { ++NumFunctions; } + size_t getExecuted() const { return Executed; } + + size_t getNumFunctions() const { return NumFunctions; } + bool isFullyCovered() const { return Executed == NumFunctions; } double getPercentCovered() const { + assert(Executed <= NumFunctions && "Covered functions over-counted"); if (NumFunctions == 0) return 0.0; return double(Executed) / double(NumFunctions) * 100.0; @@ -115,28 +138,30 @@ struct FunctionCoverageInfo { /// \brief A summary of function's code coverage. struct FunctionCoverageSummary { - StringRef Name; + std::string Name; uint64_t ExecutionCount; RegionCoverageInfo RegionCoverage; LineCoverageInfo LineCoverage; - FunctionCoverageSummary(StringRef Name) : Name(Name), ExecutionCount(0) {} + FunctionCoverageSummary(const std::string &Name) + : Name(Name), ExecutionCount(0), RegionCoverage(), LineCoverage() {} - FunctionCoverageSummary(StringRef Name, uint64_t ExecutionCount, + FunctionCoverageSummary(const std::string &Name, uint64_t ExecutionCount, const RegionCoverageInfo &RegionCoverage, const LineCoverageInfo &LineCoverage) : Name(Name), ExecutionCount(ExecutionCount), - RegionCoverage(RegionCoverage), LineCoverage(LineCoverage) { - } + RegionCoverage(RegionCoverage), LineCoverage(LineCoverage) {} /// \brief Compute the code coverage summary for the given function coverage /// mapping record. - static FunctionCoverageSummary - get(const coverage::FunctionRecord &Function); + static FunctionCoverageSummary get(const coverage::CoverageMapping &CM, + const coverage::FunctionRecord &Function); - /// \brief Update the summary with information from another instantiation - /// of this function. - void update(const FunctionCoverageSummary &Summary); + /// Compute the code coverage summary for an instantiation group \p Group, + /// given a list of summaries for each instantiation in \p Summaries. + static FunctionCoverageSummary + get(const coverage::InstantiationGroup &Group, + ArrayRef<FunctionCoverageSummary> Summaries); }; /// \brief A summary of file's code coverage. @@ -147,7 +172,9 @@ struct FileCoverageSummary { FunctionCoverageInfo FunctionCoverage; FunctionCoverageInfo InstantiationCoverage; - FileCoverageSummary(StringRef Name) : Name(Name) {} + FileCoverageSummary(StringRef Name) + : Name(Name), RegionCoverage(), LineCoverage(), FunctionCoverage(), + InstantiationCoverage() {} void addFunction(const FunctionCoverageSummary &Function) { RegionCoverage += Function.RegionCoverage; diff --git a/tools/llvm-cov/CoverageViewOptions.h b/tools/llvm-cov/CoverageViewOptions.h index 266b380b7d39..17614c4e9ba2 100644 --- a/tools/llvm-cov/CoverageViewOptions.h +++ b/tools/llvm-cov/CoverageViewOptions.h @@ -27,10 +27,12 @@ struct CoverageViewOptions { bool ShowLineNumbers; bool ShowLineStats; bool ShowRegionMarkers; - bool ShowLineStatsOrRegionMarkers; bool ShowExpandedRegions; bool ShowFunctionInstantiations; bool ShowFullFilenames; + bool ShowRegionSummary; + bool ShowInstantiationSummary; + bool ExportSummaryOnly; OutputFormat Format; std::string ShowOutputDirectory; std::vector<std::string> DemanglerOpts; diff --git a/tools/llvm-cov/SourceCoverageView.cpp b/tools/llvm-cov/SourceCoverageView.cpp index 52b8ff1747fe..8c39dab580de 100644 --- a/tools/llvm-cov/SourceCoverageView.cpp +++ b/tools/llvm-cov/SourceCoverageView.cpp @@ -84,22 +84,15 @@ CoveragePrinter::create(const CoverageViewOptions &Opts) { } unsigned SourceCoverageView::getFirstUncoveredLineNo() { - auto CheckIfUncovered = [](const coverage::CoverageSegment &S) { + const auto MinSegIt = find_if(CoverageInfo, [](const CoverageSegment &S) { return S.HasCount && S.Count == 0; - }; - // L is less than R if (1) it's an uncovered segment (has a 0 count), and (2) - // either R is not an uncovered segment, or L has a lower line number than R. - const auto MinSegIt = - std::min_element(CoverageInfo.begin(), CoverageInfo.end(), - [CheckIfUncovered](const coverage::CoverageSegment &L, - const coverage::CoverageSegment &R) { - return (CheckIfUncovered(L) && - (!CheckIfUncovered(R) || (L.Line < R.Line))); - }); - if (CheckIfUncovered(*MinSegIt)) - return (*MinSegIt).Line; + }); + // There is no uncovered line, return zero. - return 0; + if (MinSegIt == CoverageInfo.end()) + return 0; + + return (*MinSegIt).Line; } std::string SourceCoverageView::formatCount(uint64_t N) { @@ -118,9 +111,20 @@ std::string SourceCoverageView::formatCount(uint64_t N) { } bool SourceCoverageView::shouldRenderRegionMarkers( - bool LineHasMultipleRegions) const { - return getOptions().ShowRegionMarkers && - (!getOptions().ShowLineStatsOrRegionMarkers || LineHasMultipleRegions); + const LineCoverageStats &LCS) const { + if (!getOptions().ShowRegionMarkers) + return false; + + CoverageSegmentArray Segments = LCS.getLineSegments(); + if (Segments.empty()) + return false; + for (unsigned I = 0, E = Segments.size() - 1; I < E; ++I) { + const auto *CurSeg = Segments[I]; + if (!CurSeg->IsRegionEntry || CurSeg->Count == LCS.getExecutionCount()) + continue; + return true; + } + return false; } bool SourceCoverageView::hasSubViews() const { @@ -130,7 +134,7 @@ bool SourceCoverageView::hasSubViews() const { std::unique_ptr<SourceCoverageView> SourceCoverageView::create(StringRef SourceName, const MemoryBuffer &File, const CoverageViewOptions &Options, - coverage::CoverageData &&CoverageInfo) { + CoverageData &&CoverageInfo) { switch (Options.Format) { case CoverageViewOptions::OutputFormat::Text: return llvm::make_unique<SourceCoverageViewText>( @@ -150,7 +154,7 @@ std::string SourceCoverageView::getSourceName() const { } void SourceCoverageView::addExpansion( - const coverage::CounterMappingRegion &Region, + const CounterMappingRegion &Region, std::unique_ptr<SourceCoverageView> View) { ExpansionSubViews.emplace_back(Region, std::move(View)); } @@ -162,8 +166,9 @@ void SourceCoverageView::addInstantiation( } void SourceCoverageView::print(raw_ostream &OS, bool WholeFile, - bool ShowSourceName, unsigned ViewDepth) { - if (WholeFile && getOptions().hasOutputDirectory()) + bool ShowSourceName, bool ShowTitle, + unsigned ViewDepth) { + if (ShowTitle) renderTitle(OS, "Coverage Report"); renderViewHeader(OS); @@ -176,50 +181,37 @@ void SourceCoverageView::print(raw_ostream &OS, bool WholeFile, // We need the expansions and instantiations sorted so we can go through them // while we iterate lines. - std::sort(ExpansionSubViews.begin(), ExpansionSubViews.end()); - std::sort(InstantiationSubViews.begin(), InstantiationSubViews.end()); + std::stable_sort(ExpansionSubViews.begin(), ExpansionSubViews.end()); + std::stable_sort(InstantiationSubViews.begin(), InstantiationSubViews.end()); auto NextESV = ExpansionSubViews.begin(); auto EndESV = ExpansionSubViews.end(); auto NextISV = InstantiationSubViews.begin(); auto EndISV = InstantiationSubViews.end(); // Get the coverage information for the file. - auto NextSegment = CoverageInfo.begin(); + auto StartSegment = CoverageInfo.begin(); auto EndSegment = CoverageInfo.end(); + LineCoverageIterator LCI{CoverageInfo, 1}; + LineCoverageIterator LCIEnd = LCI.getEnd(); - unsigned FirstLine = NextSegment != EndSegment ? NextSegment->Line : 0; - const coverage::CoverageSegment *WrappedSegment = nullptr; - SmallVector<const coverage::CoverageSegment *, 8> LineSegments; - for (line_iterator LI(File, /*SkipBlanks=*/false); !LI.is_at_eof(); ++LI) { + unsigned FirstLine = StartSegment != EndSegment ? StartSegment->Line : 0; + for (line_iterator LI(File, /*SkipBlanks=*/false); !LI.is_at_eof(); + ++LI, ++LCI) { // If we aren't rendering the whole file, we need to filter out the prologue // and epilogue. if (!WholeFile) { - if (NextSegment == EndSegment) + if (LCI == LCIEnd) break; else if (LI.line_number() < FirstLine) continue; } - // Collect the coverage information relevant to this line. - if (LineSegments.size()) - WrappedSegment = LineSegments.back(); - LineSegments.clear(); - while (NextSegment != EndSegment && NextSegment->Line == LI.line_number()) - LineSegments.push_back(&*NextSegment++); - - // Calculate a count to be for the line as a whole. - LineCoverageStats LineCount; - if (WrappedSegment && WrappedSegment->HasCount) - LineCount.addRegionCount(WrappedSegment->Count); - for (const auto *S : LineSegments) - if (S->HasCount && S->IsRegionEntry) - LineCount.addRegionStartCount(S->Count); - renderLinePrefix(OS, ViewDepth); if (getOptions().ShowLineNumbers) renderLineNumberColumn(OS, LI.line_number()); + if (getOptions().ShowLineStats) - renderLineCoverageColumn(OS, LineCount); + renderLineCoverageColumn(OS, *LCI); // If there are expansion subviews, we want to highlight the first one. unsigned ExpansionColumn = 0; @@ -228,12 +220,11 @@ void SourceCoverageView::print(raw_ostream &OS, bool WholeFile, ExpansionColumn = NextESV->getStartCol(); // Display the source code for the current line. - renderLine(OS, {*LI, LI.line_number()}, WrappedSegment, LineSegments, - ExpansionColumn, ViewDepth); + renderLine(OS, {*LI, LI.line_number()}, *LCI, ExpansionColumn, ViewDepth); // Show the region markers. - if (shouldRenderRegionMarkers(LineCount.hasMultipleRegions())) - renderRegionMarkers(OS, LineSegments, ViewDepth); + if (shouldRenderRegionMarkers(*LCI)) + renderRegionMarkers(OS, *LCI, ViewDepth); // Show the expansions and instantiations for this line. bool RenderedSubView = false; @@ -245,8 +236,8 @@ void SourceCoverageView::print(raw_ostream &OS, bool WholeFile, // this subview. if (RenderedSubView) { ExpansionColumn = NextESV->getStartCol(); - renderExpansionSite(OS, {*LI, LI.line_number()}, WrappedSegment, - LineSegments, ExpansionColumn, ViewDepth); + renderExpansionSite(OS, {*LI, LI.line_number()}, *LCI, ExpansionColumn, + ViewDepth); renderViewDivider(OS, ViewDepth + 1); } diff --git a/tools/llvm-cov/SourceCoverageView.h b/tools/llvm-cov/SourceCoverageView.h index 9cb608fed608..7f58ea5d7be8 100644 --- a/tools/llvm-cov/SourceCoverageView.h +++ b/tools/llvm-cov/SourceCoverageView.h @@ -15,20 +15,24 @@ #define LLVM_COV_SOURCECOVERAGEVIEW_H #include "CoverageViewOptions.h" +#include "CoverageSummaryInfo.h" #include "llvm/ProfileData/Coverage/CoverageMapping.h" #include "llvm/Support/MemoryBuffer.h" #include <vector> namespace llvm { +using namespace coverage; + +class CoverageFiltersMatchAll; class SourceCoverageView; /// \brief A view that represents a macro or include expansion. struct ExpansionView { - coverage::CounterMappingRegion Region; + CounterMappingRegion Region; std::unique_ptr<SourceCoverageView> View; - ExpansionView(const coverage::CounterMappingRegion &Region, + ExpansionView(const CounterMappingRegion &Region, std::unique_ptr<SourceCoverageView> View) : Region(Region), View(std::move(View)) {} ExpansionView(ExpansionView &&RHS) @@ -64,30 +68,6 @@ struct InstantiationView { } }; -/// \brief Coverage statistics for a single line. -struct LineCoverageStats { - uint64_t ExecutionCount; - unsigned RegionCount; - bool Mapped; - - LineCoverageStats() : ExecutionCount(0), RegionCount(0), Mapped(false) {} - - bool isMapped() const { return Mapped; } - - bool hasMultipleRegions() const { return RegionCount > 1; } - - void addRegionStartCount(uint64_t Count) { - // The max of all region starts is the most interesting value. - addRegionCount(RegionCount ? std::max(ExecutionCount, Count) : Count); - ++RegionCount; - } - - void addRegionCount(uint64_t Count) { - Mapped = true; - ExecutionCount = Count; - } -}; - /// \brief A file manager that handles format-aware file creation. class CoveragePrinter { public: @@ -134,7 +114,8 @@ public: /// \brief Create an index which lists reports for the given source files. virtual Error createIndexFile(ArrayRef<std::string> SourceFiles, - const coverage::CoverageMapping &Coverage) = 0; + const CoverageMapping &Coverage, + const CoverageFiltersMatchAll &Filters) = 0; /// @} }; @@ -155,7 +136,7 @@ class SourceCoverageView { const CoverageViewOptions &Options; /// Complete coverage information about the source on display. - coverage::CoverageData CoverageInfo; + CoverageData CoverageInfo; /// A container for all expansions (e.g macros) in the source on display. std::vector<ExpansionView> ExpansionSubViews; @@ -175,7 +156,7 @@ protected: LineRef(StringRef Line, int64_t LineNo) : Line(Line), LineNo(LineNo) {} }; - using CoverageSegmentArray = ArrayRef<const coverage::CoverageSegment *>; + using CoverageSegmentArray = ArrayRef<const CoverageSegment *>; /// @name Rendering Interface /// @{ @@ -200,8 +181,7 @@ protected: /// \brief Render a source line with highlighting. virtual void renderLine(raw_ostream &OS, LineRef L, - const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, + const LineCoverageStats &LCS, unsigned ExpansionCol, unsigned ViewDepth) = 0; /// \brief Render the line's execution count column. @@ -213,15 +193,14 @@ protected: /// \brief Render all the region's execution counts on a line. virtual void renderRegionMarkers(raw_ostream &OS, - CoverageSegmentArray Segments, + const LineCoverageStats &Line, unsigned ViewDepth) = 0; /// \brief Render the site of an expansion. - virtual void - renderExpansionSite(raw_ostream &OS, LineRef L, - const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, - unsigned ViewDepth) = 0; + virtual void renderExpansionSite(raw_ostream &OS, LineRef L, + const LineCoverageStats &LCS, + unsigned ExpansionCol, + unsigned ViewDepth) = 0; /// \brief Render an expansion view and any nested views. virtual void renderExpansionView(raw_ostream &OS, ExpansionView &ESV, @@ -246,22 +225,21 @@ protected: static std::string formatCount(uint64_t N); /// \brief Check if region marker output is expected for a line. - bool shouldRenderRegionMarkers(bool LineHasMultipleRegions) const; + bool shouldRenderRegionMarkers(const LineCoverageStats &LCS) const; /// \brief Check if there are any sub-views attached to this view. bool hasSubViews() const; SourceCoverageView(StringRef SourceName, const MemoryBuffer &File, const CoverageViewOptions &Options, - coverage::CoverageData &&CoverageInfo) + CoverageData &&CoverageInfo) : SourceName(SourceName), File(File), Options(Options), CoverageInfo(std::move(CoverageInfo)) {} public: static std::unique_ptr<SourceCoverageView> create(StringRef SourceName, const MemoryBuffer &File, - const CoverageViewOptions &Options, - coverage::CoverageData &&CoverageInfo); + const CoverageViewOptions &Options, CoverageData &&CoverageInfo); virtual ~SourceCoverageView() {} @@ -271,7 +249,7 @@ public: const CoverageViewOptions &getOptions() const { return Options; } /// \brief Add an expansion subview to this view. - void addExpansion(const coverage::CounterMappingRegion &Region, + void addExpansion(const CounterMappingRegion &Region, std::unique_ptr<SourceCoverageView> View); /// \brief Add a function instantiation subview to this view. @@ -281,7 +259,7 @@ public: /// \brief Print the code coverage information for a specific portion of a /// source file to the output stream. void print(raw_ostream &OS, bool WholeFile, bool ShowSourceName, - unsigned ViewDepth = 0); + bool ShowTitle, unsigned ViewDepth = 0); }; } // namespace llvm diff --git a/tools/llvm-cov/SourceCoverageViewHTML.cpp b/tools/llvm-cov/SourceCoverageViewHTML.cpp index 64b888e89d79..e45c6f4cb473 100644 --- a/tools/llvm-cov/SourceCoverageViewHTML.cpp +++ b/tools/llvm-cov/SourceCoverageViewHTML.cpp @@ -16,7 +16,6 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" -#include "llvm/Support/FileSystem.h" #include "llvm/Support/Format.h" #include "llvm/Support/Path.h" @@ -285,15 +284,31 @@ void CoveragePrinterHTML::closeViewFile(OwnedStream OS) { } /// Emit column labels for the table in the index. -static void emitColumnLabelsForIndex(raw_ostream &OS) { +static void emitColumnLabelsForIndex(raw_ostream &OS, + const CoverageViewOptions &Opts) { SmallVector<std::string, 4> Columns; Columns.emplace_back(tag("td", "Filename", "column-entry-left")); - for (const char *Label : {"Function Coverage", "Instantiation Coverage", - "Line Coverage", "Region Coverage"}) - Columns.emplace_back(tag("td", Label, "column-entry")); + Columns.emplace_back(tag("td", "Function Coverage", "column-entry")); + if (Opts.ShowInstantiationSummary) + Columns.emplace_back(tag("td", "Instantiation Coverage", "column-entry")); + Columns.emplace_back(tag("td", "Line Coverage", "column-entry")); + if (Opts.ShowRegionSummary) + Columns.emplace_back(tag("td", "Region Coverage", "column-entry")); OS << tag("tr", join(Columns.begin(), Columns.end(), "")); } +std::string +CoveragePrinterHTML::buildLinkToFile(StringRef SF, + const FileCoverageSummary &FCS) const { + SmallString<128> LinkTextStr(sys::path::relative_path(FCS.Name)); + sys::path::remove_dots(LinkTextStr, /*remove_dot_dots=*/true); + sys::path::native(LinkTextStr); + std::string LinkText = escape(LinkTextStr, Opts); + std::string LinkTarget = + escape(getOutputPath(SF, "html", /*InToplevel=*/false), Opts); + return a(LinkTarget, LinkText); +} + /// Render a file coverage summary (\p FCS) in a table row. If \p IsTotals is /// false, link the summary to \p SF. void CoveragePrinterHTML::emitFileSummary(raw_ostream &OS, StringRef SF, @@ -326,34 +341,31 @@ void CoveragePrinterHTML::emitFileSummary(raw_ostream &OS, StringRef SF, if (IsTotals) { Filename = "TOTALS"; } else { - SmallString<128> LinkTextStr(sys::path::relative_path(FCS.Name)); - sys::path::remove_dots(LinkTextStr, /*remove_dot_dots=*/true); - sys::path::native(LinkTextStr); - std::string LinkText = escape(LinkTextStr, Opts); - std::string LinkTarget = - escape(getOutputPath(SF, "html", /*InToplevel=*/false), Opts); - Filename = a(LinkTarget, LinkText); + Filename = buildLinkToFile(SF, FCS); } Columns.emplace_back(tag("td", tag("pre", Filename))); - AddCoverageTripleToColumn(FCS.FunctionCoverage.Executed, - FCS.FunctionCoverage.NumFunctions, + AddCoverageTripleToColumn(FCS.FunctionCoverage.getExecuted(), + FCS.FunctionCoverage.getNumFunctions(), FCS.FunctionCoverage.getPercentCovered()); - AddCoverageTripleToColumn(FCS.InstantiationCoverage.Executed, - FCS.InstantiationCoverage.NumFunctions, - FCS.InstantiationCoverage.getPercentCovered()); - AddCoverageTripleToColumn(FCS.LineCoverage.Covered, FCS.LineCoverage.NumLines, + if (Opts.ShowInstantiationSummary) + AddCoverageTripleToColumn(FCS.InstantiationCoverage.getExecuted(), + FCS.InstantiationCoverage.getNumFunctions(), + FCS.InstantiationCoverage.getPercentCovered()); + AddCoverageTripleToColumn(FCS.LineCoverage.getCovered(), + FCS.LineCoverage.getNumLines(), FCS.LineCoverage.getPercentCovered()); - AddCoverageTripleToColumn(FCS.RegionCoverage.Covered, - FCS.RegionCoverage.NumRegions, - FCS.RegionCoverage.getPercentCovered()); + if (Opts.ShowRegionSummary) + AddCoverageTripleToColumn(FCS.RegionCoverage.getCovered(), + FCS.RegionCoverage.getNumRegions(), + FCS.RegionCoverage.getPercentCovered()); OS << tag("tr", join(Columns.begin(), Columns.end(), ""), "light-row"); } Error CoveragePrinterHTML::createIndexFile( - ArrayRef<std::string> SourceFiles, - const coverage::CoverageMapping &Coverage) { + ArrayRef<std::string> SourceFiles, const CoverageMapping &Coverage, + const CoverageFiltersMatchAll &Filters) { // Emit the default stylesheet. auto CSSOrErr = createOutputStream("style", "css", /*InToplevel=*/true); if (Error E = CSSOrErr.takeError()) @@ -387,16 +399,39 @@ Error CoveragePrinterHTML::createIndexFile( " for information about interpreting this report."); // Emit a table containing links to reports for each file in the covmapping. + // Exclude files which don't contain any regions. OSRef << BeginCenteredDiv << BeginTable; - emitColumnLabelsForIndex(OSRef); + emitColumnLabelsForIndex(OSRef, Opts); FileCoverageSummary Totals("TOTALS"); - auto FileReports = - CoverageReport::prepareFileReports(Coverage, Totals, SourceFiles); - for (unsigned I = 0, E = FileReports.size(); I < E; ++I) - emitFileSummary(OSRef, SourceFiles[I], FileReports[I]); + auto FileReports = CoverageReport::prepareFileReports( + Coverage, Totals, SourceFiles, Opts, Filters); + bool EmptyFiles = false; + for (unsigned I = 0, E = FileReports.size(); I < E; ++I) { + if (FileReports[I].FunctionCoverage.getNumFunctions()) + emitFileSummary(OSRef, SourceFiles[I], FileReports[I]); + else + EmptyFiles = true; + } emitFileSummary(OSRef, "Totals", Totals, /*IsTotals=*/true); - OSRef << EndTable << EndCenteredDiv - << tag("h5", escape(Opts.getLLVMVersionString(), Opts)); + OSRef << EndTable << EndCenteredDiv; + + // Emit links to files which don't contain any functions. These are normally + // not very useful, but could be relevant for code which abuses the + // preprocessor. + if (EmptyFiles && Filters.empty()) { + OSRef << tag("p", "Files which contain no functions. (These " + "files contain code pulled into other files " + "by the preprocessor.)\n"); + OSRef << BeginCenteredDiv << BeginTable; + for (unsigned I = 0, E = FileReports.size(); I < E; ++I) + if (!FileReports[I].FunctionCoverage.getNumFunctions()) { + std::string Link = buildLinkToFile(SourceFiles[I], FileReports[I]); + OSRef << tag("tr", tag("td", tag("pre", Link)), "light-row") << '\n'; + } + OSRef << EndTable << EndCenteredDiv; + } + + OSRef << tag("h5", escape(Opts.getLLVMVersionString(), Opts)); emitEpilog(OSRef); return Error::success(); @@ -431,9 +466,9 @@ void SourceCoverageViewHTML::renderViewDivider(raw_ostream &, unsigned) { // The table-based output makes view dividers unnecessary. } -void SourceCoverageViewHTML::renderLine( - raw_ostream &OS, LineRef L, const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, unsigned) { +void SourceCoverageViewHTML::renderLine(raw_ostream &OS, LineRef L, + const LineCoverageStats &LCS, + unsigned ExpansionCol, unsigned) { StringRef Line = L.Line; unsigned LineNo = L.LineNo; @@ -445,6 +480,7 @@ void SourceCoverageViewHTML::renderLine( // at the end of the line. Both are required but may be empty. SmallVector<std::string, 8> Snippets; + CoverageSegmentArray Segments = LCS.getLineSegments(); unsigned LCol = 1; auto Snip = [&](unsigned Start, unsigned Len) { @@ -469,7 +505,7 @@ void SourceCoverageViewHTML::renderLine( // 1 to set the highlight for snippet 2, segment 2 to set the highlight for // snippet 3, and so on. - Optional<std::string> Color; + Optional<StringRef> Color; SmallVector<std::pair<unsigned, unsigned>, 2> HighlightedRanges; auto Highlight = [&](const std::string &Snippet, unsigned LC, unsigned RC) { if (getOptions().Debug) @@ -477,11 +513,12 @@ void SourceCoverageViewHTML::renderLine( return tag("span", Snippet, Color.getValue()); }; - auto CheckIfUncovered = [](const coverage::CoverageSegment *S) { - return S && S->HasCount && S->Count == 0; + auto CheckIfUncovered = [&](const CoverageSegment *S) { + return S && (!S->IsGapRegion || (Color && *Color == "red")) && + S->HasCount && S->Count == 0; }; - if (CheckIfUncovered(WrappedSegment)) { + if (CheckIfUncovered(LCS.getWrappedSegment())) { Color = "red"; if (!Snippets[0].empty()) Snippets[0] = Highlight(Snippets[0], 1, 1 + Snippets[0].size()); @@ -489,10 +526,10 @@ void SourceCoverageViewHTML::renderLine( for (unsigned I = 0, E = Segments.size(); I < E; ++I) { const auto *CurSeg = Segments[I]; - if (CurSeg->Col == ExpansionCol) - Color = "cyan"; - else if (CheckIfUncovered(CurSeg)) + if (CheckIfUncovered(CurSeg)) Color = "red"; + else if (CurSeg->Col == ExpansionCol) + Color = "cyan"; else Color = None; @@ -518,25 +555,23 @@ void SourceCoverageViewHTML::renderLine( // 4. Snippets[1:N+1] correspond to \p Segments[0:N]: use these to generate // sub-line region count tooltips if needed. - bool HasMultipleRegions = [&] { - unsigned RegionCount = 0; - for (const auto *S : Segments) - if (S->HasCount && S->IsRegionEntry) - if (++RegionCount > 1) - return true; - return false; - }(); - - if (shouldRenderRegionMarkers(HasMultipleRegions)) { - for (unsigned I = 0, E = Segments.size(); I < E; ++I) { + if (shouldRenderRegionMarkers(LCS)) { + // Just consider the segments which start *and* end on this line. + for (unsigned I = 0, E = Segments.size() - 1; I < E; ++I) { const auto *CurSeg = Segments[I]; - if (!CurSeg->IsRegionEntry || !CurSeg->HasCount) + if (!CurSeg->IsRegionEntry) + continue; + if (CurSeg->Count == LCS.getExecutionCount()) continue; Snippets[I + 1] = tag("div", Snippets[I + 1] + tag("span", formatCount(CurSeg->Count), "tooltip-content"), "tooltip"); + + if (getOptions().Debug) + errs() << "Marker at " << CurSeg->Line << ":" << CurSeg->Col << " = " + << formatCount(CurSeg->Count) << "\n"; } } @@ -556,9 +591,9 @@ void SourceCoverageViewHTML::renderLineCoverageColumn( raw_ostream &OS, const LineCoverageStats &Line) { std::string Count = ""; if (Line.isMapped()) - Count = tag("pre", formatCount(Line.ExecutionCount)); + Count = tag("pre", formatCount(Line.getExecutionCount())); std::string CoverageClass = - (Line.ExecutionCount > 0) ? "covered-line" : "uncovered-line"; + (Line.getExecutionCount() > 0) ? "covered-line" : "uncovered-line"; OS << tag("td", Count, CoverageClass); } @@ -571,16 +606,17 @@ void SourceCoverageViewHTML::renderLineNumberColumn(raw_ostream &OS, } void SourceCoverageViewHTML::renderRegionMarkers(raw_ostream &, - CoverageSegmentArray, + const LineCoverageStats &Line, unsigned) { // Region markers are rendered in-line using tooltips. } -void SourceCoverageViewHTML::renderExpansionSite( - raw_ostream &OS, LineRef L, const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, unsigned ViewDepth) { +void SourceCoverageViewHTML::renderExpansionSite(raw_ostream &OS, LineRef L, + const LineCoverageStats &LCS, + unsigned ExpansionCol, + unsigned ViewDepth) { // Render the line containing the expansion site. No extra formatting needed. - renderLine(OS, L, WrappedSegment, Segments, ExpansionCol, ViewDepth); + renderLine(OS, L, LCS, ExpansionCol, ViewDepth); } void SourceCoverageViewHTML::renderExpansionView(raw_ostream &OS, @@ -588,7 +624,7 @@ void SourceCoverageViewHTML::renderExpansionView(raw_ostream &OS, unsigned ViewDepth) { OS << BeginExpansionDiv; ESV.View->print(OS, /*WholeFile=*/false, /*ShowSourceName=*/false, - ViewDepth + 1); + /*ShowTitle=*/false, ViewDepth + 1); OS << EndExpansionDiv; } @@ -604,7 +640,7 @@ void SourceCoverageViewHTML::renderInstantiationView(raw_ostream &OS, << EndSourceNameDiv; else ISV.View->print(OS, /*WholeFile=*/false, /*ShowSourceName=*/true, - ViewDepth); + /*ShowTitle=*/false, ViewDepth); OS << EndExpansionDiv; } diff --git a/tools/llvm-cov/SourceCoverageViewHTML.h b/tools/llvm-cov/SourceCoverageViewHTML.h index 94b08a5e7fc4..91b4ad4e220c 100644 --- a/tools/llvm-cov/SourceCoverageViewHTML.h +++ b/tools/llvm-cov/SourceCoverageViewHTML.h @@ -18,6 +18,8 @@ namespace llvm { +using namespace coverage; + struct FileCoverageSummary; /// \brief A coverage printer for html output. @@ -29,7 +31,8 @@ public: void closeViewFile(OwnedStream OS) override; Error createIndexFile(ArrayRef<std::string> SourceFiles, - const coverage::CoverageMapping &Coverage) override; + const coverage::CoverageMapping &Coverage, + const CoverageFiltersMatchAll &Filters) override; CoveragePrinterHTML(const CoverageViewOptions &Opts) : CoveragePrinter(Opts) {} @@ -38,6 +41,8 @@ private: void emitFileSummary(raw_ostream &OS, StringRef SF, const FileCoverageSummary &FCS, bool IsTotals = false) const; + std::string buildLinkToFile(StringRef SF, + const FileCoverageSummary &FCS) const; }; /// \brief A code coverage view which supports html-based rendering. @@ -54,14 +59,11 @@ class SourceCoverageViewHTML : public SourceCoverageView { void renderViewDivider(raw_ostream &OS, unsigned ViewDepth) override; - void renderLine(raw_ostream &OS, LineRef L, - const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, - unsigned ViewDepth) override; + void renderLine(raw_ostream &OS, LineRef L, const LineCoverageStats &LCS, + unsigned ExpansionCol, unsigned ViewDepth) override; void renderExpansionSite(raw_ostream &OS, LineRef L, - const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, + const LineCoverageStats &LCS, unsigned ExpansionCol, unsigned ViewDepth) override; void renderExpansionView(raw_ostream &OS, ExpansionView &ESV, @@ -75,7 +77,7 @@ class SourceCoverageViewHTML : public SourceCoverageView { void renderLineNumberColumn(raw_ostream &OS, unsigned LineNo) override; - void renderRegionMarkers(raw_ostream &OS, CoverageSegmentArray Segments, + void renderRegionMarkers(raw_ostream &OS, const LineCoverageStats &Line, unsigned ViewDepth) override; void renderTitle(raw_ostream &OS, StringRef Title) override; diff --git a/tools/llvm-cov/SourceCoverageViewText.cpp b/tools/llvm-cov/SourceCoverageViewText.cpp index 4ad120f642eb..2480ee9f416a 100644 --- a/tools/llvm-cov/SourceCoverageViewText.cpp +++ b/tools/llvm-cov/SourceCoverageViewText.cpp @@ -29,8 +29,8 @@ void CoveragePrinterText::closeViewFile(OwnedStream OS) { } Error CoveragePrinterText::createIndexFile( - ArrayRef<std::string> SourceFiles, - const coverage::CoverageMapping &Coverage) { + ArrayRef<std::string> SourceFiles, const CoverageMapping &Coverage, + const CoverageFiltersMatchAll &Filters) { auto OSOrErr = createOutputStream("index", "txt", /*InToplevel=*/true); if (Error E = OSOrErr.takeError()) return E; @@ -38,7 +38,7 @@ Error CoveragePrinterText::createIndexFile( raw_ostream &OSRef = *OS.get(); CoverageReport Report(Opts, Coverage); - Report.renderFileReports(OSRef, SourceFiles); + Report.renderFileReports(OSRef, SourceFiles, Filters); Opts.colored_ostream(OSRef, raw_ostream::CYAN) << "\n" << Opts.getLLVMVersionString(); @@ -93,18 +93,21 @@ void SourceCoverageViewText::renderViewDivider(raw_ostream &OS, OS << '\n'; } -void SourceCoverageViewText::renderLine( - raw_ostream &OS, LineRef L, - const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, unsigned ViewDepth) { +void SourceCoverageViewText::renderLine(raw_ostream &OS, LineRef L, + const LineCoverageStats &LCS, + unsigned ExpansionCol, + unsigned ViewDepth) { StringRef Line = L.Line; unsigned LineNumber = L.LineNo; + auto *WrappedSegment = LCS.getWrappedSegment(); + CoverageSegmentArray Segments = LCS.getLineSegments(); Optional<raw_ostream::Colors> Highlight; SmallVector<std::pair<unsigned, unsigned>, 2> HighlightedRanges; // The first segment overlaps from a previous line, so we treat it specially. - if (WrappedSegment && WrappedSegment->HasCount && WrappedSegment->Count == 0) + if (WrappedSegment && !WrappedSegment->IsGapRegion && + WrappedSegment->HasCount && WrappedSegment->Count == 0) Highlight = raw_ostream::RED; // Output each segment of the line, possibly highlighted. @@ -118,10 +121,11 @@ void SourceCoverageViewText::renderLine( if (getOptions().Debug && Highlight) HighlightedRanges.push_back(std::make_pair(Col, End)); Col = End; - if (Col == ExpansionCol) - Highlight = raw_ostream::CYAN; - else if (S->HasCount && S->Count == 0) + if ((!S->IsGapRegion || (Highlight && *Highlight == raw_ostream::RED)) && + S->HasCount && S->Count == 0) Highlight = raw_ostream::RED; + else if (Col == ExpansionCol) + Highlight = raw_ostream::CYAN; else Highlight = None; } @@ -147,7 +151,7 @@ void SourceCoverageViewText::renderLineCoverageColumn( OS.indent(LineCoverageColumnWidth) << '|'; return; } - std::string C = formatCount(Line.ExecutionCount); + std::string C = formatCount(Line.getExecutionCount()); OS.indent(LineCoverageColumnWidth - C.size()); colored_ostream(OS, raw_ostream::MAGENTA, Line.hasMultipleRegions() && getOptions().Colors) @@ -166,15 +170,24 @@ void SourceCoverageViewText::renderLineNumberColumn(raw_ostream &OS, OS.indent(LineNumberColumnWidth - Str.size()) << Str << '|'; } -void SourceCoverageViewText::renderRegionMarkers( - raw_ostream &OS, CoverageSegmentArray Segments, unsigned ViewDepth) { +void SourceCoverageViewText::renderRegionMarkers(raw_ostream &OS, + const LineCoverageStats &Line, + unsigned ViewDepth) { renderLinePrefix(OS, ViewDepth); OS.indent(getCombinedColumnWidth(getOptions())); + CoverageSegmentArray Segments = Line.getLineSegments(); + + // Just consider the segments which start *and* end on this line. + if (Segments.size() > 1) + Segments = Segments.drop_back(); + unsigned PrevColumn = 1; for (const auto *S : Segments) { if (!S->IsRegionEntry) continue; + if (S->Count == Line.getExecutionCount()) + continue; // Skip to the new region. if (S->Col > PrevColumn) OS.indent(S->Col - PrevColumn); @@ -182,21 +195,21 @@ void SourceCoverageViewText::renderRegionMarkers( std::string C = formatCount(S->Count); PrevColumn += C.size(); OS << '^' << C; - } - OS << '\n'; - if (getOptions().Debug) - for (const auto *S : Segments) + if (getOptions().Debug) errs() << "Marker at " << S->Line << ":" << S->Col << " = " - << formatCount(S->Count) << (S->IsRegionEntry ? "\n" : " (pop)\n"); + << formatCount(S->Count) << "\n"; + } + OS << '\n'; } -void SourceCoverageViewText::renderExpansionSite( - raw_ostream &OS, LineRef L, const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, unsigned ViewDepth) { +void SourceCoverageViewText::renderExpansionSite(raw_ostream &OS, LineRef L, + const LineCoverageStats &LCS, + unsigned ExpansionCol, + unsigned ViewDepth) { renderLinePrefix(OS, ViewDepth); OS.indent(getCombinedColumnWidth(getOptions()) + (ViewDepth == 0 ? 0 : 1)); - renderLine(OS, L, WrappedSegment, Segments, ExpansionCol, ViewDepth); + renderLine(OS, L, LCS, ExpansionCol, ViewDepth); } void SourceCoverageViewText::renderExpansionView(raw_ostream &OS, @@ -207,7 +220,7 @@ void SourceCoverageViewText::renderExpansionView(raw_ostream &OS, errs() << "Expansion at line " << ESV.getLine() << ", " << ESV.getStartCol() << " -> " << ESV.getEndCol() << '\n'; ESV.View->print(OS, /*WholeFile=*/false, /*ShowSourceName=*/false, - ViewDepth + 1); + /*ShowTitle=*/false, ViewDepth + 1); } void SourceCoverageViewText::renderInstantiationView(raw_ostream &OS, @@ -220,7 +233,7 @@ void SourceCoverageViewText::renderInstantiationView(raw_ostream &OS, << "Unexecuted instantiation: " << ISV.FunctionName << "\n"; else ISV.View->print(OS, /*WholeFile=*/false, /*ShowSourceName=*/true, - ViewDepth); + /*ShowTitle=*/false, ViewDepth); } void SourceCoverageViewText::renderTitle(raw_ostream &OS, StringRef Title) { diff --git a/tools/llvm-cov/SourceCoverageViewText.h b/tools/llvm-cov/SourceCoverageViewText.h index c3f20de92978..cabf91975df3 100644 --- a/tools/llvm-cov/SourceCoverageViewText.h +++ b/tools/llvm-cov/SourceCoverageViewText.h @@ -18,6 +18,8 @@ namespace llvm { +using namespace coverage; + /// \brief A coverage printer for text output. class CoveragePrinterText : public CoveragePrinter { public: @@ -27,7 +29,8 @@ public: void closeViewFile(OwnedStream OS) override; Error createIndexFile(ArrayRef<std::string> SourceFiles, - const coverage::CoverageMapping &Coverage) override; + const CoverageMapping &Coverage, + const CoverageFiltersMatchAll &Filters) override; CoveragePrinterText(const CoverageViewOptions &Opts) : CoveragePrinter(Opts) {} @@ -47,14 +50,11 @@ class SourceCoverageViewText : public SourceCoverageView { void renderViewDivider(raw_ostream &OS, unsigned ViewDepth) override; - void renderLine(raw_ostream &OS, LineRef L, - const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, - unsigned ViewDepth) override; + void renderLine(raw_ostream &OS, LineRef L, const LineCoverageStats &LCS, + unsigned ExpansionCol, unsigned ViewDepth) override; void renderExpansionSite(raw_ostream &OS, LineRef L, - const coverage::CoverageSegment *WrappedSegment, - CoverageSegmentArray Segments, unsigned ExpansionCol, + const LineCoverageStats &LCS, unsigned ExpansionCol, unsigned ViewDepth) override; void renderExpansionView(raw_ostream &OS, ExpansionView &ESV, @@ -68,7 +68,7 @@ class SourceCoverageViewText : public SourceCoverageView { void renderLineNumberColumn(raw_ostream &OS, unsigned LineNo) override; - void renderRegionMarkers(raw_ostream &OS, CoverageSegmentArray Segments, + void renderRegionMarkers(raw_ostream &OS, const LineCoverageStats &Line, unsigned ViewDepth) override; void renderTitle(raw_ostream &OS, StringRef Title) override; @@ -79,7 +79,7 @@ class SourceCoverageViewText : public SourceCoverageView { public: SourceCoverageViewText(StringRef SourceName, const MemoryBuffer &File, const CoverageViewOptions &Options, - coverage::CoverageData &&CoverageInfo) + CoverageData &&CoverageInfo) : SourceCoverageView(SourceName, File, Options, std::move(CoverageInfo)) { } }; diff --git a/tools/llvm-cov/gcov.cpp b/tools/llvm-cov/gcov.cpp index 4df7f015fd18..7776f2aa9a68 100644 --- a/tools/llvm-cov/gcov.cpp +++ b/tools/llvm-cov/gcov.cpp @@ -11,11 +11,11 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ProfileData/GCOV.h" #include "llvm/ADT/SmallString.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Errc.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/GCOV.h" #include "llvm/Support/Path.h" #include <system_error> using namespace llvm; diff --git a/tools/llvm-cvtres/llvm-cvtres.cpp b/tools/llvm-cvtres/llvm-cvtres.cpp index 36c15925e84f..2f33afdb0be9 100644 --- a/tools/llvm-cvtres/llvm-cvtres.cpp +++ b/tools/llvm-cvtres/llvm-cvtres.cpp @@ -126,6 +126,7 @@ int main(int argc_, const char *argv_[]) { std::string MachineString = InputArgs.getLastArgValue(OPT_MACHINE).upper(); MachineType = StringSwitch<COFF::MachineTypes>(MachineString) .Case("ARM", COFF::IMAGE_FILE_MACHINE_ARMNT) + .Case("ARM64", COFF::IMAGE_FILE_MACHINE_ARM64) .Case("X64", COFF::IMAGE_FILE_MACHINE_AMD64) .Case("X86", COFF::IMAGE_FILE_MACHINE_I386) .Default(COFF::IMAGE_FILE_MACHINE_UNKNOWN); @@ -155,6 +156,9 @@ int main(int argc_, const char *argv_[]) { if (Verbose) { outs() << "Machine: "; switch (MachineType) { + case COFF::IMAGE_FILE_MACHINE_ARM64: + outs() << "ARM64\n"; + break; case COFF::IMAGE_FILE_MACHINE_ARMNT: outs() << "ARM\n"; break; @@ -202,7 +206,7 @@ int main(int argc_, const char *argv_[]) { auto FileOrErr = FileOutputBuffer::create(OutputFile, OutputBuffer->getBufferSize()); if (!FileOrErr) - reportError(OutputFile, FileOrErr.getError()); + reportError(OutputFile, errorToErrorCode(FileOrErr.takeError())); std::unique_ptr<FileOutputBuffer> FileBuffer = std::move(*FileOrErr); std::copy(OutputBuffer->getBufferStart(), OutputBuffer->getBufferEnd(), FileBuffer->getBufferStart()); diff --git a/tools/llvm-cxxdump/llvm-cxxdump.cpp b/tools/llvm-cxxdump/llvm-cxxdump.cpp index b10759ad05c0..9b687e4fbe22 100644 --- a/tools/llvm-cxxdump/llvm-cxxdump.cpp +++ b/tools/llvm-cxxdump/llvm-cxxdump.cpp @@ -549,8 +549,7 @@ int main(int argc, const char *argv[]) { if (opts::InputFilenames.size() == 0) opts::InputFilenames.push_back("-"); - std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(), - dumpInput); + llvm::for_each(opts::InputFilenames, dumpInput); return EXIT_SUCCESS; } diff --git a/tools/llvm-cxxfilt/CMakeLists.txt b/tools/llvm-cxxfilt/CMakeLists.txt index 488064d08dab..2a78acad80a8 100644 --- a/tools/llvm-cxxfilt/CMakeLists.txt +++ b/tools/llvm-cxxfilt/CMakeLists.txt @@ -6,3 +6,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_tool(llvm-cxxfilt llvm-cxxfilt.cpp ) + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(c++filt llvm-cxxfilt) +endif() diff --git a/tools/llvm-cxxfilt/llvm-cxxfilt.cpp b/tools/llvm-cxxfilt/llvm-cxxfilt.cpp index 13024fbeaeaa..9c6a1612fa08 100644 --- a/tools/llvm-cxxfilt/llvm-cxxfilt.cpp +++ b/tools/llvm-cxxfilt/llvm-cxxfilt.cpp @@ -75,6 +75,7 @@ static void demangle(llvm::raw_ostream &OS, const std::string &Mangled) { } OS << (Undecorated ? Undecorated : Mangled) << '\n'; + OS.flush(); free(Undecorated); } diff --git a/tools/llvm-demangle-fuzzer/CMakeLists.txt b/tools/llvm-demangle-fuzzer/CMakeLists.txt new file mode 100644 index 000000000000..34a04b43fafd --- /dev/null +++ b/tools/llvm-demangle-fuzzer/CMakeLists.txt @@ -0,0 +1,9 @@ +set(LLVM_LINK_COMPONENTS + Demangle + FuzzMutate + Support +) + +add_llvm_fuzzer(llvm-demangle-fuzzer + llvm-demangle-fuzzer.cpp + DUMMY_MAIN DummyDemanglerFuzzer.cpp) diff --git a/tools/llvm-demangle-fuzzer/DummyDemanglerFuzzer.cpp b/tools/llvm-demangle-fuzzer/DummyDemanglerFuzzer.cpp new file mode 100644 index 000000000000..a2bf9f1b807e --- /dev/null +++ b/tools/llvm-demangle-fuzzer/DummyDemanglerFuzzer.cpp @@ -0,0 +1,19 @@ +//===--- DummyDemanglerMain.cpp - Entry point to sanity check the fuzzer --===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementation of main so we can build and test without linking libFuzzer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/FuzzMutate/FuzzerCLI.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); +int main(int argc, char *argv[]) { + return llvm::runFuzzerOnInputs(argc, argv, LLVMFuzzerTestOneInput); +} diff --git a/tools/llvm-demangle-fuzzer/llvm-demangle-fuzzer.cpp b/tools/llvm-demangle-fuzzer/llvm-demangle-fuzzer.cpp new file mode 100644 index 000000000000..07c290a0be5c --- /dev/null +++ b/tools/llvm-demangle-fuzzer/llvm-demangle-fuzzer.cpp @@ -0,0 +1,24 @@ +//===--- llvm-demangle-fuzzer.cpp - Fuzzer for the Itanium Demangler ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Demangle/Demangle.h" + +#include <cstdint> +#include <cstdlib> +#include <string> + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + std::string NullTerminatedString((const char *)Data, Size); + int status = 0; + if (char *demangle = llvm::itaniumDemangle(NullTerminatedString.c_str(), nullptr, + nullptr, &status)) + free(demangle); + + return 0; +} diff --git a/tools/llvm-diff/DiffConsumer.cpp b/tools/llvm-diff/DiffConsumer.cpp index e16775010fef..ec189df27521 100644 --- a/tools/llvm-diff/DiffConsumer.cpp +++ b/tools/llvm-diff/DiffConsumer.cpp @@ -13,7 +13,6 @@ #include "DiffConsumer.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Module.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Debug.h" diff --git a/tools/llvm-diff/DiffLog.cpp b/tools/llvm-diff/DiffLog.cpp index 898749e73bd2..50c0c4cff2fc 100644 --- a/tools/llvm-diff/DiffLog.cpp +++ b/tools/llvm-diff/DiffLog.cpp @@ -14,7 +14,6 @@ #include "DiffLog.h" #include "DiffConsumer.h" #include "llvm/ADT/StringRef.h" -#include "llvm/IR/Instructions.h" using namespace llvm; diff --git a/tools/llvm-dis/llvm-dis.cpp b/tools/llvm-dis/llvm-dis.cpp index 0c1d723a7b10..c91aa1c71a15 100644 --- a/tools/llvm-dis/llvm-dis.cpp +++ b/tools/llvm-dis/llvm-dis.cpp @@ -51,8 +51,13 @@ static cl::opt<bool> DontPrint("disable-output", cl::desc("Don't output the .ll file"), cl::Hidden); static cl::opt<bool> -ShowAnnotations("show-annotations", - cl::desc("Add informational comments to the .ll file")); + SetImporting("set-importing", + cl::desc("Set lazy loading to pretend to import a module"), + cl::Hidden); + +static cl::opt<bool> + ShowAnnotations("show-annotations", + cl::desc("Add informational comments to the .ll file")); static cl::opt<bool> PreserveAssemblyUseListOrder( "preserve-ll-uselistorder", @@ -117,34 +122,38 @@ public: } }; -} // end anon namespace - -static void diagnosticHandler(const DiagnosticInfo &DI, void *Context) { - raw_ostream &OS = errs(); - OS << (char *)Context << ": "; - switch (DI.getSeverity()) { - case DS_Error: OS << "error: "; break; - case DS_Warning: OS << "warning: "; break; - case DS_Remark: OS << "remark: "; break; - case DS_Note: OS << "note: "; break; - } +struct LLVMDisDiagnosticHandler : public DiagnosticHandler { + char *Prefix; + LLVMDisDiagnosticHandler(char *PrefixPtr) : Prefix(PrefixPtr) {} + bool handleDiagnostics(const DiagnosticInfo &DI) override { + raw_ostream &OS = errs(); + OS << Prefix << ": "; + switch (DI.getSeverity()) { + case DS_Error: OS << "error: "; break; + case DS_Warning: OS << "warning: "; break; + case DS_Remark: OS << "remark: "; break; + case DS_Note: OS << "note: "; break; + } - DiagnosticPrinterRawOStream DP(OS); - DI.print(DP); - OS << '\n'; + DiagnosticPrinterRawOStream DP(OS); + DI.print(DP); + OS << '\n'; - if (DI.getSeverity() == DS_Error) - exit(1); -} + if (DI.getSeverity() == DS_Error) + exit(1); + return true; + } +}; +} // end anon namespace static ExitOnError ExitOnErr; static std::unique_ptr<Module> openInputFile(LLVMContext &Context) { std::unique_ptr<MemoryBuffer> MB = ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(InputFilename))); - std::unique_ptr<Module> M = - ExitOnErr(getOwningLazyBitcodeModule(std::move(MB), Context, - /*ShouldLazyLoadMetadata=*/true)); + std::unique_ptr<Module> M = ExitOnErr(getOwningLazyBitcodeModule( + std::move(MB), Context, + /*ShouldLazyLoadMetadata=*/true, SetImporting)); if (MaterializeMetadata) ExitOnErr(M->materializeMetadata()); else @@ -161,9 +170,8 @@ int main(int argc, char **argv) { LLVMContext Context; llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. - - Context.setDiagnosticHandler(diagnosticHandler, argv[0]); - + Context.setDiagnosticHandler( + llvm::make_unique<LLVMDisDiagnosticHandler>(argv[0])); cl::ParseCommandLineOptions(argc, argv, "llvm .bc -> .ll disassembler\n"); std::unique_ptr<Module> M = openInputFile(Context); @@ -183,8 +191,8 @@ int main(int argc, char **argv) { } std::error_code EC; - std::unique_ptr<tool_output_file> Out( - new tool_output_file(OutputFilename, EC, sys::fs::F_None)); + std::unique_ptr<ToolOutputFile> Out( + new ToolOutputFile(OutputFilename, EC, sys::fs::F_None)); if (EC) { errs() << EC.message() << '\n'; return 1; diff --git a/tools/llvm-dwarfdump/CMakeLists.txt b/tools/llvm-dwarfdump/CMakeLists.txt index 9a2e53f5a4bb..77620e0faaf8 100644 --- a/tools/llvm-dwarfdump/CMakeLists.txt +++ b/tools/llvm-dwarfdump/CMakeLists.txt @@ -1,13 +1,15 @@ set(LLVM_LINK_COMPONENTS DebugInfoDWARF + AllTargetsDescs + AllTargetsInfos + MC Object Support ) add_llvm_tool(llvm-dwarfdump + Statistics.cpp llvm-dwarfdump.cpp ) -if(LLVM_USE_SANITIZE_COVERAGE) - add_subdirectory(fuzzer) -endif() +add_subdirectory(fuzzer) diff --git a/tools/llvm-dwarfdump/Statistics.cpp b/tools/llvm-dwarfdump/Statistics.cpp new file mode 100644 index 000000000000..9a7454a52624 --- /dev/null +++ b/tools/llvm-dwarfdump/Statistics.cpp @@ -0,0 +1,239 @@ +#include "llvm/ADT/DenseMap.h" +#include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" +#include "llvm/Object/ObjectFile.h" + +#define DEBUG_TYPE "dwarfdump" +using namespace llvm; +using namespace object; + +/// Holds statistics for one function (or other entity that has a PC range and +/// contains variables, such as a compile unit). +struct PerFunctionStats { + /// Number of inlined instances of this function. + unsigned NumFnInlined = 0; + /// Number of variables with location across all inlined instances. + unsigned TotalVarWithLoc = 0; + /// Number of constants with location across all inlined instances. + unsigned ConstantMembers = 0; + /// List of all Variables in this function. + SmallDenseSet<uint32_t, 4> VarsInFunction; + /// Compile units also cover a PC range, but have this flag set to false. + bool IsFunction = false; +}; + +/// Holds accumulated global statistics about local variables. +struct GlobalStats { + /// Total number of PC range bytes covered by DW_AT_locations. + unsigned ScopeBytesCovered = 0; + /// Total number of PC range bytes in each variable's enclosing scope, + /// starting from the first definition of the variable. + unsigned ScopeBytesFromFirstDefinition = 0; +}; + +/// Extract the low pc from a Die. +static uint64_t getLowPC(DWARFDie Die) { + if (Die.getAddressRanges().size()) + return Die.getAddressRanges()[0].LowPC; + return dwarf::toAddress(Die.find(dwarf::DW_AT_low_pc), 0); +} + +/// Collect debug info quality metrics for one DIE. +static void collectStatsForDie(DWARFDie Die, std::string Prefix, + uint64_t ScopeLowPC, uint64_t BytesInScope, + StringMap<PerFunctionStats> &FnStatMap, + GlobalStats &GlobalStats) { + bool HasLoc = false; + uint64_t BytesCovered = 0; + uint64_t OffsetToFirstDefinition = 0; + if (Die.find(dwarf::DW_AT_const_value)) { + // This catches constant members *and* variables. + HasLoc = true; + BytesCovered = BytesInScope; + } else if (Die.getTag() == dwarf::DW_TAG_variable || + Die.getTag() == dwarf::DW_TAG_formal_parameter) { + // Handle variables and function arguments. + auto FormValue = Die.find(dwarf::DW_AT_location); + HasLoc = FormValue.hasValue(); + if (HasLoc) { + // Get PC coverage. + if (auto DebugLocOffset = FormValue->getAsSectionOffset()) { + auto *DebugLoc = Die.getDwarfUnit()->getContext().getDebugLoc(); + if (auto List = DebugLoc->getLocationListAtOffset(*DebugLocOffset)) { + for (auto Entry : List->Entries) + BytesCovered += Entry.End - Entry.Begin; + if (List->Entries.size()) { + uint64_t FirstDef = List->Entries[0].Begin; + uint64_t UnitOfs = getLowPC(Die.getDwarfUnit()->getUnitDIE()); + // Ranges sometimes start before the lexical scope. + if (UnitOfs + FirstDef >= ScopeLowPC) + OffsetToFirstDefinition = UnitOfs + FirstDef - ScopeLowPC; + // Or even after it. Count that as a failure. + if (OffsetToFirstDefinition > BytesInScope) + OffsetToFirstDefinition = 0; + } + } + assert(BytesInScope); + } else { + // Assume the entire range is covered by a single location. + BytesCovered = BytesInScope; + } + } + } else { + // Not a variable or constant member. + return; + } + + // Collect PC range coverage data. + auto &FnStats = FnStatMap[Prefix]; + if (DWARFDie D = + Die.getAttributeValueAsReferencedDie(dwarf::DW_AT_abstract_origin)) + Die = D; + // This is a unique ID for the variable inside the current object file. + unsigned CanonicalDieOffset = Die.getOffset(); + FnStats.VarsInFunction.insert(CanonicalDieOffset); + if (BytesInScope) { + FnStats.TotalVarWithLoc += (unsigned)HasLoc; + // Adjust for the fact the variables often start their lifetime in the + // middle of the scope. + BytesInScope -= OffsetToFirstDefinition; + // Turns out we have a lot of ranges that extend past the lexical scope. + GlobalStats.ScopeBytesCovered += std::min(BytesInScope, BytesCovered); + GlobalStats.ScopeBytesFromFirstDefinition += BytesInScope; + assert(GlobalStats.ScopeBytesCovered <= + GlobalStats.ScopeBytesFromFirstDefinition); + } else { + FnStats.ConstantMembers++; + } +} + +/// Recursively collect debug info quality metrics. +static void collectStatsRecursive(DWARFDie Die, std::string Prefix, + uint64_t ScopeLowPC, uint64_t BytesInScope, + StringMap<PerFunctionStats> &FnStatMap, + GlobalStats &GlobalStats) { + // Handle any kind of lexical scope. + if (Die.getTag() == dwarf::DW_TAG_subprogram || + Die.getTag() == dwarf::DW_TAG_inlined_subroutine || + Die.getTag() == dwarf::DW_TAG_lexical_block) { + // Ignore forward declarations. + if (Die.find(dwarf::DW_AT_declaration)) + return; + + // Count the function. + if (Die.getTag() != dwarf::DW_TAG_lexical_block) { + StringRef Name = Die.getName(DINameKind::LinkageName); + if (Name.empty()) + Name = Die.getName(DINameKind::ShortName); + Prefix = Name; + // Skip over abstract origins. + if (Die.find(dwarf::DW_AT_inline)) + return; + // We've seen an (inlined) instance of this function. + auto &FnStats = FnStatMap[Name]; + FnStats.NumFnInlined++; + FnStats.IsFunction = true; + } + + // PC Ranges. + auto Ranges = Die.getAddressRanges(); + uint64_t BytesInThisScope = 0; + for (auto Range : Ranges) + BytesInThisScope += Range.HighPC - Range.LowPC; + ScopeLowPC = getLowPC(Die); + + if (BytesInThisScope) + BytesInScope = BytesInThisScope; + } else { + // Not a scope, visit the Die itself. It could be a variable. + collectStatsForDie(Die, Prefix, ScopeLowPC, BytesInScope, FnStatMap, + GlobalStats); + } + + // Traverse children. + DWARFDie Child = Die.getFirstChild(); + while (Child) { + collectStatsRecursive(Child, Prefix, ScopeLowPC, BytesInScope, FnStatMap, + GlobalStats); + Child = Child.getSibling(); + } +} + +/// Print machine-readable output. +/// The machine-readable format is single-line JSON output. +/// \{ +static void printDatum(raw_ostream &OS, const char *Key, StringRef Value) { + OS << ",\"" << Key << "\":\"" << Value << '"'; + DEBUG(llvm::dbgs() << Key << ": " << Value << '\n'); +} +static void printDatum(raw_ostream &OS, const char *Key, uint64_t Value) { + OS << ",\"" << Key << "\":" << Value; + DEBUG(llvm::dbgs() << Key << ": " << Value << '\n'); +} +/// \} + +/// Collect debug info quality metrics for an entire DIContext. +/// +/// Do the impossible and reduce the quality of the debug info down to a few +/// numbers. The idea is to condense the data into numbers that can be tracked +/// over time to identify trends in newer compiler versions and gauge the effect +/// of particular optimizations. The raw numbers themselves are not particularly +/// useful, only the delta between compiling the same program with different +/// compilers is. +bool collectStatsForObjectFile(ObjectFile &Obj, DWARFContext &DICtx, + Twine Filename, raw_ostream &OS) { + StringRef FormatName = Obj.getFileFormatName(); + GlobalStats GlobalStats; + StringMap<PerFunctionStats> Statistics; + for (const auto &CU : static_cast<DWARFContext *>(&DICtx)->compile_units()) + if (DWARFDie CUDie = CU->getUnitDIE(false)) + collectStatsRecursive(CUDie, "/", 0, 0, Statistics, GlobalStats); + + /// The version number should be increased every time the algorithm is changed + /// (including bug fixes). New metrics may be added without increasing the + /// version. + unsigned Version = 1; + unsigned VarTotal = 0; + unsigned VarUnique = 0; + unsigned VarWithLoc = 0; + unsigned NumFunctions = 0; + unsigned NumInlinedFunctions = 0; + for (auto &Entry : Statistics) { + PerFunctionStats &Stats = Entry.getValue(); + unsigned TotalVars = Stats.VarsInFunction.size() * Stats.NumFnInlined; + unsigned Constants = Stats.ConstantMembers; + VarWithLoc += Stats.TotalVarWithLoc + Constants; + VarTotal += TotalVars + Constants; + VarUnique += Stats.VarsInFunction.size(); + DEBUG(for (auto V : Stats.VarsInFunction) + llvm::dbgs() << Entry.getKey() << ": " << V << "\n"); + NumFunctions += Stats.IsFunction; + NumInlinedFunctions += Stats.IsFunction * Stats.NumFnInlined; + } + + // Print summary. + OS.SetBufferSize(1024); + OS << "{\"version\":\"" << Version << '"'; + DEBUG(llvm::dbgs() << "Variable location quality metrics\n"; + llvm::dbgs() << "---------------------------------\n"); + printDatum(OS, "file", Filename.str()); + printDatum(OS, "format", FormatName); + printDatum(OS, "source functions", NumFunctions); + printDatum(OS, "inlined functions", NumInlinedFunctions); + printDatum(OS, "unique source variables", VarUnique); + printDatum(OS, "source variables", VarTotal); + printDatum(OS, "variables with location", VarWithLoc); + printDatum(OS, "scope bytes total", + GlobalStats.ScopeBytesFromFirstDefinition); + printDatum(OS, "scope bytes covered", GlobalStats.ScopeBytesCovered); + OS << "}\n"; + DEBUG( + llvm::dbgs() << "Total Availability: " + << (int)std::round((VarWithLoc * 100.0) / VarTotal) << "%\n"; + llvm::dbgs() << "PC Ranges covered: " + << (int)std::round((GlobalStats.ScopeBytesCovered * 100.0) / + GlobalStats.ScopeBytesFromFirstDefinition) + << "%\n"); + return true; +} diff --git a/tools/llvm-dwarfdump/fuzzer/CMakeLists.txt b/tools/llvm-dwarfdump/fuzzer/CMakeLists.txt index 1de35a3de478..318c4f7bfce4 100644 --- a/tools/llvm-dwarfdump/fuzzer/CMakeLists.txt +++ b/tools/llvm-dwarfdump/fuzzer/CMakeLists.txt @@ -4,11 +4,7 @@ set(LLVM_LINK_COMPONENTS Support ) -add_llvm_executable(llvm-dwarfdump-fuzzer +add_llvm_fuzzer(llvm-dwarfdump-fuzzer EXCLUDE_FROM_ALL llvm-dwarfdump-fuzzer.cpp ) - -target_link_libraries(llvm-dwarfdump-fuzzer - LLVMFuzzer - ) diff --git a/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp b/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp index 32e173f9d1fc..53c74df40280 100644 --- a/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp +++ b/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp @@ -20,7 +20,7 @@ using namespace llvm; using namespace object; -extern "C" void LLVMFuzzerTestOneInput(uint8_t *data, size_t size) { +extern "C" int LLVMFuzzerTestOneInput(uint8_t *data, size_t size) { std::unique_ptr<MemoryBuffer> Buff = MemoryBuffer::getMemBuffer( StringRef((const char *)data, size), "", false); @@ -28,9 +28,14 @@ extern "C" void LLVMFuzzerTestOneInput(uint8_t *data, size_t size) { ObjectFile::createObjectFile(Buff->getMemBufferRef()); if (auto E = ObjOrErr.takeError()) { consumeError(std::move(E)); - return; + return 0; } ObjectFile &Obj = *ObjOrErr.get(); - std::unique_ptr<DIContext> DICtx(new DWARFContextInMemory(Obj)); - DICtx->dump(nulls(), DIDT_All); + std::unique_ptr<DIContext> DICtx = DWARFContext::create(Obj); + + + DIDumpOptions opts; + opts.DumpType = DIDT_All; + DICtx->dump(nulls(), opts); + return 0; } diff --git a/tools/llvm-dwarfdump/llvm-dwarfdump.cpp b/tools/llvm-dwarfdump/llvm-dwarfdump.cpp index ec5e554d4f5f..e4e34efff842 100644 --- a/tools/llvm-dwarfdump/llvm-dwarfdump.cpp +++ b/tools/llvm-dwarfdump/llvm-dwarfdump.cpp @@ -12,12 +12,13 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ADT/Triple.h" #include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/Object/Archive.h" #include "llvm/Object/MachOUniversal.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/Object/RelocVisitor.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Format.h" @@ -25,120 +26,379 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Regex.h" #include "llvm/Support/Signals.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <cstring> -#include <string> -#include <system_error> using namespace llvm; using namespace object; -static cl::list<std::string> -InputFilenames(cl::Positional, cl::desc("<input object files or .dSYM bundles>"), - cl::ZeroOrMore); - -static cl::opt<DIDumpType> DumpType( - "debug-dump", cl::init(DIDT_All), cl::desc("Dump of debug sections:"), - cl::values( - clEnumValN(DIDT_All, "all", "Dump all debug sections"), - clEnumValN(DIDT_Abbrev, "abbrev", ".debug_abbrev"), - clEnumValN(DIDT_AbbrevDwo, "abbrev.dwo", ".debug_abbrev.dwo"), - clEnumValN(DIDT_AppleNames, "apple_names", ".apple_names"), - clEnumValN(DIDT_AppleTypes, "apple_types", ".apple_types"), - clEnumValN(DIDT_AppleNamespaces, "apple_namespaces", - ".apple_namespaces"), - clEnumValN(DIDT_AppleObjC, "apple_objc", ".apple_objc"), - clEnumValN(DIDT_Aranges, "aranges", ".debug_aranges"), - clEnumValN(DIDT_Info, "info", ".debug_info"), - clEnumValN(DIDT_InfoDwo, "info.dwo", ".debug_info.dwo"), - clEnumValN(DIDT_Types, "types", ".debug_types"), - clEnumValN(DIDT_TypesDwo, "types.dwo", ".debug_types.dwo"), - clEnumValN(DIDT_Line, "line", ".debug_line"), - clEnumValN(DIDT_LineDwo, "line.dwo", ".debug_line.dwo"), - clEnumValN(DIDT_Loc, "loc", ".debug_loc"), - clEnumValN(DIDT_LocDwo, "loc.dwo", ".debug_loc.dwo"), - clEnumValN(DIDT_Frames, "frames", ".debug_frame"), - clEnumValN(DIDT_Macro, "macro", ".debug_macinfo"), - clEnumValN(DIDT_Ranges, "ranges", ".debug_ranges"), - clEnumValN(DIDT_Pubnames, "pubnames", ".debug_pubnames"), - clEnumValN(DIDT_Pubtypes, "pubtypes", ".debug_pubtypes"), - clEnumValN(DIDT_GnuPubnames, "gnu_pubnames", ".debug_gnu_pubnames"), - clEnumValN(DIDT_GnuPubtypes, "gnu_pubtypes", ".debug_gnu_pubtypes"), - clEnumValN(DIDT_Str, "str", ".debug_str"), - clEnumValN(DIDT_StrOffsets, "str_offsets", ".debug_str_offsets"), - clEnumValN(DIDT_StrDwo, "str.dwo", ".debug_str.dwo"), - clEnumValN(DIDT_StrOffsetsDwo, "str_offsets.dwo", - ".debug_str_offsets.dwo"), - clEnumValN(DIDT_CUIndex, "cu_index", ".debug_cu_index"), - clEnumValN(DIDT_GdbIndex, "gdb_index", ".gdb_index"), - clEnumValN(DIDT_TUIndex, "tu_index", ".debug_tu_index"))); +/// Parser for options that take an optional offest argument. +/// @{ +struct OffsetOption { + uint64_t Val = 0; + bool HasValue = false; + bool IsRequested = false; +}; -static cl::opt<bool> - SummarizeTypes("summarize-types", - cl::desc("Abbreviate the description of type unit entries")); +namespace llvm { +namespace cl { +template <> +class parser<OffsetOption> final : public basic_parser<OffsetOption> { +public: + parser(Option &O) : basic_parser(O) {} + + /// Return true on error. + bool parse(Option &O, StringRef ArgName, StringRef Arg, OffsetOption &Val) { + if (Arg == "") { + Val.Val = 0; + Val.HasValue = false; + Val.IsRequested = true; + return false; + } + if (Arg.getAsInteger(0, Val.Val)) + return O.error("'" + Arg + "' value invalid for integer argument!"); + Val.HasValue = true; + Val.IsRequested = true; + return false; + } + + enum ValueExpected getValueExpectedFlagDefault() const { + return ValueOptional; + } + + void printOptionInfo(const Option &O, size_t GlobalWidth) const { + outs() << " -" << O.ArgStr; + Option::printHelpStr(O.HelpStr, GlobalWidth, getOptionWidth(O)); + } + + void printOptionDiff(const Option &O, OffsetOption V, OptVal Default, + size_t GlobalWidth) const { + printOptionName(O, GlobalWidth); + outs() << "[=offset]"; + } + + // An out-of-line virtual method to provide a 'home' for this class. + void anchor() override {}; +}; +} // cl +} // llvm -static cl::opt<bool> Verify("verify", cl::desc("Verify the DWARF debug info")); +/// @} +/// Command line options. +/// @{ -static cl::opt<bool> Quiet("quiet", - cl::desc("Use with -verify to not emit to STDOUT.")); +namespace { +using namespace cl; -static cl::opt<bool> Brief("brief", cl::desc("Print fewer low-level details")); +OptionCategory DwarfDumpCategory("Specific Options"); +static opt<bool> Help("h", desc("Alias for -help"), Hidden, + cat(DwarfDumpCategory)); +static list<std::string> + InputFilenames(Positional, desc("<input object files or .dSYM bundles>"), + ZeroOrMore, cat(DwarfDumpCategory)); -static void error(StringRef Filename, std::error_code EC) { +cl::OptionCategory SectionCategory("Section-specific Dump Options", + "These control which sections are dumped. " + "Where applicable these parameters take an " + "optional =<offset> argument to dump only " + "the entry at the specified offset."); + +static opt<bool> DumpAll("all", desc("Dump all debug info sections"), + cat(SectionCategory)); +static alias DumpAllAlias("a", desc("Alias for -all"), aliasopt(DumpAll)); + +// Options for dumping specific sections. +static unsigned DumpType = DIDT_Null; +static std::array<llvm::Optional<uint64_t>, (unsigned)DIDT_ID_Count> + DumpOffsets; +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME) \ + static opt<OffsetOption> Dump##ENUM_NAME( \ + CMDLINE_NAME, desc("Dump the " ELF_NAME " section"), \ + cat(SectionCategory)); +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION + +static alias DumpDebugFrameAlias("eh-frame", desc("Alias for -debug-frame"), + NotHidden, cat(SectionCategory), + aliasopt(DumpDebugFrame)); +static list<std::string> + ArchFilters("arch", + desc("Dump debug information for the specified CPU " + "architecture only. Architectures may be specified by " + "name or by number. This option can be specified " + "multiple times, once for each desired architecture."), + cat(DwarfDumpCategory)); +static opt<bool> + Diff("diff", + desc("Emit diff-friendly output by omitting offsets and addresses."), + cat(DwarfDumpCategory)); +static list<std::string> + Find("find", + desc("Search for the exact match for <name> in the accelerator tables " + "and print the matching debug information entries. When no " + "accelerator tables are available, the slower but more complete " + "-name option can be used instead."), + value_desc("name"), cat(DwarfDumpCategory)); +static alias FindAlias("f", desc("Alias for -find."), aliasopt(Find)); +static opt<bool> + IgnoreCase("ignore-case", + desc("Ignore case distinctions in when searching by name."), + value_desc("i"), cat(DwarfDumpCategory)); +static alias IgnoreCaseAlias("i", desc("Alias for -ignore-case."), + aliasopt(IgnoreCase)); +static list<std::string> Name( + "name", + desc("Find and print all debug info entries whose name (DW_AT_name " + "attribute) matches the exact text in <pattern>. When used with the " + "the -regex option <pattern> is interpreted as a regular expression."), + value_desc("pattern"), cat(DwarfDumpCategory)); +static alias NameAlias("n", desc("Alias for -name"), aliasopt(Name)); +static opt<unsigned> + Lookup("lookup", + desc("Lookup <address> in the debug information and print out any" + "available file, function, block and line table details."), + value_desc("address"), cat(DwarfDumpCategory)); +static opt<std::string> + OutputFilename("out-file", cl::init(""), + cl::desc("Redirect output to the specified file."), + cl::value_desc("filename")); +static alias OutputFilenameAlias("o", desc("Alias for -out-file."), + aliasopt(OutputFilename), + cat(DwarfDumpCategory)); +static opt<bool> + UseRegex("regex", + desc("Treat any <pattern> strings as regular expressions when " + "searching instead of just as an exact string match."), + cat(DwarfDumpCategory)); +static alias RegexAlias("x", desc("Alias for -regex"), aliasopt(UseRegex)); +static opt<bool> + ShowChildren("show-children", + desc("Show a debug info entry's children when selectively " + "printing with the =<offset> option."), + cat(DwarfDumpCategory)); +static alias ShowChildrenAlias("c", desc("Alias for -show-children."), + aliasopt(ShowChildren)); +static opt<bool> + ShowParents("show-parents", + desc("Show a debug info entry's parents when selectively " + "printing with the =<offset> option."), + cat(DwarfDumpCategory)); +static alias ShowParentsAlias("p", desc("Alias for -show-parents."), + aliasopt(ShowParents)); +static opt<bool> + ShowForm("show-form", + desc("Show DWARF form types after the DWARF attribute types."), + cat(DwarfDumpCategory)); +static alias ShowFormAlias("F", desc("Alias for -show-form."), + aliasopt(ShowForm), cat(DwarfDumpCategory)); +static opt<unsigned> RecurseDepth( + "recurse-depth", + desc("Only recurse to a depth of N when displaying debug info entries."), + cat(DwarfDumpCategory), init(-1U), value_desc("N")); +static alias RecurseDepthAlias("r", desc("Alias for -recurse-depth."), + aliasopt(RecurseDepth)); + +static opt<bool> + SummarizeTypes("summarize-types", + desc("Abbreviate the description of type unit entries."), + cat(DwarfDumpCategory)); +static cl::opt<bool> + Statistics("statistics", + cl::desc("Emit JSON-formatted debug info quality metrics."), + cat(DwarfDumpCategory)); +static opt<bool> Verify("verify", desc("Verify the DWARF debug info."), + cat(DwarfDumpCategory)); +static opt<bool> Quiet("quiet", desc("Use with -verify to not emit to STDOUT."), + cat(DwarfDumpCategory)); +static opt<bool> DumpUUID("uuid", desc("Show the UUID for each architecture."), + cat(DwarfDumpCategory)); +static alias DumpUUIDAlias("u", desc("Alias for -uuid."), aliasopt(DumpUUID)); +static opt<bool> Verbose("verbose", + desc("Print more low-level encoding details."), + cat(DwarfDumpCategory)); +static alias VerboseAlias("v", desc("Alias for -verbose."), aliasopt(Verbose), + cat(DwarfDumpCategory)); +} // namespace +/// @} +//===----------------------------------------------------------------------===// + +static void error(StringRef Prefix, std::error_code EC) { if (!EC) return; - errs() << Filename << ": " << EC.message() << "\n"; + errs() << Prefix << ": " << EC.message() << "\n"; exit(1); } -static void DumpObjectFile(ObjectFile &Obj, Twine Filename) { - std::unique_ptr<DIContext> DICtx(new DWARFContextInMemory(Obj)); - - outs() << Filename.str() << ":\tfile format " << Obj.getFileFormatName() - << "\n\n"; - - - // Dump the complete DWARF structure. +static DIDumpOptions getDumpOpts() { DIDumpOptions DumpOpts; DumpOpts.DumpType = DumpType; + DumpOpts.RecurseDepth = RecurseDepth; + DumpOpts.ShowAddresses = !Diff; + DumpOpts.ShowChildren = ShowChildren; + DumpOpts.ShowParents = ShowParents; + DumpOpts.ShowForm = ShowForm; DumpOpts.SummarizeTypes = SummarizeTypes; - DumpOpts.Brief = Brief; - DICtx->dump(outs(), DumpOpts); + DumpOpts.Verbose = Verbose; + // In -verify mode, print DIEs without children in error messages. + if (Verify) + return DumpOpts.noImplicitRecursion(); + return DumpOpts; } -static void DumpInput(StringRef Filename) { - ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr = - MemoryBuffer::getFileOrSTDIN(Filename); - error(Filename, BuffOrErr.getError()); - std::unique_ptr<MemoryBuffer> Buff = std::move(BuffOrErr.get()); +static uint32_t getCPUType(MachOObjectFile &MachO) { + if (MachO.is64Bit()) + return MachO.getHeader64().cputype; + else + return MachO.getHeader().cputype; +} - Expected<std::unique_ptr<Binary>> BinOrErr = - object::createBinary(Buff->getMemBufferRef()); - if (!BinOrErr) - error(Filename, errorToErrorCode(BinOrErr.takeError())); +/// Return true if the object file has not been filtered by an --arch option. +static bool filterArch(ObjectFile &Obj) { + if (ArchFilters.empty()) + return true; - if (auto *Obj = dyn_cast<ObjectFile>(BinOrErr->get())) - DumpObjectFile(*Obj, Filename); - else if (auto *Fat = dyn_cast<MachOUniversalBinary>(BinOrErr->get())) - for (auto &ObjForArch : Fat->objects()) { - auto MachOOrErr = ObjForArch.getAsObjectFile(); - error(Filename, errorToErrorCode(MachOOrErr.takeError())); - DumpObjectFile(**MachOOrErr, - Filename + " (" + ObjForArch.getArchFlagName() + ")"); + if (auto *MachO = dyn_cast<MachOObjectFile>(&Obj)) { + std::string ObjArch = + Triple::getArchTypeName(MachO->getArchTriple().getArch()); + + for (auto Arch : ArchFilters) { + // Match name. + if (Arch == ObjArch) + return true; + + // Match architecture number. + unsigned Value; + if (!StringRef(Arch).getAsInteger(0, Value)) + if (Value == getCPUType(*MachO)) + return true; + } + } + return false; +} + +using HandlerFn = std::function<bool(ObjectFile &, DWARFContext &DICtx, Twine, + raw_ostream &)>; + +/// Print only DIEs that have a certain name. +static void filterByName(const StringSet<> &Names, + DWARFContext::cu_iterator_range CUs, raw_ostream &OS) { + for (const auto &CU : CUs) + for (const auto &Entry : CU->dies()) { + DWARFDie Die = {CU.get(), &Entry}; + if (const char *NamePtr = Die.getName(DINameKind::ShortName)) { + std::string Name = + (IgnoreCase && !UseRegex) ? StringRef(NamePtr).lower() : NamePtr; + // Match regular expression. + if (UseRegex) + for (auto Pattern : Names.keys()) { + Regex RE(Pattern, IgnoreCase ? Regex::IgnoreCase : Regex::NoFlags); + std::string Error; + if (!RE.isValid(Error)) { + errs() << "error in regular expression: " << Error << "\n"; + exit(1); + } + if (RE.match(Name)) + Die.dump(OS, 0, getDumpOpts()); + } + // Match full text. + else if (Names.count(Name)) + Die.dump(OS, 0, getDumpOpts()); + } } + } -static bool VerifyObjectFile(ObjectFile &Obj, Twine Filename) { - std::unique_ptr<DIContext> DICtx(new DWARFContextInMemory(Obj)); - +/// Handle the --lookup option and dump the DIEs and line info for the given +/// address. +static bool lookup(DWARFContext &DICtx, uint64_t Address, raw_ostream &OS) { + auto DIEsForAddr = DICtx.getDIEsForAddress(Lookup); + + if (!DIEsForAddr) + return false; + + DIDumpOptions DumpOpts = getDumpOpts(); + DumpOpts.RecurseDepth = 0; + DIEsForAddr.CompileUnit->dump(OS, DumpOpts); + if (DIEsForAddr.FunctionDIE) { + DIEsForAddr.FunctionDIE.dump(OS, 2, DumpOpts); + if (DIEsForAddr.BlockDIE) + DIEsForAddr.BlockDIE.dump(OS, 4, DumpOpts); + } + + if (DILineInfo LineInfo = DICtx.getLineInfoForAddress(Lookup)) + LineInfo.dump(OS); + + return true; +} + +bool collectStatsForObjectFile(ObjectFile &Obj, DWARFContext &DICtx, + Twine Filename, raw_ostream &OS); + +static bool dumpObjectFile(ObjectFile &Obj, DWARFContext &DICtx, Twine Filename, + raw_ostream &OS) { + logAllUnhandledErrors(DICtx.loadRegisterInfo(Obj), errs(), + Filename.str() + ": "); + // The UUID dump already contains all the same information. + if (!(DumpType & DIDT_UUID) || DumpType == DIDT_All) + OS << Filename << ":\tfile format " << Obj.getFileFormatName() << '\n'; + + // Handle the --lookup option. + if (Lookup) + return lookup(DICtx, Lookup, OS); + + // Handle the --name option. + if (!Name.empty()) { + StringSet<> Names; + for (auto name : Name) + Names.insert((IgnoreCase && !UseRegex) ? StringRef(name).lower() : name); + + filterByName(Names, DICtx.compile_units(), OS); + filterByName(Names, DICtx.dwo_compile_units(), OS); + return true; + } + + // Handle the --find option and lower it to --debug-info=<offset>. + if (!Find.empty()) { + DumpOffsets[DIDT_ID_DebugInfo] = [&]() -> llvm::Optional<uint64_t> { + for (auto Name : Find) { + auto find = [&](const DWARFAcceleratorTable &Accel) + -> llvm::Optional<uint64_t> { + for (auto Entry : Accel.equal_range(Name)) + for (auto Atom : Entry) + if (auto Offset = Atom.getAsSectionOffset()) + return Offset; + return None; + }; + if (auto Offset = find(DICtx.getAppleNames())) + return DumpOffsets[DIDT_ID_DebugInfo] = *Offset; + if (auto Offset = find(DICtx.getAppleTypes())) + return DumpOffsets[DIDT_ID_DebugInfo] = *Offset; + if (auto Offset = find(DICtx.getAppleNamespaces())) + return DumpOffsets[DIDT_ID_DebugInfo] = *Offset; + } + return None; + }(); + // Early exit if --find was specified but the current file doesn't have it. + if (!DumpOffsets[DIDT_ID_DebugInfo]) + return true; + } + + // Dump the complete DWARF structure. + DICtx.dump(OS, getDumpOpts(), DumpOffsets); + return true; +} + +static bool verifyObjectFile(ObjectFile &Obj, DWARFContext &DICtx, + Twine Filename, raw_ostream &OS) { // Verify the DWARF and exit with non-zero exit status if verification // fails. - raw_ostream &stream = Quiet ? nulls() : outs(); + raw_ostream &stream = Quiet ? nulls() : OS; stream << "Verifying " << Filename.str() << ":\tfile format " << Obj.getFileFormatName() << "\n"; - bool Result = DICtx->verify(stream, DumpType); + bool Result = DICtx.verify(stream, getDumpOpts()); if (Result) stream << "No errors.\n"; else @@ -146,30 +406,72 @@ static bool VerifyObjectFile(ObjectFile &Obj, Twine Filename) { return Result; } -static bool VerifyInput(StringRef Filename) { - ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr = - MemoryBuffer::getFileOrSTDIN(Filename); - error(Filename, BuffOrErr.getError()); - std::unique_ptr<MemoryBuffer> Buff = std::move(BuffOrErr.get()); - - Expected<std::unique_ptr<Binary>> BinOrErr = - object::createBinary(Buff->getMemBufferRef()); - if (!BinOrErr) - error(Filename, errorToErrorCode(BinOrErr.takeError())); - +static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer, + HandlerFn HandleObj, raw_ostream &OS); + +static bool handleArchive(StringRef Filename, Archive &Arch, + HandlerFn HandleObj, raw_ostream &OS) { + bool Result = true; + Error Err = Error::success(); + for (auto Child : Arch.children(Err)) { + auto BuffOrErr = Child.getMemoryBufferRef(); + error(Filename, errorToErrorCode(BuffOrErr.takeError())); + auto NameOrErr = Child.getName(); + error(Filename, errorToErrorCode(NameOrErr.takeError())); + std::string Name = (Filename + "(" + NameOrErr.get() + ")").str(); + Result &= handleBuffer(Name, BuffOrErr.get(), HandleObj, OS); + } + error(Filename, errorToErrorCode(std::move(Err))); + + return Result; +} + +static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer, + HandlerFn HandleObj, raw_ostream &OS) { + Expected<std::unique_ptr<Binary>> BinOrErr = object::createBinary(Buffer); + error(Filename, errorToErrorCode(BinOrErr.takeError())); + bool Result = true; - if (auto *Obj = dyn_cast<ObjectFile>(BinOrErr->get())) - Result = VerifyObjectFile(*Obj, Filename); + if (auto *Obj = dyn_cast<ObjectFile>(BinOrErr->get())) { + if (filterArch(*Obj)) { + std::unique_ptr<DWARFContext> DICtx = DWARFContext::create(*Obj); + Result = HandleObj(*Obj, *DICtx, Filename, OS); + } + } else if (auto *Fat = dyn_cast<MachOUniversalBinary>(BinOrErr->get())) for (auto &ObjForArch : Fat->objects()) { - auto MachOOrErr = ObjForArch.getAsObjectFile(); - error(Filename, errorToErrorCode(MachOOrErr.takeError())); - if (!VerifyObjectFile(**MachOOrErr, Filename + " (" + ObjForArch.getArchFlagName() + ")")) - Result = false; + std::string ObjName = + (Filename + "(" + ObjForArch.getArchFlagName() + ")").str(); + if (auto MachOOrErr = ObjForArch.getAsObjectFile()) { + auto &Obj = **MachOOrErr; + if (filterArch(Obj)) { + std::unique_ptr<DWARFContext> DICtx = DWARFContext::create(Obj); + Result &= HandleObj(Obj, *DICtx, ObjName, OS); + } + continue; + } else + consumeError(MachOOrErr.takeError()); + if (auto ArchiveOrErr = ObjForArch.getAsArchive()) { + error(ObjName, errorToErrorCode(ArchiveOrErr.takeError())); + Result &= handleArchive(ObjName, *ArchiveOrErr.get(), HandleObj, OS); + continue; + } else + consumeError(ArchiveOrErr.takeError()); } + else if (auto *Arch = dyn_cast<Archive>(BinOrErr->get())) + Result = handleArchive(Filename, *Arch, HandleObj, OS); return Result; } +static bool handleFile(StringRef Filename, HandlerFn HandleObj, + raw_ostream &OS) { + ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr = + MemoryBuffer::getFileOrSTDIN(Filename); + error(Filename, BuffOrErr.getError()); + std::unique_ptr<MemoryBuffer> Buffer = std::move(BuffOrErr.get()); + return handleBuffer(Filename, *Buffer, HandleObj, OS); +} + /// If the input path is a .dSYM bundle (as created by the dsymutil tool), /// replace it with individual entries for each of the object files inside the /// bundle otherwise return the input path. @@ -209,7 +511,59 @@ int main(int argc, char **argv) { PrettyStackTraceProgram X(argc, argv); llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. - cl::ParseCommandLineOptions(argc, argv, "llvm dwarf dumper\n"); + llvm::InitializeAllTargetInfos(); + llvm::InitializeAllTargetMCs(); + + HideUnrelatedOptions({&DwarfDumpCategory, &SectionCategory}); + cl::ParseCommandLineOptions( + argc, argv, + "pretty-print DWARF debug information in object files" + " and debug info archives.\n"); + + if (Help) { + PrintHelpMessage(/*Hidden =*/false, /*Categorized =*/true); + return 0; + } + + std::unique_ptr<ToolOutputFile> OutputFile; + if (!OutputFilename.empty()) { + std::error_code EC; + OutputFile = llvm::make_unique<ToolOutputFile>(OutputFilename, EC, + sys::fs::F_None); + error("Unable to open output file" + OutputFilename, EC); + // Don't remove output file if we exit with an error. + OutputFile->keep(); + } + + raw_ostream &OS = OutputFile ? OutputFile->os() : outs(); + bool OffsetRequested = false; + + // Defaults to dumping all sections, unless brief mode is specified in which + // case only the .debug_info section in dumped. +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME) \ + if (Dump##ENUM_NAME.IsRequested) { \ + DumpType |= DIDT_##ENUM_NAME; \ + if (Dump##ENUM_NAME.HasValue) { \ + DumpOffsets[DIDT_ID_##ENUM_NAME] = Dump##ENUM_NAME.Val; \ + OffsetRequested = true; \ + } \ + } +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION + if (DumpUUID) + DumpType |= DIDT_UUID; + if (DumpAll) + DumpType = DIDT_All; + if (DumpType == DIDT_Null) { + if (Verbose) + DumpType = DIDT_All; + else + DumpType = DIDT_DebugInfo; + } + + // Unless dumping a specific DIE, default to --show-children. + if (!ShowChildren && !Verify && !OffsetRequested && Name.empty() && Find.empty()) + ShowChildren = true; // Defaults to a.out if no filenames specified. if (InputFilenames.size() == 0) @@ -224,11 +578,16 @@ int main(int argc, char **argv) { if (Verify) { // If we encountered errors during verify, exit with a non-zero exit status. - if (!std::all_of(Objects.begin(), Objects.end(), VerifyInput)) + if (!std::all_of(Objects.begin(), Objects.end(), [&](std::string Object) { + return handleFile(Object, verifyObjectFile, OS); + })) exit(1); - } else { - std::for_each(Objects.begin(), Objects.end(), DumpInput); - } + } else if (Statistics) + for (auto Object : Objects) + handleFile(Object, collectStatsForObjectFile, OS); + else + for (auto Object : Objects) + handleFile(Object, dumpObjectFile, OS); return EXIT_SUCCESS; } diff --git a/tools/llvm-dwp/CMakeLists.txt b/tools/llvm-dwp/CMakeLists.txt index 98d67e04fe6a..1b5fbddc1f75 100644 --- a/tools/llvm-dwp/CMakeLists.txt +++ b/tools/llvm-dwp/CMakeLists.txt @@ -15,3 +15,7 @@ add_llvm_tool(llvm-dwp DEPENDS intrinsics_gen ) + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(dwp llvm-dwp) +endif() diff --git a/tools/llvm-dwp/llvm-dwp.cpp b/tools/llvm-dwp/llvm-dwp.cpp index 7a6922d8a3fc..dbbe61bf3b06 100644 --- a/tools/llvm-dwp/llvm-dwp.cpp +++ b/tools/llvm-dwp/llvm-dwp.cpp @@ -15,43 +15,43 @@ #include "DWPStringPool.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSet.h" -#include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" #include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h" +#include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCTargetOptionsCommandFlags.h" +#include "llvm/MC/MCTargetOptionsCommandFlags.def" #include "llvm/Object/Decompressor.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/Support/Compression.h" #include "llvm/Support/DataExtractor.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Options.h" +#include "llvm/Support/Path.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" -#include <deque> -#include <iostream> -#include <memory> using namespace llvm; using namespace llvm::object; using namespace cl; OptionCategory DwpCategory("Specific Options"); -static list<std::string> InputFiles(Positional, OneOrMore, +static list<std::string> InputFiles(Positional, ZeroOrMore, desc("<input files>"), cat(DwpCategory)); +static list<std::string> ExecFilenames( + "e", ZeroOrMore, + desc("Specify the executable/library files to get the list of *.dwo from"), + value_desc("filename"), cat(DwpCategory)); + static opt<std::string> OutputFilename(Required, "o", desc("Specify the output file."), value_desc("filename"), @@ -113,7 +113,7 @@ struct CompileUnitIdentifiers { }; static Expected<const char *> -getIndexedString(dwarf::Form Form, DataExtractor InfoData, +getIndexedString(dwarf::Form Form, DataExtractor InfoData, uint32_t &InfoOffset, StringRef StrOffsets, StringRef Str) { if (Form == dwarf::DW_FORM_string) return InfoData.getCStr(&InfoOffset); @@ -463,6 +463,35 @@ buildDuplicateError(const std::pair<uint64_t, UnitIndexEntry> &PrevE, " and " + buildDWODescription(ID.Name, DWPName, ID.DWOName)); } +static Expected<SmallVector<std::string, 16>> +getDWOFilenames(StringRef ExecFilename) { + auto ErrOrObj = object::ObjectFile::createObjectFile(ExecFilename); + if (!ErrOrObj) + return ErrOrObj.takeError(); + + const ObjectFile &Obj = *ErrOrObj.get().getBinary(); + std::unique_ptr<DWARFContext> DWARFCtx = DWARFContext::create(Obj); + + SmallVector<std::string, 16> DWOPaths; + for (const auto &CU : DWARFCtx->compile_units()) { + const DWARFDie &Die = CU->getUnitDIE(); + std::string DWOName = dwarf::toString( + Die.find({dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}), ""); + if (DWOName.empty()) + continue; + std::string DWOCompDir = + dwarf::toString(Die.find(dwarf::DW_AT_comp_dir), ""); + if (!DWOCompDir.empty()) { + SmallString<16> DWOPath; + sys::path::append(DWOPath, DWOCompDir, DWOName); + DWOPaths.emplace_back(DWOPath.data(), DWOPath.size()); + } else { + DWOPaths.push_back(std::move(DWOName)); + } + } + return std::move(DWOPaths); +} + static Error write(MCStreamer &Out, ArrayRef<std::string> Inputs) { const auto &MCOFI = *Out.getContext().getObjectFileInfo(); MCSection *const StrSection = MCOFI.getDwarfStrDWOSection(); @@ -642,7 +671,7 @@ int main(int argc, char **argv) { MCObjectFileInfo MOFI; MCContext MC(MAI.get(), MRI.get(), &MOFI); - MOFI.InitMCObjectFileInfo(TheTriple, /*PIC*/ false, CodeModel::Default, MC); + MOFI.InitMCObjectFileInfo(TheTriple, /*PIC*/ false, MC); MCTargetOptions Options; auto MAB = TheTarget->createMCAsmBackend(*MRI, TripleName, "", Options); @@ -670,13 +699,26 @@ int main(int argc, char **argv) { MCTargetOptions MCOptions = InitMCTargetOptionsFromFlags(); std::unique_ptr<MCStreamer> MS(TheTarget->createMCObjectStreamer( - TheTriple, MC, *MAB, OutFile, MCE, *MSTI, MCOptions.MCRelaxAll, + TheTriple, MC, std::unique_ptr<MCAsmBackend>(MAB), OutFile, + std::unique_ptr<MCCodeEmitter>(MCE), *MSTI, MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible, /*DWARFMustBeAtTheEnd*/ false)); if (!MS) return error("no object streamer for target " + TripleName, Context); - if (auto Err = write(*MS, InputFiles)) { + std::vector<std::string> DWOFilenames = InputFiles; + for (const auto &ExecFilename : ExecFilenames) { + auto DWOs = getDWOFilenames(ExecFilename); + if (!DWOs) { + logAllUnhandledErrors(DWOs.takeError(), errs(), "error: "); + return 1; + } + DWOFilenames.insert(DWOFilenames.end(), + std::make_move_iterator(DWOs->begin()), + std::make_move_iterator(DWOs->end())); + } + + if (auto Err = write(*MS, DWOFilenames)) { logAllUnhandledErrors(std::move(Err), errs(), "error: "); return 1; } diff --git a/tools/llvm-extract/llvm-extract.cpp b/tools/llvm-extract/llvm-extract.cpp index d868db7f78ad..c39ffa58fbf7 100644 --- a/tools/llvm-extract/llvm-extract.cpp +++ b/tools/llvm-extract/llvm-extract.cpp @@ -296,7 +296,7 @@ int main(int argc, char **argv) { Passes.add(createStripDeadPrototypesPass()); // Remove dead func decls std::error_code EC; - tool_output_file Out(OutputFilename, EC, sys::fs::F_None); + ToolOutputFile Out(OutputFilename, EC, sys::fs::F_None); if (EC) { errs() << EC.message() << '\n'; return 1; diff --git a/tools/llvm-go/llvm-go.go b/tools/llvm-go/llvm-go.go index 604509f83b9a..4d65de623462 100644 --- a/tools/llvm-go/llvm-go.go +++ b/tools/llvm-go/llvm-go.go @@ -92,7 +92,7 @@ func llvmFlags() compilerFlags { ldflags := llvmConfig(args...) if runtime.GOOS != "darwin" { // OS X doesn't like -rpath with cgo. See: - // https://code.google.com/p/go/issues/detail?id=7293 + // https://github.com/golang/go/issues/7293 ldflags = "-Wl,-rpath," + llvmConfig("--libdir") + " " + ldflags } return compilerFlags{ diff --git a/tools/llvm-isel-fuzzer/CMakeLists.txt b/tools/llvm-isel-fuzzer/CMakeLists.txt new file mode 100644 index 000000000000..21f3d14f1749 --- /dev/null +++ b/tools/llvm-isel-fuzzer/CMakeLists.txt @@ -0,0 +1,18 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + Analysis + AsmPrinter + BitReader + BitWriter + CodeGen + Core + FuzzMutate + IRReader + MC + ScalarOpts + SelectionDAG + Support + Target +) +add_llvm_fuzzer(llvm-isel-fuzzer llvm-isel-fuzzer.cpp + DUMMY_MAIN DummyISelFuzzer.cpp) diff --git a/tools/llvm-isel-fuzzer/DummyISelFuzzer.cpp b/tools/llvm-isel-fuzzer/DummyISelFuzzer.cpp new file mode 100644 index 000000000000..ec9c74e3040d --- /dev/null +++ b/tools/llvm-isel-fuzzer/DummyISelFuzzer.cpp @@ -0,0 +1,21 @@ +//===--- DummyFuzzerMain.cpp - Entry point to sanity check the fuzzer -----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementation of main so we can build and test without linking libFuzzer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/FuzzMutate/FuzzerCLI.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); +extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv); +int main(int argc, char *argv[]) { + return llvm::runFuzzerOnInputs(argc, argv, LLVMFuzzerTestOneInput, + LLVMFuzzerInitialize); +} diff --git a/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp b/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp new file mode 100644 index 000000000000..764134d18b7e --- /dev/null +++ b/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp @@ -0,0 +1,170 @@ +//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Tool to fuzz instruction selection using libFuzzer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CodeGen/CommandFlags.def" +#include "llvm/FuzzMutate/FuzzerCLI.h" +#include "llvm/FuzzMutate/IRMutator.h" +#include "llvm/FuzzMutate/Operations.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IRReader/IRReader.h" +#include "llvm/Support/DataTypes.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Target/TargetMachine.h" + +#define DEBUG_TYPE "isel-fuzzer" + +using namespace llvm; + +static cl::opt<char> +OptLevel("O", + cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] " + "(default = '-O2')"), + cl::Prefix, + cl::ZeroOrMore, + cl::init(' ')); + +static cl::opt<std::string> +TargetTriple("mtriple", cl::desc("Override target triple for module")); + +static std::unique_ptr<TargetMachine> TM; +static std::unique_ptr<IRMutator> Mutator; + +std::unique_ptr<IRMutator> createISelMutator() { + std::vector<TypeGetter> Types{ + Type::getInt1Ty, Type::getInt8Ty, Type::getInt16Ty, Type::getInt32Ty, + Type::getInt64Ty, Type::getFloatTy, Type::getDoubleTy}; + + std::vector<std::unique_ptr<IRMutationStrategy>> Strategies; + Strategies.emplace_back( + new InjectorIRStrategy(InjectorIRStrategy::getDefaultOps())); + Strategies.emplace_back(new InstDeleterIRStrategy()); + + return llvm::make_unique<IRMutator>(std::move(Types), std::move(Strategies)); +} + +extern "C" LLVM_ATTRIBUTE_USED size_t LLVMFuzzerCustomMutator( + uint8_t *Data, size_t Size, size_t MaxSize, unsigned int Seed) { + LLVMContext Context; + std::unique_ptr<Module> M; + if (Size <= 1) + // We get bogus data given an empty corpus - just create a new module. + M.reset(new Module("M", Context)); + else + M = parseModule(Data, Size, Context); + + Mutator->mutateModule(*M, Seed, Size, MaxSize); + + return writeModule(*M, Data, MaxSize); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + if (Size <= 1) + // We get bogus data given an empty corpus - ignore it. + return 0; + + LLVMContext Context; + auto M = parseModule(Data, Size, Context); + if (!M || verifyModule(*M, &errs())) { + errs() << "error: input module is broken!\n"; + return 0; + } + + // Set up the module to build for our target. + M->setTargetTriple(TM->getTargetTriple().normalize()); + M->setDataLayout(TM->createDataLayout()); + + // Build up a PM to do instruction selection. + legacy::PassManager PM; + TargetLibraryInfoImpl TLII(TM->getTargetTriple()); + PM.add(new TargetLibraryInfoWrapperPass(TLII)); + raw_null_ostream OS; + TM->addPassesToEmitFile(PM, OS, TargetMachine::CGFT_Null); + PM.run(*M); + + return 0; +} + +static void handleLLVMFatalError(void *, const std::string &Message, bool) { + // TODO: Would it be better to call into the fuzzer internals directly? + dbgs() << "LLVM ERROR: " << Message << "\n" + << "Aborting to trigger fuzzer exit handling.\n"; + abort(); +} + +extern "C" LLVM_ATTRIBUTE_USED int LLVMFuzzerInitialize(int *argc, + char ***argv) { + EnableDebugBuffering = true; + + InitializeAllTargets(); + InitializeAllTargetMCs(); + InitializeAllAsmPrinters(); + InitializeAllAsmParsers(); + + handleExecNameEncodedBEOpts(*argv[0]); + parseFuzzerCLOpts(*argc, *argv); + + if (TargetTriple.empty()) { + errs() << *argv[0] << ": -mtriple must be specified\n"; + exit(1); + } + + Triple TheTriple = Triple(Triple::normalize(TargetTriple)); + + // Get the target specific parser. + std::string Error; + const Target *TheTarget = + TargetRegistry::lookupTarget(MArch, TheTriple, Error); + if (!TheTarget) { + errs() << argv[0] << ": " << Error; + return 1; + } + + // Set up the pipeline like llc does. + std::string CPUStr = getCPUStr(), FeaturesStr = getFeaturesStr(); + + CodeGenOpt::Level OLvl = CodeGenOpt::Default; + switch (OptLevel) { + default: + errs() << argv[0] << ": invalid optimization level.\n"; + return 1; + case ' ': break; + case '0': OLvl = CodeGenOpt::None; break; + case '1': OLvl = CodeGenOpt::Less; break; + case '2': OLvl = CodeGenOpt::Default; break; + case '3': OLvl = CodeGenOpt::Aggressive; break; + } + + TargetOptions Options = InitTargetOptionsFromCodeGenFlags(); + TM.reset(TheTarget->createTargetMachine(TheTriple.getTriple(), CPUStr, + FeaturesStr, Options, getRelocModel(), + getCodeModel(), OLvl)); + assert(TM && "Could not allocate target machine!"); + + // Make sure we print the summary and the current unit when LLVM errors out. + install_fatal_error_handler(handleLLVMFatalError, nullptr); + + // Finally, create our mutator. + Mutator = createISelMutator(); + return 0; +} diff --git a/tools/llvm-link/llvm-link.cpp b/tools/llvm-link/llvm-link.cpp index 568e5f8d2d58..50f506aeaae9 100644 --- a/tools/llvm-link/llvm-link.cpp +++ b/tools/llvm-link/llvm-link.cpp @@ -182,25 +182,30 @@ Module &ModuleLazyLoaderCache::operator()(const char *argv0, } } // anonymous namespace -static void diagnosticHandler(const DiagnosticInfo &DI, void *C) { - unsigned Severity = DI.getSeverity(); - switch (Severity) { - case DS_Error: - errs() << "ERROR: "; - break; - case DS_Warning: - if (SuppressWarnings) - return; - errs() << "WARNING: "; - break; - case DS_Remark: - case DS_Note: - llvm_unreachable("Only expecting warnings and errors"); - } +namespace { +struct LLVMLinkDiagnosticHandler : public DiagnosticHandler { + bool handleDiagnostics(const DiagnosticInfo &DI) override { + unsigned Severity = DI.getSeverity(); + switch (Severity) { + case DS_Error: + errs() << "ERROR: "; + break; + case DS_Warning: + if (SuppressWarnings) + return true; + errs() << "WARNING: "; + break; + case DS_Remark: + case DS_Note: + llvm_unreachable("Only expecting warnings and errors"); + } - DiagnosticPrinterRawOStream DP(errs()); - DI.print(DP); - errs() << '\n'; + DiagnosticPrinterRawOStream DP(errs()); + DI.print(DP); + errs() << '\n'; + return true; + } +}; } /// Import any functions requested via the -import option. @@ -347,8 +352,8 @@ int main(int argc, char **argv) { ExitOnErr.setBanner(std::string(argv[0]) + ": "); LLVMContext Context; - Context.setDiagnosticHandler(diagnosticHandler, nullptr, true); - + Context.setDiagnosticHandler( + llvm::make_unique<LLVMLinkDiagnosticHandler>(), true); llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. cl::ParseCommandLineOptions(argc, argv, "llvm linker\n"); @@ -378,7 +383,7 @@ int main(int argc, char **argv) { if (DumpAsm) errs() << "Here's the assembly:\n" << *Composite; std::error_code EC; - tool_output_file Out(OutputFilename, EC, sys::fs::F_None); + ToolOutputFile Out(OutputFilename, EC, sys::fs::F_None); if (EC) { errs() << EC.message() << '\n'; return 1; diff --git a/tools/llvm-lto/llvm-lto.cpp b/tools/llvm-lto/llvm-lto.cpp index 87cd13ad70de..20c6813968be 100644 --- a/tools/llvm-lto/llvm-lto.cpp +++ b/tools/llvm-lto/llvm-lto.cpp @@ -1,4 +1,4 @@ -//===-- llvm-lto: a simple command-line program to link modules with LTO --===// +//===- llvm-lto: a simple command-line program to link modules with LTO ---===// // // The LLVM Compiler Infrastructure // @@ -12,20 +12,36 @@ // //===----------------------------------------------------------------------===// +#include "llvm-c/lto.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" +#include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" -#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/CommandFlags.def" +#include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/Verifier.h" #include "llvm/IRReader/IRReader.h" #include "llvm/LTO/legacy/LTOCodeGenerator.h" #include "llvm/LTO/legacy/LTOModule.h" #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/Signals.h" @@ -33,7 +49,19 @@ #include "llvm/Support/TargetSelect.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetOptions.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <cstdlib> #include <list> +#include <map> +#include <memory> +#include <string> +#include <system_error> +#include <tuple> +#include <utility> +#include <vector> using namespace llvm; @@ -179,10 +207,12 @@ static cl::opt<bool> CheckHasObjC( cl::desc("Only check if the module has objective-C defined in it")); namespace { + struct ModuleInfo { std::vector<bool> CanBeHidden; }; -} + +} // end anonymous namespace static void handleDiagnostics(lto_codegen_diagnostic_severity_t Severity, const char *Msg, void *) { @@ -205,34 +235,40 @@ static void handleDiagnostics(lto_codegen_diagnostic_severity_t Severity, } static std::string CurrentActivity; -static void diagnosticHandler(const DiagnosticInfo &DI, void *Context) { - raw_ostream &OS = errs(); - OS << "llvm-lto: "; - switch (DI.getSeverity()) { - case DS_Error: - OS << "error"; - break; - case DS_Warning: - OS << "warning"; - break; - case DS_Remark: - OS << "remark"; - break; - case DS_Note: - OS << "note"; - break; - } - if (!CurrentActivity.empty()) - OS << ' ' << CurrentActivity; - OS << ": "; - - DiagnosticPrinterRawOStream DP(OS); - DI.print(DP); - OS << '\n'; - if (DI.getSeverity() == DS_Error) - exit(1); -} +namespace { + struct LLVMLTODiagnosticHandler : public DiagnosticHandler { + bool handleDiagnostics(const DiagnosticInfo &DI) override { + raw_ostream &OS = errs(); + OS << "llvm-lto: "; + switch (DI.getSeverity()) { + case DS_Error: + OS << "error"; + break; + case DS_Warning: + OS << "warning"; + break; + case DS_Remark: + OS << "remark"; + break; + case DS_Note: + OS << "note"; + break; + } + if (!CurrentActivity.empty()) + OS << ' ' << CurrentActivity; + OS << ": "; + + DiagnosticPrinterRawOStream DP(OS); + DI.print(DP); + OS << '\n'; + + if (DI.getSeverity() == DS_Error) + exit(1); + return true; + } + }; + } static void error(const Twine &Msg) { errs() << "llvm-lto: " << Msg << '\n'; @@ -263,7 +299,8 @@ getLocalLTOModule(StringRef Path, std::unique_ptr<MemoryBuffer> &Buffer, Buffer = std::move(BufferOrErr.get()); CurrentActivity = ("loading file '" + Path + "'").str(); std::unique_ptr<LLVMContext> Context = llvm::make_unique<LLVMContext>(); - Context->setDiagnosticHandler(diagnosticHandler, nullptr, true); + Context->setDiagnosticHandler(llvm::make_unique<LLVMLTODiagnosticHandler>(), + true); ErrorOr<std::unique_ptr<LTOModule>> Ret = LTOModule::createInLocalContext( std::move(Context), Buffer->getBufferStart(), Buffer->getBufferSize(), Options, Path); @@ -277,7 +314,7 @@ void printIndexStats() { for (auto &Filename : InputFilenames) { ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename + "': "); std::unique_ptr<ModuleSummaryIndex> Index = - ExitOnErr(llvm::getModuleSummaryIndexForFile(Filename)); + ExitOnErr(getModuleSummaryIndexForFile(Filename)); // Skip files without a module summary. if (!Index) report_fatal_error(Filename + " does not contain an index"); @@ -396,7 +433,7 @@ std::unique_ptr<ModuleSummaryIndex> loadCombinedIndex() { report_fatal_error("Missing -thinlto-index for ThinLTO promotion stage"); ExitOnError ExitOnErr("llvm-lto: error loading file '" + ThinLTOIndex + "': "); - return ExitOnErr(llvm::getModuleSummaryIndexForFile(ThinLTOIndex)); + return ExitOnErr(getModuleSummaryIndexForFile(ThinLTOIndex)); } static std::unique_ptr<Module> loadModule(StringRef Filename, @@ -489,7 +526,6 @@ private: raw_fd_ostream OS(OutputFilename, EC, sys::fs::OpenFlags::F_None); error(EC, "error opening the file '" + OutputFilename + "'"); WriteIndexToFile(*CombinedIndex, OS); - return; } /// Load the combined index from disk, then compute and generate @@ -745,7 +781,7 @@ private: /// Load the combined index from disk, then load every file referenced by }; -} // namespace thinlto +} // end namespace thinlto int main(int argc, char **argv) { // Print a stack trace if we signal out. @@ -784,7 +820,7 @@ int main(int argc, char **argv) { std::unique_ptr<MemoryBuffer> BufferOrErr = ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(Filename))); auto Buffer = std::move(BufferOrErr.get()); - if (ExitOnErr(llvm::isBitcodeContainingObjCCategory(*Buffer))) + if (ExitOnErr(isBitcodeContainingObjCCategory(*Buffer))) outs() << "Bitcode " << Filename << " contains ObjC\n"; else outs() << "Bitcode " << Filename << " does not contain ObjC\n"; @@ -808,7 +844,8 @@ int main(int argc, char **argv) { unsigned BaseArg = 0; LLVMContext Context; - Context.setDiagnosticHandler(diagnosticHandler, nullptr, true); + Context.setDiagnosticHandler(llvm::make_unique<LLVMLTODiagnosticHandler>(), + true); LTOCodeGenerator CodeGen(Context); @@ -822,7 +859,7 @@ int main(int argc, char **argv) { CodeGen.setTargetOptions(Options); CodeGen.setShouldRestoreGlobalsLinkage(RestoreGlobalsLinkage); - llvm::StringSet<llvm::MallocAllocator> DSOSymbolsSet; + StringSet<MallocAllocator> DSOSymbolsSet; for (unsigned i = 0; i < DSOSymbols.size(); ++i) DSOSymbolsSet.insert(DSOSymbols[i]); @@ -899,7 +936,7 @@ int main(int argc, char **argv) { error("writing merged module failed."); } - std::list<tool_output_file> OSs; + std::list<ToolOutputFile> OSs; std::vector<raw_pwrite_stream *> OSPtrs; for (unsigned I = 0; I != Parallelism; ++I) { std::string PartFilename = OutputFilename; @@ -916,7 +953,7 @@ int main(int argc, char **argv) { // Diagnostic messages should have been printed by the handler. error("error compiling the code"); - for (tool_output_file &OS : OSs) + for (ToolOutputFile &OS : OSs) OS.keep(); } else { if (Parallelism != 1) diff --git a/tools/llvm-lto2/llvm-lto2.cpp b/tools/llvm-lto2/llvm-lto2.cpp index 5426e040cd7c..70aae0f41507 100644 --- a/tools/llvm-lto2/llvm-lto2.cpp +++ b/tools/llvm-lto2/llvm-lto2.cpp @@ -17,7 +17,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Bitcode/BitcodeReader.h" -#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/CommandFlags.def" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/LTO/Caching.h" #include "llvm/LTO/LTO.h" @@ -100,11 +100,19 @@ static cl::opt<bool> OptRemarksWithHotness( cl::desc("Whether to include hotness informations in the remarks.\n" "Has effect only if -pass-remarks-output is specified.")); +static cl::opt<std::string> + SamplePGOFile("lto-sample-profile-file", + cl::desc("Specify a SamplePGO profile file")); + static cl::opt<bool> UseNewPM("use-new-pm", cl::desc("Run LTO passes using the new pass manager"), cl::init(false), cl::Hidden); +static cl::opt<bool> + DebugPassManager("debug-pass-manager", cl::init(false), cl::Hidden, + cl::desc("Print pass management debugging information")); + static void check(Error E, std::string Msg) { if (!E) return; @@ -189,7 +197,9 @@ static int run(int argc, char **argv) { Conf.MAttrs = MAttrs; if (auto RM = getRelocModel()) Conf.RelocModel = *RM; - Conf.CodeModel = CMModel; + Conf.CodeModel = getCodeModel(); + + Conf.DebugPassManager = DebugPassManager; if (SaveTemps) check(Conf.addSaveTemps(OutputFilename + "."), @@ -199,6 +209,8 @@ static int run(int argc, char **argv) { Conf.RemarksFilename = OptRemarksOutput; Conf.RemarksWithHotness = OptRemarksWithHotness; + Conf.SampleProfile = SamplePGOFile; + // Run a custom pipeline, if asked for. Conf.OptPipeline = OptPipeline; Conf.AAPipeline = AAPipeline; @@ -284,7 +296,8 @@ static int run(int argc, char **argv) { return llvm::make_unique<lto::NativeObjectStream>(std::move(S)); }; - auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) { + auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB, + StringRef Path) { *AddStream(Task)->OS << MB->getBuffer(); }; @@ -355,6 +368,9 @@ static int dumpSymtab(int argc, char **argv) { if (TT.isOSBinFormatCOFF() && Sym.isWeak() && Sym.isIndirect()) outs() << " fallback " << Sym.getCOFFWeakExternalFallback() << '\n'; + + if (!Sym.getSectionName().empty()) + outs() << " section " << Sym.getSectionName() << "\n"; } outs() << '\n'; diff --git a/tools/llvm-mc-assemble-fuzzer/CMakeLists.txt b/tools/llvm-mc-assemble-fuzzer/CMakeLists.txt index c5fb62166cfd..3545d53503b7 100644 --- a/tools/llvm-mc-assemble-fuzzer/CMakeLists.txt +++ b/tools/llvm-mc-assemble-fuzzer/CMakeLists.txt @@ -1,19 +1,10 @@ -if( LLVM_USE_SANITIZE_COVERAGE ) - include_directories(BEFORE - ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/Fuzzer) - - set(LLVM_LINK_COMPONENTS - AllTargetsAsmPrinters - AllTargetsAsmParsers - AllTargetsDescs - AllTargetsInfos - MC - MCParser - Support - ) - add_llvm_tool(llvm-mc-assemble-fuzzer - llvm-mc-assemble-fuzzer.cpp) - target_link_libraries(llvm-mc-assemble-fuzzer - LLVMFuzzer - ) -endif() +set(LLVM_LINK_COMPONENTS + AllTargetsAsmPrinters + AllTargetsAsmParsers + AllTargetsDescs + AllTargetsInfos + MC + MCParser + Support +) +add_llvm_fuzzer(llvm-mc-assemble-fuzzer llvm-mc-assemble-fuzzer.cpp) diff --git a/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp b/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp index 0344d8cd8c9a..efbf3bc29232 100644 --- a/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp +++ b/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp @@ -9,11 +9,11 @@ // //===----------------------------------------------------------------------===// -#include "FuzzerInterface.h" #include "llvm-c/Target.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInstPrinter.h" #include "llvm/MC/MCInstrInfo.h" @@ -24,7 +24,7 @@ #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptionsCommandFlags.h" +#include "llvm/MC/MCTargetOptionsCommandFlags.def" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileUtilities.h" @@ -85,7 +85,7 @@ class LLVMFuzzerInputBuffer : public MemoryBuffer { public: LLVMFuzzerInputBuffer(const uint8_t *data_, size_t size_) - : Data(reinterpret_cast<const char *>(data_)), + : Data(reinterpret_cast<const char *>(data_)), Size(size_) { init(Data, Data+Size, false); } @@ -172,8 +172,7 @@ int AssembleOneInput(const uint8_t *Data, size_t Size) { MCContext Ctx(MAI.get(), MRI.get(), &MOFI, &SrcMgr); static const bool UsePIC = false; - static const CodeModel::Model CMModel = CodeModel::Default; - MOFI.InitMCObjectFileInfo(TheTriple, UsePIC, CMModel, Ctx); + MOFI.InitMCObjectFileInfo(TheTriple, UsePIC, Ctx); const unsigned OutputAsmVariant = 0; std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo()); @@ -211,8 +210,8 @@ int AssembleOneInput(const uint8_t *Data, size_t Size) { std::error_code EC; const std::string OutputFilename = "-"; - auto Out = llvm::make_unique<tool_output_file>(OutputFilename, EC, - sys::fs::F_None); + auto Out = + llvm::make_unique<ToolOutputFile>(OutputFilename, EC, sys::fs::F_None); if (EC) { errs() << EC.message() << '\n'; abort(); @@ -232,7 +231,8 @@ int AssembleOneInput(const uint8_t *Data, size_t Size) { MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*MRI, TripleName, MCPU, MCOptions); Str.reset(TheTarget->createMCObjectStreamer( - TheTriple, Ctx, *MAB, *OS, CE, *STI, MCOptions.MCRelaxAll, + TheTriple, Ctx, std::unique_ptr<MCAsmBackend>(MAB), *OS, + std::unique_ptr<MCCodeEmitter>(CE), *STI, MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible, /*DWARFMustBeAtTheEnd*/ false)); } @@ -244,11 +244,12 @@ int AssembleOneInput(const uint8_t *Data, size_t Size) { return 0; } -int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { return AssembleOneInput(Data, Size); } -int LLVMFuzzerInitialize(int *argc, char ***argv) { +extern "C" LLVM_ATTRIBUTE_USED int LLVMFuzzerInitialize(int *argc, + char ***argv) { // The command line is unusual compared to other fuzzers due to the need to // specify the target. Options like -triple, -mcpu, and -mattr work like // their counterparts in llvm-mc, while -fuzzer-args collects options for the diff --git a/tools/llvm-mc-disassemble-fuzzer/CMakeLists.txt b/tools/llvm-mc-disassemble-fuzzer/CMakeLists.txt index c539f823e57f..60b08062d09f 100644 --- a/tools/llvm-mc-disassemble-fuzzer/CMakeLists.txt +++ b/tools/llvm-mc-disassemble-fuzzer/CMakeLists.txt @@ -1,21 +1,11 @@ -if( LLVM_USE_SANITIZE_COVERAGE ) - include_directories(BEFORE - ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/Fuzzer) - - set(LLVM_LINK_COMPONENTS - AllTargetsAsmPrinters - AllTargetsDescs - AllTargetsDisassemblers - AllTargetsInfos - MC - MCDisassembler - MCParser - Support - ) - add_llvm_tool(llvm-mc-disassemble-fuzzer - llvm-mc-disassemble-fuzzer.cpp) - - target_link_libraries(llvm-mc-disassemble-fuzzer - LLVMFuzzer - ) -endif() +set(LLVM_LINK_COMPONENTS + AllTargetsAsmPrinters + AllTargetsDescs + AllTargetsDisassemblers + AllTargetsInfos + MC + MCDisassembler + MCParser + Support +) +add_llvm_fuzzer(llvm-mc-disassemble-fuzzer llvm-mc-disassemble-fuzzer.cpp) diff --git a/tools/llvm-mc-disassemble-fuzzer/llvm-mc-disassemble-fuzzer.cpp b/tools/llvm-mc-disassemble-fuzzer/llvm-mc-disassemble-fuzzer.cpp index 643afe64073e..36d1f7ca9948 100644 --- a/tools/llvm-mc-disassemble-fuzzer/llvm-mc-disassemble-fuzzer.cpp +++ b/tools/llvm-mc-disassemble-fuzzer/llvm-mc-disassemble-fuzzer.cpp @@ -9,7 +9,6 @@ // //===----------------------------------------------------------------------===// -#include "FuzzerInterface.h" #include "llvm-c/Disassembler.h" #include "llvm-c/Target.h" #include "llvm/MC/SubtargetFeature.h" @@ -74,11 +73,12 @@ int DisassembleOneInput(const uint8_t *Data, size_t Size) { return 0; } -int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { return DisassembleOneInput(Data, Size); } -int LLVMFuzzerInitialize(int *argc, char ***argv) { +extern "C" LLVM_ATTRIBUTE_USED int LLVMFuzzerInitialize(int *argc, + char ***argv) { // The command line is unusual compared to other fuzzers due to the need to // specify the target. Options like -triple, -mcpu, and -mattr work like // their counterparts in llvm-mc, while -fuzzer-args collects options for the diff --git a/tools/llvm-mc/llvm-mc.cpp b/tools/llvm-mc/llvm-mc.cpp index 8782588dfdd8..e925346eb2d1 100644 --- a/tools/llvm-mc/llvm-mc.cpp +++ b/tools/llvm-mc/llvm-mc.cpp @@ -15,6 +15,7 @@ #include "Disassembler.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInstPrinter.h" #include "llvm/MC/MCInstrInfo.h" @@ -22,10 +23,9 @@ #include "llvm/MC/MCParser/AsmLexer.h" #include "llvm/MC/MCParser/MCTargetAsmParser.h" #include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptionsCommandFlags.h" +#include "llvm/MC/MCTargetOptionsCommandFlags.def" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compression.h" #include "llvm/Support/FileUtilities.h" @@ -131,20 +131,10 @@ MAttrs("mattr", static cl::opt<bool> PIC("position-independent", cl::desc("Position independent"), cl::init(false)); -static cl::opt<llvm::CodeModel::Model> -CMModel("code-model", - cl::desc("Choose code model"), - cl::init(CodeModel::Default), - cl::values(clEnumValN(CodeModel::Default, "default", - "Target default code model"), - clEnumValN(CodeModel::Small, "small", - "Small code model"), - clEnumValN(CodeModel::Kernel, "kernel", - "Kernel code model"), - clEnumValN(CodeModel::Medium, "medium", - "Medium code model"), - clEnumValN(CodeModel::Large, "large", - "Large code model"))); +static cl::opt<bool> + LargeCodeModel("large-code-model", + cl::desc("Create cfi directives that assume the code might " + "be more than 2gb away")); static cl::opt<bool> NoInitialTextSection("n", cl::desc("Don't assume assembly file starts " @@ -207,13 +197,13 @@ static const Target *GetTarget(const char *ProgName) { return TheTarget; } -static std::unique_ptr<tool_output_file> GetOutputStream() { +static std::unique_ptr<ToolOutputFile> GetOutputStream() { if (OutputFilename == "") OutputFilename = "-"; std::error_code EC; - auto Out = llvm::make_unique<tool_output_file>(OutputFilename, EC, - sys::fs::F_None); + auto Out = + llvm::make_unique<ToolOutputFile>(OutputFilename, EC, sys::fs::F_None); if (EC) { errs() << EC.message() << '\n'; return nullptr; @@ -506,7 +496,7 @@ int main(int argc, char **argv) { // MCObjectFileInfo needs a MCContext reference in order to initialize itself. MCObjectFileInfo MOFI; MCContext Ctx(MAI.get(), MRI.get(), &MOFI, &SrcMgr); - MOFI.InitMCObjectFileInfo(TheTriple, PIC, CMModel, Ctx); + MOFI.InitMCObjectFileInfo(TheTriple, PIC, Ctx, LargeCodeModel); if (SaveTempLabels) Ctx.setAllowTemporaryLabels(false); @@ -544,7 +534,7 @@ int main(int argc, char **argv) { FeaturesStr = Features.getString(); } - std::unique_ptr<tool_output_file> Out = GetOutputStream(); + std::unique_ptr<ToolOutputFile> Out = GetOutputStream(); if (!Out) return 1; @@ -601,7 +591,8 @@ int main(int argc, char **argv) { MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*MRI, TripleName, MCPU, MCOptions); Str.reset(TheTarget->createMCObjectStreamer( - TheTriple, Ctx, *MAB, *OS, CE, *STI, MCOptions.MCRelaxAll, + TheTriple, Ctx, std::unique_ptr<MCAsmBackend>(MAB), *OS, + std::unique_ptr<MCCodeEmitter>(CE), *STI, MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible, /*DWARFMustBeAtTheEnd*/ false)); if (NoExecStack) diff --git a/tools/llvm-mcmarkup/llvm-mcmarkup.cpp b/tools/llvm-mcmarkup/llvm-mcmarkup.cpp index 0be3c715eee4..711493ad8307 100644 --- a/tools/llvm-mcmarkup/llvm-mcmarkup.cpp +++ b/tools/llvm-mcmarkup/llvm-mcmarkup.cpp @@ -12,14 +12,12 @@ //===----------------------------------------------------------------------===// #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Format.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/Signals.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" -#include <system_error> using namespace llvm; static cl::list<std::string> @@ -220,7 +218,6 @@ int main(int argc, char **argv) { if (InputFilenames.size() == 0) InputFilenames.push_back("-"); - std::for_each(InputFilenames.begin(), InputFilenames.end(), - parseMCMarkup); + llvm::for_each(InputFilenames, parseMCMarkup); return 0; } diff --git a/tools/llvm-modextract/llvm-modextract.cpp b/tools/llvm-modextract/llvm-modextract.cpp index 58cede1374ea..b2d21c23a094 100644 --- a/tools/llvm-modextract/llvm-modextract.cpp +++ b/tools/llvm-modextract/llvm-modextract.cpp @@ -54,8 +54,8 @@ int main(int argc, char **argv) { } std::error_code EC; - std::unique_ptr<tool_output_file> Out( - new tool_output_file(OutputFilename, EC, sys::fs::F_None)); + std::unique_ptr<ToolOutputFile> Out( + new ToolOutputFile(OutputFilename, EC, sys::fs::F_None)); ExitOnErr(errorCodeToError(EC)); if (BinaryExtract) { diff --git a/tools/llvm-mt/CMakeLists.txt b/tools/llvm-mt/CMakeLists.txt index d97cc4fe446f..e4e994680921 100644 --- a/tools/llvm-mt/CMakeLists.txt +++ b/tools/llvm-mt/CMakeLists.txt @@ -1,6 +1,7 @@ set(LLVM_LINK_COMPONENTS Option Support + WindowsManifest ) set(LLVM_TARGET_DEFINITIONS Opts.td) diff --git a/tools/llvm-mt/LLVMBuild.txt b/tools/llvm-mt/LLVMBuild.txt index 894d3527924b..02fc9ace7552 100644 --- a/tools/llvm-mt/LLVMBuild.txt +++ b/tools/llvm-mt/LLVMBuild.txt @@ -19,4 +19,4 @@ type = Tool name = llvm-mt parent = Tools -required_libraries = Option Support +required_libraries = Option Support WindowsManifest diff --git a/tools/llvm-mt/llvm-mt.cpp b/tools/llvm-mt/llvm-mt.cpp index 05c9238c7c64..944af22cf9c8 100644 --- a/tools/llvm-mt/llvm-mt.cpp +++ b/tools/llvm-mt/llvm-mt.cpp @@ -16,12 +16,15 @@ #include "llvm/Option/ArgList.h" #include "llvm/Option/Option.h" #include "llvm/Support/Error.h" +#include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/Process.h" #include "llvm/Support/Signals.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/WindowsManifest/WindowsManifestMerger.h" #include <system_error> @@ -67,6 +70,22 @@ LLVM_ATTRIBUTE_NORETURN void reportError(Twine Msg) { exit(1); } +static void reportError(StringRef Input, std::error_code EC) { + reportError(Twine(Input) + ": " + EC.message()); +} + +void error(std::error_code EC) { + if (EC) + reportError(EC.message()); +} + +void error(Error EC) { + if (EC) + handleAllErrors(std::move(EC), [&](const ErrorInfoBase &EI) { + reportError(EI.message()); + }); +} + int main(int argc, const char **argv) { sys::PrintStackTraceOnErrorSignal(argv[0]); PrettyStackTraceProgram X(argc, argv); @@ -77,7 +96,6 @@ int main(int argc, const char **argv) { SpecificBumpPtrAllocator<char> ArgAllocator; ExitOnErr(errorCodeToError(sys::Process::GetArgumentVector( argv_buf, makeArrayRef(argv, argc), ArgAllocator))); - llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. CvtResOptTable T; @@ -85,6 +103,9 @@ int main(int argc, const char **argv) { ArrayRef<const char *> ArgsArr = makeArrayRef(argv + 1, argc); opt::InputArgList InputArgs = T.ParseArgs(ArgsArr, MAI, MAC); + for (auto *Arg : InputArgs.filtered(OPT_INPUT)) + reportError(Twine("invalid option ") + Arg->getSpelling()); + for (auto &Arg : InputArgs) { if (Arg->getOption().matches(OPT_unsupported)) { outs() << "llvm-mt: ignoring unsupported '" << Arg->getOption().getName() @@ -104,7 +125,6 @@ int main(int argc, const char **argv) { } StringRef OutputFile; - if (InputArgs.hasArg(OPT_out)) { OutputFile = InputArgs.getLastArgValue(OPT_out); } else if (InputFiles.size() == 1) { @@ -113,5 +133,27 @@ int main(int argc, const char **argv) { reportError("no output file specified"); } + windows_manifest::WindowsManifestMerger Merger; + + for (const auto &File : InputFiles) { + ErrorOr<std::unique_ptr<MemoryBuffer>> ManifestOrErr = + MemoryBuffer::getFile(File); + if (!ManifestOrErr) + reportError(File, ManifestOrErr.getError()); + MemoryBuffer &Manifest = *ManifestOrErr.get(); + error(Merger.merge(Manifest)); + } + + std::unique_ptr<MemoryBuffer> OutputBuffer = Merger.getMergedManifest(); + if (!OutputBuffer) + reportError("empty manifest not written"); + Expected<std::unique_ptr<FileOutputBuffer>> FileOrErr = + FileOutputBuffer::create(OutputFile, OutputBuffer->getBufferSize()); + if (!FileOrErr) + reportError(OutputFile, errorToErrorCode(FileOrErr.takeError())); + std::unique_ptr<FileOutputBuffer> FileBuffer = std::move(*FileOrErr); + std::copy(OutputBuffer->getBufferStart(), OutputBuffer->getBufferEnd(), + FileBuffer->getBufferStart()); + error(FileBuffer->commit()); return 0; } diff --git a/tools/llvm-nm/CMakeLists.txt b/tools/llvm-nm/CMakeLists.txt index 08bcd5f30898..f093cc4328ae 100644 --- a/tools/llvm-nm/CMakeLists.txt +++ b/tools/llvm-nm/CMakeLists.txt @@ -14,3 +14,7 @@ add_llvm_tool(llvm-nm DEPENDS intrinsics_gen ) + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(nm llvm-nm) +endif() diff --git a/tools/llvm-nm/llvm-nm.cpp b/tools/llvm-nm/llvm-nm.cpp index ea47891250f7..b6ac9c20a946 100644 --- a/tools/llvm-nm/llvm-nm.cpp +++ b/tools/llvm-nm/llvm-nm.cpp @@ -20,10 +20,7 @@ #include "llvm/BinaryFormat/COFF.h" #include "llvm/Demangle/Demangle.h" #include "llvm/IR/Function.h" -#include "llvm/IR/GlobalAlias.h" -#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" #include "llvm/Object/Archive.h" #include "llvm/Object/COFF.h" #include "llvm/Object/COFFImportFile.h" @@ -43,13 +40,7 @@ #include "llvm/Support/Signals.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <cctype> -#include <cerrno> -#include <cstring> -#include <system_error> #include <vector> -#include <string.h> using namespace llvm; using namespace object; @@ -85,9 +76,11 @@ cl::alias DefinedOnly2("U", cl::desc("Alias for --defined-only"), cl::aliasopt(DefinedOnly), cl::Grouping); cl::opt<bool> ExternalOnly("extern-only", - cl::desc("Show only external symbols")); + cl::desc("Show only external symbols"), + cl::ZeroOrMore); cl::alias ExternalOnly2("g", cl::desc("Alias for --extern-only"), - cl::aliasopt(ExternalOnly), cl::Grouping); + cl::aliasopt(ExternalOnly), cl::Grouping, + cl::ZeroOrMore); cl::opt<bool> BSDFormat("B", cl::desc("Alias for --format=bsd"), cl::Grouping); @@ -486,6 +479,10 @@ static void darwinPrintSymbol(SymbolicFile &Obj, SymbolListT::iterator I, break; } Sec = *SecOrErr; + if (Sec == MachO->section_end()) { + outs() << "(?,?) "; + break; + } } else { Sec = I->Section; } @@ -709,9 +706,13 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName, } else if (OutputFormat == bsd && MultipleFiles && printName) { outs() << "\n" << CurrentFilename << ":\n"; } else if (OutputFormat == sysv) { - outs() << "\n\nSymbols from " << CurrentFilename << ":\n\n" - << "Name Value Class Type" - << " Size Line Section\n"; + outs() << "\n\nSymbols from " << CurrentFilename << ":\n\n"; + if (isSymbolList64Bit(Obj)) + outs() << "Name Value Class Type" + << " Size Line Section\n"; + else + outs() << "Name Value Class Type" + << " Size Line Section\n"; } } @@ -938,6 +939,10 @@ static char getSymbolNMTypeChar(COFFObjectFile &Obj, symbol_iterator I) { section_iterator SecI = *SecIOrErr; const coff_section *Section = Obj.getCOFFSection(*SecI); Characteristics = Section->Characteristics; + StringRef SectionName; + Obj.getSectionName(Section, SectionName); + if (SectionName.startswith(".idata")) + return 'i'; } switch (Symb.getSectionNumber()) { @@ -993,6 +998,8 @@ static char getSymbolNMTypeChar(MachOObjectFile &Obj, basic_symbol_iterator I) { return 's'; } section_iterator Sec = *SecOrErr; + if (Sec == Obj.section_end()) + return 's'; DataRefImpl Ref = Sec->getRawDataRefImpl(); StringRef SectionName; Obj.getSectionName(Ref, SectionName); @@ -1226,7 +1233,8 @@ dumpSymbolNamesFromObject(SymbolicFile &Obj, bool printName, if (DyldInfoOnly || AddDyldInfo || HFlags & MachO::MH_NLIST_OUTOFSYNC_WITH_DYLDINFO) { unsigned ExportsAdded = 0; - for (const llvm::object::ExportEntry &Entry : MachO->exports()) { + Error Err = Error::success(); + for (const llvm::object::ExportEntry &Entry : MachO->exports(Err)) { bool found = false; bool ReExport = false; if (!DyldInfoOnly) { @@ -1362,6 +1370,8 @@ dumpSymbolNamesFromObject(SymbolicFile &Obj, bool printName, } } } + if (Err) + error(std::move(Err), MachO->getFileName()); // Set the symbol names and indirect names for the added symbols. if (ExportsAdded) { EOS.flush(); @@ -1958,8 +1968,7 @@ int main(int argc, char **argv) { if (NoDyldInfo && (AddDyldInfo || DyldInfoOnly)) error("-no-dyldinfo can't be used with -add-dyldinfo or -dyldinfo-only"); - std::for_each(InputFilenames.begin(), InputFilenames.end(), - dumpSymbolNamesFromFile); + llvm::for_each(InputFilenames, dumpSymbolNamesFromFile); if (HadError) return 1; diff --git a/tools/llvm-objcopy/CMakeLists.txt b/tools/llvm-objcopy/CMakeLists.txt new file mode 100644 index 000000000000..05aa727ab9d8 --- /dev/null +++ b/tools/llvm-objcopy/CMakeLists.txt @@ -0,0 +1,13 @@ +set(LLVM_LINK_COMPONENTS + Object + Support + MC + ) +add_llvm_tool(llvm-objcopy + llvm-objcopy.cpp + Object.cpp + ) + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(objcopy llvm-objcopy) +endif() diff --git a/tools/llvm-objcopy/LLVMBuild.txt b/tools/llvm-objcopy/LLVMBuild.txt new file mode 100644 index 000000000000..0a3473a222b0 --- /dev/null +++ b/tools/llvm-objcopy/LLVMBuild.txt @@ -0,0 +1,21 @@ +;===- ./tools/llvm-objcopy/LLVMBuild.txt -----------------------*- Conf -*--===; +; +; The LLVM Compiler Infrastructure +; +; This file is distributed under the University of Illinois Open Source +; License. See LICENSE.TXT for details. +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; +[component_0] +type = Tool +name = llvm-objcopy +parent = Tools +required_libraries = Object Support MC diff --git a/tools/llvm-objcopy/Object.cpp b/tools/llvm-objcopy/Object.cpp new file mode 100644 index 000000000000..bd5bcd7fc188 --- /dev/null +++ b/tools/llvm-objcopy/Object.cpp @@ -0,0 +1,936 @@ +//===- Object.cpp ---------------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "Object.h" +#include "llvm-objcopy.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileOutputBuffer.h" +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <utility> +#include <vector> + +using namespace llvm; +using namespace object; +using namespace ELF; + +template <class ELFT> void Segment::writeHeader(FileOutputBuffer &Out) const { + using Elf_Ehdr = typename ELFT::Ehdr; + using Elf_Phdr = typename ELFT::Phdr; + + uint8_t *Buf = Out.getBufferStart(); + Buf += sizeof(Elf_Ehdr) + Index * sizeof(Elf_Phdr); + Elf_Phdr &Phdr = *reinterpret_cast<Elf_Phdr *>(Buf); + Phdr.p_type = Type; + Phdr.p_flags = Flags; + Phdr.p_offset = Offset; + Phdr.p_vaddr = VAddr; + Phdr.p_paddr = PAddr; + Phdr.p_filesz = FileSize; + Phdr.p_memsz = MemSize; + Phdr.p_align = Align; +} + +void Segment::writeSegment(FileOutputBuffer &Out) const { + uint8_t *Buf = Out.getBufferStart() + Offset; + // We want to maintain segments' interstitial data and contents exactly. + // This lets us just copy segments directly. + std::copy(std::begin(Contents), std::end(Contents), Buf); +} + +void SectionBase::removeSectionReferences(const SectionBase *Sec) {} +void SectionBase::initialize(SectionTableRef SecTable) {} +void SectionBase::finalize() {} + +template <class ELFT> +void SectionBase::writeHeader(FileOutputBuffer &Out) const { + uint8_t *Buf = Out.getBufferStart(); + Buf += HeaderOffset; + typename ELFT::Shdr &Shdr = *reinterpret_cast<typename ELFT::Shdr *>(Buf); + Shdr.sh_name = NameIndex; + Shdr.sh_type = Type; + Shdr.sh_flags = Flags; + Shdr.sh_addr = Addr; + Shdr.sh_offset = Offset; + Shdr.sh_size = Size; + Shdr.sh_link = Link; + Shdr.sh_info = Info; + Shdr.sh_addralign = Align; + Shdr.sh_entsize = EntrySize; +} + +void Section::writeSection(FileOutputBuffer &Out) const { + if (Type == SHT_NOBITS) + return; + uint8_t *Buf = Out.getBufferStart() + Offset; + std::copy(std::begin(Contents), std::end(Contents), Buf); +} + +void StringTableSection::addString(StringRef Name) { + StrTabBuilder.add(Name); + Size = StrTabBuilder.getSize(); +} + +uint32_t StringTableSection::findIndex(StringRef Name) const { + return StrTabBuilder.getOffset(Name); +} + +void StringTableSection::finalize() { StrTabBuilder.finalize(); } + +void StringTableSection::writeSection(FileOutputBuffer &Out) const { + StrTabBuilder.write(Out.getBufferStart() + Offset); +} + +static bool isValidReservedSectionIndex(uint16_t Index, uint16_t Machine) { + switch (Index) { + case SHN_ABS: + case SHN_COMMON: + return true; + } + if (Machine == EM_HEXAGON) { + switch (Index) { + case SHN_HEXAGON_SCOMMON: + case SHN_HEXAGON_SCOMMON_2: + case SHN_HEXAGON_SCOMMON_4: + case SHN_HEXAGON_SCOMMON_8: + return true; + } + } + return false; +} + +uint16_t Symbol::getShndx() const { + if (DefinedIn != nullptr) { + return DefinedIn->Index; + } + switch (ShndxType) { + // This means that we don't have a defined section but we do need to + // output a legitimate section index. + case SYMBOL_SIMPLE_INDEX: + return SHN_UNDEF; + case SYMBOL_ABS: + case SYMBOL_COMMON: + case SYMBOL_HEXAGON_SCOMMON: + case SYMBOL_HEXAGON_SCOMMON_2: + case SYMBOL_HEXAGON_SCOMMON_4: + case SYMBOL_HEXAGON_SCOMMON_8: + return static_cast<uint16_t>(ShndxType); + } + llvm_unreachable("Symbol with invalid ShndxType encountered"); +} + +void SymbolTableSection::addSymbol(StringRef Name, uint8_t Bind, uint8_t Type, + SectionBase *DefinedIn, uint64_t Value, + uint16_t Shndx, uint64_t Sz) { + Symbol Sym; + Sym.Name = Name; + Sym.Binding = Bind; + Sym.Type = Type; + Sym.DefinedIn = DefinedIn; + if (DefinedIn == nullptr) { + if (Shndx >= SHN_LORESERVE) + Sym.ShndxType = static_cast<SymbolShndxType>(Shndx); + else + Sym.ShndxType = SYMBOL_SIMPLE_INDEX; + } + Sym.Value = Value; + Sym.Size = Sz; + Sym.Index = Symbols.size(); + Symbols.emplace_back(llvm::make_unique<Symbol>(Sym)); + Size += this->EntrySize; +} + +void SymbolTableSection::removeSectionReferences(const SectionBase *Sec) { + if (SymbolNames == Sec) { + error("String table " + SymbolNames->Name + + " cannot be removed because it is referenced by the symbol table " + + this->Name); + } + auto Iter = + std::remove_if(std::begin(Symbols), std::end(Symbols), + [=](const SymPtr &Sym) { return Sym->DefinedIn == Sec; }); + Size -= (std::end(Symbols) - Iter) * this->EntrySize; + Symbols.erase(Iter, std::end(Symbols)); +} + +void SymbolTableSection::initialize(SectionTableRef SecTable) { + Size = 0; + setStrTab(SecTable.getSectionOfType<StringTableSection>( + Link, + "Symbol table has link index of " + Twine(Link) + + " which is not a valid index", + "Symbol table has link index of " + Twine(Link) + + " which is not a string table")); +} + +void SymbolTableSection::finalize() { + // Make sure SymbolNames is finalized before getting name indexes. + SymbolNames->finalize(); + + uint32_t MaxLocalIndex = 0; + for (auto &Sym : Symbols) { + Sym->NameIndex = SymbolNames->findIndex(Sym->Name); + if (Sym->Binding == STB_LOCAL) + MaxLocalIndex = std::max(MaxLocalIndex, Sym->Index); + } + // Now we need to set the Link and Info fields. + Link = SymbolNames->Index; + Info = MaxLocalIndex + 1; +} + +void SymbolTableSection::addSymbolNames() { + // Add all of our strings to SymbolNames so that SymbolNames has the right + // size before layout is decided. + for (auto &Sym : Symbols) + SymbolNames->addString(Sym->Name); +} + +const Symbol *SymbolTableSection::getSymbolByIndex(uint32_t Index) const { + if (Symbols.size() <= Index) + error("Invalid symbol index: " + Twine(Index)); + return Symbols[Index].get(); +} + +template <class ELFT> +void SymbolTableSectionImpl<ELFT>::writeSection(FileOutputBuffer &Out) const { + uint8_t *Buf = Out.getBufferStart(); + Buf += Offset; + typename ELFT::Sym *Sym = reinterpret_cast<typename ELFT::Sym *>(Buf); + // Loop though symbols setting each entry of the symbol table. + for (auto &Symbol : Symbols) { + Sym->st_name = Symbol->NameIndex; + Sym->st_value = Symbol->Value; + Sym->st_size = Symbol->Size; + Sym->setBinding(Symbol->Binding); + Sym->setType(Symbol->Type); + Sym->st_shndx = Symbol->getShndx(); + ++Sym; + } +} + +template <class SymTabType> +void RelocSectionWithSymtabBase<SymTabType>::removeSectionReferences( + const SectionBase *Sec) { + if (Symbols == Sec) { + error("Symbol table " + Symbols->Name + " cannot be removed because it is " + "referenced by the relocation " + "section " + + this->Name); + } +} + +template <class SymTabType> +void RelocSectionWithSymtabBase<SymTabType>::initialize( + SectionTableRef SecTable) { + setSymTab(SecTable.getSectionOfType<SymTabType>( + Link, + "Link field value " + Twine(Link) + " in section " + Name + " is invalid", + "Link field value " + Twine(Link) + " in section " + Name + + " is not a symbol table")); + + if (Info != SHN_UNDEF) + setSection(SecTable.getSection(Info, + "Info field value " + Twine(Info) + + " in section " + Name + " is invalid")); + else + setSection(nullptr); +} + +template <class SymTabType> +void RelocSectionWithSymtabBase<SymTabType>::finalize() { + this->Link = Symbols->Index; + if (SecToApplyRel != nullptr) + this->Info = SecToApplyRel->Index; +} + +template <class ELFT> +void setAddend(Elf_Rel_Impl<ELFT, false> &Rel, uint64_t Addend) {} + +template <class ELFT> +void setAddend(Elf_Rel_Impl<ELFT, true> &Rela, uint64_t Addend) { + Rela.r_addend = Addend; +} + +template <class ELFT> +template <class T> +void RelocationSection<ELFT>::writeRel(T *Buf) const { + for (const auto &Reloc : Relocations) { + Buf->r_offset = Reloc.Offset; + setAddend(*Buf, Reloc.Addend); + Buf->setSymbolAndType(Reloc.RelocSymbol->Index, Reloc.Type, false); + ++Buf; + } +} + +template <class ELFT> +void RelocationSection<ELFT>::writeSection(FileOutputBuffer &Out) const { + uint8_t *Buf = Out.getBufferStart() + Offset; + if (Type == SHT_REL) + writeRel(reinterpret_cast<Elf_Rel *>(Buf)); + else + writeRel(reinterpret_cast<Elf_Rela *>(Buf)); +} + +void DynamicRelocationSection::writeSection(FileOutputBuffer &Out) const { + std::copy(std::begin(Contents), std::end(Contents), + Out.getBufferStart() + Offset); +} + +void SectionWithStrTab::removeSectionReferences(const SectionBase *Sec) { + if (StrTab == Sec) { + error("String table " + StrTab->Name + " cannot be removed because it is " + "referenced by the section " + + this->Name); + } +} + +bool SectionWithStrTab::classof(const SectionBase *S) { + return isa<DynamicSymbolTableSection>(S) || isa<DynamicSection>(S); +} + +void SectionWithStrTab::initialize(SectionTableRef SecTable) { + auto StrTab = SecTable.getSection(Link, + "Link field value " + Twine(Link) + + " in section " + Name + " is invalid"); + if (StrTab->Type != SHT_STRTAB) { + error("Link field value " + Twine(Link) + " in section " + Name + + " is not a string table"); + } + setStrTab(StrTab); +} + +void SectionWithStrTab::finalize() { this->Link = StrTab->Index; } + +// Returns true IFF a section is wholly inside the range of a segment +static bool sectionWithinSegment(const SectionBase &Section, + const Segment &Segment) { + // If a section is empty it should be treated like it has a size of 1. This is + // to clarify the case when an empty section lies on a boundary between two + // segments and ensures that the section "belongs" to the second segment and + // not the first. + uint64_t SecSize = Section.Size ? Section.Size : 1; + return Segment.Offset <= Section.OriginalOffset && + Segment.Offset + Segment.FileSize >= Section.OriginalOffset + SecSize; +} + +// Returns true IFF a segment's original offset is inside of another segment's +// range. +static bool segmentOverlapsSegment(const Segment &Child, + const Segment &Parent) { + + return Parent.OriginalOffset <= Child.OriginalOffset && + Parent.OriginalOffset + Parent.FileSize > Child.OriginalOffset; +} + +static bool compareSegments(const Segment *A, const Segment *B) { + // Any segment without a parent segment should come before a segment + // that has a parent segment. + if (A->OriginalOffset < B->OriginalOffset) + return true; + if (A->OriginalOffset > B->OriginalOffset) + return false; + return A->Index < B->Index; +} + +template <class ELFT> +void Object<ELFT>::readProgramHeaders(const ELFFile<ELFT> &ElfFile) { + uint32_t Index = 0; + for (const auto &Phdr : unwrapOrError(ElfFile.program_headers())) { + ArrayRef<uint8_t> Data{ElfFile.base() + Phdr.p_offset, + (size_t)Phdr.p_filesz}; + Segments.emplace_back(llvm::make_unique<Segment>(Data)); + Segment &Seg = *Segments.back(); + Seg.Type = Phdr.p_type; + Seg.Flags = Phdr.p_flags; + Seg.OriginalOffset = Phdr.p_offset; + Seg.Offset = Phdr.p_offset; + Seg.VAddr = Phdr.p_vaddr; + Seg.PAddr = Phdr.p_paddr; + Seg.FileSize = Phdr.p_filesz; + Seg.MemSize = Phdr.p_memsz; + Seg.Align = Phdr.p_align; + Seg.Index = Index++; + for (auto &Section : Sections) { + if (sectionWithinSegment(*Section, Seg)) { + Seg.addSection(&*Section); + if (!Section->ParentSegment || + Section->ParentSegment->Offset > Seg.Offset) { + Section->ParentSegment = &Seg; + } + } + } + } + // Now we do an O(n^2) loop through the segments in order to match up + // segments. + for (auto &Child : Segments) { + for (auto &Parent : Segments) { + // Every segment will overlap with itself but we don't want a segment to + // be it's own parent so we avoid that situation. + if (&Child != &Parent && segmentOverlapsSegment(*Child, *Parent)) { + // We want a canonical "most parental" segment but this requires + // inspecting the ParentSegment. + if (compareSegments(Parent.get(), Child.get())) + if (Child->ParentSegment == nullptr || + compareSegments(Parent.get(), Child->ParentSegment)) { + Child->ParentSegment = Parent.get(); + } + } + } + } +} + +template <class ELFT> +void Object<ELFT>::initSymbolTable(const object::ELFFile<ELFT> &ElfFile, + SymbolTableSection *SymTab, + SectionTableRef SecTable) { + const Elf_Shdr &Shdr = *unwrapOrError(ElfFile.getSection(SymTab->Index)); + StringRef StrTabData = unwrapOrError(ElfFile.getStringTableForSymtab(Shdr)); + + for (const auto &Sym : unwrapOrError(ElfFile.symbols(&Shdr))) { + SectionBase *DefSection = nullptr; + StringRef Name = unwrapOrError(Sym.getName(StrTabData)); + + if (Sym.st_shndx >= SHN_LORESERVE) { + if (!isValidReservedSectionIndex(Sym.st_shndx, Machine)) { + error( + "Symbol '" + Name + + "' has unsupported value greater than or equal to SHN_LORESERVE: " + + Twine(Sym.st_shndx)); + } + } else if (Sym.st_shndx != SHN_UNDEF) { + DefSection = SecTable.getSection( + Sym.st_shndx, + "Symbol '" + Name + "' is defined in invalid section with index " + + Twine(Sym.st_shndx)); + } + + SymTab->addSymbol(Name, Sym.getBinding(), Sym.getType(), DefSection, + Sym.getValue(), Sym.st_shndx, Sym.st_size); + } +} + +template <class ELFT> +static void getAddend(uint64_t &ToSet, const Elf_Rel_Impl<ELFT, false> &Rel) {} + +template <class ELFT> +static void getAddend(uint64_t &ToSet, const Elf_Rel_Impl<ELFT, true> &Rela) { + ToSet = Rela.r_addend; +} + +template <class ELFT, class T> +void initRelocations(RelocationSection<ELFT> *Relocs, + SymbolTableSection *SymbolTable, T RelRange) { + for (const auto &Rel : RelRange) { + Relocation ToAdd; + ToAdd.Offset = Rel.r_offset; + getAddend(ToAdd.Addend, Rel); + ToAdd.Type = Rel.getType(false); + ToAdd.RelocSymbol = SymbolTable->getSymbolByIndex(Rel.getSymbol(false)); + Relocs->addRelocation(ToAdd); + } +} + +SectionBase *SectionTableRef::getSection(uint16_t Index, Twine ErrMsg) { + if (Index == SHN_UNDEF || Index > Sections.size()) + error(ErrMsg); + return Sections[Index - 1].get(); +} + +template <class T> +T *SectionTableRef::getSectionOfType(uint16_t Index, Twine IndexErrMsg, + Twine TypeErrMsg) { + if (T *Sec = dyn_cast<T>(getSection(Index, IndexErrMsg))) + return Sec; + error(TypeErrMsg); +} + +template <class ELFT> +std::unique_ptr<SectionBase> +Object<ELFT>::makeSection(const object::ELFFile<ELFT> &ElfFile, + const Elf_Shdr &Shdr) { + ArrayRef<uint8_t> Data; + switch (Shdr.sh_type) { + case SHT_REL: + case SHT_RELA: + if (Shdr.sh_flags & SHF_ALLOC) { + Data = unwrapOrError(ElfFile.getSectionContents(&Shdr)); + return llvm::make_unique<DynamicRelocationSection>(Data); + } + return llvm::make_unique<RelocationSection<ELFT>>(); + case SHT_STRTAB: + // If a string table is allocated we don't want to mess with it. That would + // mean altering the memory image. There are no special link types or + // anything so we can just use a Section. + if (Shdr.sh_flags & SHF_ALLOC) { + Data = unwrapOrError(ElfFile.getSectionContents(&Shdr)); + return llvm::make_unique<Section>(Data); + } + return llvm::make_unique<StringTableSection>(); + case SHT_HASH: + case SHT_GNU_HASH: + // Hash tables should refer to SHT_DYNSYM which we're not going to change. + // Because of this we don't need to mess with the hash tables either. + Data = unwrapOrError(ElfFile.getSectionContents(&Shdr)); + return llvm::make_unique<Section>(Data); + case SHT_DYNSYM: + Data = unwrapOrError(ElfFile.getSectionContents(&Shdr)); + return llvm::make_unique<DynamicSymbolTableSection>(Data); + case SHT_DYNAMIC: + Data = unwrapOrError(ElfFile.getSectionContents(&Shdr)); + return llvm::make_unique<DynamicSection>(Data); + case SHT_SYMTAB: { + auto SymTab = llvm::make_unique<SymbolTableSectionImpl<ELFT>>(); + SymbolTable = SymTab.get(); + return std::move(SymTab); + } + case SHT_NOBITS: + return llvm::make_unique<Section>(Data); + default: + Data = unwrapOrError(ElfFile.getSectionContents(&Shdr)); + return llvm::make_unique<Section>(Data); + } +} + +template <class ELFT> +SectionTableRef Object<ELFT>::readSectionHeaders(const ELFFile<ELFT> &ElfFile) { + uint32_t Index = 0; + for (const auto &Shdr : unwrapOrError(ElfFile.sections())) { + if (Index == 0) { + ++Index; + continue; + } + SecPtr Sec = makeSection(ElfFile, Shdr); + Sec->Name = unwrapOrError(ElfFile.getSectionName(&Shdr)); + Sec->Type = Shdr.sh_type; + Sec->Flags = Shdr.sh_flags; + Sec->Addr = Shdr.sh_addr; + Sec->Offset = Shdr.sh_offset; + Sec->OriginalOffset = Shdr.sh_offset; + Sec->Size = Shdr.sh_size; + Sec->Link = Shdr.sh_link; + Sec->Info = Shdr.sh_info; + Sec->Align = Shdr.sh_addralign; + Sec->EntrySize = Shdr.sh_entsize; + Sec->Index = Index++; + Sections.push_back(std::move(Sec)); + } + + SectionTableRef SecTable(Sections); + + // Now that all of the sections have been added we can fill out some extra + // details about symbol tables. We need the symbol table filled out before + // any relocations. + if (SymbolTable) { + SymbolTable->initialize(SecTable); + initSymbolTable(ElfFile, SymbolTable, SecTable); + } + + // Now that all sections and symbols have been added we can add + // relocations that reference symbols and set the link and info fields for + // relocation sections. + for (auto &Section : Sections) { + if (Section.get() == SymbolTable) + continue; + Section->initialize(SecTable); + if (auto RelSec = dyn_cast<RelocationSection<ELFT>>(Section.get())) { + auto Shdr = unwrapOrError(ElfFile.sections()).begin() + RelSec->Index; + if (RelSec->Type == SHT_REL) + initRelocations(RelSec, SymbolTable, unwrapOrError(ElfFile.rels(Shdr))); + else + initRelocations(RelSec, SymbolTable, + unwrapOrError(ElfFile.relas(Shdr))); + } + } + + return SecTable; +} + +template <class ELFT> Object<ELFT>::Object(const ELFObjectFile<ELFT> &Obj) { + const auto &ElfFile = *Obj.getELFFile(); + const auto &Ehdr = *ElfFile.getHeader(); + + std::copy(Ehdr.e_ident, Ehdr.e_ident + 16, Ident); + Type = Ehdr.e_type; + Machine = Ehdr.e_machine; + Version = Ehdr.e_version; + Entry = Ehdr.e_entry; + Flags = Ehdr.e_flags; + + SectionTableRef SecTable = readSectionHeaders(ElfFile); + readProgramHeaders(ElfFile); + + SectionNames = SecTable.getSectionOfType<StringTableSection>( + Ehdr.e_shstrndx, + "e_shstrndx field value " + Twine(Ehdr.e_shstrndx) + " in elf header " + + " is invalid", + "e_shstrndx field value " + Twine(Ehdr.e_shstrndx) + " in elf header " + + " is not a string table"); +} + +template <class ELFT> +void Object<ELFT>::writeHeader(FileOutputBuffer &Out) const { + uint8_t *Buf = Out.getBufferStart(); + Elf_Ehdr &Ehdr = *reinterpret_cast<Elf_Ehdr *>(Buf); + std::copy(Ident, Ident + 16, Ehdr.e_ident); + Ehdr.e_type = Type; + Ehdr.e_machine = Machine; + Ehdr.e_version = Version; + Ehdr.e_entry = Entry; + Ehdr.e_phoff = sizeof(Elf_Ehdr); + Ehdr.e_flags = Flags; + Ehdr.e_ehsize = sizeof(Elf_Ehdr); + Ehdr.e_phentsize = sizeof(Elf_Phdr); + Ehdr.e_phnum = Segments.size(); + Ehdr.e_shentsize = sizeof(Elf_Shdr); + if (WriteSectionHeaders) { + Ehdr.e_shoff = SHOffset; + Ehdr.e_shnum = Sections.size() + 1; + Ehdr.e_shstrndx = SectionNames->Index; + } else { + Ehdr.e_shoff = 0; + Ehdr.e_shnum = 0; + Ehdr.e_shstrndx = 0; + } +} + +template <class ELFT> +void Object<ELFT>::writeProgramHeaders(FileOutputBuffer &Out) const { + for (auto &Phdr : Segments) + Phdr->template writeHeader<ELFT>(Out); +} + +template <class ELFT> +void Object<ELFT>::writeSectionHeaders(FileOutputBuffer &Out) const { + uint8_t *Buf = Out.getBufferStart() + SHOffset; + // This reference serves to write the dummy section header at the begining + // of the file. It is not used for anything else + Elf_Shdr &Shdr = *reinterpret_cast<Elf_Shdr *>(Buf); + Shdr.sh_name = 0; + Shdr.sh_type = SHT_NULL; + Shdr.sh_flags = 0; + Shdr.sh_addr = 0; + Shdr.sh_offset = 0; + Shdr.sh_size = 0; + Shdr.sh_link = 0; + Shdr.sh_info = 0; + Shdr.sh_addralign = 0; + Shdr.sh_entsize = 0; + + for (auto &Section : Sections) + Section->template writeHeader<ELFT>(Out); +} + +template <class ELFT> +void Object<ELFT>::writeSectionData(FileOutputBuffer &Out) const { + for (auto &Section : Sections) + Section->writeSection(Out); +} + +template <class ELFT> +void Object<ELFT>::removeSections( + std::function<bool(const SectionBase &)> ToRemove) { + + auto Iter = std::stable_partition( + std::begin(Sections), std::end(Sections), [=](const SecPtr &Sec) { + if (ToRemove(*Sec)) + return false; + if (auto RelSec = dyn_cast<RelocationSectionBase>(Sec.get())) { + if (auto ToRelSec = RelSec->getSection()) + return !ToRemove(*ToRelSec); + } + return true; + }); + if (SymbolTable != nullptr && ToRemove(*SymbolTable)) + SymbolTable = nullptr; + if (ToRemove(*SectionNames)) { + if (WriteSectionHeaders) + error("Cannot remove " + SectionNames->Name + + " because it is the section header string table."); + SectionNames = nullptr; + } + // Now make sure there are no remaining references to the sections that will + // be removed. Sometimes it is impossible to remove a reference so we emit + // an error here instead. + for (auto &RemoveSec : make_range(Iter, std::end(Sections))) { + for (auto &Segment : Segments) + Segment->removeSection(RemoveSec.get()); + for (auto &KeepSec : make_range(std::begin(Sections), Iter)) + KeepSec->removeSectionReferences(RemoveSec.get()); + } + // Now finally get rid of them all togethor. + Sections.erase(Iter, std::end(Sections)); +} + +template <class ELFT> void ELFObject<ELFT>::sortSections() { + // Put all sections in offset order. Maintain the ordering as closely as + // possible while meeting that demand however. + auto CompareSections = [](const SecPtr &A, const SecPtr &B) { + return A->OriginalOffset < B->OriginalOffset; + }; + std::stable_sort(std::begin(this->Sections), std::end(this->Sections), + CompareSections); +} + +static uint64_t alignToAddr(uint64_t Offset, uint64_t Addr, uint64_t Align) { + // Calculate Diff such that (Offset + Diff) & -Align == Addr & -Align. + if (Align == 0) + Align = 1; + auto Diff = + static_cast<int64_t>(Addr % Align) - static_cast<int64_t>(Offset % Align); + // We only want to add to Offset, however, so if Diff < 0 we can add Align and + // (Offset + Diff) & -Align == Addr & -Align will still hold. + if (Diff < 0) + Diff += Align; + return Offset + Diff; +} + +// Orders segments such that if x = y->ParentSegment then y comes before x. +static void OrderSegments(std::vector<Segment *> &Segments) { + std::stable_sort(std::begin(Segments), std::end(Segments), compareSegments); +} + +// This function finds a consistent layout for a list of segments starting from +// an Offset. It assumes that Segments have been sorted by OrderSegments and +// returns an Offset one past the end of the last segment. +static uint64_t LayoutSegments(std::vector<Segment *> &Segments, + uint64_t Offset) { + assert(std::is_sorted(std::begin(Segments), std::end(Segments), + compareSegments)); + // The only way a segment should move is if a section was between two + // segments and that section was removed. If that section isn't in a segment + // then it's acceptable, but not ideal, to simply move it to after the + // segments. So we can simply layout segments one after the other accounting + // for alignment. + for (auto &Segment : Segments) { + // We assume that segments have been ordered by OriginalOffset and Index + // such that a parent segment will always come before a child segment in + // OrderedSegments. This means that the Offset of the ParentSegment should + // already be set and we can set our offset relative to it. + if (Segment->ParentSegment != nullptr) { + auto Parent = Segment->ParentSegment; + Segment->Offset = + Parent->Offset + Segment->OriginalOffset - Parent->OriginalOffset; + } else { + Offset = alignToAddr(Offset, Segment->VAddr, Segment->Align); + Segment->Offset = Offset; + } + Offset = std::max(Offset, Segment->Offset + Segment->FileSize); + } + return Offset; +} + +// This function finds a consistent layout for a list of sections. It assumes +// that the ->ParentSegment of each section has already been laid out. The +// supplied starting Offset is used for the starting offset of any section that +// does not have a ParentSegment. It returns either the offset given if all +// sections had a ParentSegment or an offset one past the last section if there +// was a section that didn't have a ParentSegment. +template <class SecPtr> +static uint64_t LayoutSections(std::vector<SecPtr> &Sections, uint64_t Offset) { + // Now the offset of every segment has been set we can assign the offsets + // of each section. For sections that are covered by a segment we should use + // the segment's original offset and the section's original offset to compute + // the offset from the start of the segment. Using the offset from the start + // of the segment we can assign a new offset to the section. For sections not + // covered by segments we can just bump Offset to the next valid location. + uint32_t Index = 1; + for (auto &Section : Sections) { + Section->Index = Index++; + if (Section->ParentSegment != nullptr) { + auto Segment = Section->ParentSegment; + Section->Offset = + Segment->Offset + (Section->OriginalOffset - Segment->OriginalOffset); + } else { + Offset = alignTo(Offset, Section->Align == 0 ? 1 : Section->Align); + Section->Offset = Offset; + if (Section->Type != SHT_NOBITS) + Offset += Section->Size; + } + } + return Offset; +} + +template <class ELFT> void ELFObject<ELFT>::assignOffsets() { + // We need a temporary list of segments that has a special order to it + // so that we know that anytime ->ParentSegment is set that segment has + // already had its offset properly set. + std::vector<Segment *> OrderedSegments; + for (auto &Segment : this->Segments) + OrderedSegments.push_back(Segment.get()); + OrderSegments(OrderedSegments); + // The size of ELF + program headers will not change so it is ok to assume + // that the first offset of the first segment is a good place to start + // outputting sections. This covers both the standard case and the PT_PHDR + // case. + uint64_t Offset; + if (!OrderedSegments.empty()) { + Offset = OrderedSegments[0]->Offset; + } else { + Offset = sizeof(Elf_Ehdr); + } + Offset = LayoutSegments(OrderedSegments, Offset); + Offset = LayoutSections(this->Sections, Offset); + // If we need to write the section header table out then we need to align the + // Offset so that SHOffset is valid. + if (this->WriteSectionHeaders) + Offset = alignTo(Offset, sizeof(typename ELFT::Addr)); + this->SHOffset = Offset; +} + +template <class ELFT> size_t ELFObject<ELFT>::totalSize() const { + // We already have the section header offset so we can calculate the total + // size by just adding up the size of each section header. + auto NullSectionSize = this->WriteSectionHeaders ? sizeof(Elf_Shdr) : 0; + return this->SHOffset + this->Sections.size() * sizeof(Elf_Shdr) + + NullSectionSize; +} + +template <class ELFT> void ELFObject<ELFT>::write(FileOutputBuffer &Out) const { + this->writeHeader(Out); + this->writeProgramHeaders(Out); + this->writeSectionData(Out); + if (this->WriteSectionHeaders) + this->writeSectionHeaders(Out); +} + +template <class ELFT> void ELFObject<ELFT>::finalize() { + // Make sure we add the names of all the sections. + if (this->SectionNames != nullptr) + for (const auto &Section : this->Sections) { + this->SectionNames->addString(Section->Name); + } + // Make sure we add the names of all the symbols. + if (this->SymbolTable != nullptr) + this->SymbolTable->addSymbolNames(); + + sortSections(); + assignOffsets(); + + // Finalize SectionNames first so that we can assign name indexes. + if (this->SectionNames != nullptr) + this->SectionNames->finalize(); + // Finally now that all offsets and indexes have been set we can finalize any + // remaining issues. + uint64_t Offset = this->SHOffset + sizeof(Elf_Shdr); + for (auto &Section : this->Sections) { + Section->HeaderOffset = Offset; + Offset += sizeof(Elf_Shdr); + if (this->WriteSectionHeaders) + Section->NameIndex = this->SectionNames->findIndex(Section->Name); + Section->finalize(); + } +} + +template <class ELFT> size_t BinaryObject<ELFT>::totalSize() const { + return TotalSize; +} + +template <class ELFT> +void BinaryObject<ELFT>::write(FileOutputBuffer &Out) const { + for (auto &Section : this->Sections) { + if ((Section->Flags & SHF_ALLOC) == 0) + continue; + Section->writeSection(Out); + } +} + +template <class ELFT> void BinaryObject<ELFT>::finalize() { + // TODO: Create a filter range to construct OrderedSegments from so that this + // code can be deduped with assignOffsets above. This should also solve the + // todo below for LayoutSections. + // We need a temporary list of segments that has a special order to it + // so that we know that anytime ->ParentSegment is set that segment has + // already had it's offset properly set. We only want to consider the segments + // that will affect layout of allocated sections so we only add those. + std::vector<Segment *> OrderedSegments; + for (auto &Section : this->Sections) { + if ((Section->Flags & SHF_ALLOC) != 0 && + Section->ParentSegment != nullptr) { + OrderedSegments.push_back(Section->ParentSegment); + } + } + OrderSegments(OrderedSegments); + // Because we add a ParentSegment for each section we might have duplicate + // segments in OrderedSegments. If there were duplicates then LayoutSegments + // would do very strange things. + auto End = + std::unique(std::begin(OrderedSegments), std::end(OrderedSegments)); + OrderedSegments.erase(End, std::end(OrderedSegments)); + + // Modify the first segment so that there is no gap at the start. This allows + // our layout algorithm to proceed as expected while not out writing out the + // gap at the start. + if (!OrderedSegments.empty()) { + auto Seg = OrderedSegments[0]; + auto Sec = Seg->firstSection(); + auto Diff = Sec->OriginalOffset - Seg->OriginalOffset; + Seg->OriginalOffset += Diff; + // The size needs to be shrunk as well + Seg->FileSize -= Diff; + Seg->MemSize -= Diff; + // The VAddr needs to be adjusted so that the alignment is correct as well + Seg->VAddr += Diff; + Seg->PAddr = Seg->VAddr; + // We don't want this to be shifted by alignment so we need to set the + // alignment to zero. + Seg->Align = 0; + } + + uint64_t Offset = LayoutSegments(OrderedSegments, 0); + + // TODO: generalize LayoutSections to take a range. Pass a special range + // constructed from an iterator that skips values for which a predicate does + // not hold. Then pass such a range to LayoutSections instead of constructing + // AllocatedSections here. + std::vector<SectionBase *> AllocatedSections; + for (auto &Section : this->Sections) { + if ((Section->Flags & SHF_ALLOC) == 0) + continue; + AllocatedSections.push_back(Section.get()); + } + LayoutSections(AllocatedSections, Offset); + + // Now that every section has been laid out we just need to compute the total + // file size. This might not be the same as the offset returned by + // LayoutSections, because we want to truncate the last segment to the end of + // its last section, to match GNU objcopy's behaviour. + TotalSize = 0; + for (const auto &Section : AllocatedSections) { + if (Section->Type != SHT_NOBITS) + TotalSize = std::max(TotalSize, Section->Offset + Section->Size); + } +} + +namespace llvm { + +template class Object<ELF64LE>; +template class Object<ELF64BE>; +template class Object<ELF32LE>; +template class Object<ELF32BE>; + +template class ELFObject<ELF64LE>; +template class ELFObject<ELF64BE>; +template class ELFObject<ELF32LE>; +template class ELFObject<ELF32BE>; + +template class BinaryObject<ELF64LE>; +template class BinaryObject<ELF64BE>; +template class BinaryObject<ELF32LE>; +template class BinaryObject<ELF32BE>; + +} // end namespace llvm diff --git a/tools/llvm-objcopy/Object.h b/tools/llvm-objcopy/Object.h new file mode 100644 index 000000000000..9f98c04ad9bb --- /dev/null +++ b/tools/llvm-objcopy/Object.h @@ -0,0 +1,417 @@ +//===- Object.h -------------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_OBJCOPY_OBJECT_H +#define LLVM_TOOLS_OBJCOPY_OBJECT_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/StringTableBuilder.h" +#include "llvm/Object/ELFObjectFile.h" +#include <cstddef> +#include <cstdint> +#include <functional> +#include <memory> +#include <set> +#include <vector> + +namespace llvm { + +class FileOutputBuffer; +class SectionBase; +class Segment; + +class SectionTableRef { +private: + ArrayRef<std::unique_ptr<SectionBase>> Sections; + +public: + SectionTableRef(ArrayRef<std::unique_ptr<SectionBase>> Secs) + : Sections(Secs) {} + SectionTableRef(const SectionTableRef &) = default; + + SectionBase *getSection(uint16_t Index, Twine ErrMsg); + + template <class T> + T *getSectionOfType(uint16_t Index, Twine IndexErrMsg, Twine TypeErrMsg); +}; + +class SectionBase { +public: + StringRef Name; + Segment *ParentSegment = nullptr; + uint64_t HeaderOffset; + uint64_t OriginalOffset; + uint32_t Index; + + uint64_t Addr = 0; + uint64_t Align = 1; + uint32_t EntrySize = 0; + uint64_t Flags = 0; + uint64_t Info = 0; + uint64_t Link = ELF::SHN_UNDEF; + uint64_t NameIndex = 0; + uint64_t Offset = 0; + uint64_t Size = 0; + uint64_t Type = ELF::SHT_NULL; + + virtual ~SectionBase() = default; + + virtual void initialize(SectionTableRef SecTable); + virtual void finalize(); + virtual void removeSectionReferences(const SectionBase *Sec); + template <class ELFT> void writeHeader(FileOutputBuffer &Out) const; + virtual void writeSection(FileOutputBuffer &Out) const = 0; +}; + +class Segment { +private: + struct SectionCompare { + bool operator()(const SectionBase *Lhs, const SectionBase *Rhs) const { + // Some sections might have the same address if one of them is empty. To + // fix this we can use the lexicographic ordering on ->Addr and the + // address of the actully stored section. + if (Lhs->OriginalOffset == Rhs->OriginalOffset) + return Lhs < Rhs; + return Lhs->OriginalOffset < Rhs->OriginalOffset; + } + }; + + std::set<const SectionBase *, SectionCompare> Sections; + ArrayRef<uint8_t> Contents; + +public: + uint64_t Align; + uint64_t FileSize; + uint32_t Flags; + uint32_t Index; + uint64_t MemSize; + uint64_t Offset; + uint64_t PAddr; + uint64_t Type; + uint64_t VAddr; + + uint64_t OriginalOffset; + Segment *ParentSegment = nullptr; + + Segment(ArrayRef<uint8_t> Data) : Contents(Data) {} + + const SectionBase *firstSection() const { + if (!Sections.empty()) + return *Sections.begin(); + return nullptr; + } + + void removeSection(const SectionBase *Sec) { Sections.erase(Sec); } + void addSection(const SectionBase *Sec) { Sections.insert(Sec); } + template <class ELFT> void writeHeader(FileOutputBuffer &Out) const; + void writeSegment(FileOutputBuffer &Out) const; +}; + +class Section : public SectionBase { +private: + ArrayRef<uint8_t> Contents; + +public: + Section(ArrayRef<uint8_t> Data) : Contents(Data) {} + + void writeSection(FileOutputBuffer &Out) const override; +}; + +// There are two types of string tables that can exist, dynamic and not dynamic. +// In the dynamic case the string table is allocated. Changing a dynamic string +// table would mean altering virtual addresses and thus the memory image. So +// dynamic string tables should not have an interface to modify them or +// reconstruct them. This type lets us reconstruct a string table. To avoid +// this class being used for dynamic string tables (which has happened) the +// classof method checks that the particular instance is not allocated. This +// then agrees with the makeSection method used to construct most sections. +class StringTableSection : public SectionBase { +private: + StringTableBuilder StrTabBuilder; + +public: + StringTableSection() : StrTabBuilder(StringTableBuilder::ELF) { + Type = ELF::SHT_STRTAB; + } + + void addString(StringRef Name); + uint32_t findIndex(StringRef Name) const; + void finalize() override; + void writeSection(FileOutputBuffer &Out) const override; + + static bool classof(const SectionBase *S) { + if (S->Flags & ELF::SHF_ALLOC) + return false; + return S->Type == ELF::SHT_STRTAB; + } +}; + +// Symbols have a st_shndx field that normally stores an index but occasionally +// stores a different special value. This enum keeps track of what the st_shndx +// field means. Most of the values are just copies of the special SHN_* values. +// SYMBOL_SIMPLE_INDEX means that the st_shndx is just an index of a section. +enum SymbolShndxType { + SYMBOL_SIMPLE_INDEX = 0, + SYMBOL_ABS = ELF::SHN_ABS, + SYMBOL_COMMON = ELF::SHN_COMMON, + SYMBOL_HEXAGON_SCOMMON = ELF::SHN_HEXAGON_SCOMMON, + SYMBOL_HEXAGON_SCOMMON_2 = ELF::SHN_HEXAGON_SCOMMON_2, + SYMBOL_HEXAGON_SCOMMON_4 = ELF::SHN_HEXAGON_SCOMMON_4, + SYMBOL_HEXAGON_SCOMMON_8 = ELF::SHN_HEXAGON_SCOMMON_8, +}; + +struct Symbol { + uint8_t Binding; + SectionBase *DefinedIn = nullptr; + SymbolShndxType ShndxType; + uint32_t Index; + StringRef Name; + uint32_t NameIndex; + uint64_t Size; + uint8_t Type; + uint64_t Value; + + uint16_t getShndx() const; +}; + +class SymbolTableSection : public SectionBase { +protected: + std::vector<std::unique_ptr<Symbol>> Symbols; + StringTableSection *SymbolNames = nullptr; + + using SymPtr = std::unique_ptr<Symbol>; + +public: + void setStrTab(StringTableSection *StrTab) { SymbolNames = StrTab; } + void addSymbol(StringRef Name, uint8_t Bind, uint8_t Type, + SectionBase *DefinedIn, uint64_t Value, uint16_t Shndx, + uint64_t Sz); + void addSymbolNames(); + const SectionBase *getStrTab() const { return SymbolNames; } + const Symbol *getSymbolByIndex(uint32_t Index) const; + void removeSectionReferences(const SectionBase *Sec) override; + void initialize(SectionTableRef SecTable) override; + void finalize() override; + + static bool classof(const SectionBase *S) { + return S->Type == ELF::SHT_SYMTAB; + } +}; + +// Only writeSection depends on the ELF type so we implement it in a subclass. +template <class ELFT> class SymbolTableSectionImpl : public SymbolTableSection { + void writeSection(FileOutputBuffer &Out) const override; +}; + +struct Relocation { + const Symbol *RelocSymbol = nullptr; + uint64_t Offset; + uint64_t Addend; + uint32_t Type; +}; + +// All relocation sections denote relocations to apply to another section. +// However, some relocation sections use a dynamic symbol table and others use +// a regular symbol table. Because the types of the two symbol tables differ in +// our system (because they should behave differently) we can't uniformly +// represent all relocations with the same base class if we expose an interface +// that mentions the symbol table type. So we split the two base types into two +// different classes, one which handles the section the relocation is applied to +// and another which handles the symbol table type. The symbol table type is +// taken as a type parameter to the class (see RelocSectionWithSymtabBase). +class RelocationSectionBase : public SectionBase { +protected: + SectionBase *SecToApplyRel = nullptr; + +public: + const SectionBase *getSection() const { return SecToApplyRel; } + void setSection(SectionBase *Sec) { SecToApplyRel = Sec; } + + static bool classof(const SectionBase *S) { + return S->Type == ELF::SHT_REL || S->Type == ELF::SHT_RELA; + } +}; + +// Takes the symbol table type to use as a parameter so that we can deduplicate +// that code between the two symbol table types. +template <class SymTabType> +class RelocSectionWithSymtabBase : public RelocationSectionBase { +private: + SymTabType *Symbols = nullptr; + +protected: + RelocSectionWithSymtabBase() = default; + +public: + void setSymTab(SymTabType *StrTab) { Symbols = StrTab; } + void removeSectionReferences(const SectionBase *Sec) override; + void initialize(SectionTableRef SecTable) override; + void finalize() override; +}; + +template <class ELFT> +class RelocationSection + : public RelocSectionWithSymtabBase<SymbolTableSection> { +private: + using Elf_Rel = typename ELFT::Rel; + using Elf_Rela = typename ELFT::Rela; + + std::vector<Relocation> Relocations; + + template <class T> void writeRel(T *Buf) const; + +public: + void addRelocation(Relocation Rel) { Relocations.push_back(Rel); } + void writeSection(FileOutputBuffer &Out) const override; + + static bool classof(const SectionBase *S) { + if (S->Flags & ELF::SHF_ALLOC) + return false; + return S->Type == ELF::SHT_REL || S->Type == ELF::SHT_RELA; + } +}; + +class SectionWithStrTab : public Section { +private: + const SectionBase *StrTab = nullptr; + +public: + SectionWithStrTab(ArrayRef<uint8_t> Data) : Section(Data) {} + + void setStrTab(const SectionBase *StringTable) { StrTab = StringTable; } + void removeSectionReferences(const SectionBase *Sec) override; + void initialize(SectionTableRef SecTable) override; + void finalize() override; + static bool classof(const SectionBase *S); +}; + +class DynamicSymbolTableSection : public SectionWithStrTab { +public: + DynamicSymbolTableSection(ArrayRef<uint8_t> Data) : SectionWithStrTab(Data) {} + + static bool classof(const SectionBase *S) { + return S->Type == ELF::SHT_DYNSYM; + } +}; + +class DynamicSection : public SectionWithStrTab { +public: + DynamicSection(ArrayRef<uint8_t> Data) : SectionWithStrTab(Data) {} + + static bool classof(const SectionBase *S) { + return S->Type == ELF::SHT_DYNAMIC; + } +}; + +class DynamicRelocationSection + : public RelocSectionWithSymtabBase<DynamicSymbolTableSection> { +private: + ArrayRef<uint8_t> Contents; + +public: + DynamicRelocationSection(ArrayRef<uint8_t> Data) : Contents(Data) {} + + void writeSection(FileOutputBuffer &Out) const override; + + static bool classof(const SectionBase *S) { + if (!(S->Flags & ELF::SHF_ALLOC)) + return false; + return S->Type == ELF::SHT_REL || S->Type == ELF::SHT_RELA; + } +}; + +template <class ELFT> class Object { +private: + using SecPtr = std::unique_ptr<SectionBase>; + using SegPtr = std::unique_ptr<Segment>; + + using Elf_Shdr = typename ELFT::Shdr; + using Elf_Ehdr = typename ELFT::Ehdr; + using Elf_Phdr = typename ELFT::Phdr; + + void initSymbolTable(const object::ELFFile<ELFT> &ElfFile, + SymbolTableSection *SymTab, SectionTableRef SecTable); + SecPtr makeSection(const object::ELFFile<ELFT> &ElfFile, + const Elf_Shdr &Shdr); + void readProgramHeaders(const object::ELFFile<ELFT> &ElfFile); + SectionTableRef readSectionHeaders(const object::ELFFile<ELFT> &ElfFile); + +protected: + StringTableSection *SectionNames = nullptr; + SymbolTableSection *SymbolTable = nullptr; + std::vector<SecPtr> Sections; + std::vector<SegPtr> Segments; + + void writeHeader(FileOutputBuffer &Out) const; + void writeProgramHeaders(FileOutputBuffer &Out) const; + void writeSectionData(FileOutputBuffer &Out) const; + void writeSectionHeaders(FileOutputBuffer &Out) const; + +public: + uint8_t Ident[16]; + uint64_t Entry; + uint64_t SHOffset; + uint32_t Type; + uint32_t Machine; + uint32_t Version; + uint32_t Flags; + bool WriteSectionHeaders = true; + + Object(const object::ELFObjectFile<ELFT> &Obj); + virtual ~Object() = default; + + const SymbolTableSection *getSymTab() const { return SymbolTable; } + const SectionBase *getSectionHeaderStrTab() const { return SectionNames; } + void removeSections(std::function<bool(const SectionBase &)> ToRemove); + virtual size_t totalSize() const = 0; + virtual void finalize() = 0; + virtual void write(FileOutputBuffer &Out) const = 0; +}; + +template <class ELFT> class ELFObject : public Object<ELFT> { +private: + using SecPtr = std::unique_ptr<SectionBase>; + using SegPtr = std::unique_ptr<Segment>; + + using Elf_Shdr = typename ELFT::Shdr; + using Elf_Ehdr = typename ELFT::Ehdr; + using Elf_Phdr = typename ELFT::Phdr; + + void sortSections(); + void assignOffsets(); + +public: + ELFObject(const object::ELFObjectFile<ELFT> &Obj) : Object<ELFT>(Obj) {} + + void finalize() override; + size_t totalSize() const override; + void write(FileOutputBuffer &Out) const override; +}; + +template <class ELFT> class BinaryObject : public Object<ELFT> { +private: + using SecPtr = std::unique_ptr<SectionBase>; + using SegPtr = std::unique_ptr<Segment>; + + uint64_t TotalSize; + +public: + BinaryObject(const object::ELFObjectFile<ELFT> &Obj) : Object<ELFT>(Obj) {} + + void finalize() override; + size_t totalSize() const override; + void write(FileOutputBuffer &Out) const override; +}; + +} // end namespace llvm + +#endif // LLVM_TOOLS_OBJCOPY_OBJECT_H diff --git a/tools/llvm-objcopy/llvm-objcopy.cpp b/tools/llvm-objcopy/llvm-objcopy.cpp new file mode 100644 index 000000000000..9d60ae426390 --- /dev/null +++ b/tools/llvm-objcopy/llvm-objcopy.cpp @@ -0,0 +1,325 @@ +//===- llvm-objcopy.cpp ---------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm-objcopy.h" +#include "Object.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ELFTypes.h" +#include "llvm/Object/Error.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/FileOutputBuffer.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cassert> +#include <cstdlib> +#include <functional> +#include <iterator> +#include <memory> +#include <string> +#include <system_error> +#include <utility> + +using namespace llvm; +using namespace object; +using namespace ELF; + +// The name this program was invoked as. +static StringRef ToolName; + +namespace llvm { + +LLVM_ATTRIBUTE_NORETURN void error(Twine Message) { + errs() << ToolName << ": " << Message << ".\n"; + errs().flush(); + exit(1); +} + +LLVM_ATTRIBUTE_NORETURN void reportError(StringRef File, std::error_code EC) { + assert(EC); + errs() << ToolName << ": '" << File << "': " << EC.message() << ".\n"; + exit(1); +} + +LLVM_ATTRIBUTE_NORETURN void reportError(StringRef File, Error E) { + assert(E); + std::string Buf; + raw_string_ostream OS(Buf); + logAllUnhandledErrors(std::move(E), OS, ""); + OS.flush(); + errs() << ToolName << ": '" << File << "': " << Buf; + exit(1); +} + +} // end namespace llvm + +static cl::opt<std::string> InputFilename(cl::Positional, cl::desc("<input>")); +static cl::opt<std::string> OutputFilename(cl::Positional, cl::desc("<output>"), + cl::init("-")); +static cl::opt<std::string> + OutputFormat("O", cl::desc("Set output format to one of the following:" + "\n\tbinary")); +static cl::list<std::string> ToRemove("remove-section", + cl::desc("Remove <section>"), + cl::value_desc("section")); +static cl::alias ToRemoveA("R", cl::desc("Alias for remove-section"), + cl::aliasopt(ToRemove)); +static cl::opt<bool> StripAll( + "strip-all", + cl::desc( + "Removes non-allocated sections other than .gnu.warning* sections")); +static cl::opt<bool> + StripAllGNU("strip-all-gnu", + cl::desc("Removes symbol, relocation, and debug information")); +static cl::list<std::string> Keep("keep", cl::desc("Keep <section>"), + cl::value_desc("section")); +static cl::list<std::string> OnlyKeep("only-keep", + cl::desc("Remove all but <section>"), + cl::value_desc("section")); +static cl::alias OnlyKeepA("j", cl::desc("Alias for only-keep"), + cl::aliasopt(OnlyKeep)); +static cl::opt<bool> StripDebug("strip-debug", + cl::desc("Removes all debug information")); +static cl::opt<bool> StripSections("strip-sections", + cl::desc("Remove all section headers")); +static cl::opt<bool> + StripNonAlloc("strip-non-alloc", + cl::desc("Remove all non-allocated sections")); +static cl::opt<bool> + StripDWO("strip-dwo", cl::desc("Remove all DWARF .dwo sections from file")); +static cl::opt<bool> ExtractDWO( + "extract-dwo", + cl::desc("Remove all sections that are not DWARF .dwo sections from file")); +static cl::opt<std::string> + SplitDWO("split-dwo", + cl::desc("Equivalent to extract-dwo on the input file to " + "<dwo-file>, then strip-dwo on the input file"), + cl::value_desc("dwo-file")); + +using SectionPred = std::function<bool(const SectionBase &Sec)>; + +bool IsDWOSection(const SectionBase &Sec) { return Sec.Name.endswith(".dwo"); } + +template <class ELFT> +bool OnlyKeepDWOPred(const Object<ELFT> &Obj, const SectionBase &Sec) { + // We can't remove the section header string table. + if (&Sec == Obj.getSectionHeaderStrTab()) + return false; + // Short of keeping the string table we want to keep everything that is a DWO + // section and remove everything else. + return !IsDWOSection(Sec); +} + +template <class ELFT> +void WriteObjectFile(const Object<ELFT> &Obj, StringRef File) { + std::unique_ptr<FileOutputBuffer> Buffer; + Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr = + FileOutputBuffer::create(File, Obj.totalSize(), + FileOutputBuffer::F_executable); + handleAllErrors(BufferOrErr.takeError(), [](const ErrorInfoBase &) { + error("failed to open " + OutputFilename); + }); + Buffer = std::move(*BufferOrErr); + + Obj.write(*Buffer); + if (auto E = Buffer->commit()) + reportError(File, errorToErrorCode(std::move(E))); +} + +template <class ELFT> +void SplitDWOToFile(const ELFObjectFile<ELFT> &ObjFile, StringRef File) { + // Construct a second output file for the DWO sections. + ELFObject<ELFT> DWOFile(ObjFile); + + DWOFile.removeSections([&](const SectionBase &Sec) { + return OnlyKeepDWOPred<ELFT>(DWOFile, Sec); + }); + DWOFile.finalize(); + WriteObjectFile(DWOFile, File); +} + +// This function handles the high level operations of GNU objcopy including +// handling command line options. It's important to outline certain properties +// we expect to hold of the command line operations. Any operation that "keeps" +// should keep regardless of a remove. Additionally any removal should respect +// any previous removals. Lastly whether or not something is removed shouldn't +// depend a) on the order the options occur in or b) on some opaque priority +// system. The only priority is that keeps/copies overrule removes. +template <class ELFT> void CopyBinary(const ELFObjectFile<ELFT> &ObjFile) { + std::unique_ptr<Object<ELFT>> Obj; + + if (!OutputFormat.empty() && OutputFormat != "binary") + error("invalid output format '" + OutputFormat + "'"); + if (!OutputFormat.empty() && OutputFormat == "binary") + Obj = llvm::make_unique<BinaryObject<ELFT>>(ObjFile); + else + Obj = llvm::make_unique<ELFObject<ELFT>>(ObjFile); + + if (!SplitDWO.empty()) + SplitDWOToFile<ELFT>(ObjFile, SplitDWO.getValue()); + + SectionPred RemovePred = [](const SectionBase &) { return false; }; + + // Removes: + + if (!ToRemove.empty()) { + RemovePred = [&](const SectionBase &Sec) { + return std::find(std::begin(ToRemove), std::end(ToRemove), Sec.Name) != + std::end(ToRemove); + }; + } + + if (StripDWO || !SplitDWO.empty()) + RemovePred = [RemovePred](const SectionBase &Sec) { + return IsDWOSection(Sec) || RemovePred(Sec); + }; + + if (ExtractDWO) + RemovePred = [RemovePred, &Obj](const SectionBase &Sec) { + return OnlyKeepDWOPred(*Obj, Sec) || RemovePred(Sec); + }; + + if (StripAllGNU) + RemovePred = [RemovePred, &Obj](const SectionBase &Sec) { + if (RemovePred(Sec)) + return true; + if ((Sec.Flags & SHF_ALLOC) != 0) + return false; + if (&Sec == Obj->getSectionHeaderStrTab()) + return false; + switch (Sec.Type) { + case SHT_SYMTAB: + case SHT_REL: + case SHT_RELA: + case SHT_STRTAB: + return true; + } + return Sec.Name.startswith(".debug"); + }; + + if (StripSections) { + RemovePred = [RemovePred](const SectionBase &Sec) { + return RemovePred(Sec) || (Sec.Flags & SHF_ALLOC) == 0; + }; + Obj->WriteSectionHeaders = false; + } + + if (StripDebug) { + RemovePred = [RemovePred](const SectionBase &Sec) { + return RemovePred(Sec) || Sec.Name.startswith(".debug"); + }; + } + + if (StripNonAlloc) + RemovePred = [RemovePred, &Obj](const SectionBase &Sec) { + if (RemovePred(Sec)) + return true; + if (&Sec == Obj->getSectionHeaderStrTab()) + return false; + return (Sec.Flags & SHF_ALLOC) == 0; + }; + + if (StripAll) + RemovePred = [RemovePred, &Obj](const SectionBase &Sec) { + if (RemovePred(Sec)) + return true; + if (&Sec == Obj->getSectionHeaderStrTab()) + return false; + if (Sec.Name.startswith(".gnu.warning")) + return false; + return (Sec.Flags & SHF_ALLOC) == 0; + }; + + // Explicit copies: + + if (!OnlyKeep.empty()) { + RemovePred = [RemovePred, &Obj](const SectionBase &Sec) { + // Explicitly keep these sections regardless of previous removes. + if (std::find(std::begin(OnlyKeep), std::end(OnlyKeep), Sec.Name) != + std::end(OnlyKeep)) + return false; + + // Allow all implicit removes. + if (RemovePred(Sec)) { + return true; + } + + // Keep special sections. + if (Obj->getSectionHeaderStrTab() == &Sec) { + return false; + } + if (Obj->getSymTab() == &Sec || Obj->getSymTab()->getStrTab() == &Sec) { + return false; + } + // Remove everything else. + return true; + }; + } + + if (!Keep.empty()) { + RemovePred = [RemovePred](const SectionBase &Sec) { + // Explicitly keep these sections regardless of previous removes. + if (std::find(std::begin(Keep), std::end(Keep), Sec.Name) != + std::end(Keep)) + return false; + // Otherwise defer to RemovePred. + return RemovePred(Sec); + }; + } + + Obj->removeSections(RemovePred); + Obj->finalize(); + WriteObjectFile(*Obj, OutputFilename.getValue()); +} + +int main(int argc, char **argv) { + // Print a stack trace if we signal out. + sys::PrintStackTraceOnErrorSignal(argv[0]); + PrettyStackTraceProgram X(argc, argv); + llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. + cl::ParseCommandLineOptions(argc, argv, "llvm objcopy utility\n"); + ToolName = argv[0]; + if (InputFilename.empty()) { + cl::PrintHelpMessage(); + return 2; + } + Expected<OwningBinary<Binary>> BinaryOrErr = createBinary(InputFilename); + if (!BinaryOrErr) + reportError(InputFilename, BinaryOrErr.takeError()); + Binary &Binary = *BinaryOrErr.get().getBinary(); + if (auto *o = dyn_cast<ELFObjectFile<ELF64LE>>(&Binary)) { + CopyBinary(*o); + return 0; + } + if (auto *o = dyn_cast<ELFObjectFile<ELF32LE>>(&Binary)) { + CopyBinary(*o); + return 0; + } + if (auto *o = dyn_cast<ELFObjectFile<ELF64BE>>(&Binary)) { + CopyBinary(*o); + return 0; + } + if (auto *o = dyn_cast<ELFObjectFile<ELF32BE>>(&Binary)) { + CopyBinary(*o); + return 0; + } + reportError(InputFilename, object_error::invalid_file_type); +} diff --git a/tools/llvm-objcopy/llvm-objcopy.h b/tools/llvm-objcopy/llvm-objcopy.h new file mode 100644 index 000000000000..6732e410d8e0 --- /dev/null +++ b/tools/llvm-objcopy/llvm-objcopy.h @@ -0,0 +1,37 @@ +//===- llvm-objcopy.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_OBJCOPY_OBJCOPY_H +#define LLVM_TOOLS_OBJCOPY_OBJCOPY_H + +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include <string> + +namespace llvm { + +LLVM_ATTRIBUTE_NORETURN extern void error(Twine Message); + +// This is taken from llvm-readobj. +// [see here](llvm/tools/llvm-readobj/llvm-readobj.h:38) +template <class T> T unwrapOrError(Expected<T> EO) { + if (EO) + return *EO; + std::string Buf; + raw_string_ostream OS(Buf); + logAllUnhandledErrors(EO.takeError(), OS, ""); + OS.flush(); + error(Buf); +} + +} // end namespace llvm + +#endif // LLVM_TOOLS_OBJCOPY_OBJCOPY_H diff --git a/tools/llvm-objdump/CMakeLists.txt b/tools/llvm-objdump/CMakeLists.txt index 27e6145dfc13..177c98166ef1 100644 --- a/tools/llvm-objdump/CMakeLists.txt +++ b/tools/llvm-objdump/CMakeLists.txt @@ -23,5 +23,9 @@ add_llvm_tool(llvm-objdump ) if(HAVE_LIBXAR) - target_link_libraries(llvm-objdump ${XAR_LIB}) + target_link_libraries(llvm-objdump PRIVATE ${XAR_LIB}) +endif() + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(objdump llvm-objdump) endif() diff --git a/tools/llvm-objdump/COFFDump.cpp b/tools/llvm-objdump/COFFDump.cpp index db549bbe3eec..780d1e9e6111 100644 --- a/tools/llvm-objdump/COFFDump.cpp +++ b/tools/llvm-objdump/COFFDump.cpp @@ -20,12 +20,8 @@ #include "llvm/Object/COFFImportFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Format.h" -#include "llvm/Support/SourceMgr.h" #include "llvm/Support/Win64EH.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <cstring> -#include <system_error> using namespace llvm; using namespace object; @@ -641,9 +637,9 @@ void llvm::printCOFFSymbolTable(const object::COFFImportFile *i) { void llvm::printCOFFSymbolTable(const COFFObjectFile *coff) { for (unsigned SI = 0, SE = coff->getNumberOfSymbols(); SI != SE; ++SI) { - ErrorOr<COFFSymbolRef> Symbol = coff->getSymbol(SI); + Expected<COFFSymbolRef> Symbol = coff->getSymbol(SI); StringRef Name; - error(Symbol.getError()); + error(errorToErrorCode(Symbol.takeError())); error(coff->getSymbolName(*Symbol, Name)); outs() << "[" << format("%2d", SI) << "]" diff --git a/tools/llvm-objdump/MachODump.cpp b/tools/llvm-objdump/MachODump.cpp index 05c2c3f7c4cb..9908c2f2d016 100644 --- a/tools/llvm-objdump/MachODump.cpp +++ b/tools/llvm-objdump/MachODump.cpp @@ -202,6 +202,35 @@ typedef std::pair<uint64_t, DiceRef> DiceTableEntry; typedef std::vector<DiceTableEntry> DiceTable; typedef DiceTable::iterator dice_table_iterator; +#ifdef HAVE_LIBXAR +namespace { +struct ScopedXarFile { + xar_t xar; + ScopedXarFile(const char *filename, int32_t flags) + : xar(xar_open(filename, flags)) {} + ~ScopedXarFile() { + if (xar) + xar_close(xar); + } + ScopedXarFile(const ScopedXarFile &) = delete; + ScopedXarFile &operator=(const ScopedXarFile &) = delete; + operator xar_t() { return xar; } +}; + +struct ScopedXarIter { + xar_iter_t iter; + ScopedXarIter() : iter(xar_iter_new()) {} + ~ScopedXarIter() { + if (iter) + xar_iter_free(iter); + } + ScopedXarIter(const ScopedXarIter &) = delete; + ScopedXarIter &operator=(const ScopedXarIter &) = delete; + operator xar_iter_t() { return iter; } +}; +} // namespace +#endif // defined(HAVE_LIBXAR) + // This is used to search for a data in code table entry for the PC being // disassembled. The j parameter has the PC in j.first. A single data in code // table entry can cover many bytes for each of its Kind's. So if the offset, @@ -438,6 +467,333 @@ static void PrintIndirectSymbols(MachOObjectFile *O, bool verbose) { } } +static void PrintRType(const uint64_t cputype, const unsigned r_type) { + static char const *generic_r_types[] = { + "VANILLA ", "PAIR ", "SECTDIF ", "PBLAPTR ", "LOCSDIF ", "TLV ", + " 6 (?) ", " 7 (?) ", " 8 (?) ", " 9 (?) ", " 10 (?) ", " 11 (?) ", + " 12 (?) ", " 13 (?) ", " 14 (?) ", " 15 (?) " + }; + static char const *x86_64_r_types[] = { + "UNSIGND ", "SIGNED ", "BRANCH ", "GOT_LD ", "GOT ", "SUB ", + "SIGNED1 ", "SIGNED2 ", "SIGNED4 ", "TLV ", " 10 (?) ", " 11 (?) ", + " 12 (?) ", " 13 (?) ", " 14 (?) ", " 15 (?) " + }; + static char const *arm_r_types[] = { + "VANILLA ", "PAIR ", "SECTDIFF", "LOCSDIF ", "PBLAPTR ", + "BR24 ", "T_BR22 ", "T_BR32 ", "HALF ", "HALFDIF ", + " 10 (?) ", " 11 (?) ", " 12 (?) ", " 13 (?) ", " 14 (?) ", " 15 (?) " + }; + static char const *arm64_r_types[] = { + "UNSIGND ", "SUB ", "BR26 ", "PAGE21 ", "PAGOF12 ", + "GOTLDP ", "GOTLDPOF", "PTRTGOT ", "TLVLDP ", "TLVLDPOF", + "ADDEND ", " 11 (?) ", " 12 (?) ", " 13 (?) ", " 14 (?) ", " 15 (?) " + }; + + if (r_type > 0xf){ + outs() << format("%-7u", r_type) << " "; + return; + } + switch (cputype) { + case MachO::CPU_TYPE_I386: + outs() << generic_r_types[r_type]; + break; + case MachO::CPU_TYPE_X86_64: + outs() << x86_64_r_types[r_type]; + break; + case MachO::CPU_TYPE_ARM: + outs() << arm_r_types[r_type]; + break; + case MachO::CPU_TYPE_ARM64: + outs() << arm64_r_types[r_type]; + break; + default: + outs() << format("%-7u ", r_type); + } +} + +static void PrintRLength(const uint64_t cputype, const unsigned r_type, + const unsigned r_length, const bool previous_arm_half){ + if (cputype == MachO::CPU_TYPE_ARM && + (r_type == llvm::MachO::ARM_RELOC_HALF || + r_type == llvm::MachO::ARM_RELOC_HALF_SECTDIFF || + previous_arm_half == true)) { + if ((r_length & 0x1) == 0) + outs() << "lo/"; + else + outs() << "hi/"; + if ((r_length & 0x1) == 0) + outs() << "arm "; + else + outs() << "thm "; + } else { + switch (r_length) { + case 0: + outs() << "byte "; + break; + case 1: + outs() << "word "; + break; + case 2: + outs() << "long "; + break; + case 3: + if (cputype == MachO::CPU_TYPE_X86_64) + outs() << "quad "; + else + outs() << format("?(%2d) ", r_length); + break; + default: + outs() << format("?(%2d) ", r_length); + } + } +} + +static void PrintRelocationEntries(const MachOObjectFile *O, + const relocation_iterator Begin, + const relocation_iterator End, + const uint64_t cputype, + const bool verbose) { + const MachO::symtab_command Symtab = O->getSymtabLoadCommand(); + bool previous_arm_half = false; + bool previous_sectdiff = false; + uint32_t sectdiff_r_type = 0; + + for (relocation_iterator Reloc = Begin; Reloc != End; ++Reloc) { + const DataRefImpl Rel = Reloc->getRawDataRefImpl(); + const MachO::any_relocation_info RE = O->getRelocation(Rel); + const unsigned r_type = O->getAnyRelocationType(RE); + const bool r_scattered = O->isRelocationScattered(RE); + const unsigned r_pcrel = O->getAnyRelocationPCRel(RE); + const unsigned r_length = O->getAnyRelocationLength(RE); + const unsigned r_address = O->getAnyRelocationAddress(RE); + const bool r_extern = (r_scattered ? false : + O->getPlainRelocationExternal(RE)); + const uint32_t r_value = (r_scattered ? + O->getScatteredRelocationValue(RE) : 0); + const unsigned r_symbolnum = (r_scattered ? 0 : + O->getPlainRelocationSymbolNum(RE)); + + if (r_scattered && cputype != MachO::CPU_TYPE_X86_64) { + if (verbose) { + // scattered: address + if ((cputype == MachO::CPU_TYPE_I386 && + r_type == llvm::MachO::GENERIC_RELOC_PAIR) || + (cputype == MachO::CPU_TYPE_ARM && + r_type == llvm::MachO::ARM_RELOC_PAIR)) + outs() << " "; + else + outs() << format("%08x ", (unsigned int)r_address); + + // scattered: pcrel + if (r_pcrel) + outs() << "True "; + else + outs() << "False "; + + // scattered: length + PrintRLength(cputype, r_type, r_length, previous_arm_half); + + // scattered: extern & type + outs() << "n/a "; + PrintRType(cputype, r_type); + + // scattered: scattered & value + outs() << format("True 0x%08x", (unsigned int)r_value); + if (previous_sectdiff == false) { + if ((cputype == MachO::CPU_TYPE_ARM && + r_type == llvm::MachO::ARM_RELOC_PAIR)) + outs() << format(" half = 0x%04x ", (unsigned int)r_address); + } + else if (cputype == MachO::CPU_TYPE_ARM && + sectdiff_r_type == llvm::MachO::ARM_RELOC_HALF_SECTDIFF) + outs() << format(" other_half = 0x%04x ", (unsigned int)r_address); + if ((cputype == MachO::CPU_TYPE_I386 && + (r_type == llvm::MachO::GENERIC_RELOC_SECTDIFF || + r_type == llvm::MachO::GENERIC_RELOC_LOCAL_SECTDIFF)) || + (cputype == MachO::CPU_TYPE_ARM && + (sectdiff_r_type == llvm::MachO::ARM_RELOC_SECTDIFF || + sectdiff_r_type == llvm::MachO::ARM_RELOC_LOCAL_SECTDIFF || + sectdiff_r_type == llvm::MachO::ARM_RELOC_HALF_SECTDIFF))) { + previous_sectdiff = true; + sectdiff_r_type = r_type; + } + else { + previous_sectdiff = false; + sectdiff_r_type = 0; + } + if (cputype == MachO::CPU_TYPE_ARM && + (r_type == llvm::MachO::ARM_RELOC_HALF || + r_type == llvm::MachO::ARM_RELOC_HALF_SECTDIFF)) + previous_arm_half = true; + else + previous_arm_half = false; + outs() << "\n"; + } + else { + // scattered: address pcrel length extern type scattered value + outs() << format("%08x %1d %-2d n/a %-7d 1 0x%08x\n", + (unsigned int)r_address, r_pcrel, r_length, r_type, + (unsigned int)r_value); + } + } + else { + if (verbose) { + // plain: address + if (cputype == MachO::CPU_TYPE_ARM && + r_type == llvm::MachO::ARM_RELOC_PAIR) + outs() << " "; + else + outs() << format("%08x ", (unsigned int)r_address); + + // plain: pcrel + if (r_pcrel) + outs() << "True "; + else + outs() << "False "; + + // plain: length + PrintRLength(cputype, r_type, r_length, previous_arm_half); + + if (r_extern) { + // plain: extern & type & scattered + outs() << "True "; + PrintRType(cputype, r_type); + outs() << "False "; + + // plain: symbolnum/value + if (r_symbolnum > Symtab.nsyms) + outs() << format("?(%d)\n", r_symbolnum); + else { + SymbolRef Symbol = *O->getSymbolByIndex(r_symbolnum); + Expected<StringRef> SymNameNext = Symbol.getName(); + const char *name = NULL; + if (SymNameNext) + name = SymNameNext->data(); + if (name == NULL) + outs() << format("?(%d)\n", r_symbolnum); + else + outs() << name << "\n"; + } + } + else { + // plain: extern & type & scattered + outs() << "False "; + PrintRType(cputype, r_type); + outs() << "False "; + + // plain: symbolnum/value + if (cputype == MachO::CPU_TYPE_ARM && + r_type == llvm::MachO::ARM_RELOC_PAIR) + outs() << format("other_half = 0x%04x\n", (unsigned int)r_address); + else if (cputype == MachO::CPU_TYPE_ARM64 && + r_type == llvm::MachO::ARM64_RELOC_ADDEND) + outs() << format("addend = 0x%06x\n", (unsigned int)r_symbolnum); + else { + outs() << format("%d ", r_symbolnum); + if (r_symbolnum == llvm::MachO::R_ABS) + outs() << "R_ABS\n"; + else { + // in this case, r_symbolnum is actually a 1-based section number + uint32_t nsects = O->section_end()->getRawDataRefImpl().d.a; + if (r_symbolnum > 0 && r_symbolnum <= nsects) { + llvm::object::DataRefImpl DRI; + DRI.d.a = r_symbolnum-1; + StringRef SegName = O->getSectionFinalSegmentName(DRI); + StringRef SectName; + if (O->getSectionName(DRI, SectName)) + outs() << "(?,?)\n"; + else + outs() << "(" << SegName << "," << SectName << ")\n"; + } + else { + outs() << "(?,?)\n"; + } + } + } + } + if (cputype == MachO::CPU_TYPE_ARM && + (r_type == llvm::MachO::ARM_RELOC_HALF || + r_type == llvm::MachO::ARM_RELOC_HALF_SECTDIFF)) + previous_arm_half = true; + else + previous_arm_half = false; + } + else { + // plain: address pcrel length extern type scattered symbolnum/section + outs() << format("%08x %1d %-2d %1d %-7d 0 %d\n", + (unsigned int)r_address, r_pcrel, r_length, r_extern, + r_type, r_symbolnum); + } + } + } +} + +static void PrintRelocations(const MachOObjectFile *O, const bool verbose) { + const uint64_t cputype = O->getHeader().cputype; + const MachO::dysymtab_command Dysymtab = O->getDysymtabLoadCommand(); + if (Dysymtab.nextrel != 0) { + outs() << "External relocation information " << Dysymtab.nextrel + << " entries"; + outs() << "\naddress pcrel length extern type scattered " + "symbolnum/value\n"; + PrintRelocationEntries(O, O->extrel_begin(), O->extrel_end(), cputype, + verbose); + } + if (Dysymtab.nlocrel != 0) { + outs() << format("Local relocation information %u entries", + Dysymtab.nlocrel); + outs() << "\naddress pcrel length extern type scattered " + "symbolnum/value\n"; + PrintRelocationEntries(O, O->locrel_begin(), O->locrel_end(), cputype, + verbose); + } + for (const auto &Load : O->load_commands()) { + if (Load.C.cmd == MachO::LC_SEGMENT_64) { + const MachO::segment_command_64 Seg = O->getSegment64LoadCommand(Load); + for (unsigned J = 0; J < Seg.nsects; ++J) { + const MachO::section_64 Sec = O->getSection64(Load, J); + if (Sec.nreloc != 0) { + DataRefImpl DRI; + DRI.d.a = J; + const StringRef SegName = O->getSectionFinalSegmentName(DRI); + StringRef SectName; + if (O->getSectionName(DRI, SectName)) + outs() << "Relocation information (" << SegName << ",?) " + << format("%u entries", Sec.nreloc); + else + outs() << "Relocation information (" << SegName << "," + << SectName << format(") %u entries", Sec.nreloc); + outs() << "\naddress pcrel length extern type scattered " + "symbolnum/value\n"; + PrintRelocationEntries(O, O->section_rel_begin(DRI), + O->section_rel_end(DRI), cputype, verbose); + } + } + } else if (Load.C.cmd == MachO::LC_SEGMENT) { + const MachO::segment_command Seg = O->getSegmentLoadCommand(Load); + for (unsigned J = 0; J < Seg.nsects; ++J) { + const MachO::section Sec = O->getSection(Load, J); + if (Sec.nreloc != 0) { + DataRefImpl DRI; + DRI.d.a = J; + const StringRef SegName = O->getSectionFinalSegmentName(DRI); + StringRef SectName; + if (O->getSectionName(DRI, SectName)) + outs() << "Relocation information (" << SegName << ",?) " + << format("%u entries", Sec.nreloc); + else + outs() << "Relocation information (" << SegName << "," + << SectName << format(") %u entries", Sec.nreloc); + outs() << "\naddress pcrel length extern type scattered " + "symbolnum/value\n"; + PrintRelocationEntries(O, O->section_rel_begin(DRI), + O->section_rel_end(DRI), cputype, verbose); + } + } + } + } +} + static void PrintDataInCodeTable(MachOObjectFile *O, bool verbose) { MachO::linkedit_data_command DIC = O->getDataInCodeLoadCommand(); uint32_t nentries = DIC.datasize / sizeof(struct MachO::data_in_code_entry); @@ -1192,9 +1548,10 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF, // If we are doing some processing here on the Mach-O file print the header // info. And don't print it otherwise like in the case of printing the // UniversalHeaders or ArchiveHeaders. - if (Disassemble || PrivateHeaders || ExportsTrie || Rebase || Bind || SymbolTable || - LazyBind || WeakBind || IndirectSymbols || DataInCode || LinkOptHints || - DylibsUsed || DylibId || ObjcMetaData || (FilterSections.size() != 0)) { + if (Disassemble || Relocations || PrivateHeaders || ExportsTrie || Rebase || + Bind || SymbolTable || LazyBind || WeakBind || IndirectSymbols || + DataInCode || LinkOptHints || DylibsUsed || DylibId || ObjcMetaData || + (FilterSections.size() != 0)) { if (!NoLeadingHeaders) { outs() << Name; if (!ArchiveMemberName.empty()) @@ -1226,7 +1583,7 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF, if (Disassemble) { if (MachOOF->getHeader().filetype == MachO::MH_KEXT_BUNDLE && - MachOOF->getHeader().cputype == MachO::CPU_TYPE_ARM64) + MachOOF->getHeader().cputype == MachO::CPU_TYPE_ARM64) DisassembleMachO(FileName, MachOOF, "__TEXT_EXEC", "__text"); else DisassembleMachO(FileName, MachOOF, "__TEXT", "__text"); @@ -1238,7 +1595,7 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF, if (LinkOptHints) PrintLinkOptHints(MachOOF); if (Relocations) - PrintRelocations(MachOOF); + PrintRelocations(MachOOF, !NonVerbose); if (SectionHeaders) PrintSectionHeaders(MachOOF); if (SectionContents) @@ -1275,11 +1632,10 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF, printWeakBindTable(MachOOF); if (DwarfDumpType != DIDT_Null) { - std::unique_ptr<DIContext> DICtx(new DWARFContextInMemory(*MachOOF)); + std::unique_ptr<DIContext> DICtx = DWARFContext::create(*MachOOF); // Dump the complete DWARF structure. DIDumpOptions DumpOpts; DumpOpts.DumpType = DwarfDumpType; - DumpOpts.DumpEH = true; DICtx->dump(outs(), DumpOpts); } } @@ -5803,14 +6159,12 @@ static void PrintModeVerbose(uint32_t mode) { } static void PrintXarFilesSummary(const char *XarFilename, xar_t xar) { - xar_iter_t xi; xar_file_t xf; - xar_iter_t xp; const char *key, *type, *mode, *user, *group, *size, *mtime, *name, *m; char *endp; uint32_t mode_value; - xi = xar_iter_new(); + ScopedXarIter xi; if (!xi) { errs() << "Can't obtain an xar iterator for xar archive " << XarFilename << "\n"; @@ -5819,7 +6173,7 @@ static void PrintXarFilesSummary(const char *XarFilename, xar_t xar) { // Go through the xar's files. for (xf = xar_file_first(xar, xi); xf; xf = xar_file_next(xi)) { - xp = xar_iter_new(); + ScopedXarIter xp; if(!xp){ errs() << "Can't obtain an xar iterator for xar archive " << XarFilename << "\n"; @@ -5833,7 +6187,7 @@ static void PrintXarFilesSummary(const char *XarFilename, xar_t xar) { mtime = nullptr; name = nullptr; for(key = xar_prop_first(xf, xp); key; key = xar_prop_next(xp)){ - const char *val = nullptr; + const char *val = nullptr; xar_prop_get(xf, key, &val); #if 0 // Useful for debugging. outs() << "key: " << key << " value: " << val << "\n"; @@ -5949,7 +6303,7 @@ static void DumpBitcodeSection(MachOObjectFile *O, const char *sect, errs() << XarEC.message() << "\n"; return; } - tool_output_file XarFile(XarFilename, FD); + ToolOutputFile XarFile(XarFilename, FD); raw_fd_ostream &XarOut = XarFile.os(); StringRef XarContents(sect, size); XarOut << XarContents; @@ -5957,7 +6311,7 @@ static void DumpBitcodeSection(MachOObjectFile *O, const char *sect, if (XarOut.has_error()) return; - xar_t xar = xar_open(XarFilename.c_str(), READ); + ScopedXarFile xar(XarFilename.c_str(), READ); if (!xar) { errs() << "Can't create temporary xar archive " << XarFilename << "\n"; return; @@ -5997,41 +6351,38 @@ static void DumpBitcodeSection(MachOObjectFile *O, const char *sect, outs() << Buffer->getBuffer() << "\n"; // TODO: Go through the xar's files. - xar_iter_t xi = xar_iter_new(); + ScopedXarIter xi; if(!xi){ errs() << "Can't obtain an xar iterator for xar archive " << XarFilename.c_str() << "\n"; - xar_close(xar); return; } for(xar_file_t xf = xar_file_first(xar, xi); xf; xf = xar_file_next(xi)){ const char *key; - xar_iter_t xp; const char *member_name, *member_type, *member_size_string; size_t member_size; - xp = xar_iter_new(); + ScopedXarIter xp; if(!xp){ errs() << "Can't obtain an xar iterator for xar archive " - << XarFilename.c_str() << "\n"; - xar_close(xar); + << XarFilename.c_str() << "\n"; return; } member_name = NULL; member_type = NULL; member_size_string = NULL; for(key = xar_prop_first(xf, xp); key; key = xar_prop_next(xp)){ - const char *val = nullptr; + const char *val = nullptr; xar_prop_get(xf, key, &val); #if 0 // Useful for debugging. outs() << "key: " << key << " value: " << val << "\n"; #endif - if(strcmp(key, "name") == 0) - member_name = val; - if(strcmp(key, "type") == 0) - member_type = val; - if(strcmp(key, "data/size") == 0) - member_size_string = val; + if (strcmp(key, "name") == 0) + member_name = val; + if (strcmp(key, "type") == 0) + member_type = val; + if (strcmp(key, "data/size") == 0) + member_size_string = val; } /* * If we find a file with a name, date/size and type properties @@ -6044,44 +6395,42 @@ static void DumpBitcodeSection(MachOObjectFile *O, const char *sect, char *endptr; member_size = strtoul(member_size_string, &endptr, 10); if (*endptr == '\0' && member_size != 0) { - char *buffer = (char *) ::operator new (member_size); - if (xar_extract_tobuffersz(xar, xf, &buffer, &member_size) == 0) { + char *buffer; + if (xar_extract_tobuffersz(xar, xf, &buffer, &member_size) == 0) { #if 0 // Useful for debugging. - outs() << "xar member: " << member_name << " extracted\n"; + outs() << "xar member: " << member_name << " extracted\n"; #endif // Set the XarMemberName we want to see printed in the header. - std::string OldXarMemberName; - // If XarMemberName is already set this is nested. So - // save the old name and create the nested name. - if (!XarMemberName.empty()) { - OldXarMemberName = XarMemberName; + std::string OldXarMemberName; + // If XarMemberName is already set this is nested. So + // save the old name and create the nested name. + if (!XarMemberName.empty()) { + OldXarMemberName = XarMemberName; XarMemberName = - (Twine("[") + XarMemberName + "]" + member_name).str(); - } else { - OldXarMemberName = ""; - XarMemberName = member_name; - } - // See if this is could be a xar file (nested). - if (member_size >= sizeof(struct xar_header)) { + (Twine("[") + XarMemberName + "]" + member_name).str(); + } else { + OldXarMemberName = ""; + XarMemberName = member_name; + } + // See if this is could be a xar file (nested). + if (member_size >= sizeof(struct xar_header)) { #if 0 // Useful for debugging. - outs() << "could be a xar file: " << member_name << "\n"; + outs() << "could be a xar file: " << member_name << "\n"; #endif - memcpy((char *)&XarHeader, buffer, sizeof(struct xar_header)); + memcpy((char *)&XarHeader, buffer, sizeof(struct xar_header)); if (sys::IsLittleEndianHost) - swapStruct(XarHeader); - if(XarHeader.magic == XAR_HEADER_MAGIC) - DumpBitcodeSection(O, buffer, member_size, verbose, + swapStruct(XarHeader); + if (XarHeader.magic == XAR_HEADER_MAGIC) + DumpBitcodeSection(O, buffer, member_size, verbose, PrintXarHeader, PrintXarFileHeaders, - XarMemberName); - } - XarMemberName = OldXarMemberName; - } - delete buffer; + XarMemberName); + } + XarMemberName = OldXarMemberName; + delete buffer; + } } } - xar_iter_free(xp); } - xar_close(xar); } #endif // defined(HAVE_LIBXAR) @@ -6437,8 +6786,11 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF, // GetTarget prints out stuff. return; } + std::string MachOMCPU; if (MCPU.empty() && McpuDefault) - MCPU = McpuDefault; + MachOMCPU = McpuDefault; + else + MachOMCPU = MCPU; std::unique_ptr<const MCInstrInfo> InstrInfo(TheTarget->createMCInstrInfo()); std::unique_ptr<const MCInstrInfo> ThumbInstrInfo; @@ -6460,7 +6812,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF, std::unique_ptr<const MCAsmInfo> AsmInfo( TheTarget->createMCAsmInfo(*MRI, TripleName)); std::unique_ptr<const MCSubtargetInfo> STI( - TheTarget->createMCSubtargetInfo(TripleName, MCPU, FeaturesStr)); + TheTarget->createMCSubtargetInfo(TripleName, MachOMCPU, FeaturesStr)); MCContext Ctx(AsmInfo.get(), MRI.get(), nullptr); std::unique_ptr<MCDisassembler> DisAsm( TheTarget->createMCDisassembler(*STI, Ctx)); @@ -6510,7 +6862,8 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF, ThumbAsmInfo.reset( ThumbTarget->createMCAsmInfo(*ThumbMRI, ThumbTripleName)); ThumbSTI.reset( - ThumbTarget->createMCSubtargetInfo(ThumbTripleName, MCPU, FeaturesStr)); + ThumbTarget->createMCSubtargetInfo(ThumbTripleName, MachOMCPU, + FeaturesStr)); ThumbCtx.reset(new MCContext(ThumbAsmInfo.get(), ThumbMRI.get(), nullptr)); ThumbDisAsm.reset(ThumbTarget->createMCDisassembler(*ThumbSTI, *ThumbCtx)); MCContext *PtrThumbCtx = ThumbCtx.get(); @@ -6594,7 +6947,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF, } // Setup the DIContext - diContext.reset(new DWARFContextInMemory(*DbgObj)); + diContext = DWARFContext::create(*DbgObj); } if (FilterSections.size() == 0) @@ -6703,7 +7056,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF, if (!DisSymName.empty() && DisSymName == SymName) { outs() << "-dis-symname: " << DisSymName << " not in the section\n"; return; - } + } continue; } // The __mh_execute_header is special and we need to deal with that fact @@ -9402,7 +9755,8 @@ void llvm::printMachOExportsTrie(const object::MachOObjectFile *Obj) { } } } - for (const llvm::object::ExportEntry &Entry : Obj->exports()) { + Error Err = Error::success(); + for (const llvm::object::ExportEntry &Entry : Obj->exports(Err)) { uint64_t Flags = Entry.flags(); bool ReExport = (Flags & MachO::EXPORT_SYMBOL_FLAGS_REEXPORT); bool WeakDef = (Flags & MachO::EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION); @@ -9455,6 +9809,8 @@ void llvm::printMachOExportsTrie(const object::MachOObjectFile *Obj) { } outs() << "\n"; } + if (Err) + report_error(Obj->getFileName(), std::move(Err)); } //===----------------------------------------------------------------------===// @@ -9608,3 +9964,4 @@ static const char *get_dyld_bind_info_symbolname(uint64_t ReferenceValue, auto name = info->bindtable->lookup(ReferenceValue); return !name.empty() ? name.data() : nullptr; } + diff --git a/tools/llvm-objdump/llvm-objdump.cpp b/tools/llvm-objdump/llvm-objdump.cpp index 74593e6202aa..79204c6e9533 100644 --- a/tools/llvm-objdump/llvm-objdump.cpp +++ b/tools/llvm-objdump/llvm-objdump.cpp @@ -62,8 +62,8 @@ #include <cctype> #include <cstring> #include <system_error> -#include <utility> #include <unordered_map> +#include <utility> using namespace llvm; using namespace object; @@ -191,7 +191,7 @@ cl::opt<bool> PrintFaultMaps("fault-map-section", cl::opt<DIDumpType> llvm::DwarfDumpType( "dwarf", cl::init(DIDT_Null), cl::desc("Dump of dwarf debug sections:"), - cl::values(clEnumValN(DIDT_Frames, "frames", ".debug_frame"))); + cl::values(clEnumValN(DIDT_DebugFrame, "frames", ".debug_frame"))); cl::opt<bool> PrintSource( "source", @@ -362,29 +362,11 @@ static const Target *getTarget(const ObjectFile *Obj = nullptr) { llvm::Triple TheTriple("unknown-unknown-unknown"); if (TripleName.empty()) { if (Obj) { - auto Arch = Obj->getArch(); - TheTriple.setArch(Triple::ArchType(Arch)); - - // For ARM targets, try to use the build attributes to build determine - // the build target. Target features are also added, but later during - // disassembly. - if (Arch == Triple::arm || Arch == Triple::armeb) { - Obj->setARMSubArch(TheTriple); - } - - // TheTriple defaults to ELF, and COFF doesn't have an environment: - // the best we can do here is indicate that it is mach-o. - if (Obj->isMachO()) - TheTriple.setObjectFormat(Triple::MachO); - - if (Obj->isCOFF()) { - const auto COFFObj = dyn_cast<COFFObjectFile>(Obj); - if (COFFObj->getArch() == Triple::thumb) - TheTriple.setTriple("thumbv7-windows"); - } + TheTriple = Obj->makeTriple(); } } else { TheTriple.setTriple(Triple::normalize(TripleName)); + // Use the triple, but also try to combine with ARM build attributes. if (Obj) { auto Arch = Obj->getArch(); @@ -418,7 +400,7 @@ namespace { class SourcePrinter { protected: DILineInfo OldLineInfo; - const ObjectFile *Obj; + const ObjectFile *Obj = nullptr; std::unique_ptr<symbolize::LLVMSymbolizer> Symbolizer; // File name to file contents of source std::unordered_map<std::string, std::unique_ptr<MemoryBuffer>> SourceCache; @@ -426,22 +408,22 @@ protected: std::unordered_map<std::string, std::vector<StringRef>> LineCache; private: - bool cacheSource(std::string File); + bool cacheSource(const std::string& File); public: - virtual ~SourcePrinter() {} - SourcePrinter() : Obj(nullptr), Symbolizer(nullptr) {} + SourcePrinter() = default; SourcePrinter(const ObjectFile *Obj, StringRef DefaultArch) : Obj(Obj) { symbolize::LLVMSymbolizer::Options SymbolizerOpts( DILineInfoSpecifier::FunctionNameKind::None, true, false, false, DefaultArch); Symbolizer.reset(new symbolize::LLVMSymbolizer(SymbolizerOpts)); } + virtual ~SourcePrinter() = default; virtual void printSourceLine(raw_ostream &OS, uint64_t Address, StringRef Delimiter = "; "); }; -bool SourcePrinter::cacheSource(std::string File) { +bool SourcePrinter::cacheSource(const std::string& File) { auto BufferOrError = MemoryBuffer::getFile(File); if (!BufferOrError) return false; @@ -509,7 +491,7 @@ static bool isArmElf(const ObjectFile *Obj) { class PrettyPrinter { public: - virtual ~PrettyPrinter(){} + virtual ~PrettyPrinter() = default; virtual void printInst(MCInstPrinter &IP, const MCInst *MI, ArrayRef<uint8_t> Bytes, uint64_t Address, raw_ostream &OS, StringRef Annot, @@ -883,8 +865,19 @@ static void printRelocationTargetName(const MachOObjectFile *O, } else { section_iterator SI = O->section_begin(); // Adjust for the fact that sections are 1-indexed. - advance(SI, Val - 1); - SI->getName(S); + if (Val == 0) { + fmt << "0 (?,?)"; + return; + } + uint32_t i = Val - 1; + while (i != 0 && SI != O->section_end()) { + i--; + advance(SI, 1); + } + if (SI == O->section_end()) + fmt << Val << " (?,?)"; + else + SI->getName(S); } fmt << S; @@ -1223,7 +1216,7 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) { MCObjectFileInfo MOFI; MCContext Ctx(AsmInfo.get(), MRI.get(), &MOFI); // FIXME: for now initialize MCObjectFileInfo with default values - MOFI.InitMCObjectFileInfo(Triple(TripleName), false, CodeModel::Default, Ctx); + MOFI.InitMCObjectFileInfo(Triple(TripleName), false, Ctx); std::unique_ptr<MCDisassembler> DisAsm( TheTarget->createMCDisassembler(*STI, Ctx)); @@ -2081,11 +2074,10 @@ static void DumpObject(ObjectFile *o, const Archive *a = nullptr) { if (PrintFaultMaps) printFaultMaps(o); if (DwarfDumpType != DIDT_Null) { - std::unique_ptr<DIContext> DICtx(new DWARFContextInMemory(*o)); + std::unique_ptr<DIContext> DICtx = DWARFContext::create(*o); // Dump the complete DWARF structure. DIDumpOptions DumpOpts; DumpOpts.DumpType = DwarfDumpType; - DumpOpts.DumpEH = true; DICtx->dump(outs(), DumpOpts); } } @@ -2205,8 +2197,7 @@ int main(int argc, char **argv) { return 2; } - std::for_each(InputFilenames.begin(), InputFilenames.end(), - DumpInput); + llvm::for_each(InputFilenames, DumpInput); return EXIT_SUCCESS; } diff --git a/tools/llvm-opt-fuzzer/CMakeLists.txt b/tools/llvm-opt-fuzzer/CMakeLists.txt new file mode 100644 index 000000000000..d2fb07f96fc8 --- /dev/null +++ b/tools/llvm-opt-fuzzer/CMakeLists.txt @@ -0,0 +1,25 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + Analysis + BitReader + BitWriter + CodeGen + Core + Coroutines + IPO + IRReader + InstCombine + Instrumentation + FuzzMutate + MC + ObjCARCOpts + ScalarOpts + Support + Target + TransformUtils + Vectorize + Passes +) + +add_llvm_fuzzer(llvm-opt-fuzzer llvm-opt-fuzzer.cpp + DUMMY_MAIN DummyOptFuzzer.cpp) diff --git a/tools/llvm-opt-fuzzer/DummyOptFuzzer.cpp b/tools/llvm-opt-fuzzer/DummyOptFuzzer.cpp new file mode 100644 index 000000000000..115923fe33c2 --- /dev/null +++ b/tools/llvm-opt-fuzzer/DummyOptFuzzer.cpp @@ -0,0 +1,21 @@ +//===--- DummyOptFuzzer.cpp - Entry point to sanity check the fuzzer ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementation of main so we can build and test without linking libFuzzer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/FuzzMutate/FuzzerCLI.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); +extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv); +int main(int argc, char *argv[]) { + return llvm::runFuzzerOnInputs(argc, argv, LLVMFuzzerTestOneInput, + LLVMFuzzerInitialize); +} diff --git a/tools/llvm-opt-fuzzer/llvm-opt-fuzzer.cpp b/tools/llvm-opt-fuzzer/llvm-opt-fuzzer.cpp new file mode 100644 index 000000000000..8187bbcea668 --- /dev/null +++ b/tools/llvm-opt-fuzzer/llvm-opt-fuzzer.cpp @@ -0,0 +1,222 @@ +//===--- llvm-opt-fuzzer.cpp - Fuzzer for instruction selection ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Tool to fuzz optimization passes using libFuzzer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CodeGen/CommandFlags.def" +#include "llvm/FuzzMutate/FuzzerCLI.h" +#include "llvm/FuzzMutate/IRMutator.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" + +using namespace llvm; + +static cl::opt<std::string> + TargetTripleStr("mtriple", cl::desc("Override target triple for module")); + +// Passes to run for this fuzzer instance. Expects new pass manager syntax. +static cl::opt<std::string> PassPipeline( + "passes", + cl::desc("A textual description of the pass pipeline for testing")); + +static std::unique_ptr<IRMutator> Mutator; +static std::unique_ptr<TargetMachine> TM; + +std::unique_ptr<IRMutator> createOptMutator() { + std::vector<TypeGetter> Types{ + Type::getInt1Ty, Type::getInt8Ty, Type::getInt16Ty, Type::getInt32Ty, + Type::getInt64Ty, Type::getFloatTy, Type::getDoubleTy}; + + std::vector<std::unique_ptr<IRMutationStrategy>> Strategies; + Strategies.push_back( + llvm::make_unique<InjectorIRStrategy>( + InjectorIRStrategy::getDefaultOps())); + Strategies.push_back( + llvm::make_unique<InstDeleterIRStrategy>()); + + return llvm::make_unique<IRMutator>(std::move(Types), std::move(Strategies)); +} + +extern "C" LLVM_ATTRIBUTE_USED size_t LLVMFuzzerCustomMutator( + uint8_t *Data, size_t Size, size_t MaxSize, unsigned int Seed) { + + assert(Mutator && + "IR mutator should have been created during fuzzer initialization"); + + LLVMContext Context; + auto M = parseModule(Data, Size, Context); + if (!M || verifyModule(*M, &errs())) { + errs() << "error: mutator input module is broken!\n"; + return 0; + } + + Mutator->mutateModule(*M, Seed, Size, MaxSize); + +#ifndef NDEBUG + if (verifyModule(*M, &errs())) { + errs() << "mutation result doesn't pass verification\n"; + M->dump(); + abort(); + } +#endif + + return writeModule(*M, Data, MaxSize); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + assert(TM && "Should have been created during fuzzer initialization"); + + if (Size <= 1) + // We get bogus data given an empty corpus - ignore it. + return 0; + + // Parse module + // + + LLVMContext Context; + auto M = parseModule(Data, Size, Context); + if (!M || verifyModule(*M, &errs())) { + errs() << "error: input module is broken!\n"; + return 0; + } + + // Set up target dependant options + // + + M->setTargetTriple(TM->getTargetTriple().normalize()); + M->setDataLayout(TM->createDataLayout()); + setFunctionAttributes(TM->getTargetCPU(), TM->getTargetFeatureString(), *M); + + // Create pass pipeline + // + + PassBuilder PB(TM.get()); + + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModulePassManager MPM; + ModuleAnalysisManager MAM; + + FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + bool Ok = PB.parsePassPipeline(MPM, PassPipeline, false, false); + assert(Ok && "Should have been checked during fuzzer initialization"); + (void)Ok; // silence unused variable warning on release builds + + // Run passes which we need to test + // + + MPM.run(*M, MAM); + + // Check that passes resulted in a correct code + if (verifyModule(*M, &errs())) { + errs() << "Transformation resulted in an invalid module\n"; + abort(); + } + + return 0; +} + +static void handleLLVMFatalError(void *, const std::string &Message, bool) { + // TODO: Would it be better to call into the fuzzer internals directly? + dbgs() << "LLVM ERROR: " << Message << "\n" + << "Aborting to trigger fuzzer exit handling.\n"; + abort(); +} + +extern "C" LLVM_ATTRIBUTE_USED int LLVMFuzzerInitialize( + int *argc, char ***argv) { + EnableDebugBuffering = true; + + // Make sure we print the summary and the current unit when LLVM errors out. + install_fatal_error_handler(handleLLVMFatalError, nullptr); + + // Initialize llvm + // + + InitializeAllTargets(); + InitializeAllTargetMCs(); + + PassRegistry &Registry = *PassRegistry::getPassRegistry(); + initializeCore(Registry); + initializeCoroutines(Registry); + initializeScalarOpts(Registry); + initializeObjCARCOpts(Registry); + initializeVectorization(Registry); + initializeIPO(Registry); + initializeAnalysis(Registry); + initializeTransformUtils(Registry); + initializeInstCombine(Registry); + initializeInstrumentation(Registry); + initializeTarget(Registry); + + // Parse input options + // + + handleExecNameEncodedOptimizerOpts(*argv[0]); + parseFuzzerCLOpts(*argc, *argv); + + // Create TargetMachine + // + + if (TargetTripleStr.empty()) { + errs() << *argv[0] << ": -mtriple must be specified\n"; + exit(1); + } + Triple TargetTriple = Triple(Triple::normalize(TargetTripleStr)); + + std::string Error; + const Target *TheTarget = + TargetRegistry::lookupTarget(MArch, TargetTriple, Error); + if (!TheTarget) { + errs() << *argv[0] << ": " << Error; + exit(1); + } + + TargetOptions Options = InitTargetOptionsFromCodeGenFlags(); + TM.reset(TheTarget->createTargetMachine( + TargetTriple.getTriple(), getCPUStr(), getFeaturesStr(), + Options, getRelocModel(), getCodeModel(), CodeGenOpt::Default)); + assert(TM && "Could not allocate target machine!"); + + // Check that pass pipeline is specified and correct + // + + if (PassPipeline.empty()) { + errs() << *argv[0] << ": at least one pass should be specified\n"; + exit(1); + } + + PassBuilder PB(TM.get()); + ModulePassManager MPM; + if (!PB.parsePassPipeline(MPM, PassPipeline, false, false)) { + errs() << *argv[0] << ": can't parse pass pipeline\n"; + exit(1); + } + + // Create mutator + // + + Mutator = createOptMutator(); + + return 0; +} diff --git a/tools/llvm-opt-report/OptReport.cpp b/tools/llvm-opt-report/OptReport.cpp index 4f45dd9f2aa2..3c6115db6ac0 100644 --- a/tools/llvm-opt-report/OptReport.cpp +++ b/tools/llvm-opt-report/OptReport.cpp @@ -514,8 +514,10 @@ int main(int argc, const char **argv) { "A tool to generate an optimization report from YAML optimization" " record files.\n"); - if (Help) + if (Help) { cl::PrintHelpMessage(); + return 0; + } LocationInfoTy LocationInfo; if (!readLocationInfo(LocationInfo)) diff --git a/tools/llvm-pdbutil/BytesOutputStyle.cpp b/tools/llvm-pdbutil/BytesOutputStyle.cpp index 9e5a28c2b98b..2b96c8f986aa 100644 --- a/tools/llvm-pdbutil/BytesOutputStyle.cpp +++ b/tools/llvm-pdbutil/BytesOutputStyle.cpp @@ -15,6 +15,7 @@ #include "llvm/DebugInfo/CodeView/Formatters.h" #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" +#include "llvm/DebugInfo/MSF/MSFCommon.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/DbiStream.h" #include "llvm/DebugInfo/PDB/Native/InfoStream.h" @@ -120,6 +121,11 @@ Error BytesOutputStyle::dump() { P.NewLine(); } + if (opts::bytes::Fpm) { + dumpFpm(); + P.NewLine(); + } + if (!opts::bytes::DumpStreamData.empty()) { dumpStreamBytes(); P.NewLine(); @@ -401,27 +407,6 @@ void BytesOutputStyle::dumpModuleC11() { }); } -static std::string formatChunkKind(DebugSubsectionKind Kind) { - switch (Kind) { - RETURN_CASE(DebugSubsectionKind, None, "none"); - RETURN_CASE(DebugSubsectionKind, Symbols, "symbols"); - RETURN_CASE(DebugSubsectionKind, Lines, "lines"); - RETURN_CASE(DebugSubsectionKind, StringTable, "strings"); - RETURN_CASE(DebugSubsectionKind, FileChecksums, "checksums"); - RETURN_CASE(DebugSubsectionKind, FrameData, "frames"); - RETURN_CASE(DebugSubsectionKind, InlineeLines, "inlinee lines"); - RETURN_CASE(DebugSubsectionKind, CrossScopeImports, "xmi"); - RETURN_CASE(DebugSubsectionKind, CrossScopeExports, "xme"); - RETURN_CASE(DebugSubsectionKind, ILLines, "il lines"); - RETURN_CASE(DebugSubsectionKind, FuncMDTokenMap, "func md token map"); - RETURN_CASE(DebugSubsectionKind, TypeMDTokenMap, "type md token map"); - RETURN_CASE(DebugSubsectionKind, MergedAssemblyInput, - "merged assembly input"); - RETURN_CASE(DebugSubsectionKind, CoffSymbolRVA, "coff symbol rva"); - } - return formatUnknownEnum(Kind); -} - void BytesOutputStyle::dumpModuleC13() { printHeader(P, "Debug Chunks"); @@ -480,6 +465,13 @@ BytesOutputStyle::initializeTypes(uint32_t StreamIdx) { return *TypeCollection; } +void BytesOutputStyle::dumpFpm() { + printHeader(P, "Free Page Map"); + + msf::MSFStreamLayout FpmLayout = File.getFpmStreamLayout(); + P.formatMsfStreamBlocks(File, FpmLayout); +} + void BytesOutputStyle::dumpStreamBytes() { if (StreamPurposes.empty()) discoverStreamPurposes(File, StreamPurposes); @@ -495,7 +487,8 @@ void BytesOutputStyle::dumpStreamBytes() { P.formatLine("Stream {0}: Not present", Spec.SI); continue; } - P.formatMsfStreamData("Data", File, Spec.SI, StreamPurposes[Spec.SI], - Spec.Begin, Spec.Size); + P.formatMsfStreamData("Data", File, Spec.SI, + StreamPurposes[Spec.SI].getShortName(), Spec.Begin, + Spec.Size); } } diff --git a/tools/llvm-pdbutil/BytesOutputStyle.h b/tools/llvm-pdbutil/BytesOutputStyle.h index c162163fdd01..aa5342998e56 100644 --- a/tools/llvm-pdbutil/BytesOutputStyle.h +++ b/tools/llvm-pdbutil/BytesOutputStyle.h @@ -12,6 +12,7 @@ #include "LinePrinter.h" #include "OutputStyle.h" +#include "StreamUtil.h" #include "llvm/Support/Error.h" @@ -35,6 +36,7 @@ private: void dumpNameMap(); void dumpBlockRanges(uint32_t Min, uint32_t Max); void dumpByteRanges(uint32_t Min, uint32_t Max); + void dumpFpm(); void dumpStreamBytes(); void dumpSectionContributions(); @@ -59,7 +61,7 @@ private: PDBFile &File; LinePrinter P; ExitOnError Err; - SmallVector<std::string, 8> StreamPurposes; + SmallVector<StreamInfo, 8> StreamPurposes; }; } // namespace pdb } // namespace llvm diff --git a/tools/llvm-pdbutil/CMakeLists.txt b/tools/llvm-pdbutil/CMakeLists.txt index bc28e6bdd7ea..ab4ce1ac30e0 100644 --- a/tools/llvm-pdbutil/CMakeLists.txt +++ b/tools/llvm-pdbutil/CMakeLists.txt @@ -1,4 +1,5 @@ set(LLVM_LINK_COMPONENTS + BinaryFormat DebugInfoCodeView DebugInfoMSF DebugInfoPDB @@ -13,6 +14,7 @@ add_llvm_tool(llvm-pdbutil Diff.cpp DiffPrinter.cpp DumpOutputStyle.cpp + InputFile.cpp llvm-pdbutil.cpp FormatUtil.cpp LinePrinter.cpp @@ -32,7 +34,3 @@ add_llvm_tool(llvm-pdbutil StreamUtil.cpp YAMLOutputStyle.cpp ) - -if(LLVM_USE_SANITIZE_COVERAGE) - add_subdirectory(fuzzer) -endif() diff --git a/tools/llvm-pdbutil/Diff.cpp b/tools/llvm-pdbutil/Diff.cpp index aad4e1bf1427..286dc51c29b6 100644 --- a/tools/llvm-pdbutil/Diff.cpp +++ b/tools/llvm-pdbutil/Diff.cpp @@ -23,7 +23,6 @@ #include "llvm/DebugInfo/PDB/Native/PDBStringTable.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/Support/FileSystem.h" #include "llvm/Support/FormatAdapters.h" #include "llvm/Support/FormatProviders.h" #include "llvm/Support/FormatVariadic.h" @@ -135,29 +134,28 @@ struct BinaryPathProvider { struct StreamPurposeProvider { explicit StreamPurposeProvider(uint32_t MaxLen) : MaxLen(MaxLen) {} - DiffResult compare(const std::pair<StreamPurpose, std::string> &L, - const std::pair<StreamPurpose, std::string> &R) { - if (L.first != R.first) + DiffResult compare(const StreamInfo &L, const StreamInfo &R) { + if (L.getPurpose() != R.getPurpose()) return DiffResult::DIFFERENT; - if (L.first == StreamPurpose::ModuleStream) { + if (L.getPurpose() == StreamPurpose::ModuleStream) { BinaryPathProvider PathProvider(MaxLen); - return PathProvider.compare(L.second, R.second); + return PathProvider.compare(L.getShortName(), R.getShortName()); } - return (L.second == R.second) ? DiffResult::IDENTICAL - : DiffResult::DIFFERENT; + return (L.getShortName() == R.getShortName()) ? DiffResult::IDENTICAL + : DiffResult::DIFFERENT; } - std::string format(const std::pair<StreamPurpose, std::string> &P, - bool Right) { - if (P.first == StreamPurpose::Other) - return truncateStringBack(P.second, MaxLen); - if (P.first == StreamPurpose::NamedStream) - return truncateQuotedNameBack("Named Stream", P.second, MaxLen); + std::string format(const StreamInfo &P, bool Right) { + if (P.getPurpose() == StreamPurpose::Other || + P.getPurpose() == StreamPurpose::Symbols) + return truncateStringBack(P.getShortName(), MaxLen); + if (P.getPurpose() == StreamPurpose::NamedStream) + return truncateQuotedNameBack("Named Stream", P.getShortName(), MaxLen); - assert(P.first == StreamPurpose::ModuleStream); + assert(P.getPurpose() == StreamPurpose::ModuleStream); uint32_t ExtraChars = strlen("Module \"\""); BinaryPathProvider PathProvider(MaxLen - ExtraChars); - std::string Result = PathProvider.format(P.second, Right); + std::string Result = PathProvider.format(P.getShortName(), Right); return formatv("Module \"{0}\"", Result); } @@ -256,8 +254,8 @@ Error DiffStyle::diffStreamDirectory() { truncateStringFront(File1.getFilePath(), 18), truncateStringFront(File2.getFilePath(), 18)); - SmallVector<std::pair<StreamPurpose, std::string>, 32> P; - SmallVector<std::pair<StreamPurpose, std::string>, 32> Q; + SmallVector<StreamInfo, 32> P; + SmallVector<StreamInfo, 32> Q; discoverStreamPurposes(File1, P); discoverStreamPurposes(File2, Q); D.print("Stream Count", File1.getNumStreams(), File2.getNumStreams()); @@ -425,45 +423,82 @@ Error DiffStyle::diffInfoStream() { return Error::success(); } -static std::vector<std::pair<uint32_t, DbiModuleDescriptor>> +typedef std::pair<uint32_t, DbiModuleDescriptor> IndexedModuleDescriptor; +typedef std::vector<IndexedModuleDescriptor> IndexedModuleDescriptorList; + +static IndexedModuleDescriptorList getModuleDescriptors(const DbiModuleList &ML) { - std::vector<std::pair<uint32_t, DbiModuleDescriptor>> List; + IndexedModuleDescriptorList List; List.reserve(ML.getModuleCount()); for (uint32_t I = 0; I < ML.getModuleCount(); ++I) List.emplace_back(I, ML.getModuleDescriptor(I)); return List; } -static void -diffOneModule(DiffPrinter &D, - const std::pair<uint32_t, DbiModuleDescriptor> Item, - std::vector<std::pair<uint32_t, DbiModuleDescriptor>> &Other, - bool ItemIsRight) { +static IndexedModuleDescriptorList::iterator +findOverrideEquivalentModule(uint32_t Modi, + IndexedModuleDescriptorList &OtherList) { + auto &EqMap = opts::diff::Equivalences; + + auto Iter = EqMap.find(Modi); + if (Iter == EqMap.end()) + return OtherList.end(); + + uint32_t EqValue = Iter->second; + + return llvm::find_if(OtherList, + [EqValue](const IndexedModuleDescriptor &Desc) { + return Desc.first == EqValue; + }); +} + +static IndexedModuleDescriptorList::iterator +findEquivalentModule(const IndexedModuleDescriptor &Item, + IndexedModuleDescriptorList &OtherList, bool ItemIsRight) { + + if (!ItemIsRight) { + uint32_t Modi = Item.first; + auto OverrideIter = findOverrideEquivalentModule(Modi, OtherList); + if (OverrideIter != OtherList.end()) + return OverrideIter; + } + + BinaryPathProvider PathProvider(28); + + auto Iter = OtherList.begin(); + auto End = OtherList.end(); + for (; Iter != End; ++Iter) { + const IndexedModuleDescriptor *Left = &Item; + const IndexedModuleDescriptor *Right = &*Iter; + if (ItemIsRight) + std::swap(Left, Right); + DiffResult Result = PathProvider.compare(Left->second.getModuleName(), + Right->second.getModuleName()); + if (Result == DiffResult::EQUIVALENT || Result == DiffResult::IDENTICAL) + return Iter; + } + return OtherList.end(); +} + +static void diffOneModule(DiffPrinter &D, const IndexedModuleDescriptor &Item, + IndexedModuleDescriptorList &Other, + bool ItemIsRight) { StreamPurposeProvider HeaderProvider(70); - std::pair<StreamPurpose, std::string> Header; - Header.first = StreamPurpose::ModuleStream; - Header.second = Item.second.getModuleName(); - D.printFullRow(HeaderProvider.format(Header, ItemIsRight)); + StreamInfo Info = StreamInfo::createModuleStream( + Item.second.getModuleName(), Item.second.getModuleStreamIndex(), + Item.first); + D.printFullRow(HeaderProvider.format(Info, ItemIsRight)); const auto *L = &Item; - BinaryPathProvider PathProvider(28); - auto Iter = llvm::find_if( - Other, [&PathProvider, ItemIsRight, - L](const std::pair<uint32_t, DbiModuleDescriptor> &Other) { - const auto *Left = L; - const auto *Right = &Other; - if (ItemIsRight) - std::swap(Left, Right); - DiffResult Result = PathProvider.compare(Left->second.getModuleName(), - Right->second.getModuleName()); - return Result == DiffResult::EQUIVALENT || - Result == DiffResult::IDENTICAL; - }); + auto Iter = findEquivalentModule(Item, Other, ItemIsRight); if (Iter == Other.end()) { // We didn't find this module at all on the other side. Just print one row // and continue. - D.print<ModiProvider>("- Modi", Item.first, None); + if (ItemIsRight) + D.print<ModiProvider>("- Modi", None, Item.first); + else + D.print<ModiProvider>("- Modi", Item.first, None); return; } @@ -472,6 +507,7 @@ diffOneModule(DiffPrinter &D, if (ItemIsRight) std::swap(L, R); + BinaryPathProvider PathProvider(28); D.print<ModiProvider>("- Modi", L->first, R->first); D.print<BinaryPathProvider>("- Obj File Name", L->second.getObjFileName(), R->second.getObjFileName(), PathProvider); diff --git a/tools/llvm-pdbutil/DumpOutputStyle.cpp b/tools/llvm-pdbutil/DumpOutputStyle.cpp index 01c7481c3086..dd8436728baf 100644 --- a/tools/llvm-pdbutil/DumpOutputStyle.cpp +++ b/tools/llvm-pdbutil/DumpOutputStyle.cpp @@ -10,6 +10,7 @@ #include "DumpOutputStyle.h" #include "FormatUtil.h" +#include "InputFile.h" #include "MinimalSymbolDumper.h" #include "MinimalTypeDumper.h" #include "StreamUtil.h" @@ -21,28 +22,21 @@ #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" #include "llvm/DebugInfo/CodeView/DebugCrossExSubsection.h" #include "llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h" #include "llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h" #include "llvm/DebugInfo/CodeView/DebugLinesSubsection.h" #include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h" #include "llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugUnknownSubsection.h" -#include "llvm/DebugInfo/CodeView/EnumTables.h" #include "llvm/DebugInfo/CodeView/Formatters.h" #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" #include "llvm/DebugInfo/CodeView/Line.h" #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" -#include "llvm/DebugInfo/CodeView/SymbolDumper.h" #include "llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h" #include "llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h" -#include "llvm/DebugInfo/CodeView/TypeDumpVisitor.h" +#include "llvm/DebugInfo/CodeView/TypeHashing.h" #include "llvm/DebugInfo/CodeView/TypeIndexDiscovery.h" -#include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" #include "llvm/DebugInfo/PDB/Native/DbiStream.h" -#include "llvm/DebugInfo/PDB/Native/EnumTables.h" #include "llvm/DebugInfo/PDB/Native/GlobalsStream.h" #include "llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h" #include "llvm/DebugInfo/PDB/Native/InfoStream.h" @@ -50,24 +44,27 @@ #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/PublicsStream.h" #include "llvm/DebugInfo/PDB/Native/RawError.h" +#include "llvm/DebugInfo/PDB/Native/SymbolStream.h" #include "llvm/DebugInfo/PDB/Native/TpiHashing.h" #include "llvm/DebugInfo/PDB/Native/TpiStream.h" -#include "llvm/DebugInfo/PDB/PDBExtras.h" #include "llvm/Object/COFF.h" #include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/FormatAdapters.h" #include "llvm/Support/FormatVariadic.h" -#include <unordered_map> +#include <cctype> using namespace llvm; using namespace llvm::codeview; using namespace llvm::msf; using namespace llvm::pdb; -DumpOutputStyle::DumpOutputStyle(PDBFile &File) +DumpOutputStyle::DumpOutputStyle(InputFile &File) : File(File), P(2, false, outs()) {} +PDBFile &DumpOutputStyle::getPdb() { return File.pdb(); } +object::COFFObjectFile &DumpOutputStyle::getObj() { return File.obj(); } + Error DumpOutputStyle::dump() { if (opts::dump::DumpSummary) { if (auto EC = dumpFileSummary()) @@ -81,6 +78,18 @@ Error DumpOutputStyle::dump() { P.NewLine(); } + if (opts::dump::DumpSymbolStats) { + if (auto EC = dumpSymbolStats()) + return EC; + P.NewLine(); + } + + if (opts::dump::DumpUdtStats) { + if (auto EC = dumpUdtStats()) + return EC; + P.NewLine(); + } + if (opts::dump::DumpStringTable) { if (auto EC = dumpStringTable()) return EC; @@ -117,15 +126,27 @@ Error DumpOutputStyle::dump() { return EC; } - if (opts::dump::DumpTypes || !opts::dump::DumpTypeIndex.empty() || - opts::dump::DumpTypeExtras) { - if (auto EC = dumpTpiStream(StreamTPI)) - return EC; + if (File.isObj()) { + if (opts::dump::DumpTypes || !opts::dump::DumpTypeIndex.empty() || + opts::dump::DumpTypeExtras) + if (auto EC = dumpTypesFromObjectFile()) + return EC; + } else { + if (opts::dump::DumpTypes || !opts::dump::DumpTypeIndex.empty() || + opts::dump::DumpTypeExtras) { + if (auto EC = dumpTpiStream(StreamTPI)) + return EC; + } + + if (opts::dump::DumpIds || !opts::dump::DumpIdIndex.empty() || + opts::dump::DumpIdExtras) { + if (auto EC = dumpTpiStream(StreamIPI)) + return EC; + } } - if (opts::dump::DumpIds || !opts::dump::DumpIdIndex.empty() || - opts::dump::DumpIdExtras) { - if (auto EC = dumpTpiStream(StreamIPI)) + if (opts::dump::DumpGlobals) { + if (auto EC = dumpGlobals()) return EC; } @@ -135,7 +156,13 @@ Error DumpOutputStyle::dump() { } if (opts::dump::DumpSymbols) { - if (auto EC = dumpModuleSyms()) + auto EC = File.isPdb() ? dumpModuleSymsForPdb() : dumpModuleSymsForObj(); + if (EC) + return EC; + } + + if (opts::dump::DumpSectionHeaders) { + if (auto EC = dumpSectionHeaders()) return EC; } @@ -161,25 +188,30 @@ static void printHeader(LinePrinter &P, const Twine &S) { Error DumpOutputStyle::dumpFileSummary() { printHeader(P, "Summary"); - ExitOnError Err("Invalid PDB Format"); + ExitOnError Err("Invalid PDB Format: "); AutoIndent Indent(P); - P.formatLine("Block Size: {0}", File.getBlockSize()); - P.formatLine("Number of blocks: {0}", File.getBlockCount()); - P.formatLine("Number of streams: {0}", File.getNumStreams()); + if (File.isObj()) { + P.formatLine("Dumping File summary is not valid for object files"); + return Error::success(); + } - auto &PS = Err(File.getPDBInfoStream()); + P.formatLine("Block Size: {0}", getPdb().getBlockSize()); + P.formatLine("Number of blocks: {0}", getPdb().getBlockCount()); + P.formatLine("Number of streams: {0}", getPdb().getNumStreams()); + + auto &PS = Err(getPdb().getPDBInfoStream()); P.formatLine("Signature: {0}", PS.getSignature()); P.formatLine("Age: {0}", PS.getAge()); P.formatLine("GUID: {0}", fmt_guid(PS.getGuid().Guid)); P.formatLine("Features: {0:x+}", static_cast<uint32_t>(PS.getFeatures())); - P.formatLine("Has Debug Info: {0}", File.hasPDBDbiStream()); - P.formatLine("Has Types: {0}", File.hasPDBTpiStream()); - P.formatLine("Has IDs: {0}", File.hasPDBIpiStream()); - P.formatLine("Has Globals: {0}", File.hasPDBGlobalsStream()); - P.formatLine("Has Publics: {0}", File.hasPDBPublicsStream()); - if (File.hasPDBDbiStream()) { - auto &DBI = Err(File.getPDBDbiStream()); + P.formatLine("Has Debug Info: {0}", getPdb().hasPDBDbiStream()); + P.formatLine("Has Types: {0}", getPdb().hasPDBTpiStream()); + P.formatLine("Has IDs: {0}", getPdb().hasPDBIpiStream()); + P.formatLine("Has Globals: {0}", getPdb().hasPDBGlobalsStream()); + P.formatLine("Has Publics: {0}", getPdb().hasPDBPublicsStream()); + if (getPdb().hasPDBDbiStream()) { + auto &DBI = Err(getPdb().getPDBDbiStream()); P.formatLine("Is incrementally linked: {0}", DBI.isIncrementallyLinked()); P.formatLine("Has conflicting types: {0}", DBI.hasCTypes()); P.formatLine("Is stripped: {0}", DBI.isStripped()); @@ -188,22 +220,123 @@ Error DumpOutputStyle::dumpFileSummary() { return Error::success(); } +static StatCollection getSymbolStats(const SymbolGroup &SG, + StatCollection &CumulativeStats) { + StatCollection Stats; + if (SG.getFile().isPdb()) { + // For PDB files, all symbols are packed into one stream. + for (const auto &S : SG.getPdbModuleStream().symbols(nullptr)) { + Stats.update(S.kind(), S.length()); + CumulativeStats.update(S.kind(), S.length()); + } + return Stats; + } + + for (const auto &SS : SG.getDebugSubsections()) { + // For object files, all symbols are spread across multiple Symbol + // subsections of a given .debug$S section. + if (SS.kind() != DebugSubsectionKind::Symbols) + continue; + DebugSymbolsSubsectionRef Symbols; + BinaryStreamReader Reader(SS.getRecordData()); + cantFail(Symbols.initialize(Reader)); + for (const auto &S : Symbols) { + Stats.update(S.kind(), S.length()); + CumulativeStats.update(S.kind(), S.length()); + } + } + return Stats; +} + +static StatCollection getChunkStats(const SymbolGroup &SG, + StatCollection &CumulativeStats) { + StatCollection Stats; + for (const auto &Chunk : SG.getDebugSubsections()) { + Stats.update(uint32_t(Chunk.kind()), Chunk.getRecordLength()); + CumulativeStats.update(uint32_t(Chunk.kind()), Chunk.getRecordLength()); + } + return Stats; +} + +static inline std::string formatModuleDetailKind(DebugSubsectionKind K) { + return formatChunkKind(K, false); +} + +static inline std::string formatModuleDetailKind(SymbolKind K) { + return formatSymbolKind(K); +} + +template <typename Kind> +static void printModuleDetailStats(LinePrinter &P, StringRef Label, + const StatCollection &Stats) { + P.NewLine(); + P.formatLine(" {0}", Label); + AutoIndent Indent(P); + P.formatLine("{0,40}: {1,7} entries ({2,8} bytes)", "Total", + Stats.Totals.Count, Stats.Totals.Size); + P.formatLine("{0}", fmt_repeat('-', 74)); + for (const auto &K : Stats.Individual) { + std::string KindName = formatModuleDetailKind(Kind(K.first)); + P.formatLine("{0,40}: {1,7} entries ({2,8} bytes)", KindName, + K.second.Count, K.second.Size); + } +} + +static bool isMyCode(const SymbolGroup &Group) { + if (Group.getFile().isObj()) + return true; + + StringRef Name = Group.name(); + if (Name.startswith("Import:")) + return false; + if (Name.endswith_lower(".dll")) + return false; + if (Name.equals_lower("* linker *")) + return false; + if (Name.startswith_lower("f:\\binaries\\Intermediate\\vctools")) + return false; + if (Name.startswith_lower("f:\\dd\\vctools\\crt")) + return false; + return true; +} + +static bool shouldDumpSymbolGroup(uint32_t Idx, const SymbolGroup &Group) { + if (opts::dump::JustMyCode && !isMyCode(Group)) + return false; + + // If the arg was not specified on the command line, always dump all modules. + if (opts::dump::DumpModi.getNumOccurrences() == 0) + return true; + + // Otherwise, only dump if this is the same module specified. + return (opts::dump::DumpModi == Idx); +} + Error DumpOutputStyle::dumpStreamSummary() { printHeader(P, "Streams"); + AutoIndent Indent(P); + if (File.isObj()) { + P.formatLine("Dumping streams is not valid for object files"); + return Error::success(); + } + if (StreamPurposes.empty()) - discoverStreamPurposes(File, StreamPurposes); + discoverStreamPurposes(getPdb(), StreamPurposes); - AutoIndent Indent(P); - uint32_t StreamCount = File.getNumStreams(); + uint32_t StreamCount = getPdb().getNumStreams(); + uint32_t MaxStreamSize = getPdb().getMaxStreamSize(); for (uint16_t StreamIdx = 0; StreamIdx < StreamCount; ++StreamIdx) { P.formatLine( - "Stream {0}: [{1}] ({2} bytes)", + "Stream {0} ({1} bytes): [{2}]", fmt_align(StreamIdx, AlignStyle::Right, NumDigits(StreamCount)), - StreamPurposes[StreamIdx], File.getStreamByteSize(StreamIdx)); + fmt_align(getPdb().getStreamByteSize(StreamIdx), AlignStyle::Right, + NumDigits(MaxStreamSize)), + StreamPurposes[StreamIdx].getLongName()); + if (opts::dump::DumpStreamBlocks) { - auto Blocks = File.getStreamBlockList(StreamIdx); + auto Blocks = getPdb().getStreamBlockList(StreamIdx); std::vector<uint32_t> BV(Blocks.begin(), Blocks.end()); P.formatLine(" {0} Blocks: [{1}]", fmt_repeat(' ', NumDigits(StreamCount)), @@ -216,7 +349,7 @@ Error DumpOutputStyle::dumpStreamSummary() { static Expected<ModuleDebugStreamRef> getModuleDebugStream(PDBFile &File, uint32_t Index) { - ExitOnError Err("Unexpected error"); + ExitOnError Err("Unexpected error: "); auto &Dbi = Err(File.getPDBDbiStream()); const auto &Modules = Dbi.modules(); @@ -227,9 +360,7 @@ static Expected<ModuleDebugStreamRef> getModuleDebugStream(PDBFile &File, return make_error<RawError>(raw_error_code::no_stream, "Module stream not present"); - auto ModStreamData = MappedBlockStream::createIndexedStream( - File.getMsfLayout(), File.getMsfBuffer(), ModiStream, - File.getAllocator()); + auto ModStreamData = File.createIndexedStream(ModiStream); ModuleDebugStreamRef ModS(Modi, std::move(ModStreamData)); if (auto EC = ModS.reload()) @@ -239,216 +370,345 @@ static Expected<ModuleDebugStreamRef> getModuleDebugStream(PDBFile &File, return std::move(ModS); } -static std::string formatChecksumKind(FileChecksumKind Kind) { - switch (Kind) { - RETURN_CASE(FileChecksumKind, None, "None"); - RETURN_CASE(FileChecksumKind, MD5, "MD5"); - RETURN_CASE(FileChecksumKind, SHA1, "SHA-1"); - RETURN_CASE(FileChecksumKind, SHA256, "SHA-256"); +template <typename CallbackT> +static void +iterateOneModule(InputFile &File, const Optional<PrintScope> &HeaderScope, + const SymbolGroup &SG, uint32_t Modi, CallbackT Callback) { + if (HeaderScope) { + HeaderScope->P.formatLine( + "Mod {0:4} | `{1}`: ", + fmt_align(Modi, AlignStyle::Right, HeaderScope->LabelWidth), SG.name()); } - return formatUnknownEnum(Kind); -} -namespace { -class StringsAndChecksumsPrinter { - const DebugStringTableSubsectionRef &extractStringTable(PDBFile &File) { - ExitOnError Err("Unexpected error processing modules"); - return Err(File.getStringTable()).getStringTable(); - } - - template <typename... Args> - void formatInternal(LinePrinter &Printer, bool Append, - Args &&... args) const { - if (Append) - Printer.format(std::forward<Args>(args)...); - else - Printer.formatLine(std::forward<Args>(args)...); - } - -public: - StringsAndChecksumsPrinter(PDBFile &File, uint32_t Modi) - : Records(extractStringTable(File)) { - auto MDS = getModuleDebugStream(File, Modi); - if (!MDS) { - consumeError(MDS.takeError()); - return; - } + AutoIndent Indent(HeaderScope); + Callback(Modi, SG); +} - DebugStream = llvm::make_unique<ModuleDebugStreamRef>(std::move(*MDS)); - Records.initialize(MDS->subsections()); - if (Records.hasChecksums()) { - for (const auto &Entry : Records.checksums()) { - auto S = Records.strings().getString(Entry.FileNameOffset); - if (!S) - continue; - ChecksumsByFile[*S] = Entry; - } - } +template <typename CallbackT> +static void iterateSymbolGroups(InputFile &Input, + const Optional<PrintScope> &HeaderScope, + CallbackT Callback) { + AutoIndent Indent(HeaderScope); + + ExitOnError Err("Unexpected error processing modules: "); + + if (opts::dump::DumpModi.getNumOccurrences() > 0) { + assert(opts::dump::DumpModi.getNumOccurrences() == 1); + uint32_t Modi = opts::dump::DumpModi; + SymbolGroup SG(&Input, Modi); + iterateOneModule(Input, withLabelWidth(HeaderScope, NumDigits(Modi)), SG, + Modi, Callback); + return; } - Expected<StringRef> getNameFromStringTable(uint32_t Offset) const { - return Records.strings().getString(Offset); - } + uint32_t I = 0; - void formatFromFileName(LinePrinter &Printer, StringRef File, - bool Append = false) const { - auto FC = ChecksumsByFile.find(File); - if (FC == ChecksumsByFile.end()) { - formatInternal(Printer, Append, "- (no checksum) {0}", File); - return; - } + for (const auto &SG : Input.symbol_groups()) { + if (shouldDumpSymbolGroup(I, SG)) + iterateOneModule(Input, withLabelWidth(HeaderScope, NumDigits(I)), SG, I, + Callback); - formatInternal(Printer, Append, "- ({0}: {1}) {2}", - formatChecksumKind(FC->getValue().Kind), - toHex(FC->getValue().Checksum), File); + ++I; } +} - void formatFromChecksumsOffset(LinePrinter &Printer, uint32_t Offset, - bool Append = false) const { - if (!Records.hasChecksums()) { - formatInternal(Printer, Append, "(unknown file name offset {0})", Offset); - return; - } +template <typename SubsectionT> +static void iterateModuleSubsections( + InputFile &File, const Optional<PrintScope> &HeaderScope, + llvm::function_ref<void(uint32_t, const SymbolGroup &, SubsectionT &)> + Callback) { - auto Iter = Records.checksums().getArray().at(Offset); - if (Iter == Records.checksums().getArray().end()) { - formatInternal(Printer, Append, "(unknown file name offset {0})", Offset); - return; - } + iterateSymbolGroups(File, HeaderScope, + [&](uint32_t Modi, const SymbolGroup &SG) { + for (const auto &SS : SG.getDebugSubsections()) { + SubsectionT Subsection; - uint32_t FO = Iter->FileNameOffset; - auto ExpectedFile = getNameFromStringTable(FO); - if (!ExpectedFile) { - formatInternal(Printer, Append, "(unknown file name offset {0})", Offset); - consumeError(ExpectedFile.takeError()); - return; - } - if (Iter->Kind == FileChecksumKind::None) { - formatInternal(Printer, Append, "{0} (no checksum)", *ExpectedFile); - } else { - formatInternal(Printer, Append, "{0} ({1}: {2})", *ExpectedFile, - formatChecksumKind(Iter->Kind), toHex(Iter->Checksum)); - } - } + if (SS.kind() != Subsection.kind()) + continue; - std::unique_ptr<ModuleDebugStreamRef> DebugStream; - StringsAndChecksumsRef Records; - StringMap<FileChecksumEntry> ChecksumsByFile; -}; -} // namespace + BinaryStreamReader Reader(SS.getRecordData()); + if (auto EC = Subsection.initialize(Reader)) + continue; + Callback(Modi, SG, Subsection); + } + }); +} -template <typename CallbackT> -static void iterateModules(PDBFile &File, LinePrinter &P, uint32_t IndentLevel, - CallbackT Callback) { +Error DumpOutputStyle::dumpModules() { + printHeader(P, "Modules"); AutoIndent Indent(P); - if (!File.hasPDBDbiStream()) { + + if (File.isObj()) { + P.formatLine("Dumping modules is not supported for object files"); + return Error::success(); + } + + if (!getPdb().hasPDBDbiStream()) { P.formatLine("DBI Stream not present"); - return; + return Error::success(); } - ExitOnError Err("Unexpected error processing modules"); + ExitOnError Err("Unexpected error processing modules: "); - auto &Stream = Err(File.getPDBDbiStream()); + auto &Stream = Err(getPdb().getPDBDbiStream()); const DbiModuleList &Modules = Stream.modules(); - uint32_t Count = Modules.getModuleCount(); - uint32_t Digits = NumDigits(Count); - for (uint32_t I = 0; I < Count; ++I) { - auto Modi = Modules.getModuleDescriptor(I); - P.formatLine("Mod {0:4} | `{1}`: ", fmt_align(I, AlignStyle::Right, Digits), - Modi.getModuleName()); + iterateSymbolGroups( + File, PrintScope{P, 11}, [&](uint32_t Modi, const SymbolGroup &Strings) { + auto Desc = Modules.getModuleDescriptor(Modi); + P.formatLine("Obj: `{0}`: ", Desc.getObjFileName()); + P.formatLine("debug stream: {0}, # files: {1}, has ec info: {2}", + Desc.getModuleStreamIndex(), Desc.getNumberOfFiles(), + Desc.hasECInfo()); + StringRef PdbFilePath = + Err(Stream.getECName(Desc.getPdbFilePathNameIndex())); + StringRef SrcFilePath = + Err(Stream.getECName(Desc.getSourceFileNameIndex())); + P.formatLine("pdb file ni: {0} `{1}`, src file ni: {2} `{3}`", + Desc.getPdbFilePathNameIndex(), PdbFilePath, + Desc.getSourceFileNameIndex(), SrcFilePath); + }); + return Error::success(); +} + +Error DumpOutputStyle::dumpModuleFiles() { + printHeader(P, "Files"); - StringsAndChecksumsPrinter Strings(File, I); - AutoIndent Indent2(P, IndentLevel); - Callback(I, Strings); + if (File.isObj()) { + P.formatLine("Dumping files is not valid for object files"); + return Error::success(); } + + ExitOnError Err("Unexpected error processing modules: "); + + iterateSymbolGroups(File, PrintScope{P, 11}, + [this, &Err](uint32_t Modi, const SymbolGroup &Strings) { + auto &Stream = Err(getPdb().getPDBDbiStream()); + + const DbiModuleList &Modules = Stream.modules(); + for (const auto &F : Modules.source_files(Modi)) { + Strings.formatFromFileName(P, F); + } + }); + return Error::success(); } -template <typename SubsectionT> -static void iterateModuleSubsections( - PDBFile &File, LinePrinter &P, uint32_t IndentLevel, - llvm::function_ref<void(uint32_t, StringsAndChecksumsPrinter &, - SubsectionT &)> - Callback) { +Error DumpOutputStyle::dumpSymbolStats() { + printHeader(P, "Module Stats"); - iterateModules( - File, P, IndentLevel, - [&File, &Callback](uint32_t Modi, StringsAndChecksumsPrinter &Strings) { - auto MDS = getModuleDebugStream(File, Modi); - if (!MDS) { - consumeError(MDS.takeError()); - return; - } + ExitOnError Err("Unexpected error processing modules: "); - for (const auto &SS : MDS->subsections()) { - SubsectionT Subsection; + StatCollection SymStats; + StatCollection ChunkStats; - if (SS.kind() != Subsection.kind()) - continue; + Optional<PrintScope> Scope; + if (File.isPdb()) + Scope.emplace(P, 2); - BinaryStreamReader Reader(SS.getRecordData()); - if (auto EC = Subsection.initialize(Reader)) - continue; - Callback(Modi, Strings, Subsection); - } - }); -} + iterateSymbolGroups(File, Scope, [&](uint32_t Modi, const SymbolGroup &SG) { + StatCollection SS = getSymbolStats(SG, SymStats); + StatCollection CS = getChunkStats(SG, ChunkStats); -Error DumpOutputStyle::dumpModules() { - printHeader(P, "Modules"); + if (SG.getFile().isPdb()) { + AutoIndent Indent(P); + auto Modules = cantFail(File.pdb().getPDBDbiStream()).modules(); + uint32_t ModCount = Modules.getModuleCount(); + DbiModuleDescriptor Desc = Modules.getModuleDescriptor(Modi); + uint32_t StreamIdx = Desc.getModuleStreamIndex(); - AutoIndent Indent(P); - if (!File.hasPDBDbiStream()) { - P.formatLine("DBI Stream not present"); - return Error::success(); + if (StreamIdx == kInvalidStreamIndex) { + P.formatLine("Mod {0} (debug info not present): [{1}]", + fmt_align(Modi, AlignStyle::Right, NumDigits(ModCount)), + Desc.getModuleName()); + return; + } + P.formatLine("Stream {0}, {1} bytes", StreamIdx, + getPdb().getStreamByteSize(StreamIdx)); + + printModuleDetailStats<SymbolKind>(P, "Symbols", SS); + printModuleDetailStats<DebugSubsectionKind>(P, "Chunks", CS); + } + }); + + P.printLine(" Summary |"); + AutoIndent Indent(P, 4); + if (SymStats.Totals.Count > 0) { + printModuleDetailStats<SymbolKind>(P, "Symbols", SymStats); + printModuleDetailStats<DebugSubsectionKind>(P, "Chunks", ChunkStats); } - ExitOnError Err("Unexpected error processing modules"); + return Error::success(); +} + +static bool isValidNamespaceIdentifier(StringRef S) { + if (S.empty()) + return false; - auto &Stream = Err(File.getPDBDbiStream()); + if (std::isdigit(S[0])) + return false; - const DbiModuleList &Modules = Stream.modules(); - uint32_t Count = Modules.getModuleCount(); - uint32_t Digits = NumDigits(Count); - for (uint32_t I = 0; I < Count; ++I) { - auto Modi = Modules.getModuleDescriptor(I); - P.formatLine("Mod {0:4} | Name: `{1}`: ", - fmt_align(I, AlignStyle::Right, Digits), Modi.getModuleName()); - P.formatLine(" Obj: `{0}`: ", Modi.getObjFileName()); - P.formatLine(" debug stream: {0}, # files: {1}, has ec info: {2}", - Modi.getModuleStreamIndex(), Modi.getNumberOfFiles(), - Modi.hasECInfo()); - StringRef PdbFilePath = - Err(Stream.getECName(Modi.getPdbFilePathNameIndex())); - StringRef SrcFilePath = - Err(Stream.getECName(Modi.getSourceFileNameIndex())); - P.formatLine(" pdb file ni: {0} `{1}`, src file ni: {2} `{3}`", - Modi.getPdbFilePathNameIndex(), PdbFilePath, - Modi.getSourceFileNameIndex(), SrcFilePath); + return llvm::all_of(S, [](char C) { return std::isalnum(C); }); +} + +namespace { +constexpr uint32_t kNoneUdtKind = 0; +constexpr uint32_t kSimpleUdtKind = 1; +constexpr uint32_t kUnknownUdtKind = 2; +const StringRef NoneLabel("<none type>"); +const StringRef SimpleLabel("<simple type>"); +const StringRef UnknownLabel("<unknown type>"); + +} // namespace + +static StringRef getUdtStatLabel(uint32_t Kind) { + if (Kind == kNoneUdtKind) + return NoneLabel; + + if (Kind == kSimpleUdtKind) + return SimpleLabel; + + if (Kind == kUnknownUdtKind) + return UnknownLabel; + + return formatTypeLeafKind(static_cast<TypeLeafKind>(Kind)); +} + +static uint32_t getLongestTypeLeafName(const StatCollection &Stats) { + size_t L = 0; + for (const auto &Stat : Stats.Individual) { + StringRef Label = getUdtStatLabel(Stat.first); + L = std::max(L, Label.size()); } - return Error::success(); + return static_cast<uint32_t>(L); } -Error DumpOutputStyle::dumpModuleFiles() { - printHeader(P, "Files"); +Error DumpOutputStyle::dumpUdtStats() { + printHeader(P, "S_UDT Record Stats"); - ExitOnError Err("Unexpected error processing modules"); + StatCollection UdtStats; + StatCollection UdtTargetStats; + AutoIndent Indent(P, 4); - iterateModules( - File, P, 11, - [this, &Err](uint32_t Modi, StringsAndChecksumsPrinter &Strings) { - auto &Stream = Err(File.getPDBDbiStream()); + auto &TpiTypes = File.types(); - const DbiModuleList &Modules = Stream.modules(); - for (const auto &F : Modules.source_files(Modi)) { - Strings.formatFromFileName(P, F); - } - }); + StringMap<StatCollection::Stat> NamespacedStats; + + size_t LongestNamespace = 0; + auto HandleOneSymbol = [&](const CVSymbol &Sym) { + if (Sym.kind() != SymbolKind::S_UDT) + return; + UdtStats.update(SymbolKind::S_UDT, Sym.length()); + + UDTSym UDT = cantFail(SymbolDeserializer::deserializeAs<UDTSym>(Sym)); + + uint32_t Kind = 0; + uint32_t RecordSize = 0; + + if (UDT.Type.isNoneType()) + Kind = kNoneUdtKind; + else if (UDT.Type.isSimple()) + Kind = kSimpleUdtKind; + else if (Optional<CVType> T = TpiTypes.tryGetType(UDT.Type)) { + Kind = T->kind(); + RecordSize = T->length(); + } else + Kind = kUnknownUdtKind; + + UdtTargetStats.update(Kind, RecordSize); + + size_t Pos = UDT.Name.find("::"); + if (Pos == StringRef::npos) + return; + + StringRef Scope = UDT.Name.take_front(Pos); + if (Scope.empty() || !isValidNamespaceIdentifier(Scope)) + return; + + LongestNamespace = std::max(LongestNamespace, Scope.size()); + NamespacedStats[Scope].update(RecordSize); + }; + + P.NewLine(); + + if (File.isPdb()) { + if (!getPdb().hasPDBGlobalsStream()) { + P.printLine("- Error: globals stream not present"); + return Error::success(); + } + + auto &SymbolRecords = cantFail(getPdb().getPDBSymbolStream()); + auto ExpGlobals = getPdb().getPDBGlobalsStream(); + if (!ExpGlobals) + return ExpGlobals.takeError(); + + for (uint32_t PubSymOff : ExpGlobals->getGlobalsTable()) { + CVSymbol Sym = SymbolRecords.readRecord(PubSymOff); + HandleOneSymbol(Sym); + } + } else { + for (const auto &Sec : File.symbol_groups()) { + for (const auto &SS : Sec.getDebugSubsections()) { + if (SS.kind() != DebugSubsectionKind::Symbols) + continue; + + DebugSymbolsSubsectionRef Symbols; + BinaryStreamReader Reader(SS.getRecordData()); + cantFail(Symbols.initialize(Reader)); + for (const auto &S : Symbols) + HandleOneSymbol(S); + } + } + } + + LongestNamespace += StringRef(" namespace ''").size(); + size_t LongestTypeLeafKind = getLongestTypeLeafName(UdtTargetStats); + size_t FieldWidth = std::max(LongestNamespace, LongestTypeLeafKind); + + // Compute the max number of digits for count and size fields, including comma + // separators. + StringRef CountHeader("Count"); + StringRef SizeHeader("Size"); + size_t CD = NumDigits(UdtStats.Totals.Count); + CD += (CD - 1) / 3; + CD = std::max(CD, CountHeader.size()); + + size_t SD = NumDigits(UdtStats.Totals.Size); + SD += (SD - 1) / 3; + SD = std::max(SD, SizeHeader.size()); + + uint32_t TableWidth = FieldWidth + 3 + CD + 2 + SD + 1; + + P.formatLine("{0} | {1} {2}", + fmt_align("Record Kind", AlignStyle::Right, FieldWidth), + fmt_align(CountHeader, AlignStyle::Right, CD), + fmt_align(SizeHeader, AlignStyle::Right, SD)); + + P.formatLine("{0}", fmt_repeat('-', TableWidth)); + for (const auto &Stat : UdtTargetStats.Individual) { + StringRef Label = getUdtStatLabel(Stat.first); + P.formatLine("{0} | {1:N} {2:N}", + fmt_align(Label, AlignStyle::Right, FieldWidth), + fmt_align(Stat.second.Count, AlignStyle::Right, CD), + fmt_align(Stat.second.Size, AlignStyle::Right, SD)); + } + P.formatLine("{0}", fmt_repeat('-', TableWidth)); + P.formatLine("{0} | {1:N} {2:N}", + fmt_align("Total (S_UDT)", AlignStyle::Right, FieldWidth), + fmt_align(UdtStats.Totals.Count, AlignStyle::Right, CD), + fmt_align(UdtStats.Totals.Size, AlignStyle::Right, SD)); + P.formatLine("{0}", fmt_repeat('-', TableWidth)); + for (const auto &Stat : NamespacedStats) { + std::string Label = formatv("namespace '{0}'", Stat.getKey()); + P.formatLine("{0} | {1:N} {2:N}", + fmt_align(Label, AlignStyle::Right, FieldWidth), + fmt_align(Stat.second.Count, AlignStyle::Right, CD), + fmt_align(Stat.second.Size, AlignStyle::Right, SD)); + } return Error::success(); } -static void typesetLinesAndColumns(PDBFile &File, LinePrinter &P, - uint32_t Start, const LineColumnEntry &E) { +static void typesetLinesAndColumns(LinePrinter &P, uint32_t Start, + const LineColumnEntry &E) { const uint32_t kMaxCharsPerLineNumber = 4; // 4 digit line number uint32_t MinColumnWidth = kMaxCharsPerLineNumber + 5; @@ -487,9 +747,9 @@ Error DumpOutputStyle::dumpLines() { uint32_t LastModi = UINT32_MAX; uint32_t LastNameIndex = UINT32_MAX; iterateModuleSubsections<DebugLinesSubsectionRef>( - File, P, 4, + File, PrintScope{P, 4}, [this, &LastModi, &LastNameIndex](uint32_t Modi, - StringsAndChecksumsPrinter &Strings, + const SymbolGroup &Strings, DebugLinesSubsectionRef &Lines) { uint16_t Segment = Lines.header()->RelocSegment; uint32_t Begin = Lines.header()->RelocOffset; @@ -510,7 +770,7 @@ Error DumpOutputStyle::dumpLines() { P.format("line/addr entries = {0}", Count); P.NewLine(); - typesetLinesAndColumns(File, P, Begin, Block); + typesetLinesAndColumns(P, Begin, Block); } }); @@ -521,8 +781,8 @@ Error DumpOutputStyle::dumpInlineeLines() { printHeader(P, "Inlinee Lines"); iterateModuleSubsections<DebugInlineeLinesSubsectionRef>( - File, P, 2, - [this](uint32_t Modi, StringsAndChecksumsPrinter &Strings, + File, PrintScope{P, 2}, + [this](uint32_t Modi, const SymbolGroup &Strings, DebugInlineeLinesSubsectionRef &Lines) { P.formatLine("{0,+8} | {1,+5} | {2}", "Inlinee", "Line", "Source File"); for (const auto &Entry : Lines) { @@ -539,8 +799,8 @@ Error DumpOutputStyle::dumpInlineeLines() { Error DumpOutputStyle::dumpXmi() { printHeader(P, "Cross Module Imports"); iterateModuleSubsections<DebugCrossModuleImportsSubsectionRef>( - File, P, 2, - [this](uint32_t Modi, StringsAndChecksumsPrinter &Strings, + File, PrintScope{P, 2}, + [this](uint32_t Modi, const SymbolGroup &Strings, DebugCrossModuleImportsSubsectionRef &Imports) { P.formatLine("{0,=32} | {1}", "Imported Module", "Type IDs"); @@ -575,8 +835,8 @@ Error DumpOutputStyle::dumpXme() { printHeader(P, "Cross Module Exports"); iterateModuleSubsections<DebugCrossModuleExportsSubsectionRef>( - File, P, 2, - [this](uint32_t Modi, StringsAndChecksumsPrinter &Strings, + File, PrintScope{P, 2}, + [this](uint32_t Modi, const SymbolGroup &Strings, DebugCrossModuleExportsSubsectionRef &Exports) { P.formatLine("{0,-10} | {1}", "Local ID", "Global ID"); for (const auto &Export : Exports) { @@ -591,8 +851,13 @@ Error DumpOutputStyle::dumpXme() { Error DumpOutputStyle::dumpStringTable() { printHeader(P, "String Table"); + if (File.isObj()) { + P.formatLine("Dumping string table is not supported for object files"); + return Error::success(); + } + AutoIndent Indent(P); - auto IS = File.getStringTable(); + auto IS = getPdb().getStringTable(); if (!IS) { P.formatLine("Not present in file"); consumeError(IS.takeError()); @@ -646,15 +911,17 @@ static void buildDepSet(LazyRandomTypeCollection &Types, } } -static void dumpFullTypeStream(LinePrinter &Printer, - LazyRandomTypeCollection &Types, - TpiStream &Stream, bool Bytes, bool Extras) { - Printer.formatLine("Showing {0:N} records", Stream.getNumTypeRecords()); - uint32_t Width = - NumDigits(TypeIndex::FirstNonSimpleIndex + Stream.getNumTypeRecords()); +static void +dumpFullTypeStream(LinePrinter &Printer, LazyRandomTypeCollection &Types, + uint32_t NumTypeRecords, uint32_t NumHashBuckets, + FixedStreamArray<support::ulittle32_t> HashValues, + bool Bytes, bool Extras) { + + Printer.formatLine("Showing {0:N} records", NumTypeRecords); + uint32_t Width = NumDigits(TypeIndex::FirstNonSimpleIndex + NumTypeRecords); MinimalTypeDumpVisitor V(Printer, Width + 2, Bytes, Extras, Types, - Stream.getNumHashBuckets(), Stream.getHashValues()); + NumHashBuckets, HashValues); if (auto EC = codeview::visitTypeStream(Types, V)) { Printer.formatLine("An error occurred dumping type records: {0}", @@ -700,25 +967,81 @@ static void dumpPartialTypeStream(LinePrinter &Printer, } } +Error DumpOutputStyle::dumpTypesFromObjectFile() { + LazyRandomTypeCollection Types(100); + + for (const auto &S : getObj().sections()) { + StringRef SectionName; + if (auto EC = S.getName(SectionName)) + return errorCodeToError(EC); + + if (SectionName != ".debug$T") + continue; + StringRef Contents; + if (auto EC = S.getContents(Contents)) + return errorCodeToError(EC); + + uint32_t Magic; + BinaryStreamReader Reader(Contents, llvm::support::little); + if (auto EC = Reader.readInteger(Magic)) + return EC; + if (Magic != COFF::DEBUG_SECTION_MAGIC) + return make_error<StringError>("Invalid CodeView debug section.", + inconvertibleErrorCode()); + + Types.reset(Reader, 100); + + if (opts::dump::DumpTypes) { + dumpFullTypeStream(P, Types, 0, 0, {}, opts::dump::DumpTypeData, false); + } else if (opts::dump::DumpTypeExtras) { + auto LocalHashes = LocallyHashedType::hashTypeCollection(Types); + auto GlobalHashes = GloballyHashedType::hashTypeCollection(Types); + assert(LocalHashes.size() == GlobalHashes.size()); + + P.formatLine("Local / Global hashes:"); + TypeIndex TI(TypeIndex::FirstNonSimpleIndex); + for (const auto &H : zip(LocalHashes, GlobalHashes)) { + AutoIndent Indent2(P); + LocallyHashedType &L = std::get<0>(H); + GloballyHashedType &G = std::get<1>(H); + + P.formatLine("TI: {0}, LocalHash: {1:X}, GlobalHash: {2}", TI, L, G); + + ++TI; + } + P.NewLine(); + } + } + + return Error::success(); +} + Error DumpOutputStyle::dumpTpiStream(uint32_t StreamIdx) { assert(StreamIdx == StreamTPI || StreamIdx == StreamIPI); + if (StreamIdx == StreamTPI) { + printHeader(P, "Types (TPI Stream)"); + } else if (StreamIdx == StreamIPI) { + printHeader(P, "Types (IPI Stream)"); + } + + AutoIndent Indent(P); + assert(!File.isObj()); + bool Present = false; bool DumpTypes = false; bool DumpBytes = false; bool DumpExtras = false; std::vector<uint32_t> Indices; if (StreamIdx == StreamTPI) { - printHeader(P, "Types (TPI Stream)"); - Present = File.hasPDBTpiStream(); + Present = getPdb().hasPDBTpiStream(); DumpTypes = opts::dump::DumpTypes; DumpBytes = opts::dump::DumpTypeData; DumpExtras = opts::dump::DumpTypeExtras; Indices.assign(opts::dump::DumpTypeIndex.begin(), opts::dump::DumpTypeIndex.end()); } else if (StreamIdx == StreamIPI) { - printHeader(P, "Types (IPI Stream)"); - Present = File.hasPDBIpiStream(); + Present = getPdb().hasPDBIpiStream(); DumpTypes = opts::dump::DumpIds; DumpBytes = opts::dump::DumpIdData; DumpExtras = opts::dump::DumpIdExtras; @@ -726,22 +1049,23 @@ Error DumpOutputStyle::dumpTpiStream(uint32_t StreamIdx) { opts::dump::DumpIdIndex.end()); } - AutoIndent Indent(P); if (!Present) { P.formatLine("Stream not present"); return Error::success(); } - ExitOnError Err("Unexpected error processing types"); + ExitOnError Err("Unexpected error processing types: "); - auto &Stream = Err((StreamIdx == StreamTPI) ? File.getPDBTpiStream() - : File.getPDBIpiStream()); + auto &Stream = Err((StreamIdx == StreamTPI) ? getPdb().getPDBTpiStream() + : getPdb().getPDBIpiStream()); - auto &Types = Err(initializeTypes(StreamIdx)); + auto &Types = (StreamIdx == StreamTPI) ? File.types() : File.ids(); if (DumpTypes || !Indices.empty()) { if (Indices.empty()) - dumpFullTypeStream(P, Types, Stream, DumpBytes, DumpExtras); + dumpFullTypeStream(P, Types, Stream.getNumTypeRecords(), + Stream.getNumHashBuckets(), Stream.getHashValues(), + DumpBytes, DumpExtras); else { std::vector<TypeIndex> TiList(Indices.begin(), Indices.end()); dumpPartialTypeStream(P, Types, Stream, TiList, DumpBytes, DumpExtras, @@ -761,7 +1085,7 @@ Error DumpOutputStyle::dumpTpiStream(uint32_t StreamIdx) { P.NewLine(); P.formatLine("Hash Adjusters:"); auto &Adjusters = Stream.getHashAdjusters(); - auto &Strings = Err(File.getStringTable()); + auto &Strings = Err(getPdb().getStringTable()); for (const auto &A : Adjusters) { AutoIndent Indent2(P); auto ExpectedStr = Strings.getStringForID(A.first); @@ -777,178 +1101,225 @@ Error DumpOutputStyle::dumpTpiStream(uint32_t StreamIdx) { return Error::success(); } -Expected<codeview::LazyRandomTypeCollection &> -DumpOutputStyle::initializeTypes(uint32_t SN) { - auto &TypeCollection = (SN == StreamTPI) ? TpiTypes : IpiTypes; - auto Tpi = - (SN == StreamTPI) ? File.getPDBTpiStream() : File.getPDBIpiStream(); - if (!Tpi) - return Tpi.takeError(); +Error DumpOutputStyle::dumpModuleSymsForObj() { + printHeader(P, "Symbols"); - if (!TypeCollection) { - auto &Types = Tpi->typeArray(); - uint32_t Count = Tpi->getNumTypeRecords(); - auto Offsets = Tpi->getTypeIndexOffsets(); - TypeCollection = - llvm::make_unique<LazyRandomTypeCollection>(Types, Count, Offsets); - } + AutoIndent Indent(P); + + ExitOnError Err("Unexpected error processing symbols: "); + + auto &Types = File.types(); + + SymbolVisitorCallbackPipeline Pipeline; + SymbolDeserializer Deserializer(nullptr, CodeViewContainer::ObjectFile); + MinimalSymbolDumper Dumper(P, opts::dump::DumpSymRecordBytes, Types, Types); + + Pipeline.addCallbackToPipeline(Deserializer); + Pipeline.addCallbackToPipeline(Dumper); + CVSymbolVisitor Visitor(Pipeline); + + std::unique_ptr<llvm::Error> SymbolError; + + iterateModuleSubsections<DebugSymbolsSubsectionRef>( + File, PrintScope{P, 2}, + [&](uint32_t Modi, const SymbolGroup &Strings, + DebugSymbolsSubsectionRef &Symbols) { + for (auto Symbol : Symbols) { + if (auto EC = Visitor.visitSymbolRecord(Symbol)) { + SymbolError = llvm::make_unique<Error>(std::move(EC)); + return; + } + } + }); + + if (SymbolError) + return std::move(*SymbolError); - return *TypeCollection; + return Error::success(); } -Error DumpOutputStyle::dumpModuleSyms() { +Error DumpOutputStyle::dumpModuleSymsForPdb() { printHeader(P, "Symbols"); AutoIndent Indent(P); - if (!File.hasPDBDbiStream()) { + if (!getPdb().hasPDBDbiStream()) { P.formatLine("DBI Stream not present"); return Error::success(); } - ExitOnError Err("Unexpected error processing symbols"); + ExitOnError Err("Unexpected error processing symbols: "); + + auto &Ids = File.ids(); + auto &Types = File.types(); - auto &Stream = Err(File.getPDBDbiStream()); + iterateSymbolGroups( + File, PrintScope{P, 2}, [&](uint32_t I, const SymbolGroup &Strings) { + auto ExpectedModS = getModuleDebugStream(File.pdb(), I); + if (!ExpectedModS) { + P.formatLine("Error loading module stream {0}. {1}", I, + toString(ExpectedModS.takeError())); + return; + } - auto &Types = Err(initializeTypes(StreamTPI)); + ModuleDebugStreamRef &ModS = *ExpectedModS; - const DbiModuleList &Modules = Stream.modules(); - uint32_t Count = Modules.getModuleCount(); - uint32_t Digits = NumDigits(Count); - for (uint32_t I = 0; I < Count; ++I) { - auto Modi = Modules.getModuleDescriptor(I); - P.formatLine("Mod {0:4} | `{1}`: ", fmt_align(I, AlignStyle::Right, Digits), - Modi.getModuleName()); - uint16_t ModiStream = Modi.getModuleStreamIndex(); - if (ModiStream == kInvalidStreamIndex) { - P.formatLine(" <symbols not present>"); - continue; - } - auto ModStreamData = MappedBlockStream::createIndexedStream( - File.getMsfLayout(), File.getMsfBuffer(), ModiStream, - File.getAllocator()); - - ModuleDebugStreamRef ModS(Modi, std::move(ModStreamData)); - if (auto EC = ModS.reload()) { - P.formatLine("Error loading module stream {0}. {1}", I, - toString(std::move(EC))); - continue; - } + SymbolVisitorCallbackPipeline Pipeline; + SymbolDeserializer Deserializer(nullptr, CodeViewContainer::Pdb); + MinimalSymbolDumper Dumper(P, opts::dump::DumpSymRecordBytes, Ids, + Types); - SymbolVisitorCallbackPipeline Pipeline; - SymbolDeserializer Deserializer(nullptr, CodeViewContainer::Pdb); - MinimalSymbolDumper Dumper(P, opts::dump::DumpSymRecordBytes, Types); + Pipeline.addCallbackToPipeline(Deserializer); + Pipeline.addCallbackToPipeline(Dumper); + CVSymbolVisitor Visitor(Pipeline); + auto SS = ModS.getSymbolsSubstream(); + if (auto EC = + Visitor.visitSymbolStream(ModS.getSymbolArray(), SS.Offset)) { + P.formatLine("Error while processing symbol records. {0}", + toString(std::move(EC))); + return; + } + }); + return Error::success(); +} - Pipeline.addCallbackToPipeline(Deserializer); - Pipeline.addCallbackToPipeline(Dumper); - CVSymbolVisitor Visitor(Pipeline); - auto SS = ModS.getSymbolsSubstream(); - if (auto EC = Visitor.visitSymbolStream(ModS.getSymbolArray(), SS.Offset)) { - P.formatLine("Error while processing symbol records. {0}", - toString(std::move(EC))); - continue; - } +Error DumpOutputStyle::dumpGlobals() { + printHeader(P, "Global Symbols"); + AutoIndent Indent(P); + + if (File.isObj()) { + P.formatLine("Dumping Globals is not supported for object files"); + return Error::success(); + } + + if (!getPdb().hasPDBGlobalsStream()) { + P.formatLine("Globals stream not present"); + return Error::success(); } + ExitOnError Err("Error dumping globals stream: "); + auto &Globals = Err(getPdb().getPDBGlobalsStream()); + + const GSIHashTable &Table = Globals.getGlobalsTable(); + Err(dumpSymbolsFromGSI(Table, opts::dump::DumpGlobalExtras)); return Error::success(); } Error DumpOutputStyle::dumpPublics() { printHeader(P, "Public Symbols"); - AutoIndent Indent(P); - if (!File.hasPDBPublicsStream()) { - P.formatLine("Publics stream not present"); + + if (File.isObj()) { + P.formatLine("Dumping Globals is not supported for object files"); return Error::success(); } - ExitOnError Err("Error dumping publics stream"); + if (!getPdb().hasPDBPublicsStream()) { + P.formatLine("Publics stream not present"); + return Error::success(); + } + ExitOnError Err("Error dumping publics stream: "); + auto &Publics = Err(getPdb().getPDBPublicsStream()); + + const GSIHashTable &PublicsTable = Publics.getPublicsTable(); + if (opts::dump::DumpPublicExtras) { + P.printLine("Publics Header"); + AutoIndent Indent(P); + P.formatLine("sym hash = {0}, thunk table addr = {1}", Publics.getSymHash(), + formatSegmentOffset(Publics.getThunkTableSection(), + Publics.getThunkTableOffset())); + } + Err(dumpSymbolsFromGSI(PublicsTable, opts::dump::DumpPublicExtras)); - auto &Types = Err(initializeTypes(StreamTPI)); - auto &Publics = Err(File.getPDBPublicsStream()); - SymbolVisitorCallbackPipeline Pipeline; - SymbolDeserializer Deserializer(nullptr, CodeViewContainer::Pdb); - MinimalSymbolDumper Dumper(P, opts::dump::DumpSymRecordBytes, Types); + // Skip the rest if we aren't dumping extras. + if (!opts::dump::DumpPublicExtras) + return Error::success(); - Pipeline.addCallbackToPipeline(Deserializer); - Pipeline.addCallbackToPipeline(Dumper); - CVSymbolVisitor Visitor(Pipeline); + P.formatLine("Address Map"); + { + // These are offsets into the publics stream sorted by secidx:secrel. + AutoIndent Indent2(P); + for (uint32_t Addr : Publics.getAddressMap()) + P.formatLine("off = {0}", Addr); + } - auto ExpectedSymbols = Publics.getSymbolArray(); - if (!ExpectedSymbols) { - P.formatLine("Could not read public symbol record stream"); - return Error::success(); + // The thunk map is optional debug info used for ILT thunks. + if (!Publics.getThunkMap().empty()) { + P.formatLine("Thunk Map"); + AutoIndent Indent2(P); + for (uint32_t Addr : Publics.getThunkMap()) + P.formatLine("{0:x8}", Addr); } - if (auto EC = Visitor.visitSymbolStream(*ExpectedSymbols, 0)) - P.formatLine("Error while processing public symbol records. {0}", - toString(std::move(EC))); + // The section offsets table appears to be empty when incremental linking + // isn't in use. + if (!Publics.getSectionOffsets().empty()) { + P.formatLine("Section Offsets"); + AutoIndent Indent2(P); + for (const SectionOffset &SO : Publics.getSectionOffsets()) + P.formatLine("{0:x4}:{1:x8}", uint16_t(SO.Isect), uint32_t(SO.Off)); + } return Error::success(); } -static std::string formatSectionCharacteristics(uint32_t IndentLevel, - uint32_t C) { - using SC = COFF::SectionCharacteristics; - std::vector<std::string> Opts; - if (C == COFF::SC_Invalid) - return "invalid"; - if (C == 0) - return "none"; +Error DumpOutputStyle::dumpSymbolsFromGSI(const GSIHashTable &Table, + bool HashExtras) { + auto ExpectedSyms = getPdb().getPDBSymbolStream(); + if (!ExpectedSyms) + return ExpectedSyms.takeError(); + auto &Types = File.types(); + auto &Ids = File.ids(); + + if (HashExtras) { + P.printLine("GSI Header"); + AutoIndent Indent(P); + P.formatLine("sig = {0:X}, hdr = {1:X}, hr size = {2}, num buckets = {3}", + Table.getVerSignature(), Table.getVerHeader(), + Table.getHashRecordSize(), Table.getNumBuckets()); + } + + { + P.printLine("Records"); + SymbolVisitorCallbackPipeline Pipeline; + SymbolDeserializer Deserializer(nullptr, CodeViewContainer::Pdb); + MinimalSymbolDumper Dumper(P, opts::dump::DumpSymRecordBytes, Ids, Types); + + Pipeline.addCallbackToPipeline(Deserializer); + Pipeline.addCallbackToPipeline(Dumper); + CVSymbolVisitor Visitor(Pipeline); + + BinaryStreamRef SymStream = + ExpectedSyms->getSymbolArray().getUnderlyingStream(); + for (uint32_t PubSymOff : Table) { + Expected<CVSymbol> Sym = readSymbolFromStream(SymStream, PubSymOff); + if (!Sym) + return Sym.takeError(); + if (auto E = Visitor.visitSymbolRecord(*Sym, PubSymOff)) + return E; + } + } + + // Return early if we aren't dumping public hash table and address map info. + if (!HashExtras) + return Error::success(); + + P.formatLine("Hash Entries"); + { + AutoIndent Indent2(P); + for (const PSHashRecord &HR : Table.HashRecords) + P.formatLine("off = {0}, refcnt = {1}", uint32_t(HR.Off), + uint32_t(HR.CRef)); + } + + // FIXME: Dump the bitmap. - PUSH_FLAG(SC, IMAGE_SCN_TYPE_NOLOAD, C, "IMAGE_SCN_TYPE_NOLOAD"); - PUSH_FLAG(SC, IMAGE_SCN_TYPE_NO_PAD, C, "IMAGE_SCN_TYPE_NO_PAD"); - PUSH_FLAG(SC, IMAGE_SCN_CNT_CODE, C, "IMAGE_SCN_CNT_CODE"); - PUSH_FLAG(SC, IMAGE_SCN_CNT_INITIALIZED_DATA, C, - "IMAGE_SCN_CNT_INITIALIZED_DATA"); - PUSH_FLAG(SC, IMAGE_SCN_CNT_UNINITIALIZED_DATA, C, - "IMAGE_SCN_CNT_UNINITIALIZED_DATA"); - PUSH_FLAG(SC, IMAGE_SCN_LNK_OTHER, C, "IMAGE_SCN_LNK_OTHER"); - PUSH_FLAG(SC, IMAGE_SCN_LNK_INFO, C, "IMAGE_SCN_LNK_INFO"); - PUSH_FLAG(SC, IMAGE_SCN_LNK_REMOVE, C, "IMAGE_SCN_LNK_REMOVE"); - PUSH_FLAG(SC, IMAGE_SCN_LNK_COMDAT, C, "IMAGE_SCN_LNK_COMDAT"); - PUSH_FLAG(SC, IMAGE_SCN_GPREL, C, "IMAGE_SCN_GPREL"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_PURGEABLE, C, "IMAGE_SCN_MEM_PURGEABLE"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_16BIT, C, "IMAGE_SCN_MEM_16BIT"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_LOCKED, C, "IMAGE_SCN_MEM_LOCKED"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_PRELOAD, C, "IMAGE_SCN_MEM_PRELOAD"); - PUSH_FLAG(SC, IMAGE_SCN_GPREL, C, "IMAGE_SCN_GPREL"); - PUSH_FLAG(SC, IMAGE_SCN_GPREL, C, "IMAGE_SCN_GPREL"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_1BYTES, C, - "IMAGE_SCN_ALIGN_1BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_2BYTES, C, - "IMAGE_SCN_ALIGN_2BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_4BYTES, C, - "IMAGE_SCN_ALIGN_4BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_8BYTES, C, - "IMAGE_SCN_ALIGN_8BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_16BYTES, C, - "IMAGE_SCN_ALIGN_16BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_32BYTES, C, - "IMAGE_SCN_ALIGN_32BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_64BYTES, C, - "IMAGE_SCN_ALIGN_64BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_128BYTES, C, - "IMAGE_SCN_ALIGN_128BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_256BYTES, C, - "IMAGE_SCN_ALIGN_256BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_512BYTES, C, - "IMAGE_SCN_ALIGN_512BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_1024BYTES, C, - "IMAGE_SCN_ALIGN_1024BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_2048BYTES, C, - "IMAGE_SCN_ALIGN_2048BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_4096BYTES, C, - "IMAGE_SCN_ALIGN_4096BYTES"); - PUSH_MASKED_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_8192BYTES, C, - "IMAGE_SCN_ALIGN_8192BYTES"); - PUSH_FLAG(SC, IMAGE_SCN_LNK_NRELOC_OVFL, C, "IMAGE_SCN_LNK_NRELOC_OVFL"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_DISCARDABLE, C, "IMAGE_SCN_MEM_DISCARDABLE"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_NOT_CACHED, C, "IMAGE_SCN_MEM_NOT_CACHED"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_NOT_PAGED, C, "IMAGE_SCN_MEM_NOT_PAGED"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_SHARED, C, "IMAGE_SCN_MEM_SHARED"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_EXECUTE, C, "IMAGE_SCN_MEM_EXECUTE"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_READ, C, "IMAGE_SCN_MEM_READ"); - PUSH_FLAG(SC, IMAGE_SCN_MEM_WRITE, C, "IMAGE_SCN_MEM_WRITE"); - return typesetItemList(Opts, IndentLevel, 3, " | "); + P.formatLine("Hash Buckets"); + { + AutoIndent Indent2(P); + for (uint32_t Hash : Table.HashBuckets) + P.formatLine("{0:x8}", Hash); + } + + return Error::success(); } static std::string formatSegMapDescriptorFlag(uint32_t IndentLevel, @@ -967,69 +1338,197 @@ static std::string formatSegMapDescriptorFlag(uint32_t IndentLevel, return typesetItemList(Opts, IndentLevel, 4, " | "); } +Error DumpOutputStyle::dumpSectionHeaders() { + dumpSectionHeaders("Section Headers", DbgHeaderType::SectionHdr); + dumpSectionHeaders("Original Section Headers", DbgHeaderType::SectionHdrOrig); + return Error::success(); +} + +static Expected<std::pair<std::unique_ptr<MappedBlockStream>, + ArrayRef<llvm::object::coff_section>>> +loadSectionHeaders(PDBFile &File, DbgHeaderType Type) { + if (!File.hasPDBDbiStream()) + return make_error<StringError>( + "Section headers require a DBI Stream, which could not be loaded", + inconvertibleErrorCode()); + + auto &Dbi = cantFail(File.getPDBDbiStream()); + uint32_t SI = Dbi.getDebugStreamIndex(Type); + + if (SI == kInvalidStreamIndex) + return make_error<StringError>( + "PDB does not contain the requested image section header type", + inconvertibleErrorCode()); + + auto Stream = File.createIndexedStream(SI); + if (!Stream) + return make_error<StringError>("Could not load the required stream data", + inconvertibleErrorCode()); + + ArrayRef<object::coff_section> Headers; + if (Stream->getLength() % sizeof(object::coff_section) != 0) + return make_error<StringError>( + "Section header array size is not a multiple of section header size", + inconvertibleErrorCode()); + + uint32_t NumHeaders = Stream->getLength() / sizeof(object::coff_section); + BinaryStreamReader Reader(*Stream); + cantFail(Reader.readArray(Headers, NumHeaders)); + return std::make_pair(std::move(Stream), Headers); +} + +void DumpOutputStyle::dumpSectionHeaders(StringRef Label, DbgHeaderType Type) { + printHeader(P, Label); + + AutoIndent Indent(P); + if (File.isObj()) { + P.formatLine("Dumping Section Headers is not supported for object files"); + return; + } + + ExitOnError Err("Error dumping section headers: "); + std::unique_ptr<MappedBlockStream> Stream; + ArrayRef<object::coff_section> Headers; + auto ExpectedHeaders = loadSectionHeaders(getPdb(), Type); + if (!ExpectedHeaders) { + P.printLine(toString(ExpectedHeaders.takeError())); + return; + } + std::tie(Stream, Headers) = std::move(*ExpectedHeaders); + + uint32_t I = 1; + for (const auto &Header : Headers) { + P.NewLine(); + P.formatLine("SECTION HEADER #{0}", I); + P.formatLine("{0,8} name", Header.Name); + P.formatLine("{0,8:X-} virtual size", uint32_t(Header.VirtualSize)); + P.formatLine("{0,8:X-} virtual address", uint32_t(Header.VirtualAddress)); + P.formatLine("{0,8:X-} size of raw data", uint32_t(Header.SizeOfRawData)); + P.formatLine("{0,8:X-} file pointer to raw data", + uint32_t(Header.PointerToRawData)); + P.formatLine("{0,8:X-} file pointer to relocation table", + uint32_t(Header.PointerToRelocations)); + P.formatLine("{0,8:X-} file pointer to line numbers", + uint32_t(Header.PointerToLinenumbers)); + P.formatLine("{0,8:X-} number of relocations", + uint32_t(Header.NumberOfRelocations)); + P.formatLine("{0,8:X-} number of line numbers", + uint32_t(Header.NumberOfLinenumbers)); + P.formatLine("{0,8:X-} flags", uint32_t(Header.Characteristics)); + AutoIndent IndentMore(P, 9); + P.formatLine("{0}", formatSectionCharacteristics( + P.getIndentLevel(), Header.Characteristics, 1, "")); + ++I; + } + return; +} + +std::vector<std::string> getSectionNames(PDBFile &File) { + auto ExpectedHeaders = loadSectionHeaders(File, DbgHeaderType::SectionHdr); + if (!ExpectedHeaders) + return {}; + + std::unique_ptr<MappedBlockStream> Stream; + ArrayRef<object::coff_section> Headers; + std::tie(Stream, Headers) = std::move(*ExpectedHeaders); + std::vector<std::string> Names; + for (const auto &H : Headers) + Names.push_back(H.Name); + return Names; +} + Error DumpOutputStyle::dumpSectionContribs() { printHeader(P, "Section Contributions"); - ExitOnError Err("Error dumping publics stream"); AutoIndent Indent(P); - if (!File.hasPDBDbiStream()) { + if (File.isObj()) { + P.formatLine( + "Dumping section contributions is not supported for object files"); + return Error::success(); + } + + ExitOnError Err("Error dumping section contributions: "); + if (!getPdb().hasPDBDbiStream()) { P.formatLine( "Section contribs require a DBI Stream, which could not be loaded"); return Error::success(); } - auto &Dbi = Err(File.getPDBDbiStream()); + auto &Dbi = Err(getPdb().getPDBDbiStream()); class Visitor : public ISectionContribVisitor { public: - Visitor(LinePrinter &P) : P(P) {} + Visitor(LinePrinter &P, ArrayRef<std::string> Names) : P(P), Names(Names) { + auto Max = std::max_element( + Names.begin(), Names.end(), + [](StringRef S1, StringRef S2) { return S1.size() < S2.size(); }); + MaxNameLen = (Max == Names.end() ? 0 : Max->size()); + } void visit(const SectionContrib &SC) override { - P.formatLine( - "SC | mod = {2}, {0}, size = {1}, data crc = {3}, reloc crc = {4}", - formatSegmentOffset(SC.ISect, SC.Off), fmtle(SC.Size), fmtle(SC.Imod), - fmtle(SC.DataCrc), fmtle(SC.RelocCrc)); + assert(SC.ISect > 0); + std::string NameInsert; + if (SC.ISect < Names.size()) { + StringRef SectionName = Names[SC.ISect - 1]; + NameInsert = formatv("[{0}]", SectionName).str(); + } else + NameInsert = "[???]"; + P.formatLine("SC{5} | mod = {2}, {0}, size = {1}, data crc = {3}, reloc " + "crc = {4}", + formatSegmentOffset(SC.ISect, SC.Off), fmtle(SC.Size), + fmtle(SC.Imod), fmtle(SC.DataCrc), fmtle(SC.RelocCrc), + fmt_align(NameInsert, AlignStyle::Left, MaxNameLen + 2)); + AutoIndent Indent(P, MaxNameLen + 2); P.formatLine(" {0}", formatSectionCharacteristics(P.getIndentLevel() + 6, - SC.Characteristics)); + SC.Characteristics, 3, " | ")); } void visit(const SectionContrib2 &SC) override { - P.formatLine("SC2 | mod = {2}, {0}, size = {1}, data crc = {3}, reloc " - "crc = {4}, coff section = {5}", - formatSegmentOffset(SC.Base.ISect, SC.Base.Off), - fmtle(SC.Base.Size), fmtle(SC.Base.Imod), - fmtle(SC.Base.DataCrc), fmtle(SC.Base.RelocCrc), - fmtle(SC.ISectCoff)); - P.formatLine(" {0}", - formatSectionCharacteristics(P.getIndentLevel() + 6, - SC.Base.Characteristics)); + P.formatLine( + "SC2[{6}] | mod = {2}, {0}, size = {1}, data crc = {3}, reloc " + "crc = {4}, coff section = {5}", + formatSegmentOffset(SC.Base.ISect, SC.Base.Off), fmtle(SC.Base.Size), + fmtle(SC.Base.Imod), fmtle(SC.Base.DataCrc), fmtle(SC.Base.RelocCrc), + fmtle(SC.ISectCoff)); + P.formatLine(" {0}", formatSectionCharacteristics( + P.getIndentLevel() + 6, + SC.Base.Characteristics, 3, " | ")); } private: LinePrinter &P; + uint32_t MaxNameLen; + ArrayRef<std::string> Names; }; - Visitor V(P); + std::vector<std::string> Names = getSectionNames(getPdb()); + Visitor V(P, makeArrayRef(Names)); Dbi.visitSectionContributions(V); return Error::success(); } Error DumpOutputStyle::dumpSectionMap() { printHeader(P, "Section Map"); - ExitOnError Err("Error dumping section map"); - AutoIndent Indent(P); - if (!File.hasPDBDbiStream()) { + + if (File.isObj()) { + P.formatLine("Dumping section map is not supported for object files"); + return Error::success(); + } + + ExitOnError Err("Error dumping section map: "); + + if (!getPdb().hasPDBDbiStream()) { P.formatLine("Dumping the section map requires a DBI Stream, which could " "not be loaded"); return Error::success(); } - auto &Dbi = Err(File.getPDBDbiStream()); + auto &Dbi = Err(getPdb().getPDBDbiStream()); uint32_t I = 0; for (auto &M : Dbi.getSectionMap()) { P.formatLine( - "Section {0:4} | ovl = {0}, group = {1}, frame = {2}, name = {3}", I, + "Section {0:4} | ovl = {1}, group = {2}, frame = {3}, name = {4}", I, fmtle(M.Ovl), fmtle(M.Group), fmtle(M.Frame), fmtle(M.SecName)); P.formatLine(" class = {0}, offset = {1}, size = {2}", fmtle(M.ClassName), fmtle(M.Offset), fmtle(M.SecByteLength)); diff --git a/tools/llvm-pdbutil/DumpOutputStyle.h b/tools/llvm-pdbutil/DumpOutputStyle.h index 4c52289f052e..3ce2884b2712 100644 --- a/tools/llvm-pdbutil/DumpOutputStyle.h +++ b/tools/llvm-pdbutil/DumpOutputStyle.h @@ -12,9 +12,12 @@ #include "LinePrinter.h" #include "OutputStyle.h" +#include "StreamUtil.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/PDB/Native/RawConstants.h" #include <string> @@ -25,36 +28,75 @@ namespace codeview { class LazyRandomTypeCollection; } +namespace object { +class COFFObjectFile; +} + namespace pdb { +class GSIHashTable; +class InputFile; + +struct StatCollection { + struct Stat { + Stat() {} + Stat(uint32_t Count, uint32_t Size) : Count(Count), Size(Size) {} + uint32_t Count = 0; + uint32_t Size = 0; + + void update(uint32_t RecordSize) { + ++Count; + Size += RecordSize; + } + }; + + void update(uint32_t Kind, uint32_t RecordSize) { + Totals.update(RecordSize); + auto Iter = Individual.try_emplace(Kind, 1, RecordSize); + if (!Iter.second) + Iter.first->second.update(RecordSize); + } + Stat Totals; + DenseMap<uint32_t, Stat> Individual; +}; + class DumpOutputStyle : public OutputStyle { + public: - DumpOutputStyle(PDBFile &File); + DumpOutputStyle(InputFile &File); Error dump() override; private: - Expected<codeview::LazyRandomTypeCollection &> initializeTypes(uint32_t SN); + PDBFile &getPdb(); + object::COFFObjectFile &getObj(); Error dumpFileSummary(); Error dumpStreamSummary(); + Error dumpSymbolStats(); + Error dumpUdtStats(); Error dumpStringTable(); Error dumpLines(); Error dumpInlineeLines(); Error dumpXmi(); Error dumpXme(); Error dumpTpiStream(uint32_t StreamIdx); + Error dumpTypesFromObjectFile(); Error dumpModules(); Error dumpModuleFiles(); - Error dumpModuleSyms(); + Error dumpModuleSymsForPdb(); + Error dumpModuleSymsForObj(); + Error dumpGlobals(); Error dumpPublics(); + Error dumpSymbolsFromGSI(const GSIHashTable &Table, bool HashExtras); + Error dumpSectionHeaders(); Error dumpSectionContribs(); Error dumpSectionMap(); - PDBFile &File; + void dumpSectionHeaders(StringRef Label, DbgHeaderType Type); + + InputFile &File; LinePrinter P; - std::unique_ptr<codeview::LazyRandomTypeCollection> TpiTypes; - std::unique_ptr<codeview::LazyRandomTypeCollection> IpiTypes; - SmallVector<std::string, 32> StreamPurposes; + SmallVector<StreamInfo, 32> StreamPurposes; }; } // namespace pdb } // namespace llvm diff --git a/tools/llvm-pdbutil/FormatUtil.cpp b/tools/llvm-pdbutil/FormatUtil.cpp index 02030272dd4d..f55d478127d6 100644 --- a/tools/llvm-pdbutil/FormatUtil.cpp +++ b/tools/llvm-pdbutil/FormatUtil.cpp @@ -10,10 +10,13 @@ #include "FormatUtil.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/BinaryFormat/COFF.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/Support/FormatAdapters.h" #include "llvm/Support/FormatVariadic.h" using namespace llvm; +using namespace llvm::codeview; using namespace llvm::pdb; std::string llvm::pdb::truncateStringBack(StringRef S, uint32_t MaxLen) { @@ -96,6 +99,160 @@ std::string llvm::pdb::typesetStringList(uint32_t IndentLevel, return Result; } +std::string llvm::pdb::formatChunkKind(DebugSubsectionKind Kind, + bool Friendly) { + if (Friendly) { + switch (Kind) { + RETURN_CASE(DebugSubsectionKind, None, "none"); + RETURN_CASE(DebugSubsectionKind, Symbols, "symbols"); + RETURN_CASE(DebugSubsectionKind, Lines, "lines"); + RETURN_CASE(DebugSubsectionKind, StringTable, "strings"); + RETURN_CASE(DebugSubsectionKind, FileChecksums, "checksums"); + RETURN_CASE(DebugSubsectionKind, FrameData, "frames"); + RETURN_CASE(DebugSubsectionKind, InlineeLines, "inlinee lines"); + RETURN_CASE(DebugSubsectionKind, CrossScopeImports, "xmi"); + RETURN_CASE(DebugSubsectionKind, CrossScopeExports, "xme"); + RETURN_CASE(DebugSubsectionKind, ILLines, "il lines"); + RETURN_CASE(DebugSubsectionKind, FuncMDTokenMap, "func md token map"); + RETURN_CASE(DebugSubsectionKind, TypeMDTokenMap, "type md token map"); + RETURN_CASE(DebugSubsectionKind, MergedAssemblyInput, + "merged assembly input"); + RETURN_CASE(DebugSubsectionKind, CoffSymbolRVA, "coff symbol rva"); + } + } else { + switch (Kind) { + RETURN_CASE(DebugSubsectionKind, None, "none"); + RETURN_CASE(DebugSubsectionKind, Symbols, "DEBUG_S_SYMBOLS"); + RETURN_CASE(DebugSubsectionKind, Lines, "DEBUG_S_LINES"); + RETURN_CASE(DebugSubsectionKind, StringTable, "DEBUG_S_STRINGTABLE"); + RETURN_CASE(DebugSubsectionKind, FileChecksums, "DEBUG_S_FILECHKSMS"); + RETURN_CASE(DebugSubsectionKind, FrameData, "DEBUG_S_FRAMEDATA"); + RETURN_CASE(DebugSubsectionKind, InlineeLines, "DEBUG_S_INLINEELINES"); + RETURN_CASE(DebugSubsectionKind, CrossScopeImports, + "DEBUG_S_CROSSSCOPEIMPORTS"); + RETURN_CASE(DebugSubsectionKind, CrossScopeExports, + "DEBUG_S_CROSSSCOPEEXPORTS"); + RETURN_CASE(DebugSubsectionKind, ILLines, "DEBUG_S_IL_LINES"); + RETURN_CASE(DebugSubsectionKind, FuncMDTokenMap, + "DEBUG_S_FUNC_MDTOKEN_MAP"); + RETURN_CASE(DebugSubsectionKind, TypeMDTokenMap, + "DEBUG_S_TYPE_MDTOKEN_MAP"); + RETURN_CASE(DebugSubsectionKind, MergedAssemblyInput, + "DEBUG_S_MERGED_ASSEMBLYINPUT"); + RETURN_CASE(DebugSubsectionKind, CoffSymbolRVA, + "DEBUG_S_COFF_SYMBOL_RVA"); + } + } + return formatUnknownEnum(Kind); +} + +std::string llvm::pdb::formatSymbolKind(SymbolKind K) { + switch (uint32_t(K)) { +#define SYMBOL_RECORD(EnumName, value, name) \ + case EnumName: \ + return #EnumName; +#define CV_SYMBOL(EnumName, value) SYMBOL_RECORD(EnumName, value, EnumName) +#include "llvm/DebugInfo/CodeView/CodeViewSymbols.def" + } + return formatUnknownEnum(K); +} + +StringRef llvm::pdb::formatTypeLeafKind(TypeLeafKind K) { + switch (K) { +#define TYPE_RECORD(EnumName, value, name) \ + case EnumName: \ + return #EnumName; +#include "llvm/DebugInfo/CodeView/CodeViewTypes.def" + default: + llvm_unreachable("Unknown type leaf kind!"); + } + return ""; +} + std::string llvm::pdb::formatSegmentOffset(uint16_t Segment, uint32_t Offset) { return formatv("{0:4}:{1:4}", Segment, Offset); } + +#define PUSH_CHARACTERISTIC_FLAG(Enum, TheOpt, Value, Style, Descriptive) \ + PUSH_FLAG(Enum, TheOpt, Value, \ + ((Style == CharacteristicStyle::HeaderDefinition) ? #TheOpt \ + : Descriptive)) + +#define PUSH_MASKED_CHARACTERISTIC_FLAG(Enum, Mask, TheOpt, Value, Style, \ + Descriptive) \ + PUSH_MASKED_FLAG(Enum, Mask, TheOpt, Value, \ + ((Style == CharacteristicStyle::HeaderDefinition) \ + ? #TheOpt \ + : Descriptive)) + +std::string llvm::pdb::formatSectionCharacteristics(uint32_t IndentLevel, + uint32_t C, + uint32_t FlagsPerLine, + StringRef Separator, + CharacteristicStyle Style) { + using SC = COFF::SectionCharacteristics; + std::vector<std::string> Opts; + if (C == COFF::SC_Invalid) + return "invalid"; + if (C == 0) + return "none"; + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_TYPE_NOLOAD, C, Style, "noload"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_TYPE_NO_PAD, C, Style, "no padding"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_CNT_CODE, C, Style, "code"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_CNT_INITIALIZED_DATA, C, Style, + "initialized data"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_CNT_UNINITIALIZED_DATA, C, Style, + "uninitialized data"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_LNK_OTHER, C, Style, "other"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_LNK_INFO, C, Style, "info"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_LNK_REMOVE, C, Style, "remove"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_LNK_COMDAT, C, Style, "comdat"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_GPREL, C, Style, "gp rel"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_PURGEABLE, C, Style, "purgeable"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_16BIT, C, Style, "16-bit"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_LOCKED, C, Style, "locked"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_PRELOAD, C, Style, "preload"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_1BYTES, C, + Style, "1 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_2BYTES, C, + Style, "2 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_4BYTES, C, + Style, "4 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_8BYTES, C, + Style, "8 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_16BYTES, C, + Style, "16 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_32BYTES, C, + Style, "32 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_64BYTES, C, + Style, "64 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_128BYTES, C, + Style, "128 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_256BYTES, C, + Style, "256 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_512BYTES, C, + Style, "512 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_1024BYTES, C, + Style, "1024 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_2048BYTES, C, + Style, "2048 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_4096BYTES, C, + Style, "4096 byte align"); + PUSH_MASKED_CHARACTERISTIC_FLAG(SC, 0xF00000, IMAGE_SCN_ALIGN_8192BYTES, C, + Style, "8192 byte align"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_LNK_NRELOC_OVFL, C, Style, + "noreloc overflow"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_DISCARDABLE, C, Style, + "discardable"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_NOT_CACHED, C, Style, + "not cached"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_NOT_PAGED, C, Style, "not paged"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_SHARED, C, Style, "shared"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_EXECUTE, C, Style, + "execute permissions"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_READ, C, Style, + "read permissions"); + PUSH_CHARACTERISTIC_FLAG(SC, IMAGE_SCN_MEM_WRITE, C, Style, + "write permissions"); + return typesetItemList(Opts, IndentLevel, FlagsPerLine, Separator); +} diff --git a/tools/llvm-pdbutil/FormatUtil.h b/tools/llvm-pdbutil/FormatUtil.h index df32ed9360fb..9a003c9285c9 100644 --- a/tools/llvm-pdbutil/FormatUtil.h +++ b/tools/llvm-pdbutil/FormatUtil.h @@ -12,6 +12,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/Support/Endian.h" #include "llvm/Support/FormatAdapters.h" #include "llvm/Support/FormatVariadic.h" @@ -49,12 +50,26 @@ template <typename T> std::string formatUnknownEnum(T Value) { std::string formatSegmentOffset(uint16_t Segment, uint32_t Offset); +enum class CharacteristicStyle { + HeaderDefinition, // format as windows header definition + Descriptive, // format as human readable words +}; +std::string formatSectionCharacteristics( + uint32_t IndentLevel, uint32_t C, uint32_t FlagsPerLine, + StringRef Separator, + CharacteristicStyle Style = CharacteristicStyle::HeaderDefinition); + std::string typesetItemList(ArrayRef<std::string> Opts, uint32_t IndentLevel, uint32_t GroupSize, StringRef Sep); std::string typesetStringList(uint32_t IndentLevel, ArrayRef<StringRef> Strings); +std::string formatChunkKind(codeview::DebugSubsectionKind Kind, + bool Friendly = true); +std::string formatSymbolKind(codeview::SymbolKind K); +StringRef formatTypeLeafKind(codeview::TypeLeafKind K); + /// Returns the number of digits in the given integer. inline int NumDigits(uint64_t N) { if (N < 10ULL) diff --git a/tools/llvm-pdbutil/InputFile.cpp b/tools/llvm-pdbutil/InputFile.cpp new file mode 100644 index 000000000000..8b05381174df --- /dev/null +++ b/tools/llvm-pdbutil/InputFile.cpp @@ -0,0 +1,469 @@ +//===- InputFile.cpp ------------------------------------------ *- C++ --*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "InputFile.h" + +#include "FormatUtil.h" +#include "LinePrinter.h" + +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" +#include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" +#include "llvm/DebugInfo/PDB/Native/DbiStream.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/Native/PDBFile.h" +#include "llvm/DebugInfo/PDB/Native/PDBStringTable.h" +#include "llvm/DebugInfo/PDB/Native/RawError.h" +#include "llvm/DebugInfo/PDB/Native/TpiStream.h" +#include "llvm/DebugInfo/PDB/PDB.h" +#include "llvm/Object/COFF.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormatVariadic.h" + +using namespace llvm; +using namespace llvm::codeview; +using namespace llvm::object; +using namespace llvm::pdb; + +InputFile::InputFile() {} +InputFile::~InputFile() {} + +static Expected<ModuleDebugStreamRef> +getModuleDebugStream(PDBFile &File, StringRef &ModuleName, uint32_t Index) { + ExitOnError Err("Unexpected error: "); + + auto &Dbi = Err(File.getPDBDbiStream()); + const auto &Modules = Dbi.modules(); + auto Modi = Modules.getModuleDescriptor(Index); + + ModuleName = Modi.getModuleName(); + + uint16_t ModiStream = Modi.getModuleStreamIndex(); + if (ModiStream == kInvalidStreamIndex) + return make_error<RawError>(raw_error_code::no_stream, + "Module stream not present"); + + auto ModStreamData = File.createIndexedStream(ModiStream); + + ModuleDebugStreamRef ModS(Modi, std::move(ModStreamData)); + if (auto EC = ModS.reload()) + return make_error<RawError>(raw_error_code::corrupt_file, + "Invalid module stream"); + + return std::move(ModS); +} + +static inline bool isCodeViewDebugSubsection(object::SectionRef Section, + StringRef Name, + BinaryStreamReader &Reader) { + StringRef SectionName, Contents; + if (Section.getName(SectionName)) + return false; + + if (SectionName != Name) + return false; + + if (Section.getContents(Contents)) + return false; + + Reader = BinaryStreamReader(Contents, support::little); + uint32_t Magic; + if (Reader.bytesRemaining() < sizeof(uint32_t)) + return false; + cantFail(Reader.readInteger(Magic)); + if (Magic != COFF::DEBUG_SECTION_MAGIC) + return false; + return true; +} + +static inline bool isDebugSSection(object::SectionRef Section, + DebugSubsectionArray &Subsections) { + BinaryStreamReader Reader; + if (!isCodeViewDebugSubsection(Section, ".debug$S", Reader)) + return false; + + cantFail(Reader.readArray(Subsections, Reader.bytesRemaining())); + return true; +} + +static bool isDebugTSection(SectionRef Section, CVTypeArray &Types) { + BinaryStreamReader Reader; + if (!isCodeViewDebugSubsection(Section, ".debug$T", Reader)) + return false; + cantFail(Reader.readArray(Types, Reader.bytesRemaining())); + return true; +} + +static std::string formatChecksumKind(FileChecksumKind Kind) { + switch (Kind) { + RETURN_CASE(FileChecksumKind, None, "None"); + RETURN_CASE(FileChecksumKind, MD5, "MD5"); + RETURN_CASE(FileChecksumKind, SHA1, "SHA-1"); + RETURN_CASE(FileChecksumKind, SHA256, "SHA-256"); + } + return formatUnknownEnum(Kind); +} + +static const DebugStringTableSubsectionRef &extractStringTable(PDBFile &File) { + return cantFail(File.getStringTable()).getStringTable(); +} + +template <typename... Args> +static void formatInternal(LinePrinter &Printer, bool Append, Args &&... args) { + if (Append) + Printer.format(std::forward<Args>(args)...); + else + Printer.formatLine(std::forward<Args>(args)...); +} + +SymbolGroup::SymbolGroup(InputFile *File, uint32_t GroupIndex) : File(File) { + if (!File) + return; + + if (File->isPdb()) + initializeForPdb(GroupIndex); + else { + Name = ".debug$S"; + uint32_t I = 0; + for (const auto &S : File->obj().sections()) { + DebugSubsectionArray SS; + if (!isDebugSSection(S, SS)) + continue; + + if (!SC.hasChecksums() || !SC.hasStrings()) + SC.initialize(SS); + + if (I == GroupIndex) + Subsections = SS; + + if (SC.hasChecksums() && SC.hasStrings()) + break; + } + rebuildChecksumMap(); + } +} + +StringRef SymbolGroup::name() const { return Name; } + +void SymbolGroup::updateDebugS(const codeview::DebugSubsectionArray &SS) { + Subsections = SS; +} + +void SymbolGroup::updatePdbModi(uint32_t Modi) { initializeForPdb(Modi); } + +void SymbolGroup::initializeForPdb(uint32_t Modi) { + assert(File && File->isPdb()); + + // PDB always uses the same string table, but each module has its own + // checksums. So we only set the strings if they're not already set. + if (!SC.hasStrings()) + SC.setStrings(extractStringTable(File->pdb())); + + SC.resetChecksums(); + auto MDS = getModuleDebugStream(File->pdb(), Name, Modi); + if (!MDS) { + consumeError(MDS.takeError()); + return; + } + + DebugStream = std::make_shared<ModuleDebugStreamRef>(std::move(*MDS)); + Subsections = DebugStream->getSubsectionsArray(); + SC.initialize(Subsections); + rebuildChecksumMap(); +} + +void SymbolGroup::rebuildChecksumMap() { + if (!SC.hasChecksums()) + return; + + for (const auto &Entry : SC.checksums()) { + auto S = SC.strings().getString(Entry.FileNameOffset); + if (!S) + continue; + ChecksumsByFile[*S] = Entry; + } +} + +const ModuleDebugStreamRef &SymbolGroup::getPdbModuleStream() const { + assert(File && File->isPdb() && DebugStream); + return *DebugStream; +} + +Expected<StringRef> SymbolGroup::getNameFromStringTable(uint32_t Offset) const { + return SC.strings().getString(Offset); +} + +void SymbolGroup::formatFromFileName(LinePrinter &Printer, StringRef File, + bool Append) const { + auto FC = ChecksumsByFile.find(File); + if (FC == ChecksumsByFile.end()) { + formatInternal(Printer, Append, "- (no checksum) {0}", File); + return; + } + + formatInternal(Printer, Append, "- ({0}: {1}) {2}", + formatChecksumKind(FC->getValue().Kind), + toHex(FC->getValue().Checksum), File); +} + +void SymbolGroup::formatFromChecksumsOffset(LinePrinter &Printer, + uint32_t Offset, + bool Append) const { + if (!SC.hasChecksums()) { + formatInternal(Printer, Append, "(unknown file name offset {0})", Offset); + return; + } + + auto Iter = SC.checksums().getArray().at(Offset); + if (Iter == SC.checksums().getArray().end()) { + formatInternal(Printer, Append, "(unknown file name offset {0})", Offset); + return; + } + + uint32_t FO = Iter->FileNameOffset; + auto ExpectedFile = getNameFromStringTable(FO); + if (!ExpectedFile) { + formatInternal(Printer, Append, "(unknown file name offset {0})", Offset); + consumeError(ExpectedFile.takeError()); + return; + } + if (Iter->Kind == FileChecksumKind::None) { + formatInternal(Printer, Append, "{0} (no checksum)", *ExpectedFile); + } else { + formatInternal(Printer, Append, "{0} ({1}: {2})", *ExpectedFile, + formatChecksumKind(Iter->Kind), toHex(Iter->Checksum)); + } +} + +Expected<InputFile> InputFile::open(StringRef Path) { + InputFile IF; + if (!llvm::sys::fs::exists(Path)) + return make_error<StringError>(formatv("File {0} not found", Path), + inconvertibleErrorCode()); + + file_magic Magic; + if (auto EC = identify_magic(Path, Magic)) + return make_error<StringError>( + formatv("Unable to identify file type for file {0}", Path), EC); + + if (Magic == file_magic::coff_object) { + Expected<OwningBinary<Binary>> BinaryOrErr = createBinary(Path); + if (!BinaryOrErr) + return BinaryOrErr.takeError(); + + IF.CoffObject = std::move(*BinaryOrErr); + IF.PdbOrObj = llvm::cast<COFFObjectFile>(IF.CoffObject.getBinary()); + return std::move(IF); + } + + if (Magic == file_magic::unknown) { + std::unique_ptr<IPDBSession> Session; + if (auto Err = loadDataForPDB(PDB_ReaderType::Native, Path, Session)) + return std::move(Err); + + IF.PdbSession.reset(static_cast<NativeSession *>(Session.release())); + IF.PdbOrObj = &IF.PdbSession->getPDBFile(); + + return std::move(IF); + } + + return make_error<StringError>( + formatv("File {0} is not a supported file type", Path), + inconvertibleErrorCode()); +} + +PDBFile &InputFile::pdb() { + assert(isPdb()); + return *PdbOrObj.get<PDBFile *>(); +} + +const PDBFile &InputFile::pdb() const { + assert(isPdb()); + return *PdbOrObj.get<PDBFile *>(); +} + +object::COFFObjectFile &InputFile::obj() { + assert(isObj()); + return *PdbOrObj.get<object::COFFObjectFile *>(); +} + +const object::COFFObjectFile &InputFile::obj() const { + assert(isObj()); + return *PdbOrObj.get<object::COFFObjectFile *>(); +} + +bool InputFile::hasTypes() const { + if (isPdb()) + return pdb().hasPDBTpiStream(); + + for (const auto &Section : obj().sections()) { + CVTypeArray Types; + if (isDebugTSection(Section, Types)) + return true; + } + return false; +} + +bool InputFile::hasIds() const { + if (isObj()) + return false; + return pdb().hasPDBIpiStream(); +} + +bool InputFile::isPdb() const { return PdbOrObj.is<PDBFile *>(); } + +bool InputFile::isObj() const { + return PdbOrObj.is<object::COFFObjectFile *>(); +} + +codeview::LazyRandomTypeCollection & +InputFile::getOrCreateTypeCollection(TypeCollectionKind Kind) { + if (Types && Kind == kTypes) + return *Types; + if (Ids && Kind == kIds) + return *Ids; + + if (Kind == kIds) { + assert(isPdb() && pdb().hasPDBIpiStream()); + } + + // If the collection was already initialized, we should have just returned it + // in step 1. + if (isPdb()) { + TypeCollectionPtr &Collection = (Kind == kIds) ? Ids : Types; + auto &Stream = cantFail((Kind == kIds) ? pdb().getPDBIpiStream() + : pdb().getPDBTpiStream()); + + auto &Array = Stream.typeArray(); + uint32_t Count = Stream.getNumTypeRecords(); + auto Offsets = Stream.getTypeIndexOffsets(); + Collection = + llvm::make_unique<LazyRandomTypeCollection>(Array, Count, Offsets); + return *Collection; + } + + assert(isObj()); + assert(Kind == kTypes); + assert(!Types); + + for (const auto &Section : obj().sections()) { + CVTypeArray Records; + if (!isDebugTSection(Section, Records)) + continue; + + Types = llvm::make_unique<LazyRandomTypeCollection>(Records, 100); + return *Types; + } + + Types = llvm::make_unique<LazyRandomTypeCollection>(100); + return *Types; +} + +codeview::LazyRandomTypeCollection &InputFile::types() { + return getOrCreateTypeCollection(kTypes); +} + +codeview::LazyRandomTypeCollection &InputFile::ids() { + // Object files have only one type stream that contains both types and ids. + // Similarly, some PDBs don't contain an IPI stream, and for those both types + // and IDs are in the same stream. + if (isObj() || !pdb().hasPDBIpiStream()) + return types(); + + return getOrCreateTypeCollection(kIds); +} + +iterator_range<SymbolGroupIterator> InputFile::symbol_groups() { + return make_range<SymbolGroupIterator>(symbol_groups_begin(), + symbol_groups_end()); +} + +SymbolGroupIterator InputFile::symbol_groups_begin() { + return SymbolGroupIterator(*this); +} + +SymbolGroupIterator InputFile::symbol_groups_end() { + return SymbolGroupIterator(); +} + +SymbolGroupIterator::SymbolGroupIterator() : Value(nullptr) {} + +SymbolGroupIterator::SymbolGroupIterator(InputFile &File) : Value(&File) { + if (File.isObj()) { + SectionIter = File.obj().section_begin(); + scanToNextDebugS(); + } +} + +bool SymbolGroupIterator::operator==(const SymbolGroupIterator &R) const { + bool E = isEnd(); + bool RE = R.isEnd(); + if (E || RE) + return E == RE; + + if (Value.File != R.Value.File) + return false; + return Index == R.Index; +} + +const SymbolGroup &SymbolGroupIterator::operator*() const { + assert(!isEnd()); + return Value; +} +SymbolGroup &SymbolGroupIterator::operator*() { + assert(!isEnd()); + return Value; +} + +SymbolGroupIterator &SymbolGroupIterator::operator++() { + assert(Value.File && !isEnd()); + ++Index; + if (isEnd()) + return *this; + + if (Value.File->isPdb()) { + Value.updatePdbModi(Index); + return *this; + } + + scanToNextDebugS(); + return *this; +} + +void SymbolGroupIterator::scanToNextDebugS() { + assert(SectionIter.hasValue()); + auto End = Value.File->obj().section_end(); + auto &Iter = *SectionIter; + assert(!isEnd()); + + while (++Iter != End) { + DebugSubsectionArray SS; + SectionRef SR = *Iter; + if (!isDebugSSection(SR, SS)) + continue; + + Value.updateDebugS(SS); + return; + } +} + +bool SymbolGroupIterator::isEnd() const { + if (!Value.File) + return true; + if (Value.File->isPdb()) { + auto &Dbi = cantFail(Value.File->pdb().getPDBDbiStream()); + uint32_t Count = Dbi.modules().getModuleCount(); + assert(Index <= Count); + return Index == Count; + } + + assert(SectionIter.hasValue()); + return *SectionIter == Value.File->obj().section_end(); +} diff --git a/tools/llvm-pdbutil/InputFile.h b/tools/llvm-pdbutil/InputFile.h new file mode 100644 index 000000000000..8063439133c2 --- /dev/null +++ b/tools/llvm-pdbutil/InputFile.h @@ -0,0 +1,147 @@ +//===- InputFile.h -------------------------------------------- *- C++ --*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVMPDBDUMP_INPUTFILE_H +#define LLVM_TOOLS_LLVMPDBDUMP_INPUTFILE_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/iterator.h" +#include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" +#include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" +#include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace codeview { +class LazyRandomTypeCollection; +} +namespace object { +class COFFObjectFile; +class SectionRef; +} // namespace object + +namespace pdb { +class InputFile; +class LinePrinter; +class PDBFile; +class NativeSession; +class SymbolGroupIterator; +class SymbolGroup; + +class InputFile { + InputFile(); + + std::unique_ptr<NativeSession> PdbSession; + object::OwningBinary<object::Binary> CoffObject; + PointerUnion<PDBFile *, object::COFFObjectFile *> PdbOrObj; + + using TypeCollectionPtr = std::unique_ptr<codeview::LazyRandomTypeCollection>; + + TypeCollectionPtr Types; + TypeCollectionPtr Ids; + + enum TypeCollectionKind { kTypes, kIds }; + codeview::LazyRandomTypeCollection & + getOrCreateTypeCollection(TypeCollectionKind Kind); + +public: + ~InputFile(); + InputFile(InputFile &&Other) = default; + + static Expected<InputFile> open(StringRef Path); + + PDBFile &pdb(); + const PDBFile &pdb() const; + object::COFFObjectFile &obj(); + const object::COFFObjectFile &obj() const; + + bool hasTypes() const; + bool hasIds() const; + + codeview::LazyRandomTypeCollection &types(); + codeview::LazyRandomTypeCollection &ids(); + + iterator_range<SymbolGroupIterator> symbol_groups(); + SymbolGroupIterator symbol_groups_begin(); + SymbolGroupIterator symbol_groups_end(); + + bool isPdb() const; + bool isObj() const; +}; + +class SymbolGroup { + friend class SymbolGroupIterator; + +public: + explicit SymbolGroup(InputFile *File, uint32_t GroupIndex = 0); + + Expected<StringRef> getNameFromStringTable(uint32_t Offset) const; + + void formatFromFileName(LinePrinter &Printer, StringRef File, + bool Append = false) const; + + void formatFromChecksumsOffset(LinePrinter &Printer, uint32_t Offset, + bool Append = false) const; + + StringRef name() const; + + codeview::DebugSubsectionArray getDebugSubsections() const { + return Subsections; + } + const ModuleDebugStreamRef &getPdbModuleStream() const; + + const InputFile &getFile() const { return *File; } + InputFile &getFile() { return *File; } + +private: + void initializeForPdb(uint32_t Modi); + void updatePdbModi(uint32_t Modi); + void updateDebugS(const codeview::DebugSubsectionArray &SS); + + void rebuildChecksumMap(); + InputFile *File = nullptr; + StringRef Name; + codeview::DebugSubsectionArray Subsections; + std::shared_ptr<ModuleDebugStreamRef> DebugStream; + codeview::StringsAndChecksumsRef SC; + StringMap<codeview::FileChecksumEntry> ChecksumsByFile; +}; + +class SymbolGroupIterator + : public iterator_facade_base<SymbolGroupIterator, + std::forward_iterator_tag, SymbolGroup> { +public: + SymbolGroupIterator(); + explicit SymbolGroupIterator(InputFile &File); + SymbolGroupIterator(const SymbolGroupIterator &Other) = default; + SymbolGroupIterator &operator=(const SymbolGroupIterator &R) = default; + + const SymbolGroup &operator*() const; + SymbolGroup &operator*(); + + bool operator==(const SymbolGroupIterator &R) const; + SymbolGroupIterator &operator++(); + +private: + void scanToNextDebugS(); + bool isEnd() const; + + uint32_t Index = 0; + Optional<object::section_iterator> SectionIter; + SymbolGroup Value; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/tools/llvm-pdbutil/LinePrinter.cpp b/tools/llvm-pdbutil/LinePrinter.cpp index 6e4d95af944a..e80a1762450b 100644 --- a/tools/llvm-pdbutil/LinePrinter.cpp +++ b/tools/llvm-pdbutil/LinePrinter.cpp @@ -13,7 +13,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/DebugInfo/MSF/MSFCommon.h" -#include "llvm/DebugInfo/MSF/MSFStreamLayout.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/UDTLayout.h" @@ -246,6 +245,30 @@ void LinePrinter::formatMsfStreamData(StringRef Label, PDBFile &File, OS << ")"; } +void LinePrinter::formatMsfStreamBlocks( + PDBFile &File, const msf::MSFStreamLayout &StreamLayout) { + auto Blocks = makeArrayRef(StreamLayout.Blocks); + uint32_t L = StreamLayout.Length; + + while (L > 0) { + NewLine(); + assert(!Blocks.empty()); + OS << formatv("Block {0} (\n", uint32_t(Blocks.front())); + uint32_t UsedBytes = std::min(L, File.getBlockSize()); + ArrayRef<uint8_t> BlockData = + cantFail(File.getBlockData(Blocks.front(), File.getBlockSize())); + uint64_t BaseOffset = Blocks.front(); + BaseOffset *= File.getBlockSize(); + OS << format_bytes_with_ascii(BlockData, BaseOffset, 32, 4, + CurrentIndent + IndentSpaces, true); + NewLine(); + OS << ")"; + NewLine(); + L -= UsedBytes; + Blocks = Blocks.drop_front(); + } +} + bool LinePrinter::IsTypeExcluded(llvm::StringRef TypeName, uint32_t Size) { if (IsItemExcluded(TypeName, IncludeTypeFilters, ExcludeTypeFilters)) return true; diff --git a/tools/llvm-pdbutil/LinePrinter.h b/tools/llvm-pdbutil/LinePrinter.h index 68ce321a27ec..09bde28f516a 100644 --- a/tools/llvm-pdbutil/LinePrinter.h +++ b/tools/llvm-pdbutil/LinePrinter.h @@ -60,6 +60,7 @@ public: void formatMsfStreamData(StringRef Label, PDBFile &File, const msf::MSFStreamLayout &Stream, BinarySubstreamRef Substream); + void formatMsfStreamBlocks(PDBFile &File, const msf::MSFStreamLayout &Stream); bool hasColor() const { return UseColor; } raw_ostream &getStream() { return OS; } @@ -92,14 +93,41 @@ private: std::list<Regex> IncludeSymbolFilters; }; +struct PrintScope { + explicit PrintScope(LinePrinter &P, uint32_t IndentLevel) + : P(P), IndentLevel(IndentLevel) {} + explicit PrintScope(const PrintScope &Other, uint32_t LabelWidth) + : P(Other.P), IndentLevel(Other.IndentLevel), LabelWidth(LabelWidth) {} + + LinePrinter &P; + uint32_t IndentLevel; + uint32_t LabelWidth = 0; +}; + +inline Optional<PrintScope> withLabelWidth(const Optional<PrintScope> &Scope, + uint32_t W) { + if (!Scope) + return None; + return PrintScope{*Scope, W}; +} + struct AutoIndent { explicit AutoIndent(LinePrinter &L, uint32_t Amount = 0) - : L(L), Amount(Amount) { + : L(&L), Amount(Amount) { L.Indent(Amount); } - ~AutoIndent() { L.Unindent(Amount); } + explicit AutoIndent(const Optional<PrintScope> &Scope) { + if (Scope.hasValue()) { + L = &Scope->P; + Amount = Scope->IndentLevel; + } + } + ~AutoIndent() { + if (L) + L->Unindent(Amount); + } - LinePrinter &L; + LinePrinter *L = nullptr; uint32_t Amount = 0; }; diff --git a/tools/llvm-pdbutil/MinimalSymbolDumper.cpp b/tools/llvm-pdbutil/MinimalSymbolDumper.cpp index d93843649db0..40a0e46efd48 100644 --- a/tools/llvm-pdbutil/MinimalSymbolDumper.cpp +++ b/tools/llvm-pdbutil/MinimalSymbolDumper.cpp @@ -24,18 +24,6 @@ using namespace llvm; using namespace llvm::codeview; using namespace llvm::pdb; -static StringRef getSymbolKindName(SymbolKind K) { - switch (K) { -#define SYMBOL_RECORD(EnumName, value, name) \ - case EnumName: \ - return #EnumName; -#include "llvm/DebugInfo/CodeView/CodeViewSymbols.def" - default: - llvm_unreachable("Unknown symbol kind!"); - } - return ""; -} - static std::string formatLocalSymFlags(uint32_t IndentLevel, LocalSymFlags Flags) { std::vector<std::string> Opts; @@ -216,6 +204,7 @@ static std::string formatSourceLanguage(SourceLanguage Lang) { RETURN_CASE(SourceLanguage, JScript, "javascript"); RETURN_CASE(SourceLanguage, MSIL, "msil"); RETURN_CASE(SourceLanguage, HLSL, "hlsl"); + RETURN_CASE(SourceLanguage, D, "d"); } return formatUnknownEnum(Lang); } @@ -269,6 +258,7 @@ static std::string formatMachineType(CPUType Cpu) { RETURN_CASE(CPUType, ARM_XMAC, "arm xmac"); RETURN_CASE(CPUType, ARM_WMMX, "arm wmmx"); RETURN_CASE(CPUType, ARM7, "arm 7"); + RETURN_CASE(CPUType, ARM64, "arm64"); RETURN_CASE(CPUType, Omni, "omni"); RETURN_CASE(CPUType, Ia64, "intel itanium ia64"); RETURN_CASE(CPUType, Ia64_2, "intel itanium ia64 2"); @@ -297,57 +287,11 @@ static std::string formatCookieKind(FrameCookieKind Kind) { static std::string formatRegisterId(RegisterId Id) { switch (Id) { - RETURN_CASE(RegisterId, VFrame, "vframe"); - RETURN_CASE(RegisterId, AL, "al"); - RETURN_CASE(RegisterId, CL, "cl"); - RETURN_CASE(RegisterId, DL, "dl"); - RETURN_CASE(RegisterId, BL, "bl"); - RETURN_CASE(RegisterId, AH, "ah"); - RETURN_CASE(RegisterId, CH, "ch"); - RETURN_CASE(RegisterId, DH, "dh"); - RETURN_CASE(RegisterId, BH, "bh"); - RETURN_CASE(RegisterId, AX, "ax"); - RETURN_CASE(RegisterId, CX, "cx"); - RETURN_CASE(RegisterId, DX, "dx"); - RETURN_CASE(RegisterId, BX, "bx"); - RETURN_CASE(RegisterId, SP, "sp"); - RETURN_CASE(RegisterId, BP, "bp"); - RETURN_CASE(RegisterId, SI, "si"); - RETURN_CASE(RegisterId, DI, "di"); - RETURN_CASE(RegisterId, EAX, "eax"); - RETURN_CASE(RegisterId, ECX, "ecx"); - RETURN_CASE(RegisterId, EDX, "edx"); - RETURN_CASE(RegisterId, EBX, "ebx"); - RETURN_CASE(RegisterId, ESP, "esp"); - RETURN_CASE(RegisterId, EBP, "ebp"); - RETURN_CASE(RegisterId, ESI, "esi"); - RETURN_CASE(RegisterId, EDI, "edi"); - RETURN_CASE(RegisterId, ES, "es"); - RETURN_CASE(RegisterId, CS, "cs"); - RETURN_CASE(RegisterId, SS, "ss"); - RETURN_CASE(RegisterId, DS, "ds"); - RETURN_CASE(RegisterId, FS, "fs"); - RETURN_CASE(RegisterId, GS, "gs"); - RETURN_CASE(RegisterId, IP, "ip"); - RETURN_CASE(RegisterId, RAX, "rax"); - RETURN_CASE(RegisterId, RBX, "rbx"); - RETURN_CASE(RegisterId, RCX, "rcx"); - RETURN_CASE(RegisterId, RDX, "rdx"); - RETURN_CASE(RegisterId, RSI, "rsi"); - RETURN_CASE(RegisterId, RDI, "rdi"); - RETURN_CASE(RegisterId, RBP, "rbp"); - RETURN_CASE(RegisterId, RSP, "rsp"); - RETURN_CASE(RegisterId, R8, "r8"); - RETURN_CASE(RegisterId, R9, "r9"); - RETURN_CASE(RegisterId, R10, "r10"); - RETURN_CASE(RegisterId, R11, "r11"); - RETURN_CASE(RegisterId, R12, "r12"); - RETURN_CASE(RegisterId, R13, "r13"); - RETURN_CASE(RegisterId, R14, "r14"); - RETURN_CASE(RegisterId, R15, "r15"); - default: - return formatUnknownEnum(Id); +#define CV_REGISTER(name, val) RETURN_CASE(RegisterId, name, #name) +#include "llvm/DebugInfo/CodeView/CodeViewRegisters.def" +#undef CV_REGISTER } + return formatUnknownEnum(Id); } static std::string formatRange(LocalVariableAddrRange Range) { @@ -377,20 +321,26 @@ Error MinimalSymbolDumper::visitSymbolBegin(codeview::CVSymbol &Record, // append to the existing line. P.formatLine("{0} | {1} [size = {2}]", fmt_align(Offset, AlignStyle::Right, 6), - getSymbolKindName(Record.Type), Record.length()); + formatSymbolKind(Record.Type), Record.length()); P.Indent(); return Error::success(); } Error MinimalSymbolDumper::visitSymbolEnd(CVSymbol &Record) { + if (RecordBytes) { + AutoIndent Indent(P, 7); + P.formatBinary("bytes", Record.content(), 0); + } P.Unindent(); return Error::success(); } -std::string MinimalSymbolDumper::typeIndex(TypeIndex TI) const { - if (TI.isSimple()) +std::string MinimalSymbolDumper::typeOrIdIndex(codeview::TypeIndex TI, + bool IsType) const { + if (TI.isSimple() || TI.isDecoratedItemId()) return formatv("{0}", TI).str(); - StringRef Name = Types.getTypeName(TI); + auto &Container = IsType ? Types : Ids; + StringRef Name = Container.getTypeName(TI); if (Name.size() > 32) { Name = Name.take_front(32); return formatv("{0} ({1}...)", TI, Name); @@ -398,6 +348,14 @@ std::string MinimalSymbolDumper::typeIndex(TypeIndex TI) const { return formatv("{0} ({1})", TI, Name); } +std::string MinimalSymbolDumper::idIndex(codeview::TypeIndex TI) const { + return typeOrIdIndex(TI, false); +} + +std::string MinimalSymbolDumper::typeIndex(TypeIndex TI) const { + return typeOrIdIndex(TI, true); +} + Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR, BlockSym &Block) { P.format(" `{0}`", Block.Name); AutoIndent Indent(P, 7); @@ -434,18 +392,27 @@ Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR, SectionSym &Section) { P.format(" `{0}`", Section.Name); AutoIndent Indent(P, 7); - P.formatLine("length = {0}, alignment = {1}, rva = {2}, section # = {3}, " - "characteristics = {4}", + P.formatLine("length = {0}, alignment = {1}, rva = {2}, section # = {3}", Section.Length, Section.Alignment, Section.Rva, - Section.SectionNumber, Section.Characteristics); + Section.SectionNumber); + P.printLine("characteristics ="); + AutoIndent Indent2(P, 2); + P.printLine(formatSectionCharacteristics(P.getIndentLevel(), + Section.Characteristics, 1, "", + CharacteristicStyle::Descriptive)); return Error::success(); } Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR, CoffGroupSym &CG) { P.format(" `{0}`", CG.Name); AutoIndent Indent(P, 7); - P.formatLine("length = {0}, addr = {1}, characteristics = {2}", CG.Size, - formatSegmentOffset(CG.Segment, CG.Offset), CG.Characteristics); + P.formatLine("length = {0}, addr = {1}", CG.Size, + formatSegmentOffset(CG.Segment, CG.Offset)); + P.printLine("characteristics ="); + AutoIndent Indent2(P, 2); + P.printLine(formatSectionCharacteristics(P.getIndentLevel(), + CG.Characteristics, 1, "", + CharacteristicStyle::Descriptive)); return Error::success(); } @@ -660,7 +627,7 @@ Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR, InlineSiteSym &IS) { StringRef Annotations(reinterpret_cast<const char *>(Bytes.begin()), Bytes.size()); - P.formatLine("inlinee = {0}, parent = {1}, end = {2}", typeIndex(IS.Inlinee), + P.formatLine("inlinee = {0}, parent = {1}, end = {2}", idIndex(IS.Inlinee), IS.Parent, IS.End); P.formatLine("annotations = {0}", toHex(Annotations)); return Error::success(); @@ -725,9 +692,19 @@ Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR, ProcSym &Proc) { Proc.Parent, Proc.End, formatSegmentOffset(Proc.Segment, Proc.CodeOffset), Proc.CodeSize); - // FIXME: It seems FunctionType is sometimes an id and sometimes a type. + bool IsType = true; + switch (Proc.getKind()) { + case SymbolRecordKind::GlobalProcIdSym: + case SymbolRecordKind::ProcIdSym: + case SymbolRecordKind::DPCProcIdSym: + IsType = false; + break; + default: + break; + } P.formatLine("type = `{0}`, debug start = {1}, debug end = {2}, flags = {3}", - typeIndex(Proc.FunctionType), Proc.DbgStart, Proc.DbgEnd, + typeOrIdIndex(Proc.FunctionType, IsType), Proc.DbgStart, + Proc.DbgEnd, formatProcSymFlags(P.getIndentLevel() + 9, Proc.Flags)); return Error::success(); } @@ -740,7 +717,7 @@ Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR, Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR, CallerSym &Caller) { AutoIndent Indent(P, 7); for (const auto &I : Caller.Indices) { - P.formatLine("callee: {0}", typeIndex(I)); + P.formatLine("callee: {0}", idIndex(I)); } return Error::success(); } diff --git a/tools/llvm-pdbutil/MinimalSymbolDumper.h b/tools/llvm-pdbutil/MinimalSymbolDumper.h index 5e30959ea9c0..d9e9861d5b30 100644 --- a/tools/llvm-pdbutil/MinimalSymbolDumper.h +++ b/tools/llvm-pdbutil/MinimalSymbolDumper.h @@ -23,8 +23,9 @@ class LinePrinter; class MinimalSymbolDumper : public codeview::SymbolVisitorCallbacks { public: MinimalSymbolDumper(LinePrinter &P, bool RecordBytes, + codeview::LazyRandomTypeCollection &Ids, codeview::LazyRandomTypeCollection &Types) - : P(P), Types(Types) {} + : P(P), RecordBytes(RecordBytes), Ids(Ids), Types(Types) {} Error visitSymbolBegin(codeview::CVSymbol &Record) override; Error visitSymbolBegin(codeview::CVSymbol &Record, uint32_t Offset) override; @@ -37,9 +38,14 @@ public: #include "llvm/DebugInfo/CodeView/CodeViewSymbols.def" private: + std::string typeOrIdIndex(codeview::TypeIndex TI, bool IsType) const; + std::string typeIndex(codeview::TypeIndex TI) const; + std::string idIndex(codeview::TypeIndex TI) const; LinePrinter &P; + bool RecordBytes; + codeview::LazyRandomTypeCollection &Ids; codeview::LazyRandomTypeCollection &Types; }; } // namespace pdb diff --git a/tools/llvm-pdbutil/MinimalTypeDumper.cpp b/tools/llvm-pdbutil/MinimalTypeDumper.cpp index 0079b9e7eaa4..fae89920e0b8 100644 --- a/tools/llvm-pdbutil/MinimalTypeDumper.cpp +++ b/tools/llvm-pdbutil/MinimalTypeDumper.cpp @@ -26,18 +26,6 @@ using namespace llvm; using namespace llvm::codeview; using namespace llvm::pdb; -static StringRef getLeafTypeName(TypeLeafKind K) { - switch (K) { -#define TYPE_RECORD(EnumName, value, name) \ - case EnumName: \ - return #EnumName; -#include "llvm/DebugInfo/CodeView/CodeViewTypes.def" - default: - llvm_unreachable("Unknown type leaf kind!"); - } - return ""; -} - static std::string formatClassOptions(uint32_t IndentLevel, ClassOptions Options) { std::vector<std::string> Opts; @@ -212,7 +200,7 @@ Error MinimalTypeDumpVisitor::visitTypeBegin(CVType &Record, TypeIndex Index) { if (!Hashes) { P.formatLine("{0} | {1} [size = {2}]", fmt_align(Index, AlignStyle::Right, Width), - getLeafTypeName(Record.Type), Record.length()); + formatTypeLeafKind(Record.Type), Record.length()); } else { std::string H; if (Index.toArrayIndex() >= HashValues.size()) { @@ -231,7 +219,7 @@ Error MinimalTypeDumpVisitor::visitTypeBegin(CVType &Record, TypeIndex Index) { } P.formatLine("{0} | {1} [size = {2}, hash = {3}]", fmt_align(Index, AlignStyle::Right, Width), - getLeafTypeName(Record.Type), Record.length(), H); + formatTypeLeafKind(Record.Type), Record.length(), H); } P.Indent(Width + 3); return Error::success(); @@ -246,7 +234,7 @@ Error MinimalTypeDumpVisitor::visitTypeEnd(CVType &Record) { } Error MinimalTypeDumpVisitor::visitMemberBegin(CVMemberRecord &Record) { - P.formatLine("- {0}", getLeafTypeName(Record.Kind)); + P.formatLine("- {0}", formatTypeLeafKind(Record.Kind)); return Error::success(); } diff --git a/tools/llvm-pdbutil/PdbYaml.cpp b/tools/llvm-pdbutil/PdbYaml.cpp index 9c3beb566d2c..eb39708a27e9 100644 --- a/tools/llvm-pdbutil/PdbYaml.cpp +++ b/tools/llvm-pdbutil/PdbYaml.cpp @@ -10,17 +10,10 @@ #include "PdbYaml.h" #include "llvm/ADT/StringExtras.h" -#include "llvm/DebugInfo/CodeView/CVSymbolVisitor.h" #include "llvm/DebugInfo/CodeView/CVTypeVisitor.h" -#include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h" -#include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" -#include "llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h" -#include "llvm/DebugInfo/CodeView/TypeSerializer.h" -#include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/DebugInfo/PDB/Native/TpiHashing.h" -#include "llvm/DebugInfo/PDB/PDBExtras.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/ObjectYAML/CodeViewYAMLDebugSections.h" #include "llvm/ObjectYAML/CodeViewYAMLTypes.h" diff --git a/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp b/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp index fcda312e65e9..10b3d9ee7304 100644 --- a/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp +++ b/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp @@ -9,7 +9,6 @@ #include "PrettyBuiltinDumper.h" #include "LinePrinter.h" -#include "llvm-pdbutil.h" #include "llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h" diff --git a/tools/llvm-pdbutil/PrettyEnumDumper.cpp b/tools/llvm-pdbutil/PrettyEnumDumper.cpp index 7aff5b93d986..bf22e75e3949 100644 --- a/tools/llvm-pdbutil/PrettyEnumDumper.cpp +++ b/tools/llvm-pdbutil/PrettyEnumDumper.cpp @@ -26,25 +26,29 @@ void EnumDumper::start(const PDBSymbolTypeEnum &Symbol) { WithColor(Printer, PDB_ColorItem::Keyword).get() << "enum "; WithColor(Printer, PDB_ColorItem::Type).get() << Symbol.getName(); if (!opts::pretty::NoEnumDefs) { - auto BuiltinType = Symbol.getUnderlyingType(); - if (BuiltinType->getBuiltinType() != PDB_BuiltinType::Int || - BuiltinType->getLength() != 4) { + auto UnderlyingType = Symbol.getUnderlyingType(); + if (!UnderlyingType) + return; + if (UnderlyingType->getBuiltinType() != PDB_BuiltinType::Int || + UnderlyingType->getLength() != 4) { Printer << " : "; BuiltinDumper Dumper(Printer); - Dumper.start(*BuiltinType); + Dumper.start(*UnderlyingType); } + auto EnumValues = Symbol.findAllChildren<PDBSymbolData>(); Printer << " {"; Printer.Indent(); - auto EnumValues = Symbol.findAllChildren<PDBSymbolData>(); - while (auto EnumValue = EnumValues->getNext()) { - if (EnumValue->getDataKind() != PDB_DataKind::Constant) - continue; - Printer.NewLine(); - WithColor(Printer, PDB_ColorItem::Identifier).get() - << EnumValue->getName(); - Printer << " = "; - WithColor(Printer, PDB_ColorItem::LiteralValue).get() - << EnumValue->getValue(); + if (EnumValues && EnumValues->getChildCount() > 0) { + while (auto EnumValue = EnumValues->getNext()) { + if (EnumValue->getDataKind() != PDB_DataKind::Constant) + continue; + Printer.NewLine(); + WithColor(Printer, PDB_ColorItem::Identifier).get() + << EnumValue->getName(); + Printer << " = "; + WithColor(Printer, PDB_ColorItem::LiteralValue).get() + << EnumValue->getValue(); + } } Printer.Unindent(); Printer.NewLine(); diff --git a/tools/llvm-pdbutil/PrettyFunctionDumper.cpp b/tools/llvm-pdbutil/PrettyFunctionDumper.cpp index 06d72410359f..0bffc73f6c74 100644 --- a/tools/llvm-pdbutil/PrettyFunctionDumper.cpp +++ b/tools/llvm-pdbutil/PrettyFunctionDumper.cpp @@ -10,7 +10,6 @@ #include "PrettyFunctionDumper.h" #include "LinePrinter.h" #include "PrettyBuiltinDumper.h" -#include "llvm-pdbutil.h" #include "llvm/DebugInfo/PDB/IPDBSession.h" #include "llvm/DebugInfo/PDB/PDBExtras.h" diff --git a/tools/llvm-pdbutil/PrettyTypedefDumper.cpp b/tools/llvm-pdbutil/PrettyTypedefDumper.cpp index 2266e6ea2bef..ba3b4c8035c5 100644 --- a/tools/llvm-pdbutil/PrettyTypedefDumper.cpp +++ b/tools/llvm-pdbutil/PrettyTypedefDumper.cpp @@ -12,7 +12,6 @@ #include "LinePrinter.h" #include "PrettyBuiltinDumper.h" #include "PrettyFunctionDumper.h" -#include "llvm-pdbutil.h" #include "llvm/DebugInfo/PDB/IPDBSession.h" #include "llvm/DebugInfo/PDB/PDBExtras.h" diff --git a/tools/llvm-pdbutil/StreamUtil.cpp b/tools/llvm-pdbutil/StreamUtil.cpp index 4d352004dec3..991c99aa8686 100644 --- a/tools/llvm-pdbutil/StreamUtil.cpp +++ b/tools/llvm-pdbutil/StreamUtil.cpp @@ -22,9 +22,57 @@ using namespace llvm; using namespace llvm::pdb; -void llvm::pdb::discoverStreamPurposes( - PDBFile &File, - SmallVectorImpl<std::pair<StreamPurpose, std::string>> &Purposes) { +std::string StreamInfo::getLongName() const { + if (Purpose == StreamPurpose::NamedStream) + return formatv("Named Stream \"{0}\"", Name).str(); + if (Purpose == StreamPurpose::ModuleStream) + return formatv("Module \"{0}\"", Name).str(); + return Name; +} + +StreamInfo StreamInfo::createStream(StreamPurpose Purpose, StringRef Name, + uint32_t StreamIndex) { + StreamInfo Result; + Result.Name = Name; + Result.StreamIndex = StreamIndex; + Result.Purpose = Purpose; + return Result; +} + +StreamInfo StreamInfo::createModuleStream(StringRef Module, + uint32_t StreamIndex, uint32_t Modi) { + StreamInfo Result; + Result.Name = Module; + Result.StreamIndex = StreamIndex; + Result.ModuleIndex = Modi; + Result.Purpose = StreamPurpose::ModuleStream; + return Result; +} + +static inline StreamInfo otherStream(StringRef Label, uint32_t Idx) { + return StreamInfo::createStream(StreamPurpose::Other, Label, Idx); +} + +static inline StreamInfo namedStream(StringRef Label, uint32_t Idx) { + return StreamInfo::createStream(StreamPurpose::NamedStream, Label, Idx); +} + +static inline StreamInfo symbolStream(StringRef Label, uint32_t Idx) { + return StreamInfo::createStream(StreamPurpose::Symbols, Label, Idx); +} + +static inline StreamInfo moduleStream(StringRef Label, uint32_t StreamIdx, + uint32_t Modi) { + return StreamInfo::createModuleStream(Label, StreamIdx, Modi); +} + +struct IndexedModuleDescriptor { + uint32_t Modi; + DbiModuleDescriptor Descriptor; +}; + +void llvm::pdb::discoverStreamPurposes(PDBFile &File, + SmallVectorImpl<StreamInfo> &Streams) { // It's OK if we fail to load some of these streams, we still attempt to print // what we can. auto Dbi = File.getPDBDbiStream(); @@ -33,16 +81,18 @@ void llvm::pdb::discoverStreamPurposes( auto Info = File.getPDBInfoStream(); uint32_t StreamCount = File.getNumStreams(); - DenseMap<uint16_t, DbiModuleDescriptor> ModStreams; + DenseMap<uint16_t, IndexedModuleDescriptor> ModStreams; DenseMap<uint16_t, std::string> NamedStreams; if (Dbi) { const DbiModuleList &Modules = Dbi->modules(); for (uint32_t I = 0; I < Modules.getModuleCount(); ++I) { - DbiModuleDescriptor Descriptor = Modules.getModuleDescriptor(I); - uint16_t SN = Descriptor.getModuleStreamIndex(); + IndexedModuleDescriptor IMD; + IMD.Modi = I; + IMD.Descriptor = Modules.getModuleDescriptor(I); + uint16_t SN = IMD.Descriptor.getModuleStreamIndex(); if (SN != kInvalidStreamIndex) - ModStreams[SN] = Descriptor; + ModStreams[SN] = IMD; } } if (Info) { @@ -52,77 +102,76 @@ void llvm::pdb::discoverStreamPurposes( } } - Purposes.resize(StreamCount); + Streams.resize(StreamCount); for (uint16_t StreamIdx = 0; StreamIdx < StreamCount; ++StreamIdx) { - std::pair<StreamPurpose, std::string> Value; if (StreamIdx == OldMSFDirectory) - Value = std::make_pair(StreamPurpose::Other, "Old MSF Directory"); + Streams[StreamIdx] = otherStream("Old MSF Directory", StreamIdx); else if (StreamIdx == StreamPDB) - Value = std::make_pair(StreamPurpose::Other, "PDB Stream"); + Streams[StreamIdx] = otherStream("PDB Stream", StreamIdx); else if (StreamIdx == StreamDBI) - Value = std::make_pair(StreamPurpose::Other, "DBI Stream"); + Streams[StreamIdx] = otherStream("DBI Stream", StreamIdx); else if (StreamIdx == StreamTPI) - Value = std::make_pair(StreamPurpose::Other, "TPI Stream"); + Streams[StreamIdx] = otherStream("TPI Stream", StreamIdx); else if (StreamIdx == StreamIPI) - Value = std::make_pair(StreamPurpose::Other, "IPI Stream"); + Streams[StreamIdx] = otherStream("IPI Stream", StreamIdx); else if (Dbi && StreamIdx == Dbi->getGlobalSymbolStreamIndex()) - Value = std::make_pair(StreamPurpose::Other, "Global Symbol Hash"); + Streams[StreamIdx] = otherStream("Global Symbol Hash", StreamIdx); else if (Dbi && StreamIdx == Dbi->getPublicSymbolStreamIndex()) - Value = std::make_pair(StreamPurpose::Other, "Public Symbol Hash"); + Streams[StreamIdx] = otherStream("Public Symbol Hash", StreamIdx); else if (Dbi && StreamIdx == Dbi->getSymRecordStreamIndex()) - Value = std::make_pair(StreamPurpose::Other, "Public Symbol Records"); + Streams[StreamIdx] = symbolStream("Symbol Records", StreamIdx); else if (Tpi && StreamIdx == Tpi->getTypeHashStreamIndex()) - Value = std::make_pair(StreamPurpose::Other, "TPI Hash"); + Streams[StreamIdx] = otherStream("TPI Hash", StreamIdx); else if (Tpi && StreamIdx == Tpi->getTypeHashStreamAuxIndex()) - Value = std::make_pair(StreamPurpose::Other, "TPI Aux Hash"); + Streams[StreamIdx] = otherStream("TPI Aux Hash", StreamIdx); else if (Ipi && StreamIdx == Ipi->getTypeHashStreamIndex()) - Value = std::make_pair(StreamPurpose::Other, "IPI Hash"); + Streams[StreamIdx] = otherStream("IPI Hash", StreamIdx); else if (Ipi && StreamIdx == Ipi->getTypeHashStreamAuxIndex()) - Value = std::make_pair(StreamPurpose::Other, "IPI Aux Hash"); + Streams[StreamIdx] = otherStream("IPI Aux Hash", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::Exception)) - Value = std::make_pair(StreamPurpose::Other, "Exception Data"); + Streams[StreamIdx] = otherStream("Exception Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::Fixup)) - Value = std::make_pair(StreamPurpose::Other, "Fixup Data"); + Streams[StreamIdx] = otherStream("Fixup Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::FPO)) - Value = std::make_pair(StreamPurpose::Other, "FPO Data"); + Streams[StreamIdx] = otherStream("FPO Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::NewFPO)) - Value = std::make_pair(StreamPurpose::Other, "New FPO Data"); + Streams[StreamIdx] = otherStream("New FPO Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::OmapFromSrc)) - Value = std::make_pair(StreamPurpose::Other, "Omap From Source Data"); + Streams[StreamIdx] = otherStream("Omap From Source Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::OmapToSrc)) - Value = std::make_pair(StreamPurpose::Other, "Omap To Source Data"); + Streams[StreamIdx] = otherStream("Omap To Source Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::Pdata)) - Value = std::make_pair(StreamPurpose::Other, "Pdata"); + Streams[StreamIdx] = otherStream("Pdata", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::SectionHdr)) - Value = std::make_pair(StreamPurpose::Other, "Section Header Data"); + Streams[StreamIdx] = otherStream("Section Header Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::SectionHdrOrig)) - Value = - std::make_pair(StreamPurpose::Other, "Section Header Original Data"); + Streams[StreamIdx] = + otherStream("Section Header Original Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::TokenRidMap)) - Value = std::make_pair(StreamPurpose::Other, "Token Rid Data"); + Streams[StreamIdx] = otherStream("Token Rid Data", StreamIdx); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::Xdata)) - Value = std::make_pair(StreamPurpose::Other, "Xdata"); + Streams[StreamIdx] = otherStream("Xdata", StreamIdx); else { auto ModIter = ModStreams.find(StreamIdx); auto NSIter = NamedStreams.find(StreamIdx); if (ModIter != ModStreams.end()) { - Value = std::make_pair(StreamPurpose::ModuleStream, - ModIter->second.getModuleName()); + Streams[StreamIdx] = + moduleStream(ModIter->second.Descriptor.getModuleName(), StreamIdx, + ModIter->second.Modi); } else if (NSIter != NamedStreams.end()) { - Value = std::make_pair(StreamPurpose::NamedStream, NSIter->second); + Streams[StreamIdx] = namedStream(NSIter->second, StreamIdx); } else { - Value = std::make_pair(StreamPurpose::Other, "???"); + Streams[StreamIdx] = otherStream("???", StreamIdx); } } - Purposes[StreamIdx] = Value; } // Consume errors from missing streams. @@ -135,18 +184,3 @@ void llvm::pdb::discoverStreamPurposes( if (!Info) consumeError(Info.takeError()); } - -void llvm::pdb::discoverStreamPurposes(PDBFile &File, - SmallVectorImpl<std::string> &Purposes) { - SmallVector<std::pair<StreamPurpose, std::string>, 24> SP; - discoverStreamPurposes(File, SP); - Purposes.reserve(SP.size()); - for (const auto &P : SP) { - if (P.first == StreamPurpose::NamedStream) - Purposes.push_back(formatv("Named Stream \"{0}\"", P.second)); - else if (P.first == StreamPurpose::ModuleStream) - Purposes.push_back(formatv("Module \"{0}\"", P.second)); - else - Purposes.push_back(P.second); - } -} diff --git a/tools/llvm-pdbutil/StreamUtil.h b/tools/llvm-pdbutil/StreamUtil.h index f49c0a0eceb6..443267ca3290 100644 --- a/tools/llvm-pdbutil/StreamUtil.h +++ b/tools/llvm-pdbutil/StreamUtil.h @@ -10,20 +10,41 @@ #ifndef LLVM_TOOLS_LLVMPDBDUMP_STREAMUTIL_H #define LLVM_TOOLS_LLVMPDBDUMP_STREAMUTIL_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" #include <string> namespace llvm { namespace pdb { class PDBFile; -enum class StreamPurpose { NamedStream, ModuleStream, Other }; +enum class StreamPurpose { NamedStream, ModuleStream, Symbols, Other }; + +struct StreamInfo { +public: + StreamInfo() {} + + uint32_t getModuleIndex() const { return *ModuleIndex; } + StreamPurpose getPurpose() const { return Purpose; } + StringRef getShortName() const { return Name; } + uint32_t getStreamIndex() const { return StreamIndex; } + std::string getLongName() const; + + static StreamInfo createStream(StreamPurpose Purpose, StringRef Name, + uint32_t StreamIndex); + static StreamInfo createModuleStream(StringRef Module, uint32_t StreamIndex, + uint32_t Modi); + +private: + StreamPurpose Purpose; + uint32_t StreamIndex; + std::string Name; + Optional<uint32_t> ModuleIndex; +}; void discoverStreamPurposes(PDBFile &File, - SmallVectorImpl<std::string> &Purposes); -void discoverStreamPurposes( - PDBFile &File, - SmallVectorImpl<std::pair<StreamPurpose, std::string>> &Purposes); + SmallVectorImpl<StreamInfo> &Streams); } } diff --git a/tools/llvm-pdbutil/YAMLOutputStyle.cpp b/tools/llvm-pdbutil/YAMLOutputStyle.cpp index ae3138efb13a..a7afbf1242c5 100644 --- a/tools/llvm-pdbutil/YAMLOutputStyle.cpp +++ b/tools/llvm-pdbutil/YAMLOutputStyle.cpp @@ -13,11 +13,8 @@ #include "llvm-pdbutil.h" #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h" -#include "llvm/DebugInfo/CodeView/DebugLinesSubsection.h" #include "llvm/DebugInfo/CodeView/DebugSubsection.h" #include "llvm/DebugInfo/CodeView/DebugUnknownSubsection.h" -#include "llvm/DebugInfo/CodeView/Line.h" #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/DbiStream.h" @@ -25,7 +22,6 @@ #include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/Native/RawError.h" #include "llvm/DebugInfo/PDB/Native/TpiStream.h" using namespace llvm; diff --git a/tools/llvm-pdbutil/fuzzer/CMakeLists.txt b/tools/llvm-pdbutil/fuzzer/CMakeLists.txt deleted file mode 100644 index 6af00476577f..000000000000 --- a/tools/llvm-pdbutil/fuzzer/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -set(LLVM_LINK_COMPONENTS - DebugInfoCodeView - DebugInfoPDB - Object - Support - ) - -add_llvm_executable(llvm-pdbutil-fuzzer - EXCLUDE_FROM_ALL - llvm-pdbutil-fuzzer.cpp - ) - -target_link_libraries(llvm-pdbutil-fuzzer - LLVMFuzzer - ) diff --git a/tools/llvm-pdbutil/fuzzer/llvm-pdbutil-fuzzer.cpp b/tools/llvm-pdbutil/fuzzer/llvm-pdbutil-fuzzer.cpp deleted file mode 100644 index 4edb53e261ff..000000000000 --- a/tools/llvm-pdbutil/fuzzer/llvm-pdbutil-fuzzer.cpp +++ /dev/null @@ -1,105 +0,0 @@ -//===-- llvm-pdbutil-fuzzer.cpp - Fuzz the llvm-pdbutil tool --------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// \brief This file implements a function that runs llvm-pdbutil -/// on a single input. This function is then linked into the Fuzzer library. -/// -//===----------------------------------------------------------------------===// -#include "llvm/ADT/STLExtras.h" -#include "llvm/DebugInfo/CodeView/BinaryByteStream.h" -#include "llvm/DebugInfo/CodeView/SymbolDumper.h" -#include "llvm/DebugInfo/CodeView/TypeDumper.h" -#include "llvm/DebugInfo/PDB/Raw/DbiStream.h" -#include "llvm/DebugInfo/PDB/Raw/IPDBStreamData.h" -#include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h" -#include "llvm/DebugInfo/PDB/Raw/ModuleDebugStream.h" -#include "llvm/DebugInfo/PDB/Raw/PDBFile.h" -#include "llvm/DebugInfo/PDB/Raw/RawSession.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/ScopedPrinter.h" - -using namespace llvm; - -namespace { -// We need a class which behaves like an immutable BinaryByteStream, but whose -// data -// is backed by an llvm::MemoryBuffer. It also needs to own the underlying -// MemoryBuffer, so this simple adapter is a good way to achieve that. -class InputByteStream : public codeview::BinaryByteStream<false> { -public: - explicit InputByteStream(std::unique_ptr<MemoryBuffer> Buffer) - : BinaryByteStream(ArrayRef<uint8_t>(Buffer->getBuffer().bytes_begin(), - Buffer->getBuffer().bytes_end())), - MemBuffer(std::move(Buffer)) {} - - std::unique_ptr<MemoryBuffer> MemBuffer; -}; -} - -extern "C" int LLVMFuzzerTestOneInput(uint8_t *data, size_t size) { - std::unique_ptr<MemoryBuffer> Buff = MemoryBuffer::getMemBuffer( - StringRef((const char *)data, size), "", false); - - ScopedPrinter P(nulls()); - codeview::CVTypeDumper TD(&P, false); - - auto InputStream = llvm::make_unique<InputByteStream>(std::move(Buff)); - std::unique_ptr<pdb::PDBFile> File(new pdb::PDBFile(std::move(InputStream))); - if (auto E = File->parseFileHeaders()) { - consumeError(std::move(E)); - return 0; - } - if (auto E = File->parseStreamData()) { - consumeError(std::move(E)); - return 0; - } - - auto DbiS = File->getPDBDbiStream(); - if (auto E = DbiS.takeError()) { - consumeError(std::move(E)); - return 0; - } - auto TpiS = File->getPDBTpiStream(); - if (auto E = TpiS.takeError()) { - consumeError(std::move(E)); - return 0; - } - auto IpiS = File->getPDBIpiStream(); - if (auto E = IpiS.takeError()) { - consumeError(std::move(E)); - return 0; - } - auto InfoS = File->getPDBInfoStream(); - if (auto E = InfoS.takeError()) { - consumeError(std::move(E)); - return 0; - } - pdb::DbiStream &DS = DbiS.get(); - - for (auto &Modi : DS.modules()) { - auto ModStreamData = pdb::MappedBlockStream::createIndexedStream( - Modi.Info.getModuleStreamIndex(), *File, File->getAllocator()); - if (!ModStreamData) { - consumeError(ModStreamData.takeError()); - return 0; - } - pdb::ModuleDebugStreamRef ModS(Modi.Info, std::move(*ModStreamData)); - if (auto E = ModS.reload()) { - consumeError(std::move(E)); - return 0; - } - codeview::CVSymbolDumper SD(P, TD, nullptr, false); - bool HadError = false; - for (auto &S : ModS.symbols(&HadError)) { - SD.dump(S); - } - } - return 0; -} diff --git a/tools/llvm-pdbutil/llvm-pdbutil.cpp b/tools/llvm-pdbutil/llvm-pdbutil.cpp index f2bd194622ed..089f7256536f 100644 --- a/tools/llvm-pdbutil/llvm-pdbutil.cpp +++ b/tools/llvm-pdbutil/llvm-pdbutil.cpp @@ -17,6 +17,7 @@ #include "BytesOutputStyle.h" #include "Diff.h" #include "DumpOutputStyle.h" +#include "InputFile.h" #include "LinePrinter.h" #include "OutputStyle.h" #include "PrettyCompilandDumper.h" @@ -31,23 +32,23 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/BinaryFormat/Magic.h" #include "llvm/Config/config.h" +#include "llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h" #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" #include "llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h" #include "llvm/DebugInfo/CodeView/DebugLinesSubsection.h" #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" +#include "llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h" #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" #include "llvm/DebugInfo/CodeView/TypeStreamMerger.h" -#include "llvm/DebugInfo/CodeView/TypeTableBuilder.h" #include "llvm/DebugInfo/MSF/MSFBuilder.h" #include "llvm/DebugInfo/PDB/GenericError.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/IPDBSession.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h" -#include "llvm/DebugInfo/PDB/Native/DbiStream.h" #include "llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h" -#include "llvm/DebugInfo/PDB/Native/InfoStream.h" #include "llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h" #include "llvm/DebugInfo/PDB/Native/NativeSession.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" @@ -70,6 +71,7 @@ #include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Format.h" +#include "llvm/Support/LineIterator.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" @@ -80,6 +82,7 @@ #include "llvm/Support/Signals.h" #include "llvm/Support/raw_ostream.h" + using namespace llvm; using namespace llvm::codeview; using namespace llvm::msf; @@ -293,6 +296,13 @@ cl::opt<bool> cl::desc("Print a column with the result status"), cl::Optional, cl::sub(DiffSubcommand)); +cl::list<std::string> + RawModiEquivalences("modi-equivalence", cl::ZeroOrMore, + cl::value_desc("left,right"), + cl::desc("Modules with the specified indices will be " + "treated as referring to the same module"), + cl::sub(DiffSubcommand)); + cl::opt<std::string> LeftRoot( "left-bin-root", cl::Optional, cl::desc("Treats the specified path as the root of the tree containing " @@ -310,6 +320,8 @@ cl::opt<std::string> Left(cl::Positional, cl::desc("<left>"), cl::sub(DiffSubcommand)); cl::opt<std::string> Right(cl::Positional, cl::desc("<right>"), cl::sub(DiffSubcommand)); + +llvm::DenseMap<uint32_t, uint32_t> Equivalences; } cl::OptionCategory FileOptions("Module & File Options"); @@ -342,6 +354,8 @@ cl::list<std::string> cl::opt<bool> NameMap("name-map", cl::desc("Dump bytes of PDB Name Map"), cl::sub(BytesSubcommand), cl::cat(PdbBytes)); +cl::opt<bool> Fpm("fpm", cl::desc("Dump free page map"), + cl::sub(BytesSubcommand), cl::cat(MsfBytes)); cl::opt<bool> SectionContributions("sc", cl::desc("Dump section contributions"), cl::sub(BytesSubcommand), cl::cat(DbiBytes)); @@ -407,6 +421,15 @@ cl::opt<bool> DumpStreamBlocks( "stream-blocks", cl::desc("Add block information to the output of -streams"), cl::cat(MsfOptions), cl::sub(DumpSubcommand)); +cl::opt<bool> DumpSymbolStats( + "sym-stats", + cl::desc("Dump a detailed breakdown of symbol usage/size for each module"), + cl::cat(MsfOptions), cl::sub(DumpSubcommand)); + +cl::opt<bool> DumpUdtStats( + "udt-stats", + cl::desc("Dump a detailed breakdown of S_UDT record usage / stats"), + cl::cat(MsfOptions), cl::sub(DumpSubcommand)); // TYPE OPTIONS cl::opt<bool> DumpTypes("types", @@ -450,8 +473,15 @@ cl::opt<bool> DumpTypeDependents( cl::cat(TypeOptions), cl::sub(DumpSubcommand)); // SYMBOL OPTIONS +cl::opt<bool> DumpGlobals("globals", cl::desc("dump Globals symbol records"), + cl::cat(SymbolOptions), cl::sub(DumpSubcommand)); +cl::opt<bool> DumpGlobalExtras("global-extras", cl::desc("dump Globals hashes"), + cl::cat(SymbolOptions), cl::sub(DumpSubcommand)); cl::opt<bool> DumpPublics("publics", cl::desc("dump Publics stream data"), cl::cat(SymbolOptions), cl::sub(DumpSubcommand)); +cl::opt<bool> DumpPublicExtras("public-extras", + cl::desc("dump Publics hashes and address maps"), + cl::cat(SymbolOptions), cl::sub(DumpSubcommand)); cl::opt<bool> DumpSymbols("symbols", cl::desc("dump module symbols"), cl::cat(SymbolOptions), cl::sub(DumpSubcommand)); @@ -485,6 +515,14 @@ cl::opt<bool> DumpXme( cl::desc( "dump cross module exports (DEBUG_S_CROSSSCOPEEXPORTS subsection)"), cl::cat(FileOptions), cl::sub(DumpSubcommand)); +cl::opt<uint32_t> DumpModi("modi", cl::Optional, + cl::desc("For all options that iterate over " + "modules, limit to the specified module"), + cl::cat(FileOptions), cl::sub(DumpSubcommand)); +cl::opt<bool> JustMyCode("jmc", cl::Optional, + cl::desc("For all options that iterate over modules, " + "ignore modules from system libraries"), + cl::cat(FileOptions), cl::sub(DumpSubcommand)); // MISCELLANEOUS OPTIONS cl::opt<bool> DumpStringTable("string-table", cl::desc("dump PDB String Table"), @@ -496,6 +534,9 @@ cl::opt<bool> DumpSectionContribs("section-contribs", cl::sub(DumpSubcommand)); cl::opt<bool> DumpSectionMap("section-map", cl::desc("dump section map"), cl::cat(MiscOptions), cl::sub(DumpSubcommand)); +cl::opt<bool> DumpSectionHeaders("section-headers", + cl::desc("Dump image section headers"), + cl::cat(MiscOptions), cl::sub(DumpSubcommand)); cl::opt<bool> RawAll("all", cl::desc("Implies most other options."), cl::cat(MiscOptions), cl::sub(DumpSubcommand)); @@ -660,7 +701,7 @@ static void yamlToPdb(StringRef Path) { ModiBuilder.setObjFileName(MI.Obj); for (auto S : MI.SourceFiles) - ExitOnErr(DbiBuilder.addModuleSourceFile(MI.Mod, S)); + ExitOnErr(DbiBuilder.addModuleSourceFile(ModiBuilder, S)); if (MI.Modi.hasValue()) { const auto &ModiStream = *MI.Modi; for (auto Symbol : ModiStream.Symbols) { @@ -683,8 +724,9 @@ static void yamlToPdb(StringRef Path) { auto &TpiBuilder = Builder.getTpiBuilder(); const auto &Tpi = YamlObj.TpiStream.getValueOr(DefaultTpiStream); TpiBuilder.setVersionHeader(Tpi.Version); + AppendingTypeTableBuilder TS(Allocator); for (const auto &R : Tpi.Records) { - CVType Type = R.toCodeViewRecord(Allocator); + CVType Type = R.toCodeViewRecord(TS); TpiBuilder.addTypeRecord(Type.RecordData, None); } @@ -692,7 +734,7 @@ static void yamlToPdb(StringRef Path) { auto &IpiBuilder = Builder.getIpiBuilder(); IpiBuilder.setVersionHeader(Ipi.Version); for (const auto &R : Ipi.Records) { - CVType Type = R.toCodeViewRecord(Allocator); + CVType Type = R.toCodeViewRecord(TS); IpiBuilder.addTypeRecord(Type.RecordData, None); } @@ -719,11 +761,10 @@ static void pdb2Yaml(StringRef Path) { } static void dumpRaw(StringRef Path) { - std::unique_ptr<IPDBSession> Session; - auto &File = loadPDB(Path, Session); - auto O = llvm::make_unique<DumpOutputStyle>(File); + InputFile IF = ExitOnErr(InputFile::open(Path)); + auto O = llvm::make_unique<DumpOutputStyle>(IF); ExitOnErr(O->dump()); } @@ -945,8 +986,8 @@ static void dumpPretty(StringRef Path) { static void mergePdbs() { BumpPtrAllocator Allocator; - TypeTableBuilder MergedTpi(Allocator); - TypeTableBuilder MergedIpi(Allocator); + MergingTypeTableBuilder MergedTpi(Allocator); + MergingTypeTableBuilder MergedIpi(Allocator); // Create a Tpi and Ipi type table with all types from all input files. for (const auto &Path : opts::merge::InputFilenames) { @@ -976,11 +1017,11 @@ static void mergePdbs() { auto &DestTpi = Builder.getTpiBuilder(); auto &DestIpi = Builder.getIpiBuilder(); - MergedTpi.ForEachRecord([&DestTpi](TypeIndex TI, ArrayRef<uint8_t> Data) { - DestTpi.addTypeRecord(Data, None); + MergedTpi.ForEachRecord([&DestTpi](TypeIndex TI, const CVType &Type) { + DestTpi.addTypeRecord(Type.RecordData, None); }); - MergedIpi.ForEachRecord([&DestIpi](TypeIndex TI, ArrayRef<uint8_t> Data) { - DestIpi.addTypeRecord(Data, None); + MergedIpi.ForEachRecord([&DestIpi](TypeIndex TI, const CVType &Type) { + DestIpi.addTypeRecord(Type.RecordData, None); }); Builder.getInfoBuilder().addFeature(PdbRaw_FeatureSig::VC140); @@ -1058,25 +1099,28 @@ int main(int argc_, const char *argv_[]) { if (opts::DumpSubcommand) { if (opts::dump::RawAll) { - opts::dump::DumpLines = true; + opts::dump::DumpGlobals = true; opts::dump::DumpInlineeLines = true; - opts::dump::DumpXme = true; - opts::dump::DumpXmi = true; opts::dump::DumpIds = true; + opts::dump::DumpIdExtras = true; + opts::dump::DumpLines = true; + opts::dump::DumpModules = true; + opts::dump::DumpModuleFiles = true; opts::dump::DumpPublics = true; opts::dump::DumpSectionContribs = true; + opts::dump::DumpSectionHeaders = true; opts::dump::DumpSectionMap = true; opts::dump::DumpStreams = true; opts::dump::DumpStreamBlocks = true; opts::dump::DumpStringTable = true; opts::dump::DumpSummary = true; opts::dump::DumpSymbols = true; - opts::dump::DumpIds = true; - opts::dump::DumpIdExtras = true; + opts::dump::DumpSymbolStats = true; opts::dump::DumpTypes = true; opts::dump::DumpTypeExtras = true; - opts::dump::DumpModules = true; - opts::dump::DumpModuleFiles = true; + opts::dump::DumpUdtStats = true; + opts::dump::DumpXme = true; + opts::dump::DumpXmi = true; } } if (opts::PdbToYamlSubcommand) { @@ -1156,15 +1200,25 @@ int main(int argc_, const char *argv_[]) { opts::pretty::ExcludeCompilands.push_back( "d:\\\\th.obj.x86fre\\\\minkernel"); } - std::for_each(opts::pretty::InputFilenames.begin(), - opts::pretty::InputFilenames.end(), dumpPretty); + llvm::for_each(opts::pretty::InputFilenames, dumpPretty); } else if (opts::DumpSubcommand) { - std::for_each(opts::dump::InputFilenames.begin(), - opts::dump::InputFilenames.end(), dumpRaw); + llvm::for_each(opts::dump::InputFilenames, dumpRaw); } else if (opts::BytesSubcommand) { - std::for_each(opts::bytes::InputFilenames.begin(), - opts::bytes::InputFilenames.end(), dumpBytes); + llvm::for_each(opts::bytes::InputFilenames, dumpBytes); } else if (opts::DiffSubcommand) { + for (StringRef S : opts::diff::RawModiEquivalences) { + StringRef Left; + StringRef Right; + std::tie(Left, Right) = S.split(','); + uint32_t X, Y; + if (!to_integer(Left, X) || !to_integer(Right, Y)) { + errs() << formatv("invalid value {0} specified for modi equivalence\n", + S); + exit(1); + } + opts::diff::Equivalences[X] = Y; + } + diff(opts::diff::Left, opts::diff::Right); } else if (opts::MergeSubcommand) { if (opts::merge::InputFilenames.size() < 2) { diff --git a/tools/llvm-pdbutil/llvm-pdbutil.h b/tools/llvm-pdbutil/llvm-pdbutil.h index 4e92e639a127..3ce03d5880af 100644 --- a/tools/llvm-pdbutil/llvm-pdbutil.h +++ b/tools/llvm-pdbutil/llvm-pdbutil.h @@ -10,7 +10,9 @@ #ifndef LLVM_TOOLS_LLVMPDBDUMP_LLVMPDBDUMP_H #define LLVM_TOOLS_LLVMPDBDUMP_LLVMPDBDUMP_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/PointerUnion.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/raw_ostream.h" @@ -18,11 +20,17 @@ #include <stdint.h> namespace llvm { +namespace object { +class COFFObjectFile; +} namespace pdb { class PDBSymbolData; class PDBSymbolFunc; +class PDBFile; uint32_t getTypeLength(const PDBSymbolData &Symbol); } +typedef llvm::PointerUnion<object::COFFObjectFile *, pdb::PDBFile *> + PdbOrCoffObj; } namespace opts { @@ -102,6 +110,7 @@ extern llvm::Optional<NumberRange> DumpBlockRange; extern llvm::Optional<NumberRange> DumpByteRange; extern llvm::cl::list<std::string> DumpStreamData; extern llvm::cl::opt<bool> NameMap; +extern llvm::cl::opt<bool> Fpm; extern llvm::cl::opt<bool> SectionContributions; extern llvm::cl::opt<bool> SectionMap; @@ -123,7 +132,10 @@ extern llvm::cl::opt<bool> SplitChunks; namespace dump { extern llvm::cl::opt<bool> DumpSummary; +extern llvm::cl::opt<bool> DumpFpm; extern llvm::cl::opt<bool> DumpStreams; +extern llvm::cl::opt<bool> DumpSymbolStats; +extern llvm::cl::opt<bool> DumpUdtStats; extern llvm::cl::opt<bool> DumpStreamBlocks; extern llvm::cl::opt<bool> DumpLines; @@ -136,14 +148,20 @@ extern llvm::cl::opt<bool> DumpTypeData; extern llvm::cl::opt<bool> DumpTypeExtras; extern llvm::cl::list<uint32_t> DumpTypeIndex; extern llvm::cl::opt<bool> DumpTypeDependents; +extern llvm::cl::opt<bool> DumpSectionHeaders; extern llvm::cl::opt<bool> DumpIds; extern llvm::cl::opt<bool> DumpIdData; extern llvm::cl::opt<bool> DumpIdExtras; extern llvm::cl::list<uint32_t> DumpIdIndex; +extern llvm::cl::opt<uint32_t> DumpModi; +extern llvm::cl::opt<bool> JustMyCode; extern llvm::cl::opt<bool> DumpSymbols; extern llvm::cl::opt<bool> DumpSymRecordBytes; +extern llvm::cl::opt<bool> DumpGlobals; +extern llvm::cl::opt<bool> DumpGlobalExtras; extern llvm::cl::opt<bool> DumpPublics; +extern llvm::cl::opt<bool> DumpPublicExtras; extern llvm::cl::opt<bool> DumpSectionContribs; extern llvm::cl::opt<bool> DumpSectionMap; extern llvm::cl::opt<bool> DumpModules; @@ -172,6 +190,7 @@ extern llvm::cl::opt<bool> DumpModuleSyms; namespace diff { extern llvm::cl::opt<bool> PrintValueColumns; extern llvm::cl::opt<bool> PrintResultColumn; +extern llvm::DenseMap<uint32_t, uint32_t> Equivalences; extern llvm::cl::opt<std::string> LeftRoot; extern llvm::cl::opt<std::string> RightRoot; } // namespace diff diff --git a/tools/llvm-profdata/llvm-profdata.cpp b/tools/llvm-profdata/llvm-profdata.cpp index eee242107dab..9afd0ae92eae 100644 --- a/tools/llvm-profdata/llvm-profdata.cpp +++ b/tools/llvm-profdata/llvm-profdata.cpp @@ -37,14 +37,19 @@ using namespace llvm; enum ProfileFormat { PF_None = 0, PF_Text, PF_Binary, PF_GCC }; -static void exitWithError(const Twine &Message, StringRef Whence = "", - StringRef Hint = "") { - errs() << "error: "; +static void warn(StringRef Prefix, Twine Message, std::string Whence = "", + std::string Hint = "") { + errs() << Prefix; if (!Whence.empty()) errs() << Whence << ": "; errs() << Message << "\n"; if (!Hint.empty()) errs() << Hint << "\n"; +} + +static void exitWithError(Twine Message, std::string Whence = "", + std::string Hint = "") { + warn("error: ", Message, Whence, Hint); ::exit(1); } @@ -119,7 +124,7 @@ struct WriterContext { std::mutex Lock; InstrProfWriter Writer; Error Err; - StringRef ErrWhence; + std::string ErrWhence; std::mutex &ErrLock; SmallSet<instrprof_error, 4> &WriterErrorCodes; @@ -129,6 +134,22 @@ struct WriterContext { ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) {} }; +/// Determine whether an error is fatal for profile merging. +static bool isFatalError(instrprof_error IPE) { + switch (IPE) { + default: + return true; + case instrprof_error::success: + case instrprof_error::eof: + case instrprof_error::unknown_function: + case instrprof_error::hash_mismatch: + case instrprof_error::count_mismatch: + case instrprof_error::counter_overflow: + case instrprof_error::value_site_count_mismatch: + return false; + } +} + /// Load an input into a writer context. static void loadInput(const WeightedFile &Input, WriterContext *WC) { std::unique_lock<std::mutex> CtxGuard{WC->Lock}; @@ -137,6 +158,9 @@ static void loadInput(const WeightedFile &Input, WriterContext *WC) { if (WC->Err) return; + // Copy the filename, because llvm::ThreadPool copied the input "const + // WeightedFile &" by value, making a reference to the filename within it + // invalid outside of this packaged task. WC->ErrWhence = Input.Filename; auto ReaderOrErr = InstrProfReader::create(Input.Filename); @@ -174,12 +198,22 @@ static void loadInput(const WeightedFile &Input, WriterContext *WC) { FuncName, firstTime); }); } - if (Reader->hasError()) - WC->Err = Reader->getError(); + if (Reader->hasError()) { + if (Error E = Reader->getError()) { + instrprof_error IPE = InstrProfError::take(std::move(E)); + if (isFatalError(IPE)) + WC->Err = make_error<InstrProfError>(IPE); + } + } } /// Merge the \p Src writer context into \p Dst. static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) { + // If we've already seen a hard error, continuing with the merge would + // clobber it. + if (Dst->Err || Src->Err) + return; + bool Reported = false; Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer), [&](Error E) { if (Reported) { @@ -211,8 +245,8 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs, // If NumThreads is not specified, auto-detect a good default. if (NumThreads == 0) - NumThreads = std::max(1U, std::min(std::thread::hardware_concurrency(), - unsigned(Inputs.size() / 2))); + NumThreads = + std::min(hardware_concurrency(), unsigned((Inputs.size() + 1) / 2)); // Initialize the writer contexts. SmallVector<std::unique_ptr<WriterContext>, 4> Contexts; @@ -254,10 +288,20 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs, } // Handle deferred hard errors encountered during merging. - for (std::unique_ptr<WriterContext> &WC : Contexts) - if (WC->Err) + for (std::unique_ptr<WriterContext> &WC : Contexts) { + if (!WC->Err) + continue; + if (!WC->Err.isA<InstrProfError>()) exitWithError(std::move(WC->Err), WC->ErrWhence); + instrprof_error IPE = InstrProfError::take(std::move(WC->Err)); + if (isFatalError(IPE)) + exitWithError(make_error<InstrProfError>(IPE), WC->ErrWhence); + else + warn("warning: ", toString(make_error<InstrProfError>(IPE)), + WC->ErrWhence); + } + InstrProfWriter &Writer = Contexts[0]->Writer; if (OutputFormat == PF_Text) { if (Error E = Writer.writeText(Output)) @@ -625,6 +669,8 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts, if (ShowCounts && TextFormat) return 0; std::unique_ptr<ProfileSummary> PS(Builder.getSummary()); + OS << "Instrumentation level: " + << (Reader->isIRLevelProfile() ? "IR" : "Front-end") << "\n"; if (ShowAllFunctions || !ShowFunction.empty()) OS << "Functions shown: " << ShownFunctions << "\n"; OS << "Total functions: " << PS->getNumFunctions() << "\n"; diff --git a/tools/llvm-rc/CMakeLists.txt b/tools/llvm-rc/CMakeLists.txt new file mode 100644 index 000000000000..e5c0eb25d7bc --- /dev/null +++ b/tools/llvm-rc/CMakeLists.txt @@ -0,0 +1,17 @@ +set(LLVM_LINK_COMPONENTS + Option + Support + ) + +set(LLVM_TARGET_DEFINITIONS Opts.td) + +tablegen(LLVM Opts.inc -gen-opt-parser-defs) +add_public_tablegen_target(RcTableGen) + +add_llvm_tool(llvm-rc + llvm-rc.cpp + ResourceFileWriter.cpp + ResourceScriptParser.cpp + ResourceScriptStmt.cpp + ResourceScriptToken.cpp + ) diff --git a/tools/llvm-rc/LLVMBuild.txt b/tools/llvm-rc/LLVMBuild.txt new file mode 100644 index 000000000000..fec4d3369883 --- /dev/null +++ b/tools/llvm-rc/LLVMBuild.txt @@ -0,0 +1,22 @@ +;===- ./tools/llvm-rc/LLVMBuild.txt ----------------------------*- Conf -*--===; +; +; The LLVM Compiler Infrastructure +; +; This file is distributed under the University of Illinois Open Source +; License. See LICENSE.TXT for details. +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Tool +name = llvm-rc +parent = Tools +required_libraries = Option diff --git a/tools/llvm-rc/Opts.td b/tools/llvm-rc/Opts.td new file mode 100644 index 000000000000..9792aa582cbb --- /dev/null +++ b/tools/llvm-rc/Opts.td @@ -0,0 +1,56 @@ +include "llvm/Option/OptParser.td" + +// All the switches can be preceded by either '/' or '-'. +// These options seem to be important for the tool +// and should be implemented. + +def FILEOUT : Separate<[ "/", "-" ], "FO">, + HelpText<"Change the output file location.">; + +def DEFINE : Separate<[ "/", "-" ], "D">, + HelpText<"Define a symbol for the C preprocessor.">; +def UNDEF : Separate<[ "/", "-" ], "U">, + HelpText<"Undefine a symbol for the C preprocessor.">; + +def LANG_ID : Separate<[ "/", "-" ], "L">, + HelpText<"Set the default language identifier.">; +def LANG_NAME : Separate<[ "/", "-" ], "LN">, + HelpText<"Set the default language name.">; + +def INCLUDE : Separate<[ "/", "-" ], "I">, HelpText<"Add an include path.">; +def NOINCLUDE : Flag<[ "/", "-" ], "X">, HelpText<"Ignore 'include' variable.">; + +def ADD_NULL : Flag<[ "/", "-" ], "N">, + HelpText<"Null-terminate all strings in the string table.">; + +def DUPID_NOWARN : Flag<[ "/", "-" ], "Y">, + HelpText<"Suppress warnings on duplicate resource IDs.">; + +def VERBOSE : Flag<[ "/", "-" ], "V">, HelpText<"Be verbose.">; +def HELP : Flag<[ "/", "-" ], "?">, HelpText<"Display this help and exit.">; +def H : Flag<[ "/", "-" ], "H">, + Alias<HELP>, + HelpText<"Display this help and exit.">; + +def DRY_RUN : Flag<[ "/", "-" ], "dry-run">, + HelpText<"Don't compile the input; only try to parse it.">; + +// Unused switches (at least for now). These will stay unimplemented +// in an early stage of development and can be ignored. However, we need to +// parse them in order to preserve the compatibility with the original tool. + +def NOLOGO : Flag<[ "/", "-" ], "NOLOGO">; +def R : Flag<[ "/", "-" ], "R">; +def SL : Flag<[ "/", "-" ], "SL">; + +// (Codepages support.) +def C : Flag<[ "/", "-" ], "C">; +def W : Flag<[ "/", "-" ], "W">; + +// (Support of MUI and similar.) +def FM : Separate<[ "/", "-" ], "FM">; +def Q : Separate<[ "/", "-" ], "Q">; +def G : Flag<[ "/", "-" ], "G">; +def GN : Flag<[ "/", "-" ], "GN">; +def G1 : Flag<[ "/", "-" ], "G1">; +def G2 : Flag<[ "/", "-" ], "G2">; diff --git a/tools/llvm-rc/ResourceFileWriter.cpp b/tools/llvm-rc/ResourceFileWriter.cpp new file mode 100644 index 000000000000..f141dc7e3564 --- /dev/null +++ b/tools/llvm-rc/ResourceFileWriter.cpp @@ -0,0 +1,1449 @@ +//===-- ResourceFileWriter.cpp --------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This implements the visitor serializing resources to a .res stream. +// +//===---------------------------------------------------------------------===// + +#include "ResourceFileWriter.h" + +#include "llvm/Object/WindowsResource.h" +#include "llvm/Support/ConvertUTF.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/EndianStream.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" + +using namespace llvm::support; + +// Take an expression returning llvm::Error and forward the error if it exists. +#define RETURN_IF_ERROR(Expr) \ + if (auto Err = (Expr)) \ + return Err; + +namespace llvm { +namespace rc { + +// Class that employs RAII to save the current FileWriter object state +// and revert to it as soon as we leave the scope. This is useful if resources +// declare their own resource-local statements. +class ContextKeeper { + ResourceFileWriter *FileWriter; + ResourceFileWriter::ObjectInfo SavedInfo; + +public: + ContextKeeper(ResourceFileWriter *V) + : FileWriter(V), SavedInfo(V->ObjectData) {} + ~ContextKeeper() { FileWriter->ObjectData = SavedInfo; } +}; + +static Error createError(const Twine &Message, + std::errc Type = std::errc::invalid_argument) { + return make_error<StringError>(Message, std::make_error_code(Type)); +} + +static Error checkNumberFits(uint32_t Number, size_t MaxBits, + const Twine &FieldName) { + assert(1 <= MaxBits && MaxBits <= 32); + if (!(Number >> MaxBits)) + return Error::success(); + return createError(FieldName + " (" + Twine(Number) + ") does not fit in " + + Twine(MaxBits) + " bits.", + std::errc::value_too_large); +} + +template <typename FitType> +static Error checkNumberFits(uint32_t Number, const Twine &FieldName) { + return checkNumberFits(Number, sizeof(FitType) * 8, FieldName); +} + +// A similar function for signed integers. +template <typename FitType> +static Error checkSignedNumberFits(uint32_t Number, const Twine &FieldName, + bool CanBeNegative) { + int32_t SignedNum = Number; + if (SignedNum < std::numeric_limits<FitType>::min() || + SignedNum > std::numeric_limits<FitType>::max()) + return createError(FieldName + " (" + Twine(SignedNum) + + ") does not fit in " + Twine(sizeof(FitType) * 8) + + "-bit signed integer type.", + std::errc::value_too_large); + + if (!CanBeNegative && SignedNum < 0) + return createError(FieldName + " (" + Twine(SignedNum) + + ") cannot be negative."); + + return Error::success(); +} + +static Error checkRCInt(RCInt Number, const Twine &FieldName) { + if (Number.isLong()) + return Error::success(); + return checkNumberFits<uint16_t>(Number, FieldName); +} + +static Error checkIntOrString(IntOrString Value, const Twine &FieldName) { + if (!Value.isInt()) + return Error::success(); + return checkNumberFits<uint16_t>(Value.getInt(), FieldName); +} + +static bool stripQuotes(StringRef &Str, bool &IsLongString) { + if (!Str.contains('"')) + return false; + + // Just take the contents of the string, checking if it's been marked long. + IsLongString = Str.startswith_lower("L"); + if (IsLongString) + Str = Str.drop_front(); + + bool StripSuccess = Str.consume_front("\"") && Str.consume_back("\""); + (void)StripSuccess; + assert(StripSuccess && "Strings should be enclosed in quotes."); + return true; +} + +// Describes a way to handle '\0' characters when processing the string. +// rc.exe tool sometimes behaves in a weird way in postprocessing. +// If the string to be output is equivalent to a C-string (e.g. in MENU +// titles), string is (predictably) truncated after first 0-byte. +// When outputting a string table, the behavior is equivalent to appending +// '\0\0' at the end of the string, and then stripping the string +// before the first '\0\0' occurrence. +// Finally, when handling strings in user-defined resources, 0-bytes +// aren't stripped, nor do they terminate the string. + +enum class NullHandlingMethod { + UserResource, // Don't terminate string on '\0'. + CutAtNull, // Terminate string on '\0'. + CutAtDoubleNull // Terminate string on '\0\0'; strip final '\0'. +}; + +// Parses an identifier or string and returns a processed version of it: +// * String the string boundary quotes. +// * Squash "" to a single ". +// * Replace the escape sequences with their processed version. +// For identifiers, this is no-op. +static Error processString(StringRef Str, NullHandlingMethod NullHandler, + bool &IsLongString, SmallVectorImpl<UTF16> &Result) { + bool IsString = stripQuotes(Str, IsLongString); + SmallVector<UTF16, 128> Chars; + convertUTF8ToUTF16String(Str, Chars); + + if (!IsString) { + // It's an identifier if it's not a string. Make all characters uppercase. + for (UTF16 &Ch : Chars) { + assert(Ch <= 0x7F && "We didn't allow identifiers to be non-ASCII"); + Ch = toupper(Ch); + } + Result.swap(Chars); + return Error::success(); + } + Result.reserve(Chars.size()); + size_t Pos = 0; + + auto AddRes = [&Result, NullHandler, IsLongString](UTF16 Char) -> Error { + if (!IsLongString) { + if (NullHandler == NullHandlingMethod::UserResource) { + // Narrow strings in user-defined resources are *not* output in + // UTF-16 format. + if (Char > 0xFF) + return createError("Non-8-bit codepoint (" + Twine(Char) + + ") can't occur in a user-defined narrow string"); + + } else { + // In case of narrow non-user strings, Windows RC converts + // [0x80, 0xFF] chars according to the current codepage. + // There is no 'codepage' concept settled in every supported platform, + // so we should reject such inputs. + if (Char > 0x7F && Char <= 0xFF) + return createError("Non-ASCII 8-bit codepoint (" + Twine(Char) + + ") can't " + "occur in a non-Unicode string"); + } + } + + Result.push_back(Char); + return Error::success(); + }; + + while (Pos < Chars.size()) { + UTF16 CurChar = Chars[Pos]; + ++Pos; + + // Strip double "". + if (CurChar == '"') { + if (Pos == Chars.size() || Chars[Pos] != '"') + return createError("Expected \"\""); + ++Pos; + RETURN_IF_ERROR(AddRes('"')); + continue; + } + + if (CurChar == '\\') { + UTF16 TypeChar = Chars[Pos]; + ++Pos; + + if (TypeChar == 'x' || TypeChar == 'X') { + // Read a hex number. Max number of characters to read differs between + // narrow and wide strings. + UTF16 ReadInt = 0; + size_t RemainingChars = IsLongString ? 4 : 2; + // We don't want to read non-ASCII hex digits. std:: functions past + // 0xFF invoke UB. + // + // FIXME: actually, Microsoft version probably doesn't check this + // condition and uses their Unicode version of 'isxdigit'. However, + // there are some hex-digit Unicode character outside of ASCII, and + // some of these are actually accepted by rc.exe, the notable example + // being fullwidth forms (U+FF10..U+FF19 etc.) These can be written + // instead of ASCII digits in \x... escape sequence and get accepted. + // However, the resulting hexcodes seem totally unpredictable. + // We think it's infeasible to try to reproduce this behavior, nor to + // put effort in order to detect it. + while (RemainingChars && Pos < Chars.size() && Chars[Pos] < 0x80) { + if (!isxdigit(Chars[Pos])) + break; + char Digit = tolower(Chars[Pos]); + ++Pos; + + ReadInt <<= 4; + if (isdigit(Digit)) + ReadInt |= Digit - '0'; + else + ReadInt |= Digit - 'a' + 10; + + --RemainingChars; + } + + RETURN_IF_ERROR(AddRes(ReadInt)); + continue; + } + + if (TypeChar >= '0' && TypeChar < '8') { + // Read an octal number. Note that we've already read the first digit. + UTF16 ReadInt = TypeChar - '0'; + size_t RemainingChars = IsLongString ? 6 : 2; + + while (RemainingChars && Pos < Chars.size() && Chars[Pos] >= '0' && + Chars[Pos] < '8') { + ReadInt <<= 3; + ReadInt |= Chars[Pos] - '0'; + --RemainingChars; + ++Pos; + } + + RETURN_IF_ERROR(AddRes(ReadInt)); + + continue; + } + + switch (TypeChar) { + case 'A': + case 'a': + // Windows '\a' translates into '\b' (Backspace). + RETURN_IF_ERROR(AddRes('\b')); + break; + + case 'n': // Somehow, RC doesn't recognize '\N' and '\R'. + RETURN_IF_ERROR(AddRes('\n')); + break; + + case 'r': + RETURN_IF_ERROR(AddRes('\r')); + break; + + case 'T': + case 't': + RETURN_IF_ERROR(AddRes('\t')); + break; + + case '\\': + RETURN_IF_ERROR(AddRes('\\')); + break; + + case '"': + // RC accepts \" only if another " comes afterwards; then, \"" means + // a single ". + if (Pos == Chars.size() || Chars[Pos] != '"') + return createError("Expected \\\"\""); + ++Pos; + RETURN_IF_ERROR(AddRes('"')); + break; + + default: + // If TypeChar means nothing, \ is should be output to stdout with + // following char. However, rc.exe consumes these characters when + // dealing with wide strings. + if (!IsLongString) { + RETURN_IF_ERROR(AddRes('\\')); + RETURN_IF_ERROR(AddRes(TypeChar)); + } + break; + } + + continue; + } + + // If nothing interesting happens, just output the character. + RETURN_IF_ERROR(AddRes(CurChar)); + } + + switch (NullHandler) { + case NullHandlingMethod::CutAtNull: + for (size_t Pos = 0; Pos < Result.size(); ++Pos) + if (Result[Pos] == '\0') + Result.resize(Pos); + break; + + case NullHandlingMethod::CutAtDoubleNull: + for (size_t Pos = 0; Pos + 1 < Result.size(); ++Pos) + if (Result[Pos] == '\0' && Result[Pos + 1] == '\0') + Result.resize(Pos); + if (Result.size() > 0 && Result.back() == '\0') + Result.pop_back(); + break; + + case NullHandlingMethod::UserResource: + break; + } + + return Error::success(); +} + +uint64_t ResourceFileWriter::writeObject(const ArrayRef<uint8_t> Data) { + uint64_t Result = tell(); + FS->write((const char *)Data.begin(), Data.size()); + return Result; +} + +Error ResourceFileWriter::writeCString(StringRef Str, bool WriteTerminator) { + SmallVector<UTF16, 128> ProcessedString; + bool IsLongString; + RETURN_IF_ERROR(processString(Str, NullHandlingMethod::CutAtNull, + IsLongString, ProcessedString)); + for (auto Ch : ProcessedString) + writeInt<uint16_t>(Ch); + if (WriteTerminator) + writeInt<uint16_t>(0); + return Error::success(); +} + +Error ResourceFileWriter::writeIdentifier(const IntOrString &Ident) { + return writeIntOrString(Ident); +} + +Error ResourceFileWriter::writeIntOrString(const IntOrString &Value) { + if (!Value.isInt()) + return writeCString(Value.getString()); + + writeInt<uint16_t>(0xFFFF); + writeInt<uint16_t>(Value.getInt()); + return Error::success(); +} + +void ResourceFileWriter::writeRCInt(RCInt Value) { + if (Value.isLong()) + writeInt<uint32_t>(Value); + else + writeInt<uint16_t>(Value); +} + +Error ResourceFileWriter::appendFile(StringRef Filename) { + bool IsLong; + stripQuotes(Filename, IsLong); + + auto File = loadFile(Filename); + if (!File) + return File.takeError(); + + *FS << (*File)->getBuffer(); + return Error::success(); +} + +void ResourceFileWriter::padStream(uint64_t Length) { + assert(Length > 0); + uint64_t Location = tell(); + Location %= Length; + uint64_t Pad = (Length - Location) % Length; + for (uint64_t i = 0; i < Pad; ++i) + writeInt<uint8_t>(0); +} + +Error ResourceFileWriter::handleError(Error Err, const RCResource *Res) { + if (Err) + return joinErrors(createError("Error in " + Res->getResourceTypeName() + + " statement (ID " + Twine(Res->ResName) + + "): "), + std::move(Err)); + return Error::success(); +} + +Error ResourceFileWriter::visitNullResource(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeNullBody); +} + +Error ResourceFileWriter::visitAcceleratorsResource(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeAcceleratorsBody); +} + +Error ResourceFileWriter::visitCursorResource(const RCResource *Res) { + return handleError(visitIconOrCursorResource(Res), Res); +} + +Error ResourceFileWriter::visitDialogResource(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeDialogBody); +} + +Error ResourceFileWriter::visitIconResource(const RCResource *Res) { + return handleError(visitIconOrCursorResource(Res), Res); +} + +Error ResourceFileWriter::visitCaptionStmt(const CaptionStmt *Stmt) { + ObjectData.Caption = Stmt->Value; + return Error::success(); +} + +Error ResourceFileWriter::visitHTMLResource(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeHTMLBody); +} + +Error ResourceFileWriter::visitMenuResource(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeMenuBody); +} + +Error ResourceFileWriter::visitStringTableResource(const RCResource *Base) { + const auto *Res = cast<StringTableResource>(Base); + + ContextKeeper RAII(this); + RETURN_IF_ERROR(Res->applyStmts(this)); + + for (auto &String : Res->Table) { + RETURN_IF_ERROR(checkNumberFits<uint16_t>(String.first, "String ID")); + uint16_t BundleID = String.first >> 4; + StringTableInfo::BundleKey Key(BundleID, ObjectData.LanguageInfo); + auto &BundleData = StringTableData.BundleData; + auto Iter = BundleData.find(Key); + + if (Iter == BundleData.end()) { + // Need to create a bundle. + StringTableData.BundleList.push_back(Key); + auto EmplaceResult = + BundleData.emplace(Key, StringTableInfo::Bundle(ObjectData)); + assert(EmplaceResult.second && "Could not create a bundle"); + Iter = EmplaceResult.first; + } + + RETURN_IF_ERROR( + insertStringIntoBundle(Iter->second, String.first, String.second)); + } + + return Error::success(); +} + +Error ResourceFileWriter::visitUserDefinedResource(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeUserDefinedBody); +} + +Error ResourceFileWriter::visitVersionInfoResource(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeVersionInfoBody); +} + +Error ResourceFileWriter::visitCharacteristicsStmt( + const CharacteristicsStmt *Stmt) { + ObjectData.Characteristics = Stmt->Value; + return Error::success(); +} + +Error ResourceFileWriter::visitFontStmt(const FontStmt *Stmt) { + RETURN_IF_ERROR(checkNumberFits<uint16_t>(Stmt->Size, "Font size")); + RETURN_IF_ERROR(checkNumberFits<uint16_t>(Stmt->Weight, "Font weight")); + RETURN_IF_ERROR(checkNumberFits<uint8_t>(Stmt->Charset, "Font charset")); + ObjectInfo::FontInfo Font{Stmt->Size, Stmt->Name, Stmt->Weight, Stmt->Italic, + Stmt->Charset}; + ObjectData.Font.emplace(Font); + return Error::success(); +} + +Error ResourceFileWriter::visitLanguageStmt(const LanguageResource *Stmt) { + RETURN_IF_ERROR(checkNumberFits(Stmt->Lang, 10, "Primary language ID")); + RETURN_IF_ERROR(checkNumberFits(Stmt->SubLang, 6, "Sublanguage ID")); + ObjectData.LanguageInfo = Stmt->Lang | (Stmt->SubLang << 10); + return Error::success(); +} + +Error ResourceFileWriter::visitStyleStmt(const StyleStmt *Stmt) { + ObjectData.Style = Stmt->Value; + return Error::success(); +} + +Error ResourceFileWriter::visitVersionStmt(const VersionStmt *Stmt) { + ObjectData.VersionInfo = Stmt->Value; + return Error::success(); +} + +Error ResourceFileWriter::writeResource( + const RCResource *Res, + Error (ResourceFileWriter::*BodyWriter)(const RCResource *)) { + // We don't know the sizes yet. + object::WinResHeaderPrefix HeaderPrefix{ulittle32_t(0U), ulittle32_t(0U)}; + uint64_t HeaderLoc = writeObject(HeaderPrefix); + + auto ResType = Res->getResourceType(); + RETURN_IF_ERROR(checkIntOrString(ResType, "Resource type")); + RETURN_IF_ERROR(checkIntOrString(Res->ResName, "Resource ID")); + RETURN_IF_ERROR(handleError(writeIdentifier(ResType), Res)); + RETURN_IF_ERROR(handleError(writeIdentifier(Res->ResName), Res)); + + // Apply the resource-local optional statements. + ContextKeeper RAII(this); + RETURN_IF_ERROR(handleError(Res->applyStmts(this), Res)); + + padStream(sizeof(uint32_t)); + object::WinResHeaderSuffix HeaderSuffix{ + ulittle32_t(0), // DataVersion; seems to always be 0 + ulittle16_t(Res->getMemoryFlags()), ulittle16_t(ObjectData.LanguageInfo), + ulittle32_t(ObjectData.VersionInfo), + ulittle32_t(ObjectData.Characteristics)}; + writeObject(HeaderSuffix); + + uint64_t DataLoc = tell(); + RETURN_IF_ERROR(handleError((this->*BodyWriter)(Res), Res)); + // RETURN_IF_ERROR(handleError(dumpResource(Ctx))); + + // Update the sizes. + HeaderPrefix.DataSize = tell() - DataLoc; + HeaderPrefix.HeaderSize = DataLoc - HeaderLoc; + writeObjectAt(HeaderPrefix, HeaderLoc); + padStream(sizeof(uint32_t)); + + return Error::success(); +} + +// --- NullResource helpers. --- // + +Error ResourceFileWriter::writeNullBody(const RCResource *) { + return Error::success(); +} + +// --- AcceleratorsResource helpers. --- // + +Error ResourceFileWriter::writeSingleAccelerator( + const AcceleratorsResource::Accelerator &Obj, bool IsLastItem) { + using Accelerator = AcceleratorsResource::Accelerator; + using Opt = Accelerator::Options; + + struct AccelTableEntry { + ulittle16_t Flags; + ulittle16_t ANSICode; + ulittle16_t Id; + uint16_t Padding; + } Entry{ulittle16_t(0), ulittle16_t(0), ulittle16_t(0), 0}; + + bool IsASCII = Obj.Flags & Opt::ASCII, IsVirtKey = Obj.Flags & Opt::VIRTKEY; + + // Remove ASCII flags (which doesn't occur in .res files). + Entry.Flags = Obj.Flags & ~Opt::ASCII; + + if (IsLastItem) + Entry.Flags |= 0x80; + + RETURN_IF_ERROR(checkNumberFits<uint16_t>(Obj.Id, "ACCELERATORS entry ID")); + Entry.Id = ulittle16_t(Obj.Id); + + auto createAccError = [&Obj](const char *Msg) { + return createError("Accelerator ID " + Twine(Obj.Id) + ": " + Msg); + }; + + if (IsASCII && IsVirtKey) + return createAccError("Accelerator can't be both ASCII and VIRTKEY"); + + if (!IsVirtKey && (Obj.Flags & (Opt::ALT | Opt::SHIFT | Opt::CONTROL))) + return createAccError("Can only apply ALT, SHIFT or CONTROL to VIRTKEY" + " accelerators"); + + if (Obj.Event.isInt()) { + if (!IsASCII && !IsVirtKey) + return createAccError( + "Accelerator with a numeric event must be either ASCII" + " or VIRTKEY"); + + uint32_t EventVal = Obj.Event.getInt(); + RETURN_IF_ERROR( + checkNumberFits<uint16_t>(EventVal, "Numeric event key ID")); + Entry.ANSICode = ulittle16_t(EventVal); + writeObject(Entry); + return Error::success(); + } + + StringRef Str = Obj.Event.getString(); + bool IsWide; + stripQuotes(Str, IsWide); + + if (Str.size() == 0 || Str.size() > 2) + return createAccError( + "Accelerator string events should have length 1 or 2"); + + if (Str[0] == '^') { + if (Str.size() == 1) + return createAccError("No character following '^' in accelerator event"); + if (IsVirtKey) + return createAccError( + "VIRTKEY accelerator events can't be preceded by '^'"); + + char Ch = Str[1]; + if (Ch >= 'a' && Ch <= 'z') + Entry.ANSICode = ulittle16_t(Ch - 'a' + 1); + else if (Ch >= 'A' && Ch <= 'Z') + Entry.ANSICode = ulittle16_t(Ch - 'A' + 1); + else + return createAccError("Control character accelerator event should be" + " alphabetic"); + + writeObject(Entry); + return Error::success(); + } + + if (Str.size() == 2) + return createAccError("Event string should be one-character, possibly" + " preceded by '^'"); + + uint8_t EventCh = Str[0]; + // The original tool just warns in this situation. We chose to fail. + if (IsVirtKey && !isalnum(EventCh)) + return createAccError("Non-alphanumeric characters cannot describe virtual" + " keys"); + if (EventCh > 0x7F) + return createAccError("Non-ASCII description of accelerator"); + + if (IsVirtKey) + EventCh = toupper(EventCh); + Entry.ANSICode = ulittle16_t(EventCh); + writeObject(Entry); + return Error::success(); +} + +Error ResourceFileWriter::writeAcceleratorsBody(const RCResource *Base) { + auto *Res = cast<AcceleratorsResource>(Base); + size_t AcceleratorId = 0; + for (auto &Acc : Res->Accelerators) { + ++AcceleratorId; + RETURN_IF_ERROR( + writeSingleAccelerator(Acc, AcceleratorId == Res->Accelerators.size())); + } + return Error::success(); +} + +// --- CursorResource and IconResource helpers. --- // + +// ICONRESDIR structure. Describes a single icon in resouce group. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms648016.aspx +struct IconResDir { + uint8_t Width; + uint8_t Height; + uint8_t ColorCount; + uint8_t Reserved; +}; + +// CURSORDIR structure. Describes a single cursor in resource group. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms648011(v=vs.85).aspx +struct CursorDir { + ulittle16_t Width; + ulittle16_t Height; +}; + +// RESDIRENTRY structure, stripped from the last item. Stripping made +// for compatibility with RESDIR. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms648026(v=vs.85).aspx +struct ResourceDirEntryStart { + union { + CursorDir Cursor; // Used in CURSOR resources. + IconResDir Icon; // Used in .ico and .cur files, and ICON resources. + }; + ulittle16_t Planes; // HotspotX (.cur files but not CURSOR resource). + ulittle16_t BitCount; // HotspotY (.cur files but not CURSOR resource). + ulittle32_t Size; + // ulittle32_t ImageOffset; // Offset to image data (ICONDIRENTRY only). + // ulittle16_t IconID; // Resource icon ID (RESDIR only). +}; + +// BITMAPINFOHEADER structure. Describes basic information about the bitmap +// being read. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/dd183376(v=vs.85).aspx +struct BitmapInfoHeader { + ulittle32_t Size; + ulittle32_t Width; + ulittle32_t Height; + ulittle16_t Planes; + ulittle16_t BitCount; + ulittle32_t Compression; + ulittle32_t SizeImage; + ulittle32_t XPelsPerMeter; + ulittle32_t YPelsPerMeter; + ulittle32_t ClrUsed; + ulittle32_t ClrImportant; +}; + +// Group icon directory header. Called ICONDIR in .ico/.cur files and +// NEWHEADER in .res files. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms648023(v=vs.85).aspx +struct GroupIconDir { + ulittle16_t Reserved; // Always 0. + ulittle16_t ResType; // 1 for icons, 2 for cursors. + ulittle16_t ResCount; // Number of items. +}; + +enum class IconCursorGroupType { Icon, Cursor }; + +class SingleIconCursorResource : public RCResource { +public: + IconCursorGroupType Type; + const ResourceDirEntryStart &Header; + ArrayRef<uint8_t> Image; + + SingleIconCursorResource(IconCursorGroupType ResourceType, + const ResourceDirEntryStart &HeaderEntry, + ArrayRef<uint8_t> ImageData) + : Type(ResourceType), Header(HeaderEntry), Image(ImageData) {} + + Twine getResourceTypeName() const override { return "Icon/cursor image"; } + IntOrString getResourceType() const override { + return Type == IconCursorGroupType::Icon ? RkSingleIcon : RkSingleCursor; + } + uint16_t getMemoryFlags() const override { + return MfDiscardable | MfMoveable; + } + ResourceKind getKind() const override { return RkSingleCursorOrIconRes; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkSingleCursorOrIconRes; + } +}; + +class IconCursorGroupResource : public RCResource { +public: + IconCursorGroupType Type; + GroupIconDir Header; + std::vector<ResourceDirEntryStart> ItemEntries; + + IconCursorGroupResource(IconCursorGroupType ResourceType, + const GroupIconDir &HeaderData, + std::vector<ResourceDirEntryStart> &&Entries) + : Type(ResourceType), Header(HeaderData), + ItemEntries(std::move(Entries)) {} + + Twine getResourceTypeName() const override { return "Icon/cursor group"; } + IntOrString getResourceType() const override { + return Type == IconCursorGroupType::Icon ? RkIconGroup : RkCursorGroup; + } + ResourceKind getKind() const override { return RkCursorOrIconGroupRes; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkCursorOrIconGroupRes; + } +}; + +Error ResourceFileWriter::writeSingleIconOrCursorBody(const RCResource *Base) { + auto *Res = cast<SingleIconCursorResource>(Base); + if (Res->Type == IconCursorGroupType::Cursor) { + // In case of cursors, two WORDS are appended to the beginning + // of the resource: HotspotX (Planes in RESDIRENTRY), + // and HotspotY (BitCount). + // + // Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms648026.aspx + // (Remarks section). + writeObject(Res->Header.Planes); + writeObject(Res->Header.BitCount); + } + + writeObject(Res->Image); + return Error::success(); +} + +Error ResourceFileWriter::writeIconOrCursorGroupBody(const RCResource *Base) { + auto *Res = cast<IconCursorGroupResource>(Base); + writeObject(Res->Header); + for (auto Item : Res->ItemEntries) { + writeObject(Item); + writeInt(IconCursorID++); + } + return Error::success(); +} + +Error ResourceFileWriter::visitSingleIconOrCursor(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeSingleIconOrCursorBody); +} + +Error ResourceFileWriter::visitIconOrCursorGroup(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeIconOrCursorGroupBody); +} + +Error ResourceFileWriter::visitIconOrCursorResource(const RCResource *Base) { + IconCursorGroupType Type; + StringRef FileStr; + IntOrString ResName = Base->ResName; + + if (auto *IconRes = dyn_cast<IconResource>(Base)) { + FileStr = IconRes->IconLoc; + Type = IconCursorGroupType::Icon; + } else { + auto *CursorRes = dyn_cast<CursorResource>(Base); + FileStr = CursorRes->CursorLoc; + Type = IconCursorGroupType::Cursor; + } + + bool IsLong; + stripQuotes(FileStr, IsLong); + auto File = loadFile(FileStr); + + if (!File) + return File.takeError(); + + BinaryStreamReader Reader((*File)->getBuffer(), support::little); + + // Read the file headers. + // - At the beginning, ICONDIR/NEWHEADER header. + // - Then, a number of RESDIR headers follow. These contain offsets + // to data. + const GroupIconDir *Header; + + RETURN_IF_ERROR(Reader.readObject(Header)); + if (Header->Reserved != 0) + return createError("Incorrect icon/cursor Reserved field; should be 0."); + uint16_t NeededType = Type == IconCursorGroupType::Icon ? 1 : 2; + if (Header->ResType != NeededType) + return createError("Incorrect icon/cursor ResType field; should be " + + Twine(NeededType) + "."); + + uint16_t NumItems = Header->ResCount; + + // Read single ico/cur headers. + std::vector<ResourceDirEntryStart> ItemEntries; + ItemEntries.reserve(NumItems); + std::vector<uint32_t> ItemOffsets(NumItems); + for (size_t ID = 0; ID < NumItems; ++ID) { + const ResourceDirEntryStart *Object; + RETURN_IF_ERROR(Reader.readObject(Object)); + ItemEntries.push_back(*Object); + RETURN_IF_ERROR(Reader.readInteger(ItemOffsets[ID])); + } + + // Now write each icon/cursors one by one. At first, all the contents + // without ICO/CUR header. This is described by SingleIconCursorResource. + for (size_t ID = 0; ID < NumItems; ++ID) { + // Load the fragment of file. + Reader.setOffset(ItemOffsets[ID]); + ArrayRef<uint8_t> Image; + RETURN_IF_ERROR(Reader.readArray(Image, ItemEntries[ID].Size)); + SingleIconCursorResource SingleRes(Type, ItemEntries[ID], Image); + SingleRes.setName(IconCursorID + ID); + RETURN_IF_ERROR(visitSingleIconOrCursor(&SingleRes)); + } + + // Now, write all the headers concatenated into a separate resource. + for (size_t ID = 0; ID < NumItems; ++ID) { + if (Type == IconCursorGroupType::Icon) { + // rc.exe seems to always set NumPlanes to 1. No idea why it happens. + ItemEntries[ID].Planes = 1; + continue; + } + + // We need to rewrite the cursor headers. + const auto &OldHeader = ItemEntries[ID]; + ResourceDirEntryStart NewHeader; + NewHeader.Cursor.Width = OldHeader.Icon.Width; + // Each cursor in fact stores two bitmaps, one under another. + // Height provided in cursor definition describes the height of the + // cursor, whereas the value existing in resource definition describes + // the height of the bitmap. Therefore, we need to double this height. + NewHeader.Cursor.Height = OldHeader.Icon.Height * 2; + + // Now, we actually need to read the bitmap header to find + // the number of planes and the number of bits per pixel. + Reader.setOffset(ItemOffsets[ID]); + const BitmapInfoHeader *BMPHeader; + RETURN_IF_ERROR(Reader.readObject(BMPHeader)); + NewHeader.Planes = BMPHeader->Planes; + NewHeader.BitCount = BMPHeader->BitCount; + + // Two WORDs were written at the beginning of the resource (hotspot + // location). This is reflected in Size field. + NewHeader.Size = OldHeader.Size + 2 * sizeof(uint16_t); + + ItemEntries[ID] = NewHeader; + } + + IconCursorGroupResource HeaderRes(Type, *Header, std::move(ItemEntries)); + HeaderRes.setName(ResName); + RETURN_IF_ERROR(visitIconOrCursorGroup(&HeaderRes)); + + return Error::success(); +} + +// --- DialogResource helpers. --- // + +Error ResourceFileWriter::writeSingleDialogControl(const Control &Ctl, + bool IsExtended) { + // Each control should be aligned to DWORD. + padStream(sizeof(uint32_t)); + + auto TypeInfo = Control::SupportedCtls.lookup(Ctl.Type); + uint32_t CtlStyle = TypeInfo.Style | Ctl.Style.getValueOr(0); + uint32_t CtlExtStyle = Ctl.ExtStyle.getValueOr(0); + + // DIALOG(EX) item header prefix. + if (!IsExtended) { + struct { + ulittle32_t Style; + ulittle32_t ExtStyle; + } Prefix{ulittle32_t(CtlStyle), ulittle32_t(CtlExtStyle)}; + writeObject(Prefix); + } else { + struct { + ulittle32_t HelpID; + ulittle32_t ExtStyle; + ulittle32_t Style; + } Prefix{ulittle32_t(Ctl.HelpID.getValueOr(0)), ulittle32_t(CtlExtStyle), + ulittle32_t(CtlStyle)}; + writeObject(Prefix); + } + + // Common fixed-length part. + RETURN_IF_ERROR(checkSignedNumberFits<int16_t>( + Ctl.X, "Dialog control x-coordinate", true)); + RETURN_IF_ERROR(checkSignedNumberFits<int16_t>( + Ctl.Y, "Dialog control y-coordinate", true)); + RETURN_IF_ERROR( + checkSignedNumberFits<int16_t>(Ctl.Width, "Dialog control width", false)); + RETURN_IF_ERROR(checkSignedNumberFits<int16_t>( + Ctl.Height, "Dialog control height", false)); + struct { + ulittle16_t X; + ulittle16_t Y; + ulittle16_t Width; + ulittle16_t Height; + } Middle{ulittle16_t(Ctl.X), ulittle16_t(Ctl.Y), ulittle16_t(Ctl.Width), + ulittle16_t(Ctl.Height)}; + writeObject(Middle); + + // ID; it's 16-bit in DIALOG and 32-bit in DIALOGEX. + if (!IsExtended) { + RETURN_IF_ERROR(checkNumberFits<uint16_t>( + Ctl.ID, "Control ID in simple DIALOG resource")); + writeInt<uint16_t>(Ctl.ID); + } else { + writeInt<uint32_t>(Ctl.ID); + } + + // Window class - either 0xFFFF + 16-bit integer or a string. + RETURN_IF_ERROR(writeIntOrString(IntOrString(TypeInfo.CtlClass))); + + // Element caption/reference ID. ID is preceded by 0xFFFF. + RETURN_IF_ERROR(checkIntOrString(Ctl.Title, "Control reference ID")); + RETURN_IF_ERROR(writeIntOrString(Ctl.Title)); + + // # bytes of extra creation data count. Don't pass any. + writeInt<uint16_t>(0); + + return Error::success(); +} + +Error ResourceFileWriter::writeDialogBody(const RCResource *Base) { + auto *Res = cast<DialogResource>(Base); + + // Default style: WS_POPUP | WS_BORDER | WS_SYSMENU. + const uint32_t DefaultStyle = 0x80880000; + const uint32_t StyleFontFlag = 0x40; + const uint32_t StyleCaptionFlag = 0x00C00000; + + uint32_t UsedStyle = ObjectData.Style.getValueOr(DefaultStyle); + if (ObjectData.Font) + UsedStyle |= StyleFontFlag; + else + UsedStyle &= ~StyleFontFlag; + + // Actually, in case of empty (but existent) caption, the examined field + // is equal to "\"\"". That's why empty captions are still noticed. + if (ObjectData.Caption != "") + UsedStyle |= StyleCaptionFlag; + + const uint16_t DialogExMagic = 0xFFFF; + + // Write DIALOG(EX) header prefix. These are pretty different. + if (!Res->IsExtended) { + // We cannot let the higher word of DefaultStyle be equal to 0xFFFF. + // In such a case, whole object (in .res file) is equivalent to a + // DIALOGEX. It might lead to access violation/segmentation fault in + // resource readers. For example, + // 1 DIALOG 0, 0, 0, 65432 + // STYLE 0xFFFF0001 {} + // would be compiled to a DIALOGEX with 65432 controls. + if ((UsedStyle >> 16) == DialogExMagic) + return createError("16 higher bits of DIALOG resource style cannot be" + " equal to 0xFFFF"); + + struct { + ulittle32_t Style; + ulittle32_t ExtStyle; + } Prefix{ulittle32_t(UsedStyle), + ulittle32_t(0)}; // As of now, we don't keep EXSTYLE. + + writeObject(Prefix); + } else { + struct { + ulittle16_t Version; + ulittle16_t Magic; + ulittle32_t HelpID; + ulittle32_t ExtStyle; + ulittle32_t Style; + } Prefix{ulittle16_t(1), ulittle16_t(DialogExMagic), + ulittle32_t(Res->HelpID), ulittle32_t(0), ulittle32_t(UsedStyle)}; + + writeObject(Prefix); + } + + // Now, a common part. First, fixed-length fields. + RETURN_IF_ERROR(checkNumberFits<uint16_t>(Res->Controls.size(), + "Number of dialog controls")); + RETURN_IF_ERROR( + checkSignedNumberFits<int16_t>(Res->X, "Dialog x-coordinate", true)); + RETURN_IF_ERROR( + checkSignedNumberFits<int16_t>(Res->Y, "Dialog y-coordinate", true)); + RETURN_IF_ERROR( + checkSignedNumberFits<int16_t>(Res->Width, "Dialog width", false)); + RETURN_IF_ERROR( + checkSignedNumberFits<int16_t>(Res->Height, "Dialog height", false)); + struct { + ulittle16_t Count; + ulittle16_t PosX; + ulittle16_t PosY; + ulittle16_t DialogWidth; + ulittle16_t DialogHeight; + } Middle{ulittle16_t(Res->Controls.size()), ulittle16_t(Res->X), + ulittle16_t(Res->Y), ulittle16_t(Res->Width), + ulittle16_t(Res->Height)}; + writeObject(Middle); + + // MENU field. As of now, we don't keep them in the state and can peacefully + // think there is no menu attached to the dialog. + writeInt<uint16_t>(0); + + // Window CLASS field. Not kept here. + writeInt<uint16_t>(0); + + // Window title or a single word equal to 0. + RETURN_IF_ERROR(writeCString(ObjectData.Caption)); + + // If there *is* a window font declared, output its data. + auto &Font = ObjectData.Font; + if (Font) { + writeInt<uint16_t>(Font->Size); + // Additional description occurs only in DIALOGEX. + if (Res->IsExtended) { + writeInt<uint16_t>(Font->Weight); + writeInt<uint8_t>(Font->IsItalic); + writeInt<uint8_t>(Font->Charset); + } + RETURN_IF_ERROR(writeCString(Font->Typeface)); + } + + auto handleCtlError = [&](Error &&Err, const Control &Ctl) -> Error { + if (!Err) + return Error::success(); + return joinErrors(createError("Error in " + Twine(Ctl.Type) + + " control (ID " + Twine(Ctl.ID) + "):"), + std::move(Err)); + }; + + for (auto &Ctl : Res->Controls) + RETURN_IF_ERROR( + handleCtlError(writeSingleDialogControl(Ctl, Res->IsExtended), Ctl)); + + return Error::success(); +} + +// --- HTMLResource helpers. --- // + +Error ResourceFileWriter::writeHTMLBody(const RCResource *Base) { + return appendFile(cast<HTMLResource>(Base)->HTMLLoc); +} + +// --- MenuResource helpers. --- // + +Error ResourceFileWriter::writeMenuDefinition( + const std::unique_ptr<MenuDefinition> &Def, uint16_t Flags) { + assert(Def); + const MenuDefinition *DefPtr = Def.get(); + + if (auto *MenuItemPtr = dyn_cast<MenuItem>(DefPtr)) { + writeInt<uint16_t>(Flags); + RETURN_IF_ERROR( + checkNumberFits<uint16_t>(MenuItemPtr->Id, "MENUITEM action ID")); + writeInt<uint16_t>(MenuItemPtr->Id); + RETURN_IF_ERROR(writeCString(MenuItemPtr->Name)); + return Error::success(); + } + + if (isa<MenuSeparator>(DefPtr)) { + writeInt<uint16_t>(Flags); + writeInt<uint32_t>(0); + return Error::success(); + } + + auto *PopupPtr = cast<PopupItem>(DefPtr); + writeInt<uint16_t>(Flags); + RETURN_IF_ERROR(writeCString(PopupPtr->Name)); + return writeMenuDefinitionList(PopupPtr->SubItems); +} + +Error ResourceFileWriter::writeMenuDefinitionList( + const MenuDefinitionList &List) { + for (auto &Def : List.Definitions) { + uint16_t Flags = Def->getResFlags(); + // Last element receives an additional 0x80 flag. + const uint16_t LastElementFlag = 0x0080; + if (&Def == &List.Definitions.back()) + Flags |= LastElementFlag; + + RETURN_IF_ERROR(writeMenuDefinition(Def, Flags)); + } + return Error::success(); +} + +Error ResourceFileWriter::writeMenuBody(const RCResource *Base) { + // At first, MENUHEADER structure. In fact, these are two WORDs equal to 0. + // Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms648018.aspx + writeInt<uint32_t>(0); + + return writeMenuDefinitionList(cast<MenuResource>(Base)->Elements); +} + +// --- StringTableResource helpers. --- // + +class BundleResource : public RCResource { +public: + using BundleType = ResourceFileWriter::StringTableInfo::Bundle; + BundleType Bundle; + + BundleResource(const BundleType &StrBundle) : Bundle(StrBundle) {} + IntOrString getResourceType() const override { return 6; } + + ResourceKind getKind() const override { return RkStringTableBundle; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkStringTableBundle; + } +}; + +Error ResourceFileWriter::visitStringTableBundle(const RCResource *Res) { + return writeResource(Res, &ResourceFileWriter::writeStringTableBundleBody); +} + +Error ResourceFileWriter::insertStringIntoBundle( + StringTableInfo::Bundle &Bundle, uint16_t StringID, StringRef String) { + uint16_t StringLoc = StringID & 15; + if (Bundle.Data[StringLoc]) + return createError("Multiple STRINGTABLE strings located under ID " + + Twine(StringID)); + Bundle.Data[StringLoc] = String; + return Error::success(); +} + +Error ResourceFileWriter::writeStringTableBundleBody(const RCResource *Base) { + auto *Res = cast<BundleResource>(Base); + for (size_t ID = 0; ID < Res->Bundle.Data.size(); ++ID) { + // The string format is a tiny bit different here. We + // first output the size of the string, and then the string itself + // (which is not null-terminated). + bool IsLongString; + SmallVector<UTF16, 128> Data; + RETURN_IF_ERROR(processString(Res->Bundle.Data[ID].getValueOr(StringRef()), + NullHandlingMethod::CutAtDoubleNull, + IsLongString, Data)); + if (AppendNull && Res->Bundle.Data[ID]) + Data.push_back('\0'); + RETURN_IF_ERROR( + checkNumberFits<uint16_t>(Data.size(), "STRINGTABLE string size")); + writeInt<uint16_t>(Data.size()); + for (auto Char : Data) + writeInt(Char); + } + return Error::success(); +} + +Error ResourceFileWriter::dumpAllStringTables() { + for (auto Key : StringTableData.BundleList) { + auto Iter = StringTableData.BundleData.find(Key); + assert(Iter != StringTableData.BundleData.end()); + + // For a moment, revert the context info to moment of bundle declaration. + ContextKeeper RAII(this); + ObjectData = Iter->second.DeclTimeInfo; + + BundleResource Res(Iter->second); + // Bundle #(k+1) contains keys [16k, 16k + 15]. + Res.setName(Key.first + 1); + RETURN_IF_ERROR(visitStringTableBundle(&Res)); + } + return Error::success(); +} + +// --- UserDefinedResource helpers. --- // + +Error ResourceFileWriter::writeUserDefinedBody(const RCResource *Base) { + auto *Res = cast<UserDefinedResource>(Base); + + if (Res->IsFileResource) + return appendFile(Res->FileLoc); + + for (auto &Elem : Res->Contents) { + if (Elem.isInt()) { + RETURN_IF_ERROR( + checkRCInt(Elem.getInt(), "Number in user-defined resource")); + writeRCInt(Elem.getInt()); + continue; + } + + SmallVector<UTF16, 128> ProcessedString; + bool IsLongString; + RETURN_IF_ERROR(processString(Elem.getString(), + NullHandlingMethod::UserResource, + IsLongString, ProcessedString)); + + for (auto Ch : ProcessedString) { + if (IsLongString) { + writeInt(Ch); + continue; + } + + RETURN_IF_ERROR(checkNumberFits<uint8_t>( + Ch, "Character in narrow string in user-defined resource")); + writeInt<uint8_t>(Ch); + } + } + + return Error::success(); +} + +// --- VersionInfoResourceResource helpers. --- // + +Error ResourceFileWriter::writeVersionInfoBlock(const VersionInfoBlock &Blk) { + // Output the header if the block has name. + bool OutputHeader = Blk.Name != ""; + uint64_t LengthLoc; + + if (OutputHeader) { + LengthLoc = writeInt<uint16_t>(0); + writeInt<uint16_t>(0); + writeInt<uint16_t>(1); // true + RETURN_IF_ERROR(writeCString(Blk.Name)); + padStream(sizeof(uint32_t)); + } + + for (const std::unique_ptr<VersionInfoStmt> &Item : Blk.Stmts) { + VersionInfoStmt *ItemPtr = Item.get(); + + if (auto *BlockPtr = dyn_cast<VersionInfoBlock>(ItemPtr)) { + RETURN_IF_ERROR(writeVersionInfoBlock(*BlockPtr)); + continue; + } + + auto *ValuePtr = cast<VersionInfoValue>(ItemPtr); + RETURN_IF_ERROR(writeVersionInfoValue(*ValuePtr)); + } + + if (OutputHeader) { + uint64_t CurLoc = tell(); + writeObjectAt(ulittle16_t(CurLoc - LengthLoc), LengthLoc); + } + + padStream(sizeof(uint32_t)); + return Error::success(); +} + +Error ResourceFileWriter::writeVersionInfoValue(const VersionInfoValue &Val) { + // rc has a peculiar algorithm to output VERSIONINFO VALUEs. Each VALUE + // is a mapping from the key (string) to the value (a sequence of ints or + // a sequence of strings). + // + // If integers are to be written: width of each integer written depends on + // whether it's been declared 'long' (it's DWORD then) or not (it's WORD). + // ValueLength defined in structure referenced below is then the total + // number of bytes taken by these integers. + // + // If strings are to be written: characters are always WORDs. + // Moreover, '\0' character is written after the last string, and between + // every two strings separated by comma (if strings are not comma-separated, + // they're simply concatenated). ValueLength is equal to the number of WORDs + // written (that is, half of the bytes written). + // + // Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms646994.aspx + bool HasStrings = false, HasInts = false; + for (auto &Item : Val.Values) + (Item.isInt() ? HasInts : HasStrings) = true; + + assert((HasStrings || HasInts) && "VALUE must have at least one argument"); + if (HasStrings && HasInts) + return createError(Twine("VALUE ") + Val.Key + + " cannot contain both strings and integers"); + + auto LengthLoc = writeInt<uint16_t>(0); + auto ValLengthLoc = writeInt<uint16_t>(0); + writeInt<uint16_t>(HasStrings); + RETURN_IF_ERROR(writeCString(Val.Key)); + padStream(sizeof(uint32_t)); + + auto DataLoc = tell(); + for (size_t Id = 0; Id < Val.Values.size(); ++Id) { + auto &Item = Val.Values[Id]; + if (Item.isInt()) { + auto Value = Item.getInt(); + RETURN_IF_ERROR(checkRCInt(Value, "VERSIONINFO integer value")); + writeRCInt(Value); + continue; + } + + bool WriteTerminator = + Id == Val.Values.size() - 1 || Val.HasPrecedingComma[Id + 1]; + RETURN_IF_ERROR(writeCString(Item.getString(), WriteTerminator)); + } + + auto CurLoc = tell(); + auto ValueLength = CurLoc - DataLoc; + if (HasStrings) { + assert(ValueLength % 2 == 0); + ValueLength /= 2; + } + writeObjectAt(ulittle16_t(CurLoc - LengthLoc), LengthLoc); + writeObjectAt(ulittle16_t(ValueLength), ValLengthLoc); + padStream(sizeof(uint32_t)); + return Error::success(); +} + +template <typename Ty> +static Ty getWithDefault(const StringMap<Ty> &Map, StringRef Key, + const Ty &Default) { + auto Iter = Map.find(Key); + if (Iter != Map.end()) + return Iter->getValue(); + return Default; +} + +Error ResourceFileWriter::writeVersionInfoBody(const RCResource *Base) { + auto *Res = cast<VersionInfoResource>(Base); + + const auto &FixedData = Res->FixedData; + + struct /* VS_FIXEDFILEINFO */ { + ulittle32_t Signature = ulittle32_t(0xFEEF04BD); + ulittle32_t StructVersion = ulittle32_t(0x10000); + // It's weird to have most-significant DWORD first on the little-endian + // machines, but let it be this way. + ulittle32_t FileVersionMS; + ulittle32_t FileVersionLS; + ulittle32_t ProductVersionMS; + ulittle32_t ProductVersionLS; + ulittle32_t FileFlagsMask; + ulittle32_t FileFlags; + ulittle32_t FileOS; + ulittle32_t FileType; + ulittle32_t FileSubtype; + // MS implementation seems to always set these fields to 0. + ulittle32_t FileDateMS = ulittle32_t(0); + ulittle32_t FileDateLS = ulittle32_t(0); + } FixedInfo; + + // First, VS_VERSIONINFO. + auto LengthLoc = writeInt<uint16_t>(0); + writeInt<uint16_t>(sizeof(FixedInfo)); + writeInt<uint16_t>(0); + cantFail(writeCString("VS_VERSION_INFO")); + padStream(sizeof(uint32_t)); + + using VersionInfoFixed = VersionInfoResource::VersionInfoFixed; + auto GetField = [&](VersionInfoFixed::VersionInfoFixedType Type) { + static const SmallVector<uint32_t, 4> DefaultOut{0, 0, 0, 0}; + if (!FixedData.IsTypePresent[(int)Type]) + return DefaultOut; + return FixedData.FixedInfo[(int)Type]; + }; + + auto FileVer = GetField(VersionInfoFixed::FtFileVersion); + RETURN_IF_ERROR(checkNumberFits<uint16_t>( + *std::max_element(FileVer.begin(), FileVer.end()), "FILEVERSION fields")); + FixedInfo.FileVersionMS = (FileVer[0] << 16) | FileVer[1]; + FixedInfo.FileVersionLS = (FileVer[2] << 16) | FileVer[3]; + + auto ProdVer = GetField(VersionInfoFixed::FtProductVersion); + RETURN_IF_ERROR(checkNumberFits<uint16_t>( + *std::max_element(ProdVer.begin(), ProdVer.end()), + "PRODUCTVERSION fields")); + FixedInfo.ProductVersionMS = (ProdVer[0] << 16) | ProdVer[1]; + FixedInfo.ProductVersionLS = (ProdVer[2] << 16) | ProdVer[3]; + + FixedInfo.FileFlagsMask = GetField(VersionInfoFixed::FtFileFlagsMask)[0]; + FixedInfo.FileFlags = GetField(VersionInfoFixed::FtFileFlags)[0]; + FixedInfo.FileOS = GetField(VersionInfoFixed::FtFileOS)[0]; + FixedInfo.FileType = GetField(VersionInfoFixed::FtFileType)[0]; + FixedInfo.FileSubtype = GetField(VersionInfoFixed::FtFileSubtype)[0]; + + writeObject(FixedInfo); + padStream(sizeof(uint32_t)); + + RETURN_IF_ERROR(writeVersionInfoBlock(Res->MainBlock)); + + // FIXME: check overflow? + writeObjectAt(ulittle16_t(tell() - LengthLoc), LengthLoc); + + return Error::success(); +} + +Expected<std::unique_ptr<MemoryBuffer>> +ResourceFileWriter::loadFile(StringRef File) const { + SmallString<128> Path; + SmallString<128> Cwd; + std::unique_ptr<MemoryBuffer> Result; + + // 1. The current working directory. + sys::fs::current_path(Cwd); + Path.assign(Cwd.begin(), Cwd.end()); + sys::path::append(Path, File); + if (sys::fs::exists(Path)) + return errorOrToExpected(MemoryBuffer::getFile(Path, -1, false)); + + // 2. The directory of the input resource file, if it is different from the + // current + // working directory. + StringRef InputFileDir = sys::path::parent_path(Params.InputFilePath); + Path.assign(InputFileDir.begin(), InputFileDir.end()); + sys::path::append(Path, File); + if (sys::fs::exists(Path)) + return errorOrToExpected(MemoryBuffer::getFile(Path, -1, false)); + + // 3. All of the include directories specified on the command line. + for (StringRef ForceInclude : Params.Include) { + Path.assign(ForceInclude.begin(), ForceInclude.end()); + sys::path::append(Path, File); + if (sys::fs::exists(Path)) + return errorOrToExpected(MemoryBuffer::getFile(Path, -1, false)); + } + + if (auto Result = + llvm::sys::Process::FindInEnvPath("INCLUDE", File, Params.NoInclude)) + return errorOrToExpected(MemoryBuffer::getFile(*Result, -1, false)); + + return make_error<StringError>("error : file not found : " + Twine(File), + inconvertibleErrorCode()); +} + +} // namespace rc +} // namespace llvm diff --git a/tools/llvm-rc/ResourceFileWriter.h b/tools/llvm-rc/ResourceFileWriter.h new file mode 100644 index 000000000000..20bd4bd73679 --- /dev/null +++ b/tools/llvm-rc/ResourceFileWriter.h @@ -0,0 +1,195 @@ +//===-- ResourceSerializator.h ----------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This defines a visitor serializing resources to a .res stream. +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVMRC_RESOURCESERIALIZATOR_H +#define LLVM_TOOLS_LLVMRC_RESOURCESERIALIZATOR_H + +#include "ResourceScriptStmt.h" +#include "ResourceVisitor.h" + +#include "llvm/Support/Endian.h" + +namespace llvm { + +class MemoryBuffer; + +namespace rc { + +struct SearchParams { + std::vector<std::string> Include; // Additional folders to search for files. + std::vector<std::string> NoInclude; // Folders to exclude from file search. + StringRef InputFilePath; // The full path of the input file. +}; + +class ResourceFileWriter : public Visitor { +public: + ResourceFileWriter(const SearchParams &Params, + std::unique_ptr<raw_fd_ostream> Stream) + : Params(Params), FS(std::move(Stream)), IconCursorID(1) { + assert(FS && "Output stream needs to be provided to the serializator"); + } + + Error visitNullResource(const RCResource *) override; + Error visitAcceleratorsResource(const RCResource *) override; + Error visitCursorResource(const RCResource *) override; + Error visitDialogResource(const RCResource *) override; + Error visitHTMLResource(const RCResource *) override; + Error visitIconResource(const RCResource *) override; + Error visitMenuResource(const RCResource *) override; + Error visitVersionInfoResource(const RCResource *) override; + Error visitStringTableResource(const RCResource *) override; + Error visitUserDefinedResource(const RCResource *) override; + + Error visitCaptionStmt(const CaptionStmt *) override; + Error visitCharacteristicsStmt(const CharacteristicsStmt *) override; + Error visitFontStmt(const FontStmt *) override; + Error visitLanguageStmt(const LanguageResource *) override; + Error visitStyleStmt(const StyleStmt *) override; + Error visitVersionStmt(const VersionStmt *) override; + + // Stringtables are output at the end of .res file. We need a separate + // function to do it. + Error dumpAllStringTables(); + + bool AppendNull; // Append '\0' to each existing STRINGTABLE element? + + struct ObjectInfo { + uint16_t LanguageInfo; + uint32_t Characteristics; + uint32_t VersionInfo; + + Optional<uint32_t> Style; + StringRef Caption; + struct FontInfo { + uint32_t Size; + StringRef Typeface; + uint32_t Weight; + bool IsItalic; + uint32_t Charset; + }; + Optional<FontInfo> Font; + + ObjectInfo() : LanguageInfo(0), Characteristics(0), VersionInfo(0) {} + } ObjectData; + + struct StringTableInfo { + // Each STRINGTABLE bundle depends on ID of the bundle and language + // description. + using BundleKey = std::pair<uint16_t, uint16_t>; + // Each bundle is in fact an array of 16 strings. + struct Bundle { + std::array<Optional<StringRef>, 16> Data; + ObjectInfo DeclTimeInfo; + Bundle(const ObjectInfo &Info) : DeclTimeInfo(Info) {} + }; + std::map<BundleKey, Bundle> BundleData; + // Bundles are listed in the order of their first occurence. + std::vector<BundleKey> BundleList; + } StringTableData; + +private: + Error handleError(Error Err, const RCResource *Res); + + Error + writeResource(const RCResource *Res, + Error (ResourceFileWriter::*BodyWriter)(const RCResource *)); + + // NullResource + Error writeNullBody(const RCResource *); + + // AcceleratorsResource + Error writeSingleAccelerator(const AcceleratorsResource::Accelerator &, + bool IsLastItem); + Error writeAcceleratorsBody(const RCResource *); + + // CursorResource and IconResource + Error visitIconOrCursorResource(const RCResource *); + Error visitIconOrCursorGroup(const RCResource *); + Error visitSingleIconOrCursor(const RCResource *); + Error writeSingleIconOrCursorBody(const RCResource *); + Error writeIconOrCursorGroupBody(const RCResource *); + + // DialogResource + Error writeSingleDialogControl(const Control &, bool IsExtended); + Error writeDialogBody(const RCResource *); + + // HTMLResource + Error writeHTMLBody(const RCResource *); + + // MenuResource + Error writeMenuDefinition(const std::unique_ptr<MenuDefinition> &, + uint16_t Flags); + Error writeMenuDefinitionList(const MenuDefinitionList &List); + Error writeMenuBody(const RCResource *); + + // StringTableResource + Error visitStringTableBundle(const RCResource *); + Error writeStringTableBundleBody(const RCResource *); + Error insertStringIntoBundle(StringTableInfo::Bundle &Bundle, + uint16_t StringID, StringRef String); + + // User defined resource + Error writeUserDefinedBody(const RCResource *); + + // VersionInfoResource + Error writeVersionInfoBody(const RCResource *); + Error writeVersionInfoBlock(const VersionInfoBlock &); + Error writeVersionInfoValue(const VersionInfoValue &); + + const SearchParams &Params; + + // Output stream handling. + std::unique_ptr<raw_fd_ostream> FS; + + uint64_t tell() const { return FS->tell(); } + + uint64_t writeObject(const ArrayRef<uint8_t> Data); + + template <typename T> uint64_t writeInt(const T &Value) { + support::detail::packed_endian_specific_integral<T, support::little, + support::unaligned> + Object(Value); + return writeObject(Object); + } + + template <typename T> uint64_t writeObject(const T &Value) { + return writeObject(ArrayRef<uint8_t>( + reinterpret_cast<const uint8_t *>(&Value), sizeof(T))); + } + + template <typename T> void writeObjectAt(const T &Value, uint64_t Position) { + FS->pwrite((const char *)&Value, sizeof(T), Position); + } + + Error writeCString(StringRef Str, bool WriteTerminator = true); + + Error writeIdentifier(const IntOrString &Ident); + Error writeIntOrString(const IntOrString &Data); + + void writeRCInt(RCInt); + + Error appendFile(StringRef Filename); + + void padStream(uint64_t Length); + + Expected<std::unique_ptr<MemoryBuffer>> loadFile(StringRef File) const; + + // Icon and cursor IDs are allocated starting from 1 and increasing for + // each icon/cursor dumped. This maintains the current ID to be allocated. + uint16_t IconCursorID; +}; + +} // namespace rc +} // namespace llvm + +#endif diff --git a/tools/llvm-rc/ResourceScriptParser.cpp b/tools/llvm-rc/ResourceScriptParser.cpp new file mode 100644 index 000000000000..cf1579ee2a11 --- /dev/null +++ b/tools/llvm-rc/ResourceScriptParser.cpp @@ -0,0 +1,714 @@ +//===-- ResourceScriptParser.cpp --------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This implements the parser defined in ResourceScriptParser.h. +// +//===---------------------------------------------------------------------===// + +#include "ResourceScriptParser.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" + +// Take an expression returning llvm::Error and forward the error if it exists. +#define RETURN_IF_ERROR(Expr) \ + if (auto Err = (Expr)) \ + return std::move(Err); + +// Take an expression returning llvm::Expected<T> and assign it to Var or +// forward the error out of the function. +#define ASSIGN_OR_RETURN(Var, Expr) \ + auto Var = (Expr); \ + if (!Var) \ + return Var.takeError(); + +namespace llvm { +namespace rc { + +RCParser::ParserError::ParserError(const Twine &Expected, const LocIter CurLoc, + const LocIter End) + : ErrorLoc(CurLoc), FileEnd(End) { + CurMessage = "Error parsing file: expected " + Expected.str() + ", got " + + (CurLoc == End ? "<EOF>" : CurLoc->value()).str(); +} + +char RCParser::ParserError::ID = 0; + +RCParser::RCParser(std::vector<RCToken> TokenList) + : Tokens(std::move(TokenList)), CurLoc(Tokens.begin()), End(Tokens.end()) {} + +bool RCParser::isEof() const { return CurLoc == End; } + +RCParser::ParseType RCParser::parseSingleResource() { + // The first thing we read is usually a resource's name. However, in some + // cases (LANGUAGE and STRINGTABLE) the resources don't have their names + // and the first token to be read is the type. + ASSIGN_OR_RETURN(NameToken, readTypeOrName()); + + if (NameToken->equalsLower("LANGUAGE")) + return parseLanguageResource(); + else if (NameToken->equalsLower("STRINGTABLE")) + return parseStringTableResource(); + + // If it's not an unnamed resource, what we've just read is a name. Now, + // read resource type; + ASSIGN_OR_RETURN(TypeToken, readTypeOrName()); + + ParseType Result = std::unique_ptr<RCResource>(); + (void)!Result; + + if (TypeToken->equalsLower("ACCELERATORS")) + Result = parseAcceleratorsResource(); + else if (TypeToken->equalsLower("CURSOR")) + Result = parseCursorResource(); + else if (TypeToken->equalsLower("DIALOG")) + Result = parseDialogResource(false); + else if (TypeToken->equalsLower("DIALOGEX")) + Result = parseDialogResource(true); + else if (TypeToken->equalsLower("ICON")) + Result = parseIconResource(); + else if (TypeToken->equalsLower("HTML")) + Result = parseHTMLResource(); + else if (TypeToken->equalsLower("MENU")) + Result = parseMenuResource(); + else if (TypeToken->equalsLower("VERSIONINFO")) + Result = parseVersionInfoResource(); + else + Result = parseUserDefinedResource(*TypeToken); + + if (Result) + (*Result)->setName(*NameToken); + + return Result; +} + +bool RCParser::isNextTokenKind(Kind TokenKind) const { + return !isEof() && look().kind() == TokenKind; +} + +const RCToken &RCParser::look() const { + assert(!isEof()); + return *CurLoc; +} + +const RCToken &RCParser::read() { + assert(!isEof()); + return *CurLoc++; +} + +void RCParser::consume() { + assert(!isEof()); + CurLoc++; +} + +// An integer description might consist of a single integer or +// an arithmetic expression evaluating to the integer. The expressions +// can contain the following tokens: <int> ( ) + - | & ~. Their meaning +// is the same as in C++. +// The operators in the original RC implementation have the following +// precedence: +// 1) Unary operators (- ~), +// 2) Binary operators (+ - & |), with no precedence. +// +// The following grammar is used to parse the expressions Exp1: +// Exp1 ::= Exp2 || Exp1 + Exp2 || Exp1 - Exp2 || Exp1 | Exp2 || Exp1 & Exp2 +// Exp2 ::= -Exp2 || ~Exp2 || Int || (Exp1). +// (More conveniently, Exp1 is a non-empty sequence of Exp2 expressions, +// separated by binary operators.) +// +// Expressions of type Exp1 are read by parseIntExpr1(Inner) method, while Exp2 +// is read by parseIntExpr2(). +// +// The original Microsoft tool handles multiple unary operators incorrectly. +// For example, in 16-bit little-endian integers: +// 1 => 01 00, -1 => ff ff, --1 => ff ff, ---1 => 01 00; +// 1 => 01 00, ~1 => fe ff, ~~1 => fd ff, ~~~1 => fc ff. +// Our implementation differs from the original one and handles these +// operators correctly: +// 1 => 01 00, -1 => ff ff, --1 => 01 00, ---1 => ff ff; +// 1 => 01 00, ~1 => fe ff, ~~1 => 01 00, ~~~1 => fe ff. + +Expected<RCInt> RCParser::readInt() { return parseIntExpr1(); } + +Expected<RCInt> RCParser::parseIntExpr1() { + // Exp1 ::= Exp2 || Exp1 + Exp2 || Exp1 - Exp2 || Exp1 | Exp2 || Exp1 & Exp2. + ASSIGN_OR_RETURN(FirstResult, parseIntExpr2()); + RCInt Result = *FirstResult; + + while (!isEof() && look().isBinaryOp()) { + auto OpToken = read(); + ASSIGN_OR_RETURN(NextResult, parseIntExpr2()); + + switch (OpToken.kind()) { + case Kind::Plus: + Result += *NextResult; + break; + + case Kind::Minus: + Result -= *NextResult; + break; + + case Kind::Pipe: + Result |= *NextResult; + break; + + case Kind::Amp: + Result &= *NextResult; + break; + + default: + llvm_unreachable("Already processed all binary ops."); + } + } + + return Result; +} + +Expected<RCInt> RCParser::parseIntExpr2() { + // Exp2 ::= -Exp2 || ~Exp2 || Int || (Exp1). + static const char ErrorMsg[] = "'-', '~', integer or '('"; + + if (isEof()) + return getExpectedError(ErrorMsg); + + switch (look().kind()) { + case Kind::Minus: { + consume(); + ASSIGN_OR_RETURN(Result, parseIntExpr2()); + return -(*Result); + } + + case Kind::Tilde: { + consume(); + ASSIGN_OR_RETURN(Result, parseIntExpr2()); + return ~(*Result); + } + + case Kind::Int: + return RCInt(read()); + + case Kind::LeftParen: { + consume(); + ASSIGN_OR_RETURN(Result, parseIntExpr1()); + RETURN_IF_ERROR(consumeType(Kind::RightParen)); + return *Result; + } + + default: + return getExpectedError(ErrorMsg); + } +} + +Expected<StringRef> RCParser::readString() { + if (!isNextTokenKind(Kind::String)) + return getExpectedError("string"); + return read().value(); +} + +Expected<StringRef> RCParser::readIdentifier() { + if (!isNextTokenKind(Kind::Identifier)) + return getExpectedError("identifier"); + return read().value(); +} + +Expected<IntOrString> RCParser::readIntOrString() { + if (!isNextTokenKind(Kind::Int) && !isNextTokenKind(Kind::String)) + return getExpectedError("int or string"); + return IntOrString(read()); +} + +Expected<IntOrString> RCParser::readTypeOrName() { + // We suggest that the correct resource name or type should be either an + // identifier or an integer. The original RC tool is much more liberal. + if (!isNextTokenKind(Kind::Identifier) && !isNextTokenKind(Kind::Int)) + return getExpectedError("int or identifier"); + return IntOrString(read()); +} + +Error RCParser::consumeType(Kind TokenKind) { + if (isNextTokenKind(TokenKind)) { + consume(); + return Error::success(); + } + + switch (TokenKind) { +#define TOKEN(TokenName) \ + case Kind::TokenName: \ + return getExpectedError(#TokenName); +#define SHORT_TOKEN(TokenName, TokenCh) \ + case Kind::TokenName: \ + return getExpectedError(#TokenCh); +#include "ResourceScriptTokenList.def" + } + + llvm_unreachable("All case options exhausted."); +} + +bool RCParser::consumeOptionalType(Kind TokenKind) { + if (isNextTokenKind(TokenKind)) { + consume(); + return true; + } + + return false; +} + +Expected<SmallVector<RCInt, 8>> RCParser::readIntsWithCommas(size_t MinCount, + size_t MaxCount) { + assert(MinCount <= MaxCount); + + SmallVector<RCInt, 8> Result; + + auto FailureHandler = + [&](llvm::Error Err) -> Expected<SmallVector<RCInt, 8>> { + if (Result.size() < MinCount) + return std::move(Err); + consumeError(std::move(Err)); + return Result; + }; + + for (size_t i = 0; i < MaxCount; ++i) { + // Try to read a comma unless we read the first token. + // Sometimes RC tool requires them and sometimes not. We decide to + // always require them. + if (i >= 1) { + if (auto CommaError = consumeType(Kind::Comma)) + return FailureHandler(std::move(CommaError)); + } + + if (auto IntResult = readInt()) + Result.push_back(*IntResult); + else + return FailureHandler(IntResult.takeError()); + } + + return std::move(Result); +} + +Expected<uint32_t> RCParser::parseFlags(ArrayRef<StringRef> FlagDesc, + ArrayRef<uint32_t> FlagValues) { + assert(!FlagDesc.empty()); + assert(FlagDesc.size() == FlagValues.size()); + + uint32_t Result = 0; + while (isNextTokenKind(Kind::Comma)) { + consume(); + ASSIGN_OR_RETURN(FlagResult, readIdentifier()); + bool FoundFlag = false; + + for (size_t FlagId = 0; FlagId < FlagDesc.size(); ++FlagId) { + if (!FlagResult->equals_lower(FlagDesc[FlagId])) + continue; + + Result |= FlagValues[FlagId]; + FoundFlag = true; + break; + } + + if (!FoundFlag) + return getExpectedError(join(FlagDesc, "/"), true); + } + + return Result; +} + +Expected<OptionalStmtList> +RCParser::parseOptionalStatements(OptStmtType StmtsType) { + OptionalStmtList Result; + + // The last statement is always followed by the start of the block. + while (!isNextTokenKind(Kind::BlockBegin)) { + ASSIGN_OR_RETURN(SingleParse, parseSingleOptionalStatement(StmtsType)); + Result.addStmt(std::move(*SingleParse)); + } + + return std::move(Result); +} + +Expected<std::unique_ptr<OptionalStmt>> +RCParser::parseSingleOptionalStatement(OptStmtType StmtsType) { + ASSIGN_OR_RETURN(TypeToken, readIdentifier()); + if (TypeToken->equals_lower("CHARACTERISTICS")) + return parseCharacteristicsStmt(); + if (TypeToken->equals_lower("LANGUAGE")) + return parseLanguageStmt(); + if (TypeToken->equals_lower("VERSION")) + return parseVersionStmt(); + + if (StmtsType != OptStmtType::BasicStmt) { + if (TypeToken->equals_lower("CAPTION")) + return parseCaptionStmt(); + if (TypeToken->equals_lower("FONT")) + return parseFontStmt(StmtsType); + if (TypeToken->equals_lower("STYLE")) + return parseStyleStmt(); + } + + return getExpectedError("optional statement type, BEGIN or '{'", + /* IsAlreadyRead = */ true); +} + +RCParser::ParseType RCParser::parseLanguageResource() { + // Read LANGUAGE as an optional statement. If it's read correctly, we can + // upcast it to RCResource. + return parseLanguageStmt(); +} + +RCParser::ParseType RCParser::parseAcceleratorsResource() { + ASSIGN_OR_RETURN(OptStatements, parseOptionalStatements()); + RETURN_IF_ERROR(consumeType(Kind::BlockBegin)); + + auto Accels = + llvm::make_unique<AcceleratorsResource>(std::move(*OptStatements)); + + while (!consumeOptionalType(Kind::BlockEnd)) { + ASSIGN_OR_RETURN(EventResult, readIntOrString()); + RETURN_IF_ERROR(consumeType(Kind::Comma)); + ASSIGN_OR_RETURN(IDResult, readInt()); + ASSIGN_OR_RETURN( + FlagsResult, + parseFlags(AcceleratorsResource::Accelerator::OptionsStr, + AcceleratorsResource::Accelerator::OptionsFlags)); + Accels->addAccelerator(*EventResult, *IDResult, *FlagsResult); + } + + return std::move(Accels); +} + +RCParser::ParseType RCParser::parseCursorResource() { + ASSIGN_OR_RETURN(Arg, readString()); + return llvm::make_unique<CursorResource>(*Arg); +} + +RCParser::ParseType RCParser::parseDialogResource(bool IsExtended) { + // Dialog resources have the following format of the arguments: + // DIALOG: x, y, width, height [opt stmts...] {controls...} + // DIALOGEX: x, y, width, height [, helpID] [opt stmts...] {controls...} + // These are very similar, so we parse them together. + ASSIGN_OR_RETURN(LocResult, readIntsWithCommas(4, 4)); + + uint32_t HelpID = 0; // When HelpID is unset, it's assumed to be 0. + if (IsExtended && consumeOptionalType(Kind::Comma)) { + ASSIGN_OR_RETURN(HelpIDResult, readInt()); + HelpID = *HelpIDResult; + } + + ASSIGN_OR_RETURN(OptStatements, parseOptionalStatements( + IsExtended ? OptStmtType::DialogExStmt + : OptStmtType::DialogStmt)); + + assert(isNextTokenKind(Kind::BlockBegin) && + "parseOptionalStatements, when successful, halts on BlockBegin."); + consume(); + + auto Dialog = llvm::make_unique<DialogResource>( + (*LocResult)[0], (*LocResult)[1], (*LocResult)[2], (*LocResult)[3], + HelpID, std::move(*OptStatements), IsExtended); + + while (!consumeOptionalType(Kind::BlockEnd)) { + ASSIGN_OR_RETURN(ControlDefResult, parseControl()); + Dialog->addControl(std::move(*ControlDefResult)); + } + + return std::move(Dialog); +} + +RCParser::ParseType RCParser::parseUserDefinedResource(IntOrString Type) { + if (isEof()) + return getExpectedError("filename, '{' or BEGIN"); + + // Check if this is a file resource. + if (look().kind() == Kind::String) + return llvm::make_unique<UserDefinedResource>(Type, read().value()); + + RETURN_IF_ERROR(consumeType(Kind::BlockBegin)); + std::vector<IntOrString> Data; + + // Consume comma before each consecutive token except the first one. + bool ConsumeComma = false; + while (!consumeOptionalType(Kind::BlockEnd)) { + if (ConsumeComma) + RETURN_IF_ERROR(consumeType(Kind::Comma)); + ConsumeComma = true; + + ASSIGN_OR_RETURN(Item, readIntOrString()); + Data.push_back(*Item); + } + + return llvm::make_unique<UserDefinedResource>(Type, std::move(Data)); +} + +RCParser::ParseType RCParser::parseVersionInfoResource() { + ASSIGN_OR_RETURN(FixedResult, parseVersionInfoFixed()); + ASSIGN_OR_RETURN(BlockResult, parseVersionInfoBlockContents(StringRef())); + return llvm::make_unique<VersionInfoResource>(std::move(**BlockResult), + std::move(*FixedResult)); +} + +Expected<Control> RCParser::parseControl() { + // Each control definition (except CONTROL) follows one of the schemes below + // depending on the control class: + // [class] text, id, x, y, width, height [, style] [, exstyle] [, helpID] + // [class] id, x, y, width, height [, style] [, exstyle] [, helpID] + // Note that control ids must be integers. + // Text might be either a string or an integer pointing to resource ID. + ASSIGN_OR_RETURN(ClassResult, readIdentifier()); + std::string ClassUpper = ClassResult->upper(); + auto CtlInfo = Control::SupportedCtls.find(ClassUpper); + if (CtlInfo == Control::SupportedCtls.end()) + return getExpectedError("control type, END or '}'", true); + + // Read caption if necessary. + IntOrString Caption{StringRef()}; + if (CtlInfo->getValue().HasTitle) { + ASSIGN_OR_RETURN(CaptionResult, readIntOrString()); + RETURN_IF_ERROR(consumeType(Kind::Comma)); + Caption = *CaptionResult; + } + + ASSIGN_OR_RETURN(Args, readIntsWithCommas(5, 8)); + + auto TakeOptArg = [&Args](size_t Id) -> Optional<uint32_t> { + return Args->size() > Id ? (uint32_t)(*Args)[Id] : Optional<uint32_t>(); + }; + + return Control(*ClassResult, Caption, (*Args)[0], (*Args)[1], (*Args)[2], + (*Args)[3], (*Args)[4], TakeOptArg(5), TakeOptArg(6), + TakeOptArg(7)); +} + +RCParser::ParseType RCParser::parseIconResource() { + ASSIGN_OR_RETURN(Arg, readString()); + return llvm::make_unique<IconResource>(*Arg); +} + +RCParser::ParseType RCParser::parseHTMLResource() { + ASSIGN_OR_RETURN(Arg, readString()); + return llvm::make_unique<HTMLResource>(*Arg); +} + +RCParser::ParseType RCParser::parseMenuResource() { + ASSIGN_OR_RETURN(OptStatements, parseOptionalStatements()); + ASSIGN_OR_RETURN(Items, parseMenuItemsList()); + return llvm::make_unique<MenuResource>(std::move(*OptStatements), + std::move(*Items)); +} + +Expected<MenuDefinitionList> RCParser::parseMenuItemsList() { + RETURN_IF_ERROR(consumeType(Kind::BlockBegin)); + + MenuDefinitionList List; + + // Read a set of items. Each item is of one of three kinds: + // MENUITEM SEPARATOR + // MENUITEM caption:String, result:Int [, menu flags]... + // POPUP caption:String [, menu flags]... { items... } + while (!consumeOptionalType(Kind::BlockEnd)) { + ASSIGN_OR_RETURN(ItemTypeResult, readIdentifier()); + + bool IsMenuItem = ItemTypeResult->equals_lower("MENUITEM"); + bool IsPopup = ItemTypeResult->equals_lower("POPUP"); + if (!IsMenuItem && !IsPopup) + return getExpectedError("MENUITEM, POPUP, END or '}'", true); + + if (IsMenuItem && isNextTokenKind(Kind::Identifier)) { + // Now, expecting SEPARATOR. + ASSIGN_OR_RETURN(SeparatorResult, readIdentifier()); + if (SeparatorResult->equals_lower("SEPARATOR")) { + List.addDefinition(llvm::make_unique<MenuSeparator>()); + continue; + } + + return getExpectedError("SEPARATOR or string", true); + } + + // Not a separator. Read the caption. + ASSIGN_OR_RETURN(CaptionResult, readString()); + + // If MENUITEM, expect also a comma and an integer. + uint32_t MenuResult = -1; + + if (IsMenuItem) { + RETURN_IF_ERROR(consumeType(Kind::Comma)); + ASSIGN_OR_RETURN(IntResult, readInt()); + MenuResult = *IntResult; + } + + ASSIGN_OR_RETURN(FlagsResult, parseFlags(MenuDefinition::OptionsStr, + MenuDefinition::OptionsFlags)); + + if (IsPopup) { + // If POPUP, read submenu items recursively. + ASSIGN_OR_RETURN(SubMenuResult, parseMenuItemsList()); + List.addDefinition(llvm::make_unique<PopupItem>( + *CaptionResult, *FlagsResult, std::move(*SubMenuResult))); + continue; + } + + assert(IsMenuItem); + List.addDefinition( + llvm::make_unique<MenuItem>(*CaptionResult, MenuResult, *FlagsResult)); + } + + return std::move(List); +} + +RCParser::ParseType RCParser::parseStringTableResource() { + ASSIGN_OR_RETURN(OptStatements, parseOptionalStatements()); + RETURN_IF_ERROR(consumeType(Kind::BlockBegin)); + + auto Table = + llvm::make_unique<StringTableResource>(std::move(*OptStatements)); + + // Read strings until we reach the end of the block. + while (!consumeOptionalType(Kind::BlockEnd)) { + // Each definition consists of string's ID (an integer) and a string. + // Some examples in documentation suggest that there might be a comma in + // between, however we strictly adhere to the single statement definition. + ASSIGN_OR_RETURN(IDResult, readInt()); + ASSIGN_OR_RETURN(StrResult, readString()); + Table->addString(*IDResult, *StrResult); + } + + return std::move(Table); +} + +Expected<std::unique_ptr<VersionInfoBlock>> +RCParser::parseVersionInfoBlockContents(StringRef BlockName) { + RETURN_IF_ERROR(consumeType(Kind::BlockBegin)); + + auto Contents = llvm::make_unique<VersionInfoBlock>(BlockName); + + while (!isNextTokenKind(Kind::BlockEnd)) { + ASSIGN_OR_RETURN(Stmt, parseVersionInfoStmt()); + Contents->addStmt(std::move(*Stmt)); + } + + consume(); // Consume BlockEnd. + + return std::move(Contents); +} + +Expected<std::unique_ptr<VersionInfoStmt>> RCParser::parseVersionInfoStmt() { + // Expect either BLOCK or VALUE, then a name or a key (a string). + ASSIGN_OR_RETURN(TypeResult, readIdentifier()); + + if (TypeResult->equals_lower("BLOCK")) { + ASSIGN_OR_RETURN(NameResult, readString()); + return parseVersionInfoBlockContents(*NameResult); + } + + if (TypeResult->equals_lower("VALUE")) { + ASSIGN_OR_RETURN(KeyResult, readString()); + // Read a non-empty list of strings and/or ints, each + // possibly preceded by a comma. Unfortunately, the tool behavior depends + // on them existing or not, so we need to memorize where we found them. + std::vector<IntOrString> Values; + std::vector<bool> PrecedingCommas; + RETURN_IF_ERROR(consumeType(Kind::Comma)); + while (!isNextTokenKind(Kind::Identifier) && + !isNextTokenKind(Kind::BlockEnd)) { + // Try to eat a comma if it's not the first statement. + bool HadComma = Values.size() > 0 && consumeOptionalType(Kind::Comma); + ASSIGN_OR_RETURN(ValueResult, readIntOrString()); + Values.push_back(*ValueResult); + PrecedingCommas.push_back(HadComma); + } + return llvm::make_unique<VersionInfoValue>(*KeyResult, std::move(Values), + std::move(PrecedingCommas)); + } + + return getExpectedError("BLOCK or VALUE", true); +} + +Expected<VersionInfoResource::VersionInfoFixed> +RCParser::parseVersionInfoFixed() { + using RetType = VersionInfoResource::VersionInfoFixed; + RetType Result; + + // Read until the beginning of the block. + while (!isNextTokenKind(Kind::BlockBegin)) { + ASSIGN_OR_RETURN(TypeResult, readIdentifier()); + auto FixedType = RetType::getFixedType(*TypeResult); + + if (!RetType::isTypeSupported(FixedType)) + return getExpectedError("fixed VERSIONINFO statement type", true); + if (Result.IsTypePresent[FixedType]) + return getExpectedError("yet unread fixed VERSIONINFO statement type", + true); + + // VERSION variations take multiple integers. + size_t NumInts = RetType::isVersionType(FixedType) ? 4 : 1; + ASSIGN_OR_RETURN(ArgsResult, readIntsWithCommas(NumInts, NumInts)); + SmallVector<uint32_t, 4> ArgInts(ArgsResult->begin(), ArgsResult->end()); + Result.setValue(FixedType, ArgInts); + } + + return Result; +} + +RCParser::ParseOptionType RCParser::parseLanguageStmt() { + ASSIGN_OR_RETURN(Args, readIntsWithCommas(/* min = */ 2, /* max = */ 2)); + return llvm::make_unique<LanguageResource>((*Args)[0], (*Args)[1]); +} + +RCParser::ParseOptionType RCParser::parseCharacteristicsStmt() { + ASSIGN_OR_RETURN(Arg, readInt()); + return llvm::make_unique<CharacteristicsStmt>(*Arg); +} + +RCParser::ParseOptionType RCParser::parseVersionStmt() { + ASSIGN_OR_RETURN(Arg, readInt()); + return llvm::make_unique<VersionStmt>(*Arg); +} + +RCParser::ParseOptionType RCParser::parseCaptionStmt() { + ASSIGN_OR_RETURN(Arg, readString()); + return llvm::make_unique<CaptionStmt>(*Arg); +} + +RCParser::ParseOptionType RCParser::parseFontStmt(OptStmtType DialogType) { + assert(DialogType != OptStmtType::BasicStmt); + + ASSIGN_OR_RETURN(SizeResult, readInt()); + RETURN_IF_ERROR(consumeType(Kind::Comma)); + ASSIGN_OR_RETURN(NameResult, readString()); + + // Default values for the optional arguments. + uint32_t FontWeight = 0; + bool FontItalic = false; + uint32_t FontCharset = 1; + if (DialogType == OptStmtType::DialogExStmt) { + if (consumeOptionalType(Kind::Comma)) { + ASSIGN_OR_RETURN(Args, readIntsWithCommas(/* min = */ 0, /* max = */ 3)); + if (Args->size() >= 1) + FontWeight = (*Args)[0]; + if (Args->size() >= 2) + FontItalic = (*Args)[1] != 0; + if (Args->size() >= 3) + FontCharset = (*Args)[2]; + } + } + return llvm::make_unique<FontStmt>(*SizeResult, *NameResult, FontWeight, + FontItalic, FontCharset); +} + +RCParser::ParseOptionType RCParser::parseStyleStmt() { + ASSIGN_OR_RETURN(Arg, readInt()); + return llvm::make_unique<StyleStmt>(*Arg); +} + +Error RCParser::getExpectedError(const Twine &Message, bool IsAlreadyRead) { + return make_error<ParserError>( + Message, IsAlreadyRead ? std::prev(CurLoc) : CurLoc, End); +} + +} // namespace rc +} // namespace llvm diff --git a/tools/llvm-rc/ResourceScriptParser.h b/tools/llvm-rc/ResourceScriptParser.h new file mode 100644 index 000000000000..84fdfd5a5860 --- /dev/null +++ b/tools/llvm-rc/ResourceScriptParser.h @@ -0,0 +1,187 @@ +//===-- ResourceScriptParser.h ----------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This defines the RC scripts parser. It takes a sequence of RC tokens +// and then provides the method to parse the resources one by one. +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVMRC_RESOURCESCRIPTPARSER_H +#define LLVM_TOOLS_LLVMRC_RESOURCESCRIPTPARSER_H + +#include "ResourceScriptStmt.h" +#include "ResourceScriptToken.h" + +#include "llvm/Support/Compiler.h" +#include "llvm/Support/raw_ostream.h" + +#include <system_error> +#include <vector> + +namespace llvm { +namespace opt { +class InputArgList; +} +namespace rc { + +class RCParser { +public: + using LocIter = std::vector<RCToken>::iterator; + using ParseType = Expected<std::unique_ptr<RCResource>>; + using ParseOptionType = Expected<std::unique_ptr<OptionalStmt>>; + + // Class describing a single failure of parser. + class ParserError : public ErrorInfo<ParserError> { + public: + ParserError(const Twine &Expected, const LocIter CurLoc, const LocIter End); + + void log(raw_ostream &OS) const override { OS << CurMessage; } + std::error_code convertToErrorCode() const override { + return std::make_error_code(std::errc::invalid_argument); + } + const std::string &getMessage() const { return CurMessage; } + + static char ID; // Keep llvm::Error happy. + + private: + std::string CurMessage; + LocIter ErrorLoc, FileEnd; + }; + + explicit RCParser(std::vector<RCToken> TokenList); + + // Reads and returns a single resource definition, or error message if any + // occurred. + ParseType parseSingleResource(); + + bool isEof() const; + +private: + using Kind = RCToken::Kind; + + // Checks if the current parser state points to the token of type TokenKind. + bool isNextTokenKind(Kind TokenKind) const; + + // These methods assume that the parser is not in EOF state. + + // Take a look at the current token. Do not fetch it. + const RCToken &look() const; + // Read the current token and advance the state by one token. + const RCToken &read(); + // Advance the state by one token, discarding the current token. + void consume(); + + // The following methods try to read a single token, check if it has the + // correct type and then parse it. + // Each integer can be written as an arithmetic expression producing an + // unsigned 32-bit integer. + Expected<RCInt> readInt(); // Parse an integer. + Expected<StringRef> readString(); // Parse a string. + Expected<StringRef> readIdentifier(); // Parse an identifier. + Expected<IntOrString> readIntOrString(); // Parse an integer or a string. + Expected<IntOrString> readTypeOrName(); // Parse an integer or an identifier. + + // Helper integer expression parsing methods. + Expected<RCInt> parseIntExpr1(); + Expected<RCInt> parseIntExpr2(); + + // Advance the state by one, discarding the current token. + // If the discarded token had an incorrect type, fail. + Error consumeType(Kind TokenKind); + + // Check the current token type. If it's TokenKind, discard it. + // Return true if the parser consumed this token successfully. + bool consumeOptionalType(Kind TokenKind); + + // Read at least MinCount, and at most MaxCount integers separated by + // commas. The parser stops reading after fetching MaxCount integers + // or after an error occurs. Whenever the parser reads a comma, it + // expects an integer to follow. + Expected<SmallVector<RCInt, 8>> readIntsWithCommas(size_t MinCount, + size_t MaxCount); + + // Read an unknown number of flags preceded by commas. Each correct flag + // has an entry in FlagDesc array of length NumFlags. In case i-th + // flag (0-based) has been read, the result is OR-ed with FlagValues[i]. + // As long as parser has a comma to read, it expects to be fed with + // a correct flag afterwards. + Expected<uint32_t> parseFlags(ArrayRef<StringRef> FlagDesc, + ArrayRef<uint32_t> FlagValues); + + // Reads a set of optional statements. These can change the behavior of + // a number of resource types (e.g. STRINGTABLE, MENU or DIALOG) if provided + // before the main block with the contents of the resource. + // Usually, resources use a basic set of optional statements: + // CHARACTERISTICS, LANGUAGE, VERSION + // However, DIALOG and DIALOGEX extend this list by the following items: + // CAPTION, CLASS, EXSTYLE, FONT, MENU, STYLE + // UseExtendedStatements flag (off by default) allows the parser to read + // the additional types of statements. + // + // Ref (to the list of all optional statements): + // msdn.microsoft.com/en-us/library/windows/desktop/aa381002(v=vs.85).aspx + enum class OptStmtType { BasicStmt, DialogStmt, DialogExStmt }; + + Expected<OptionalStmtList> + parseOptionalStatements(OptStmtType StmtsType = OptStmtType::BasicStmt); + + // Read a single optional statement. + Expected<std::unique_ptr<OptionalStmt>> + parseSingleOptionalStatement(OptStmtType StmtsType = OptStmtType::BasicStmt); + + // Top-level resource parsers. + ParseType parseLanguageResource(); + ParseType parseAcceleratorsResource(); + ParseType parseCursorResource(); + ParseType parseDialogResource(bool IsExtended); + ParseType parseIconResource(); + ParseType parseHTMLResource(); + ParseType parseMenuResource(); + ParseType parseStringTableResource(); + ParseType parseUserDefinedResource(IntOrString Type); + ParseType parseVersionInfoResource(); + + // Helper DIALOG parser - a single control. + Expected<Control> parseControl(); + + // Helper MENU parser. + Expected<MenuDefinitionList> parseMenuItemsList(); + + // Helper VERSIONINFO parser - read the contents of a single BLOCK statement, + // from BEGIN to END. + Expected<std::unique_ptr<VersionInfoBlock>> + parseVersionInfoBlockContents(StringRef BlockName); + // Helper VERSIONINFO parser - read either VALUE or BLOCK statement. + Expected<std::unique_ptr<VersionInfoStmt>> parseVersionInfoStmt(); + // Helper VERSIONINFO parser - read fixed VERSIONINFO statements. + Expected<VersionInfoResource::VersionInfoFixed> parseVersionInfoFixed(); + + // Optional statement parsers. + ParseOptionType parseLanguageStmt(); + ParseOptionType parseCharacteristicsStmt(); + ParseOptionType parseVersionStmt(); + ParseOptionType parseCaptionStmt(); + ParseOptionType parseFontStmt(OptStmtType DialogType); + ParseOptionType parseStyleStmt(); + + // Raises an error. If IsAlreadyRead = false (default), this complains about + // the token that couldn't be parsed. If the flag is on, this complains about + // the correctly read token that makes no sense (that is, the current parser + // state is beyond the erroneous token.) + Error getExpectedError(const Twine &Message, bool IsAlreadyRead = false); + + std::vector<RCToken> Tokens; + LocIter CurLoc; + const LocIter End; +}; + +} // namespace rc +} // namespace llvm + +#endif diff --git a/tools/llvm-rc/ResourceScriptStmt.cpp b/tools/llvm-rc/ResourceScriptStmt.cpp new file mode 100644 index 000000000000..42505cc76d0e --- /dev/null +++ b/tools/llvm-rc/ResourceScriptStmt.cpp @@ -0,0 +1,266 @@ +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This implements methods defined in ResourceScriptStmt.h. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa380599(v=vs.85).aspx +// +//===---------------------------------------------------------------------===// + +#include "ResourceScriptStmt.h" + +namespace llvm { +namespace rc { + +raw_ostream &operator<<(raw_ostream &OS, const IntOrString &Item) { + if (Item.IsInt) + return OS << Item.Data.Int; + else + return OS << Item.Data.String; +} + +raw_ostream &OptionalStmtList::log(raw_ostream &OS) const { + for (const auto &Stmt : Statements) { + OS << " Option: "; + Stmt->log(OS); + } + return OS; +} + +raw_ostream &LanguageResource::log(raw_ostream &OS) const { + return OS << "Language: " << Lang << ", Sublanguage: " << SubLang << "\n"; +} + +StringRef AcceleratorsResource::Accelerator::OptionsStr + [AcceleratorsResource::Accelerator::NumFlags] = { + "ASCII", "VIRTKEY", "NOINVERT", "ALT", "SHIFT", "CONTROL"}; + +uint32_t AcceleratorsResource::Accelerator::OptionsFlags + [AcceleratorsResource::Accelerator::NumFlags] = {ASCII, VIRTKEY, NOINVERT, + ALT, SHIFT, CONTROL}; + +raw_ostream &AcceleratorsResource::log(raw_ostream &OS) const { + OS << "Accelerators (" << ResName << "): \n"; + OptStatements->log(OS); + for (const auto &Acc : Accelerators) { + OS << " Accelerator: " << Acc.Event << " " << Acc.Id; + for (size_t i = 0; i < Accelerator::NumFlags; ++i) + if (Acc.Flags & Accelerator::OptionsFlags[i]) + OS << " " << Accelerator::OptionsStr[i]; + OS << "\n"; + } + return OS; +} + +raw_ostream &CursorResource::log(raw_ostream &OS) const { + return OS << "Cursor (" << ResName << "): " << CursorLoc << "\n"; +} + +raw_ostream &IconResource::log(raw_ostream &OS) const { + return OS << "Icon (" << ResName << "): " << IconLoc << "\n"; +} + +raw_ostream &HTMLResource::log(raw_ostream &OS) const { + return OS << "HTML (" << ResName << "): " << HTMLLoc << "\n"; +} + +StringRef MenuDefinition::OptionsStr[MenuDefinition::NumFlags] = { + "CHECKED", "GRAYED", "HELP", "INACTIVE", "MENUBARBREAK", "MENUBREAK"}; + +uint32_t MenuDefinition::OptionsFlags[MenuDefinition::NumFlags] = { + CHECKED, GRAYED, HELP, INACTIVE, MENUBARBREAK, MENUBREAK}; + +raw_ostream &MenuDefinition::logFlags(raw_ostream &OS, uint16_t Flags) { + for (size_t i = 0; i < NumFlags; ++i) + if (Flags & OptionsFlags[i]) + OS << " " << OptionsStr[i]; + return OS; +} + +raw_ostream &MenuDefinitionList::log(raw_ostream &OS) const { + OS << " Menu list starts\n"; + for (auto &Item : Definitions) + Item->log(OS); + return OS << " Menu list ends\n"; +} + +raw_ostream &MenuItem::log(raw_ostream &OS) const { + OS << " MenuItem (" << Name << "), ID = " << Id; + logFlags(OS, Flags); + return OS << "\n"; +} + +raw_ostream &MenuSeparator::log(raw_ostream &OS) const { + return OS << " Menu separator\n"; +} + +raw_ostream &PopupItem::log(raw_ostream &OS) const { + OS << " Popup (" << Name << ")"; + logFlags(OS, Flags); + OS << ":\n"; + return SubItems.log(OS); +} + +raw_ostream &MenuResource::log(raw_ostream &OS) const { + OS << "Menu (" << ResName << "):\n"; + OptStatements->log(OS); + return Elements.log(OS); +} + +raw_ostream &StringTableResource::log(raw_ostream &OS) const { + OS << "StringTable:\n"; + OptStatements->log(OS); + for (const auto &String : Table) + OS << " " << String.first << " => " << String.second << "\n"; + return OS; +} + +const StringMap<Control::CtlInfo> Control::SupportedCtls = { + {"LTEXT", CtlInfo{0x50020000, ClsStatic, true}}, + {"CTEXT", CtlInfo{0x50020001, ClsStatic, true}}, + {"RTEXT", CtlInfo{0x50020002, ClsStatic, true}}, + {"PUSHBUTTON", CtlInfo{0x50010000, ClsButton, true}}, + {"DEFPUSHBUTTON", CtlInfo{0x50010001, ClsButton, true}}, + {"EDITTEXT", CtlInfo{0x50810000, ClsEdit, false}}, +}; + +raw_ostream &Control::log(raw_ostream &OS) const { + OS << " Control (" << ID << "): " << Type << ", title: " << Title + << ", loc: (" << X << ", " << Y << "), size: [" << Width << ", " << Height + << "]"; + if (Style) + OS << ", style: " << *Style; + if (ExtStyle) + OS << ", ext. style: " << *ExtStyle; + if (HelpID) + OS << ", help ID: " << *HelpID; + return OS << "\n"; +} + +raw_ostream &DialogResource::log(raw_ostream &OS) const { + OS << "Dialog" << (IsExtended ? "Ex" : "") << " (" << ResName << "): loc: (" + << X << ", " << Y << "), size: [" << Width << ", " << Height + << "], help ID: " << HelpID << "\n"; + OptStatements->log(OS); + for (auto &Ctl : Controls) + Ctl.log(OS); + return OS; +} + +raw_ostream &VersionInfoBlock::log(raw_ostream &OS) const { + OS << " Start of block (name: " << Name << ")\n"; + for (auto &Stmt : Stmts) + Stmt->log(OS); + return OS << " End of block\n"; +} + +raw_ostream &VersionInfoValue::log(raw_ostream &OS) const { + OS << " " << Key << " =>"; + size_t NumValues = Values.size(); + for (size_t Id = 0; Id < NumValues; ++Id) { + if (Id > 0 && HasPrecedingComma[Id]) + OS << ","; + OS << " " << Values[Id]; + } + return OS << "\n"; +} + +using VersionInfoFixed = VersionInfoResource::VersionInfoFixed; +using VersionInfoFixedType = VersionInfoFixed::VersionInfoFixedType; + +const StringRef + VersionInfoFixed::FixedFieldsNames[VersionInfoFixed::FtNumTypes] = { + "", "FILEVERSION", "PRODUCTVERSION", "FILEFLAGSMASK", + "FILEFLAGS", "FILEOS", "FILETYPE", "FILESUBTYPE"}; + +const StringMap<VersionInfoFixedType> VersionInfoFixed::FixedFieldsInfoMap = { + {FixedFieldsNames[FtFileVersion], FtFileVersion}, + {FixedFieldsNames[FtProductVersion], FtProductVersion}, + {FixedFieldsNames[FtFileFlagsMask], FtFileFlagsMask}, + {FixedFieldsNames[FtFileFlags], FtFileFlags}, + {FixedFieldsNames[FtFileOS], FtFileOS}, + {FixedFieldsNames[FtFileType], FtFileType}, + {FixedFieldsNames[FtFileSubtype], FtFileSubtype}}; + +VersionInfoFixedType VersionInfoFixed::getFixedType(StringRef Type) { + auto UpperType = Type.upper(); + auto Iter = FixedFieldsInfoMap.find(UpperType); + if (Iter != FixedFieldsInfoMap.end()) + return Iter->getValue(); + return FtUnknown; +} + +bool VersionInfoFixed::isTypeSupported(VersionInfoFixedType Type) { + return FtUnknown < Type && Type < FtNumTypes; +} + +bool VersionInfoFixed::isVersionType(VersionInfoFixedType Type) { + switch (Type) { + case FtFileVersion: + case FtProductVersion: + return true; + + default: + return false; + } +} + +raw_ostream &VersionInfoFixed::log(raw_ostream &OS) const { + for (int Type = FtUnknown; Type < FtNumTypes; ++Type) { + if (!isTypeSupported((VersionInfoFixedType)Type)) + continue; + OS << " Fixed: " << FixedFieldsNames[Type] << ":"; + for (uint32_t Val : FixedInfo[Type]) + OS << " " << Val; + OS << "\n"; + } + return OS; +} + +raw_ostream &VersionInfoResource::log(raw_ostream &OS) const { + OS << "VersionInfo (" << ResName << "):\n"; + FixedData.log(OS); + return MainBlock.log(OS); +} + +raw_ostream &UserDefinedResource::log(raw_ostream &OS) const { + OS << "User-defined (type: " << Type << ", name: " << ResName << "): "; + if (IsFileResource) + return OS << FileLoc << "\n"; + OS << "data = "; + for (auto &Item : Contents) + OS << Item << " "; + return OS << "\n"; +} + +raw_ostream &CharacteristicsStmt::log(raw_ostream &OS) const { + return OS << "Characteristics: " << Value << "\n"; +} + +raw_ostream &VersionStmt::log(raw_ostream &OS) const { + return OS << "Version: " << Value << "\n"; +} + +raw_ostream &CaptionStmt::log(raw_ostream &OS) const { + return OS << "Caption: " << Value << "\n"; +} + +raw_ostream &FontStmt::log(raw_ostream &OS) const { + OS << "Font: size = " << Size << ", face = " << Name + << ", weight = " << Weight; + if (Italic) + OS << ", italic"; + return OS << ", charset = " << Charset << "\n"; +} + +raw_ostream &StyleStmt::log(raw_ostream &OS) const { + return OS << "Style: " << Value << "\n"; +} + +} // namespace rc +} // namespace llvm diff --git a/tools/llvm-rc/ResourceScriptStmt.h b/tools/llvm-rc/ResourceScriptStmt.h new file mode 100644 index 000000000000..e44120b770f3 --- /dev/null +++ b/tools/llvm-rc/ResourceScriptStmt.h @@ -0,0 +1,827 @@ +//===-- ResourceScriptStmt.h ------------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This lists all the resource and statement types occurring in RC scripts. +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVMRC_RESOURCESCRIPTSTMT_H +#define LLVM_TOOLS_LLVMRC_RESOURCESCRIPTSTMT_H + +#include "ResourceScriptToken.h" +#include "ResourceVisitor.h" + +#include "llvm/ADT/StringSet.h" + +namespace llvm { +namespace rc { + +// Integer wrapper that also holds information whether the user declared +// the integer to be long (by appending L to the end of the integer) or not. +// It allows to be implicitly cast from and to uint32_t in order +// to be compatible with the parts of code that don't care about the integers +// being marked long. +class RCInt { + uint32_t Val; + bool Long; + +public: + RCInt(const RCToken &Token) + : Val(Token.intValue()), Long(Token.isLongInt()) {} + RCInt(uint32_t Value) : Val(Value), Long(false) {} + RCInt(uint32_t Value, bool IsLong) : Val(Value), Long(IsLong) {} + operator uint32_t() const { return Val; } + bool isLong() const { return Long; } + + RCInt &operator+=(const RCInt &Rhs) { + std::tie(Val, Long) = std::make_pair(Val + Rhs.Val, Long | Rhs.Long); + return *this; + } + + RCInt &operator-=(const RCInt &Rhs) { + std::tie(Val, Long) = std::make_pair(Val - Rhs.Val, Long | Rhs.Long); + return *this; + } + + RCInt &operator|=(const RCInt &Rhs) { + std::tie(Val, Long) = std::make_pair(Val | Rhs.Val, Long | Rhs.Long); + return *this; + } + + RCInt &operator&=(const RCInt &Rhs) { + std::tie(Val, Long) = std::make_pair(Val & Rhs.Val, Long | Rhs.Long); + return *this; + } + + RCInt operator-() const { return {-Val, Long}; } + RCInt operator~() const { return {~Val, Long}; } + + friend raw_ostream &operator<<(raw_ostream &OS, const RCInt &Int) { + return OS << Int.Val << (Int.Long ? "L" : ""); + } +}; + +// A class holding a name - either an integer or a reference to the string. +class IntOrString { +private: + union Data { + RCInt Int; + StringRef String; + Data(RCInt Value) : Int(Value) {} + Data(const StringRef Value) : String(Value) {} + Data(const RCToken &Token) { + if (Token.kind() == RCToken::Kind::Int) + Int = RCInt(Token); + else + String = Token.value(); + } + } Data; + bool IsInt; + +public: + IntOrString() : IntOrString(RCInt(0)) {} + IntOrString(uint32_t Value) : Data(Value), IsInt(1) {} + IntOrString(RCInt Value) : Data(Value), IsInt(1) {} + IntOrString(StringRef Value) : Data(Value), IsInt(0) {} + IntOrString(const RCToken &Token) + : Data(Token), IsInt(Token.kind() == RCToken::Kind::Int) {} + + bool equalsLower(const char *Str) { + return !IsInt && Data.String.equals_lower(Str); + } + + bool isInt() const { return IsInt; } + + RCInt getInt() const { + assert(IsInt); + return Data.Int; + } + + const StringRef &getString() const { + assert(!IsInt); + return Data.String; + } + + operator Twine() const { + return isInt() ? Twine(getInt()) : Twine(getString()); + } + + friend raw_ostream &operator<<(raw_ostream &, const IntOrString &); +}; + +enum ResourceKind { + // These resource kinds have corresponding .res resource type IDs + // (TYPE in RESOURCEHEADER structure). The numeric value assigned to each + // kind is equal to this type ID. + RkNull = 0, + RkSingleCursor = 1, + RkSingleIcon = 3, + RkMenu = 4, + RkDialog = 5, + RkStringTableBundle = 6, + RkAccelerators = 9, + RkCursorGroup = 12, + RkIconGroup = 14, + RkVersionInfo = 16, + RkHTML = 23, + + // These kinds don't have assigned type IDs (they might be the resources + // of invalid kind, expand to many resource structures in .res files, + // or have variable type ID). In order to avoid ID clashes with IDs above, + // we assign the kinds the values 256 and larger. + RkInvalid = 256, + RkBase, + RkCursor, + RkIcon, + RkStringTable, + RkUser, + RkSingleCursorOrIconRes, + RkCursorOrIconGroupRes, +}; + +// Non-zero memory flags. +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms648027(v=vs.85).aspx +enum MemoryFlags { + MfMoveable = 0x10, + MfPure = 0x20, + MfPreload = 0x40, + MfDiscardable = 0x1000 +}; + +// Base resource. All the resources should derive from this base. +class RCResource { +public: + IntOrString ResName; + void setName(const IntOrString &Name) { ResName = Name; } + virtual raw_ostream &log(raw_ostream &OS) const { + return OS << "Base statement\n"; + }; + virtual ~RCResource() {} + + virtual Error visit(Visitor *) const { + llvm_unreachable("This is unable to call methods from Visitor base"); + } + + // Apply the statements attached to this resource. Generic resources + // don't have any. + virtual Error applyStmts(Visitor *) const { return Error::success(); } + + // By default, memory flags are DISCARDABLE | PURE | MOVEABLE. + virtual uint16_t getMemoryFlags() const { + return MfDiscardable | MfPure | MfMoveable; + } + virtual ResourceKind getKind() const { return RkBase; } + static bool classof(const RCResource *Res) { return true; } + + virtual IntOrString getResourceType() const { + llvm_unreachable("This cannot be called on objects without types."); + } + virtual Twine getResourceTypeName() const { + llvm_unreachable("This cannot be called on objects without types."); + }; +}; + +// An empty resource. It has no content, type 0, ID 0 and all of its +// characteristics are equal to 0. +class NullResource : public RCResource { +public: + raw_ostream &log(raw_ostream &OS) const override { + return OS << "Null resource\n"; + } + Error visit(Visitor *V) const override { return V->visitNullResource(this); } + IntOrString getResourceType() const override { return 0; } + Twine getResourceTypeName() const override { return "(NULL)"; } + uint16_t getMemoryFlags() const override { return 0; } +}; + +// Optional statement base. All such statements should derive from this base. +class OptionalStmt : public RCResource {}; + +class OptionalStmtList : public OptionalStmt { + std::vector<std::unique_ptr<OptionalStmt>> Statements; + +public: + OptionalStmtList() {} + raw_ostream &log(raw_ostream &OS) const override; + + void addStmt(std::unique_ptr<OptionalStmt> Stmt) { + Statements.push_back(std::move(Stmt)); + } + + Error visit(Visitor *V) const override { + for (auto &StmtPtr : Statements) + if (auto Err = StmtPtr->visit(V)) + return Err; + return Error::success(); + } +}; + +class OptStatementsRCResource : public RCResource { +public: + std::unique_ptr<OptionalStmtList> OptStatements; + + OptStatementsRCResource(OptionalStmtList &&Stmts) + : OptStatements(llvm::make_unique<OptionalStmtList>(std::move(Stmts))) {} + + virtual Error applyStmts(Visitor *V) const { return OptStatements->visit(V); } +}; + +// LANGUAGE statement. It can occur both as a top-level statement (in such +// a situation, it changes the default language until the end of the file) +// and as an optional resource statement (then it changes the language +// of a single resource). +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381019(v=vs.85).aspx +class LanguageResource : public OptionalStmt { +public: + uint32_t Lang, SubLang; + + LanguageResource(uint32_t LangId, uint32_t SubLangId) + : Lang(LangId), SubLang(SubLangId) {} + raw_ostream &log(raw_ostream &) const override; + + // This is not a regular top-level statement; when it occurs, it just + // modifies the language context. + Error visit(Visitor *V) const override { return V->visitLanguageStmt(this); } + Twine getResourceTypeName() const override { return "LANGUAGE"; } +}; + +// ACCELERATORS resource. Defines a named table of accelerators for the app. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa380610(v=vs.85).aspx +class AcceleratorsResource : public OptStatementsRCResource { +public: + class Accelerator { + public: + IntOrString Event; + uint32_t Id; + uint16_t Flags; + + enum Options { + // This is actually 0x0000 (accelerator is assumed to be ASCII if it's + // not VIRTKEY). However, rc.exe behavior is different in situations + // "only ASCII defined" and "neither ASCII nor VIRTKEY defined". + // Therefore, we include ASCII as another flag. This must be zeroed + // when serialized. + ASCII = 0x8000, + VIRTKEY = 0x0001, + NOINVERT = 0x0002, + ALT = 0x0010, + SHIFT = 0x0004, + CONTROL = 0x0008 + }; + + static constexpr size_t NumFlags = 6; + static StringRef OptionsStr[NumFlags]; + static uint32_t OptionsFlags[NumFlags]; + }; + + std::vector<Accelerator> Accelerators; + + using OptStatementsRCResource::OptStatementsRCResource; + void addAccelerator(IntOrString Event, uint32_t Id, uint16_t Flags) { + Accelerators.push_back(Accelerator{Event, Id, Flags}); + } + raw_ostream &log(raw_ostream &) const override; + + IntOrString getResourceType() const override { return RkAccelerators; } + uint16_t getMemoryFlags() const override { + return MfPure | MfMoveable; + } + Twine getResourceTypeName() const override { return "ACCELERATORS"; } + + Error visit(Visitor *V) const override { + return V->visitAcceleratorsResource(this); + } + ResourceKind getKind() const override { return RkAccelerators; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkAccelerators; + } +}; + +// CURSOR resource. Represents a single cursor (".cur") file. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa380920(v=vs.85).aspx +class CursorResource : public RCResource { +public: + StringRef CursorLoc; + + CursorResource(StringRef Location) : CursorLoc(Location) {} + raw_ostream &log(raw_ostream &) const override; + + Twine getResourceTypeName() const override { return "CURSOR"; } + Error visit(Visitor *V) const override { + return V->visitCursorResource(this); + } + ResourceKind getKind() const override { return RkCursor; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkCursor; + } +}; + +// ICON resource. Represents a single ".ico" file containing a group of icons. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381018(v=vs.85).aspx +class IconResource : public RCResource { +public: + StringRef IconLoc; + + IconResource(StringRef Location) : IconLoc(Location) {} + raw_ostream &log(raw_ostream &) const override; + + Twine getResourceTypeName() const override { return "ICON"; } + Error visit(Visitor *V) const override { return V->visitIconResource(this); } + ResourceKind getKind() const override { return RkIcon; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkIcon; + } +}; + +// HTML resource. Represents a local webpage that is to be embedded into the +// resulting resource file. It embeds a file only - no additional resources +// (images etc.) are included with this resource. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa966018(v=vs.85).aspx +class HTMLResource : public RCResource { +public: + StringRef HTMLLoc; + + HTMLResource(StringRef Location) : HTMLLoc(Location) {} + raw_ostream &log(raw_ostream &) const override; + + Error visit(Visitor *V) const override { return V->visitHTMLResource(this); } + + // Curiously, file resources don't have DISCARDABLE flag set. + uint16_t getMemoryFlags() const override { return MfPure | MfMoveable; } + IntOrString getResourceType() const override { return RkHTML; } + Twine getResourceTypeName() const override { return "HTML"; } + ResourceKind getKind() const override { return RkHTML; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkHTML; + } +}; + +// -- MENU resource and its helper classes -- +// This resource describes the contents of an application menu +// (usually located in the upper part of the dialog.) +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381025(v=vs.85).aspx + +// Description of a single submenu item. +class MenuDefinition { +public: + enum Options { + CHECKED = 0x0008, + GRAYED = 0x0001, + HELP = 0x4000, + INACTIVE = 0x0002, + MENUBARBREAK = 0x0020, + MENUBREAK = 0x0040 + }; + + enum MenuDefKind { MkBase, MkSeparator, MkMenuItem, MkPopup }; + + static constexpr size_t NumFlags = 6; + static StringRef OptionsStr[NumFlags]; + static uint32_t OptionsFlags[NumFlags]; + static raw_ostream &logFlags(raw_ostream &, uint16_t Flags); + virtual raw_ostream &log(raw_ostream &OS) const { + return OS << "Base menu definition\n"; + } + virtual ~MenuDefinition() {} + + virtual uint16_t getResFlags() const { return 0; } + virtual MenuDefKind getKind() const { return MkBase; } +}; + +// Recursive description of a whole submenu. +class MenuDefinitionList : public MenuDefinition { +public: + std::vector<std::unique_ptr<MenuDefinition>> Definitions; + + void addDefinition(std::unique_ptr<MenuDefinition> Def) { + Definitions.push_back(std::move(Def)); + } + raw_ostream &log(raw_ostream &) const override; +}; + +// Separator in MENU definition (MENUITEM SEPARATOR). +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381024(v=vs.85).aspx +class MenuSeparator : public MenuDefinition { +public: + raw_ostream &log(raw_ostream &) const override; + + MenuDefKind getKind() const override { return MkSeparator; } + static bool classof(const MenuDefinition *D) { + return D->getKind() == MkSeparator; + } +}; + +// MENUITEM statement definition. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381024(v=vs.85).aspx +class MenuItem : public MenuDefinition { +public: + StringRef Name; + uint32_t Id; + uint16_t Flags; + + MenuItem(StringRef Caption, uint32_t ItemId, uint16_t ItemFlags) + : Name(Caption), Id(ItemId), Flags(ItemFlags) {} + raw_ostream &log(raw_ostream &) const override; + + uint16_t getResFlags() const override { return Flags; } + MenuDefKind getKind() const override { return MkMenuItem; } + static bool classof(const MenuDefinition *D) { + return D->getKind() == MkMenuItem; + } +}; + +// POPUP statement definition. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381030(v=vs.85).aspx +class PopupItem : public MenuDefinition { +public: + StringRef Name; + uint16_t Flags; + MenuDefinitionList SubItems; + + PopupItem(StringRef Caption, uint16_t ItemFlags, + MenuDefinitionList &&SubItemsList) + : Name(Caption), Flags(ItemFlags), SubItems(std::move(SubItemsList)) {} + raw_ostream &log(raw_ostream &) const override; + + // This has an additional (0x10) flag. It doesn't match with documented + // 0x01 flag, though. + uint16_t getResFlags() const override { return Flags | 0x10; } + MenuDefKind getKind() const override { return MkPopup; } + static bool classof(const MenuDefinition *D) { + return D->getKind() == MkPopup; + } +}; + +// Menu resource definition. +class MenuResource : public OptStatementsRCResource { +public: + MenuDefinitionList Elements; + + MenuResource(OptionalStmtList &&OptStmts, MenuDefinitionList &&Items) + : OptStatementsRCResource(std::move(OptStmts)), + Elements(std::move(Items)) {} + raw_ostream &log(raw_ostream &) const override; + + IntOrString getResourceType() const override { return RkMenu; } + Twine getResourceTypeName() const override { return "MENU"; } + Error visit(Visitor *V) const override { return V->visitMenuResource(this); } + ResourceKind getKind() const override { return RkMenu; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkMenu; + } +}; + +// STRINGTABLE resource. Contains a list of strings, each having its unique ID. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381050(v=vs.85).aspx +class StringTableResource : public OptStatementsRCResource { +public: + std::vector<std::pair<uint32_t, StringRef>> Table; + + using OptStatementsRCResource::OptStatementsRCResource; + void addString(uint32_t ID, StringRef String) { + Table.emplace_back(ID, String); + } + raw_ostream &log(raw_ostream &) const override; + Twine getResourceTypeName() const override { return "STRINGTABLE"; } + Error visit(Visitor *V) const override { + return V->visitStringTableResource(this); + } +}; + +// -- DIALOG(EX) resource and its helper classes -- +// +// This resource describes dialog boxes and controls residing inside them. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381003(v=vs.85).aspx +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381002(v=vs.85).aspx + +// Single control definition. +class Control { +public: + StringRef Type; + IntOrString Title; + uint32_t ID, X, Y, Width, Height; + Optional<uint32_t> Style, ExtStyle, HelpID; + + // Control classes as described in DLGITEMTEMPLATEEX documentation. + // + // Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms645389.aspx + enum CtlClasses { + ClsButton = 0x80, + ClsEdit = 0x81, + ClsStatic = 0x82, + ClsListBox = 0x83, + ClsScrollBar = 0x84, + ClsComboBox = 0x85 + }; + + // Simple information about a single control type. + struct CtlInfo { + uint32_t Style; + uint16_t CtlClass; + bool HasTitle; + }; + + Control(StringRef CtlType, IntOrString CtlTitle, uint32_t CtlID, + uint32_t PosX, uint32_t PosY, uint32_t ItemWidth, uint32_t ItemHeight, + Optional<uint32_t> ItemStyle, Optional<uint32_t> ExtItemStyle, + Optional<uint32_t> CtlHelpID) + : Type(CtlType), Title(CtlTitle), ID(CtlID), X(PosX), Y(PosY), + Width(ItemWidth), Height(ItemHeight), Style(ItemStyle), + ExtStyle(ExtItemStyle), HelpID(CtlHelpID) {} + + static const StringMap<CtlInfo> SupportedCtls; + + raw_ostream &log(raw_ostream &) const; +}; + +// Single dialog definition. We don't create distinct classes for DIALOG and +// DIALOGEX because of their being too similar to each other. We only have a +// flag determining the type of the dialog box. +class DialogResource : public OptStatementsRCResource { +public: + uint32_t X, Y, Width, Height, HelpID; + std::vector<Control> Controls; + bool IsExtended; + + DialogResource(uint32_t PosX, uint32_t PosY, uint32_t DlgWidth, + uint32_t DlgHeight, uint32_t DlgHelpID, + OptionalStmtList &&OptStmts, bool IsDialogEx) + : OptStatementsRCResource(std::move(OptStmts)), X(PosX), Y(PosY), + Width(DlgWidth), Height(DlgHeight), HelpID(DlgHelpID), + IsExtended(IsDialogEx) {} + + void addControl(Control &&Ctl) { Controls.push_back(std::move(Ctl)); } + + raw_ostream &log(raw_ostream &) const override; + + // It was a weird design decision to assign the same resource type number + // both for DIALOG and DIALOGEX (and the same structure version number). + // It makes it possible for DIALOG to be mistaken for DIALOGEX. + IntOrString getResourceType() const override { return RkDialog; } + Twine getResourceTypeName() const override { + return "DIALOG" + Twine(IsExtended ? "EX" : ""); + } + Error visit(Visitor *V) const override { + return V->visitDialogResource(this); + } + ResourceKind getKind() const override { return RkDialog; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkDialog; + } +}; + +// User-defined resource. It is either: +// * a link to the file, e.g. NAME TYPE "filename", +// * or contains a list of integers and strings, e.g. NAME TYPE {1, "a", 2}. +class UserDefinedResource : public RCResource { +public: + IntOrString Type; + StringRef FileLoc; + std::vector<IntOrString> Contents; + bool IsFileResource; + + UserDefinedResource(IntOrString ResourceType, StringRef FileLocation) + : Type(ResourceType), FileLoc(FileLocation), IsFileResource(true) {} + UserDefinedResource(IntOrString ResourceType, std::vector<IntOrString> &&Data) + : Type(ResourceType), Contents(std::move(Data)), IsFileResource(false) {} + + raw_ostream &log(raw_ostream &) const override; + IntOrString getResourceType() const override { return Type; } + Twine getResourceTypeName() const override { return Type; } + uint16_t getMemoryFlags() const override { return MfPure | MfMoveable; } + + Error visit(Visitor *V) const override { + return V->visitUserDefinedResource(this); + } + ResourceKind getKind() const override { return RkUser; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkUser; + } +}; + +// -- VERSIONINFO resource and its helper classes -- +// +// This resource lists the version information on the executable/library. +// The declaration consists of the following items: +// * A number of fixed optional version statements (e.g. FILEVERSION, FILEOS) +// * BEGIN +// * A number of BLOCK and/or VALUE statements. BLOCK recursively defines +// another block of version information, whereas VALUE defines a +// key -> value correspondence. There might be more than one value +// corresponding to the single key. +// * END +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381058(v=vs.85).aspx + +// A single VERSIONINFO statement; +class VersionInfoStmt { +public: + enum StmtKind { StBase = 0, StBlock = 1, StValue = 2 }; + + virtual raw_ostream &log(raw_ostream &OS) const { return OS << "VI stmt\n"; } + virtual ~VersionInfoStmt() {} + + virtual StmtKind getKind() const { return StBase; } + static bool classof(const VersionInfoStmt *S) { + return S->getKind() == StBase; + } +}; + +// BLOCK definition; also the main VERSIONINFO declaration is considered a +// BLOCK, although it has no name. +// The correct top-level blocks are "VarFileInfo" and "StringFileInfo". We don't +// care about them at the parsing phase. +class VersionInfoBlock : public VersionInfoStmt { +public: + std::vector<std::unique_ptr<VersionInfoStmt>> Stmts; + StringRef Name; + + VersionInfoBlock(StringRef BlockName) : Name(BlockName) {} + void addStmt(std::unique_ptr<VersionInfoStmt> Stmt) { + Stmts.push_back(std::move(Stmt)); + } + raw_ostream &log(raw_ostream &) const override; + + StmtKind getKind() const override { return StBlock; } + static bool classof(const VersionInfoStmt *S) { + return S->getKind() == StBlock; + } +}; + +class VersionInfoValue : public VersionInfoStmt { +public: + StringRef Key; + std::vector<IntOrString> Values; + std::vector<bool> HasPrecedingComma; + + VersionInfoValue(StringRef InfoKey, std::vector<IntOrString> &&Vals, + std::vector<bool> &&CommasBeforeVals) + : Key(InfoKey), Values(std::move(Vals)), + HasPrecedingComma(std::move(CommasBeforeVals)) {} + raw_ostream &log(raw_ostream &) const override; + + StmtKind getKind() const override { return StValue; } + static bool classof(const VersionInfoStmt *S) { + return S->getKind() == StValue; + } +}; + +class VersionInfoResource : public RCResource { +public: + // A class listing fixed VERSIONINFO statements (occuring before main BEGIN). + // If any of these is not specified, it is assumed by the original tool to + // be equal to 0. + class VersionInfoFixed { + public: + enum VersionInfoFixedType { + FtUnknown, + FtFileVersion, + FtProductVersion, + FtFileFlagsMask, + FtFileFlags, + FtFileOS, + FtFileType, + FtFileSubtype, + FtNumTypes + }; + + private: + static const StringMap<VersionInfoFixedType> FixedFieldsInfoMap; + static const StringRef FixedFieldsNames[FtNumTypes]; + + public: + SmallVector<uint32_t, 4> FixedInfo[FtNumTypes]; + SmallVector<bool, FtNumTypes> IsTypePresent; + + static VersionInfoFixedType getFixedType(StringRef Type); + static bool isTypeSupported(VersionInfoFixedType Type); + static bool isVersionType(VersionInfoFixedType Type); + + VersionInfoFixed() : IsTypePresent(FtNumTypes, false) {} + + void setValue(VersionInfoFixedType Type, ArrayRef<uint32_t> Value) { + FixedInfo[Type] = SmallVector<uint32_t, 4>(Value.begin(), Value.end()); + IsTypePresent[Type] = true; + } + + raw_ostream &log(raw_ostream &) const; + }; + + VersionInfoBlock MainBlock; + VersionInfoFixed FixedData; + + VersionInfoResource(VersionInfoBlock &&TopLevelBlock, + VersionInfoFixed &&FixedInfo) + : MainBlock(std::move(TopLevelBlock)), FixedData(std::move(FixedInfo)) {} + + raw_ostream &log(raw_ostream &) const override; + IntOrString getResourceType() const override { return RkVersionInfo; } + uint16_t getMemoryFlags() const override { return MfMoveable | MfPure; } + Twine getResourceTypeName() const override { return "VERSIONINFO"; } + Error visit(Visitor *V) const override { + return V->visitVersionInfoResource(this); + } + ResourceKind getKind() const override { return RkVersionInfo; } + static bool classof(const RCResource *Res) { + return Res->getKind() == RkVersionInfo; + } +}; + +// CHARACTERISTICS optional statement. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa380872(v=vs.85).aspx +class CharacteristicsStmt : public OptionalStmt { +public: + uint32_t Value; + + CharacteristicsStmt(uint32_t Characteristic) : Value(Characteristic) {} + raw_ostream &log(raw_ostream &) const override; + + Twine getResourceTypeName() const override { return "CHARACTERISTICS"; } + Error visit(Visitor *V) const override { + return V->visitCharacteristicsStmt(this); + } +}; + +// VERSION optional statement. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381059(v=vs.85).aspx +class VersionStmt : public OptionalStmt { +public: + uint32_t Value; + + VersionStmt(uint32_t Version) : Value(Version) {} + raw_ostream &log(raw_ostream &) const override; + + Twine getResourceTypeName() const override { return "VERSION"; } + Error visit(Visitor *V) const override { return V->visitVersionStmt(this); } +}; + +// CAPTION optional statement. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa380778(v=vs.85).aspx +class CaptionStmt : public OptionalStmt { +public: + StringRef Value; + + CaptionStmt(StringRef Caption) : Value(Caption) {} + raw_ostream &log(raw_ostream &) const override; + Twine getResourceTypeName() const override { return "CAPTION"; } + Error visit(Visitor *V) const override { return V->visitCaptionStmt(this); } +}; + +// FONT optional statement. +// Note that the documentation is inaccurate: it expects five arguments to be +// given, however the example provides only two. In fact, the original tool +// expects two arguments - point size and name of the typeface. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381013(v=vs.85).aspx +class FontStmt : public OptionalStmt { +public: + uint32_t Size, Weight, Charset; + StringRef Name; + bool Italic; + + FontStmt(uint32_t FontSize, StringRef FontName, uint32_t FontWeight, + bool FontItalic, uint32_t FontCharset) + : Size(FontSize), Weight(FontWeight), Charset(FontCharset), + Name(FontName), Italic(FontItalic) {} + raw_ostream &log(raw_ostream &) const override; + Twine getResourceTypeName() const override { return "FONT"; } + Error visit(Visitor *V) const override { return V->visitFontStmt(this); } +}; + +// STYLE optional statement. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381051(v=vs.85).aspx +class StyleStmt : public OptionalStmt { +public: + uint32_t Value; + + StyleStmt(uint32_t Style) : Value(Style) {} + raw_ostream &log(raw_ostream &) const override; + Twine getResourceTypeName() const override { return "STYLE"; } + Error visit(Visitor *V) const override { return V->visitStyleStmt(this); } +}; + +} // namespace rc +} // namespace llvm + +#endif diff --git a/tools/llvm-rc/ResourceScriptToken.cpp b/tools/llvm-rc/ResourceScriptToken.cpp new file mode 100644 index 000000000000..7bbf0d16f044 --- /dev/null +++ b/tools/llvm-rc/ResourceScriptToken.cpp @@ -0,0 +1,366 @@ +//===-- ResourceScriptToken.cpp ---------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file implements an interface defined in ResourceScriptToken.h. +// In particular, it defines an .rc script tokenizer. +// +//===---------------------------------------------------------------------===// + +#include "ResourceScriptToken.h" +#include "llvm/Support/raw_ostream.h" + +#include <algorithm> +#include <cassert> +#include <cctype> +#include <cstdlib> +#include <utility> + +using namespace llvm; + +using Kind = RCToken::Kind; + +// Checks if Representation is a correct description of an RC integer. +// It should be a 32-bit unsigned integer, either decimal, octal (0[0-7]+), +// or hexadecimal (0x[0-9a-f]+). It might be followed by a single 'L' +// character (that is the difference between our representation and +// StringRef's one). If Representation is correct, 'true' is returned and +// the return value is put back in Num. +static bool rcGetAsInteger(StringRef Representation, uint32_t &Num) { + size_t Length = Representation.size(); + if (Length == 0) + return false; + // Strip the last 'L' if unnecessary. + if (std::toupper(Representation.back()) == 'L') + Representation = Representation.drop_back(1); + + return !Representation.getAsInteger<uint32_t>(0, Num); +} + +RCToken::RCToken(RCToken::Kind RCTokenKind, StringRef Value) + : TokenKind(RCTokenKind), TokenValue(Value) {} + +uint32_t RCToken::intValue() const { + assert(TokenKind == Kind::Int); + // We assume that the token already is a correct integer (checked by + // rcGetAsInteger). + uint32_t Result; + bool IsSuccess = rcGetAsInteger(TokenValue, Result); + assert(IsSuccess); + (void)IsSuccess; // Silence the compiler warning when -DNDEBUG flag is on. + return Result; +} + +bool RCToken::isLongInt() const { + return TokenKind == Kind::Int && std::toupper(TokenValue.back()) == 'L'; +} + +StringRef RCToken::value() const { return TokenValue; } + +Kind RCToken::kind() const { return TokenKind; } + +bool RCToken::isBinaryOp() const { + switch (TokenKind) { + case Kind::Plus: + case Kind::Minus: + case Kind::Pipe: + case Kind::Amp: + return true; + default: + return false; + } +} + +static Error getStringError(const Twine &message) { + return make_error<StringError>("Error parsing file: " + message, + inconvertibleErrorCode()); +} + +namespace { + +class Tokenizer { +public: + Tokenizer(StringRef Input) : Data(Input), DataLength(Input.size()) {} + + Expected<std::vector<RCToken>> run(); + +private: + // All 'advancing' methods return boolean values; if they're equal to false, + // the stream has ended or failed. + bool advance(size_t Amount = 1); + bool skipWhitespaces(); + + // Consumes a token. If any problem occurred, a non-empty Error is returned. + Error consumeToken(const Kind TokenKind); + + // Check if tokenizer is about to read FollowingChars. + bool willNowRead(StringRef FollowingChars) const; + + // Check if tokenizer can start reading an identifier at current position. + // The original tool did non specify the rules to determine what is a correct + // identifier. We assume they should follow the C convention: + // [a-zA-Z_][a-zA-Z0-9_]*. + bool canStartIdentifier() const; + // Check if tokenizer can continue reading an identifier. + bool canContinueIdentifier() const; + + // Check if tokenizer can start reading an integer. + // A correct integer always starts with a 0-9 digit, + // can contain characters 0-9A-Fa-f (digits), + // Ll (marking the integer is 32-bit), Xx (marking the representation + // is hexadecimal). As some kind of separator should come after the + // integer, we can consume the integer until a non-alphanumeric + // character. + bool canStartInt() const; + bool canContinueInt() const; + + bool canStartString() const; + + // Check if tokenizer can start reading a single line comment (e.g. a comment + // that begins with '//') + bool canStartLineComment() const; + + // Check if tokenizer can start or finish reading a block comment (e.g. a + // comment that begins with '/*' and ends with '*/') + bool canStartBlockComment() const; + + // Throw away all remaining characters on the current line. + void skipCurrentLine(); + + bool streamEof() const; + + // Classify the token that is about to be read from the current position. + Kind classifyCurrentToken() const; + + // Process the Kind::Identifier token - check if it is + // an identifier describing a block start or end. + void processIdentifier(RCToken &token) const; + + StringRef Data; + size_t DataLength, Pos; +}; + +void Tokenizer::skipCurrentLine() { + Pos = Data.find_first_of("\r\n", Pos); + Pos = Data.find_first_not_of("\r\n", Pos); + + if (Pos == StringRef::npos) + Pos = DataLength; +} + +Expected<std::vector<RCToken>> Tokenizer::run() { + Pos = 0; + std::vector<RCToken> Result; + + // Consume an optional UTF-8 Byte Order Mark. + if (willNowRead("\xef\xbb\xbf")) + advance(3); + + while (!streamEof()) { + if (!skipWhitespaces()) + break; + + Kind TokenKind = classifyCurrentToken(); + if (TokenKind == Kind::Invalid) + return getStringError("Invalid token found at position " + Twine(Pos)); + + const size_t TokenStart = Pos; + if (Error TokenError = consumeToken(TokenKind)) + return std::move(TokenError); + + // Comments are just deleted, don't bother saving them. + if (TokenKind == Kind::LineComment || TokenKind == Kind::StartComment) + continue; + + RCToken Token(TokenKind, Data.take_front(Pos).drop_front(TokenStart)); + if (TokenKind == Kind::Identifier) { + processIdentifier(Token); + } else if (TokenKind == Kind::Int) { + uint32_t TokenInt; + if (!rcGetAsInteger(Token.value(), TokenInt)) { + // The integer has incorrect format or cannot be represented in + // a 32-bit integer. + return getStringError("Integer invalid or too large: " + + Token.value().str()); + } + } + + Result.push_back(Token); + } + + return Result; +} + +bool Tokenizer::advance(size_t Amount) { + Pos += Amount; + return !streamEof(); +} + +bool Tokenizer::skipWhitespaces() { + while (!streamEof() && std::isspace(Data[Pos])) + advance(); + return !streamEof(); +} + +Error Tokenizer::consumeToken(const Kind TokenKind) { + switch (TokenKind) { + // One-character token consumption. +#define TOKEN(Name) +#define SHORT_TOKEN(Name, Ch) case Kind::Name: +#include "ResourceScriptTokenList.def" + advance(); + return Error::success(); + + case Kind::LineComment: + advance(2); + skipCurrentLine(); + return Error::success(); + + case Kind::StartComment: { + advance(2); + auto EndPos = Data.find("*/", Pos); + if (EndPos == StringRef::npos) + return getStringError( + "Unclosed multi-line comment beginning at position " + Twine(Pos)); + advance(EndPos - Pos); + advance(2); + return Error::success(); + } + case Kind::Identifier: + while (!streamEof() && canContinueIdentifier()) + advance(); + return Error::success(); + + case Kind::Int: + while (!streamEof() && canContinueInt()) + advance(); + return Error::success(); + + case Kind::String: + // Consume the preceding 'L', if there is any. + if (std::toupper(Data[Pos]) == 'L') + advance(); + // Consume the double-quote. + advance(); + + // Consume the characters until the end of the file, line or string. + while (true) { + if (streamEof()) { + return getStringError("Unterminated string literal."); + } else if (Data[Pos] == '"') { + // Consume the ending double-quote. + advance(); + // However, if another '"' follows this double-quote, the string didn't + // end and we just included '"' into the string. + if (!willNowRead("\"")) + return Error::success(); + } else if (Data[Pos] == '\n') { + return getStringError("String literal not terminated in the line."); + } + + advance(); + } + + case Kind::Invalid: + assert(false && "Cannot consume an invalid token."); + } + + llvm_unreachable("Unknown RCToken::Kind"); +} + +bool Tokenizer::willNowRead(StringRef FollowingChars) const { + return Data.drop_front(Pos).startswith(FollowingChars); +} + +bool Tokenizer::canStartIdentifier() const { + assert(!streamEof()); + + const char CurChar = Data[Pos]; + return std::isalpha(CurChar) || CurChar == '_'; +} + +bool Tokenizer::canContinueIdentifier() const { + assert(!streamEof()); + const char CurChar = Data[Pos]; + return std::isalnum(CurChar) || CurChar == '_'; +} + +bool Tokenizer::canStartInt() const { + assert(!streamEof()); + return std::isdigit(Data[Pos]); +} + +bool Tokenizer::canStartBlockComment() const { + assert(!streamEof()); + return Data.drop_front(Pos).startswith("/*"); +} + +bool Tokenizer::canStartLineComment() const { + assert(!streamEof()); + return Data.drop_front(Pos).startswith("//"); +} + +bool Tokenizer::canContinueInt() const { + assert(!streamEof()); + return std::isalnum(Data[Pos]); +} + +bool Tokenizer::canStartString() const { + return willNowRead("\"") || willNowRead("L\"") || willNowRead("l\""); +} + +bool Tokenizer::streamEof() const { return Pos == DataLength; } + +Kind Tokenizer::classifyCurrentToken() const { + if (canStartBlockComment()) + return Kind::StartComment; + if (canStartLineComment()) + return Kind::LineComment; + + if (canStartInt()) + return Kind::Int; + if (canStartString()) + return Kind::String; + // BEGIN and END are at this point of lexing recognized as identifiers. + if (canStartIdentifier()) + return Kind::Identifier; + + const char CurChar = Data[Pos]; + + switch (CurChar) { + // One-character token classification. +#define TOKEN(Name) +#define SHORT_TOKEN(Name, Ch) \ + case Ch: \ + return Kind::Name; +#include "ResourceScriptTokenList.def" + + default: + return Kind::Invalid; + } +} + +void Tokenizer::processIdentifier(RCToken &Token) const { + assert(Token.kind() == Kind::Identifier); + StringRef Name = Token.value(); + + if (Name.equals_lower("begin")) + Token = RCToken(Kind::BlockBegin, Name); + else if (Name.equals_lower("end")) + Token = RCToken(Kind::BlockEnd, Name); +} + +} // anonymous namespace + +namespace llvm { + +Expected<std::vector<RCToken>> tokenizeRC(StringRef Input) { + return Tokenizer(Input).run(); +} + +} // namespace llvm diff --git a/tools/llvm-rc/ResourceScriptToken.h b/tools/llvm-rc/ResourceScriptToken.h new file mode 100644 index 000000000000..0f108b50ed10 --- /dev/null +++ b/tools/llvm-rc/ResourceScriptToken.h @@ -0,0 +1,83 @@ +//===-- ResourceScriptToken.h -----------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This declares the .rc script tokens and defines an interface for tokenizing +// the input data. The list of available tokens is located at +// ResourceScriptTokenList.def. +// +// Note that the tokenizer does not support comments or preprocessor +// directives. The preprocessor should do its work on the .rc file before +// running llvm-rc. +// +// As for now, it is possible to parse ASCII files only (the behavior on +// UTF files might be undefined). However, it already consumes UTF-8 BOM, if +// there is any. Thus, ASCII-compatible UTF-8 files are tokenized correctly. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa380599(v=vs.85).aspx +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVMRC_RESOURCESCRIPTTOKEN_H +#define LLVM_TOOLS_LLVMRC_RESOURCESCRIPTTOKEN_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +#include <cstdint> +#include <map> +#include <string> +#include <vector> + +namespace llvm { + +// A definition of a single resource script token. Each token has its kind +// (declared in ResourceScriptTokenList) and holds a value - a reference +// representation of the token. +// RCToken does not claim ownership on its value. A memory buffer containing +// the token value should be stored in a safe place and cannot be freed +// nor reallocated. +class RCToken { +public: + enum class Kind { +#define TOKEN(Name) Name, +#define SHORT_TOKEN(Name, Ch) Name, +#include "ResourceScriptTokenList.def" + }; + + RCToken(RCToken::Kind RCTokenKind, StringRef Value); + + // Get an integer value of the integer token. + uint32_t intValue() const; + bool isLongInt() const; + + StringRef value() const; + Kind kind() const; + + // Check if a token describes a binary operator. + bool isBinaryOp() const; + +private: + Kind TokenKind; + StringRef TokenValue; +}; + +// Tokenize Input. +// In case no error occured, the return value contains +// tokens in order they were in the input file. +// In case of any error, the return value contains +// a textual representation of error. +// +// Tokens returned by this function hold only references to the parts +// of the Input. Memory buffer containing Input cannot be freed, +// modified or reallocated. +Expected<std::vector<RCToken>> tokenizeRC(StringRef Input); + +} // namespace llvm + +#endif diff --git a/tools/llvm-rc/ResourceScriptTokenList.def b/tools/llvm-rc/ResourceScriptTokenList.def new file mode 100644 index 000000000000..085b728ff520 --- /dev/null +++ b/tools/llvm-rc/ResourceScriptTokenList.def @@ -0,0 +1,40 @@ +//===-- ResourceScriptTokenList.h -------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This is a part of llvm-rc tokenizer. It lists all the possible tokens +// that might occur in a correct .rc script. +// +//===---------------------------------------------------------------------===// + + +// Long tokens. They might consist of more than one character. +TOKEN(Invalid) // Invalid token. Should not occur in a valid script. +TOKEN(Int) // Integer (decimal, octal or hexadecimal). +TOKEN(String) // String value. +TOKEN(Identifier) // Script identifier (resource name or type). +TOKEN(LineComment) // Beginning of single-line comment. +TOKEN(StartComment) // Beginning of multi-line comment. + +// Short tokens. They usually consist of exactly one character. +// The definitions are of the form SHORT_TOKEN(TokenName, TokenChar). +// TokenChar is the one-character token representation occuring in the correct +// .rc scripts. +SHORT_TOKEN(BlockBegin, '{') // Start of the script block; can also be BEGIN. +SHORT_TOKEN(BlockEnd, '}') // End of the block; can also be END. +SHORT_TOKEN(Comma, ',') // Comma - resource arguments separator. +SHORT_TOKEN(Plus, '+') // Addition operator. +SHORT_TOKEN(Minus, '-') // Subtraction operator. +SHORT_TOKEN(Pipe, '|') // Bitwise-OR operator. +SHORT_TOKEN(Amp, '&') // Bitwise-AND operator. +SHORT_TOKEN(Tilde, '~') // Bitwise-NOT operator. +SHORT_TOKEN(LeftParen, '(') // Left parenthesis in the script expressions. +SHORT_TOKEN(RightParen, ')') // Right parenthesis. + +#undef TOKEN +#undef SHORT_TOKEN diff --git a/tools/llvm-rc/ResourceVisitor.h b/tools/llvm-rc/ResourceVisitor.h new file mode 100644 index 000000000000..530b4a8add2c --- /dev/null +++ b/tools/llvm-rc/ResourceVisitor.h @@ -0,0 +1,57 @@ +//===-- ResourceVisitor.h ---------------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This defines a base class visiting resource script resources. +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVMRC_RESOURCEVISITOR_H +#define LLVM_TOOLS_LLVMRC_RESOURCEVISITOR_H + +#include "llvm/Support/Error.h" + +namespace llvm { +namespace rc { + +class RCResource; + +class CaptionStmt; +class CharacteristicsStmt; +class FontStmt; +class LanguageResource; +class StyleStmt; +class VersionStmt; + +class Visitor { +public: + virtual Error visitNullResource(const RCResource *) = 0; + virtual Error visitAcceleratorsResource(const RCResource *) = 0; + virtual Error visitCursorResource(const RCResource *) = 0; + virtual Error visitDialogResource(const RCResource *) = 0; + virtual Error visitHTMLResource(const RCResource *) = 0; + virtual Error visitIconResource(const RCResource *) = 0; + virtual Error visitMenuResource(const RCResource *) = 0; + virtual Error visitStringTableResource(const RCResource *) = 0; + virtual Error visitUserDefinedResource(const RCResource *) = 0; + virtual Error visitVersionInfoResource(const RCResource *) = 0; + + virtual Error visitCaptionStmt(const CaptionStmt *) = 0; + virtual Error visitCharacteristicsStmt(const CharacteristicsStmt *) = 0; + virtual Error visitFontStmt(const FontStmt *) = 0; + virtual Error visitLanguageStmt(const LanguageResource *) = 0; + virtual Error visitStyleStmt(const StyleStmt *) = 0; + virtual Error visitVersionStmt(const VersionStmt *) = 0; + + virtual ~Visitor() {} +}; + +} // namespace rc +} // namespace llvm + +#endif diff --git a/tools/llvm-rc/llvm-rc.cpp b/tools/llvm-rc/llvm-rc.cpp new file mode 100644 index 000000000000..4a1f52e66dee --- /dev/null +++ b/tools/llvm-rc/llvm-rc.cpp @@ -0,0 +1,185 @@ +//===-- llvm-rc.cpp - Compile .rc scripts into .res -------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Compile .rc scripts into .res files. This is intended to be a +// platform-independent port of Microsoft's rc.exe tool. +// +//===----------------------------------------------------------------------===// + +#include "ResourceFileWriter.h" +#include "ResourceScriptParser.h" +#include "ResourceScriptStmt.h" +#include "ResourceScriptToken.h" + +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/raw_ostream.h" + +#include <system_error> + +using namespace llvm; +using namespace llvm::rc; + +namespace { + +// Input options tables. + +enum ID { + OPT_INVALID = 0, // This is not a correct option ID. +#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM, \ + HELPTEXT, METAVAR, VALUES) \ + OPT_##ID, +#include "Opts.inc" +#undef OPTION +}; + +#define PREFIX(NAME, VALUE) const char *const NAME[] = VALUE; +#include "Opts.inc" +#undef PREFIX + +static const opt::OptTable::Info InfoTable[] = { +#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM, \ + HELPTEXT, METAVAR, VALUES) \ + { \ + PREFIX, NAME, HELPTEXT, \ + METAVAR, OPT_##ID, opt::Option::KIND##Class, \ + PARAM, FLAGS, OPT_##GROUP, \ + OPT_##ALIAS, ALIASARGS, VALUES}, +#include "Opts.inc" +#undef OPTION +}; + +class RcOptTable : public opt::OptTable { +public: + RcOptTable() : OptTable(InfoTable, /* IgnoreCase = */ true) {} +}; + +static ExitOnError ExitOnErr; + +LLVM_ATTRIBUTE_NORETURN static void fatalError(const Twine &Message) { + errs() << Message << "\n"; + exit(1); +} + +} // anonymous namespace + +int main(int argc_, const char *argv_[]) { + sys::PrintStackTraceOnErrorSignal(argv_[0]); + PrettyStackTraceProgram X(argc_, argv_); + + ExitOnErr.setBanner("llvm-rc: "); + + SmallVector<const char *, 256> argv; + SpecificBumpPtrAllocator<char> ArgAllocator; + ExitOnErr(errorCodeToError(sys::Process::GetArgumentVector( + argv, makeArrayRef(argv_, argc_), ArgAllocator))); + + llvm_shutdown_obj Y; + + RcOptTable T; + unsigned MAI, MAC; + ArrayRef<const char *> ArgsArr = makeArrayRef(argv_ + 1, argc_); + opt::InputArgList InputArgs = T.ParseArgs(ArgsArr, MAI, MAC); + + // The tool prints nothing when invoked with no command-line arguments. + if (InputArgs.hasArg(OPT_HELP)) { + T.PrintHelp(outs(), "rc", "Resource Converter", false); + return 0; + } + + const bool BeVerbose = InputArgs.hasArg(OPT_VERBOSE); + + std::vector<std::string> InArgsInfo = InputArgs.getAllArgValues(OPT_INPUT); + if (InArgsInfo.size() != 1) { + fatalError("Exactly one input file should be provided."); + } + + // Read and tokenize the input file. + ErrorOr<std::unique_ptr<MemoryBuffer>> File = + MemoryBuffer::getFile(InArgsInfo[0]); + if (!File) { + fatalError("Error opening file '" + Twine(InArgsInfo[0]) + + "': " + File.getError().message()); + } + + std::unique_ptr<MemoryBuffer> FileContents = std::move(*File); + StringRef Contents = FileContents->getBuffer(); + + std::vector<RCToken> Tokens = ExitOnErr(tokenizeRC(Contents)); + + if (BeVerbose) { + const Twine TokenNames[] = { +#define TOKEN(Name) #Name, +#define SHORT_TOKEN(Name, Ch) #Name, +#include "ResourceScriptTokenList.def" + }; + + for (const RCToken &Token : Tokens) { + outs() << TokenNames[static_cast<int>(Token.kind())] << ": " + << Token.value(); + if (Token.kind() == RCToken::Kind::Int) + outs() << "; int value = " << Token.intValue(); + + outs() << "\n"; + } + } + + SearchParams Params; + SmallString<128> InputFile(InArgsInfo[0]); + llvm::sys::fs::make_absolute(InputFile); + Params.InputFilePath = InputFile; + Params.Include = InputArgs.getAllArgValues(OPT_INCLUDE); + Params.NoInclude = InputArgs.getAllArgValues(OPT_NOINCLUDE); + + std::unique_ptr<ResourceFileWriter> Visitor; + bool IsDryRun = InputArgs.hasArg(OPT_DRY_RUN); + + if (!IsDryRun) { + auto OutArgsInfo = InputArgs.getAllArgValues(OPT_FILEOUT); + if (OutArgsInfo.size() != 1) + fatalError( + "Exactly one output file should be provided (using /FO flag)."); + + std::error_code EC; + auto FOut = + llvm::make_unique<raw_fd_ostream>(OutArgsInfo[0], EC, sys::fs::F_RW); + if (EC) + fatalError("Error opening output file '" + OutArgsInfo[0] + + "': " + EC.message()); + Visitor = llvm::make_unique<ResourceFileWriter>(Params, std::move(FOut)); + Visitor->AppendNull = InputArgs.hasArg(OPT_ADD_NULL); + + ExitOnErr(NullResource().visit(Visitor.get())); + + // Set the default language; choose en-US arbitrarily. + ExitOnErr(LanguageResource(0x09, 0x01).visit(Visitor.get())); + } + + rc::RCParser Parser{std::move(Tokens)}; + while (!Parser.isEof()) { + auto Resource = ExitOnErr(Parser.parseSingleResource()); + if (BeVerbose) + Resource->log(outs()); + if (!IsDryRun) + ExitOnErr(Resource->visit(Visitor.get())); + } + + // STRINGTABLE resources come at the very end. + if (!IsDryRun) + ExitOnErr(Visitor->dumpAllStringTables()); + + return 0; +} diff --git a/tools/llvm-readobj/ARMEHABIPrinter.h b/tools/llvm-readobj/ARMEHABIPrinter.h index 903a246ccfd8..4417aa60fe90 100644 --- a/tools/llvm-readobj/ARMEHABIPrinter.h +++ b/tools/llvm-readobj/ARMEHABIPrinter.h @@ -35,7 +35,7 @@ class OpcodeDecoder { uint8_t Value; void (OpcodeDecoder::*Routine)(const uint8_t *Opcodes, unsigned &OI); }; - static const RingEntry Ring[]; + static ArrayRef<RingEntry> ring(); void Decode_00xxxxxx(const uint8_t *Opcodes, unsigned &OI); void Decode_01xxxxxx(const uint8_t *Opcodes, unsigned &OI); @@ -68,43 +68,48 @@ public: void Decode(const uint8_t *Opcodes, off_t Offset, size_t Length); }; -const OpcodeDecoder::RingEntry OpcodeDecoder::Ring[] = { - { 0xc0, 0x00, &OpcodeDecoder::Decode_00xxxxxx }, - { 0xc0, 0x40, &OpcodeDecoder::Decode_01xxxxxx }, - { 0xf0, 0x80, &OpcodeDecoder::Decode_1000iiii_iiiiiiii }, - { 0xff, 0x9d, &OpcodeDecoder::Decode_10011101 }, - { 0xff, 0x9f, &OpcodeDecoder::Decode_10011111 }, - { 0xf0, 0x90, &OpcodeDecoder::Decode_1001nnnn }, - { 0xf8, 0xa0, &OpcodeDecoder::Decode_10100nnn }, - { 0xf8, 0xa8, &OpcodeDecoder::Decode_10101nnn }, - { 0xff, 0xb0, &OpcodeDecoder::Decode_10110000 }, - { 0xff, 0xb1, &OpcodeDecoder::Decode_10110001_0000iiii }, - { 0xff, 0xb2, &OpcodeDecoder::Decode_10110010_uleb128 }, - { 0xff, 0xb3, &OpcodeDecoder::Decode_10110011_sssscccc }, - { 0xfc, 0xb4, &OpcodeDecoder::Decode_101101nn }, - { 0xf8, 0xb8, &OpcodeDecoder::Decode_10111nnn }, - { 0xff, 0xc6, &OpcodeDecoder::Decode_11000110_sssscccc }, - { 0xff, 0xc7, &OpcodeDecoder::Decode_11000111_0000iiii }, - { 0xff, 0xc8, &OpcodeDecoder::Decode_11001000_sssscccc }, - { 0xff, 0xc9, &OpcodeDecoder::Decode_11001001_sssscccc }, - { 0xc8, 0xc8, &OpcodeDecoder::Decode_11001yyy }, - { 0xf8, 0xc0, &OpcodeDecoder::Decode_11000nnn }, - { 0xf8, 0xd0, &OpcodeDecoder::Decode_11010nnn }, - { 0xc0, 0xc0, &OpcodeDecoder::Decode_11xxxyyy }, -}; +inline ArrayRef<OpcodeDecoder::RingEntry> OpcodeDecoder::ring() { + static const OpcodeDecoder::RingEntry Ring[] = { + {0xc0, 0x00, &OpcodeDecoder::Decode_00xxxxxx}, + {0xc0, 0x40, &OpcodeDecoder::Decode_01xxxxxx}, + {0xf0, 0x80, &OpcodeDecoder::Decode_1000iiii_iiiiiiii}, + {0xff, 0x9d, &OpcodeDecoder::Decode_10011101}, + {0xff, 0x9f, &OpcodeDecoder::Decode_10011111}, + {0xf0, 0x90, &OpcodeDecoder::Decode_1001nnnn}, + {0xf8, 0xa0, &OpcodeDecoder::Decode_10100nnn}, + {0xf8, 0xa8, &OpcodeDecoder::Decode_10101nnn}, + {0xff, 0xb0, &OpcodeDecoder::Decode_10110000}, + {0xff, 0xb1, &OpcodeDecoder::Decode_10110001_0000iiii}, + {0xff, 0xb2, &OpcodeDecoder::Decode_10110010_uleb128}, + {0xff, 0xb3, &OpcodeDecoder::Decode_10110011_sssscccc}, + {0xfc, 0xb4, &OpcodeDecoder::Decode_101101nn}, + {0xf8, 0xb8, &OpcodeDecoder::Decode_10111nnn}, + {0xff, 0xc6, &OpcodeDecoder::Decode_11000110_sssscccc}, + {0xff, 0xc7, &OpcodeDecoder::Decode_11000111_0000iiii}, + {0xff, 0xc8, &OpcodeDecoder::Decode_11001000_sssscccc}, + {0xff, 0xc9, &OpcodeDecoder::Decode_11001001_sssscccc}, + {0xc8, 0xc8, &OpcodeDecoder::Decode_11001yyy}, + {0xf8, 0xc0, &OpcodeDecoder::Decode_11000nnn}, + {0xf8, 0xd0, &OpcodeDecoder::Decode_11010nnn}, + {0xc0, 0xc0, &OpcodeDecoder::Decode_11xxxyyy}, + }; + return makeArrayRef(Ring); +} -void OpcodeDecoder::Decode_00xxxxxx(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_00xxxxxx(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; vsp = vsp + %u\n", Opcode, ((Opcode & 0x3f) << 2) + 4); } -void OpcodeDecoder::Decode_01xxxxxx(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_01xxxxxx(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; vsp = vsp - %u\n", Opcode, ((Opcode & 0x3f) << 2) + 4); } -void OpcodeDecoder::Decode_1000iiii_iiiiiiii(const uint8_t *Opcodes, - unsigned &OI) { +inline void OpcodeDecoder::Decode_1000iiii_iiiiiiii(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode0 = Opcodes[OI++ ^ 3]; uint8_t Opcode1 = Opcodes[OI++ ^ 3]; @@ -116,36 +121,42 @@ void OpcodeDecoder::Decode_1000iiii_iiiiiiii(const uint8_t *Opcodes, PrintGPR(GPRMask); OS << '\n'; } -void OpcodeDecoder::Decode_10011101(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_10011101(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; reserved (ARM MOVrr)\n", Opcode); } -void OpcodeDecoder::Decode_10011111(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_10011111(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; reserved (WiMMX MOVrr)\n", Opcode); } -void OpcodeDecoder::Decode_1001nnnn(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_1001nnnn(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; vsp = r%u\n", Opcode, (Opcode & 0x0f)); } -void OpcodeDecoder::Decode_10100nnn(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_10100nnn(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; pop ", Opcode); PrintGPR((((1 << ((Opcode & 0x7) + 1)) - 1) << 4)); OS << '\n'; } -void OpcodeDecoder::Decode_10101nnn(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_10101nnn(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; pop ", Opcode); PrintGPR((((1 << ((Opcode & 0x7) + 1)) - 1) << 4) | (1 << 14)); OS << '\n'; } -void OpcodeDecoder::Decode_10110000(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_10110000(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; finish\n", Opcode); } -void OpcodeDecoder::Decode_10110001_0000iiii(const uint8_t *Opcodes, - unsigned &OI) { +inline void OpcodeDecoder::Decode_10110001_0000iiii(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode0 = Opcodes[OI++ ^ 3]; uint8_t Opcode1 = Opcodes[OI++ ^ 3]; @@ -156,8 +167,8 @@ void OpcodeDecoder::Decode_10110001_0000iiii(const uint8_t *Opcodes, PrintGPR((Opcode1 & 0x0f)); OS << '\n'; } -void OpcodeDecoder::Decode_10110010_uleb128(const uint8_t *Opcodes, - unsigned &OI) { +inline void OpcodeDecoder::Decode_10110010_uleb128(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ", Opcode); @@ -173,8 +184,8 @@ void OpcodeDecoder::Decode_10110010_uleb128(const uint8_t *Opcodes, OS << format("; vsp = vsp + %" PRIu64 "\n", 0x204 + (Value << 2)); } -void OpcodeDecoder::Decode_10110011_sssscccc(const uint8_t *Opcodes, - unsigned &OI) { +inline void OpcodeDecoder::Decode_10110011_sssscccc(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode0 = Opcodes[OI++ ^ 3]; uint8_t Opcode1 = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X 0x%02X ; pop ", Opcode0, Opcode1); @@ -183,18 +194,20 @@ void OpcodeDecoder::Decode_10110011_sssscccc(const uint8_t *Opcodes, PrintRegisters((((1 << (Count + 1)) - 1) << Start), "d"); OS << '\n'; } -void OpcodeDecoder::Decode_101101nn(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_101101nn(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; spare\n", Opcode); } -void OpcodeDecoder::Decode_10111nnn(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_10111nnn(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; pop ", Opcode); PrintRegisters((((1 << ((Opcode & 0x07) + 1)) - 1) << 8), "d"); OS << '\n'; } -void OpcodeDecoder::Decode_11000110_sssscccc(const uint8_t *Opcodes, - unsigned &OI) { +inline void OpcodeDecoder::Decode_11000110_sssscccc(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode0 = Opcodes[OI++ ^ 3]; uint8_t Opcode1 = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X 0x%02X ; pop ", Opcode0, Opcode1); @@ -203,8 +216,8 @@ void OpcodeDecoder::Decode_11000110_sssscccc(const uint8_t *Opcodes, PrintRegisters((((1 << (Count + 1)) - 1) << Start), "wR"); OS << '\n'; } -void OpcodeDecoder::Decode_11000111_0000iiii(const uint8_t *Opcodes, - unsigned &OI) { +inline void OpcodeDecoder::Decode_11000111_0000iiii(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode0 = Opcodes[OI++ ^ 3]; uint8_t Opcode1 = Opcodes[OI++ ^ 3]; SW.startLine() @@ -214,8 +227,8 @@ void OpcodeDecoder::Decode_11000111_0000iiii(const uint8_t *Opcodes, PrintRegisters(Opcode1 & 0x0f, "wCGR"); OS << '\n'; } -void OpcodeDecoder::Decode_11001000_sssscccc(const uint8_t *Opcodes, - unsigned &OI) { +inline void OpcodeDecoder::Decode_11001000_sssscccc(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode0 = Opcodes[OI++ ^ 3]; uint8_t Opcode1 = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X 0x%02X ; pop ", Opcode0, Opcode1); @@ -224,8 +237,8 @@ void OpcodeDecoder::Decode_11001000_sssscccc(const uint8_t *Opcodes, PrintRegisters((((1 << (Count + 1)) - 1) << Start), "d"); OS << '\n'; } -void OpcodeDecoder::Decode_11001001_sssscccc(const uint8_t *Opcodes, - unsigned &OI) { +inline void OpcodeDecoder::Decode_11001001_sssscccc(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode0 = Opcodes[OI++ ^ 3]; uint8_t Opcode1 = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X 0x%02X ; pop ", Opcode0, Opcode1); @@ -234,28 +247,32 @@ void OpcodeDecoder::Decode_11001001_sssscccc(const uint8_t *Opcodes, PrintRegisters((((1 << (Count + 1)) - 1) << Start), "d"); OS << '\n'; } -void OpcodeDecoder::Decode_11001yyy(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_11001yyy(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; spare\n", Opcode); } -void OpcodeDecoder::Decode_11000nnn(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_11000nnn(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; pop ", Opcode); PrintRegisters((((1 << ((Opcode & 0x07) + 1)) - 1) << 10), "wR"); OS << '\n'; } -void OpcodeDecoder::Decode_11010nnn(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_11010nnn(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; pop ", Opcode); PrintRegisters((((1 << ((Opcode & 0x07) + 1)) - 1) << 8), "d"); OS << '\n'; } -void OpcodeDecoder::Decode_11xxxyyy(const uint8_t *Opcodes, unsigned &OI) { +inline void OpcodeDecoder::Decode_11xxxyyy(const uint8_t *Opcodes, + unsigned &OI) { uint8_t Opcode = Opcodes[OI++ ^ 3]; SW.startLine() << format("0x%02X ; spare\n", Opcode); } -void OpcodeDecoder::PrintGPR(uint16_t GPRMask) { +inline void OpcodeDecoder::PrintGPR(uint16_t GPRMask) { static const char *GPRRegisterNames[16] = { "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "fp", "ip", "sp", "lr", "pc" @@ -274,7 +291,7 @@ void OpcodeDecoder::PrintGPR(uint16_t GPRMask) { OS << '}'; } -void OpcodeDecoder::PrintRegisters(uint32_t VFPMask, StringRef Prefix) { +inline void OpcodeDecoder::PrintRegisters(uint32_t VFPMask, StringRef Prefix) { OS << '{'; bool Comma = false; for (unsigned RI = 0, RE = 32; RI < RE; ++RI) { @@ -288,13 +305,13 @@ void OpcodeDecoder::PrintRegisters(uint32_t VFPMask, StringRef Prefix) { OS << '}'; } -void OpcodeDecoder::Decode(const uint8_t *Opcodes, off_t Offset, size_t Length) { +inline void OpcodeDecoder::Decode(const uint8_t *Opcodes, off_t Offset, + size_t Length) { for (unsigned OCI = Offset; OCI < Length + Offset; ) { bool Decoded = false; - for (unsigned REI = 0, REE = array_lengthof(Ring); - REI != REE && !Decoded; ++REI) { - if ((Opcodes[OCI ^ 3] & Ring[REI].Mask) == Ring[REI].Value) { - (this->*Ring[REI].Routine)(Opcodes, OCI); + for (const auto &RE : ring()) { + if ((Opcodes[OCI ^ 3] & RE.Mask) == RE.Value) { + (this->*RE.Routine)(Opcodes, OCI); Decoded = true; break; } diff --git a/tools/llvm-readobj/CMakeLists.txt b/tools/llvm-readobj/CMakeLists.txt index f5b1a5b256ad..dafc9e10cfa1 100644 --- a/tools/llvm-readobj/CMakeLists.txt +++ b/tools/llvm-readobj/CMakeLists.txt @@ -19,6 +19,11 @@ add_llvm_tool(llvm-readobj ObjDumper.cpp WasmDumper.cpp Win64EHDumper.cpp + WindowsResourceDumper.cpp ) add_llvm_tool_symlink(llvm-readelf llvm-readobj) + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(readelf llvm-readobj) +endif() diff --git a/tools/llvm-readobj/COFFDumper.cpp b/tools/llvm-readobj/COFFDumper.cpp index 74c44116b127..8ac9f1a51cc5 100644 --- a/tools/llvm-readobj/COFFDumper.cpp +++ b/tools/llvm-readobj/COFFDumper.cpp @@ -31,16 +31,16 @@ #include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h" #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" #include "llvm/DebugInfo/CodeView/Line.h" +#include "llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h" #include "llvm/DebugInfo/CodeView/RecordSerialization.h" -#include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" #include "llvm/DebugInfo/CodeView/SymbolDumpDelegate.h" #include "llvm/DebugInfo/CodeView/SymbolDumper.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/CodeView/TypeDumpVisitor.h" +#include "llvm/DebugInfo/CodeView/TypeHashing.h" #include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" #include "llvm/DebugInfo/CodeView/TypeStreamMerger.h" -#include "llvm/DebugInfo/CodeView/TypeTableBuilder.h" #include "llvm/DebugInfo/CodeView/TypeTableCollection.h" #include "llvm/Object/COFF.h" #include "llvm/Object/ObjectFile.h" @@ -48,17 +48,10 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ConvertUTF.h" -#include "llvm/Support/DataExtractor.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/Path.h" #include "llvm/Support/ScopedPrinter.h" -#include "llvm/Support/SourceMgr.h" #include "llvm/Support/Win64EH.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <cstring> -#include <system_error> -#include <time.h> using namespace llvm; using namespace llvm::object; @@ -96,8 +89,9 @@ public: void printCOFFResources() override; void printCOFFLoadConfig() override; void printCodeViewDebugInfo() override; - void mergeCodeViewTypes(llvm::codeview::TypeTableBuilder &CVIDs, - llvm::codeview::TypeTableBuilder &CVTypes) override; + void + mergeCodeViewTypes(llvm::codeview::MergingTypeTableBuilder &CVIDs, + llvm::codeview::MergingTypeTableBuilder &CVTypes) override; void printStackMap() const override; private: void printSymbol(const SymbolRef &Sym); @@ -1194,8 +1188,8 @@ void COFFDumper::printFileNameForOffset(StringRef Label, uint32_t FileOffset) { W.printHex(Label, getFileNameForFileOffset(FileOffset), FileOffset); } -void COFFDumper::mergeCodeViewTypes(TypeTableBuilder &CVIDs, - TypeTableBuilder &CVTypes) { +void COFFDumper::mergeCodeViewTypes(MergingTypeTableBuilder &CVIDs, + MergingTypeTableBuilder &CVTypes) { for (const SectionRef &S : Obj->sections()) { StringRef SectionName; error(S.getName(SectionName)); @@ -1423,9 +1417,9 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) { const coff_aux_weak_external *Aux; error(getSymbolAuxData(Obj, Symbol, I, Aux)); - ErrorOr<COFFSymbolRef> Linked = Obj->getSymbol(Aux->TagIndex); + Expected<COFFSymbolRef> Linked = Obj->getSymbol(Aux->TagIndex); StringRef LinkedName; - std::error_code EC = Linked.getError(); + std::error_code EC = errorToErrorCode(Linked.takeError()); if (EC || (EC = Obj->getSymbolName(*Linked, LinkedName))) { LinkedName = ""; error(EC); @@ -1481,10 +1475,10 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) { const coff_aux_clr_token *Aux; error(getSymbolAuxData(Obj, Symbol, I, Aux)); - ErrorOr<COFFSymbolRef> ReferredSym = + Expected<COFFSymbolRef> ReferredSym = Obj->getSymbol(Aux->SymbolTableIndex); StringRef ReferredName; - std::error_code EC = ReferredSym.getError(); + std::error_code EC = errorToErrorCode(ReferredSym.takeError()); if (EC || (EC = Obj->getSymbolName(*ReferredSym, ReferredName))) { ReferredName = ""; error(EC); @@ -1627,7 +1621,7 @@ void COFFDumper::printCOFFDirectives() { } } -static StringRef getBaseRelocTypeName(uint8_t Type) { +static std::string getBaseRelocTypeName(uint8_t Type) { switch (Type) { case COFF::IMAGE_REL_BASED_ABSOLUTE: return "ABSOLUTE"; case COFF::IMAGE_REL_BASED_HIGH: return "HIGH"; @@ -1636,11 +1630,7 @@ static StringRef getBaseRelocTypeName(uint8_t Type) { case COFF::IMAGE_REL_BASED_HIGHADJ: return "HIGHADJ"; case COFF::IMAGE_REL_BASED_ARM_MOV32T: return "ARM_MOV32(T)"; case COFF::IMAGE_REL_BASED_DIR64: return "DIR64"; - default: { - static std::string Result; - Result = "unknown (" + llvm::utostr(Type) + ")"; - return Result; - } + default: return "unknown (" + llvm::utostr(Type) + ")"; } } @@ -1807,13 +1797,13 @@ void COFFDumper::printStackMap() const { StackMapV2Parser<support::big>(StackMapContentsArray)); } -void llvm::dumpCodeViewMergedTypes(ScopedPrinter &Writer, - llvm::codeview::TypeTableBuilder &IDTable, - llvm::codeview::TypeTableBuilder &CVTypes) { +void llvm::dumpCodeViewMergedTypes( + ScopedPrinter &Writer, llvm::codeview::MergingTypeTableBuilder &IDTable, + llvm::codeview::MergingTypeTableBuilder &CVTypes) { // Flatten it first, then run our dumper on it. SmallString<0> TypeBuf; - CVTypes.ForEachRecord([&](TypeIndex TI, ArrayRef<uint8_t> Record) { - TypeBuf.append(Record.begin(), Record.end()); + CVTypes.ForEachRecord([&](TypeIndex TI, const CVType &Record) { + TypeBuf.append(Record.RecordData.begin(), Record.RecordData.end()); }); TypeTableCollection TpiTypes(CVTypes.records()); diff --git a/tools/llvm-readobj/COFFImportDumper.cpp b/tools/llvm-readobj/COFFImportDumper.cpp index c5b8bf758462..3b546b3ef508 100644 --- a/tools/llvm-readobj/COFFImportDumper.cpp +++ b/tools/llvm-readobj/COFFImportDumper.cpp @@ -12,9 +12,6 @@ /// //===----------------------------------------------------------------------===// -#include "Error.h" -#include "ObjDumper.h" -#include "llvm-readobj.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/Object/COFF.h" #include "llvm/Object/COFFImportFile.h" diff --git a/tools/llvm-readobj/ELFDumper.cpp b/tools/llvm-readobj/ELFDumper.cpp index 5698420bbcc2..9678667abffe 100644 --- a/tools/llvm-readobj/ELFDumper.cpp +++ b/tools/llvm-readobj/ELFDumper.cpp @@ -18,6 +18,7 @@ #include "StackMapPrinter.h" #include "llvm-readobj.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallString.h" @@ -33,6 +34,7 @@ #include "llvm/Object/Error.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Object/StackMapParser.h" +#include "llvm/Support/AMDGPUMetadata.h" #include "llvm/Support/ARMAttributeParser.h" #include "llvm/Support/ARMBuildAttributes.h" #include "llvm/Support/Casting.h" @@ -155,8 +157,6 @@ public: void printMipsReginfo() override; void printMipsOptions() override; - void printAMDGPUCodeObjectMetadata() override; - void printStackMap() const override; void printHashHistogram() override; @@ -820,12 +820,24 @@ static const EnumEntry<unsigned> ElfOSABI[] = { {"AROS", "AROS", ELF::ELFOSABI_AROS}, {"FenixOS", "FenixOS", ELF::ELFOSABI_FENIXOS}, {"CloudABI", "CloudABI", ELF::ELFOSABI_CLOUDABI}, - {"C6000_ELFABI", "Bare-metal C6000", ELF::ELFOSABI_C6000_ELFABI}, - {"C6000_LINUX", "Linux C6000", ELF::ELFOSABI_C6000_LINUX}, - {"ARM", "ARM", ELF::ELFOSABI_ARM}, {"Standalone", "Standalone App", ELF::ELFOSABI_STANDALONE} }; +static const EnumEntry<unsigned> AMDGPUElfOSABI[] = { + {"AMDGPU_HSA", "AMDGPU - HSA", ELF::ELFOSABI_AMDGPU_HSA}, + {"AMDGPU_PAL", "AMDGPU - PAL", ELF::ELFOSABI_AMDGPU_PAL}, + {"AMDGPU_MESA3D", "AMDGPU - MESA3D", ELF::ELFOSABI_AMDGPU_MESA3D} +}; + +static const EnumEntry<unsigned> ARMElfOSABI[] = { + {"ARM", "ARM", ELF::ELFOSABI_ARM} +}; + +static const EnumEntry<unsigned> C6000ElfOSABI[] = { + {"C6000_ELFABI", "Bare-metal C6000", ELF::ELFOSABI_C6000_ELFABI}, + {"C6000_LINUX", "Linux C6000", ELF::ELFOSABI_C6000_LINUX} +}; + static const EnumEntry<unsigned> ElfMachineType[] = { ENUM_ENT(EM_NONE, "None"), ENUM_ENT(EM_M32, "WE32100"), @@ -967,7 +979,7 @@ static const EnumEntry<unsigned> ElfMachineType[] = { ENUM_ENT(EM_L10M, "EM_L10M"), ENUM_ENT(EM_K10M, "EM_K10M"), ENUM_ENT(EM_AARCH64, "AArch64"), - ENUM_ENT(EM_AVR32, "Atmel AVR 8-bit microcontroller"), + ENUM_ENT(EM_AVR32, "Atmel Corporation 32-bit microprocessor family"), ENUM_ENT(EM_STM8, "STMicroeletronics STM8 8-bit microcontroller"), ENUM_ENT(EM_TILE64, "Tilera TILE64 multicore architecture family"), ENUM_ENT(EM_TILEPRO, "Tilera TILEPro multicore architecture family"), @@ -1231,6 +1243,20 @@ static const EnumEntry<unsigned> ElfHeaderMipsFlags[] = { LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ARCH_64R6) }; +static const EnumEntry<unsigned> ElfHeaderAMDGPUFlags[] = { + LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_ARCH_NONE), + LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_ARCH_R600), + LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_ARCH_GCN) +}; + +static const EnumEntry<unsigned> ElfHeaderRISCVFlags[] = { + LLVM_READOBJ_ENUM_ENT(ELF, EF_RISCV_RVC), + LLVM_READOBJ_ENUM_ENT(ELF, EF_RISCV_FLOAT_ABI_SINGLE), + LLVM_READOBJ_ENUM_ENT(ELF, EF_RISCV_FLOAT_ABI_DOUBLE), + LLVM_READOBJ_ENUM_ENT(ELF, EF_RISCV_FLOAT_ABI_QUAD), + LLVM_READOBJ_ENUM_ENT(ELF, EF_RISCV_RVE) +}; + static const EnumEntry<unsigned> ElfSymOtherFlags[] = { LLVM_READOBJ_ENUM_ENT(ELF, STV_INTERNAL), LLVM_READOBJ_ENUM_ENT(ELF, STV_HIDDEN), @@ -1287,15 +1313,16 @@ ELFDumper<ELFT>::ELFDumper(const ELFFile<ELFT> *Obj, ScopedPrinter &Writer) switch (Sec.sh_type) { case ELF::SHT_SYMTAB: if (DotSymtabSec != nullptr) - reportError("Multilpe SHT_SYMTAB"); + reportError("Multiple SHT_SYMTAB"); DotSymtabSec = &Sec; break; case ELF::SHT_DYNSYM: if (DynSymRegion.Size) - reportError("Multilpe SHT_DYNSYM"); + reportError("Multiple SHT_DYNSYM"); DynSymRegion = createDRIFrom(&Sec); // This is only used (if Elf_Shdr present)for naming section in GNU style DynSymtabName = unwrapOrError(Obj->getSectionName(&Sec)); + DynamicStringTable = unwrapOrError(Obj->getStringTableForSymtab(Sec)); break; case ELF::SHT_SYMTAB_SHNDX: ShndxTable = unwrapOrError(Obj->getSHNDXTable(Sec)); @@ -1312,7 +1339,7 @@ ELFDumper<ELFT>::ELFDumper(const ELFFile<ELFT> *Obj, ScopedPrinter &Writer) break; case ELF::SHT_GNU_verneed: if (dot_gnu_version_r_sec != nullptr) - reportError("Multilpe SHT_GNU_verneed"); + reportError("Multiple SHT_GNU_verneed"); dot_gnu_version_r_sec = &Sec; break; } @@ -1330,8 +1357,11 @@ template <typename ELFT> void ELFDumper<ELFT>::parseDynamicTable( ArrayRef<const Elf_Phdr *> LoadSegments) { auto toMappedAddr = [&](uint64_t VAddr) -> const uint8_t * { - const Elf_Phdr *const *I = std::upper_bound( - LoadSegments.begin(), LoadSegments.end(), VAddr, compareAddr<ELFT>); + const Elf_Phdr *const *I = + std::upper_bound(LoadSegments.begin(), LoadSegments.end(), VAddr, + [](uint64_t VAddr, const Elf_Phdr_Impl<ELFT> *Phdr) { + return VAddr < Phdr->p_vaddr; + }); if (I == LoadSegments.begin()) report_fatal_error("Virtual address is not in any segment"); --I; @@ -1487,6 +1517,10 @@ static const char *getTypeString(unsigned Arch, uint64_t Type) { } } switch (Type) { + LLVM_READOBJ_TYPE_CASE(ANDROID_REL); + LLVM_READOBJ_TYPE_CASE(ANDROID_RELSZ); + LLVM_READOBJ_TYPE_CASE(ANDROID_RELA); + LLVM_READOBJ_TYPE_CASE(ANDROID_RELASZ); LLVM_READOBJ_TYPE_CASE(BIND_NOW); LLVM_READOBJ_TYPE_CASE(DEBUG); LLVM_READOBJ_TYPE_CASE(FINI); @@ -1689,6 +1723,8 @@ void ELFDumper<ELFT>::printValue(uint64_t Type, uint64_t Value) { case DT_INIT_ARRAYSZ: case DT_FINI_ARRAYSZ: case DT_PREINIT_ARRAYSZ: + case DT_ANDROID_RELSZ: + case DT_ANDROID_RELASZ: OS << Value << " (bytes)"; break; case DT_NEEDED: @@ -1874,6 +1910,7 @@ public: MipsGOTParser(ELFDumper<ELFT> *Dumper, const ELFO *Obj, Elf_Dyn_Range DynTable, ScopedPrinter &W); + void parseStaticGOT(); void parseGOT(); void parsePLT(); @@ -1890,11 +1927,12 @@ private: std::size_t getGOTTotal(ArrayRef<uint8_t> GOT) const; const GOTEntry *makeGOTIter(ArrayRef<uint8_t> GOT, std::size_t EntryNum); + void printLocalGOT(const Elf_Shdr *GOTShdr, size_t Num); void printGotEntry(uint64_t GotAddr, const GOTEntry *BeginIt, const GOTEntry *It); void printGlobalGotEntry(uint64_t GotAddr, const GOTEntry *BeginIt, const GOTEntry *It, const Elf_Sym *Sym, - StringRef StrTable, bool IsDynamic); + StringRef StrTable); void printPLTEntry(uint64_t PLTAddr, const GOTEntry *BeginIt, const GOTEntry *It, StringRef Purpose); void printPLTEntry(uint64_t PLTAddr, const GOTEntry *BeginIt, @@ -1929,6 +1967,50 @@ MipsGOTParser<ELFT>::MipsGOTParser(ELFDumper<ELFT> *Dumper, const ELFO *Obj, } } +template <class ELFT> +void MipsGOTParser<ELFT>::printLocalGOT(const Elf_Shdr *GOTShdr, size_t Num) { + ArrayRef<uint8_t> GOT = unwrapOrError(Obj->getSectionContents(GOTShdr)); + + const GOTEntry *GotBegin = makeGOTIter(GOT, 0); + const GOTEntry *GotEnd = makeGOTIter(GOT, Num); + const GOTEntry *It = GotBegin; + + W.printHex("Canonical gp value", GOTShdr->sh_addr + 0x7ff0); + { + ListScope RS(W, "Reserved entries"); + + { + DictScope D(W, "Entry"); + printGotEntry(GOTShdr->sh_addr, GotBegin, It++); + W.printString("Purpose", StringRef("Lazy resolver")); + } + + if (It != GotEnd && (*It >> (sizeof(GOTEntry) * 8 - 1)) != 0) { + DictScope D(W, "Entry"); + printGotEntry(GOTShdr->sh_addr, GotBegin, It++); + W.printString("Purpose", StringRef("Module pointer (GNU extension)")); + } + } + { + ListScope LS(W, "Local entries"); + for (; It != GotEnd; ++It) { + DictScope D(W, "Entry"); + printGotEntry(GOTShdr->sh_addr, GotBegin, It); + } + } +} + +template <class ELFT> void MipsGOTParser<ELFT>::parseStaticGOT() { + const Elf_Shdr *GOTShdr = findSectionByName(*Obj, ".got"); + if (!GOTShdr) { + W.startLine() << "Cannot find .got section.\n"; + return; + } + + DictScope GS(W, "Static GOT"); + printLocalGOT(GOTShdr, GOTShdr->sh_size / sizeof(GOTEntry)); +} + template <class ELFT> void MipsGOTParser<ELFT>::parseGOT() { // See "Global Offset Table" in Chapter 5 in the following document // for detailed GOT description. @@ -1946,10 +2028,7 @@ template <class ELFT> void MipsGOTParser<ELFT>::parseGOT() { return; } - StringRef StrTable = Dumper->getDynamicStringTable(); - const Elf_Sym *DynSymBegin = Dumper->dynamic_symbols().begin(); - const Elf_Sym *DynSymEnd = Dumper->dynamic_symbols().end(); - std::size_t DynSymTotal = std::size_t(std::distance(DynSymBegin, DynSymEnd)); + std::size_t DynSymTotal = Dumper->dynamic_symbols().size(); if (*DtGotSym > DynSymTotal) report_fatal_error("MIPS_GOTSYM exceeds a number of dynamic symbols"); @@ -1971,45 +2050,19 @@ template <class ELFT> void MipsGOTParser<ELFT>::parseGOT() { if (*DtLocalGotNum + GlobalGotNum > getGOTTotal(GOT)) report_fatal_error("Number of GOT entries exceeds the size of GOT section"); - const GOTEntry *GotBegin = makeGOTIter(GOT, 0); - const GOTEntry *GotLocalEnd = makeGOTIter(GOT, *DtLocalGotNum); - const GOTEntry *It = GotBegin; - DictScope GS(W, "Primary GOT"); + printLocalGOT(GOTShdr, *DtLocalGotNum); - W.printHex("Canonical gp value", GOTShdr->sh_addr + 0x7ff0); - { - ListScope RS(W, "Reserved entries"); - - { - DictScope D(W, "Entry"); - printGotEntry(GOTShdr->sh_addr, GotBegin, It++); - W.printString("Purpose", StringRef("Lazy resolver")); - } - - if (It != GotLocalEnd && (*It >> (sizeof(GOTEntry) * 8 - 1)) != 0) { - DictScope D(W, "Entry"); - printGotEntry(GOTShdr->sh_addr, GotBegin, It++); - W.printString("Purpose", StringRef("Module pointer (GNU extension)")); - } - } - { - ListScope LS(W, "Local entries"); - for (; It != GotLocalEnd; ++It) { - DictScope D(W, "Entry"); - printGotEntry(GOTShdr->sh_addr, GotBegin, It); - } - } { ListScope GS(W, "Global entries"); - const GOTEntry *GotGlobalEnd = - makeGOTIter(GOT, *DtLocalGotNum + GlobalGotNum); - const Elf_Sym *GotDynSym = DynSymBegin + *DtGotSym; - for (; It != GotGlobalEnd; ++It) { + const GOTEntry *GotBegin = makeGOTIter(GOT, 0); + const GOTEntry *GotEnd = makeGOTIter(GOT, *DtLocalGotNum + GlobalGotNum); + const Elf_Sym *GotDynSym = Dumper->dynamic_symbols().begin() + *DtGotSym; + for (auto It = makeGOTIter(GOT, *DtLocalGotNum); It != GotEnd; ++It) { DictScope D(W, "Entry"); - printGlobalGotEntry(GOTShdr->sh_addr, GotBegin, It, GotDynSym++, StrTable, - true); + printGlobalGotEntry(GOTShdr->sh_addr, GotBegin, It, GotDynSym++, + Dumper->getDynamicStringTable()); } } @@ -2101,9 +2154,11 @@ void MipsGOTParser<ELFT>::printGotEntry(uint64_t GotAddr, } template <class ELFT> -void MipsGOTParser<ELFT>::printGlobalGotEntry( - uint64_t GotAddr, const GOTEntry *BeginIt, const GOTEntry *It, - const Elf_Sym *Sym, StringRef StrTable, bool IsDynamic) { +void MipsGOTParser<ELFT>::printGlobalGotEntry(uint64_t GotAddr, + const GOTEntry *BeginIt, + const GOTEntry *It, + const Elf_Sym *Sym, + StringRef StrTable) { printGotEntry(GotAddr, BeginIt, It); W.printHex("Value", Sym->st_value); @@ -2115,8 +2170,7 @@ void MipsGOTParser<ELFT>::printGlobalGotEntry( Dumper->getShndxTable(), SectionName, SectionIndex); W.printHex("Section", SectionName, SectionIndex); - std::string FullSymbolName = - Dumper->getFullSymbolName(Sym, StrTable, IsDynamic); + std::string FullSymbolName = Dumper->getFullSymbolName(Sym, StrTable, true); W.printNumber("Name", FullSymbolName, Sym->st_name); } @@ -2160,8 +2214,12 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsPLTGOT() { } MipsGOTParser<ELFT> GOTParser(this, Obj, dynamic_table(), W); - GOTParser.parseGOT(); - GOTParser.parsePLT(); + if (dynamic_table().empty()) + GOTParser.parseStaticGOT(); + else { + GOTParser.parseGOT(); + GOTParser.parsePLT(); + } } static const EnumEntry<unsigned> ElfMipsISAExtType[] = { @@ -2326,36 +2384,6 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsOptions() { } } -template <class ELFT> void ELFDumper<ELFT>::printAMDGPUCodeObjectMetadata() { - const Elf_Shdr *Shdr = findSectionByName(*Obj, ".note"); - if (!Shdr) { - W.startLine() << "There is no .note section in the file.\n"; - return; - } - ArrayRef<uint8_t> Sec = unwrapOrError(Obj->getSectionContents(Shdr)); - - const uint32_t CodeObjectMetadataNoteType = 10; - for (auto I = reinterpret_cast<const Elf_Word *>(&Sec[0]), - E = I + Sec.size()/4; I != E;) { - uint32_t NameSZ = I[0]; - uint32_t DescSZ = I[1]; - uint32_t Type = I[2]; - I += 3; - - StringRef Name; - if (NameSZ) { - Name = StringRef(reinterpret_cast<const char *>(I), NameSZ - 1); - I += alignTo<4>(NameSZ)/4; - } - - if (Name == "AMD" && Type == CodeObjectMetadataNoteType) { - StringRef Desc(reinterpret_cast<const char *>(I), DescSZ); - W.printString(Desc); - } - I += alignTo<4>(DescSZ)/4; - } -} - template <class ELFT> void ELFDumper<ELFT>::printStackMap() const { const Elf_Shdr *StackMapSection = nullptr; for (const auto &Sec : unwrapOrError(Obj->sections())) { @@ -2369,7 +2397,6 @@ template <class ELFT> void ELFDumper<ELFT>::printStackMap() const { if (!StackMapSection) return; - StringRef StackMapContents; ArrayRef<uint8_t> StackMapContentsArray = unwrapOrError(Obj->getSectionContents(StackMapSection)); @@ -2441,34 +2468,91 @@ template <class ELFT> void GNUStyle<ELFT>::printFileHeaders(const ELFO *Obj) { printFields(OS, "Section header string table index:", Str); } -template <class ELFT> void GNUStyle<ELFT>::printGroupSections(const ELFO *Obj) { - uint32_t SectionIndex = 0; - bool HasGroups = false; +namespace { +struct GroupMember { + StringRef Name; + uint64_t Index; +}; + +struct GroupSection { + StringRef Name; + StringRef Signature; + uint64_t ShName; + uint64_t Index; + uint32_t Type; + std::vector<GroupMember> Members; +}; + +template <class ELFT> +std::vector<GroupSection> getGroups(const ELFFile<ELFT> *Obj) { + using Elf_Shdr = typename ELFFile<ELFT>::Elf_Shdr; + using Elf_Sym = typename ELFFile<ELFT>::Elf_Sym; + using Elf_Word = typename ELFFile<ELFT>::Elf_Word; + + std::vector<GroupSection> Ret; + uint64_t I = 0; for (const Elf_Shdr &Sec : unwrapOrError(Obj->sections())) { - if (Sec.sh_type == ELF::SHT_GROUP) { - HasGroups = true; - const Elf_Shdr *Symtab = unwrapOrError(Obj->getSection(Sec.sh_link)); - StringRef StrTable = unwrapOrError(Obj->getStringTableForSymtab(*Symtab)); - const Elf_Sym *Signature = - unwrapOrError(Obj->template getEntry<Elf_Sym>(Symtab, Sec.sh_info)); - ArrayRef<Elf_Word> Data = unwrapOrError( - Obj->template getSectionContentsAsArray<Elf_Word>(&Sec)); - StringRef Name = unwrapOrError(Obj->getSectionName(&Sec)); - OS << "\n" << getGroupType(Data[0]) << " group section [" - << format_decimal(SectionIndex, 5) << "] `" << Name << "' [" - << StrTable.data() + Signature->st_name << "] contains " - << (Data.size() - 1) << " sections:\n" - << " [Index] Name\n"; - for (auto &Ndx : Data.slice(1)) { - auto Sec = unwrapOrError(Obj->getSection(Ndx)); - const StringRef Name = unwrapOrError(Obj->getSectionName(Sec)); - OS << " [" << format_decimal(Ndx, 5) << "] " << Name - << "\n"; + ++I; + if (Sec.sh_type != ELF::SHT_GROUP) + continue; + + const Elf_Shdr *Symtab = unwrapOrError(Obj->getSection(Sec.sh_link)); + StringRef StrTable = unwrapOrError(Obj->getStringTableForSymtab(*Symtab)); + const Elf_Sym *Sym = + unwrapOrError(Obj->template getEntry<Elf_Sym>(Symtab, Sec.sh_info)); + auto Data = + unwrapOrError(Obj->template getSectionContentsAsArray<Elf_Word>(&Sec)); + + StringRef Name = unwrapOrError(Obj->getSectionName(&Sec)); + StringRef Signature = StrTable.data() + Sym->st_name; + Ret.push_back({Name, Signature, Sec.sh_name, I - 1, Data[0], {}}); + + std::vector<GroupMember> &GM = Ret.back().Members; + for (uint32_t Ndx : Data.slice(1)) { + auto Sec = unwrapOrError(Obj->getSection(Ndx)); + const StringRef Name = unwrapOrError(Obj->getSectionName(Sec)); + GM.push_back({Name, Ndx}); + } + } + return Ret; +} + +DenseMap<uint64_t, const GroupSection *> +mapSectionsToGroups(ArrayRef<GroupSection> Groups) { + DenseMap<uint64_t, const GroupSection *> Ret; + for (const GroupSection &G : Groups) + for (const GroupMember &GM : G.Members) + Ret.insert({GM.Index, &G}); + return Ret; +} + +} // namespace + +template <class ELFT> void GNUStyle<ELFT>::printGroupSections(const ELFO *Obj) { + std::vector<GroupSection> V = getGroups<ELFT>(Obj); + DenseMap<uint64_t, const GroupSection *> Map = mapSectionsToGroups(V); + for (const GroupSection &G : V) { + OS << "\n" + << getGroupType(G.Type) << " group section [" + << format_decimal(G.Index, 5) << "] `" << G.Name << "' [" << G.Signature + << "] contains " << G.Members.size() << " sections:\n" + << " [Index] Name\n"; + for (const GroupMember &GM : G.Members) { + const GroupSection *MainGroup = Map[GM.Index]; + if (MainGroup != &G) { + OS.flush(); + errs() << "Error: section [" << format_decimal(GM.Index, 5) + << "] in group section [" << format_decimal(G.Index, 5) + << "] already in group section [" + << format_decimal(MainGroup->Index, 5) << "]"; + errs().flush(); + continue; } + OS << " [" << format_decimal(GM.Index, 5) << "] " << GM.Name << "\n"; } - ++SectionIndex; } - if (!HasGroups) + + if (V.empty()) OS << "There are no section groups in this file.\n"; } @@ -2539,7 +2623,9 @@ static inline void printRelocHeader(raw_ostream &OS, bool Is64, bool IsRela) { template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) { bool HasRelocSections = false; for (const Elf_Shdr &Sec : unwrapOrError(Obj->sections())) { - if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA) + if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA && + Sec.sh_type != ELF::SHT_ANDROID_REL && + Sec.sh_type != ELF::SHT_ANDROID_RELA) continue; HasRelocSections = true; StringRef Name = unwrapOrError(Obj->getSectionName(&Sec)); @@ -2548,9 +2634,12 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) { OS << "\nRelocation section '" << Name << "' at offset 0x" << to_hexString(Offset, false) << " contains " << Entries << " entries:\n"; - printRelocHeader(OS, ELFT::Is64Bits, (Sec.sh_type == ELF::SHT_RELA)); + printRelocHeader(OS, ELFT::Is64Bits, + Sec.sh_type == ELF::SHT_RELA || + Sec.sh_type == ELF::SHT_ANDROID_RELA); const Elf_Shdr *SymTab = unwrapOrError(Obj->getSection(Sec.sh_link)); - if (Sec.sh_type == ELF::SHT_REL) { + switch (Sec.sh_type) { + case ELF::SHT_REL: for (const auto &R : unwrapOrError(Obj->rels(&Sec))) { Elf_Rela Rela; Rela.r_offset = R.r_offset; @@ -2558,9 +2647,16 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) { Rela.r_addend = 0; printRelocation(Obj, SymTab, Rela, false); } - } else { + break; + case ELF::SHT_RELA: for (const auto &R : unwrapOrError(Obj->relas(&Sec))) printRelocation(Obj, SymTab, R, true); + break; + case ELF::SHT_ANDROID_REL: + case ELF::SHT_ANDROID_RELA: + for (const auto &R : unwrapOrError(Obj->android_relas(&Sec))) + printRelocation(Obj, SymTab, R, Sec.sh_type == ELF::SHT_ANDROID_RELA); + break; } } if (!HasRelocSections) @@ -3310,7 +3406,7 @@ static std::string getGNUNoteTypeName(const uint32_t NT) { std::string string; raw_string_ostream OS(string); OS << format("Unknown note type (0x%08x)", NT); - return string; + return OS.str(); } static std::string getFreeBSDNoteTypeName(const uint32_t NT) { @@ -3338,7 +3434,30 @@ static std::string getFreeBSDNoteTypeName(const uint32_t NT) { std::string string; raw_string_ostream OS(string); OS << format("Unknown note type (0x%08x)", NT); - return string; + return OS.str(); +} + +static std::string getAMDGPUNoteTypeName(const uint32_t NT) { + static const struct { + uint32_t ID; + const char *Name; + } Notes[] = { + {ELF::NT_AMD_AMDGPU_HSA_METADATA, + "NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)"}, + {ELF::NT_AMD_AMDGPU_ISA, + "NT_AMD_AMDGPU_ISA (ISA Version)"}, + {ELF::NT_AMD_AMDGPU_PAL_METADATA, + "NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)"} + }; + + for (const auto &Note : Notes) + if (Note.ID == NT) + return std::string(Note.Name); + + std::string string; + raw_string_ostream OS(string); + OS << format("Unknown note type (0x%08x)", NT); + return OS.str(); } template <typename ELFT> @@ -3381,6 +3500,39 @@ static void printGNUNote(raw_ostream &OS, uint32_t NoteType, OS << '\n'; } +template <typename ELFT> +static void printAMDGPUNote(raw_ostream &OS, uint32_t NoteType, + ArrayRef<typename ELFFile<ELFT>::Elf_Word> Words, + size_t Size) { + switch (NoteType) { + default: + return; + case ELF::NT_AMD_AMDGPU_HSA_METADATA: + OS << " HSA Metadata:\n" + << StringRef(reinterpret_cast<const char *>(Words.data()), Size); + break; + case ELF::NT_AMD_AMDGPU_ISA: + OS << " ISA Version:\n" + << " " + << StringRef(reinterpret_cast<const char *>(Words.data()), Size); + break; + case ELF::NT_AMD_AMDGPU_PAL_METADATA: + const uint32_t *PALMetadataBegin = reinterpret_cast<const uint32_t *>(Words.data()); + const uint32_t *PALMetadataEnd = PALMetadataBegin + Size; + std::vector<uint32_t> PALMetadata(PALMetadataBegin, PALMetadataEnd); + std::string PALMetadataString; + auto Error = AMDGPU::PALMD::toString(PALMetadata, PALMetadataString); + OS << " PAL Metadata:\n"; + if (Error) { + OS << " Invalid"; + return; + } + OS << PALMetadataString; + break; + } + OS.flush(); +} + template <class ELFT> void GNUStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) { const Elf_Ehdr *e = Obj->getHeader(); @@ -3421,6 +3573,9 @@ void GNUStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) { printGNUNote<ELFT>(OS, Type, Descriptor, DescriptorSize); } else if (Name == "FreeBSD") { OS << getFreeBSDNoteTypeName(Type) << '\n'; + } else if (Name == "AMD") { + OS << getAMDGPUNoteTypeName(Type) << '\n'; + printAMDGPUNote<ELFT>(OS, Type, Descriptor, DescriptorSize); } else { OS << "Unknown note type: (" << format_hex(Type, 10) << ')'; } @@ -3454,13 +3609,22 @@ template <class ELFT> void LLVMStyle<ELFT>::printFileHeaders(const ELFO *Obj) { makeArrayRef(ElfDataEncoding)); W.printNumber("FileVersion", e->e_ident[ELF::EI_VERSION]); - // Handle architecture specific OS/ABI values. - if (e->e_machine == ELF::EM_AMDGPU && - e->e_ident[ELF::EI_OSABI] == ELF::ELFOSABI_AMDGPU_HSA) - W.printHex("OS/ABI", "AMDGPU_HSA", ELF::ELFOSABI_AMDGPU_HSA); - else - W.printEnum("OS/ABI", e->e_ident[ELF::EI_OSABI], - makeArrayRef(ElfOSABI)); + auto OSABI = makeArrayRef(ElfOSABI); + if (e->e_ident[ELF::EI_OSABI] >= ELF::ELFOSABI_FIRST_ARCH && + e->e_ident[ELF::EI_OSABI] <= ELF::ELFOSABI_LAST_ARCH) { + switch (e->e_machine) { + case ELF::EM_AMDGPU: + OSABI = makeArrayRef(AMDGPUElfOSABI); + break; + case ELF::EM_ARM: + OSABI = makeArrayRef(ARMElfOSABI); + break; + case ELF::EM_TI_C6000: + OSABI = makeArrayRef(C6000ElfOSABI); + break; + } + } + W.printEnum("OS/ABI", e->e_ident[ELF::EI_OSABI], OSABI); W.printNumber("ABIVersion", e->e_ident[ELF::EI_ABIVERSION]); W.printBinary("Unused", makeArrayRef(e->e_ident).slice(ELF::EI_PAD)); } @@ -3475,6 +3639,11 @@ template <class ELFT> void LLVMStyle<ELFT>::printFileHeaders(const ELFO *Obj) { W.printFlags("Flags", e->e_flags, makeArrayRef(ElfHeaderMipsFlags), unsigned(ELF::EF_MIPS_ARCH), unsigned(ELF::EF_MIPS_ABI), unsigned(ELF::EF_MIPS_MACH)); + else if (e->e_machine == EM_AMDGPU) + W.printFlags("Flags", e->e_flags, makeArrayRef(ElfHeaderAMDGPUFlags), + unsigned(ELF::EF_AMDGPU_ARCH)); + else if (e->e_machine == EM_RISCV) + W.printFlags("Flags", e->e_flags, makeArrayRef(ElfHeaderRISCVFlags)); else W.printFlags("Flags", e->e_flags); W.printNumber("HeaderSize", e->e_ehsize); @@ -3489,36 +3658,32 @@ template <class ELFT> void LLVMStyle<ELFT>::printFileHeaders(const ELFO *Obj) { template <class ELFT> void LLVMStyle<ELFT>::printGroupSections(const ELFO *Obj) { DictScope Lists(W, "Groups"); - uint32_t SectionIndex = 0; - bool HasGroups = false; - for (const Elf_Shdr &Sec : unwrapOrError(Obj->sections())) { - if (Sec.sh_type == ELF::SHT_GROUP) { - HasGroups = true; - const Elf_Shdr *Symtab = unwrapOrError(Obj->getSection(Sec.sh_link)); - StringRef StrTable = unwrapOrError(Obj->getStringTableForSymtab(*Symtab)); - const Elf_Sym *Sym = - unwrapOrError(Obj->template getEntry<Elf_Sym>(Symtab, Sec.sh_info)); - auto Data = unwrapOrError( - Obj->template getSectionContentsAsArray<Elf_Word>(&Sec)); - DictScope D(W, "Group"); - StringRef Name = unwrapOrError(Obj->getSectionName(&Sec)); - W.printNumber("Name", Name, Sec.sh_name); - W.printNumber("Index", SectionIndex); - W.printHex("Type", getGroupType(Data[0]), Data[0]); - W.startLine() << "Signature: " << StrTable.data() + Sym->st_name << "\n"; - { - ListScope L(W, "Section(s) in group"); - size_t Member = 1; - while (Member < Data.size()) { - auto Sec = unwrapOrError(Obj->getSection(Data[Member])); - const StringRef Name = unwrapOrError(Obj->getSectionName(Sec)); - W.startLine() << Name << " (" << Data[Member++] << ")\n"; - } + std::vector<GroupSection> V = getGroups<ELFT>(Obj); + DenseMap<uint64_t, const GroupSection *> Map = mapSectionsToGroups(V); + for (const GroupSection &G : V) { + DictScope D(W, "Group"); + W.printNumber("Name", G.Name, G.ShName); + W.printNumber("Index", G.Index); + W.printHex("Type", getGroupType(G.Type), G.Type); + W.startLine() << "Signature: " << G.Signature << "\n"; + + ListScope L(W, "Section(s) in group"); + for (const GroupMember &GM : G.Members) { + const GroupSection *MainGroup = Map[GM.Index]; + if (MainGroup != &G) { + W.flush(); + errs() << "Error: " << GM.Name << " (" << GM.Index + << ") in a group " + G.Name + " (" << G.Index + << ") is already in a group " + MainGroup->Name + " (" + << MainGroup->Index << ")\n"; + errs().flush(); + continue; } + W.startLine() << GM.Name << " (" << GM.Index << ")\n"; } - ++SectionIndex; } - if (!HasGroups) + + if (V.empty()) W.startLine() << "There are no group sections in the file.\n"; } @@ -3529,7 +3694,9 @@ template <class ELFT> void LLVMStyle<ELFT>::printRelocations(const ELFO *Obj) { for (const Elf_Shdr &Sec : unwrapOrError(Obj->sections())) { ++SectionNumber; - if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA) + if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA && + Sec.sh_type != ELF::SHT_ANDROID_REL && + Sec.sh_type != ELF::SHT_ANDROID_RELA) continue; StringRef Name = unwrapOrError(Obj->getSectionName(&Sec)); @@ -3562,6 +3729,11 @@ void LLVMStyle<ELFT>::printRelocations(const Elf_Shdr *Sec, const ELFO *Obj) { for (const Elf_Rela &R : unwrapOrError(Obj->relas(Sec))) printRelocation(Obj, R, SymTab); break; + case ELF::SHT_ANDROID_REL: + case ELF::SHT_ANDROID_RELA: + for (const Elf_Rela &R : unwrapOrError(Obj->android_relas(Sec))) + printRelocation(Obj, R, SymTab); + break; } } diff --git a/tools/llvm-readobj/ObjDumper.h b/tools/llvm-readobj/ObjDumper.h index 43883c2d2176..c5b331d944a2 100644 --- a/tools/llvm-readobj/ObjDumper.h +++ b/tools/llvm-readobj/ObjDumper.h @@ -19,7 +19,7 @@ class COFFImportFile; class ObjectFile; } namespace codeview { -class TypeTableBuilder; +class MergingTypeTableBuilder; } class ScopedPrinter; @@ -58,9 +58,6 @@ public: virtual void printMipsReginfo() { } virtual void printMipsOptions() { } - // Only implemented for AMDGPU ELF at this time. - virtual void printAMDGPUCodeObjectMetadata() {} - // Only implemented for PE/COFF. virtual void printCOFFImports() { } virtual void printCOFFExports() { } @@ -70,8 +67,9 @@ public: virtual void printCOFFResources() {} virtual void printCOFFLoadConfig() { } virtual void printCodeViewDebugInfo() { } - virtual void mergeCodeViewTypes(llvm::codeview::TypeTableBuilder &CVIDs, - llvm::codeview::TypeTableBuilder &CVTypes) {} + virtual void + mergeCodeViewTypes(llvm::codeview::MergingTypeTableBuilder &CVIDs, + llvm::codeview::MergingTypeTableBuilder &CVTypes) {} // Only implemented for MachO. virtual void printMachODataInCode() { } @@ -105,9 +103,9 @@ std::error_code createWasmDumper(const object::ObjectFile *Obj, void dumpCOFFImportFile(const object::COFFImportFile *File); -void dumpCodeViewMergedTypes(ScopedPrinter &Writer, - llvm::codeview::TypeTableBuilder &IDTable, - llvm::codeview::TypeTableBuilder &TypeTable); +void dumpCodeViewMergedTypes( + ScopedPrinter &Writer, llvm::codeview::MergingTypeTableBuilder &IDTable, + llvm::codeview::MergingTypeTableBuilder &TypeTable); } // namespace llvm diff --git a/tools/llvm-readobj/WasmDumper.cpp b/tools/llvm-readobj/WasmDumper.cpp index 266226d59ee8..77711e749aa0 100644 --- a/tools/llvm-readobj/WasmDumper.cpp +++ b/tools/llvm-readobj/WasmDumper.cpp @@ -83,9 +83,9 @@ void WasmDumper::printRelocation(const SectionRef &Section, bool HasAddend = false; switch (RelocType) { - case wasm::R_WEBASSEMBLY_GLOBAL_ADDR_LEB: - case wasm::R_WEBASSEMBLY_GLOBAL_ADDR_SLEB: - case wasm::R_WEBASSEMBLY_GLOBAL_ADDR_I32: + case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB: + case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB: + case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32: HasAddend = true; break; default: @@ -148,7 +148,7 @@ void WasmDumper::printSections() { const WasmSection &WasmSec = Obj->getWasmSection(Section); DictScope SectionD(W, "Section"); W.printEnum("Type", WasmSec.Type, makeArrayRef(WasmSectionTypes)); - W.printNumber("Size", (uint64_t)WasmSec.Content.size()); + W.printNumber("Size", static_cast<uint64_t>(WasmSec.Content.size())); W.printNumber("Offset", WasmSec.Offset); switch (WasmSec.Type) { case wasm::WASM_SEC_CUSTOM: @@ -156,10 +156,21 @@ void WasmDumper::printSections() { if (WasmSec.Name == "linking") { const wasm::WasmLinkingData &LinkingData = Obj->linkingData(); W.printNumber("DataSize", LinkingData.DataSize); - if (LinkingData.DataAlignment) - W.printNumber("DataAlignment", LinkingData.DataAlignment); } break; + case wasm::WASM_SEC_DATA: { + ListScope Group(W, "Segments"); + for (const WasmSegment &Segment : Obj->dataSegments()) { + const wasm::WasmDataSegment& Seg = Segment.Data; + DictScope Group(W, "Segment"); + if (!Seg.Name.empty()) + W.printString("Name", Seg.Name); + W.printNumber("Size", static_cast<uint64_t>(Seg.Content.size())); + if (Seg.Offset.Opcode == wasm::WASM_OPCODE_I32_CONST) + W.printNumber("Offset", Seg.Offset.Value.Int32); + } + break; + } case wasm::WASM_SEC_MEMORY: ListScope Group(W, "Memories"); for (const wasm::WasmLimits &Memory : Obj->memories()) { diff --git a/tools/llvm-readobj/WindowsResourceDumper.cpp b/tools/llvm-readobj/WindowsResourceDumper.cpp new file mode 100644 index 000000000000..1f568a963671 --- /dev/null +++ b/tools/llvm-readobj/WindowsResourceDumper.cpp @@ -0,0 +1,82 @@ +//===-- WindowsResourceDumper.cpp - Windows Resource printer --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the Windows resource (.res) dumper for llvm-readobj. +// +//===----------------------------------------------------------------------===// + +#include "WindowsResourceDumper.h" +#include "Error.h" +#include "llvm/Object/WindowsResource.h" +#include "llvm/Support/ConvertUTF.h" +#include "llvm/Support/ScopedPrinter.h" + +namespace llvm { +namespace object { +namespace WindowsRes { + +std::string stripUTF16(const ArrayRef<UTF16> &UTF16Str) { + std::string Result; + Result.reserve(UTF16Str.size()); + + for (UTF16 Ch : UTF16Str) { + // UTF16Str will have swapped byte order in case of big-endian machines. + // Swap it back in such a case. + uint16_t ChValue = support::endian::byte_swap(Ch, support::little); + if (ChValue <= 0xFF) + Result += ChValue; + else + Result += '?'; + } + return Result; +} + +Error Dumper::printData() { + auto EntryPtrOrErr = WinRes->getHeadEntry(); + if (!EntryPtrOrErr) + return EntryPtrOrErr.takeError(); + auto EntryPtr = *EntryPtrOrErr; + + bool IsEnd = false; + while (!IsEnd) { + printEntry(EntryPtr); + + if (auto Err = EntryPtr.moveNext(IsEnd)) + return Err; + } + return Error::success(); +} + +void Dumper::printEntry(const ResourceEntryRef &Ref) { + if (Ref.checkTypeString()) { + auto NarrowStr = stripUTF16(Ref.getTypeString()); + SW.printString("Resource type (string)", NarrowStr); + } else + SW.printNumber("Resource type (int)", Ref.getTypeID()); + + if (Ref.checkNameString()) { + auto NarrowStr = stripUTF16(Ref.getNameString()); + SW.printString("Resource name (string)", NarrowStr); + } else + SW.printNumber("Resource name (int)", Ref.getNameID()); + + SW.printNumber("Data version", Ref.getDataVersion()); + SW.printHex("Memory flags", Ref.getMemoryFlags()); + SW.printNumber("Language ID", Ref.getLanguage()); + SW.printNumber("Version (major)", Ref.getMajorVersion()); + SW.printNumber("Version (minor)", Ref.getMinorVersion()); + SW.printNumber("Characteristics", Ref.getCharacteristics()); + SW.printNumber("Data size", (uint64_t)Ref.getData().size()); + SW.printBinary("Data:", Ref.getData()); + SW.startLine() << "\n"; +} + +} // namespace WindowsRes +} // namespace object +} // namespace llvm diff --git a/tools/llvm-readobj/WindowsResourceDumper.h b/tools/llvm-readobj/WindowsResourceDumper.h new file mode 100644 index 000000000000..ca6da4046605 --- /dev/null +++ b/tools/llvm-readobj/WindowsResourceDumper.h @@ -0,0 +1,37 @@ +//===- WindowsResourceDumper.h - Windows Resource printer -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVM_READOBJ_WINDOWSRESOURCEDUMPER_H +#define LLVM_TOOLS_LLVM_READOBJ_WINDOWSRESOURCEDUMPER_H + +#include "llvm/Object/WindowsResource.h" +#include "llvm/Support/ScopedPrinter.h" + +namespace llvm { +namespace object { +namespace WindowsRes { + +class Dumper { +public: + Dumper(WindowsResource *Res, ScopedPrinter &SW) : SW(SW), WinRes(Res) {} + + Error printData(); + +private: + ScopedPrinter &SW; + WindowsResource *WinRes; + + void printEntry(const ResourceEntryRef &Ref); +}; + +} // namespace WindowsRes +} // namespace object +} // namespace llvm + +#endif diff --git a/tools/llvm-readobj/llvm-readobj.cpp b/tools/llvm-readobj/llvm-readobj.cpp index 7bfb18fab12b..c076582794fe 100644 --- a/tools/llvm-readobj/llvm-readobj.cpp +++ b/tools/llvm-readobj/llvm-readobj.cpp @@ -22,12 +22,13 @@ #include "llvm-readobj.h" #include "Error.h" #include "ObjDumper.h" -#include "llvm/DebugInfo/CodeView/TypeTableBuilder.h" +#include "WindowsResourceDumper.h" +#include "llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h" #include "llvm/Object/Archive.h" #include "llvm/Object/COFFImportFile.h" -#include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/MachOUniversal.h" #include "llvm/Object/ObjectFile.h" +#include "llvm/Object/WindowsResource.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/DataTypes.h" @@ -39,9 +40,6 @@ #include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/Signals.h" #include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include <string> -#include <system_error> using namespace llvm; using namespace llvm::object; @@ -198,11 +196,6 @@ namespace opts { cl::opt<bool> MipsOptions("mips-options", cl::desc("Display the MIPS .MIPS.options section")); - // -amdgpu-code-object-metadata - cl::opt<bool> AMDGPUCodeObjectMetadata( - "amdgpu-code-object-metadata", - cl::desc("Display AMDGPU code object metadata")); - // -coff-imports cl::opt<bool> COFFImports("coff-imports", cl::desc("Display the PE/COFF import table")); @@ -356,8 +349,8 @@ struct ReadObjTypeTableBuilder { : Allocator(), IDTable(Allocator), TypeTable(Allocator) {} llvm::BumpPtrAllocator Allocator; - llvm::codeview::TypeTableBuilder IDTable; - llvm::codeview::TypeTableBuilder TypeTable; + llvm::codeview::MergingTypeTableBuilder IDTable; + llvm::codeview::MergingTypeTableBuilder TypeTable; }; } static ReadObjTypeTableBuilder CVTypes; @@ -438,9 +431,6 @@ static void dumpObject(const ObjectFile *Obj) { if (opts::MipsOptions) Dumper->printMipsOptions(); } - if (Obj->getArch() == llvm::Triple::amdgcn) - if (opts::AMDGPUCodeObjectMetadata) - Dumper->printAMDGPUCodeObjectMetadata(); if (opts::SectionGroups) Dumper->printGroupSections(); if (opts::HashHistogram) @@ -522,6 +512,15 @@ static void dumpMachOUniversalBinary(const MachOUniversalBinary *UBinary) { } } +/// @brief Dumps \a WinRes, Windows Resource (.res) file; +static void dumpWindowsResourceFile(WindowsResource *WinRes) { + ScopedPrinter Printer{outs()}; + WindowsRes::Dumper Dumper(WinRes, Printer); + if (auto Err = Dumper.printData()) + reportError(WinRes->getFileName(), std::move(Err)); +} + + /// @brief Opens \a File and dumps it. static void dumpInput(StringRef File) { @@ -540,6 +539,8 @@ static void dumpInput(StringRef File) { dumpObject(Obj); else if (COFFImportFile *Import = dyn_cast<COFFImportFile>(&Binary)) dumpCOFFImportFile(Import); + else if (WindowsResource *WinRes = dyn_cast<WindowsResource>(&Binary)) + dumpWindowsResourceFile(WinRes); else reportError(File, readobj_error::unrecognized_file_format); } @@ -564,8 +565,7 @@ int main(int argc, const char *argv[]) { if (opts::InputFilenames.size() == 0) opts::InputFilenames.push_back("-"); - std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(), - dumpInput); + llvm::for_each(opts::InputFilenames, dumpInput); if (opts::CodeViewMergedTypes) { ScopedPrinter W(outs()); diff --git a/tools/llvm-rtdyld/llvm-rtdyld.cpp b/tools/llvm-rtdyld/llvm-rtdyld.cpp index ba130ce80be8..b09594622ca9 100644 --- a/tools/llvm-rtdyld/llvm-rtdyld.cpp +++ b/tools/llvm-rtdyld/llvm-rtdyld.cpp @@ -24,7 +24,6 @@ #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/Object/MachO.h" #include "llvm/Object/SymbolSize.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/DynamicLibrary.h" @@ -37,7 +36,6 @@ #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" #include <list> -#include <system_error> using namespace llvm; using namespace llvm::object; @@ -178,10 +176,14 @@ public: void deregisterEHFrames() override {} void preallocateSlab(uint64_t Size) { - std::string Err; - sys::MemoryBlock MB = sys::Memory::AllocateRWX(Size, nullptr, &Err); + std::error_code EC; + sys::MemoryBlock MB = + sys::Memory::allocateMappedMemory(Size, nullptr, + sys::Memory::MF_READ | + sys::Memory::MF_WRITE, + EC); if (!MB.base()) - report_fatal_error("Can't allocate enough memory: " + Err); + report_fatal_error("Can't allocate enough memory: " + EC.message()); PreallocSlab = MB; UsePreallocation = true; @@ -222,10 +224,14 @@ uint8_t *TrivialMemoryManager::allocateCodeSection(uintptr_t Size, if (UsePreallocation) return allocateFromSlab(Size, Alignment, true /* isCode */); - std::string Err; - sys::MemoryBlock MB = sys::Memory::AllocateRWX(Size, nullptr, &Err); + std::error_code EC; + sys::MemoryBlock MB = + sys::Memory::allocateMappedMemory(Size, nullptr, + sys::Memory::MF_READ | + sys::Memory::MF_WRITE, + EC); if (!MB.base()) - report_fatal_error("MemoryManager allocation failed: " + Err); + report_fatal_error("MemoryManager allocation failed: " + EC.message()); FunctionMemory.push_back(MB); return (uint8_t*)MB.base(); } @@ -242,10 +248,14 @@ uint8_t *TrivialMemoryManager::allocateDataSection(uintptr_t Size, if (UsePreallocation) return allocateFromSlab(Size, Alignment, false /* isCode */); - std::string Err; - sys::MemoryBlock MB = sys::Memory::AllocateRWX(Size, nullptr, &Err); + std::error_code EC; + sys::MemoryBlock MB = + sys::Memory::allocateMappedMemory(Size, nullptr, + sys::Memory::MF_READ | + sys::Memory::MF_WRITE, + EC); if (!MB.base()) - report_fatal_error("MemoryManager allocation failed: " + Err); + report_fatal_error("MemoryManager allocation failed: " + EC.message()); DataMemory.push_back(MB); return (uint8_t*)MB.base(); } @@ -324,8 +334,8 @@ static int printLineInfoForInput(bool LoadObjects, bool UseDebugObj) { } } - std::unique_ptr<DIContext> Context( - new DWARFContextInMemory(*SymbolObj,LoadedObjInfo.get())); + std::unique_ptr<DIContext> Context = + DWARFContext::create(*SymbolObj, LoadedObjInfo.get()); std::vector<std::pair<SymbolRef, uint64_t>> SymAddr = object::computeSymbolSizes(*SymbolObj); @@ -453,9 +463,11 @@ static int executeInput() { // Make sure the memory is executable. // setExecutable will call InvalidateInstructionCache. - std::string ErrorStr; - if (!sys::Memory::setExecutable(FM, &ErrorStr)) - ErrorAndExit("unable to mark function executable: '" + ErrorStr + "'"); + if (auto EC = sys::Memory::protectMappedMemory(FM, + sys::Memory::MF_READ | + sys::Memory::MF_EXEC)) + ErrorAndExit("unable to mark function executable: '" + EC.message() + + "'"); } // Dispatch to _main(). diff --git a/tools/llvm-shlib/CMakeLists.txt b/tools/llvm-shlib/CMakeLists.txt index 907345a94023..b2109c8758df 100644 --- a/tools/llvm-shlib/CMakeLists.txt +++ b/tools/llvm-shlib/CMakeLists.txt @@ -37,13 +37,20 @@ endif() add_llvm_library(LLVM SHARED DISABLE_LLVM_LINK_LLVM_DYLIB SONAME ${SOURCES}) list(REMOVE_DUPLICATES LIB_NAMES) -if(("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") OR (MINGW) OR (HAIKU) OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "DragonFly")) # FIXME: It should be "GNU ld for elf" +if(("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") OR (MINGW) OR (HAIKU) + OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "DragonFly") + OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "SunOS")) # FIXME: It should be "GNU ld for elf" configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/simple_version_script.map.in ${LLVM_LIBRARY_DIR}/tools/llvm-shlib/simple_version_script.map) # GNU ld doesn't resolve symbols in the version script. - set(LIB_NAMES -Wl,--version-script,${LLVM_LIBRARY_DIR}/tools/llvm-shlib/simple_version_script.map -Wl,--whole-archive ${LIB_NAMES} -Wl,--no-whole-archive) + set(LIB_NAMES -Wl,--whole-archive ${LIB_NAMES} -Wl,--no-whole-archive) + if (NOT LLVM_LINKER_IS_SOLARISLD) + # Solaris ld does not accept global: *; so there is no way to version *all* global symbols + set(LIB_NAMES -Wl,--version-script,${LLVM_LIBRARY_DIR}/tools/llvm-shlib/simple_version_script.map ${LIB_NAMES}) + endif() elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") set(LIB_NAMES -Wl,-all_load ${LIB_NAMES}) endif() diff --git a/tools/llvm-size/CMakeLists.txt b/tools/llvm-size/CMakeLists.txt index 60345739c35a..7ef4f1769b84 100644 --- a/tools/llvm-size/CMakeLists.txt +++ b/tools/llvm-size/CMakeLists.txt @@ -6,3 +6,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_tool(llvm-size llvm-size.cpp ) + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(size llvm-size) +endif() diff --git a/tools/llvm-size/llvm-size.cpp b/tools/llvm-size/llvm-size.cpp index bdb118a264e8..cf35a5795e71 100644 --- a/tools/llvm-size/llvm-size.cpp +++ b/tools/llvm-size/llvm-size.cpp @@ -883,8 +883,7 @@ int main(int argc, char **argv) { InputFilenames.push_back("a.out"); MoreThanOneFile = InputFilenames.size() > 1; - std::for_each(InputFilenames.begin(), InputFilenames.end(), - printFileSectionSizes); + llvm::for_each(InputFilenames, printFileSectionSizes); if (OutputFormat == berkeley && TotalSizes) printBerkelyTotals(); diff --git a/tools/llvm-special-case-list-fuzzer/CMakeLists.txt b/tools/llvm-special-case-list-fuzzer/CMakeLists.txt new file mode 100644 index 000000000000..f4ebf7a8ce7b --- /dev/null +++ b/tools/llvm-special-case-list-fuzzer/CMakeLists.txt @@ -0,0 +1,8 @@ +set(LLVM_LINK_COMPONENTS + Support + FuzzMutate +) + +add_llvm_fuzzer(llvm-special-case-list-fuzzer + special-case-list-fuzzer.cpp + DUMMY_MAIN DummySpecialCaseListFuzzer.cpp) diff --git a/tools/llvm-special-case-list-fuzzer/DummySpecialCaseListFuzzer.cpp b/tools/llvm-special-case-list-fuzzer/DummySpecialCaseListFuzzer.cpp new file mode 100644 index 000000000000..e447419113b9 --- /dev/null +++ b/tools/llvm-special-case-list-fuzzer/DummySpecialCaseListFuzzer.cpp @@ -0,0 +1,19 @@ +//===--- DummySpecialCaseListFuzzer.cpp -----------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementation of main so we can build and test without linking libFuzzer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/FuzzMutate/FuzzerCLI.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); +int main(int argc, char *argv[]) { + return llvm::runFuzzerOnInputs(argc, argv, LLVMFuzzerTestOneInput); +} diff --git a/tools/llvm-special-case-list-fuzzer/special-case-list-fuzzer.cpp b/tools/llvm-special-case-list-fuzzer/special-case-list-fuzzer.cpp new file mode 100644 index 000000000000..e7e310b3c7f2 --- /dev/null +++ b/tools/llvm-special-case-list-fuzzer/special-case-list-fuzzer.cpp @@ -0,0 +1,26 @@ +//===--- special-case-list-fuzzer.cpp - Fuzzer for special case lists -----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SpecialCaseList.h" + +#include <cstdlib> + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + std::unique_ptr<llvm::MemoryBuffer> Buf = llvm::MemoryBuffer::getMemBuffer( + llvm::StringRef(reinterpret_cast<const char *>(Data), Size), "", false); + + if (!Buf) + return 0; + + std::string Error; + llvm::SpecialCaseList::create(Buf.get(), Error); + + return 0; +} diff --git a/tools/llvm-split/llvm-split.cpp b/tools/llvm-split/llvm-split.cpp index d340d18fccc9..03625fb4a854 100644 --- a/tools/llvm-split/llvm-split.cpp +++ b/tools/llvm-split/llvm-split.cpp @@ -55,8 +55,8 @@ int main(int argc, char **argv) { unsigned I = 0; SplitModule(std::move(M), NumOutputs, [&](std::unique_ptr<Module> MPart) { std::error_code EC; - std::unique_ptr<tool_output_file> Out(new tool_output_file( - OutputFilename + utostr(I++), EC, sys::fs::F_None)); + std::unique_ptr<ToolOutputFile> Out( + new ToolOutputFile(OutputFilename + utostr(I++), EC, sys::fs::F_None)); if (EC) { errs() << EC.message() << '\n'; exit(1); diff --git a/tools/llvm-stress/llvm-stress.cpp b/tools/llvm-stress/llvm-stress.cpp index 3945da7020b0..d8ec11251ff6 100644 --- a/tools/llvm-stress/llvm-stress.cpp +++ b/tools/llvm-stress/llvm-stress.cpp @@ -1,4 +1,4 @@ -//===-- llvm-stress.cpp - Generate random LL files to stress-test LLVM ----===// +//===- llvm-stress.cpp - Generate random LL files to stress-test LLVM -----===// // // The LLVM Compiler Infrastructure // @@ -12,32 +12,55 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CallingConv.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRPrintingPasses.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/LegacyPassManager.h" -#include "llvm/IR/LegacyPassNameParser.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" #include "llvm/IR/Verifier.h" -#include "llvm/Support/Debug.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/PluginLoader.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/raw_ostream.h" #include <algorithm> -#include <random> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <string> +#include <system_error> #include <vector> namespace llvm { static cl::opt<unsigned> SeedCL("seed", cl::desc("Seed used for randomness"), cl::init(0)); + static cl::opt<unsigned> SizeCL("size", cl::desc("The estimated size of the generated function (# of instrs)"), cl::init(100)); + static cl::opt<std::string> OutputFilename("o", cl::desc("Override output filename"), cl::value_desc("filename")); @@ -45,6 +68,7 @@ OutputFilename("o", cl::desc("Override output filename"), static LLVMContext Context; namespace cl { + template <> class parser<Type*> final : public basic_parser<Type*> { public: parser(Option &O) : basic_parser(O) {} @@ -70,14 +94,15 @@ public: StringRef getValueName() const override { return "IR scalar type"; } }; -} +} // end namespace cl static cl::list<Type*> AdditionalScalarTypes("types", cl::CommaSeparated, cl::desc("Additional IR scalar types " "(always includes i1, i8, i16, i32, i64, float and double)")); namespace { + /// A utility class to provide a pseudo-random number generator which is /// the same across all platforms. This is somewhat close to the libc /// implementation. Note: This is not a cryptographically secure pseudorandom @@ -111,9 +136,11 @@ public: } /// Make this like a C++11 random device - typedef uint32_t result_type; + using result_type = uint32_t ; + static constexpr result_type min() { return 0; } static constexpr result_type max() { return 0x7ffff; } + uint32_t operator()() { uint32_t Val = Rand(); assert(Val <= max() && "Random value out of range"); @@ -149,18 +176,19 @@ Function *GenEmptyFunction(Module *M) { /// modifying and adding new random instructions. struct Modifier { /// Used to store the randomly generated values. - typedef std::vector<Value*> PieceTable; + using PieceTable = std::vector<Value *>; public: /// C'tor - Modifier(BasicBlock *Block, PieceTable *PT, Random *R): - BB(Block),PT(PT),Ran(R),Context(BB->getContext()) {} + Modifier(BasicBlock *Block, PieceTable *PT, Random *R) + : BB(Block), PT(PT), Ran(R), Context(BB->getContext()) {} /// virtual D'tor to silence warnings. - virtual ~Modifier() {} + virtual ~Modifier() = default; /// Add a new instruction. virtual void Act() = 0; + /// Add N new instructions, virtual void ActN(unsigned n) { for (unsigned i=0; i<n; ++i) @@ -298,16 +326,21 @@ protected: /// Basic block to populate BasicBlock *BB; + /// Value table PieceTable *PT; + /// Random number generator Random *Ran; + /// Context LLVMContext &Context; }; struct LoadModifier: public Modifier { - LoadModifier(BasicBlock *BB, PieceTable *PT, Random *R):Modifier(BB, PT, R) {} + LoadModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} + void Act() override { // Try to use predefined pointers. If non-exist, use undef pointer value; Value *Ptr = getRandomPointerValue(); @@ -317,7 +350,9 @@ struct LoadModifier: public Modifier { }; struct StoreModifier: public Modifier { - StoreModifier(BasicBlock *BB, PieceTable *PT, Random *R):Modifier(BB, PT, R) {} + StoreModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} + void Act() override { // Try to use predefined pointers. If non-exist, use undef pointer value; Value *Ptr = getRandomPointerValue(); @@ -335,7 +370,8 @@ struct StoreModifier: public Modifier { }; struct BinModifier: public Modifier { - BinModifier(BasicBlock *BB, PieceTable *PT, Random *R):Modifier(BB, PT, R) {} + BinModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} void Act() override { Value *Val0 = getRandomVal(); @@ -350,7 +386,6 @@ struct BinModifier: public Modifier { if (Val0->getType()->getScalarSizeInBits() == 1) return; - bool isFloat = Val0->getType()->getScalarType()->isFloatingPointTy(); Instruction* Term = BB->getTerminator(); unsigned R = getRandom() % (isFloat ? 7 : 13); @@ -379,7 +414,9 @@ struct BinModifier: public Modifier { /// Generate constant values. struct ConstModifier: public Modifier { - ConstModifier(BasicBlock *BB, PieceTable *PT, Random *R):Modifier(BB, PT, R) {} + ConstModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} + void Act() override { Type *Ty = pickType(); @@ -428,7 +465,8 @@ struct ConstModifier: public Modifier { }; struct AllocaModifier: public Modifier { - AllocaModifier(BasicBlock *BB, PieceTable *PT, Random *R):Modifier(BB, PT, R){} + AllocaModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} void Act() override { Type *Tp = pickType(); @@ -439,8 +477,8 @@ struct AllocaModifier: public Modifier { }; struct ExtractElementModifier: public Modifier { - ExtractElementModifier(BasicBlock *BB, PieceTable *PT, Random *R): - Modifier(BB, PT, R) {} + ExtractElementModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} void Act() override { Value *Val0 = getRandomVectorValue(); @@ -453,9 +491,10 @@ struct ExtractElementModifier: public Modifier { }; struct ShuffModifier: public Modifier { - ShuffModifier(BasicBlock *BB, PieceTable *PT, Random *R):Modifier(BB, PT, R) {} - void Act() override { + ShuffModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} + void Act() override { Value *Val0 = getRandomVectorValue(); Value *Val1 = getRandomValue(Val0->getType()); @@ -480,8 +519,8 @@ struct ShuffModifier: public Modifier { }; struct InsertElementModifier: public Modifier { - InsertElementModifier(BasicBlock *BB, PieceTable *PT, Random *R): - Modifier(BB, PT, R) {} + InsertElementModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} void Act() override { Value *Val0 = getRandomVectorValue(); @@ -493,13 +532,13 @@ struct InsertElementModifier: public Modifier { "I", BB->getTerminator()); return PT->push_back(V); } - }; struct CastModifier: public Modifier { - CastModifier(BasicBlock *BB, PieceTable *PT, Random *R):Modifier(BB, PT, R) {} - void Act() override { + CastModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} + void Act() override { Value *V = getRandomVal(); Type *VTy = V->getType(); Type *DestTy = pickScalarType(); @@ -558,7 +597,6 @@ struct CastModifier: public Modifier { return PT->push_back( new SIToFPInst(V, DestTy, "FC", BB->getTerminator())); return PT->push_back(new UIToFPInst(V, DestTy, "FC", BB->getTerminator())); - } // Both floats. @@ -574,38 +612,37 @@ struct CastModifier: public Modifier { // for which there is no defined conversion. So do nothing. } } - }; struct SelectModifier: public Modifier { - SelectModifier(BasicBlock *BB, PieceTable *PT, Random *R): - Modifier(BB, PT, R) {} + SelectModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} void Act() override { // Try a bunch of different select configuration until a valid one is found. - Value *Val0 = getRandomVal(); - Value *Val1 = getRandomValue(Val0->getType()); + Value *Val0 = getRandomVal(); + Value *Val1 = getRandomValue(Val0->getType()); - Type *CondTy = Type::getInt1Ty(Context); + Type *CondTy = Type::getInt1Ty(Context); - // If the value type is a vector, and we allow vector select, then in 50% - // of the cases generate a vector select. - if (Val0->getType()->isVectorTy() && (getRandom() % 1)) { - unsigned NumElem = cast<VectorType>(Val0->getType())->getNumElements(); - CondTy = VectorType::get(CondTy, NumElem); - } + // If the value type is a vector, and we allow vector select, then in 50% + // of the cases generate a vector select. + if (Val0->getType()->isVectorTy() && (getRandom() % 1)) { + unsigned NumElem = cast<VectorType>(Val0->getType())->getNumElements(); + CondTy = VectorType::get(CondTy, NumElem); + } - Value *Cond = getRandomValue(CondTy); - Value *V = SelectInst::Create(Cond, Val0, Val1, "Sl", BB->getTerminator()); - return PT->push_back(V); + Value *Cond = getRandomValue(CondTy); + Value *V = SelectInst::Create(Cond, Val0, Val1, "Sl", BB->getTerminator()); + return PT->push_back(V); } }; - struct CmpModifier: public Modifier { - CmpModifier(BasicBlock *BB, PieceTable *PT, Random *R):Modifier(BB, PT, R) {} - void Act() override { + CmpModifier(BasicBlock *BB, PieceTable *PT, Random *R) + : Modifier(BB, PT, R) {} + void Act() override { Value *Val0 = getRandomVal(); Value *Val1 = getRandomValue(Val0->getType()); @@ -689,7 +726,7 @@ static void IntroduceControlFlow(Function *F, Random &R) { } } -} +} // end namespace llvm int main(int argc, char **argv) { using namespace llvm; @@ -699,7 +736,7 @@ int main(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv, "llvm codegen stress-tester\n"); llvm_shutdown_obj Y; - auto M = make_unique<Module>("/tmp/autogen.bc", Context); + auto M = llvm::make_unique<Module>("/tmp/autogen.bc", Context); Function *F = GenEmptyFunction(M.get()); // Pick an initial seed value @@ -710,13 +747,13 @@ int main(int argc, char **argv) { IntroduceControlFlow(F, R); // Figure out what stream we are supposed to write to... - std::unique_ptr<tool_output_file> Out; + std::unique_ptr<ToolOutputFile> Out; // Default to standard output. if (OutputFilename.empty()) OutputFilename = "-"; std::error_code EC; - Out.reset(new tool_output_file(OutputFilename, EC, sys::fs::F_None)); + Out.reset(new ToolOutputFile(OutputFilename, EC, sys::fs::F_None)); if (EC) { errs() << EC.message() << '\n'; return 1; diff --git a/tools/llvm-strings/CMakeLists.txt b/tools/llvm-strings/CMakeLists.txt index 9339892a4997..390f11751397 100644 --- a/tools/llvm-strings/CMakeLists.txt +++ b/tools/llvm-strings/CMakeLists.txt @@ -8,3 +8,6 @@ add_llvm_tool(llvm-strings llvm-strings.cpp ) +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(strings llvm-strings) +endif() diff --git a/tools/llvm-strings/llvm-strings.cpp b/tools/llvm-strings/llvm-strings.cpp index 5053e9b97b78..638d58fce2b7 100644 --- a/tools/llvm-strings/llvm-strings.cpp +++ b/tools/llvm-strings/llvm-strings.cpp @@ -41,6 +41,12 @@ static cl::opt<int> cl::init(4)); static cl::alias MinLengthShort("n", cl::desc(""), cl::aliasopt(MinLength)); +static cl::opt<bool> + AllSections("all", + cl::desc("Check all sections, not just the data section")); +static cl::alias AllSectionsShort("a", cl::desc(""), + cl::aliasopt(AllSections)); + enum radix { none, octal, hexadecimal, decimal }; static cl::opt<radix> Radix("radix", cl::desc("print the offset within the file"), diff --git a/tools/llvm-symbolizer/CMakeLists.txt b/tools/llvm-symbolizer/CMakeLists.txt index b04c45ff7442..d9b05208afd8 100644 --- a/tools/llvm-symbolizer/CMakeLists.txt +++ b/tools/llvm-symbolizer/CMakeLists.txt @@ -14,3 +14,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_tool(llvm-symbolizer llvm-symbolizer.cpp ) + +if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(addr2line llvm-symbolizer) +endif() diff --git a/tools/llvm-symbolizer/llvm-symbolizer.cpp b/tools/llvm-symbolizer/llvm-symbolizer.cpp index c9e0cc2b3b05..b51ec513f23b 100644 --- a/tools/llvm-symbolizer/llvm-symbolizer.cpp +++ b/tools/llvm-symbolizer/llvm-symbolizer.cpp @@ -69,6 +69,10 @@ ClBinaryName("obj", cl::init(""), cl::desc("Path to object file to be symbolized (if not provided, " "object file should be specified for each input line)")); +static cl::opt<std::string> + ClDwpName("dwp", cl::init(""), + cl::desc("Path to DWP file to be use for any split CUs")); + static cl::list<std::string> ClDsymHint("dsym-hint", cl::ZeroOrMore, cl::desc("Path to .dSYM bundles to search for debug info for the " @@ -191,11 +195,13 @@ int main(int argc, char **argv) { auto ResOrErr = Symbolizer.symbolizeData(ModuleName, ModuleOffset); Printer << (error(ResOrErr) ? DIGlobal() : ResOrErr.get()); } else if (ClPrintInlining) { - auto ResOrErr = Symbolizer.symbolizeInlinedCode(ModuleName, ModuleOffset); + auto ResOrErr = + Symbolizer.symbolizeInlinedCode(ModuleName, ModuleOffset, ClDwpName); Printer << (error(ResOrErr) ? DIInliningInfo() : ResOrErr.get()); } else { - auto ResOrErr = Symbolizer.symbolizeCode(ModuleName, ModuleOffset); + auto ResOrErr = + Symbolizer.symbolizeCode(ModuleName, ModuleOffset, ClDwpName); Printer << (error(ResOrErr) ? DILineInfo() : ResOrErr.get()); } outs() << "\n"; diff --git a/tools/llvm-xray/CMakeLists.txt b/tools/llvm-xray/CMakeLists.txt index 6312e7ac47f4..b1df67f9b770 100644 --- a/tools/llvm-xray/CMakeLists.txt +++ b/tools/llvm-xray/CMakeLists.txt @@ -15,6 +15,7 @@ set(LLVM_XRAY_TOOLS xray-extract.cc xray-graph.cc xray-graph-diff.cc + xray-stacks.cc xray-registry.cc) add_llvm_tool(llvm-xray llvm-xray.cc ${LLVM_XRAY_TOOLS}) diff --git a/tools/llvm-xray/llvm-xray.cc b/tools/llvm-xray/llvm-xray.cc index 98303e7be15c..17cc9f90dd71 100644 --- a/tools/llvm-xray/llvm-xray.cc +++ b/tools/llvm-xray/llvm-xray.cc @@ -18,7 +18,6 @@ // #include "xray-registry.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/FileSystem.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -46,4 +45,5 @@ int main(int argc, char *argv[]) { // If all else fails, we still print the usage message. cl::PrintHelpMessage(false, true); + return 0; } diff --git a/tools/llvm-xray/trie-node.h b/tools/llvm-xray/trie-node.h new file mode 100644 index 000000000000..e6ba4e215b91 --- /dev/null +++ b/tools/llvm-xray/trie-node.h @@ -0,0 +1,92 @@ +//===- trie-node.h - XRay Call Stack Data Structure -----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provides a data structure and routines for working with call stacks +// of instrumented functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVM_XRAY_STACK_TRIE_H +#define LLVM_TOOLS_LLVM_XRAY_STACK_TRIE_H + +#include <forward_list> +#include <numeric> + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" + +/// A type to represent a trie of invocations. It is useful to construct a +/// graph of these nodes from reading an XRay trace, such that each function +/// call can be placed in a larger context. +/// +/// The template parameter allows users of the template to attach their own +/// data elements to each node in the invocation graph. +template <typename AssociatedData> struct TrieNode { + /// The function ID. + int32_t FuncId; + + /// The caller of this function. + TrieNode<AssociatedData> *Parent; + + /// The callees from this function. + llvm::SmallVector<TrieNode<AssociatedData> *, 4> Callees; + + /// Additional parameterized data on each node. + AssociatedData ExtraData; +}; + +/// Merges together two TrieNodes with like function ids, aggregating their +/// callee lists and durations. The caller must provide storage where new merged +/// nodes can be allocated in the form of a linked list. +template <typename T, typename Callable> +TrieNode<T> * +mergeTrieNodes(const TrieNode<T> &Left, const TrieNode<T> &Right, + /*Non-deduced pointer type for nullptr compatibility*/ + typename std::remove_reference<TrieNode<T> *>::type NewParent, + std::forward_list<TrieNode<T>> &NodeStore, + Callable &&MergeCallable) { + llvm::function_ref<T(const T &, const T &)> MergeFn( + std::forward<Callable>(MergeCallable)); + assert(Left.FuncId == Right.FuncId); + NodeStore.push_front(TrieNode<T>{ + Left.FuncId, NewParent, {}, MergeFn(Left.ExtraData, Right.ExtraData)}); + auto I = NodeStore.begin(); + auto *Node = &*I; + + // Build a map of callees from the left side. + llvm::DenseMap<int32_t, TrieNode<T> *> LeftCalleesByFuncId; + for (auto *Callee : Left.Callees) { + LeftCalleesByFuncId[Callee->FuncId] = Callee; + } + + // Iterate through the right side, either merging with the map values or + // directly adding to the Callees vector. The iteration also removes any + // merged values from the left side map. + // TODO: Unroll into iterative and explicit stack for efficiency. + for (auto *Callee : Right.Callees) { + auto iter = LeftCalleesByFuncId.find(Callee->FuncId); + if (iter != LeftCalleesByFuncId.end()) { + Node->Callees.push_back( + mergeTrieNodes(*(iter->second), *Callee, Node, NodeStore, MergeFn)); + LeftCalleesByFuncId.erase(iter); + } else { + Node->Callees.push_back(Callee); + } + } + + // Add any callees that weren't found in the right side. + for (auto MapPairIter : LeftCalleesByFuncId) { + Node->Callees.push_back(MapPairIter.second); + } + + return Node; +} + +#endif // LLVM_TOOLS_LLVM_XRAY_STACK_TRIE_H diff --git a/tools/llvm-xray/xray-account.cc b/tools/llvm-xray/xray-account.cc index 13654c3911f7..7b684aad693d 100644 --- a/tools/llvm-xray/xray-account.cc +++ b/tools/llvm-xray/xray-account.cc @@ -146,13 +146,16 @@ bool LatencyAccountant::accountRecord(const XRayRecord &Record) { auto &ThreadStack = PerThreadFunctionStack[Record.TId]; switch (Record.Type) { - case RecordTypes::ENTER: { - // Function Enter + case RecordTypes::ENTER: + case RecordTypes::ENTER_ARG: { ThreadStack.emplace_back(Record.FuncId, Record.TSC); break; } - case RecordTypes::EXIT: { - // Function Exit + case RecordTypes::EXIT: + case RecordTypes::TAIL_EXIT: { + if (ThreadStack.empty()) + return false; + if (ThreadStack.back().first == Record.FuncId) { const auto &Top = ThreadStack.back(); recordLatency(Top.first, diff(Top.second, Record.TSC)); @@ -407,6 +410,28 @@ void LatencyAccountant::exportStatsAsCSV(raw_ostream &OS, using namespace llvm::xray; +namespace llvm { +template <> struct format_provider<llvm::xray::RecordTypes> { + static void format(const llvm::xray::RecordTypes &T, raw_ostream &Stream, + StringRef Style) { + switch(T) { + case RecordTypes::ENTER: + Stream << "enter"; + break; + case RecordTypes::ENTER_ARG: + Stream << "enter-arg"; + break; + case RecordTypes::EXIT: + Stream << "exit"; + break; + case RecordTypes::TAIL_EXIT: + Stream << "tail-exit"; + break; + } + } +}; +} // namespace llvm + static CommandRegistration Unused(&Account, []() -> Error { InstrumentationMap Map; if (!AccountInstrMap.empty()) { @@ -445,11 +470,22 @@ static CommandRegistration Unused(&Account, []() -> Error { for (const auto &Record : T) { if (FCA.accountRecord(Record)) continue; + errs() + << "Error processing record: " + << llvm::formatv( + R"({{type: {0}; cpu: {1}; record-type: {2}; function-id: {3}; tsc: {4}; thread-id: {5}}})", + Record.RecordType, Record.CPU, Record.Type, Record.FuncId, + Record.TId) + << '\n'; for (const auto &ThreadStack : FCA.getPerThreadFunctionStack()) { errs() << "Thread ID: " << ThreadStack.first << "\n"; + if (ThreadStack.second.empty()) { + errs() << " (empty stack)\n"; + continue; + } auto Level = ThreadStack.second.size(); for (const auto &Entry : llvm::reverse(ThreadStack.second)) - errs() << "#" << Level-- << "\t" + errs() << " #" << Level-- << "\t" << FuncIdHelper.SymbolOrNumber(Entry.first) << '\n'; } if (!AccountKeepGoing) diff --git a/tools/llvm-xray/xray-color-helper.cc b/tools/llvm-xray/xray-color-helper.cc index 7b6a73a5552b..61314d3c766a 100644 --- a/tools/llvm-xray/xray-color-helper.cc +++ b/tools/llvm-xray/xray-color-helper.cc @@ -10,8 +10,6 @@ // A class to get a color from a specified gradient. // //===----------------------------------------------------------------------===// -#include <algorithm> -#include <iostream> #include "xray-color-helper.h" #include "llvm/Support/FormatVariadic.h" diff --git a/tools/llvm-xray/xray-converter.cc b/tools/llvm-xray/xray-converter.cc index 2583ec951495..aa0da55207b3 100644 --- a/tools/llvm-xray/xray-converter.cc +++ b/tools/llvm-xray/xray-converter.cc @@ -12,10 +12,12 @@ //===----------------------------------------------------------------------===// #include "xray-converter.h" +#include "trie-node.h" #include "xray-registry.h" #include "llvm/DebugInfo/Symbolize/Symbolize.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" @@ -32,11 +34,14 @@ static cl::SubCommand Convert("convert", "Trace Format Conversion"); static cl::opt<std::string> ConvertInput(cl::Positional, cl::desc("<xray log file>"), cl::Required, cl::sub(Convert)); -enum class ConvertFormats { BINARY, YAML }; +enum class ConvertFormats { BINARY, YAML, CHROME_TRACE_EVENT }; static cl::opt<ConvertFormats> ConvertOutputFormat( "output-format", cl::desc("output format"), cl::values(clEnumValN(ConvertFormats::BINARY, "raw", "output in binary"), - clEnumValN(ConvertFormats::YAML, "yaml", "output in yaml")), + clEnumValN(ConvertFormats::YAML, "yaml", "output in yaml"), + clEnumValN(ConvertFormats::CHROME_TRACE_EVENT, "trace_event", + "Output in chrome's trace event format. " + "May be visualized with the Catapult trace viewer.")), cl::sub(Convert)); static cl::alias ConvertOutputFormat2("f", cl::aliasopt(ConvertOutputFormat), cl::desc("Alias for -output-format"), @@ -86,7 +91,7 @@ void TraceConverter::exportAsYAML(const Trace &Records, raw_ostream &OS) { Trace.Records.push_back({R.RecordType, R.CPU, R.Type, R.FuncId, Symbolize ? FuncIdHelper.SymbolOrNumber(R.FuncId) : llvm::to_string(R.FuncId), - R.TSC, R.TId}); + R.TSC, R.TId, R.CallArgs}); } Output Out(OS, nullptr, 0); Out << Trace; @@ -123,11 +128,15 @@ void TraceConverter::exportAsRAWv1(const Trace &Records, raw_ostream &OS) { Writer.write(static_cast<uint8_t>(R.CPU)); switch (R.Type) { case RecordTypes::ENTER: + case RecordTypes::ENTER_ARG: Writer.write(uint8_t{0}); break; case RecordTypes::EXIT: Writer.write(uint8_t{1}); break; + case RecordTypes::TAIL_EXIT: + Writer.write(uint8_t{2}); + break; } Writer.write(R.FuncId); Writer.write(R.TSC); @@ -138,6 +147,192 @@ void TraceConverter::exportAsRAWv1(const Trace &Records, raw_ostream &OS) { } } +namespace { + +// A structure that allows building a dictionary of stack ids for the Chrome +// trace event format. +struct StackIdData { + // Each Stack of function calls has a unique ID. + unsigned id; + + // Bookkeeping so that IDs can be maintained uniquely across threads. + // Traversal keeps sibling pointers to other threads stacks. This is helpful + // to determine when a thread encounters a new stack and should assign a new + // unique ID. + SmallVector<TrieNode<StackIdData> *, 4> siblings; +}; + +using StackTrieNode = TrieNode<StackIdData>; + +// A helper function to find the sibling nodes for an encountered function in a +// thread of execution. Relies on the invariant that each time a new node is +// traversed in a thread, sibling bidirectional pointers are maintained. +SmallVector<StackTrieNode *, 4> +findSiblings(StackTrieNode *parent, int32_t FnId, uint32_t TId, + const DenseMap<uint32_t, SmallVector<StackTrieNode *, 4>> + &StackRootsByThreadId) { + + SmallVector<StackTrieNode *, 4> Siblings{}; + + if (parent == nullptr) { + for (auto map_iter : StackRootsByThreadId) { + // Only look for siblings in other threads. + if (map_iter.first != TId) + for (auto node_iter : map_iter.second) { + if (node_iter->FuncId == FnId) + Siblings.push_back(node_iter); + } + } + return Siblings; + } + + for (auto *ParentSibling : parent->ExtraData.siblings) + for (auto node_iter : ParentSibling->Callees) + if (node_iter->FuncId == FnId) + Siblings.push_back(node_iter); + + return Siblings; +} + +// Given a function being invoked in a thread with id TId, finds and returns the +// StackTrie representing the function call stack. If no node exists, creates +// the node. Assigns unique IDs to stacks newly encountered among all threads +// and keeps sibling links up to when creating new nodes. +StackTrieNode *findOrCreateStackNode( + StackTrieNode *Parent, int32_t FuncId, uint32_t TId, + DenseMap<uint32_t, SmallVector<StackTrieNode *, 4>> &StackRootsByThreadId, + DenseMap<unsigned, StackTrieNode *> &StacksByStackId, unsigned *id_counter, + std::forward_list<StackTrieNode> &NodeStore) { + SmallVector<StackTrieNode *, 4> &ParentCallees = + Parent == nullptr ? StackRootsByThreadId[TId] : Parent->Callees; + auto match = find_if(ParentCallees, [FuncId](StackTrieNode *ParentCallee) { + return FuncId == ParentCallee->FuncId; + }); + if (match != ParentCallees.end()) + return *match; + + SmallVector<StackTrieNode *, 4> siblings = + findSiblings(Parent, FuncId, TId, StackRootsByThreadId); + if (siblings.empty()) { + NodeStore.push_front({FuncId, Parent, {}, {(*id_counter)++, {}}}); + StackTrieNode *CurrentStack = &NodeStore.front(); + StacksByStackId[*id_counter - 1] = CurrentStack; + ParentCallees.push_back(CurrentStack); + return CurrentStack; + } + unsigned stack_id = siblings[0]->ExtraData.id; + NodeStore.push_front({FuncId, Parent, {}, {stack_id, std::move(siblings)}}); + StackTrieNode *CurrentStack = &NodeStore.front(); + for (auto *sibling : CurrentStack->ExtraData.siblings) + sibling->ExtraData.siblings.push_back(CurrentStack); + ParentCallees.push_back(CurrentStack); + return CurrentStack; +} + +void writeTraceViewerRecord(raw_ostream &OS, int32_t FuncId, uint32_t TId, + bool Symbolize, + const FuncIdConversionHelper &FuncIdHelper, + double EventTimestampUs, + const StackTrieNode &StackCursor, + StringRef FunctionPhenotype) { + OS << " "; + OS << llvm::formatv( + R"({ "name" : "{0}", "ph" : "{1}", "tid" : "{2}", "pid" : "1", )" + R"("ts" : "{3:f3}", "sf" : "{4}" })", + (Symbolize ? FuncIdHelper.SymbolOrNumber(FuncId) + : llvm::to_string(FuncId)), + FunctionPhenotype, TId, EventTimestampUs, StackCursor.ExtraData.id); +} + +} // namespace + +void TraceConverter::exportAsChromeTraceEventFormat(const Trace &Records, + raw_ostream &OS) { + const auto &FH = Records.getFileHeader(); + auto CycleFreq = FH.CycleFrequency; + + unsigned id_counter = 0; + + OS << "{\n \"traceEvents\": ["; + DenseMap<uint32_t, StackTrieNode *> StackCursorByThreadId{}; + DenseMap<uint32_t, SmallVector<StackTrieNode *, 4>> StackRootsByThreadId{}; + DenseMap<unsigned, StackTrieNode *> StacksByStackId{}; + std::forward_list<StackTrieNode> NodeStore{}; + int loop_count = 0; + for (const auto &R : Records) { + if (loop_count++ == 0) + OS << "\n"; + else + OS << ",\n"; + + // Chrome trace event format always wants data in micros. + // CyclesPerMicro = CycleHertz / 10^6 + // TSC / CyclesPerMicro == TSC * 10^6 / CycleHertz == MicroTimestamp + // Could lose some precision here by converting the TSC to a double to + // multiply by the period in micros. 52 bit mantissa is a good start though. + // TODO: Make feature request to Chrome Trace viewer to accept ticks and a + // frequency or do some more involved calculation to avoid dangers of + // conversion. + double EventTimestampUs = double(1000000) / CycleFreq * double(R.TSC); + StackTrieNode *&StackCursor = StackCursorByThreadId[R.TId]; + switch (R.Type) { + case RecordTypes::ENTER: + case RecordTypes::ENTER_ARG: + StackCursor = findOrCreateStackNode(StackCursor, R.FuncId, R.TId, + StackRootsByThreadId, StacksByStackId, + &id_counter, NodeStore); + // Each record is represented as a json dictionary with function name, + // type of B for begin or E for end, thread id, process id (faked), + // timestamp in microseconds, and a stack frame id. The ids are logged + // in an id dictionary after the events. + writeTraceViewerRecord(OS, R.FuncId, R.TId, Symbolize, FuncIdHelper, + EventTimestampUs, *StackCursor, "B"); + break; + case RecordTypes::EXIT: + case RecordTypes::TAIL_EXIT: + // No entries to record end for. + if (StackCursor == nullptr) + break; + // Should we emit an END record anyway or account this condition? + // (And/Or in loop termination below) + StackTrieNode *PreviousCursor = nullptr; + do { + writeTraceViewerRecord(OS, StackCursor->FuncId, R.TId, Symbolize, + FuncIdHelper, EventTimestampUs, *StackCursor, + "E"); + PreviousCursor = StackCursor; + StackCursor = StackCursor->Parent; + } while (PreviousCursor->FuncId != R.FuncId && StackCursor != nullptr); + break; + } + } + OS << "\n ],\n"; // Close the Trace Events array. + OS << " " + << "\"displayTimeUnit\": \"ns\",\n"; + + // The stackFrames dictionary substantially reduces size of the output file by + // avoiding repeating the entire call stack of function names for each entry. + OS << R"( "stackFrames": {)"; + int stack_frame_count = 0; + for (auto map_iter : StacksByStackId) { + if (stack_frame_count++ == 0) + OS << "\n"; + else + OS << ",\n"; + OS << " "; + OS << llvm::formatv( + R"("{0}" : { "name" : "{1}")", map_iter.first, + (Symbolize ? FuncIdHelper.SymbolOrNumber(map_iter.second->FuncId) + : llvm::to_string(map_iter.second->FuncId))); + if (map_iter.second->Parent != nullptr) + OS << llvm::formatv(R"(, "parent": "{0}")", + map_iter.second->Parent->ExtraData.id); + OS << " }"; + } + OS << "\n }\n"; // Close the stack frames map. + OS << "}\n"; // Close the JSON entry. +} + namespace llvm { namespace xray { @@ -187,6 +382,9 @@ static CommandRegistration Unused(&Convert, []() -> Error { case ConvertFormats::BINARY: TC.exportAsRAWv1(T, OS); break; + case ConvertFormats::CHROME_TRACE_EVENT: + TC.exportAsChromeTraceEventFormat(T, OS); + break; } return Error::success(); }); diff --git a/tools/llvm-xray/xray-converter.h b/tools/llvm-xray/xray-converter.h index fa0d5e132f14..5f0a3ee298eb 100644 --- a/tools/llvm-xray/xray-converter.h +++ b/tools/llvm-xray/xray-converter.h @@ -15,8 +15,8 @@ #define LLVM_TOOLS_LLVM_XRAY_XRAY_CONVERTER_H #include "func-id-helper.h" -#include "llvm/XRay/XRayRecord.h" #include "llvm/XRay/Trace.h" +#include "llvm/XRay/XRayRecord.h" namespace llvm { namespace xray { @@ -31,6 +31,11 @@ public: void exportAsYAML(const Trace &Records, raw_ostream &OS); void exportAsRAWv1(const Trace &Records, raw_ostream &OS); + + /// For this conversion, the Function records within each thread are expected + /// to be in sorted TSC order. The trace event format encodes stack traces, so + /// the linear history is essential for correct output. + void exportAsChromeTraceEventFormat(const Trace &Records, raw_ostream &OS); }; } // namespace xray diff --git a/tools/llvm-xray/xray-extract.cc b/tools/llvm-xray/xray-extract.cc index 6b72b81ab814..cd87798d0e60 100644 --- a/tools/llvm-xray/xray-extract.cc +++ b/tools/llvm-xray/xray-extract.cc @@ -13,16 +13,11 @@ // //===----------------------------------------------------------------------===// -#include <type_traits> -#include <utility> #include "func-id-helper.h" #include "xray-registry.h" -#include "llvm/BinaryFormat/ELF.h" -#include "llvm/Object/ELF.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/DataExtractor.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Format.h" diff --git a/tools/llvm-xray/xray-graph.cc b/tools/llvm-xray/xray-graph.cc index 685c24cb9187..feb676331f89 100644 --- a/tools/llvm-xray/xray-graph.cc +++ b/tools/llvm-xray/xray-graph.cc @@ -11,19 +11,12 @@ // the trace. // //===----------------------------------------------------------------------===// -#include <algorithm> -#include <cassert> -#include <cmath> -#include <system_error> -#include <utility> #include "xray-graph.h" #include "xray-registry.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormatVariadic.h" #include "llvm/XRay/InstrumentationMap.h" #include "llvm/XRay/Trace.h" -#include "llvm/XRay/YAMLXRayRecord.h" using namespace llvm; using namespace llvm::xray; @@ -208,13 +201,15 @@ Error GraphRenderer::accountRecord(const XRayRecord &Record) { auto &ThreadStack = PerThreadFunctionStack[Record.TId]; switch (Record.Type) { - case RecordTypes::ENTER: { + case RecordTypes::ENTER: + case RecordTypes::ENTER_ARG: { if (Record.FuncId != 0 && G.count(Record.FuncId) == 0) G[Record.FuncId].SymbolName = FuncIdHelper.SymbolOrNumber(Record.FuncId); ThreadStack.push_back({Record.FuncId, Record.TSC}); break; } - case RecordTypes::EXIT: { + case RecordTypes::EXIT: + case RecordTypes::TAIL_EXIT: { // FIXME: Refactor this and the account subcommand to reduce code // duplication if (ThreadStack.size() == 0 || ThreadStack.back().FuncId != Record.FuncId) { diff --git a/tools/llvm-xray/xray-record-yaml.h b/tools/llvm-xray/xray-record-yaml.h deleted file mode 100644 index abce8ff60a94..000000000000 --- a/tools/llvm-xray/xray-record-yaml.h +++ /dev/null @@ -1,102 +0,0 @@ -//===- xray-record-yaml.h - XRay Record YAML Support Definitions ----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Types and traits specialisations for YAML I/O of XRay log entries. -// -//===----------------------------------------------------------------------===// -#ifndef LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_YAML_H -#define LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_YAML_H - -#include <type_traits> - -#include "xray-record.h" -#include "llvm/Support/YAMLTraits.h" - -namespace llvm { -namespace xray { - -struct YAMLXRayFileHeader { - uint16_t Version; - uint16_t Type; - bool ConstantTSC; - bool NonstopTSC; - uint64_t CycleFrequency; -}; - -struct YAMLXRayRecord { - uint16_t RecordType; - uint8_t CPU; - RecordTypes Type; - int32_t FuncId; - std::string Function; - uint64_t TSC; - uint32_t TId; -}; - -struct YAMLXRayTrace { - YAMLXRayFileHeader Header; - std::vector<YAMLXRayRecord> Records; -}; - -using XRayRecordStorage = - std::aligned_storage<sizeof(XRayRecord), alignof(XRayRecord)>::type; - -} // namespace xray - -namespace yaml { - -// YAML Traits -// ----------- -template <> struct ScalarEnumerationTraits<xray::RecordTypes> { - static void enumeration(IO &IO, xray::RecordTypes &Type) { - IO.enumCase(Type, "function-enter", xray::RecordTypes::ENTER); - IO.enumCase(Type, "function-exit", xray::RecordTypes::EXIT); - } -}; - -template <> struct MappingTraits<xray::YAMLXRayFileHeader> { - static void mapping(IO &IO, xray::YAMLXRayFileHeader &Header) { - IO.mapRequired("version", Header.Version); - IO.mapRequired("type", Header.Type); - IO.mapRequired("constant-tsc", Header.ConstantTSC); - IO.mapRequired("nonstop-tsc", Header.NonstopTSC); - IO.mapRequired("cycle-frequency", Header.CycleFrequency); - } -}; - -template <> struct MappingTraits<xray::YAMLXRayRecord> { - static void mapping(IO &IO, xray::YAMLXRayRecord &Record) { - // FIXME: Make this type actually be descriptive - IO.mapRequired("type", Record.RecordType); - IO.mapRequired("func-id", Record.FuncId); - IO.mapOptional("function", Record.Function); - IO.mapRequired("cpu", Record.CPU); - IO.mapRequired("thread", Record.TId); - IO.mapRequired("kind", Record.Type); - IO.mapRequired("tsc", Record.TSC); - } - - static constexpr bool flow = true; -}; - -template <> struct MappingTraits<xray::YAMLXRayTrace> { - static void mapping(IO &IO, xray::YAMLXRayTrace &Trace) { - // A trace file contains two parts, the header and the list of all the - // trace records. - IO.mapRequired("header", Trace.Header); - IO.mapRequired("records", Trace.Records); - } -}; - -} // namespace yaml -} // namespace llvm - -LLVM_YAML_IS_SEQUENCE_VECTOR(xray::YAMLXRayRecord) - -#endif // LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_YAML_H diff --git a/tools/llvm-xray/xray-stacks.cc b/tools/llvm-xray/xray-stacks.cc new file mode 100644 index 000000000000..9474de047990 --- /dev/null +++ b/tools/llvm-xray/xray-stacks.cc @@ -0,0 +1,797 @@ +//===- xray-stacks.cc - XRay Function Call Stack Accounting ---------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements stack-based accounting. It takes XRay traces, and +// collates statistics across these traces to show a breakdown of time spent +// at various points of the stack to provide insight into which functions +// spend the most time in terms of a call stack. We provide a few +// sorting/filtering options for zero'ing in on the useful stacks. +// +//===----------------------------------------------------------------------===// + +#include <forward_list> +#include <numeric> + +#include "func-id-helper.h" +#include "trie-node.h" +#include "xray-registry.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormatAdapters.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/XRay/Graph.h" +#include "llvm/XRay/InstrumentationMap.h" +#include "llvm/XRay/Trace.h" + +using namespace llvm; +using namespace llvm::xray; + +static cl::SubCommand Stack("stack", "Call stack accounting"); +static cl::list<std::string> StackInputs(cl::Positional, + cl::desc("<xray trace>"), cl::Required, + cl::sub(Stack), cl::OneOrMore); + +static cl::opt<bool> + StackKeepGoing("keep-going", cl::desc("Keep going on errors encountered"), + cl::sub(Stack), cl::init(false)); +static cl::alias StackKeepGoing2("k", cl::aliasopt(StackKeepGoing), + cl::desc("Alias for -keep-going"), + cl::sub(Stack)); + +// TODO: Does there need to be an option to deduce tail or sibling calls? + +static cl::opt<std::string> StacksInstrMap( + "instr_map", + cl::desc("instrumentation map used to identify function ids. " + "Currently supports elf file instrumentation maps."), + cl::sub(Stack), cl::init("")); +static cl::alias StacksInstrMap2("m", cl::aliasopt(StacksInstrMap), + cl::desc("Alias for -instr_map"), + cl::sub(Stack)); + +static cl::opt<bool> + SeparateThreadStacks("per-thread-stacks", + cl::desc("Report top stacks within each thread id"), + cl::sub(Stack), cl::init(false)); + +static cl::opt<bool> + AggregateThreads("aggregate-threads", + cl::desc("Aggregate stack times across threads"), + cl::sub(Stack), cl::init(false)); + +static cl::opt<bool> + DumpAllStacks("all-stacks", + cl::desc("Dump sum of timings for all stacks. " + "By default separates stacks per-thread."), + cl::sub(Stack), cl::init(false)); +static cl::alias DumpAllStacksShort("all", cl::aliasopt(DumpAllStacks), + cl::desc("Alias for -all-stacks"), + cl::sub(Stack)); + +// TODO(kpw): Add other interesting formats. Perhaps chrome trace viewer format +// possibly with aggregations or just a linear trace of timings. +enum StackOutputFormat { HUMAN, FLAMETOOL }; + +static cl::opt<StackOutputFormat> StacksOutputFormat( + "stack-format", + cl::desc("The format that output stacks should be " + "output in. Only applies with all-stacks."), + cl::values( + clEnumValN(HUMAN, "human", + "Human readable output. Only valid without -all-stacks."), + clEnumValN(FLAMETOOL, "flame", + "Format consumable by Brendan Gregg's FlameGraph tool. " + "Only valid with -all-stacks.")), + cl::sub(Stack), cl::init(HUMAN)); + +// Types of values for each stack in a CallTrie. +enum class AggregationType { + TOTAL_TIME, // The total time spent in a stack and its callees. + INVOCATION_COUNT // The number of times the stack was invoked. +}; + +static cl::opt<AggregationType> RequestedAggregation( + "aggregation-type", + cl::desc("The type of aggregation to do on call stacks."), + cl::values( + clEnumValN( + AggregationType::TOTAL_TIME, "time", + "Capture the total time spent in an all invocations of a stack."), + clEnumValN(AggregationType::INVOCATION_COUNT, "count", + "Capture the number of times a stack was invoked. " + "In flamegraph mode, this count also includes invocations " + "of all callees.")), + cl::sub(Stack), cl::init(AggregationType::TOTAL_TIME)); + +/// A helper struct to work with formatv and XRayRecords. Makes it easier to +/// use instrumentation map names or addresses in formatted output. +struct format_xray_record : public FormatAdapter<XRayRecord> { + explicit format_xray_record(XRayRecord record, + const FuncIdConversionHelper &conv) + : FormatAdapter<XRayRecord>(std::move(record)), Converter(&conv) {} + void format(raw_ostream &Stream, StringRef Style) override { + Stream << formatv( + "{FuncId: \"{0}\", ThreadId: \"{1}\", RecordType: \"{2}\"}", + Converter->SymbolOrNumber(Item.FuncId), Item.TId, + DecodeRecordType(Item.RecordType)); + } + +private: + Twine DecodeRecordType(uint16_t recordType) { + switch (recordType) { + case 0: + return Twine("Fn Entry"); + case 1: + return Twine("Fn Exit"); + default: + // TODO: Add Tail exit when it is added to llvm/XRay/XRayRecord.h + return Twine("Unknown"); + } + } + + const FuncIdConversionHelper *Converter; +}; + +/// The stack command will take a set of XRay traces as arguments, and collects +/// information about the stacks of instrumented functions that appear in the +/// traces. We track the following pieces of information: +/// +/// - Total time: amount of time/cycles accounted for in the traces. +/// - Stack count: number of times a specific stack appears in the +/// traces. Only instrumented functions show up in stacks. +/// - Cumulative stack time: amount of time spent in a stack accumulated +/// across the invocations in the traces. +/// - Cumulative local time: amount of time spent in each instrumented +/// function showing up in a specific stack, accumulated across the traces. +/// +/// Example output for the kind of data we'd like to provide looks like the +/// following: +/// +/// Total time: 3.33234 s +/// Stack ID: ... +/// Stack Count: 2093 +/// # Function Local Time (%) Stack Time (%) +/// 0 main 2.34 ms 0.07% 3.33234 s 100% +/// 1 foo() 3.30000 s 99.02% 3.33 s 99.92% +/// 2 bar() 30 ms 0.90% 30 ms 0.90% +/// +/// We can also show distributions of the function call durations with +/// statistics at each level of the stack. This works by doing the following +/// algorithm: +/// +/// 1. When unwinding, record the duration of each unwound function associated +/// with the path up to which the unwinding stops. For example: +/// +/// Step Duration (? means has start time) +/// +/// push a <start time> a = ? +/// push b <start time> a = ?, a->b = ? +/// push c <start time> a = ?, a->b = ?, a->b->c = ? +/// pop c <end time> a = ?, a->b = ?, emit duration(a->b->c) +/// pop b <end time> a = ?, emit duration(a->b) +/// push c <start time> a = ?, a->c = ? +/// pop c <end time> a = ?, emit duration(a->c) +/// pop a <end time> emit duration(a) +/// +/// 2. We then account for the various stacks we've collected, and for each of +/// them will have measurements that look like the following (continuing +/// with the above simple example): +/// +/// c : [<id("a->b->c"), [durations]>, <id("a->c"), [durations]>] +/// b : [<id("a->b"), [durations]>] +/// a : [<id("a"), [durations]>] +/// +/// This allows us to compute, for each stack id, and each function that +/// shows up in the stack, some important statistics like: +/// +/// - median +/// - 99th percentile +/// - mean + stddev +/// - count +/// +/// 3. For cases where we don't have durations for some of the higher levels +/// of the stack (perhaps instrumentation wasn't activated when the stack was +/// entered), we can mark them appropriately. +/// +/// Computing this data also allows us to implement lookup by call stack nodes, +/// so that we can find functions that show up in multiple stack traces and +/// show the statistical properties of that function in various contexts. We +/// can compute information similar to the following: +/// +/// Function: 'c' +/// Stacks: 2 / 2 +/// Stack ID: ... +/// Stack Count: ... +/// # Function ... +/// 0 a ... +/// 1 b ... +/// 2 c ... +/// +/// Stack ID: ... +/// Stack Count: ... +/// # Function ... +/// 0 a ... +/// 1 c ... +/// ----------------... +/// +/// Function: 'b' +/// Stacks: 1 / 2 +/// Stack ID: ... +/// Stack Count: ... +/// # Function ... +/// 0 a ... +/// 1 b ... +/// 2 c ... +/// +/// +/// To do this we require a Trie data structure that will allow us to represent +/// all the call stacks of instrumented functions in an easily traversible +/// manner when we do the aggregations and lookups. For instrumented call +/// sequences like the following: +/// +/// a() +/// b() +/// c() +/// d() +/// c() +/// +/// We will have a representation like so: +/// +/// a -> b -> c +/// | | +/// | +--> d +/// | +/// +--> c +/// +/// We maintain a sequence of durations on the leaves and in the internal nodes +/// as we go through and process every record from the XRay trace. We also +/// maintain an index of unique functions, and provide a means of iterating +/// through all the instrumented call stacks which we know about. + +struct StackDuration { + llvm::SmallVector<int64_t, 4> TerminalDurations; + llvm::SmallVector<int64_t, 4> IntermediateDurations; +}; + +StackDuration mergeStackDuration(const StackDuration &Left, + const StackDuration &Right) { + StackDuration Data{}; + Data.TerminalDurations.reserve(Left.TerminalDurations.size() + + Right.TerminalDurations.size()); + Data.IntermediateDurations.reserve(Left.IntermediateDurations.size() + + Right.IntermediateDurations.size()); + // Aggregate the durations. + for (auto duration : Left.TerminalDurations) + Data.TerminalDurations.push_back(duration); + for (auto duration : Right.TerminalDurations) + Data.TerminalDurations.push_back(duration); + + for (auto duration : Left.IntermediateDurations) + Data.IntermediateDurations.push_back(duration); + for (auto duration : Right.IntermediateDurations) + Data.IntermediateDurations.push_back(duration); + return Data; +} + +using StackTrieNode = TrieNode<StackDuration>; + +template <AggregationType AggType> +std::size_t GetValueForStack(const StackTrieNode *Node); + +// When computing total time spent in a stack, we're adding the timings from +// its callees and the timings from when it was a leaf. +template <> +std::size_t +GetValueForStack<AggregationType::TOTAL_TIME>(const StackTrieNode *Node) { + auto TopSum = std::accumulate(Node->ExtraData.TerminalDurations.begin(), + Node->ExtraData.TerminalDurations.end(), 0uLL); + return std::accumulate(Node->ExtraData.IntermediateDurations.begin(), + Node->ExtraData.IntermediateDurations.end(), TopSum); +} + +// Calculates how many times a function was invoked. +// TODO: Hook up option to produce stacks +template <> +std::size_t +GetValueForStack<AggregationType::INVOCATION_COUNT>(const StackTrieNode *Node) { + return Node->ExtraData.TerminalDurations.size() + + Node->ExtraData.IntermediateDurations.size(); +} + +// Make sure there are implementations for each enum value. +template <AggregationType T> struct DependentFalseType : std::false_type {}; + +template <AggregationType AggType> +std::size_t GetValueForStack(const StackTrieNode *Node) { + static_assert(DependentFalseType<AggType>::value, + "No implementation found for aggregation type provided."); + return 0; +} + +class StackTrie { + // Avoid the magic number of 4 propagated through the code with an alias. + // We use this SmallVector to track the root nodes in a call graph. + using RootVector = SmallVector<StackTrieNode *, 4>; + + // We maintain pointers to the roots of the tries we see. + DenseMap<uint32_t, RootVector> Roots; + + // We make sure all the nodes are accounted for in this list. + std::forward_list<StackTrieNode> NodeStore; + + // A map of thread ids to pairs call stack trie nodes and their start times. + DenseMap<uint32_t, SmallVector<std::pair<StackTrieNode *, uint64_t>, 8>> + ThreadStackMap; + + StackTrieNode *createTrieNode(uint32_t ThreadId, int32_t FuncId, + StackTrieNode *Parent) { + NodeStore.push_front(StackTrieNode{FuncId, Parent, {}, {{}, {}}}); + auto I = NodeStore.begin(); + auto *Node = &*I; + if (!Parent) + Roots[ThreadId].push_back(Node); + return Node; + } + + StackTrieNode *findRootNode(uint32_t ThreadId, int32_t FuncId) { + const auto &RootsByThread = Roots[ThreadId]; + auto I = find_if(RootsByThread, + [&](StackTrieNode *N) { return N->FuncId == FuncId; }); + return (I == RootsByThread.end()) ? nullptr : *I; + } + +public: + enum class AccountRecordStatus { + OK, // Successfully processed + ENTRY_NOT_FOUND, // An exit record had no matching call stack entry + UNKNOWN_RECORD_TYPE + }; + + struct AccountRecordState { + // We keep track of whether the call stack is currently unwinding. + bool wasLastRecordExit; + + static AccountRecordState CreateInitialState() { return {false}; } + }; + + AccountRecordStatus accountRecord(const XRayRecord &R, + AccountRecordState *state) { + auto &TS = ThreadStackMap[R.TId]; + switch (R.Type) { + case RecordTypes::ENTER: + case RecordTypes::ENTER_ARG: { + state->wasLastRecordExit = false; + // When we encounter a new function entry, we want to record the TSC for + // that entry, and the function id. Before doing so we check the top of + // the stack to see if there are callees that already represent this + // function. + if (TS.empty()) { + auto *Root = findRootNode(R.TId, R.FuncId); + TS.emplace_back(Root ? Root : createTrieNode(R.TId, R.FuncId, nullptr), + R.TSC); + return AccountRecordStatus::OK; + } + + auto &Top = TS.back(); + auto I = find_if(Top.first->Callees, + [&](StackTrieNode *N) { return N->FuncId == R.FuncId; }); + if (I == Top.first->Callees.end()) { + // We didn't find the callee in the stack trie, so we're going to + // add to the stack then set up the pointers properly. + auto N = createTrieNode(R.TId, R.FuncId, Top.first); + Top.first->Callees.emplace_back(N); + + // Top may be invalidated after this statement. + TS.emplace_back(N, R.TSC); + } else { + // We found the callee in the stack trie, so we'll use that pointer + // instead, add it to the stack associated with the TSC. + TS.emplace_back(*I, R.TSC); + } + return AccountRecordStatus::OK; + } + case RecordTypes::EXIT: + case RecordTypes::TAIL_EXIT: { + bool wasLastRecordExit = state->wasLastRecordExit; + state->wasLastRecordExit = true; + // The exit case is more interesting, since we want to be able to deduce + // missing exit records. To do that properly, we need to look up the stack + // and see whether the exit record matches any of the entry records. If it + // does match, we attempt to record the durations as we pop the stack to + // where we see the parent. + if (TS.empty()) { + // Short circuit, and say we can't find it. + + return AccountRecordStatus::ENTRY_NOT_FOUND; + } + + auto FunctionEntryMatch = find_if( + reverse(TS), [&](const std::pair<StackTrieNode *, uint64_t> &E) { + return E.first->FuncId == R.FuncId; + }); + auto status = AccountRecordStatus::OK; + if (FunctionEntryMatch == TS.rend()) { + status = AccountRecordStatus::ENTRY_NOT_FOUND; + } else { + // Account for offset of 1 between reverse and forward iterators. We + // want the forward iterator to include the function that is exited. + ++FunctionEntryMatch; + } + auto I = FunctionEntryMatch.base(); + for (auto &E : make_range(I, TS.end() - 1)) + E.first->ExtraData.IntermediateDurations.push_back( + std::max(E.second, R.TSC) - std::min(E.second, R.TSC)); + auto &Deepest = TS.back(); + if (wasLastRecordExit) + Deepest.first->ExtraData.IntermediateDurations.push_back( + std::max(Deepest.second, R.TSC) - std::min(Deepest.second, R.TSC)); + else + Deepest.first->ExtraData.TerminalDurations.push_back( + std::max(Deepest.second, R.TSC) - std::min(Deepest.second, R.TSC)); + TS.erase(I, TS.end()); + return status; + } + } + return AccountRecordStatus::UNKNOWN_RECORD_TYPE; + } + + bool isEmpty() const { return Roots.empty(); } + + void printStack(raw_ostream &OS, const StackTrieNode *Top, + FuncIdConversionHelper &FN) { + // Traverse the pointers up to the parent, noting the sums, then print + // in reverse order (callers at top, callees down bottom). + SmallVector<const StackTrieNode *, 8> CurrentStack; + for (auto *F = Top; F != nullptr; F = F->Parent) + CurrentStack.push_back(F); + int Level = 0; + OS << formatv("{0,-5} {1,-60} {2,+12} {3,+16}\n", "lvl", "function", + "count", "sum"); + for (auto *F : + reverse(make_range(CurrentStack.begin() + 1, CurrentStack.end()))) { + auto Sum = std::accumulate(F->ExtraData.IntermediateDurations.begin(), + F->ExtraData.IntermediateDurations.end(), 0LL); + auto FuncId = FN.SymbolOrNumber(F->FuncId); + OS << formatv("#{0,-4} {1,-60} {2,+12} {3,+16}\n", Level++, + FuncId.size() > 60 ? FuncId.substr(0, 57) + "..." : FuncId, + F->ExtraData.IntermediateDurations.size(), Sum); + } + auto *Leaf = *CurrentStack.begin(); + auto LeafSum = + std::accumulate(Leaf->ExtraData.TerminalDurations.begin(), + Leaf->ExtraData.TerminalDurations.end(), 0LL); + auto LeafFuncId = FN.SymbolOrNumber(Leaf->FuncId); + OS << formatv("#{0,-4} {1,-60} {2,+12} {3,+16}\n", Level++, + LeafFuncId.size() > 60 ? LeafFuncId.substr(0, 57) + "..." + : LeafFuncId, + Leaf->ExtraData.TerminalDurations.size(), LeafSum); + OS << "\n"; + } + + /// Prints top stacks for each thread. + void printPerThread(raw_ostream &OS, FuncIdConversionHelper &FN) { + for (auto iter : Roots) { + OS << "Thread " << iter.first << ":\n"; + print(OS, FN, iter.second); + OS << "\n"; + } + } + + /// Prints timing sums for each stack in each threads. + template <AggregationType AggType> + void printAllPerThread(raw_ostream &OS, FuncIdConversionHelper &FN, + StackOutputFormat format) { + for (auto iter : Roots) { + uint32_t threadId = iter.first; + RootVector &perThreadRoots = iter.second; + bool reportThreadId = true; + printAll<AggType>(OS, FN, perThreadRoots, threadId, reportThreadId); + } + } + + /// Prints top stacks from looking at all the leaves and ignoring thread IDs. + /// Stacks that consist of the same function IDs but were called in different + /// thread IDs are not considered unique in this printout. + void printIgnoringThreads(raw_ostream &OS, FuncIdConversionHelper &FN) { + RootVector RootValues; + + // Function to pull the values out of a map iterator. + using RootsType = decltype(Roots.begin())::value_type; + auto MapValueFn = [](const RootsType &Value) { return Value.second; }; + + for (const auto &RootNodeRange : + make_range(map_iterator(Roots.begin(), MapValueFn), + map_iterator(Roots.end(), MapValueFn))) { + for (auto *RootNode : RootNodeRange) + RootValues.push_back(RootNode); + } + + print(OS, FN, RootValues); + } + + /// Creates a merged list of Tries for unique stacks that disregards their + /// thread IDs. + RootVector mergeAcrossThreads(std::forward_list<StackTrieNode> &NodeStore) { + RootVector MergedByThreadRoots; + for (auto MapIter : Roots) { + const auto &RootNodeVector = MapIter.second; + for (auto *Node : RootNodeVector) { + auto MaybeFoundIter = + find_if(MergedByThreadRoots, [Node](StackTrieNode *elem) { + return Node->FuncId == elem->FuncId; + }); + if (MaybeFoundIter == MergedByThreadRoots.end()) { + MergedByThreadRoots.push_back(Node); + } else { + MergedByThreadRoots.push_back(mergeTrieNodes( + **MaybeFoundIter, *Node, nullptr, NodeStore, mergeStackDuration)); + MergedByThreadRoots.erase(MaybeFoundIter); + } + } + } + return MergedByThreadRoots; + } + + /// Print timing sums for all stacks merged by Thread ID. + template <AggregationType AggType> + void printAllAggregatingThreads(raw_ostream &OS, FuncIdConversionHelper &FN, + StackOutputFormat format) { + std::forward_list<StackTrieNode> AggregatedNodeStore; + RootVector MergedByThreadRoots = mergeAcrossThreads(AggregatedNodeStore); + bool reportThreadId = false; + printAll<AggType>(OS, FN, MergedByThreadRoots, + /*threadId*/ 0, reportThreadId); + } + + /// Merges the trie by thread id before printing top stacks. + void printAggregatingThreads(raw_ostream &OS, FuncIdConversionHelper &FN) { + std::forward_list<StackTrieNode> AggregatedNodeStore; + RootVector MergedByThreadRoots = mergeAcrossThreads(AggregatedNodeStore); + print(OS, FN, MergedByThreadRoots); + } + + // TODO: Add a format option when more than one are supported. + template <AggregationType AggType> + void printAll(raw_ostream &OS, FuncIdConversionHelper &FN, + RootVector RootValues, uint32_t ThreadId, bool ReportThread) { + SmallVector<const StackTrieNode *, 16> S; + for (const auto *N : RootValues) { + S.clear(); + S.push_back(N); + while (!S.empty()) { + auto *Top = S.pop_back_val(); + printSingleStack<AggType>(OS, FN, ReportThread, ThreadId, Top); + for (const auto *C : Top->Callees) + S.push_back(C); + } + } + } + + /// Prints values for stacks in a format consumable for the flamegraph.pl + /// tool. This is a line based format that lists each level in the stack + /// hierarchy in a semicolon delimited form followed by a space and a numeric + /// value. If breaking down by thread, the thread ID will be added as the + /// root level of the stack. + template <AggregationType AggType> + void printSingleStack(raw_ostream &OS, FuncIdConversionHelper &Converter, + bool ReportThread, uint32_t ThreadId, + const StackTrieNode *Node) { + if (ReportThread) + OS << "thread_" << ThreadId << ";"; + SmallVector<const StackTrieNode *, 5> lineage{}; + lineage.push_back(Node); + while (lineage.back()->Parent != nullptr) + lineage.push_back(lineage.back()->Parent); + while (!lineage.empty()) { + OS << Converter.SymbolOrNumber(lineage.back()->FuncId) << ";"; + lineage.pop_back(); + } + OS << " " << GetValueForStack<AggType>(Node) << "\n"; + } + + void print(raw_ostream &OS, FuncIdConversionHelper &FN, + RootVector RootValues) { + // Go through each of the roots, and traverse the call stack, producing the + // aggregates as you go along. Remember these aggregates and stacks, and + // show summary statistics about: + // + // - Total number of unique stacks + // - Top 10 stacks by count + // - Top 10 stacks by aggregate duration + SmallVector<std::pair<const StackTrieNode *, uint64_t>, 11> + TopStacksByCount; + SmallVector<std::pair<const StackTrieNode *, uint64_t>, 11> TopStacksBySum; + auto greater_second = + [](const std::pair<const StackTrieNode *, uint64_t> &A, + const std::pair<const StackTrieNode *, uint64_t> &B) { + return A.second > B.second; + }; + uint64_t UniqueStacks = 0; + for (const auto *N : RootValues) { + SmallVector<const StackTrieNode *, 16> S; + S.emplace_back(N); + + while (!S.empty()) { + auto *Top = S.pop_back_val(); + + // We only start printing the stack (by walking up the parent pointers) + // when we get to a leaf function. + if (!Top->ExtraData.TerminalDurations.empty()) { + ++UniqueStacks; + auto TopSum = + std::accumulate(Top->ExtraData.TerminalDurations.begin(), + Top->ExtraData.TerminalDurations.end(), 0uLL); + { + auto E = std::make_pair(Top, TopSum); + TopStacksBySum.insert(std::lower_bound(TopStacksBySum.begin(), + TopStacksBySum.end(), E, + greater_second), + E); + if (TopStacksBySum.size() == 11) + TopStacksBySum.pop_back(); + } + { + auto E = + std::make_pair(Top, Top->ExtraData.TerminalDurations.size()); + TopStacksByCount.insert(std::lower_bound(TopStacksByCount.begin(), + TopStacksByCount.end(), E, + greater_second), + E); + if (TopStacksByCount.size() == 11) + TopStacksByCount.pop_back(); + } + } + for (const auto *C : Top->Callees) + S.push_back(C); + } + } + + // Now print the statistics in the end. + OS << "\n"; + OS << "Unique Stacks: " << UniqueStacks << "\n"; + OS << "Top 10 Stacks by leaf sum:\n\n"; + for (const auto &P : TopStacksBySum) { + OS << "Sum: " << P.second << "\n"; + printStack(OS, P.first, FN); + } + OS << "\n"; + OS << "Top 10 Stacks by leaf count:\n\n"; + for (const auto &P : TopStacksByCount) { + OS << "Count: " << P.second << "\n"; + printStack(OS, P.first, FN); + } + OS << "\n"; + } +}; + +std::string CreateErrorMessage(StackTrie::AccountRecordStatus Error, + const XRayRecord &Record, + const FuncIdConversionHelper &Converter) { + switch (Error) { + case StackTrie::AccountRecordStatus::ENTRY_NOT_FOUND: + return formatv("Found record {0} with no matching function entry\n", + format_xray_record(Record, Converter)); + default: + return formatv("Unknown error type for record {0}\n", + format_xray_record(Record, Converter)); + } +} + +static CommandRegistration Unused(&Stack, []() -> Error { + // Load each file provided as a command-line argument. For each one of them + // account to a single StackTrie, and just print the whole trie for now. + StackTrie ST; + InstrumentationMap Map; + if (!StacksInstrMap.empty()) { + auto InstrumentationMapOrError = loadInstrumentationMap(StacksInstrMap); + if (!InstrumentationMapOrError) + return joinErrors( + make_error<StringError>( + Twine("Cannot open instrumentation map: ") + StacksInstrMap, + std::make_error_code(std::errc::invalid_argument)), + InstrumentationMapOrError.takeError()); + Map = std::move(*InstrumentationMapOrError); + } + + if (SeparateThreadStacks && AggregateThreads) + return make_error<StringError>( + Twine("Can't specify options for per thread reporting and reporting " + "that aggregates threads."), + std::make_error_code(std::errc::invalid_argument)); + + if (!DumpAllStacks && StacksOutputFormat != HUMAN) + return make_error<StringError>( + Twine("Can't specify a non-human format without -all-stacks."), + std::make_error_code(std::errc::invalid_argument)); + + if (DumpAllStacks && StacksOutputFormat == HUMAN) + return make_error<StringError>( + Twine("You must specify a non-human format when reporting with " + "-all-stacks."), + std::make_error_code(std::errc::invalid_argument)); + + symbolize::LLVMSymbolizer::Options Opts( + symbolize::FunctionNameKind::LinkageName, true, true, false, ""); + symbolize::LLVMSymbolizer Symbolizer(Opts); + FuncIdConversionHelper FuncIdHelper(StacksInstrMap, Symbolizer, + Map.getFunctionAddresses()); + // TODO: Someday, support output to files instead of just directly to + // standard output. + for (const auto &Filename : StackInputs) { + auto TraceOrErr = loadTraceFile(Filename); + if (!TraceOrErr) { + if (!StackKeepGoing) + return joinErrors( + make_error<StringError>( + Twine("Failed loading input file '") + Filename + "'", + std::make_error_code(std::errc::invalid_argument)), + TraceOrErr.takeError()); + logAllUnhandledErrors(TraceOrErr.takeError(), errs(), ""); + continue; + } + auto &T = *TraceOrErr; + StackTrie::AccountRecordState AccountRecordState = + StackTrie::AccountRecordState::CreateInitialState(); + for (const auto &Record : T) { + auto error = ST.accountRecord(Record, &AccountRecordState); + if (error != StackTrie::AccountRecordStatus::OK) { + if (!StackKeepGoing) + return make_error<StringError>( + CreateErrorMessage(error, Record, FuncIdHelper), + make_error_code(errc::illegal_byte_sequence)); + errs() << CreateErrorMessage(error, Record, FuncIdHelper); + } + } + } + if (ST.isEmpty()) { + return make_error<StringError>( + "No instrumented calls were accounted in the input file.", + make_error_code(errc::result_out_of_range)); + } + + // Report the stacks in a long form mode for another tool to analyze. + if (DumpAllStacks) { + if (AggregateThreads) { + switch (RequestedAggregation) { + case AggregationType::TOTAL_TIME: + ST.printAllAggregatingThreads<AggregationType::TOTAL_TIME>( + outs(), FuncIdHelper, StacksOutputFormat); + break; + case AggregationType::INVOCATION_COUNT: + ST.printAllAggregatingThreads<AggregationType::INVOCATION_COUNT>( + outs(), FuncIdHelper, StacksOutputFormat); + break; + } + } else { + switch (RequestedAggregation) { + case AggregationType::TOTAL_TIME: + ST.printAllPerThread<AggregationType::TOTAL_TIME>(outs(), FuncIdHelper, + StacksOutputFormat); + break; + case AggregationType::INVOCATION_COUNT: + ST.printAllPerThread<AggregationType::INVOCATION_COUNT>( + outs(), FuncIdHelper, StacksOutputFormat); + break; + } + } + return Error::success(); + } + + // We're only outputting top stacks. + if (AggregateThreads) { + ST.printAggregatingThreads(outs(), FuncIdHelper); + } else if (SeparateThreadStacks) { + ST.printPerThread(outs(), FuncIdHelper); + } else { + ST.printIgnoringThreads(outs(), FuncIdHelper); + } + return Error::success(); +}); diff --git a/tools/lto/lto.cpp b/tools/lto/lto.cpp index 1b218a64cbf5..e6843986dbe2 100644 --- a/tools/lto/lto.cpp +++ b/tools/lto/lto.cpp @@ -15,7 +15,7 @@ #include "llvm-c/lto.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Bitcode/BitcodeReader.h" -#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/CommandFlags.def" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/LLVMContext.h" @@ -75,20 +75,23 @@ static bool parsedOptions = false; static LLVMContext *LTOContext = nullptr; -static void diagnosticHandler(const DiagnosticInfo &DI, void *Context) { - if (DI.getSeverity() != DS_Error) { - DiagnosticPrinterRawOStream DP(errs()); - DI.print(DP); - errs() << '\n'; - return; - } - sLastErrorString = ""; - { - raw_string_ostream Stream(sLastErrorString); - DiagnosticPrinterRawOStream DP(Stream); - DI.print(DP); +struct LTOToolDiagnosticHandler : public DiagnosticHandler { + bool handleDiagnostics(const DiagnosticInfo &DI) override { + if (DI.getSeverity() != DS_Error) { + DiagnosticPrinterRawOStream DP(errs()); + DI.print(DP); + errs() << '\n'; + return true; + } + sLastErrorString = ""; + { + raw_string_ostream Stream(sLastErrorString); + DiagnosticPrinterRawOStream DP(Stream); + DI.print(DP); + } + return true; } -} +}; // Initialize the configured targets if they have not been initialized. static void lto_initialize() { @@ -108,7 +111,8 @@ static void lto_initialize() { static LLVMContext Context; LTOContext = &Context; - LTOContext->setDiagnosticHandler(diagnosticHandler, nullptr, true); + LTOContext->setDiagnosticHandler( + llvm::make_unique<LTOToolDiagnosticHandler>(), true); initialized = true; } } @@ -274,7 +278,8 @@ lto_module_t lto_module_create_in_local_context(const void *mem, size_t length, // Create a local context. Ownership will be transferred to LTOModule. std::unique_ptr<LLVMContext> Context = llvm::make_unique<LLVMContext>(); - Context->setDiagnosticHandler(diagnosticHandler, nullptr, true); + Context->setDiagnosticHandler(llvm::make_unique<LTOToolDiagnosticHandler>(), + true); ErrorOr<std::unique_ptr<LTOModule>> M = LTOModule::createInLocalContext( std::move(Context), mem, length, Options, StringRef(path)); diff --git a/tools/obj2yaml/coff2yaml.cpp b/tools/obj2yaml/coff2yaml.cpp index b1a06bca1a73..6c4f8437caef 100644 --- a/tools/obj2yaml/coff2yaml.cpp +++ b/tools/obj2yaml/coff2yaml.cpp @@ -13,7 +13,6 @@ #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" #include "llvm/Object/COFF.h" #include "llvm/ObjectYAML/COFFYAML.h" -#include "llvm/ObjectYAML/CodeViewYAMLSymbols.h" #include "llvm/ObjectYAML/CodeViewYAMLTypes.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/YAMLTraits.h" @@ -172,6 +171,8 @@ void COFFDumper::dumpSections(unsigned NumSections) { NewYAMLSection.DebugS = CodeViewYAML::fromDebugS(sectionData, SC); else if (NewYAMLSection.Name == ".debug$T") NewYAMLSection.DebugT = CodeViewYAML::fromDebugT(sectionData); + else if (NewYAMLSection.Name == ".debug$H") + NewYAMLSection.DebugH = CodeViewYAML::fromDebugH(sectionData); std::vector<COFFYAML::Relocation> Relocations; for (const auto &Reloc : ObjSection.relocations()) { diff --git a/tools/obj2yaml/dwarf2yaml.cpp b/tools/obj2yaml/dwarf2yaml.cpp index d97eda30c039..91cec8b7c6ca 100644 --- a/tools/obj2yaml/dwarf2yaml.cpp +++ b/tools/obj2yaml/dwarf2yaml.cpp @@ -24,7 +24,7 @@ void dumpInitialLength(DataExtractor &Data, uint32_t &Offset, InitialLength.TotalLength64 = Data.getU64(&Offset); } -void dumpDebugAbbrev(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { +void dumpDebugAbbrev(DWARFContext &DCtx, DWARFYAML::Data &Y) { auto AbbrevSetPtr = DCtx.getDebugAbbrev(); if (AbbrevSetPtr) { for (auto AbbrvDeclSet : *AbbrevSetPtr) { @@ -39,7 +39,7 @@ void dumpDebugAbbrev(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { AttAbrv.Attribute = Attribute.Attr; AttAbrv.Form = Attribute.Form; if (AttAbrv.Form == dwarf::DW_FORM_implicit_const) - AttAbrv.Value = *Attribute.ByteSizeOrValue; + AttAbrv.Value = Attribute.getImplicitConstValue(); Abbrv.Attributes.push_back(AttAbrv); } Y.AbbrevDecls.push_back(Abbrv); @@ -48,8 +48,8 @@ void dumpDebugAbbrev(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { } } -void dumpDebugStrings(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { - StringRef RemainingTable = DCtx.getStringSection(); +void dumpDebugStrings(DWARFContext &DCtx, DWARFYAML::Data &Y) { + StringRef RemainingTable = DCtx.getDWARFObj().getStringSection(); while (RemainingTable.size() > 0) { auto SymbolPair = RemainingTable.split('\0'); RemainingTable = SymbolPair.second; @@ -57,8 +57,9 @@ void dumpDebugStrings(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { } } -void dumpDebugARanges(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { - DataExtractor ArangesData(DCtx.getARangeSection(), DCtx.isLittleEndian(), 0); +void dumpDebugARanges(DWARFContext &DCtx, DWARFYAML::Data &Y) { + DataExtractor ArangesData(DCtx.getDWARFObj().getARangeSection(), + DCtx.isLittleEndian(), 0); uint32_t Offset = 0; DWARFDebugArangeSet Set; @@ -79,7 +80,7 @@ void dumpDebugARanges(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { } } -void dumpPubSection(DWARFContextInMemory &DCtx, DWARFYAML::PubSection &Y, +void dumpPubSection(DWARFContext &DCtx, DWARFYAML::PubSection &Y, StringRef Section) { DataExtractor PubSectionData(Section, DCtx.isLittleEndian(), 0); uint32_t Offset = 0; @@ -97,21 +98,22 @@ void dumpPubSection(DWARFContextInMemory &DCtx, DWARFYAML::PubSection &Y, } } -void dumpDebugPubSections(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { +void dumpDebugPubSections(DWARFContext &DCtx, DWARFYAML::Data &Y) { + const DWARFObject &D = DCtx.getDWARFObj(); Y.PubNames.IsGNUStyle = false; - dumpPubSection(DCtx, Y.PubNames, DCtx.getPubNamesSection()); + dumpPubSection(DCtx, Y.PubNames, D.getPubNamesSection()); Y.PubTypes.IsGNUStyle = false; - dumpPubSection(DCtx, Y.PubTypes, DCtx.getPubTypesSection()); + dumpPubSection(DCtx, Y.PubTypes, D.getPubTypesSection()); Y.GNUPubNames.IsGNUStyle = true; - dumpPubSection(DCtx, Y.GNUPubNames, DCtx.getGnuPubNamesSection()); + dumpPubSection(DCtx, Y.GNUPubNames, D.getGnuPubNamesSection()); Y.GNUPubTypes.IsGNUStyle = true; - dumpPubSection(DCtx, Y.GNUPubTypes, DCtx.getGnuPubTypesSection()); + dumpPubSection(DCtx, Y.GNUPubTypes, D.getGnuPubTypesSection()); } -void dumpDebugInfo(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { +void dumpDebugInfo(DWARFContext &DCtx, DWARFYAML::Data &Y) { for (const auto &CU : DCtx.compile_units()) { DWARFYAML::Unit NewUnit; NewUnit.Length.setLength(CU->getLength()); @@ -235,7 +237,7 @@ bool dumpFileEntry(DataExtractor &Data, uint32_t &Offset, return true; } -void dumpDebugLines(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { +void dumpDebugLines(DWARFContext &DCtx, DWARFYAML::Data &Y) { for (const auto &CU : DCtx.compile_units()) { auto CUDIE = CU->getUnitDIE(); if (!CUDIE) @@ -243,8 +245,8 @@ void dumpDebugLines(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { if (auto StmtOffset = dwarf::toSectionOffset(CUDIE.find(dwarf::DW_AT_stmt_list))) { DWARFYAML::LineTable DebugLines; - DataExtractor LineData(DCtx.getLineSection().Data, DCtx.isLittleEndian(), - CU->getAddressByteSize()); + DataExtractor LineData(DCtx.getDWARFObj().getLineSection().Data, + DCtx.isLittleEndian(), CU->getAddressByteSize()); uint32_t Offset = *StmtOffset; dumpInitialLength(LineData, Offset, DebugLines.Length); uint64_t LineTableLength = DebugLines.Length.getLength(); @@ -344,7 +346,7 @@ void dumpDebugLines(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { } } -std::error_code dwarf2yaml(DWARFContextInMemory &DCtx, DWARFYAML::Data &Y) { +std::error_code dwarf2yaml(DWARFContext &DCtx, DWARFYAML::Data &Y) { dumpDebugAbbrev(DCtx, Y); dumpDebugStrings(DCtx, Y); dumpDebugARanges(DCtx, Y); diff --git a/tools/obj2yaml/elf2yaml.cpp b/tools/obj2yaml/elf2yaml.cpp index 9f9ef99265dc..f68bcf4ee402 100644 --- a/tools/obj2yaml/elf2yaml.cpp +++ b/tools/obj2yaml/elf2yaml.cpp @@ -8,7 +8,7 @@ //===----------------------------------------------------------------------===// #include "Error.h" -#include "obj2yaml.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/ObjectYAML/ELFYAML.h" @@ -27,9 +27,22 @@ class ELFDumper { typedef typename object::ELFFile<ELFT>::Elf_Rel Elf_Rel; typedef typename object::ELFFile<ELFT>::Elf_Rela Elf_Rela; + ArrayRef<Elf_Shdr> Sections; + + // If the file has multiple sections with the same name, we add a + // suffix to make them unique. + unsigned Suffix = 0; + DenseSet<StringRef> UsedSectionNames; + std::vector<std::string> SectionNames; + Expected<StringRef> getUniquedSectionName(const Elf_Shdr *Sec); + Expected<StringRef> getSymbolName(const Elf_Sym *Sym, StringRef StrTable, + const Elf_Shdr *SymTab); + const object::ELFFile<ELFT> &Obj; ArrayRef<Elf_Word> ShndxTable; + std::error_code dumpSymbols(const Elf_Shdr *Symtab, + ELFYAML::LocalGlobalWeakSymbols &Symbols); std::error_code dumpSymbol(const Elf_Sym *Sym, const Elf_Shdr *SymTab, StringRef StrTable, ELFYAML::Symbol &S); std::error_code dumpCommonSection(const Elf_Shdr *Shdr, ELFYAML::Section &S); @@ -59,7 +72,42 @@ ELFDumper<ELFT>::ELFDumper(const object::ELFFile<ELFT> &O) : Obj(O) {} template <class ELFT> -ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() { +Expected<StringRef> +ELFDumper<ELFT>::getUniquedSectionName(const Elf_Shdr *Sec) { + unsigned SecIndex = Sec - &Sections[0]; + assert(&Sections[SecIndex] == Sec); + if (!SectionNames[SecIndex].empty()) + return SectionNames[SecIndex]; + + auto NameOrErr = Obj.getSectionName(Sec); + if (!NameOrErr) + return NameOrErr; + StringRef Name = *NameOrErr; + std::string &Ret = SectionNames[SecIndex]; + Ret = Name; + while (!UsedSectionNames.insert(Ret).second) + Ret = (Name + to_string(++Suffix)).str(); + return Ret; +} + +template <class ELFT> +Expected<StringRef> ELFDumper<ELFT>::getSymbolName(const Elf_Sym *Sym, + StringRef StrTable, + const Elf_Shdr *SymTab) { + Expected<StringRef> SymbolNameOrErr = Sym->getName(StrTable); + if (!SymbolNameOrErr) + return SymbolNameOrErr; + StringRef Name = *SymbolNameOrErr; + if (Name.empty() && Sym->getType() == ELF::STT_SECTION) { + auto ShdrOrErr = Obj.getSection(Sym, SymTab, ShndxTable); + if (!ShdrOrErr) + return ShdrOrErr.takeError(); + return getUniquedSectionName(*ShdrOrErr); + } + return Name; +} + +template <class ELFT> ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() { auto Y = make_unique<ELFYAML::Object>(); // Dump header @@ -72,21 +120,26 @@ ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() { Y->Header.Entry = Obj.getHeader()->e_entry; const Elf_Shdr *Symtab = nullptr; + const Elf_Shdr *DynSymtab = nullptr; // Dump sections auto SectionsOrErr = Obj.sections(); if (!SectionsOrErr) return errorToErrorCode(SectionsOrErr.takeError()); - for (const Elf_Shdr &Sec : *SectionsOrErr) { + Sections = *SectionsOrErr; + SectionNames.resize(Sections.size()); + for (const Elf_Shdr &Sec : Sections) { switch (Sec.sh_type) { case ELF::SHT_NULL: - case ELF::SHT_DYNSYM: case ELF::SHT_STRTAB: // Do not dump these sections. break; case ELF::SHT_SYMTAB: Symtab = &Sec; break; + case ELF::SHT_DYNSYM: + DynSymtab = &Sec; + break; case ELF::SHT_SYMTAB_SHNDX: { auto TableOrErr = Obj.getSHNDXTable(Sec); if (!TableOrErr) @@ -138,44 +191,57 @@ ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() { } } - // Dump symbols + if (auto EC = dumpSymbols(Symtab, Y->Symbols)) + return EC; + if (auto EC = dumpSymbols(DynSymtab, Y->DynamicSymbols)) + return EC; + + return Y.release(); +} + +template <class ELFT> +std::error_code +ELFDumper<ELFT>::dumpSymbols(const Elf_Shdr *Symtab, + ELFYAML::LocalGlobalWeakSymbols &Symbols) { + if (!Symtab) + return std::error_code(); + auto StrTableOrErr = Obj.getStringTableForSymtab(*Symtab); if (!StrTableOrErr) return errorToErrorCode(StrTableOrErr.takeError()); StringRef StrTable = *StrTableOrErr; - bool IsFirstSym = true; auto SymtabOrErr = Obj.symbols(Symtab); if (!SymtabOrErr) return errorToErrorCode(SymtabOrErr.takeError()); - for (const Elf_Sym &Sym : *SymtabOrErr) { + + bool IsFirstSym = true; + for (const auto &Sym : *SymtabOrErr) { if (IsFirstSym) { IsFirstSym = false; continue; } ELFYAML::Symbol S; - if (std::error_code EC = - ELFDumper<ELFT>::dumpSymbol(&Sym, Symtab, StrTable, S)) + if (auto EC = dumpSymbol(&Sym, Symtab, StrTable, S)) return EC; - switch (Sym.getBinding()) - { + switch (Sym.getBinding()) { case ELF::STB_LOCAL: - Y->Symbols.Local.push_back(S); + Symbols.Local.push_back(S); break; case ELF::STB_GLOBAL: - Y->Symbols.Global.push_back(S); + Symbols.Global.push_back(S); break; case ELF::STB_WEAK: - Y->Symbols.Weak.push_back(S); + Symbols.Weak.push_back(S); break; default: llvm_unreachable("Unknown ELF symbol binding"); } } - return Y.release(); + return std::error_code(); } template <class ELFT> @@ -187,7 +253,7 @@ ELFDumper<ELFT>::dumpSymbol(const Elf_Sym *Sym, const Elf_Shdr *SymTab, S.Size = Sym->st_size; S.Other = Sym->st_other; - Expected<StringRef> SymbolNameOrErr = Sym->getName(StrTable); + Expected<StringRef> SymbolNameOrErr = getSymbolName(Sym, StrTable, SymTab); if (!SymbolNameOrErr) return errorToErrorCode(SymbolNameOrErr.takeError()); S.Name = SymbolNameOrErr.get(); @@ -199,7 +265,7 @@ ELFDumper<ELFT>::dumpSymbol(const Elf_Sym *Sym, const Elf_Shdr *SymTab, if (!Shdr) return obj2yaml_error::success; - auto NameOrErr = Obj.getSectionName(Shdr); + auto NameOrErr = getUniquedSectionName(Shdr); if (!NameOrErr) return errorToErrorCode(NameOrErr.takeError()); S.Section = NameOrErr.get(); @@ -229,7 +295,7 @@ std::error_code ELFDumper<ELFT>::dumpRelocation(const RelT *Rel, StringRef StrTab = *StrTabOrErr; if (Sym) { - Expected<StringRef> NameOrErr = Sym->getName(StrTab); + Expected<StringRef> NameOrErr = getSymbolName(Sym, StrTab, SymTab); if (!NameOrErr) return errorToErrorCode(NameOrErr.takeError()); R.Symbol = NameOrErr.get(); @@ -252,7 +318,7 @@ std::error_code ELFDumper<ELFT>::dumpCommonSection(const Elf_Shdr *Shdr, S.Address = Shdr->sh_addr; S.AddressAlign = Shdr->sh_addralign; - auto NameOrErr = Obj.getSectionName(Shdr); + auto NameOrErr = getUniquedSectionName(Shdr); if (!NameOrErr) return errorToErrorCode(NameOrErr.takeError()); S.Name = NameOrErr.get(); @@ -261,7 +327,7 @@ std::error_code ELFDumper<ELFT>::dumpCommonSection(const Elf_Shdr *Shdr, auto LinkSection = Obj.getSection(Shdr->sh_link); if (LinkSection.takeError()) return errorToErrorCode(LinkSection.takeError()); - NameOrErr = Obj.getSectionName(*LinkSection); + NameOrErr = getUniquedSectionName(*LinkSection); if (!NameOrErr) return errorToErrorCode(NameOrErr.takeError()); S.Link = NameOrErr.get(); @@ -281,7 +347,7 @@ ELFDumper<ELFT>::dumpCommonRelocationSection(const Elf_Shdr *Shdr, if (!InfoSection) return errorToErrorCode(InfoSection.takeError()); - auto NameOrErr = Obj.getSectionName(*InfoSection); + auto NameOrErr = getUniquedSectionName(*InfoSection); if (!NameOrErr) return errorToErrorCode(NameOrErr.takeError()); S.Info = NameOrErr.get(); @@ -395,7 +461,7 @@ ErrorOr<ELFYAML::Group *> ELFDumper<ELFT>::dumpGroup(const Elf_Shdr *Shdr) { auto sectionContents = Obj.getSectionContents(Shdr); if (!sectionContents) return errorToErrorCode(sectionContents.takeError()); - Expected<StringRef> symbolName = symbol->getName(StrTab); + Expected<StringRef> symbolName = getSymbolName(symbol, StrTab, Symtab); if (!symbolName) return errorToErrorCode(symbolName.takeError()); S->Info = *symbolName; @@ -410,7 +476,7 @@ ErrorOr<ELFYAML::Group *> ELFDumper<ELFT>::dumpGroup(const Elf_Shdr *Shdr) { auto sHdr = Obj.getSection(groupMembers[i]); if (!sHdr) return errorToErrorCode(sHdr.takeError()); - auto sectionName = Obj.getSectionName(*sHdr); + auto sectionName = getUniquedSectionName(*sHdr); if (!sectionName) return errorToErrorCode(sectionName.takeError()); s.sectionNameOrType = *sectionName; diff --git a/tools/obj2yaml/macho2yaml.cpp b/tools/obj2yaml/macho2yaml.cpp index a1d107dc5afb..fa81ce974ecf 100644 --- a/tools/obj2yaml/macho2yaml.cpp +++ b/tools/obj2yaml/macho2yaml.cpp @@ -35,9 +35,9 @@ class MachODumper { ArrayRef<uint8_t> OpcodeBuffer, bool Lazy = false); void dumpExportTrie(std::unique_ptr<MachOYAML::Object> &Y); void dumpSymbols(std::unique_ptr<MachOYAML::Object> &Y); - void dumpDebugAbbrev(DWARFContextInMemory &DCtx, + void dumpDebugAbbrev(DWARFContext &DCtx, std::unique_ptr<MachOYAML::Object> &Y); - void dumpDebugStrings(DWARFContextInMemory &DCtx, + void dumpDebugStrings(DWARFContext &DCtx, std::unique_ptr<MachOYAML::Object> &Y); public: @@ -187,8 +187,8 @@ Expected<std::unique_ptr<MachOYAML::Object>> MachODumper::dump() { dumpLoadCommands(Y); dumpLinkEdit(Y); - DWARFContextInMemory DICtx(Obj); - if (auto Err = dwarf2yaml(DICtx, Y->DWARF)) + std::unique_ptr<DWARFContext> DICtx = DWARFContext::create(Obj); + if (auto Err = dwarf2yaml(*DICtx, Y->DWARF)) return errorCodeToError(Err); return std::move(Y); } diff --git a/tools/obj2yaml/obj2yaml.h b/tools/obj2yaml/obj2yaml.h index 69c753296efd..3e96659462b0 100644 --- a/tools/obj2yaml/obj2yaml.h +++ b/tools/obj2yaml/obj2yaml.h @@ -29,13 +29,12 @@ std::error_code wasm2yaml(llvm::raw_ostream &Out, // Forward decls for dwarf2yaml namespace llvm { -class DWARFContextInMemory; +class DWARFContext; namespace DWARFYAML { struct Data; } } -std::error_code dwarf2yaml(llvm::DWARFContextInMemory &DCtx, - llvm::DWARFYAML::Data &Y); +std::error_code dwarf2yaml(llvm::DWARFContext &DCtx, llvm::DWARFYAML::Data &Y); #endif diff --git a/tools/obj2yaml/wasm2yaml.cpp b/tools/obj2yaml/wasm2yaml.cpp index a1da4b6a748c..1bf8149493c4 100644 --- a/tools/obj2yaml/wasm2yaml.cpp +++ b/tools/obj2yaml/wasm2yaml.cpp @@ -54,31 +54,41 @@ std::unique_ptr<WasmYAML::CustomSection> WasmDumper::dumpCustomSection(const Was if (WasmSec.Name == "name") { std::unique_ptr<WasmYAML::NameSection> NameSec = make_unique<WasmYAML::NameSection>(); for (const object::SymbolRef& Sym: Obj.symbols()) { - uint32_t Flags = Sym.getFlags(); - // Skip over symbols that come from imports or exports - if (Flags & - (object::SymbolRef::SF_Global | object::SymbolRef::SF_Undefined)) - continue; - Expected<StringRef> NameOrError = Sym.getName(); - if (!NameOrError) + const object::WasmSymbol Symbol = Obj.getWasmSymbol(Sym); + if (Symbol.Type != object::WasmSymbol::SymbolType::DEBUG_FUNCTION_NAME) continue; WasmYAML::NameEntry NameEntry; - NameEntry.Name = *NameOrError; + NameEntry.Name = Symbol.Name; NameEntry.Index = Sym.getValue(); NameSec->FunctionNames.push_back(NameEntry); } CustomSec = std::move(NameSec); } else if (WasmSec.Name == "linking") { std::unique_ptr<WasmYAML::LinkingSection> LinkingSec = make_unique<WasmYAML::LinkingSection>(); + size_t Index = 0; + for (const object::WasmSegment &Segment : Obj.dataSegments()) { + if (!Segment.Data.Name.empty()) { + WasmYAML::SegmentInfo SegmentInfo; + SegmentInfo.Name = Segment.Data.Name; + SegmentInfo.Index = Index; + SegmentInfo.Alignment = Segment.Data.Alignment; + SegmentInfo.Flags = Segment.Data.Flags; + LinkingSec->SegmentInfos.push_back(SegmentInfo); + } + Index++; + } for (const object::SymbolRef& Sym: Obj.symbols()) { const object::WasmSymbol Symbol = Obj.getWasmSymbol(Sym); if (Symbol.Flags != 0) { - WasmYAML::SymbolInfo Info = { Symbol.Name, Symbol.Flags }; - LinkingSec->SymbolInfos.push_back(Info); + WasmYAML::SymbolInfo Info{Symbol.Name, Symbol.Flags}; + LinkingSec->SymbolInfos.emplace_back(Info); } } LinkingSec->DataSize = Obj.linkingData().DataSize; - LinkingSec->DataAlignment = Obj.linkingData().DataAlignment; + for (const wasm::WasmInitFunc &Func : Obj.linkingData().InitFunctions) { + WasmYAML::InitFunction F{Func.Priority, Func.FunctionIndex}; + LinkingSec->InitFunctions.emplace_back(F); + } CustomSec = std::move(LinkingSec); } else { CustomSec = make_unique<WasmYAML::CustomSection>(WasmSec.Name); @@ -234,7 +244,7 @@ ErrorOr<WasmYAML::Object *> WasmDumper::dump() { } case wasm::WASM_SEC_DATA: { auto DataSec = make_unique<WasmYAML::DataSection>(); - for (auto &Segment : Obj.dataSegments()) { + for (const object::WasmSegment &Segment : Obj.dataSegments()) { WasmYAML::DataSegment Seg; Seg.SectionOffset = Segment.SectionOffset; Seg.MemoryIndex = Segment.Data.MemoryIndex; diff --git a/tools/opt-viewer/opt-diff.py b/tools/opt-viewer/opt-diff.py index f39432c8bc9d..6b20d82c7eec 100755 --- a/tools/opt-viewer/opt-diff.py +++ b/tools/opt-viewer/opt-diff.py @@ -46,8 +46,8 @@ if __name__ == '__main__': parser.add_argument('--output', '-o', default='diff.opt.yaml') args = parser.parse_args() - files1 = optrecord.find_opt_files([args.yaml_dir_or_file_1]) - files2 = optrecord.find_opt_files([args.yaml_dir_or_file_2]) + files1 = optrecord.find_opt_files(args.yaml_dir_or_file_1) + files2 = optrecord.find_opt_files(args.yaml_dir_or_file_2) print_progress = not args.no_progress_indicator all_remarks1, _, _ = optrecord.gather_results(files1, args.jobs, print_progress) @@ -60,5 +60,10 @@ if __name__ == '__main__': r.Added = True for r in removed: r.Added = False + + result = added | removed + for r in result: + r.recover_yaml_structure() + with open(args.output, 'w') as stream: - yaml.dump_all(added | removed, stream) + yaml.dump_all(result, stream) diff --git a/tools/opt-viewer/opt-stats.py b/tools/opt-viewer/opt-stats.py index 205b08ba8a74..5c415df1bb6d 100755 --- a/tools/opt-viewer/opt-stats.py +++ b/tools/opt-viewer/opt-stats.py @@ -13,6 +13,13 @@ import operator from collections import defaultdict from multiprocessing import cpu_count, Pool +try: + from guppy import hpy + hp = hpy() +except ImportError: + print("Memory consumption not shown because guppy is not installed") + hp = None + if __name__ == '__main__': parser = argparse.ArgumentParser(description=desc) parser.add_argument( @@ -36,7 +43,7 @@ if __name__ == '__main__': print_progress = not args.no_progress_indicator - files = optrecord.find_opt_files(args.yaml_dirs_or_files) + files = optrecord.find_opt_files(*args.yaml_dirs_or_files) if not files: parser.error("No *.opt.yaml files found") sys.exit(1) @@ -53,7 +60,12 @@ if __name__ == '__main__': byname[r.Pass + "/" + r.Name] += 1 total = len(all_remarks) - print("{:24s} {:10d}\n".format("Total number of remarks", total)) + print("{:24s} {:10d}".format("Total number of remarks", total)) + if hp: + h = hp.heap() + print("{:24s} {:10d}".format("Memory per remark", + h.size / len(all_remarks))) + print('\n') print("Top 10 remarks by pass:") for (passname, count) in sorted(bypass.items(), key=operator.itemgetter(1), diff --git a/tools/opt-viewer/opt-viewer.py b/tools/opt-viewer/opt-viewer.py index 69bcaedb7669..27b36064ced9 100755 --- a/tools/opt-viewer/opt-viewer.py +++ b/tools/opt-viewer/opt-viewer.py @@ -4,12 +4,14 @@ from __future__ import print_function import argparse import cgi +import codecs import errno import functools from multiprocessing import cpu_count import os.path import re import shutil +import sys from pygments import highlight from pygments.lexers.c_cpp import CppLexer @@ -33,6 +35,13 @@ class Context: context = Context() +def suppress(remark): + if remark.Name == 'sil.Specialized': + return remark.getArgDict()['Function'][0].startswith('\"Swift.') + elif remark.Name == 'sil.Inlined': + return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.')) + return False + class SourceFileRenderer: def __init__(self, source_dir, output_dir, filename): existing_filename = None @@ -43,7 +52,7 @@ class SourceFileRenderer: if os.path.exists(fn): existing_filename = fn - self.stream = open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w') + self.stream = codecs.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8') if existing_filename: self.source_stream = open(existing_filename) else: @@ -59,15 +68,29 @@ class SourceFileRenderer: def render_source_lines(self, stream, line_remarks): file_text = stream.read() - html_highlighted = highlight(file_text, self.cpp_lexer, self.html_formatter) - # Take off the header and footer, these must be - # reapplied line-wise, within the page structure - html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '') - html_highlighted = html_highlighted.replace('</pre></div>', '') + if args.no_highlight: + html_highlighted = file_text.decode('utf-8') + else: + html_highlighted = highlight( + file_text, + self.cpp_lexer, + self.html_formatter) + + # Note that the API is different between Python 2 and 3. On + # Python 3, pygments.highlight() returns a bytes object, so we + # have to decode. On Python 2, the output is str but since we + # support unicode characters and the output streams is unicode we + # decode too. + html_highlighted = html_highlighted.decode('utf-8') + + # Take off the header and footer, these must be + # reapplied line-wise, within the page structure + html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '') + html_highlighted = html_highlighted.replace('</pre></div>', '') for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1): - print(''' + print(u''' <tr> <td><a name=\"L{linenum}\">{linenum}</a></td> <td></td> @@ -76,13 +99,15 @@ class SourceFileRenderer: </tr>'''.format(**locals()), file=self.stream) for remark in line_remarks.get(linenum, []): - self.render_inline_remarks(remark, html_line) + if not suppress(remark): + self.render_inline_remarks(remark, html_line) def render_inline_remarks(self, r, line): inlining_context = r.DemangledFunctionName dl = context.caller_loc.get(r.Function) if dl: - link = optrecord.make_link(dl['File'], dl['Line'] - 2) + dl_dict = dict(list(dl)) + link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2) inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals()) # Column is the number of characters *including* tabs, keep those and @@ -90,7 +115,7 @@ class SourceFileRenderer: indent = line[:max(r.Column, 1) - 1] indent = re.sub('\S', ' ', indent) - print(''' + print(u''' <tr> <td></td> <td>{r.RelativeHotness}</td> @@ -105,34 +130,40 @@ class SourceFileRenderer: print(''' <html> +<meta charset="utf-8" /> <head> <link rel='stylesheet' type='text/css' href='style.css'> </head> <body> <div class="centered"> -<table> +<table class="source"> +<thead> <tr> -<td>Line</td> -<td>Hotness</td> -<td>Optimization</td> -<td>Source</td> -<td>Inline Context</td> -</tr>''', file=self.stream) +<th style="width: 2%">Line</td> +<th style="width: 3%">Hotness</td> +<th style="width: 10%">Optimization</td> +<th style="width: 70%">Source</td> +<th style="width: 15%">Inline Context</td> +</tr> +</thead> +<tbody>''', file=self.stream) self.render_source_lines(self.source_stream, line_remarks) print(''' +</tbody> </table> </body> </html>''', file=self.stream) class IndexRenderer: - def __init__(self, output_dir): - self.stream = open(os.path.join(output_dir, 'index.html'), 'w') + def __init__(self, output_dir, should_display_hotness): + self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8') + self.should_display_hotness = should_display_hotness def render_entry(self, r, odd): escaped_name = cgi.escape(r.DemangledFunctionName) - print(''' + print(u''' <tr> <td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td> <td class=\"column-entry-{odd}\">{r.RelativeHotness}</td> @@ -143,6 +174,7 @@ class IndexRenderer: def render(self, all_remarks): print(''' <html> +<meta charset="utf-8" /> <head> <link rel='stylesheet' type='text/css' href='style.css'> </head> @@ -155,8 +187,14 @@ class IndexRenderer: <td>Function</td> <td>Pass</td> </tr>''', file=self.stream) - for i, remark in enumerate(all_remarks): - self.render_entry(remark, i % 2) + + max_entries = None + if should_display_hotness: + max_entries = args.max_hottest_remarks_on_index + + for i, remark in enumerate(all_remarks[:max_entries]): + if not suppress(remark): + self.render_entry(remark, i % 2) print(''' </table> </body> @@ -176,10 +214,11 @@ def map_remarks(all_remarks): for remark in optrecord.itervalues(all_remarks): if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined": for arg in remark.Args: - caller = arg.get('Caller') + arg_dict = dict(list(arg)) + caller = arg_dict.get('Caller') if caller: try: - context.caller_loc[caller] = arg['DebugLoc'] + context.caller_loc[caller] = arg_dict['DebugLoc'] except KeyError: pass @@ -211,7 +250,7 @@ def generate_report(all_remarks, sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True) else: sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function)) - IndexRenderer(args.output_dir).render(sorted_remarks) + IndexRenderer(args.output_dir, should_display_hotness).render(sorted_remarks) shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), "style.css"), output_dir) @@ -238,7 +277,7 @@ if __name__ == '__main__': type=int, help='Max job count (defaults to %(default)s, the current CPU count)') parser.add_argument( - '-source-dir', + '--source-dir', '-s', default='', help='set source directory') @@ -249,11 +288,26 @@ if __name__ == '__main__': default=False, help='Do not display any indicator of how many YAML files were read ' 'or rendered into HTML.') + parser.add_argument( + '--max-hottest-remarks-on-index', + default=1000, + type=int, + help='Maximum number of the hottest remarks to appear on the index page') + parser.add_argument( + '--no-highlight', + action='store_true', + default=False, + help='Do not use a syntax highlighter when rendering the source code') + parser.add_argument( + '--demangler', + help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler) args = parser.parse_args() print_progress = not args.no_progress_indicator + if args.demangler: + optrecord.Remark.set_demangler(args.demangler) - files = optrecord.find_opt_files(args.yaml_dirs_or_files) + files = optrecord.find_opt_files(*args.yaml_dirs_or_files) if not files: parser.error("No *.opt.yaml files found") sys.exit(1) diff --git a/tools/opt-viewer/optrecord.py b/tools/opt-viewer/optrecord.py index 4599e12d7e68..54e791192531 100644 --- a/tools/opt-viewer/optrecord.py +++ b/tools/opt-viewer/optrecord.py @@ -17,14 +17,15 @@ import functools from multiprocessing import Lock import os, os.path import subprocess +try: + # The previously builtin function `intern()` was moved + # to the `sys` module in Python 3. + from sys import intern +except: + pass import optpmap - -p = subprocess.Popen(['c++filt', '-n'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) -p_lock = Lock() - - try: dict.iteritems except AttributeError: @@ -41,13 +42,6 @@ else: return d.iteritems() -def demangle(name): - with p_lock: - p.stdin.write((name + '\n').encode('utf-8')) - p.stdin.flush() - return p.stdout.readline().rstrip().decode('utf-8') - - def html_file_name(filename): return filename.replace('/', '_').replace('#', '_') + ".html" @@ -60,11 +54,73 @@ class Remark(yaml.YAMLObject): # Work-around for http://pyyaml.org/ticket/154. yaml_loader = Loader - def initmissing(self): + default_demangler = 'c++filt -n' + demangler_proc = None + + @classmethod + def set_demangler(cls, demangler): + cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE) + cls.demangler_lock = Lock() + + @classmethod + def demangle(cls, name): + with cls.demangler_lock: + cls.demangler_proc.stdin.write((name + '\n').encode('utf-8')) + cls.demangler_proc.stdin.flush() + return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8') + + # Intern all strings since we have lot of duplication across filenames, + # remark text. + # + # Change Args from a list of dicts to a tuple of tuples. This saves + # memory in two ways. One, a small tuple is significantly smaller than a + # small dict. Two, using tuple instead of list allows Args to be directly + # used as part of the key (in Python only immutable types are hashable). + def _reduce_memory(self): + self.Pass = intern(self.Pass) + self.Name = intern(self.Name) + try: + # Can't intern unicode strings. + self.Function = intern(self.Function) + except: + pass + + def _reduce_memory_dict(old_dict): + new_dict = dict() + for (k, v) in iteritems(old_dict): + if type(k) is str: + k = intern(k) + + if type(v) is str: + v = intern(v) + elif type(v) is dict: + # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}] + v = _reduce_memory_dict(v) + new_dict[k] = v + return tuple(new_dict.items()) + + self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args]) + + # The inverse operation of the dictonary-related memory optimization in + # _reduce_memory_dict. E.g. + # (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}] + def recover_yaml_structure(self): + def tuple_to_dict(t): + d = dict() + for (k, v) in t: + if type(v) is tuple: + v = tuple_to_dict(v) + d[k] = v + return d + + self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args] + + def canonicalize(self): if not hasattr(self, 'Hotness'): self.Hotness = 0 if not hasattr(self, 'Args'): self.Args = [] + self._reduce_memory() @property def File(self): @@ -84,30 +140,57 @@ class Remark(yaml.YAMLObject): @property def DemangledFunctionName(self): - return demangle(self.Function) + return self.demangle(self.Function) @property def Link(self): return make_link(self.File, self.Line) def getArgString(self, mapping): - mapping = mapping.copy() + mapping = dict(list(mapping)) dl = mapping.get('DebugLoc') if dl: del mapping['DebugLoc'] assert(len(mapping) == 1) - (key, value) = mapping.items()[0] + (key, value) = list(mapping.items())[0] if key == 'Caller' or key == 'Callee': - value = cgi.escape(demangle(value)) + value = cgi.escape(self.demangle(value)) if dl and key != 'Caller': - return "<a href={}>{}</a>".format( - make_link(dl['File'], dl['Line']), value) + dl_dict = dict(list(dl)) + return u"<a href={}>{}</a>".format( + make_link(dl_dict['File'], dl_dict['Line']), value) else: return value + # Return a cached dictionary for the arguments. The key for each entry is + # the argument key (e.g. 'Callee' for inlining remarks. The value is a + # list containing the value (e.g. for 'Callee' the function) and + # optionally a DebugLoc. + def getArgDict(self): + if hasattr(self, 'ArgDict'): + return self.ArgDict + self.ArgDict = {} + for arg in self.Args: + if len(arg) == 2: + if arg[0][0] == 'DebugLoc': + dbgidx = 0 + else: + assert(arg[1][0] == 'DebugLoc') + dbgidx = 1 + + key = arg[1 - dbgidx][0] + entry = (arg[1 - dbgidx][1], arg[dbgidx][1]) + else: + arg = arg[0] + key = arg[0] + entry = (arg[1], ) + + self.ArgDict[key] = entry + return self.ArgDict + def getDiffPrefix(self): if hasattr(self, 'Added'): if self.Added: @@ -129,19 +212,14 @@ class Remark(yaml.YAMLObject): @property def RelativeHotness(self): if self.max_hotness: - return "{}%".format(int(round(self.Hotness * 100 / self.max_hotness))) + return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness) else: return '' @property def key(self): - k = (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, self.Line, self.Column, self.Function) - for arg in self.Args: - for (key, value) in iteritems(arg): - if type(value) is dict: - value = tuple(value.items()) - k += (key, value) - return k + return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, + self.Line, self.Column, self.Function, self.Args) def __hash__(self): return hash(self.key) @@ -193,7 +271,7 @@ def get_remarks(input_file): with open(input_file) as f: docs = yaml.load_all(f, Loader=Loader) for remark in docs: - remark.initmissing() + remark.canonicalize() # Avoid remarks withoug debug location or if they are duplicated if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: continue @@ -214,6 +292,8 @@ def get_remarks(input_file): def gather_results(filenames, num_jobs, should_print_progress): if should_print_progress: print('Reading YAML files...') + if not Remark.demangler_proc: + Remark.set_demangler(Remark.default_demangler) remarks = optpmap.pmap( get_remarks, filenames, num_jobs, should_print_progress) max_hotness = max(entry[0] for entry in remarks) @@ -237,7 +317,7 @@ def gather_results(filenames, num_jobs, should_print_progress): return all_remarks, file_remarks, max_hotness != 0 -def find_opt_files(dirs_or_files): +def find_opt_files(*dirs_or_files): all = [] for dir_or_file in dirs_or_files: if os.path.isfile(dir_or_file): diff --git a/tools/opt-viewer/style.css b/tools/opt-viewer/style.css index 595c3e46847d..0d3347c1578c 100644 --- a/tools/opt-viewer/style.css +++ b/tools/opt-viewer/style.css @@ -1,3 +1,13 @@ +.source { + table-layout: fixed; + width: 100%; + white-space: nowrap; +} +.source td { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} .red { background-color: #ffd0d0; } diff --git a/tools/opt/CMakeLists.txt b/tools/opt/CMakeLists.txt index 518396e36028..dedc25143cf4 100644 --- a/tools/opt/CMakeLists.txt +++ b/tools/opt/CMakeLists.txt @@ -25,6 +25,7 @@ set(LLVM_NO_DEAD_STRIP 1) add_llvm_tool(opt AnalysisWrappers.cpp BreakpointPrinter.cpp + Debugify.cpp GraphPrinters.cpp NewPMDriver.cpp PassPrinters.cpp @@ -37,5 +38,5 @@ add_llvm_tool(opt export_executable_symbols(opt) if(WITH_POLLY AND LINK_POLLY_INTO_TOOLS) - target_link_libraries(opt Polly) + target_link_libraries(opt PRIVATE Polly) endif(WITH_POLLY AND LINK_POLLY_INTO_TOOLS) diff --git a/tools/opt/Debugify.cpp b/tools/opt/Debugify.cpp new file mode 100644 index 000000000000..40ee545c098d --- /dev/null +++ b/tools/opt/Debugify.cpp @@ -0,0 +1,212 @@ +//===- Debugify.cpp - Attach synthetic debug info to everything -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file This pass attaches synthetic debug info to everything. It can be used +/// to create targeted tests for debug info preservation. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DIBuilder.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/Pass.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO.h" + +using namespace llvm; + +namespace { + +bool applyDebugifyMetadata(Module &M) { + // Skip modules with debug info. + if (M.getNamedMetadata("llvm.dbg.cu")) { + errs() << "Debugify: Skipping module with debug info\n"; + return false; + } + + DIBuilder DIB(M); + LLVMContext &Ctx = M.getContext(); + + // Get a DIType which corresponds to Ty. + DenseMap<uint64_t, DIType *> TypeCache; + auto getCachedDIType = [&](Type *Ty) -> DIType * { + uint64_t Size = M.getDataLayout().getTypeAllocSizeInBits(Ty); + DIType *&DTy = TypeCache[Size]; + if (!DTy) { + std::string Name = "ty" + utostr(Size); + DTy = DIB.createBasicType(Name, Size, dwarf::DW_ATE_unsigned); + } + return DTy; + }; + + unsigned NextLine = 1; + unsigned NextVar = 1; + auto File = DIB.createFile(M.getName(), "/"); + auto CU = + DIB.createCompileUnit(dwarf::DW_LANG_C, DIB.createFile(M.getName(), "/"), + "debugify", /*isOptimized=*/true, "", 0); + + // Visit each instruction. + for (Function &F : M) { + if (F.isDeclaration()) + continue; + + auto SPType = DIB.createSubroutineType(DIB.getOrCreateTypeArray(None)); + bool IsLocalToUnit = F.hasPrivateLinkage() || F.hasInternalLinkage(); + auto SP = + DIB.createFunction(CU, F.getName(), F.getName(), File, NextLine, SPType, + IsLocalToUnit, F.hasExactDefinition(), NextLine, + DINode::FlagZero, /*isOptimized=*/true); + F.setSubprogram(SP); + for (BasicBlock &BB : F) { + // Attach debug locations. + for (Instruction &I : BB) + I.setDebugLoc(DILocation::get(Ctx, NextLine++, 1, SP)); + + // Attach debug values. + for (Instruction &I : BB) { + // Skip void-valued instructions. + if (I.getType()->isVoidTy()) + continue; + + // Skip the terminator instruction and any just-inserted intrinsics. + if (isa<TerminatorInst>(&I) || isa<DbgValueInst>(&I)) + break; + + std::string Name = utostr(NextVar++); + const DILocation *Loc = I.getDebugLoc().get(); + auto LocalVar = DIB.createAutoVariable(SP, Name, File, Loc->getLine(), + getCachedDIType(I.getType()), + /*AlwaysPreserve=*/true); + DIB.insertDbgValueIntrinsic(&I, LocalVar, DIB.createExpression(), Loc, + BB.getTerminator()); + } + } + DIB.finalizeSubprogram(SP); + } + DIB.finalize(); + + // Track the number of distinct lines and variables. + NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.debugify"); + auto *IntTy = Type::getInt32Ty(Ctx); + auto addDebugifyOperand = [&](unsigned N) { + NMD->addOperand(MDNode::get( + Ctx, ValueAsMetadata::getConstant(ConstantInt::get(IntTy, N)))); + }; + addDebugifyOperand(NextLine - 1); // Original number of lines. + addDebugifyOperand(NextVar - 1); // Original number of variables. + return true; +} + +void checkDebugifyMetadata(Module &M) { + // Skip modules without debugify metadata. + NamedMDNode *NMD = M.getNamedMetadata("llvm.debugify"); + if (!NMD) + return; + + auto getDebugifyOperand = [&](unsigned Idx) -> unsigned { + return mdconst::extract<ConstantInt>(NMD->getOperand(Idx)->getOperand(0)) + ->getZExtValue(); + }; + unsigned OriginalNumLines = getDebugifyOperand(0); + unsigned OriginalNumVars = getDebugifyOperand(1); + bool HasErrors = false; + + // Find missing lines. + BitVector MissingLines{OriginalNumLines, true}; + for (Function &F : M) { + for (Instruction &I : instructions(F)) { + if (isa<DbgValueInst>(&I)) + continue; + + auto DL = I.getDebugLoc(); + if (DL) { + MissingLines.reset(DL.getLine() - 1); + continue; + } + + outs() << "ERROR: Instruction with empty DebugLoc -- "; + I.print(outs()); + outs() << "\n"; + HasErrors = true; + } + } + for (unsigned Idx : MissingLines.set_bits()) + outs() << "WARNING: Missing line " << Idx + 1 << "\n"; + + // Find missing variables. + BitVector MissingVars{OriginalNumVars, true}; + for (Function &F : M) { + for (Instruction &I : instructions(F)) { + auto *DVI = dyn_cast<DbgValueInst>(&I); + if (!DVI) + continue; + + unsigned Var = ~0U; + (void)to_integer(DVI->getVariable()->getName(), Var, 10); + assert(Var <= OriginalNumVars && "Unexpected name for DILocalVariable"); + MissingVars.reset(Var - 1); + } + } + for (unsigned Idx : MissingVars.set_bits()) + outs() << "ERROR: Missing variable " << Idx + 1 << "\n"; + HasErrors |= MissingVars.count() > 0; + + outs() << "CheckDebugify: " << (HasErrors ? "FAIL" : "PASS") << "\n"; +} + +/// Attach synthetic debug info to everything. +struct DebugifyPass : public ModulePass { + bool runOnModule(Module &M) override { return applyDebugifyMetadata(M); } + + DebugifyPass() : ModulePass(ID) {} + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + } + + static char ID; // Pass identification. +}; + +/// Check debug info inserted by -debugify for completeness. +struct CheckDebugifyPass : public ModulePass { + bool runOnModule(Module &M) override { + checkDebugifyMetadata(M); + return false; + } + + CheckDebugifyPass() : ModulePass(ID) {} + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + } + + static char ID; // Pass identification. +}; + +} // end anonymous namespace + +char DebugifyPass::ID = 0; +static RegisterPass<DebugifyPass> X("debugify", + "Attach debug info to everything"); + +char CheckDebugifyPass::ID = 0; +static RegisterPass<CheckDebugifyPass> Y("check-debugify", + "Check debug info from -debugify"); diff --git a/tools/opt/NewPMDriver.cpp b/tools/opt/NewPMDriver.cpp index 94242d795aae..a3f16f2538c4 100644 --- a/tools/opt/NewPMDriver.cpp +++ b/tools/opt/NewPMDriver.cpp @@ -18,6 +18,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm/Config/config.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LLVMContext.h" @@ -81,6 +82,22 @@ static cl::opt<std::string> VectorizerStartEPPipeline( cl::desc("A textual description of the function pass pipeline inserted at " "the VectorizerStart extension point into default pipelines"), cl::Hidden); +enum PGOKind { NoPGO, InstrGen, InstrUse, SampleUse }; +static cl::opt<PGOKind> PGOKindFlag( + "pgo-kind", cl::init(NoPGO), cl::Hidden, + cl::desc("The kind of profile guided optimization"), + cl::values(clEnumValN(NoPGO, "nopgo", "Do not use PGO."), + clEnumValN(InstrGen, "new-pm-pgo-instr-gen-pipeline", + "Instrument the IR to generate profile."), + clEnumValN(InstrUse, "new-pm-pgo-instr-use-pipeline", + "Use instrumented profile to guide PGO."), + clEnumValN(SampleUse, "new-pm-pgo-sample-use-pipeline", + "Use sampled profile to guide PGO."))); +static cl::opt<std::string> ProfileFile( + "profile-file", cl::desc("Path to the profile."), cl::Hidden); +static cl::opt<bool> DebugInfoForProfiling( + "new-pm-debug-info-for-profiling", cl::init(false), cl::Hidden, + cl::desc("Emit special debug info to enable PGO profile generation.")); /// @}} template <typename PassManagerT> @@ -144,18 +161,46 @@ static void registerEPCallbacks(PassBuilder &PB, bool VerifyEachPass, }); } +#ifdef LINK_POLLY_INTO_TOOLS +namespace polly { +void RegisterPollyPasses(PassBuilder &); +} +#endif + bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM, - tool_output_file *Out, - tool_output_file *ThinLTOLinkOut, + ToolOutputFile *Out, ToolOutputFile *ThinLTOLinkOut, + ToolOutputFile *OptRemarkFile, StringRef PassPipeline, OutputKind OK, VerifierKind VK, bool ShouldPreserveAssemblyUseListOrder, bool ShouldPreserveBitcodeUseListOrder, bool EmitSummaryIndex, bool EmitModuleHash) { bool VerifyEachPass = VK == VK_VerifyEachPass; - PassBuilder PB(TM); + + Optional<PGOOptions> P; + switch (PGOKindFlag) { + case InstrGen: + P = PGOOptions(ProfileFile, "", "", true); + break; + case InstrUse: + P = PGOOptions("", ProfileFile, "", false); + break; + case SampleUse: + P = PGOOptions("", "", ProfileFile, false); + break; + case NoPGO: + if (DebugInfoForProfiling) + P = PGOOptions("", "", "", false, true); + else + P = None; + } + PassBuilder PB(TM, P); registerEPCallbacks(PB, VerifyEachPass, DebugPM); +#ifdef LINK_POLLY_INTO_TOOLS + polly::RegisterPollyPasses(PB); +#endif + // Specially handle the alias analysis manager so that we can register // a custom pipeline of AA passes with it. AAManager AA; @@ -221,5 +266,9 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM, if (OK == OK_OutputThinLTOBitcode && ThinLTOLinkOut) ThinLTOLinkOut->keep(); } + + if (OptRemarkFile) + OptRemarkFile->keep(); + return true; } diff --git a/tools/opt/NewPMDriver.h b/tools/opt/NewPMDriver.h index 8012e0a025c9..e5490deaeaf5 100644 --- a/tools/opt/NewPMDriver.h +++ b/tools/opt/NewPMDriver.h @@ -26,7 +26,7 @@ class StringRef; class LLVMContext; class Module; class TargetMachine; -class tool_output_file; +class ToolOutputFile; namespace opt_tool { enum OutputKind { @@ -52,9 +52,9 @@ enum VerifierKind { /// ThinLTOLinkOut is only used when OK is OK_OutputThinLTOBitcode, and can be /// nullptr. bool runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM, - tool_output_file *Out, tool_output_file *ThinLinkOut, - StringRef PassPipeline, opt_tool::OutputKind OK, - opt_tool::VerifierKind VK, + ToolOutputFile *Out, ToolOutputFile *ThinLinkOut, + ToolOutputFile *OptRemarkFile, StringRef PassPipeline, + opt_tool::OutputKind OK, opt_tool::VerifierKind VK, bool ShouldPreserveAssemblyUseListOrder, bool ShouldPreserveBitcodeUseListOrder, bool EmitSummaryIndex, bool EmitModuleHash); diff --git a/tools/opt/PassPrinters.cpp b/tools/opt/PassPrinters.cpp index 65a53038fc50..f52b52080949 100644 --- a/tools/opt/PassPrinters.cpp +++ b/tools/opt/PassPrinters.cpp @@ -11,12 +11,18 @@ /// \brief Utilities to print analysis info for various kinds of passes. /// //===----------------------------------------------------------------------===// + #include "PassPrinters.h" +#include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopPass.h" +#include "llvm/Analysis/RegionInfo.h" #include "llvm/Analysis/RegionPass.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" #include "llvm/Pass.h" +#include "llvm/Support/raw_ostream.h" #include <string> using namespace llvm; @@ -226,7 +232,8 @@ struct BasicBlockPassPrinter : public BasicBlockPass { }; char BasicBlockPassPrinter::ID = 0; -} + +} // end anonymous namespace FunctionPass *llvm::createFunctionPassPrinter(const PassInfo *PI, raw_ostream &OS, bool Quiet) { diff --git a/tools/opt/PassPrinters.h b/tools/opt/PassPrinters.h index cf46ef9e36d2..14b6e43d18e0 100644 --- a/tools/opt/PassPrinters.h +++ b/tools/opt/PassPrinters.h @@ -1,4 +1,4 @@ -//===- PassPrinters.h - Utilities to print analysis info for passes -------===// +//=- PassPrinters.h - Utilities to print analysis info for passes -*- C++ -*-=// // // The LLVM Compiler Infrastructure // @@ -11,6 +11,7 @@ /// \brief Utilities to print analysis info for various kinds of passes. /// //===----------------------------------------------------------------------===// + #ifndef LLVM_TOOLS_OPT_PASSPRINTERS_H #define LLVM_TOOLS_OPT_PASSPRINTERS_H @@ -22,8 +23,8 @@ class FunctionPass; class ModulePass; class LoopPass; class PassInfo; -class RegionPass; class raw_ostream; +class RegionPass; FunctionPass *createFunctionPassPrinter(const PassInfo *PI, raw_ostream &out, bool Quiet); @@ -42,6 +43,7 @@ RegionPass *createRegionPassPrinter(const PassInfo *PI, raw_ostream &out, BasicBlockPass *createBasicBlockPassPrinter(const PassInfo *PI, raw_ostream &out, bool Quiet); -} + +} // end namespace llvm #endif // LLVM_TOOLS_OPT_PASSPRINTERS_H diff --git a/tools/opt/opt.cpp b/tools/opt/opt.cpp index 24cce58047f1..5bc00ea35ae5 100644 --- a/tools/opt/opt.cpp +++ b/tools/opt/opt.cpp @@ -23,7 +23,7 @@ #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeWriterPass.h" -#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/CommandFlags.def" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" @@ -349,7 +349,7 @@ static TargetMachine* GetTargetMachine(Triple TheTriple, StringRef CPUStr, return TheTarget->createTargetMachine(TheTriple.getTriple(), CPUStr, FeaturesStr, Options, getRelocModel(), - CMModel, GetCodeGenOptLevel()); + getCodeModel(), GetCodeGenOptLevel()); } #ifdef LINK_POLLY_INTO_TOOLS @@ -391,6 +391,7 @@ int main(int argc, char **argv) { initializeTarget(Registry); // For codegen passes, only passes that do IR to IR transformation are // supported. + initializeExpandMemCmpPassPass(Registry); initializeScalarizeMaskedMemIntrinPass(Registry); initializeCodeGenPreparePass(Registry); initializeAtomicExpandPass(Registry); @@ -402,9 +403,11 @@ int main(int argc, char **argv) { initializePreISelIntrinsicLoweringLegacyPassPass(Registry); initializeGlobalMergePass(Registry); initializeInterleavedAccessPass(Registry); - initializeCountingFunctionInserterPass(Registry); + initializeEntryExitInstrumenterPass(Registry); + initializePostInlineEntryExitInstrumenterPass(Registry); initializeUnreachableBlockElimLegacyPassPass(Registry); initializeExpandReductionsPass(Registry); + initializeWriteBitcodePassPass(Registry); #ifdef LINK_POLLY_INTO_TOOLS polly::initializePollyPasses(Registry); @@ -430,21 +433,22 @@ int main(int argc, char **argv) { if (PassRemarksHotnessThreshold) Context.setDiagnosticsHotnessThreshold(PassRemarksHotnessThreshold); - std::unique_ptr<tool_output_file> YamlFile; + std::unique_ptr<ToolOutputFile> OptRemarkFile; if (RemarksFilename != "") { std::error_code EC; - YamlFile = llvm::make_unique<tool_output_file>(RemarksFilename, EC, - sys::fs::F_None); + OptRemarkFile = + llvm::make_unique<ToolOutputFile>(RemarksFilename, EC, sys::fs::F_None); if (EC) { errs() << EC.message() << '\n'; return 1; } Context.setDiagnosticsOutputFile( - llvm::make_unique<yaml::Output>(YamlFile->os())); + llvm::make_unique<yaml::Output>(OptRemarkFile->os())); } // Load the input module... - std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context); + std::unique_ptr<Module> M = + parseIRFile(InputFilename, Err, Context, !NoVerify); if (!M) { Err.print(argv[0], errs()); @@ -471,8 +475,8 @@ int main(int argc, char **argv) { M->setDataLayout(ClDataLayout); // Figure out what stream we are supposed to write to... - std::unique_ptr<tool_output_file> Out; - std::unique_ptr<tool_output_file> ThinLinkOut; + std::unique_ptr<ToolOutputFile> Out; + std::unique_ptr<ToolOutputFile> ThinLinkOut; if (NoOutput) { if (!OutputFilename.empty()) errs() << "WARNING: The -o (output filename) option is ignored when\n" @@ -483,7 +487,7 @@ int main(int argc, char **argv) { OutputFilename = "-"; std::error_code EC; - Out.reset(new tool_output_file(OutputFilename, EC, sys::fs::F_None)); + Out.reset(new ToolOutputFile(OutputFilename, EC, sys::fs::F_None)); if (EC) { errs() << EC.message() << '\n'; return 1; @@ -491,7 +495,7 @@ int main(int argc, char **argv) { if (!ThinLinkBitcodeFile.empty()) { ThinLinkOut.reset( - new tool_output_file(ThinLinkBitcodeFile, EC, sys::fs::F_None)); + new ToolOutputFile(ThinLinkBitcodeFile, EC, sys::fs::F_None)); if (EC) { errs() << EC.message() << '\n'; return 1; @@ -540,7 +544,8 @@ int main(int argc, char **argv) { // string. Hand off the rest of the functionality to the new code for that // layer. return runPassPipeline(argv[0], *M, TM.get(), Out.get(), ThinLinkOut.get(), - PassPipeline, OK, VK, PreserveAssemblyUseListOrder, + OptRemarkFile.get(), PassPipeline, OK, VK, + PreserveAssemblyUseListOrder, PreserveBitcodeUseListOrder, EmitSummaryIndex, EmitModuleHash) ? 0 @@ -579,8 +584,8 @@ int main(int argc, char **argv) { OutputFilename = "-"; std::error_code EC; - Out = llvm::make_unique<tool_output_file>(OutputFilename, EC, - sys::fs::F_None); + Out = llvm::make_unique<ToolOutputFile>(OutputFilename, EC, + sys::fs::F_None); if (EC) { errs() << EC.message() << '\n'; return 1; @@ -767,8 +772,8 @@ int main(int argc, char **argv) { "the compile-twice option\n"; Out->os() << BOS->str(); Out->keep(); - if (YamlFile) - YamlFile->keep(); + if (OptRemarkFile) + OptRemarkFile->keep(); return 1; } Out->os() << BOS->str(); @@ -778,8 +783,8 @@ int main(int argc, char **argv) { if (!NoOutput || PrintBreakpoints) Out->keep(); - if (YamlFile) - YamlFile->keep(); + if (OptRemarkFile) + OptRemarkFile->keep(); if (ThinLinkOut) ThinLinkOut->keep(); diff --git a/tools/sancov/coverage-report-server.py b/tools/sancov/coverage-report-server.py index 428276f95d3b..a2e161d0de58 100755 --- a/tools/sancov/coverage-report-server.py +++ b/tools/sancov/coverage-report-server.py @@ -160,7 +160,7 @@ class ServerHandler(http.server.BaseHTTPRequestHandler): linemap = self.symcov_data.compute_linemap(filename) - with open(filepath, 'r') as f: + with open(filepath, 'r', encoding='utf8') as f: content = "\n".join( ["<span class='{cls}'>{line} </span>".format( line=html.escape(line.rstrip()), diff --git a/tools/sancov/sancov.cc b/tools/sancov/sancov.cc index 7f103ebb904b..f3bc635dde8d 100644 --- a/tools/sancov/sancov.cc +++ b/tools/sancov/sancov.cc @@ -18,7 +18,6 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/MC/MCInst.h" -#include "llvm/MC/MCInstPrinter.h" #include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCObjectFileInfo.h" @@ -27,7 +26,6 @@ #include "llvm/Object/Archive.h" #include "llvm/Object/Binary.h" #include "llvm/Object/COFF.h" -#include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/MachO.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Casting.h" @@ -35,7 +33,6 @@ #include "llvm/Support/Errc.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/LineIterator.h" #include "llvm/Support/MD5.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MemoryBuffer.h" @@ -48,15 +45,10 @@ #include "llvm/Support/SpecialCaseList.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/TargetSelect.h" -#include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <set> -#include <stdio.h> -#include <string> -#include <utility> #include <vector> using namespace llvm; @@ -584,13 +576,16 @@ public: UserBlacklist(createUserBlacklist()) {} bool isBlacklisted(const DILineInfo &I) { - if (DefaultBlacklist && DefaultBlacklist->inSection("fun", I.FunctionName)) + if (DefaultBlacklist && + DefaultBlacklist->inSection("sancov", "fun", I.FunctionName)) return true; - if (DefaultBlacklist && DefaultBlacklist->inSection("src", I.FileName)) + if (DefaultBlacklist && + DefaultBlacklist->inSection("sancov", "src", I.FileName)) return true; - if (UserBlacklist && UserBlacklist->inSection("fun", I.FunctionName)) + if (UserBlacklist && + UserBlacklist->inSection("sancov", "fun", I.FunctionName)) return true; - if (UserBlacklist && UserBlacklist->inSection("src", I.FileName)) + if (UserBlacklist && UserBlacklist->inSection("sancov", "src", I.FileName)) return true; return false; } diff --git a/tools/sanstats/sanstats.cpp b/tools/sanstats/sanstats.cpp index 4463c0f0e48c..71f0207bab50 100644 --- a/tools/sanstats/sanstats.cpp +++ b/tools/sanstats/sanstats.cpp @@ -77,7 +77,7 @@ const char *ReadModule(char SizeofPtr, const char *Begin, const char *End) { return nullptr; // As the instrumentation tracks the return address and not - // the address of the call to `__sanitizer_stats_report` we + // the address of the call to `__sanitizer_stat_report` we // remove one from the address to get the correct DI. if (Expected<DILineInfo> LineInfo = Symbolizer.symbolizeCode(Filename, Addr - 1)) { diff --git a/tools/xcode-toolchain/CMakeLists.txt b/tools/xcode-toolchain/CMakeLists.txt index 978c4bc81dc8..d433c52febf1 100644 --- a/tools/xcode-toolchain/CMakeLists.txt +++ b/tools/xcode-toolchain/CMakeLists.txt @@ -93,13 +93,11 @@ add_custom_command(OUTPUT ${LLVMToolchainDir}/Info.plist COMMAND /usr/libexec/PlistBuddy -c "Add:CompatibilityVersion integer ${COMPAT_VERSION}" "${LLVMToolchainDir}/Info.plist" ) -add_custom_target(install-xcode-toolchain - DEPENDS ${LLVMToolchainDir}/Info.plist - COMMAND "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target all - COMMAND "${CMAKE_COMMAND}" - -DCMAKE_INSTALL_PREFIX=${LLVMToolchainDir}/usr/ - -P "${CMAKE_BINARY_DIR}/cmake_install.cmake" - USES_TERMINAL) +add_custom_target(build-xcode-toolchain + COMMAND "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target all) +add_llvm_install_targets(install-xcode-toolchain + DEPENDS ${LLVMToolchainDir}/Info.plist build-xcode-toolchain + PREFIX ${LLVMToolchainDir}/usr/) if(LLVM_DISTRIBUTION_COMPONENTS) if(CMAKE_CONFIGURATION_TYPES) @@ -110,13 +108,10 @@ if(LLVM_DISTRIBUTION_COMPONENTS) DEPENDS ${LLVMToolchainDir}/Info.plist distribution) foreach(target ${LLVM_DISTRIBUTION_COMPONENTS}) - add_custom_target(install-distribution-${target} - DEPENDS ${target} - COMMAND "${CMAKE_COMMAND}" - -DCMAKE_INSTALL_COMPONENT=${target} - -DCMAKE_INSTALL_PREFIX=${LLVMToolchainDir}/usr/ - -P "${CMAKE_BINARY_DIR}/cmake_install.cmake" - USES_TERMINAL) + add_llvm_install_targets(install-distribution-${target} + DEPENDS ${target} + COMPONENT ${target} + PREFIX ${LLVMToolchainDir}/usr/) add_dependencies(install-distribution-toolchain install-distribution-${target}) endforeach() endif() diff --git a/tools/yaml2obj/yaml2coff.cpp b/tools/yaml2obj/yaml2coff.cpp index 1f302fdc45a7..648317e97bb3 100644 --- a/tools/yaml2obj/yaml2coff.cpp +++ b/tools/yaml2obj/yaml2coff.cpp @@ -234,6 +234,9 @@ static bool layoutCOFF(COFFParser &CP) { } else if (S.Name == ".debug$T") { if (S.SectionData.binary_size() == 0) S.SectionData = CodeViewYAML::toDebugT(S.DebugT, CP.Allocator); + } else if (S.Name == ".debug$H") { + if (S.DebugH.hasValue() && S.SectionData.binary_size() == 0) + S.SectionData = CodeViewYAML::toDebugH(*S.DebugH, CP.Allocator); } if (S.SectionData.binary_size() > 0) { diff --git a/tools/yaml2obj/yaml2elf.cpp b/tools/yaml2obj/yaml2elf.cpp index c89f768ed6ff..21648469654a 100644 --- a/tools/yaml2obj/yaml2elf.cpp +++ b/tools/yaml2obj/yaml2elf.cpp @@ -74,6 +74,15 @@ public: Idx = I->getValue(); return false; } + /// asserts if name is not present in the map + unsigned get(StringRef Name) const { + unsigned Idx = 0; + auto missing = lookup(Name, Idx); + (void)missing; + assert(!missing && "Expected section not found in index"); + return Idx; + } + unsigned size() const { return Map.size(); } }; } // end anonymous namespace @@ -99,17 +108,23 @@ namespace { template <class ELFT> class ELFState { typedef typename object::ELFFile<ELFT>::Elf_Ehdr Elf_Ehdr; + typedef typename object::ELFFile<ELFT>::Elf_Phdr Elf_Phdr; typedef typename object::ELFFile<ELFT>::Elf_Shdr Elf_Shdr; typedef typename object::ELFFile<ELFT>::Elf_Sym Elf_Sym; typedef typename object::ELFFile<ELFT>::Elf_Rel Elf_Rel; typedef typename object::ELFFile<ELFT>::Elf_Rela Elf_Rela; + enum class SymtabType { Static, Dynamic }; + /// \brief The future ".strtab" section. StringTableBuilder DotStrtab{StringTableBuilder::ELF}; /// \brief The future ".shstrtab" section. StringTableBuilder DotShStrtab{StringTableBuilder::ELF}; + /// \brief The future ".dynstr" section. + StringTableBuilder DotDynstr{StringTableBuilder::ELF}; + NameToIdxMap SN2I; NameToIdxMap SymN2I; const ELFYAML::Object &Doc; @@ -118,15 +133,19 @@ class ELFState { bool buildSymbolIndex(std::size_t &StartIndex, const std::vector<ELFYAML::Symbol> &Symbols); void initELFHeader(Elf_Ehdr &Header); + void initProgramHeaders(std::vector<Elf_Phdr> &PHeaders); bool initSectionHeaders(std::vector<Elf_Shdr> &SHeaders, ContiguousBlobAccumulator &CBA); - void initSymtabSectionHeader(Elf_Shdr &SHeader, + void initSymtabSectionHeader(Elf_Shdr &SHeader, SymtabType STType, ContiguousBlobAccumulator &CBA); void initStrtabSectionHeader(Elf_Shdr &SHeader, StringRef Name, StringTableBuilder &STB, ContiguousBlobAccumulator &CBA); + void setProgramHeaderLayout(std::vector<Elf_Phdr> &PHeaders, + std::vector<Elf_Shdr> &SHeaders); void addSymbols(const std::vector<ELFYAML::Symbol> &Symbols, - std::vector<Elf_Sym> &Syms, unsigned SymbolBinding); + std::vector<Elf_Sym> &Syms, unsigned SymbolBinding, + const StringTableBuilder &Strtab); void writeSectionContent(Elf_Shdr &SHeader, const ELFYAML::RawContentSection &Section, ContiguousBlobAccumulator &CBA); @@ -138,15 +157,21 @@ class ELFState { bool writeSectionContent(Elf_Shdr &SHeader, const ELFYAML::MipsABIFlags &Section, ContiguousBlobAccumulator &CBA); + bool hasDynamicSymbols() const; + SmallVector<const char *, 5> implicitSectionNames() const; // - SHT_NULL entry (placed first, i.e. 0'th entry) - // - symbol table (.symtab) (placed third to last) - // - string table (.strtab) (placed second to last) - // - section header string table (.shstrtab) (placed last) - unsigned getDotSymTabSecNo() const { return Doc.Sections.size() + 1; } - unsigned getDotStrTabSecNo() const { return Doc.Sections.size() + 2; } - unsigned getDotShStrTabSecNo() const { return Doc.Sections.size() + 3; } - unsigned getSectionCount() const { return Doc.Sections.size() + 4; } + // - symbol table (.symtab) (defaults to after last yaml section) + // - string table (.strtab) (defaults to after .symtab) + // - section header string table (.shstrtab) (defaults to after .strtab) + // - dynamic symbol table (.dynsym) (defaults to after .shstrtab) + // - dynamic string table (.dynstr) (defaults to after .dynsym) + unsigned getDotSymTabSecNo() const { return SN2I.get(".symtab"); } + unsigned getDotStrTabSecNo() const { return SN2I.get(".strtab"); } + unsigned getDotShStrTabSecNo() const { return SN2I.get(".shstrtab"); } + unsigned getDotDynSymSecNo() const { return SN2I.get(".dynsym"); } + unsigned getDotDynStrSecNo() const { return SN2I.get(".dynstr"); } + unsigned getSectionCount() const { return SN2I.size() + 1; } ELFState(const ELFYAML::Object &D) : Doc(D) {} @@ -173,16 +198,32 @@ void ELFState<ELFT>::initELFHeader(Elf_Ehdr &Header) { Header.e_machine = Doc.Header.Machine; Header.e_version = EV_CURRENT; Header.e_entry = Doc.Header.Entry; + Header.e_phoff = sizeof(Header); Header.e_flags = Doc.Header.Flags; Header.e_ehsize = sizeof(Elf_Ehdr); + Header.e_phentsize = sizeof(Elf_Phdr); + Header.e_phnum = Doc.ProgramHeaders.size(); Header.e_shentsize = sizeof(Elf_Shdr); - // Immediately following the ELF header. - Header.e_shoff = sizeof(Header); + // Immediately following the ELF header and program headers. + Header.e_shoff = + sizeof(Header) + sizeof(Elf_Phdr) * Doc.ProgramHeaders.size(); Header.e_shnum = getSectionCount(); Header.e_shstrndx = getDotShStrTabSecNo(); } template <class ELFT> +void ELFState<ELFT>::initProgramHeaders(std::vector<Elf_Phdr> &PHeaders) { + for (const auto &YamlPhdr : Doc.ProgramHeaders) { + Elf_Phdr Phdr; + Phdr.p_type = YamlPhdr.Type; + Phdr.p_flags = YamlPhdr.Flags; + Phdr.p_vaddr = YamlPhdr.VAddr; + Phdr.p_paddr = YamlPhdr.PAddr; + PHeaders.push_back(Phdr); + } +} + +template <class ELFT> bool ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders, ContiguousBlobAccumulator &CBA) { // Ensure SHN_UNDEF entry is present. An all-zero section header is a @@ -191,10 +232,6 @@ bool ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders, zero(SHeader); SHeaders.push_back(SHeader); - for (const auto &Sec : Doc.Sections) - DotShStrtab.add(Sec->Name); - DotShStrtab.finalize(); - for (const auto &Sec : Doc.Sections) { zero(SHeader); SHeader.sh_name = DotShStrtab.getOffset(Sec->Name); @@ -261,13 +298,17 @@ bool ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders, template <class ELFT> void ELFState<ELFT>::initSymtabSectionHeader(Elf_Shdr &SHeader, + SymtabType STType, ContiguousBlobAccumulator &CBA) { zero(SHeader); - SHeader.sh_name = DotShStrtab.getOffset(".symtab"); - SHeader.sh_type = ELF::SHT_SYMTAB; - SHeader.sh_link = getDotStrTabSecNo(); + bool IsStatic = STType == SymtabType::Static; + SHeader.sh_name = DotShStrtab.getOffset(IsStatic ? ".symtab" : ".dynsym"); + SHeader.sh_type = IsStatic ? ELF::SHT_SYMTAB : ELF::SHT_DYNSYM; + SHeader.sh_link = IsStatic ? getDotStrTabSecNo() : getDotDynStrSecNo(); + const auto &Symbols = IsStatic ? Doc.Symbols : Doc.DynamicSymbols; + auto &Strtab = IsStatic ? DotStrtab : DotDynstr; // One greater than symbol table index of the last local symbol. - SHeader.sh_info = Doc.Symbols.Local.size() + 1; + SHeader.sh_info = Symbols.Local.size() + 1; SHeader.sh_entsize = sizeof(Elf_Sym); SHeader.sh_addralign = 8; @@ -279,18 +320,18 @@ void ELFState<ELFT>::initSymtabSectionHeader(Elf_Shdr &SHeader, Syms.push_back(Sym); } - // Add symbol names to .strtab. - for (const auto &Sym : Doc.Symbols.Local) - DotStrtab.add(Sym.Name); - for (const auto &Sym : Doc.Symbols.Global) - DotStrtab.add(Sym.Name); - for (const auto &Sym : Doc.Symbols.Weak) - DotStrtab.add(Sym.Name); - DotStrtab.finalize(); + // Add symbol names to .strtab or .dynstr. + for (const auto &Sym : Symbols.Local) + Strtab.add(Sym.Name); + for (const auto &Sym : Symbols.Global) + Strtab.add(Sym.Name); + for (const auto &Sym : Symbols.Weak) + Strtab.add(Sym.Name); + Strtab.finalize(); - addSymbols(Doc.Symbols.Local, Syms, ELF::STB_LOCAL); - addSymbols(Doc.Symbols.Global, Syms, ELF::STB_GLOBAL); - addSymbols(Doc.Symbols.Weak, Syms, ELF::STB_WEAK); + addSymbols(Symbols.Local, Syms, ELF::STB_LOCAL, Strtab); + addSymbols(Symbols.Global, Syms, ELF::STB_GLOBAL, Strtab); + addSymbols(Symbols.Weak, Syms, ELF::STB_WEAK, Strtab); writeArrayData( CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign), @@ -311,14 +352,80 @@ void ELFState<ELFT>::initStrtabSectionHeader(Elf_Shdr &SHeader, StringRef Name, } template <class ELFT> +void ELFState<ELFT>::setProgramHeaderLayout(std::vector<Elf_Phdr> &PHeaders, + std::vector<Elf_Shdr> &SHeaders) { + uint32_t PhdrIdx = 0; + for (auto &YamlPhdr : Doc.ProgramHeaders) { + auto &PHeader = PHeaders[PhdrIdx++]; + + if (YamlPhdr.Sections.size()) + PHeader.p_offset = UINT32_MAX; + else + PHeader.p_offset = 0; + + // Find the minimum offset for the program header. + for (auto SecName : YamlPhdr.Sections) { + uint32_t Index = 0; + SN2I.lookup(SecName.Section, Index); + const auto &SHeader = SHeaders[Index]; + PHeader.p_offset = std::min(PHeader.p_offset, SHeader.sh_offset); + } + + // Find the maximum offset of the end of a section in order to set p_filesz. + PHeader.p_filesz = 0; + for (auto SecName : YamlPhdr.Sections) { + uint32_t Index = 0; + SN2I.lookup(SecName.Section, Index); + const auto &SHeader = SHeaders[Index]; + uint64_t EndOfSection; + if (SHeader.sh_type == llvm::ELF::SHT_NOBITS) + EndOfSection = SHeader.sh_offset; + else + EndOfSection = SHeader.sh_offset + SHeader.sh_size; + uint64_t EndOfSegment = PHeader.p_offset + PHeader.p_filesz; + EndOfSegment = std::max(EndOfSegment, EndOfSection); + PHeader.p_filesz = EndOfSegment - PHeader.p_offset; + } + + // Find the memory size by adding the size of sections at the end of the + // segment. These should be empty (size of zero) and NOBITS sections. + PHeader.p_memsz = PHeader.p_filesz; + for (auto SecName : YamlPhdr.Sections) { + uint32_t Index = 0; + SN2I.lookup(SecName.Section, Index); + const auto &SHeader = SHeaders[Index]; + if (SHeader.sh_offset == PHeader.p_offset + PHeader.p_filesz) + PHeader.p_memsz += SHeader.sh_size; + } + + // Set the alignment of the segment to be the same as the maximum alignment + // of the sections with the same offset so that by default the segment + // has a valid and sensible alignment. + if (YamlPhdr.Align) { + PHeader.p_align = *YamlPhdr.Align; + } else { + PHeader.p_align = 1; + for (auto SecName : YamlPhdr.Sections) { + uint32_t Index = 0; + SN2I.lookup(SecName.Section, Index); + const auto &SHeader = SHeaders[Index]; + if (SHeader.sh_offset == PHeader.p_offset) + PHeader.p_align = std::max(PHeader.p_align, SHeader.sh_addralign); + } + } + } +} + +template <class ELFT> void ELFState<ELFT>::addSymbols(const std::vector<ELFYAML::Symbol> &Symbols, std::vector<Elf_Sym> &Syms, - unsigned SymbolBinding) { + unsigned SymbolBinding, + const StringTableBuilder &Strtab) { for (const auto &Sym : Symbols) { Elf_Sym Symbol; zero(Symbol); if (!Sym.Name.empty()) - Symbol.st_name = DotStrtab.getOffset(Sym.Name); + Symbol.st_name = Strtab.getOffset(Sym.Name); Symbol.setBindingAndType(SymbolBinding, Sym.Type); if (!Sym.Section.empty()) { unsigned Index; @@ -328,7 +435,10 @@ void ELFState<ELFT>::addSymbols(const std::vector<ELFYAML::Symbol> &Symbols, exit(1); } Symbol.st_shndx = Index; - } // else Symbol.st_shndex == SHN_UNDEF (== 0), since it was zero'd earlier. + } else if (Sym.Index) { + Symbol.st_shndx = *Sym.Index; + } + // else Symbol.st_shndex == SHN_UNDEF (== 0), since it was zero'd earlier. Symbol.st_value = Sym.Value; Symbol.st_other = Sym.Other; Symbol.st_size = Sym.Size; @@ -378,7 +488,8 @@ ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader, // Some special relocation, R_ARM_v4BX for instance, does not have // an external reference. So it ignores the return value of lookup() // here. - SymN2I.lookup(Rel.Symbol, SymIdx); + if (Rel.Symbol) + SymN2I.lookup(*Rel.Symbol, SymIdx); if (IsRela) { Elf_Rela REntry; @@ -458,14 +569,9 @@ bool ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader, } template <class ELFT> bool ELFState<ELFT>::buildSectionIndex() { - SN2I.addName(".symtab", getDotSymTabSecNo()); - SN2I.addName(".strtab", getDotStrTabSecNo()); - SN2I.addName(".shstrtab", getDotShStrTabSecNo()); - for (unsigned i = 0, e = Doc.Sections.size(); i != e; ++i) { StringRef Name = Doc.Sections[i]->Name; - if (Name.empty()) - continue; + DotShStrtab.add(Name); // "+ 1" to take into account the SHT_NULL entry. if (SN2I.addName(Name, i + 1)) { errs() << "error: Repeated section name: '" << Name @@ -473,6 +579,17 @@ template <class ELFT> bool ELFState<ELFT>::buildSectionIndex() { return false; } } + + auto SecNo = 1 + Doc.Sections.size(); + // Add special sections after input sections, if necessary. + for (const auto &Name : implicitSectionNames()) + if (!SN2I.addName(Name, SecNo)) { + // Account for this section, since it wasn't in the Doc + ++SecNo; + DotShStrtab.add(Name); + } + + DotShStrtab.finalize(); return true; } @@ -508,47 +625,64 @@ int ELFState<ELFT>::writeELF(raw_ostream &OS, const ELFYAML::Object &Doc) { State.initELFHeader(Header); // TODO: Flesh out section header support. - // TODO: Program headers. + + std::vector<Elf_Phdr> PHeaders; + State.initProgramHeaders(PHeaders); // XXX: This offset is tightly coupled with the order that we write // things to `OS`. - const size_t SectionContentBeginOffset = - Header.e_ehsize + Header.e_shentsize * Header.e_shnum; + const size_t SectionContentBeginOffset = Header.e_ehsize + + Header.e_phentsize * Header.e_phnum + + Header.e_shentsize * Header.e_shnum; ContiguousBlobAccumulator CBA(SectionContentBeginOffset); - // Doc might not contain .symtab, .strtab and .shstrtab sections, - // but we will emit them, so make sure to add them to ShStrTabSHeader. - State.DotShStrtab.add(".symtab"); - State.DotShStrtab.add(".strtab"); - State.DotShStrtab.add(".shstrtab"); - std::vector<Elf_Shdr> SHeaders; if(!State.initSectionHeaders(SHeaders, CBA)) return 1; - // .symtab section. - Elf_Shdr SymtabSHeader; - State.initSymtabSectionHeader(SymtabSHeader, CBA); - SHeaders.push_back(SymtabSHeader); - - // .strtab string table header. - Elf_Shdr DotStrTabSHeader; - State.initStrtabSectionHeader(DotStrTabSHeader, ".strtab", State.DotStrtab, - CBA); - SHeaders.push_back(DotStrTabSHeader); + // Populate SHeaders with implicit sections not present in the Doc + for (const auto &Name : State.implicitSectionNames()) + if (State.SN2I.get(Name) >= SHeaders.size()) + SHeaders.push_back({}); + + // Initialize the implicit sections + auto Index = State.SN2I.get(".symtab"); + State.initSymtabSectionHeader(SHeaders[Index], SymtabType::Static, CBA); + Index = State.SN2I.get(".strtab"); + State.initStrtabSectionHeader(SHeaders[Index], ".strtab", State.DotStrtab, CBA); + Index = State.SN2I.get(".shstrtab"); + State.initStrtabSectionHeader(SHeaders[Index], ".shstrtab", State.DotShStrtab, CBA); + if (State.hasDynamicSymbols()) { + Index = State.SN2I.get(".dynsym"); + State.initSymtabSectionHeader(SHeaders[Index], SymtabType::Dynamic, CBA); + SHeaders[Index].sh_flags |= ELF::SHF_ALLOC; + Index = State.SN2I.get(".dynstr"); + State.initStrtabSectionHeader(SHeaders[Index], ".dynstr", State.DotDynstr, CBA); + SHeaders[Index].sh_flags |= ELF::SHF_ALLOC; + } - // .shstrtab string table header. - Elf_Shdr ShStrTabSHeader; - State.initStrtabSectionHeader(ShStrTabSHeader, ".shstrtab", State.DotShStrtab, - CBA); - SHeaders.push_back(ShStrTabSHeader); + // Now we can decide segment offsets + State.setProgramHeaderLayout(PHeaders, SHeaders); OS.write((const char *)&Header, sizeof(Header)); + writeArrayData(OS, makeArrayRef(PHeaders)); writeArrayData(OS, makeArrayRef(SHeaders)); CBA.writeBlobToStream(OS); return 0; } +template <class ELFT> bool ELFState<ELFT>::hasDynamicSymbols() const { + return Doc.DynamicSymbols.Global.size() > 0 || + Doc.DynamicSymbols.Weak.size() > 0 || + Doc.DynamicSymbols.Local.size() > 0; +} + +template <class ELFT> SmallVector<const char *, 5> ELFState<ELFT>::implicitSectionNames() const { + if (!hasDynamicSymbols()) + return {".symtab", ".strtab", ".shstrtab"}; + return {".symtab", ".strtab", ".shstrtab", ".dynsym", ".dynstr"}; +} + static bool is64Bit(const ELFYAML::Object &Doc) { return Doc.Header.Class == ELFYAML::ELF_ELFCLASS(ELF::ELFCLASS64); } diff --git a/tools/yaml2obj/yaml2obj.cpp b/tools/yaml2obj/yaml2obj.cpp index ead4b7a86b2e..3e2a5ca7ae0f 100644 --- a/tools/yaml2obj/yaml2obj.cpp +++ b/tools/yaml2obj/yaml2obj.cpp @@ -79,8 +79,8 @@ int main(int argc, char **argv) { OutputFilename = "-"; std::error_code EC; - std::unique_ptr<tool_output_file> Out( - new tool_output_file(OutputFilename, EC, sys::fs::F_None)); + std::unique_ptr<ToolOutputFile> Out( + new ToolOutputFile(OutputFilename, EC, sys::fs::F_None)); if (EC) error("yaml2obj: Error opening '" + OutputFilename + "': " + EC.message()); diff --git a/tools/yaml2obj/yaml2wasm.cpp b/tools/yaml2obj/yaml2wasm.cpp index 059ec5f9edcd..792f7c108bc1 100644 --- a/tools/yaml2obj/yaml2wasm.cpp +++ b/tools/yaml2obj/yaml2wasm.cpp @@ -12,8 +12,6 @@ /// //===----------------------------------------------------------------------===// // -#include "yaml2obj.h" -#include "llvm/Object/Wasm.h" #include "llvm/ObjectYAML/ObjectYAML.h" #include "llvm/Support/Endian.h" #include "llvm/Support/LEB128.h" @@ -140,11 +138,6 @@ int WasmWriter::writeSectionContent(raw_ostream &OS, WasmYAML::LinkingSection &S encodeULEB128(Section.DataSize, SubSection.GetStream()); SubSection.Done(); - // DATA_ALIGNMENT subsection - encodeULEB128(wasm::WASM_DATA_ALIGNMENT, OS); - encodeULEB128(Section.DataAlignment, SubSection.GetStream()); - SubSection.Done(); - // SYMBOL_INFO subsection if (Section.SymbolInfos.size()) { encodeULEB128(wasm::WASM_SYMBOL_INFO, OS); @@ -157,6 +150,29 @@ int WasmWriter::writeSectionContent(raw_ostream &OS, WasmYAML::LinkingSection &S SubSection.Done(); } + + // SEGMENT_NAMES subsection + if (Section.SegmentInfos.size()) { + encodeULEB128(wasm::WASM_SEGMENT_INFO, OS); + encodeULEB128(Section.SegmentInfos.size(), SubSection.GetStream()); + for (const WasmYAML::SegmentInfo &SegmentInfo : Section.SegmentInfos) { + writeStringRef(SegmentInfo.Name, SubSection.GetStream()); + encodeULEB128(SegmentInfo.Alignment, SubSection.GetStream()); + encodeULEB128(SegmentInfo.Flags, SubSection.GetStream()); + } + SubSection.Done(); + } + + // INIT_FUNCS subsection + if (Section.InitFunctions.size()) { + encodeULEB128(wasm::WASM_INIT_FUNCS, OS); + encodeULEB128(Section.InitFunctions.size(), SubSection.GetStream()); + for (const WasmYAML::InitFunction &Func : Section.InitFunctions) { + encodeULEB128(Func.Priority, SubSection.GetStream()); + encodeULEB128(Func.FunctionIndex, SubSection.GetStream()); + } + SubSection.Done(); + } return 0; } @@ -370,9 +386,9 @@ int WasmWriter::writeRelocSection(raw_ostream &OS, encodeULEB128(Reloc.Offset, OS); encodeULEB128(Reloc.Index, OS); switch (Reloc.Type) { - case wasm::R_WEBASSEMBLY_GLOBAL_ADDR_LEB: - case wasm::R_WEBASSEMBLY_GLOBAL_ADDR_SLEB: - case wasm::R_WEBASSEMBLY_GLOBAL_ADDR_I32: + case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB: + case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB: + case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32: encodeULEB128(Reloc.Addend, OS); } } |