diff options
Diffstat (limited to 'lld')
93 files changed, 2656 insertions, 1766 deletions
diff --git a/lld/COFF/COFFLinkerContext.h b/lld/COFF/COFFLinkerContext.h index e5223da86ef8..a3a6f94a9413 100644 --- a/lld/COFF/COFFLinkerContext.h +++ b/lld/COFF/COFFLinkerContext.h @@ -15,12 +15,13 @@ #include "InputFiles.h" #include "SymbolTable.h" #include "Writer.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Timer.h" namespace lld { namespace coff { -class COFFLinkerContext { +class COFFLinkerContext : public CommonLinkerContext { public: COFFLinkerContext(); COFFLinkerContext(const COFFLinkerContext &) = delete; diff --git a/lld/COFF/Chunks.cpp b/lld/COFF/Chunks.cpp index 0bff11f450d1..6cabb22d98cf 100644 --- a/lld/COFF/Chunks.cpp +++ b/lld/COFF/Chunks.cpp @@ -12,7 +12,6 @@ #include "SymbolTable.h" #include "Symbols.h" #include "Writer.h" -#include "lld/Common/ErrorHandler.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/Object/COFF.h" @@ -430,7 +429,7 @@ void SectionChunk::sortRelocations() { return; warn("some relocations in " + file->getName() + " are not sorted"); MutableArrayRef<coff_relocation> newRelocs( - bAlloc.Allocate<coff_relocation>(relocsSize), relocsSize); + bAlloc().Allocate<coff_relocation>(relocsSize), relocsSize); memcpy(newRelocs.data(), relocsData, relocsSize * sizeof(coff_relocation)); llvm::sort(newRelocs, cmpByVa); setRelocs(newRelocs); @@ -635,7 +634,7 @@ void SectionChunk::printDiscardedMessage() const { // Removed by dead-stripping. If it's removed by ICF, ICF already // printed out the name, so don't repeat that here. if (sym && this == repl) - message("Discarded " + sym->getName()); + log("Discarded " + sym->getName()); } StringRef SectionChunk::getDebugName() const { diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h index 3917975e165d..4bb42c88aa93 100644 --- a/lld/COFF/Config.h +++ b/lld/COFF/Config.h @@ -43,6 +43,7 @@ static const auto I386 = llvm::COFF::IMAGE_FILE_MACHINE_I386; struct Export { StringRef name; // N in /export:N or /export:E=N StringRef extName; // E in /export:E=N + StringRef aliasTarget; // GNU specific: N in "alias == N" Symbol *sym = nullptr; uint16_t ordinal = 0; bool noname = false; @@ -63,6 +64,7 @@ struct Export { bool operator==(const Export &e) { return (name == e.name && extName == e.extName && + aliasTarget == e.aliasTarget && ordinal == e.ordinal && noname == e.noname && data == e.data && isPrivate == e.isPrivate); } @@ -282,7 +284,7 @@ struct Configuration { bool stdcallFixup = false; }; -extern Configuration *config; +extern std::unique_ptr<Configuration> config; } // namespace coff } // namespace lld diff --git a/lld/COFF/DLL.cpp b/lld/COFF/DLL.cpp index 6fec9df5617d..bfa2a6910e2b 100644 --- a/lld/COFF/DLL.cpp +++ b/lld/COFF/DLL.cpp @@ -659,14 +659,14 @@ void DelayLoadContents::create(COFFLinkerContext &ctx, Defined *h) { // Add a syntentic symbol for this load thunk, using the "__imp_load" // prefix, in case this thunk needs to be added to the list of valid // call targets for Control Flow Guard. - StringRef symName = saver.save("__imp_load_" + extName); + StringRef symName = saver().save("__imp_load_" + extName); s->loadThunkSym = cast<DefinedSynthetic>(ctx.symtab.addSynthetic(symName, t)); } } thunks.push_back(tm); StringRef tmName = - saver.save("__tailMerge_" + syms[0]->getDLLName().lower()); + saver().save("__tailMerge_" + syms[0]->getDLLName().lower()); ctx.symtab.addSynthetic(tmName, tm); // Terminate with null values. addresses.push_back(make<NullChunk>(8)); diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp index 07b60673577e..1546291e16c6 100644 --- a/lld/COFF/Driver.cpp +++ b/lld/COFF/Driver.cpp @@ -19,9 +19,7 @@ #include "Writer.h" #include "lld/Common/Args.h" #include "lld/Common/Driver.h" -#include "lld/Common/ErrorHandler.h" #include "lld/Common/Filesystem.h" -#include "lld/Common/Memory.h" #include "lld/Common/Timer.h" #include "lld/Common/Version.h" #include "llvm/ADT/Optional.h" @@ -60,39 +58,25 @@ using namespace llvm::sys; namespace lld { namespace coff { -Configuration *config; -LinkerDriver *driver; +std::unique_ptr<Configuration> config; +std::unique_ptr<LinkerDriver> driver; -bool link(ArrayRef<const char *> args, bool canExitEarly, raw_ostream &stdoutOS, - raw_ostream &stderrOS) { - lld::stdoutOS = &stdoutOS; - lld::stderrOS = &stderrOS; +bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput) { + // This driver-specific context will be freed later by lldMain(). + auto *ctx = new COFFLinkerContext; - errorHandler().cleanupCallback = []() { - freeArena(); - }; - - errorHandler().logName = args::getFilenameWithoutExe(args[0]); - errorHandler().errorLimitExceededMsg = - "too many errors emitted, stopping now" - " (use /errorlimit:0 to see all errors)"; - errorHandler().exitEarly = canExitEarly; - stderrOS.enable_colors(stderrOS.has_colors()); + ctx->e.initialize(stdoutOS, stderrOS, exitEarly, disableOutput); + ctx->e.logName = args::getFilenameWithoutExe(args[0]); + ctx->e.errorLimitExceededMsg = "too many errors emitted, stopping now" + " (use /errorlimit:0 to see all errors)"; - COFFLinkerContext ctx; - config = make<Configuration>(); - driver = make<LinkerDriver>(ctx); + config = std::make_unique<Configuration>(); + driver = std::make_unique<LinkerDriver>(*ctx); driver->linkerMain(args); - // Call exit() if we can to avoid calling destructors. - if (canExitEarly) - exitLld(errorCount() ? 1 : 0); - - bool ret = errorCount() == 0; - if (!canExitEarly) - errorHandler().reset(); - return ret; + return errorCount() == 0; } // Parse options of the form "old;new". @@ -162,7 +146,7 @@ static std::future<MBErrPair> createFutureForFile(std::string path) { static StringRef mangle(StringRef sym) { assert(config->machine != IMAGE_FILE_MACHINE_UNKNOWN); if (config->machine == I386) - return saver.save("_" + sym); + return saver().save("_" + sym); return sym; } @@ -208,17 +192,11 @@ void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> mb, ctx.symtab.addFile(make<ArchiveFile>(ctx, mbref)); break; case file_magic::bitcode: - if (lazy) - ctx.symtab.addFile(make<LazyObjFile>(ctx, mbref)); - else - ctx.symtab.addFile(make<BitcodeFile>(ctx, mbref, "", 0)); + ctx.symtab.addFile(make<BitcodeFile>(ctx, mbref, "", 0, lazy)); break; case file_magic::coff_object: case file_magic::coff_import_library: - if (lazy) - ctx.symtab.addFile(make<LazyObjFile>(ctx, mbref)); - else - ctx.symtab.addFile(make<ObjFile>(ctx, mbref)); + ctx.symtab.addFile(make<ObjFile>(ctx, mbref, lazy)); break; case file_magic::pdb: ctx.symtab.addFile(make<PDBInputFile>(ctx, mbref)); @@ -282,7 +260,8 @@ void LinkerDriver::addArchiveBuffer(MemoryBufferRef mb, StringRef symName, if (magic == file_magic::coff_object) { obj = make<ObjFile>(ctx, mb); } else if (magic == file_magic::bitcode) { - obj = make<BitcodeFile>(ctx, mb, parentName, offsetInArchive); + obj = + make<BitcodeFile>(ctx, mb, parentName, offsetInArchive, /*lazy=*/false); } else { error("unknown file type: " + mb.getBufferIdentifier()); return; @@ -363,9 +342,9 @@ void LinkerDriver::parseDirectives(InputFile *file) { Export exp = parseExport(e); if (config->machine == I386 && config->mingw) { if (!isDecorated(exp.name)) - exp.name = saver.save("_" + exp.name); + exp.name = saver().save("_" + exp.name); if (!exp.extName.empty() && !isDecorated(exp.extName)) - exp.extName = saver.save("_" + exp.extName); + exp.extName = saver().save("_" + exp.extName); } exp.directives = true; config->exports.push_back(exp); @@ -447,11 +426,11 @@ StringRef LinkerDriver::doFindFile(StringRef filename) { SmallString<128> path = dir; sys::path::append(path, filename); if (sys::fs::exists(path.str())) - return saver.save(path.str()); + return saver().save(path.str()); if (!hasExt) { path.append(".obj"); if (sys::fs::exists(path.str())) - return saver.save(path.str()); + return saver().save(path.str()); } } return filename; @@ -488,7 +467,7 @@ StringRef LinkerDriver::doFindLibMinGW(StringRef filename) { SmallString<128> s = filename; sys::path::replace_extension(s, ".a"); - StringRef libName = saver.save("lib" + s.str()); + StringRef libName = saver().save("lib" + s.str()); return doFindFile(libName); } @@ -497,7 +476,7 @@ StringRef LinkerDriver::doFindLib(StringRef filename) { // Add ".lib" to Filename if that has no file extension. bool hasExt = filename.contains('.'); if (!hasExt) - filename = saver.save(filename + ".lib"); + filename = saver().save(filename + ".lib"); StringRef ret = doFindFile(filename); // For MinGW, if the find above didn't turn up anything, try // looking for a MinGW formatted library name. @@ -530,7 +509,7 @@ void LinkerDriver::addLibSearchPaths() { Optional<std::string> envOpt = Process::GetEnv("LIB"); if (!envOpt.hasValue()) return; - StringRef env = saver.save(*envOpt); + StringRef env = saver().save(*envOpt); while (!env.empty()) { StringRef path; std::tie(path, env) = env.split(';'); @@ -815,6 +794,7 @@ static void createImportLibrary(bool asLib) { e2.Name = std::string(e1.name); e2.SymbolName = std::string(e1.symbolName); e2.ExtName = std::string(e1.extName); + e2.AliasTarget = std::string(e1.aliasTarget); e2.Ordinal = e1.ordinal; e2.Noname = e1.noname; e2.Data = e1.data; @@ -877,8 +857,8 @@ static void parseModuleDefs(StringRef path) { driver->takeBuffer(std::move(mb)); if (config->outputFile.empty()) - config->outputFile = std::string(saver.save(m.OutputFile)); - config->importName = std::string(saver.save(m.ImportName)); + config->outputFile = std::string(saver().save(m.OutputFile)); + config->importName = std::string(saver().save(m.ImportName)); if (m.ImageBase) config->imageBase = m.ImageBase; if (m.StackReserve) @@ -906,13 +886,14 @@ static void parseModuleDefs(StringRef path) { // DLL instead. This is supported by both MS and GNU linkers. if (!e1.ExtName.empty() && e1.ExtName != e1.Name && StringRef(e1.Name).contains('.')) { - e2.name = saver.save(e1.ExtName); - e2.forwardTo = saver.save(e1.Name); + e2.name = saver().save(e1.ExtName); + e2.forwardTo = saver().save(e1.Name); config->exports.push_back(e2); continue; } - e2.name = saver.save(e1.Name); - e2.extName = saver.save(e1.ExtName); + e2.name = saver().save(e1.Name); + e2.extName = saver().save(e1.ExtName); + e2.aliasTarget = saver().save(e1.AliasTarget); e2.ordinal = e1.Ordinal; e2.noname = e1.Noname; e2.data = e1.Data; @@ -1909,9 +1890,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) { Export e = parseExport(arg->getValue()); if (config->machine == I386) { if (!isDecorated(e.name)) - e.name = saver.save("_" + e.name); + e.name = saver().save("_" + e.name); if (!e.extName.empty() && !isDecorated(e.extName)) - e.extName = saver.save("_" + e.extName); + e.extName = saver().save("_" + e.extName); } config->exports.push_back(e); } diff --git a/lld/COFF/Driver.h b/lld/COFF/Driver.h index 77e67b282665..518ec1470677 100644 --- a/lld/COFF/Driver.h +++ b/lld/COFF/Driver.h @@ -30,8 +30,7 @@ namespace lld { namespace coff { -class LinkerDriver; -extern LinkerDriver *driver; +extern std::unique_ptr<class LinkerDriver> driver; using llvm::COFF::MachineTypes; using llvm::COFF::WindowsSubsystem; diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp index 0921c8e27f5a..ac0f1f972c79 100644 --- a/lld/COFF/DriverUtils.cpp +++ b/lld/COFF/DriverUtils.cpp @@ -48,17 +48,17 @@ const uint16_t RT_MANIFEST = 24; class Executor { public: - explicit Executor(StringRef s) : prog(saver.save(s)) {} - void add(StringRef s) { args.push_back(saver.save(s)); } - void add(std::string &s) { args.push_back(saver.save(s)); } - void add(Twine s) { args.push_back(saver.save(s)); } - void add(const char *s) { args.push_back(saver.save(s)); } + explicit Executor(StringRef s) : prog(saver().save(s)) {} + void add(StringRef s) { args.push_back(saver().save(s)); } + void add(std::string &s) { args.push_back(saver().save(s)); } + void add(Twine s) { args.push_back(saver().save(s)); } + void add(const char *s) { args.push_back(saver().save(s)); } void run() { ErrorOr<std::string> exeOrErr = sys::findProgramByName(prog); if (auto ec = exeOrErr.getError()) fatal("unable to find " + prog + " in PATH: " + ec.message()); - StringRef exe = saver.save(*exeOrErr); + StringRef exe = saver().save(*exeOrErr); args.insert(args.begin(), exe); if (sys::ExecuteAndWait(args[0], args) != 0) @@ -636,14 +636,14 @@ static StringRef killAt(StringRef sym, bool prefix) { sym = sym.substr(0, sym.find('@', 1)); if (!sym.startswith("@")) { if (prefix && !sym.startswith("_")) - return saver.save("_" + sym); + return saver().save("_" + sym); return sym; } // For fastcall, remove the leading @ and replace it with an // underscore, if prefixes are used. sym = sym.substr(1); if (prefix) - sym = saver.save("_" + sym); + sym = saver().save("_" + sym); return sym; } @@ -854,7 +854,7 @@ opt::InputArgList ArgParser::parse(ArrayRef<const char *> argv) { argv.data() + argv.size()); if (!args.hasArg(OPT_lldignoreenv)) addLINK(expandedArgv); - cl::ExpandResponseFiles(saver, getQuotingStyle(args), expandedArgv); + cl::ExpandResponseFiles(saver(), getQuotingStyle(args), expandedArgv); args = optTable.ParseArgs(makeArrayRef(expandedArgv).drop_front(), missingIndex, missingCount); @@ -901,7 +901,7 @@ ParsedDirectives ArgParser::parseDirectives(StringRef s) { // Handle /EXPORT and /INCLUDE in a fast path. These directives can appear for // potentially every symbol in the object, so they must be handled quickly. SmallVector<StringRef, 16> tokens; - cl::TokenizeWindowsCommandLineNoCopy(s, saver, tokens); + cl::TokenizeWindowsCommandLineNoCopy(s, saver(), tokens); for (StringRef tok : tokens) { if (tok.startswith_insensitive("/export:") || tok.startswith_insensitive("-export:")) @@ -914,7 +914,7 @@ ParsedDirectives ArgParser::parseDirectives(StringRef s) { // already copied quoted arguments for us, so those do not need to be // copied again. bool HasNul = tok.end() != s.end() && tok.data()[tok.size()] == '\0'; - rest.push_back(HasNul ? tok.data() : saver.save(tok).data()); + rest.push_back(HasNul ? tok.data() : saver().save(tok).data()); } } @@ -948,7 +948,7 @@ void ArgParser::addLINK(SmallVector<const char *, 256> &argv) { std::vector<const char *> ArgParser::tokenize(StringRef s) { SmallVector<const char *, 16> tokens; - cl::TokenizeWindowsCommandLine(s, saver, tokens); + cl::TokenizeWindowsCommandLine(s, saver(), tokens); return std::vector<const char *>(tokens.begin(), tokens.end()); } diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp index 4b38e3d1a99b..0f3f5e0ffe7c 100644 --- a/lld/COFF/InputFiles.cpp +++ b/lld/COFF/InputFiles.cpp @@ -15,8 +15,6 @@ #include "SymbolTable.h" #include "Symbols.h" #include "lld/Common/DWARF.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" #include "llvm-c/lto.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Triple.h" @@ -135,31 +133,7 @@ std::vector<MemoryBufferRef> lld::coff::getArchiveMembers(Archive *file) { return v; } -void LazyObjFile::fetch() { - if (mb.getBuffer().empty()) - return; - - InputFile *file; - if (isBitcode(mb)) - file = make<BitcodeFile>(ctx, mb, "", 0, std::move(symbols)); - else - file = make<ObjFile>(ctx, mb, std::move(symbols)); - mb = {}; - ctx.symtab.addFile(file); -} - -void LazyObjFile::parse() { - if (isBitcode(this->mb)) { - // Bitcode file. - std::unique_ptr<lto::InputFile> obj = - CHECK(lto::InputFile::create(this->mb), this); - for (const lto::InputFile::Symbol &sym : obj->symbols()) { - if (!sym.isUndefined()) - ctx.symtab.addLazyObject(this, sym.getName()); - } - return; - } - +void ObjFile::parseLazy() { // Native object file. std::unique_ptr<Binary> coffObjPtr = CHECK(createBinary(mb), this); COFFObjectFile *coffObj = cast<COFFObjectFile>(coffObjPtr.get()); @@ -929,7 +903,7 @@ ObjFile::getVariableLocation(StringRef var) { Optional<std::pair<std::string, unsigned>> ret = dwarf->getVariableLoc(var); if (!ret) return None; - return std::make_pair(saver.save(ret->first), ret->second); + return std::make_pair(saver().save(ret->first), ret->second); } // Used only for DWARF debug info, which is not common (except in MinGW @@ -964,8 +938,8 @@ void ImportFile::parse() { fatal("broken import library"); // Read names and create an __imp_ symbol. - StringRef name = saver.save(StringRef(buf + sizeof(*hdr))); - StringRef impName = saver.save("__imp_" + name); + StringRef name = saver().save(StringRef(buf + sizeof(*hdr))); + StringRef impName = saver().save("__imp_" + name); const char *nameStart = buf + sizeof(coff_import_header) + name.size() + 1; dllName = std::string(StringRef(nameStart)); StringRef extName; @@ -1006,13 +980,9 @@ void ImportFile::parse() { } BitcodeFile::BitcodeFile(COFFLinkerContext &ctx, MemoryBufferRef mb, - StringRef archiveName, uint64_t offsetInArchive) - : BitcodeFile(ctx, mb, archiveName, offsetInArchive, {}) {} - -BitcodeFile::BitcodeFile(COFFLinkerContext &ctx, MemoryBufferRef mb, StringRef archiveName, uint64_t offsetInArchive, - std::vector<Symbol *> &&symbols) - : InputFile(ctx, BitcodeKind, mb), symbols(std::move(symbols)) { + bool lazy) + : InputFile(ctx, BitcodeKind, mb, lazy) { std::string path = mb.getBufferIdentifier().str(); if (config->thinLTOIndexOnly) path = replaceThinLTOSuffix(mb.getBufferIdentifier()); @@ -1023,11 +993,12 @@ BitcodeFile::BitcodeFile(COFFLinkerContext &ctx, MemoryBufferRef mb, // into consideration at LTO time (which very likely causes undefined // symbols later in the link stage). So we append file offset to make // filename unique. - MemoryBufferRef mbref( - mb.getBuffer(), - saver.save(archiveName.empty() ? path - : archiveName + sys::path::filename(path) + - utostr(offsetInArchive))); + MemoryBufferRef mbref(mb.getBuffer(), + saver().save(archiveName.empty() + ? path + : archiveName + + sys::path::filename(path) + + utostr(offsetInArchive))); obj = check(lto::InputFile::create(mbref)); } @@ -1063,6 +1034,7 @@ FakeSectionChunk ltoDataSectionChunk(<oDataSection.section); } // namespace void BitcodeFile::parse() { + llvm::StringSaver &saver = lld::saver(); std::vector<std::pair<Symbol *, bool>> comdat(obj->getComdatTable().size()); for (size_t i = 0; i != obj->getComdatTable().size(); ++i) // FIXME: Check nodeduplicate @@ -1107,6 +1079,12 @@ void BitcodeFile::parse() { directives = obj->getCOFFLinkerOpts(); } +void BitcodeFile::parseLazy() { + for (const lto::InputFile::Symbol &sym : obj->symbols()) + if (!sym.isUndefined()) + ctx.symtab.addLazyObject(this, sym.getName()); +} + MachineTypes BitcodeFile::getMachineType() { switch (Triple(obj->getTargetTriple()).getArch()) { case Triple::x86_64: @@ -1178,11 +1156,11 @@ void DLLFile::parse() { s->nameType = ImportNameType::IMPORT_NAME; if (coffObj->getMachine() == I386) { - s->symbolName = symbolName = saver.save("_" + symbolName); + s->symbolName = symbolName = saver().save("_" + symbolName); s->nameType = ImportNameType::IMPORT_NAME_NOPREFIX; } - StringRef impName = saver.save("__imp_" + symbolName); + StringRef impName = saver().save("__imp_" + symbolName); ctx.symtab.addLazyDLLSymbol(this, s, impName); if (code) ctx.symtab.addLazyDLLSymbol(this, s, symbolName); @@ -1201,7 +1179,7 @@ void DLLFile::makeImport(DLLFile::Symbol *s) { size_t impSize = s->dllName.size() + s->symbolName.size() + 2; // +2 for NULs size_t size = sizeof(coff_import_header) + impSize; - char *buf = bAlloc.Allocate<char>(size); + char *buf = bAlloc().Allocate<char>(size); memset(buf, 0, size); char *p = buf; auto *imp = reinterpret_cast<coff_import_header *>(p); diff --git a/lld/COFF/InputFiles.h b/lld/COFF/InputFiles.h index 801c668d3ae4..2cabb54cb386 100644 --- a/lld/COFF/InputFiles.h +++ b/lld/COFF/InputFiles.h @@ -95,13 +95,17 @@ public: COFFLinkerContext &ctx; protected: - InputFile(COFFLinkerContext &c, Kind k, MemoryBufferRef m) - : mb(m), ctx(c), fileKind(k) {} + InputFile(COFFLinkerContext &c, Kind k, MemoryBufferRef m, bool lazy = false) + : mb(m), ctx(c), fileKind(k), lazy(lazy) {} StringRef directives; private: const Kind fileKind; + +public: + // True if this is a lazy ObjFile or BitcodeFile. + bool lazy = false; }; // .lib or .a file. @@ -121,33 +125,14 @@ private: llvm::DenseSet<uint64_t> seen; }; -// .obj or .o file between -start-lib and -end-lib. -class LazyObjFile : public InputFile { -public: - explicit LazyObjFile(COFFLinkerContext &ctx, MemoryBufferRef m) - : InputFile(ctx, LazyObjectKind, m) {} - static bool classof(const InputFile *f) { - return f->kind() == LazyObjectKind; - } - // Makes this object file part of the link. - void fetch(); - // Adds the symbols in this file to the symbol table as LazyObject symbols. - void parse() override; - -private: - std::vector<Symbol *> symbols; -}; - // .obj or .o file. This may be a member of an archive file. class ObjFile : public InputFile { public: - explicit ObjFile(COFFLinkerContext &ctx, MemoryBufferRef m) - : InputFile(ctx, ObjectKind, m) {} - explicit ObjFile(COFFLinkerContext &ctx, MemoryBufferRef m, - std::vector<Symbol *> &&symbols) - : InputFile(ctx, ObjectKind, m), symbols(std::move(symbols)) {} + explicit ObjFile(COFFLinkerContext &ctx, MemoryBufferRef m, bool lazy = false) + : InputFile(ctx, ObjectKind, m, lazy) {} static bool classof(const InputFile *f) { return f->kind() == ObjectKind; } void parse() override; + void parseLazy(); MachineTypes getMachineType() override; ArrayRef<Chunk *> getChunks() { return chunks; } ArrayRef<SectionChunk *> getDebugChunks() { return debugChunks; } @@ -380,15 +365,14 @@ public: // Used for LTO. class BitcodeFile : public InputFile { public: - BitcodeFile(COFFLinkerContext &ctx, MemoryBufferRef mb, StringRef archiveName, - uint64_t offsetInArchive); - explicit BitcodeFile(COFFLinkerContext &ctx, MemoryBufferRef m, + explicit BitcodeFile(COFFLinkerContext &ctx, MemoryBufferRef mb, StringRef archiveName, uint64_t offsetInArchive, - std::vector<Symbol *> &&symbols); + bool lazy); ~BitcodeFile(); static bool classof(const InputFile *f) { return f->kind() == BitcodeKind; } ArrayRef<Symbol *> getSymbols() { return symbols; } MachineTypes getMachineType() override; + void parseLazy(); std::unique_ptr<llvm::lto::InputFile> obj; private: diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index f117b62192c8..2dbe7b146402 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -11,7 +11,7 @@ #include "InputFiles.h" #include "Symbols.h" #include "lld/Common/Args.h" -#include "lld/Common/ErrorHandler.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" #include "llvm/ADT/STLExtras.h" @@ -209,8 +209,8 @@ std::vector<InputFile *> BitcodeCompiler::compile(COFFLinkerContext &ctx) { // - foo.exe.lto.1.obj // - ... StringRef ltoObjName = - saver.save(Twine(config->outputFile) + ".lto" + - (i == 0 ? Twine("") : Twine('.') + Twine(i)) + ".obj"); + saver().save(Twine(config->outputFile) + ".lto" + + (i == 0 ? Twine("") : Twine('.') + Twine(i)) + ".obj"); // Get the native object contents either from the cache or from memory. Do // not use the cached MemoryBuffer directly, or the PDB will not be diff --git a/lld/COFF/MinGW.cpp b/lld/COFF/MinGW.cpp index 148ebe5eea66..7a3a3853572f 100644 --- a/lld/COFF/MinGW.cpp +++ b/lld/COFF/MinGW.cpp @@ -11,7 +11,6 @@ #include "Driver.h" #include "InputFiles.h" #include "SymbolTable.h" -#include "lld/Common/ErrorHandler.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/Object/COFF.h" @@ -184,8 +183,8 @@ void lld::coff::writeDefFile(StringRef name) { static StringRef mangle(Twine sym) { assert(config->machine != IMAGE_FILE_MACHINE_UNKNOWN); if (config->machine == I386) - return saver.save("_" + sym); - return saver.save(sym); + return saver().save("_" + sym); + return saver().save(sym); } // Handles -wrap option. @@ -249,7 +248,7 @@ void lld::coff::wrapSymbols(COFFLinkerContext &ctx, // referenced it or not, though.) if (imp) { DefinedLocalImport *wrapimp = make<DefinedLocalImport>( - saver.save("__imp_" + w.wrap->getName()), d); + saver().save("__imp_" + w.wrap->getName()), d); ctx.symtab.localImportChunks.push_back(wrapimp->getChunk()); map[imp] = wrapimp; } diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp index a4cef1d0df3b..dea84eca5b12 100644 --- a/lld/COFF/PDB.cpp +++ b/lld/COFF/PDB.cpp @@ -16,7 +16,6 @@ #include "Symbols.h" #include "TypeMerger.h" #include "Writer.h" -#include "lld/Common/ErrorHandler.h" #include "lld/Common/Timer.h" #include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h" #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" @@ -75,7 +74,7 @@ class PDBLinker { public: PDBLinker(COFFLinkerContext &ctx) - : builder(bAlloc), tMerger(ctx, bAlloc), ctx(ctx) { + : builder(bAlloc()), tMerger(ctx, bAlloc()), ctx(ctx) { // This isn't strictly necessary, but link.exe usually puts an empty string // as the first "valid" string in the string table, so we do the same in // order to maintain as much byte-for-byte compatibility as possible. @@ -501,7 +500,7 @@ static void addGlobalSymbol(pdb::GSIStreamBuilder &builder, uint16_t modIndex, case SymbolKind::S_LPROCREF: { // sym is a temporary object, so we have to copy and reallocate the record // to stabilize it. - uint8_t *mem = bAlloc.Allocate<uint8_t>(sym.length()); + uint8_t *mem = bAlloc().Allocate<uint8_t>(sym.length()); memcpy(mem, sym.data().data(), sym.length()); builder.addGlobalSymbol(CVSymbol(makeArrayRef(mem, sym.length()))); break; @@ -1003,7 +1002,7 @@ static void warnUnusable(InputFile *f, Error e) { // Allocate memory for a .debug$S / .debug$F section and relocate it. static ArrayRef<uint8_t> relocateDebugChunk(SectionChunk &debugChunk) { - uint8_t *buffer = bAlloc.Allocate<uint8_t>(debugChunk.getSize()); + uint8_t *buffer = bAlloc().Allocate<uint8_t>(debugChunk.getSize()); assert(debugChunk.getOutputSectionIdx() == 0 && "debug sections should not be in output sections"); debugChunk.writeTo(buffer); @@ -1417,6 +1416,7 @@ static void addCommonLinkerModuleSymbols(StringRef path, ebs.Fields.push_back(path); ebs.Fields.push_back("cmd"); ebs.Fields.push_back(argStr); + llvm::BumpPtrAllocator &bAlloc = lld::bAlloc(); mod.addSymbol(codeview::SymbolSerializer::writeOneSymbol( ons, bAlloc, CodeViewContainer::Pdb)); mod.addSymbol(codeview::SymbolSerializer::writeOneSymbol( @@ -1448,7 +1448,7 @@ static void addLinkerModuleCoffGroup(PartialSection *sec, cgs.Characteristics |= llvm::COFF::IMAGE_SCN_MEM_WRITE; mod.addSymbol(codeview::SymbolSerializer::writeOneSymbol( - cgs, bAlloc, CodeViewContainer::Pdb)); + cgs, bAlloc(), CodeViewContainer::Pdb)); } static void addLinkerModuleSectionSymbol(pdb::DbiModuleDescriptorBuilder &mod, @@ -1461,7 +1461,7 @@ static void addLinkerModuleSectionSymbol(pdb::DbiModuleDescriptorBuilder &mod, sym.Rva = os.getRVA(); sym.SectionNumber = os.sectionIndex; mod.addSymbol(codeview::SymbolSerializer::writeOneSymbol( - sym, bAlloc, CodeViewContainer::Pdb)); + sym, bAlloc(), CodeViewContainer::Pdb)); // Skip COFF groups in MinGW because it adds a significant footprint to the // PDB, due to each function being in its own section @@ -1536,6 +1536,7 @@ void PDBLinker::addImportFilesToPDB() { ts.Segment = thunkOS->sectionIndex; ts.Offset = thunkChunk->getRVA() - thunkOS->getRVA(); + llvm::BumpPtrAllocator &bAlloc = lld::bAlloc(); mod->addSymbol(codeview::SymbolSerializer::writeOneSymbol( ons, bAlloc, CodeViewContainer::Pdb)); mod->addSymbol(codeview::SymbolSerializer::writeOneSymbol( diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp index 679c91ad06e6..db2db9c9272e 100644 --- a/lld/COFF/SymbolTable.cpp +++ b/lld/COFF/SymbolTable.cpp @@ -37,7 +37,21 @@ StringRef ltrim1(StringRef s, const char *chars) { void SymbolTable::addFile(InputFile *file) { log("Reading " + toString(file)); - file->parse(); + if (file->lazy) { + if (auto *f = dyn_cast<BitcodeFile>(file)) + f->parseLazy(); + else + cast<ObjFile>(file)->parseLazy(); + } else { + file->parse(); + if (auto *f = dyn_cast<ObjFile>(file)) { + ctx.objFileInstances.push_back(f); + } else if (auto *f = dyn_cast<BitcodeFile>(file)) { + ctx.bitcodeFileInstances.push_back(f); + } else if (auto *f = dyn_cast<ImportFile>(file)) { + ctx.importFileInstances.push_back(f); + } + } MachineTypes mt = file->getMachineType(); if (config->machine == IMAGE_FILE_MACHINE_UNKNOWN) { @@ -48,14 +62,6 @@ void SymbolTable::addFile(InputFile *file) { return; } - if (auto *f = dyn_cast<ObjFile>(file)) { - ctx.objFileInstances.push_back(f); - } else if (auto *f = dyn_cast<BitcodeFile>(file)) { - ctx.bitcodeFileInstances.push_back(f); - } else if (auto *f = dyn_cast<ImportFile>(file)) { - ctx.importFileInstances.push_back(f); - } - driver->parseDirectives(file); } @@ -75,9 +81,11 @@ static void forceLazy(Symbol *s) { l->file->addMember(l->sym); break; } - case Symbol::Kind::LazyObjectKind: - cast<LazyObject>(s)->file->fetch(); + case Symbol::Kind::LazyObjectKind: { + InputFile *file = cast<LazyObject>(s)->file; + file->ctx.symtab.addFile(file); break; + } case Symbol::Kind::LazyDLLSymbolKind: { auto *l = cast<LazyDLLSymbol>(s); l->file->makeImport(l->sym); @@ -126,7 +134,7 @@ getFileLineDwarf(const SectionChunk *c, uint32_t addr) { const DILineInfo &lineInfo = *optionalLineInfo; if (lineInfo.FileName == DILineInfo::BadString) return None; - return std::make_pair(saver.save(lineInfo.FileName), lineInfo.Line); + return std::make_pair(saver().save(lineInfo.FileName), lineInfo.Line); } static Optional<std::pair<StringRef, uint32_t>> @@ -562,7 +570,8 @@ void SymbolTable::addLazyArchive(ArchiveFile *f, const Archive::Symbol &sym) { f->addMember(sym); } -void SymbolTable::addLazyObject(LazyObjFile *f, StringRef n) { +void SymbolTable::addLazyObject(InputFile *f, StringRef n) { + assert(f->lazy); Symbol *s; bool wasInserted; std::tie(s, wasInserted) = insert(n, f); @@ -574,7 +583,8 @@ void SymbolTable::addLazyObject(LazyObjFile *f, StringRef n) { if (!u || u->weakAlias || s->pendingArchiveLoad) return; s->pendingArchiveLoad = true; - f->fetch(); + f->lazy = false; + addFile(f); } void SymbolTable::addLazyDLLSymbol(DLLFile *f, DLLFile::Symbol *sym, diff --git a/lld/COFF/SymbolTable.h b/lld/COFF/SymbolTable.h index 3e76b416d1a0..47f3238fd75b 100644 --- a/lld/COFF/SymbolTable.h +++ b/lld/COFF/SymbolTable.h @@ -91,7 +91,7 @@ public: Symbol *addUndefined(StringRef name, InputFile *f, bool isWeakAlias); void addLazyArchive(ArchiveFile *f, const Archive::Symbol &sym); - void addLazyObject(LazyObjFile *f, StringRef n); + void addLazyObject(InputFile *f, StringRef n); void addLazyDLLSymbol(DLLFile *f, DLLFile::Symbol *sym, StringRef n); Symbol *addAbsolute(StringRef n, COFFSymbolRef s); Symbol *addRegular(InputFile *f, StringRef n, diff --git a/lld/COFF/Symbols.cpp b/lld/COFF/Symbols.cpp index 8a6a9b27d45f..a03cb03f8d17 100644 --- a/lld/COFF/Symbols.cpp +++ b/lld/COFF/Symbols.cpp @@ -36,9 +36,9 @@ static std::string maybeDemangleSymbol(StringRef symName) { StringRef demangleInput = prefixless; if (config->machine == I386) demangleInput.consume_front("_"); - std::string demangled = demangle(std::string(demangleInput)); + std::string demangled = demangle(demangleInput, true); if (demangled != demangleInput) - return prefix + demangle(std::string(demangleInput)); + return prefix + demangle(demangleInput, true); return (prefix + prefixless).str(); } return std::string(symName); diff --git a/lld/COFF/Symbols.h b/lld/COFF/Symbols.h index bb911171b1f5..c8865d128fb8 100644 --- a/lld/COFF/Symbols.h +++ b/lld/COFF/Symbols.h @@ -305,10 +305,9 @@ public: class LazyObject : public Symbol { public: - LazyObject(LazyObjFile *f, StringRef n) - : Symbol(LazyObjectKind, n), file(f) {} + LazyObject(InputFile *f, StringRef n) : Symbol(LazyObjectKind, n), file(f) {} static bool classof(const Symbol *s) { return s->kind() == LazyObjectKind; } - LazyObjFile *file; + InputFile *file; }; // MinGW only. diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp index 0788f3519f4e..12db942f1db5 100644 --- a/lld/COFF/Writer.cpp +++ b/lld/COFF/Writer.cpp @@ -485,7 +485,7 @@ static bool createThunks(OutputSection *os, int margin) { MutableArrayRef<coff_relocation> newRelocs; if (originalRelocs.data() == curRelocs.data()) { newRelocs = makeMutableArrayRef( - bAlloc.Allocate<coff_relocation>(originalRelocs.size()), + bAlloc().Allocate<coff_relocation>(originalRelocs.size()), originalRelocs.size()); } else { newRelocs = makeMutableArrayRef( @@ -1246,7 +1246,7 @@ void Writer::mergeSections() { if (p.first == toName) continue; StringSet<> names; - while (1) { + while (true) { if (!names.insert(toName).second) fatal("/merge: cycle found for section '" + p.first + "'"); auto i = config->merge.find(toName); diff --git a/lld/Common/CommonLinkerContext.cpp b/lld/Common/CommonLinkerContext.cpp new file mode 100644 index 000000000000..50ccbb37c796 --- /dev/null +++ b/lld/Common/CommonLinkerContext.cpp @@ -0,0 +1,45 @@ +//===- CommonLinkerContext.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "lld/Common/CommonLinkerContext.h" +#include "lld/Common/ErrorHandler.h" +#include "lld/Common/Memory.h" + +using namespace llvm; +using namespace lld; + +// Reference to the current LLD instance. This is a temporary situation, until +// we pass this context everywhere by reference, or we make it a thread_local, +// as in https://reviews.llvm.org/D108850?id=370678 where each thread can be +// associated with a LLD instance. Only then will LLD be free of global +// state. +static CommonLinkerContext *lctx; + +CommonLinkerContext::CommonLinkerContext() { lctx = this; } + +CommonLinkerContext::~CommonLinkerContext() { + assert(lctx); + // Explicitly call the destructors since we created the objects with placement + // new in SpecificAlloc::create(). + for (auto &it : instances) + it.second->~SpecificAllocBase(); + lctx = nullptr; +} + +CommonLinkerContext &lld::commonContext() { + assert(lctx); + return *lctx; +} + +bool lld::hasContext() { return lctx != nullptr; } + +void CommonLinkerContext::destroy() { + if (lctx == nullptr) + return; + delete lctx; +} diff --git a/lld/Common/ErrorHandler.cpp b/lld/Common/ErrorHandler.cpp index 399b6cac7547..15b3bd058ee9 100644 --- a/lld/Common/ErrorHandler.cpp +++ b/lld/Common/ErrorHandler.cpp @@ -10,6 +10,7 @@ #include "llvm/Support/Parallel.h" +#include "lld/Common/CommonLinkerContext.h" #include "llvm/ADT/Twine.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" @@ -18,51 +19,69 @@ #include "llvm/Support/Process.h" #include "llvm/Support/Program.h" #include "llvm/Support/raw_ostream.h" -#include <mutex> #include <regex> using namespace llvm; using namespace lld; -// The functions defined in this file can be called from multiple threads, -// but lld::outs() or lld::errs() are not thread-safe. We protect them using a -// mutex. -static std::mutex mu; - -// We want to separate multi-line messages with a newline. `sep` is "\n" -// if the last messages was multi-line. Otherwise "". -static StringRef sep; - static StringRef getSeparator(const Twine &msg) { if (StringRef(msg.str()).contains('\n')) return "\n"; return ""; } -raw_ostream *lld::stdoutOS; -raw_ostream *lld::stderrOS; +ErrorHandler::~ErrorHandler() { + if (cleanupCallback) + cleanupCallback(); +} + +void ErrorHandler::initialize(llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, + bool disableOutput) { + this->stdoutOS = &stdoutOS; + this->stderrOS = &stderrOS; + stderrOS.enable_colors(stderrOS.has_colors()); + this->exitEarly = exitEarly; + this->disableOutput = disableOutput; +} -ErrorHandler &lld::errorHandler() { - static ErrorHandler handler; - return handler; +void ErrorHandler::flushStreams() { + std::lock_guard<std::mutex> lock(mu); + outs().flush(); + errs().flush(); } +ErrorHandler &lld::errorHandler() { return context().e; } + raw_ostream &lld::outs() { - if (errorHandler().disableOutput) + ErrorHandler &e = errorHandler(); + return e.outs(); +} + +raw_ostream &lld::errs() { + ErrorHandler &e = errorHandler(); + return e.errs(); +} + +raw_ostream &ErrorHandler::outs() { + if (disableOutput) return llvm::nulls(); return stdoutOS ? *stdoutOS : llvm::outs(); } -raw_ostream &lld::errs() { - if (errorHandler().disableOutput) +raw_ostream &ErrorHandler::errs() { + if (disableOutput) return llvm::nulls(); return stderrOS ? *stderrOS : llvm::errs(); } void lld::exitLld(int val) { - // Delete any temporary file, while keeping the memory mapping open. - if (errorHandler().outputBuffer) - errorHandler().outputBuffer->discard(); + if (hasContext()) { + ErrorHandler &e = errorHandler(); + // Delete any temporary file, while keeping the memory mapping open. + if (e.outputBuffer) + e.outputBuffer->discard(); + } // Re-throw a possible signal or exception once/if it was catched by // safeLldMain(). @@ -75,11 +94,9 @@ void lld::exitLld(int val) { if (!CrashRecoveryContext::GetCurrent()) llvm_shutdown(); - { - std::lock_guard<std::mutex> lock(mu); - lld::outs().flush(); - lld::errs().flush(); - } + if (hasContext()) + lld::errorHandler().flushStreams(); + // When running inside safeLldMain(), restore the control flow back to the // CrashRecoveryContext. Otherwise simply use _exit(), meanning no cleanup, // since we want to avoid further crashes on shutdown. diff --git a/lld/Common/Memory.cpp b/lld/Common/Memory.cpp index c53e1d3e6cfc..7c90ff1d799c 100644 --- a/lld/Common/Memory.cpp +++ b/lld/Common/Memory.cpp @@ -7,16 +7,19 @@ //===----------------------------------------------------------------------===// #include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" using namespace llvm; using namespace lld; -BumpPtrAllocator lld::bAlloc; -StringSaver lld::saver{bAlloc}; -std::vector<SpecificAllocBase *> lld::SpecificAllocBase::instances; - -void lld::freeArena() { - for (SpecificAllocBase *alloc : SpecificAllocBase::instances) - alloc->reset(); - bAlloc.Reset(); +SpecificAllocBase * +lld::SpecificAllocBase::getOrCreate(void *tag, size_t size, size_t align, + SpecificAllocBase *(&creator)(void *)) { + auto &instances = context().instances; + auto &instance = instances[tag]; + if (instance == nullptr) { + void *storage = context().bAlloc.Allocate(size, align); + instance = creator(storage); + } + return instance; } diff --git a/lld/Common/Strings.cpp b/lld/Common/Strings.cpp index 7bf336490dae..6e5478e335ca 100644 --- a/lld/Common/Strings.cpp +++ b/lld/Common/Strings.cpp @@ -9,7 +9,6 @@ #include "lld/Common/Strings.h" #include "lld/Common/ErrorHandler.h" #include "lld/Common/LLVM.h" -#include "llvm/Demangle/Demangle.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/GlobPattern.h" #include <algorithm> @@ -19,18 +18,6 @@ using namespace llvm; using namespace lld; -// Returns the demangled C++ symbol name for name. -std::string lld::demangleItanium(StringRef name) { - // demangleItanium() can be called for all symbols. Only demangle C++ symbols, - // to avoid getting unexpected result for a C symbol that happens to match a - // mangled type name such as "Pi" (which would demangle to "int*"). - if (!name.startswith("_Z") && !name.startswith("__Z") && - !name.startswith("___Z") && !name.startswith("____Z")) - return std::string(name); - - return demangle(std::string(name)); -} - SingleStringMatcher::SingleStringMatcher(StringRef Pattern) { if (Pattern.size() > 2 && Pattern.startswith("\"") && Pattern.endswith("\"")) { diff --git a/lld/Common/TargetOptionsCommandFlags.cpp b/lld/Common/TargetOptionsCommandFlags.cpp index d39477ed89ad..b7749c4a2032 100644 --- a/lld/Common/TargetOptionsCommandFlags.cpp +++ b/lld/Common/TargetOptionsCommandFlags.cpp @@ -7,12 +7,9 @@ //===----------------------------------------------------------------------===// #include "lld/Common/TargetOptionsCommandFlags.h" - #include "llvm/CodeGen/CommandFlags.h" #include "llvm/Target/TargetOptions.h" -static llvm::codegen::RegisterCodeGenFlags CGF; - llvm::TargetOptions lld::initTargetOptionsFromCodeGenFlags() { return llvm::codegen::InitTargetOptionsFromCodeGenFlags(llvm::Triple()); } diff --git a/lld/Common/Timer.cpp b/lld/Common/Timer.cpp index 40fecd8892c1..29838c9720b7 100644 --- a/lld/Common/Timer.cpp +++ b/lld/Common/Timer.cpp @@ -9,6 +9,7 @@ #include "lld/Common/Timer.h" #include "lld/Common/ErrorHandler.h" #include "llvm/Support/Format.h" +#include <ratio> using namespace lld; using namespace llvm; diff --git a/lld/ELF/AArch64ErrataFix.cpp b/lld/ELF/AArch64ErrataFix.cpp index 50d4c237778b..d45edf9bd8ff 100644 --- a/lld/ELF/AArch64ErrataFix.cpp +++ b/lld/ELF/AArch64ErrataFix.cpp @@ -33,7 +33,7 @@ #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Strings.h" #include "llvm/Support/Endian.h" #include "llvm/Support/raw_ostream.h" @@ -398,9 +398,9 @@ Patch843419Section::Patch843419Section(InputSection *p, uint64_t off) patchee(p), patcheeOffset(off) { this->parent = p->getParent(); patchSym = addSyntheticLocal( - saver.save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC, 0, - getSize(), *this); - addSyntheticLocal(saver.save("$x"), STT_NOTYPE, 0, 0, *this); + saver().save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC, + 0, getSize(), *this); + addSyntheticLocal(saver().save("$x"), STT_NOTYPE, 0, 0, *this); } uint64_t Patch843419Section::getLDSTAddr() const { @@ -512,7 +512,7 @@ void AArch64Err843419Patcher::insertPatches( // determine the insertion point. This is ok as we only merge into an // InputSectionDescription once per pass, and at the end of the pass // assignAddresses() will recalculate all the outSecOff values. - std::vector<InputSection *> tmp; + SmallVector<InputSection *, 0> tmp; tmp.reserve(isd.sections.size() + patches.size()); auto mergeCmp = [](const InputSection *a, const InputSection *b) { if (a->outSecOff != b->outSecOff) diff --git a/lld/ELF/ARMErrataFix.cpp b/lld/ELF/ARMErrataFix.cpp index 5ad55f1326b3..25b47b90cef8 100644 --- a/lld/ELF/ARMErrataFix.cpp +++ b/lld/ELF/ARMErrataFix.cpp @@ -22,7 +22,7 @@ #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Strings.h" #include "llvm/Support/Endian.h" #include "llvm/Support/raw_ostream.h" @@ -142,9 +142,9 @@ Patch657417Section::Patch657417Section(InputSection *p, uint64_t off, patchee(p), patcheeOffset(off), instr(instr), isARM(isARM) { parent = p->getParent(); patchSym = addSyntheticLocal( - saver.save("__CortexA8657417_" + utohexstr(getBranchAddr())), STT_FUNC, + saver().save("__CortexA8657417_" + utohexstr(getBranchAddr())), STT_FUNC, isARM ? 0 : 1, getSize(), *this); - addSyntheticLocal(saver.save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0, *this); + addSyntheticLocal(saver().save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0, *this); } uint64_t Patch657417Section::getBranchAddr() const { @@ -395,7 +395,7 @@ void ARMErr657417Patcher::insertPatches( // determine the insertion point. This is ok as we only merge into an // InputSectionDescription once per pass, and at the end of the pass // assignAddresses() will recalculate all the outSecOff values. - std::vector<InputSection *> tmp; + SmallVector<InputSection *, 0> tmp; tmp.reserve(isd.sections.size() + patches.size()); auto mergeCmp = [](const InputSection *a, const InputSection *b) { if (a->outSecOff != b->outSecOff) diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp index ca3a6aa58dc5..784d578312d7 100644 --- a/lld/ELF/Arch/AArch64.cpp +++ b/lld/ELF/Arch/AArch64.cpp @@ -568,6 +568,98 @@ void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); } +AArch64Relaxer::AArch64Relaxer(ArrayRef<Relocation> relocs) { + if (!config->relax || config->emachine != EM_AARCH64) { + safeToRelaxAdrpLdr = false; + return; + } + // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC + // always appear in pairs. + size_t i = 0; + const size_t size = relocs.size(); + for (; i != size; ++i) { + if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) { + if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) { + ++i; + continue; + } + break; + } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) { + break; + } + } + safeToRelaxAdrpLdr = i == size; +} + +bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, + const Relocation &ldrRel, uint64_t secAddr, + uint8_t *buf) const { + if (!safeToRelaxAdrpLdr) + return false; + + // When the definition of sym is not preemptible then we may + // be able to relax + // ADRP xn, :got: sym + // LDR xn, [ xn :got_lo12: sym] + // to + // ADRP xn, sym + // ADD xn, xn, :lo_12: sym + + if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE || + ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC) + return false; + // Check if the relocations apply to consecutive instructions. + if (adrpRel.offset + 4 != ldrRel.offset) + return false; + // Check if the relocations reference the same symbol and + // skip undefined, preemptible and STT_GNU_IFUNC symbols. + if (!adrpRel.sym || adrpRel.sym != ldrRel.sym || !adrpRel.sym->isDefined() || + adrpRel.sym->isPreemptible || adrpRel.sym->isGnuIFunc()) + return false; + // Check if the addends of the both relocations are zero. + if (adrpRel.addend != 0 || ldrRel.addend != 0) + return false; + uint32_t adrpInstr = read32le(buf + adrpRel.offset); + uint32_t ldrInstr = read32le(buf + ldrRel.offset); + // Check if the first instruction is ADRP and the second instruction is LDR. + if ((adrpInstr & 0x9f000000) != 0x90000000 || + (ldrInstr & 0x3b000000) != 0x39000000) + return false; + // Check the value of the sf bit. + if (!(ldrInstr >> 31)) + return false; + uint32_t adrpDestReg = adrpInstr & 0x1f; + uint32_t ldrDestReg = ldrInstr & 0x1f; + uint32_t ldrSrcReg = (ldrInstr >> 5) & 0x1f; + // Check if ADPR and LDR use the same register. + if (adrpDestReg != ldrDestReg || adrpDestReg != ldrSrcReg) + return false; + + Symbol &sym = *adrpRel.sym; + // Check if the address difference is within 4GB range. + int64_t val = + getAArch64Page(sym.getVA()) - getAArch64Page(secAddr + adrpRel.offset); + if (val != llvm::SignExtend64(val, 33)) + return false; + + Relocation adrpSymRel = {R_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21, + adrpRel.offset, /*addend=*/0, &sym}; + Relocation addRel = {R_ABS, R_AARCH64_ADD_ABS_LO12_NC, ldrRel.offset, + /*addend=*/0, &sym}; + + // adrp x_<dest_reg> + write32le(buf + adrpSymRel.offset, 0x90000000 | adrpDestReg); + // add x_<dest reg>, x_<dest reg> + write32le(buf + addRel.offset, 0x91000000 | adrpDestReg | (adrpDestReg << 5)); + + target->relocate(buf + adrpSymRel.offset, adrpSymRel, + SignExtend64(getAArch64Page(sym.getVA()) - + getAArch64Page(secAddr + adrpSymRel.offset), + 64)); + target->relocate(buf + addRel.offset, addRel, SignExtend64(sym.getVA(), 64)); + return true; +} + // AArch64 may use security features in variant PLT sequences. These are: // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target // Indicator (BTI) introduced in armv8.5-a. The additional instructions used diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp index e28b62329494..97e4d6633138 100644 --- a/lld/ELF/Arch/PPC.cpp +++ b/lld/ELF/Arch/PPC.cpp @@ -192,7 +192,7 @@ void PPC::writeGotHeader(uint8_t *buf) const { void PPC::writeGotPlt(uint8_t *buf, const Symbol &s) const { // Address of the symbol resolver stub in .glink . - write32(buf, in.plt->getVA() + in.plt->headerSize + 4 * s.pltIndex); + write32(buf, in.plt->getVA() + in.plt->headerSize + 4 * s.getPltIdx()); } bool PPC::needsThunk(RelExpr expr, RelType type, const InputFile *file, diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp index d5e73ab9ec97..d9e4fc97ea0b 100644 --- a/lld/ELF/Arch/PPC64.cpp +++ b/lld/ELF/Arch/PPC64.cpp @@ -11,8 +11,7 @@ #include "SyntheticSections.h" #include "Target.h" #include "Thunks.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "llvm/Support/Endian.h" using namespace llvm; @@ -197,7 +196,7 @@ static bool addOptional(StringRef name, uint64_t value, Symbol *sym = symtab->find(name); if (!sym || sym->isDefined()) return false; - sym->resolve(Defined{/*file=*/nullptr, saver.save(name), STB_GLOBAL, + sym->resolve(Defined{/*file=*/nullptr, saver().save(name), STB_GLOBAL, STV_HIDDEN, STT_FUNC, value, /*size=*/0, /*section=*/nullptr}); defined.push_back(cast<Defined>(sym)); @@ -1089,7 +1088,7 @@ void PPC64::writePltHeader(uint8_t *buf) const { void PPC64::writePlt(uint8_t *buf, const Symbol &sym, uint64_t /*pltEntryAddr*/) const { - int32_t offset = pltHeaderSize + sym.pltIndex * pltEntrySize; + int32_t offset = pltHeaderSize + sym.getPltIdx() * pltEntrySize; // bl __glink_PLTresolve write32(buf, 0x48000000 | ((-offset) & 0x03FFFFFc)); } diff --git a/lld/ELF/Arch/X86.cpp b/lld/ELF/Arch/X86.cpp index 2560dc883257..e084bd646912 100644 --- a/lld/ELF/Arch/X86.cpp +++ b/lld/ELF/Arch/X86.cpp @@ -220,7 +220,7 @@ void X86::writePltHeader(uint8_t *buf) const { void X86::writePlt(uint8_t *buf, const Symbol &sym, uint64_t pltEntryAddr) const { - unsigned relOff = in.relaPlt->entsize * sym.pltIndex; + unsigned relOff = in.relaPlt->entsize * sym.getPltIdx(); if (config->isPic) { const uint8_t inst[] = { 0xff, 0xa3, 0, 0, 0, 0, // jmp *foo@GOT(%ebx) @@ -502,7 +502,7 @@ IntelIBT::IntelIBT() { pltHeaderSize = 0; } void IntelIBT::writeGotPlt(uint8_t *buf, const Symbol &s) const { uint64_t va = - in.ibtPlt->getVA() + IBTPltHeaderSize + s.pltIndex * pltEntrySize; + in.ibtPlt->getVA() + IBTPltHeaderSize + s.getPltIdx() * pltEntrySize; write32le(buf, va); } @@ -600,7 +600,7 @@ void RetpolinePic::writePltHeader(uint8_t *buf) const { void RetpolinePic::writePlt(uint8_t *buf, const Symbol &sym, uint64_t pltEntryAddr) const { - unsigned relOff = in.relaPlt->entsize * sym.pltIndex; + unsigned relOff = in.relaPlt->entsize * sym.getPltIdx(); const uint8_t insn[] = { 0x50, // pushl %eax 0x8b, 0x83, 0, 0, 0, 0, // mov foo@GOT(%ebx), %eax @@ -659,7 +659,7 @@ void RetpolineNoPic::writePltHeader(uint8_t *buf) const { void RetpolineNoPic::writePlt(uint8_t *buf, const Symbol &sym, uint64_t pltEntryAddr) const { - unsigned relOff = in.relaPlt->entsize * sym.pltIndex; + unsigned relOff = in.relaPlt->entsize * sym.getPltIdx(); const uint8_t insn[] = { 0x50, // 0: pushl %eax 0xa1, 0, 0, 0, 0, // 1: mov foo_in_GOT, %eax diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp index 614b5ed59218..f1360f50f8ac 100644 --- a/lld/ELF/Arch/X86_64.cpp +++ b/lld/ELF/Arch/X86_64.cpp @@ -282,12 +282,12 @@ bool X86_64::deleteFallThruJmpInsn(InputSection &is, InputFile *file, const unsigned sizeOfJmpCCInsn = 6; // To flip, there must be atleast one JmpCC and one direct jmp. if (is.getSize() < sizeOfDirectJmpInsn + sizeOfJmpCCInsn) - return 0; + return false; unsigned rbIndex = getRelocationWithOffset(is, (is.getSize() - sizeOfDirectJmpInsn - 4)); if (rbIndex == is.relocations.size()) - return 0; + return false; Relocation &rB = is.relocations[rbIndex]; @@ -304,7 +304,8 @@ bool X86_64::deleteFallThruJmpInsn(InputSection &is, InputFile *file, JmpInsnOpcode jInvert = invertJmpOpcode(jmpOpcodeB); if (jInvert == J_UNKNOWN) return false; - is.jumpInstrMods.push_back({jInvert, (rB.offset - 1), 4}); + is.jumpInstrMod = make<JumpInstrMod>(); + *is.jumpInstrMod = {rB.offset - 1, jInvert, 4}; // Move R's values to rB except the offset. rB = {r.expr, r.type, rB.offset, r.addend, r.sym}; // Cancel R @@ -416,7 +417,7 @@ void X86_64::writePlt(uint8_t *buf, const Symbol &sym, memcpy(buf, inst, sizeof(inst)); write32le(buf + 2, sym.getGotPltVA() - pltEntryAddr - 6); - write32le(buf + 7, sym.pltIndex); + write32le(buf + 7, sym.getPltIdx()); write32le(buf + 12, in.plt->getVA() - pltEntryAddr - 16); } @@ -946,7 +947,7 @@ void X86_64::relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const { bool X86_64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end, uint8_t stOther) const { if (!config->is64) { - error("Target doesn't support split stacks."); + error("target doesn't support split stacks"); return false; } @@ -994,7 +995,7 @@ IntelIBT::IntelIBT() { pltHeaderSize = 0; } void IntelIBT::writeGotPlt(uint8_t *buf, const Symbol &s) const { uint64_t va = - in.ibtPlt->getVA() + IBTPltHeaderSize + s.pltIndex * pltEntrySize; + in.ibtPlt->getVA() + IBTPltHeaderSize + s.getPltIdx() * pltEntrySize; write64le(buf, va); } @@ -1106,7 +1107,7 @@ void Retpoline::writePlt(uint8_t *buf, const Symbol &sym, write32le(buf + 3, sym.getGotPltVA() - pltEntryAddr - 7); write32le(buf + 8, -off - 12 + 32); write32le(buf + 13, -off - 17 + 18); - write32le(buf + 18, sym.pltIndex); + write32le(buf + 18, sym.getPltIdx()); write32le(buf + 23, -off - 27); } diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index b3d5219ff57b..47bbed125cb1 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -87,8 +87,8 @@ struct SymbolVersion { struct VersionDefinition { llvm::StringRef name; uint16_t id; - std::vector<SymbolVersion> nonLocalPatterns; - std::vector<SymbolVersion> localPatterns; + SmallVector<SymbolVersion, 0> nonLocalPatterns; + SmallVector<SymbolVersion, 0> localPatterns; }; // This struct contains the global configuration for the linker. diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 6b689f50cce7..de26afddd28b 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -77,16 +77,17 @@ std::unique_ptr<LinkerDriver> elf::driver; static void setConfigs(opt::InputArgList &args); static void readConfigs(opt::InputArgList &args); -bool elf::link(ArrayRef<const char *> args, bool canExitEarly, - raw_ostream &stdoutOS, raw_ostream &stderrOS) { - lld::stdoutOS = &stdoutOS; - lld::stderrOS = &stderrOS; - - errorHandler().cleanupCallback = []() { - freeArena(); +bool elf::link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, + bool disableOutput) { + // This driver-specific context will be freed later by lldMain(). + auto *ctx = new CommonLinkerContext; + ctx->e.initialize(stdoutOS, stderrOS, exitEarly, disableOutput); + ctx->e.cleanupCallback = []() { inputSections.clear(); outputSections.clear(); + memoryBuffers.clear(); archiveFiles.clear(); binaryFiles.clear(); bitcodeFiles.clear(); @@ -95,43 +96,33 @@ bool elf::link(ArrayRef<const char *> args, bool canExitEarly, sharedFiles.clear(); backwardReferences.clear(); whyExtract.clear(); + symAux.clear(); tar = nullptr; - memset(&in, 0, sizeof(in)); + in.reset(); - partitions = {Partition()}; + partitions.clear(); + partitions.emplace_back(); SharedFile::vernauxNum = 0; }; - - errorHandler().logName = args::getFilenameWithoutExe(args[0]); - errorHandler().errorLimitExceededMsg = - "too many errors emitted, stopping now (use " - "-error-limit=0 to see all errors)"; - errorHandler().exitEarly = canExitEarly; - stderrOS.enable_colors(stderrOS.has_colors()); + ctx->e.logName = args::getFilenameWithoutExe(args[0]); + ctx->e.errorLimitExceededMsg = "too many errors emitted, stopping now (use " + "-error-limit=0 to see all errors)"; config = std::make_unique<Configuration>(); driver = std::make_unique<LinkerDriver>(); script = std::make_unique<LinkerScript>(); symtab = std::make_unique<SymbolTable>(); - partitions = {Partition()}; + partitions.clear(); + partitions.emplace_back(); config->progName = args[0]; driver->linkerMain(args); - // Exit immediately if we don't need to return to the caller. - // This saves time because the overhead of calling destructors - // for all globally-allocated objects is not negligible. - if (canExitEarly) - exitLld(errorCount() ? 1 : 0); - - bool ret = errorCount() == 0; - if (!canExitEarly) - errorHandler().reset(); - return ret; + return errorCount() == 0; } // Parses a linker -m option. @@ -232,23 +223,22 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) { std::unique_ptr<Archive> file = CHECK(Archive::create(mbref), path + ": failed to parse archive"); - // If an archive file has no symbol table, it is likely that a user - // is attempting LTO and using a default ar command that doesn't - // understand the LLVM bitcode file. It is a pretty common error, so - // we'll handle it as if it had a symbol table. + // If an archive file has no symbol table, it may be intentional (used as a + // group of lazy object files where the symbol table is not useful), or the + // user is attempting LTO and using a default ar command that doesn't + // understand the LLVM bitcode file. Treat the archive as a group of lazy + // object files. if (!file->isEmpty() && !file->hasSymbolTable()) { - // Check if all members are bitcode files. If not, ignore, which is the - // default action without the LTO hack described above. for (const std::pair<MemoryBufferRef, uint64_t> &p : - getArchiveMembers(mbref)) - if (identify_magic(p.first.getBuffer()) != file_magic::bitcode) { - error(path + ": archive has no index; run ranlib to add one"); - return; - } - - for (const std::pair<MemoryBufferRef, uint64_t> &p : - getArchiveMembers(mbref)) - files.push_back(createLazyFile(p.first, path, p.second)); + getArchiveMembers(mbref)) { + auto magic = identify_magic(p.first.getBuffer()); + if (magic == file_magic::bitcode || + magic == file_magic::elf_relocatable) + files.push_back(createLazyFile(p.first, path, p.second)); + else + error(path + ": archive member '" + p.first.getBufferIdentifier() + + "' is neither ET_REL nor LLVM bitcode"); + } return; } @@ -1256,7 +1246,7 @@ static void readConfigs(opt::InputArgList &args) { // Parse LTO options. if (auto *arg = args.getLastArg(OPT_plugin_opt_mcpu_eq)) - parseClangOption(saver.save("-mcpu=" + StringRef(arg->getValue())), + parseClangOption(saver().save("-mcpu=" + StringRef(arg->getValue())), arg->getSpelling()); for (opt::Arg *arg : args.filtered(OPT_plugin_opt_eq_minus)) @@ -1724,7 +1714,7 @@ static void handleUndefinedGlob(StringRef arg) { // symbols to the symbol table, invalidating the current iterator. std::vector<Symbol *> syms; for (Symbol *sym : symtab->symbols()) - if (pat->match(sym->getName())) + if (!sym->isPlaceholder() && pat->match(sym->getName())) syms.push_back(sym); for (Symbol *sym : syms) @@ -1847,8 +1837,7 @@ static void demoteSharedSymbols() { llvm::TimeTraceScope timeScope("Demote shared symbols"); for (Symbol *sym : symtab->symbols()) { auto *s = dyn_cast<SharedSymbol>(sym); - if (!((s && !s->getFile().isNeeded) || - (sym->isLazy() && sym->isUsedInRegularObj))) + if (!(s && !s->getFile().isNeeded) && !sym->isLazy()) continue; bool used = sym->used; @@ -1985,6 +1974,28 @@ static Symbol *addUnusedUndefined(StringRef name, return symtab->addSymbol(sym); } +static void markBuffersAsDontNeed(bool skipLinkedOutput) { + // With --thinlto-index-only, all buffers are nearly unused from now on + // (except symbol/section names used by infrequent passes). Mark input file + // buffers as MADV_DONTNEED so that these pages can be reused by the expensive + // thin link, saving memory. + if (skipLinkedOutput) { + for (MemoryBuffer &mb : llvm::make_pointee_range(memoryBuffers)) + mb.dontNeedIfMmap(); + return; + } + + // Otherwise, just mark MemoryBuffers backing BitcodeFiles. + DenseSet<const char *> bufs; + for (BitcodeFile *file : bitcodeFiles) + bufs.insert(file->mb.getBufferStart()); + for (BitcodeFile *file : lazyBitcodeFiles) + bufs.insert(file->mb.getBufferStart()); + for (MemoryBuffer &mb : llvm::make_pointee_range(memoryBuffers)) + if (bufs.count(mb.getBufferStart())) + mb.dontNeedIfMmap(); +} + // This function is where all the optimizations of link-time // optimization takes place. When LTO is in use, some input files are // not in native object file format but in the LLVM bitcode format. @@ -1992,13 +2003,17 @@ static Symbol *addUnusedUndefined(StringRef name, // using LLVM functions and replaces bitcode symbols with the results. // Because all bitcode files that the program consists of are passed to // the compiler at once, it can do a whole-program optimization. -template <class ELFT> void LinkerDriver::compileBitcodeFiles() { +template <class ELFT> +void LinkerDriver::compileBitcodeFiles(bool skipLinkedOutput) { llvm::TimeTraceScope timeScope("LTO"); // Compile bitcode files and replace bitcode symbols. lto.reset(new BitcodeCompiler); for (BitcodeFile *file : bitcodeFiles) lto->add(*file); + if (!bitcodeFiles.empty()) + markBuffersAsDontNeed(skipLinkedOutput); + for (InputFile *file : lto->compile()) { auto *obj = cast<ObjFile<ELFT>>(file); obj->parse(/*ignoreComdats=*/true); @@ -2006,7 +2021,8 @@ template <class ELFT> void LinkerDriver::compileBitcodeFiles() { // Parse '@' in symbol names for non-relocatable output. if (!config->relocatable) for (Symbol *sym : obj->getGlobalSymbols()) - sym->parseSymbolVersion(); + if (sym->hasVersionSuffix) + sym->parseSymbolVersion(); objectFiles.push_back(obj); } } @@ -2043,9 +2059,9 @@ static std::vector<WrappedSymbol> addWrappedSymbols(opt::InputArgList &args) { if (!sym) continue; - Symbol *real = addUnusedUndefined(saver.save("__real_" + name)); + Symbol *real = addUnusedUndefined(saver().save("__real_" + name)); Symbol *wrap = - addUnusedUndefined(saver.save("__wrap_" + name), sym->binding); + addUnusedUndefined(saver().save("__wrap_" + name), sym->binding); v.push_back({sym, real, wrap}); // We want to tell LTO not to inline symbols to be overwritten @@ -2080,8 +2096,10 @@ static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) { map[w.real] = w.sym; } for (Symbol *sym : symtab->symbols()) { - // Enumerate symbols with a non-default version (foo@v1). - StringRef name = sym->getName(); + // Enumerate symbols with a non-default version (foo@v1). hasVersionSuffix + // filters out most symbols but is not sufficient. + if (!sym->hasVersionSuffix) + continue; const char *suffix1 = sym->getVersionSuffix(); if (suffix1[0] != '@' || suffix1[1] == '@') continue; @@ -2090,7 +2108,7 @@ static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) { // // * There is a definition of foo@v1 and foo@@v1. // * There is a definition of foo@v1 and foo. - Defined *sym2 = dyn_cast_or_null<Defined>(symtab->find(name)); + Defined *sym2 = dyn_cast_or_null<Defined>(symtab->find(sym->getName())); if (!sym2) continue; const char *suffix2 = sym2->getVersionSuffix(); @@ -2103,6 +2121,7 @@ static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) { sym2->resolve(*sym); // Eliminate foo@v1 from the symbol table. sym->symbolKind = Symbol::PlaceholderKind; + sym->isUsedInRegularObj = false; } else if (auto *sym1 = dyn_cast<Defined>(sym)) { if (sym2->versionId > VER_NDX_GLOBAL ? config->versionDefinitions[sym2->versionId].name == suffix1 + 1 @@ -2115,6 +2134,7 @@ static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) { // defined in the same place. map.try_emplace(sym2, sym); sym2->symbolKind = Symbol::PlaceholderKind; + sym2->isUsedInRegularObj = false; } } } @@ -2357,39 +2377,45 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) { symtab->scanVersionScript(); } + // Skip the normal linked output if some LTO options are specified. + // + // For --thinlto-index-only, index file creation is performed in + // compileBitcodeFiles, so we are done afterwards. --plugin-opt=emit-llvm and + // --plugin-opt=emit-asm create output files in bitcode or assembly code, + // respectively. When only certain thinLTO modules are specified for + // compilation, the intermediate object file are the expected output. + const bool skipLinkedOutput = config->thinLTOIndexOnly || config->emitLLVM || + config->ltoEmitAsm || + !config->thinLTOModulesToCompile.empty(); + // Do link-time optimization if given files are LLVM bitcode files. // This compiles bitcode files into real object files. // // With this the symbol table should be complete. After this, no new names // except a few linker-synthesized ones will be added to the symbol table. - compileBitcodeFiles<ELFT>(); - - // Handle --exclude-libs again because lto.tmp may reference additional - // libcalls symbols defined in an excluded archive. This may override - // versionId set by scanVersionScript(). - if (args.hasArg(OPT_exclude_libs)) - excludeLibs(args); + compileBitcodeFiles<ELFT>(skipLinkedOutput); // Symbol resolution finished. Report backward reference problems. reportBackrefs(); if (errorCount()) return; - // If --thinlto-index-only is given, we should create only "index - // files" and not object files. Index file creation is already done - // in compileBitcodeFiles, so we are done if that's the case. - // Likewise, --plugin-opt=emit-llvm and --plugin-opt=emit-asm are the - // options to create output files in bitcode or assembly code - // respectively. No object files are generated. - // Also bail out here when only certain thinLTO modules are specified for - // compilation. The intermediate object file are the expected output. - if (config->thinLTOIndexOnly || config->emitLLVM || config->ltoEmitAsm || - !config->thinLTOModulesToCompile.empty()) + // Bail out if normal linked output is skipped due to LTO. + if (skipLinkedOutput) return; + // Handle --exclude-libs again because lto.tmp may reference additional + // libcalls symbols defined in an excluded archive. This may override + // versionId set by scanVersionScript(). + if (args.hasArg(OPT_exclude_libs)) + excludeLibs(args); + // Apply symbol renames for --wrap and combine foo@v1 and foo@@v1. redirectSymbols(wrapped); + // Replace common symbols with regular symbols. + replaceCommonSymbols(); + { llvm::TimeTraceScope timeScope("Aggregate sections"); // Now that we have a complete list of input files. @@ -2474,9 +2500,6 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) { if (!config->relocatable) inputSections.push_back(createCommentSection()); - // Replace common symbols with regular symbols. - replaceCommonSymbols(); - // Split SHF_MERGE and .eh_frame sections into pieces in preparation for garbage collection. splitSections<ELFT>(); diff --git a/lld/ELF/Driver.h b/lld/ELF/Driver.h index 5961e1f69472..b8cbb3b19268 100644 --- a/lld/ELF/Driver.h +++ b/lld/ELF/Driver.h @@ -34,7 +34,7 @@ private: void createFiles(llvm::opt::InputArgList &args); void inferMachineType(); template <class ELFT> void link(llvm::opt::InputArgList &args); - template <class ELFT> void compileBitcodeFiles(); + template <class ELFT> void compileBitcodeFiles(bool skipLinkedOutput); // True if we are in --whole-archive and --no-whole-archive. bool inWholeArchive = false; diff --git a/lld/ELF/DriverUtils.cpp b/lld/ELF/DriverUtils.cpp index 54d2d0ae6fb9..ac29b47abcc9 100644 --- a/lld/ELF/DriverUtils.cpp +++ b/lld/ELF/DriverUtils.cpp @@ -13,8 +13,7 @@ //===----------------------------------------------------------------------===// #include "Driver.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Reproduce.h" #include "lld/Common/Version.h" #include "llvm/ADT/Optional.h" @@ -102,7 +101,7 @@ static void concatLTOPluginOptions(SmallVectorImpl<const char *> &args) { for (size_t i = 0, e = args.size(); i != e; ++i) { StringRef s = args[i]; if ((s == "-plugin-opt" || s == "--plugin-opt") && i + 1 != e) { - v.push_back(saver.save(s + "=" + args[i + 1]).data()); + v.push_back(saver().save(s + "=" + args[i + 1]).data()); ++i; } else { v.push_back(args[i]); @@ -125,7 +124,7 @@ opt::InputArgList ELFOptTable::parse(ArrayRef<const char *> argv) { // Expand response files (arguments in the form of @<filename>) // and then parse the argument again. - cl::ExpandResponseFiles(saver, getQuotingStyle(args), vec); + cl::ExpandResponseFiles(saver(), getQuotingStyle(args), vec); concatLTOPluginOptions(vec); args = this->ParseArgs(vec, missingIndex, missingCount); diff --git a/lld/ELF/EhFrame.cpp b/lld/ELF/EhFrame.cpp index 578c640f0214..9ac2ed772073 100644 --- a/lld/ELF/EhFrame.cpp +++ b/lld/ELF/EhFrame.cpp @@ -36,7 +36,6 @@ namespace { class EhReader { public: EhReader(InputSectionBase *s, ArrayRef<uint8_t> d) : isec(s), d(d) {} - size_t readEhRecordSize(); uint8_t getFdeEncoding(); bool hasLSDA(); @@ -58,28 +57,6 @@ private: }; } -size_t elf::readEhRecordSize(InputSectionBase *s, size_t off) { - return EhReader(s, s->data().slice(off)).readEhRecordSize(); -} - -// .eh_frame section is a sequence of records. Each record starts with -// a 4 byte length field. This function reads the length. -size_t EhReader::readEhRecordSize() { - if (d.size() < 4) - failOn(d.data(), "CIE/FDE too small"); - - // First 4 bytes of CIE/FDE is the size of the record. - // If it is 0xFFFFFFFF, the next 8 bytes contain the size instead, - // but we do not support that format yet. - uint64_t v = read32(d.data()); - if (v == UINT32_MAX) - failOn(d.data(), "CIE/FDE too large"); - uint64_t size = v + 4; - if (size > d.size()) - failOn(d.data(), "CIE/FDE ends past the end of the section"); - return size; -} - // Read a byte and advance D by one byte. uint8_t EhReader::readByte() { if (d.empty()) diff --git a/lld/ELF/EhFrame.h b/lld/ELF/EhFrame.h index e44832317b3a..3b144648cf8f 100644 --- a/lld/ELF/EhFrame.h +++ b/lld/ELF/EhFrame.h @@ -16,7 +16,6 @@ namespace elf { class InputSectionBase; struct EhSectionPiece; -size_t readEhRecordSize(InputSectionBase *s, size_t off); uint8_t getFdeEncoding(EhSectionPiece *p); bool hasLSDA(const EhSectionPiece &p); } // namespace elf diff --git a/lld/ELF/ICF.cpp b/lld/ELF/ICF.cpp index ec63d2ef4d6f..3b991e8d3470 100644 --- a/lld/ELF/ICF.cpp +++ b/lld/ELF/ICF.cpp @@ -461,8 +461,9 @@ template <class ELFT> void ICF<ELFT>::run() { // Compute isPreemptible early. We may add more symbols later, so this loop // cannot be merged with the later computeIsPreemptible() pass which is used // by scanRelocations(). - for (Symbol *sym : symtab->symbols()) - sym->isPreemptible = computeIsPreemptible(*sym); + if (config->hasDynSymTab) + for (Symbol *sym : symtab->symbols()) + sym->isPreemptible = computeIsPreemptible(*sym); // Two text sections may have identical content and relocations but different // LSDA, e.g. the two functions may have catch blocks of different types. If a diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index e321b0d82920..4da371c619f4 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -13,9 +13,8 @@ #include "SymbolTable.h" #include "Symbols.h" #include "SyntheticSections.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/DWARF.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" #include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/IR/LLVMContext.h" @@ -43,6 +42,7 @@ using namespace lld::elf; bool InputFile::isInGroup; uint32_t InputFile::nextGroupId; +SmallVector<std::unique_ptr<MemoryBuffer>> elf::memoryBuffers; SmallVector<ArchiveFile *, 0> elf::archiveFiles; SmallVector<BinaryFile *, 0> elf::binaryFiles; SmallVector<BitcodeFile *, 0> elf::bitcodeFiles; @@ -110,7 +110,7 @@ Optional<MemoryBufferRef> elf::readFile(StringRef path) { // The --chroot option changes our virtual root directory. // This is useful when you are dealing with files created by --reproduce. if (!config->chroot.empty() && path.startswith("/")) - path = saver.save(config->chroot + path); + path = saver().save(config->chroot + path); log(path); config->dependencyFiles.insert(llvm::CachedHashString(path)); @@ -122,9 +122,8 @@ Optional<MemoryBufferRef> elf::readFile(StringRef path) { return None; } - std::unique_ptr<MemoryBuffer> &mb = *mbOrErr; - MemoryBufferRef mbref = mb->getMemBufferRef(); - make<std::unique_ptr<MemoryBuffer>>(std::move(mb)); // take MB ownership + MemoryBufferRef mbref = (*mbOrErr)->getMemBufferRef(); + memoryBuffers.push_back(std::move(*mbOrErr)); // take MB ownership if (tar) tar->append(relativeToRoot(path), mbref.getBuffer()); @@ -552,7 +551,7 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats) { std::vector<ArrayRef<Elf_Word>> selectedGroups; - for (size_t i = 0, e = objSections.size(); i < e; ++i) { + for (size_t i = 0; i != size; ++i) { if (this->sections[i] == &InputSection::discarded) continue; const Elf_Shdr &sec = objSections[i]; @@ -638,21 +637,61 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats) { // such cases, the relocation section would attempt to reference a target // section that has not yet been created. For simplicity, delay creation of // relocation sections until now. - for (size_t i = 0, e = objSections.size(); i < e; ++i) { + for (size_t i = 0; i != size; ++i) { if (this->sections[i] == &InputSection::discarded) continue; const Elf_Shdr &sec = objSections[i]; - if (sec.sh_type == SHT_REL || sec.sh_type == SHT_RELA) - this->sections[i] = createInputSection(i, sec, shstrtab); + if (sec.sh_type == SHT_REL || sec.sh_type == SHT_RELA) { + // Find a relocation target section and associate this section with that. + // Target may have been discarded if it is in a different section group + // and the group is discarded, even though it's a violation of the spec. + // We handle that situation gracefully by discarding dangling relocation + // sections. + const uint32_t info = sec.sh_info; + InputSectionBase *s = getRelocTarget(i, sec, info); + if (!s) + continue; + + // ELF spec allows mergeable sections with relocations, but they are rare, + // and it is in practice hard to merge such sections by contents, because + // applying relocations at end of linking changes section contents. So, we + // simply handle such sections as non-mergeable ones. Degrading like this + // is acceptable because section merging is optional. + if (auto *ms = dyn_cast<MergeInputSection>(s)) { + s = make<InputSection>(ms->file, ms->flags, ms->type, ms->alignment, + ms->data(), ms->name); + sections[info] = s; + } + + if (s->relSecIdx != 0) + error( + toString(s) + + ": multiple relocation sections to one section are not supported"); + s->relSecIdx = i; + + // Relocation sections are usually removed from the output, so return + // `nullptr` for the normal case. However, if -r or --emit-relocs is + // specified, we need to copy them to the output. (Some post link analysis + // tools specify --emit-relocs to obtain the information.) + if (config->copyRelocs) { + auto *isec = make<InputSection>( + *this, sec, check(obj.getSectionName(sec, shstrtab))); + // If the relocated section is discarded (due to /DISCARD/ or + // --gc-sections), the relocation section should be discarded as well. + s->dependentSections.push_back(isec); + sections[i] = isec; + } + continue; + } // A SHF_LINK_ORDER section with sh_link=0 is handled as if it did not have // the flag. - if (!(sec.sh_flags & SHF_LINK_ORDER) || !sec.sh_link) + if (!sec.sh_link || !(sec.sh_flags & SHF_LINK_ORDER)) continue; InputSectionBase *linkSec = nullptr; - if (sec.sh_link < this->sections.size()) + if (sec.sh_link < size) linkSec = this->sections[sec.sh_link]; if (!linkSec) fatal(toString(this) + ": invalid sh_link index: " + Twine(sec.sh_link)); @@ -828,9 +867,9 @@ template <class ELFT> static uint32_t readAndFeatures(const InputSection &sec) { } template <class ELFT> -InputSectionBase *ObjFile<ELFT>::getRelocTarget(uint32_t idx, StringRef name, - const Elf_Shdr &sec) { - uint32_t info = sec.sh_info; +InputSectionBase *ObjFile<ELFT>::getRelocTarget(uint32_t idx, + const Elf_Shdr &sec, + uint32_t info) { if (info < this->sections.size()) { InputSectionBase *target = this->sections[info]; @@ -844,18 +883,11 @@ InputSectionBase *ObjFile<ELFT>::getRelocTarget(uint32_t idx, StringRef name, return target; } - error(toString(this) + Twine(": relocation section ") + name + " (index " + - Twine(idx) + ") has invalid sh_info (" + Twine(info) + ")"); + error(toString(this) + Twine(": relocation section (index ") + Twine(idx) + + ") has invalid sh_info (" + Twine(info) + ")"); return nullptr; } -// Create a regular InputSection class that has the same contents -// as a given section. -static InputSection *toRegularSection(MergeInputSection *sec) { - return make<InputSection>(sec->file, sec->flags, sec->type, sec->alignment, - sec->data(), sec->name); -} - template <class ELFT> InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx, const Elf_Shdr &sec, @@ -879,8 +911,8 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx, // to work. In a full implementation we would merge all attribute // sections. if (in.attributes == nullptr) { - in.attributes = make<InputSection>(*this, sec, name); - return in.attributes; + in.attributes = std::make_unique<InputSection>(*this, sec, name); + return in.attributes.get(); } return &InputSection::discarded; } @@ -901,17 +933,14 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx, // standard extensions to enable. In a full implementation we would merge // all attribute sections. if (in.attributes == nullptr) { - in.attributes = make<InputSection>(*this, sec, name); - return in.attributes; + in.attributes = std::make_unique<InputSection>(*this, sec, name); + return in.attributes.get(); } return &InputSection::discarded; } } - switch (sec.sh_type) { - case SHT_LLVM_DEPENDENT_LIBRARIES: { - if (config->relocatable) - break; + if (sec.sh_type == SHT_LLVM_DEPENDENT_LIBRARIES && !config->relocatable) { ArrayRef<char> data = CHECK(this->getObj().template getSectionContentsAsArray<char>(sec), this); if (!data.empty() && data.back() != '\0') { @@ -927,45 +956,6 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx, } return &InputSection::discarded; } - case SHT_RELA: - case SHT_REL: { - // Find a relocation target section and associate this section with that. - // Target may have been discarded if it is in a different section group - // and the group is discarded, even though it's a violation of the - // spec. We handle that situation gracefully by discarding dangling - // relocation sections. - InputSectionBase *target = getRelocTarget(idx, name, sec); - if (!target) - return nullptr; - - // ELF spec allows mergeable sections with relocations, but they are - // rare, and it is in practice hard to merge such sections by contents, - // because applying relocations at end of linking changes section - // contents. So, we simply handle such sections as non-mergeable ones. - // Degrading like this is acceptable because section merging is optional. - if (auto *ms = dyn_cast<MergeInputSection>(target)) { - target = toRegularSection(ms); - this->sections[sec.sh_info] = target; - } - - if (target->relSecIdx != 0) - fatal(toString(this) + - ": multiple relocation sections to one section are not supported"); - target->relSecIdx = idx; - - // Relocation sections are usually removed from the output, so return - // `nullptr` for the normal case. However, if -r or --emit-relocs is - // specified, we need to copy them to the output. (Some post link analysis - // tools specify --emit-relocs to obtain the information.) - if (!config->copyRelocs) - return nullptr; - InputSection *relocSec = make<InputSection>(*this, sec, name); - // If the relocated section is discarded (due to /DISCARD/ or - // --gc-sections), the relocation section should be discarded as well. - target->dependentSections.push_back(relocSec); - return relocSec; - } - } if (name.startswith(".n")) { // The GNU linker uses .note.GNU-stack section as a marker indicating @@ -1076,7 +1066,7 @@ template <class ELFT> void ObjFile<ELFT>::initializeSymbols() { sourceFile = CHECK(eSym.getName(stringTable), this); if (LLVM_UNLIKELY(stringTable.size() <= eSym.st_name)) fatal(toString(this) + ": invalid symbol name offset"); - StringRefZ name = stringTable.data() + eSym.st_name; + StringRef name(stringTable.data() + eSym.st_name); symbols[i] = reinterpret_cast<Symbol *>(locals + i); if (eSym.st_shndx == SHN_UNDEF || sec == &InputSection::discarded) @@ -1460,9 +1450,10 @@ template <class ELFT> void SharedFile::parse() { } // DSOs are uniquified not by filename but by soname. - DenseMap<StringRef, SharedFile *>::iterator it; + DenseMap<CachedHashStringRef, SharedFile *>::iterator it; bool wasInserted; - std::tie(it, wasInserted) = symtab->soNames.try_emplace(soName, this); + std::tie(it, wasInserted) = + symtab->soNames.try_emplace(CachedHashStringRef(soName), this); // If a DSO appears more than once on the command line with and without // --as-needed, --no-as-needed takes precedence over --as-needed because a @@ -1526,8 +1517,8 @@ template <class ELFT> void SharedFile::parse() { } StringRef verName = stringTable.data() + verneeds[idx]; versionedNameBuffer.clear(); - name = - saver.save((name + "@" + verName).toStringRef(versionedNameBuffer)); + name = saver().save( + (name + "@" + verName).toStringRef(versionedNameBuffer)); } Symbol *s = symtab.addSymbol( Undefined{this, name, sym.getBinding(), sym.st_other, sym.getType()}); @@ -1569,7 +1560,7 @@ template <class ELFT> void SharedFile::parse() { reinterpret_cast<const Elf_Verdef *>(verdefs[idx])->getAux()->vda_name; versionedNameBuffer.clear(); name = (name + "@" + verName).toStringRef(versionedNameBuffer); - symtab.addSymbol(SharedSymbol{*this, saver.save(name), sym.getBinding(), + symtab.addSymbol(SharedSymbol{*this, saver().save(name), sym.getBinding(), sym.st_other, sym.getType(), sym.st_value, sym.st_size, alignment, idx}); } @@ -1652,11 +1643,10 @@ BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName, // into consideration at LTO time (which very likely causes undefined // symbols later in the link stage). So we append file offset to make // filename unique. - StringRef name = - archiveName.empty() - ? saver.save(path) - : saver.save(archiveName + "(" + path::filename(path) + " at " + - utostr(offsetInArchive) + ")"); + StringRef name = archiveName.empty() + ? saver().save(path) + : saver().save(archiveName + "(" + path::filename(path) + + " at " + utostr(offsetInArchive) + ")"); MemoryBufferRef mbref(mb.getBuffer(), name); obj = CHECK(lto::InputFile::create(mbref), this); @@ -1680,34 +1670,42 @@ static uint8_t mapVisibility(GlobalValue::VisibilityTypes gvVisibility) { } template <class ELFT> -static Symbol *createBitcodeSymbol(const std::vector<bool> &keptComdats, - const lto::InputFile::Symbol &objSym, - BitcodeFile &f) { - StringRef name = saver.save(objSym.getName()); +static void +createBitcodeSymbol(Symbol *&sym, const std::vector<bool> &keptComdats, + const lto::InputFile::Symbol &objSym, BitcodeFile &f) { uint8_t binding = objSym.isWeak() ? STB_WEAK : STB_GLOBAL; uint8_t type = objSym.isTLS() ? STT_TLS : STT_NOTYPE; uint8_t visibility = mapVisibility(objSym.getVisibility()); bool canOmitFromDynSym = objSym.canBeOmittedFromSymbolTable(); + StringRef name; + if (sym) { + name = sym->getName(); + } else { + name = saver().save(objSym.getName()); + sym = symtab->insert(name); + } + int c = objSym.getComdatIndex(); if (objSym.isUndefined() || (c != -1 && !keptComdats[c])) { Undefined newSym(&f, name, binding, visibility, type); if (canOmitFromDynSym) newSym.exportDynamic = false; - Symbol *ret = symtab->addSymbol(newSym); - ret->referenced = true; - return ret; + sym->resolve(newSym); + sym->referenced = true; + return; } - if (objSym.isCommon()) - return symtab->addSymbol( - CommonSymbol{&f, name, binding, visibility, STT_OBJECT, - objSym.getCommonAlignment(), objSym.getCommonSize()}); - - Defined newSym(&f, name, binding, visibility, type, 0, 0, nullptr); - if (canOmitFromDynSym) - newSym.exportDynamic = false; - return symtab->addSymbol(newSym); + if (objSym.isCommon()) { + sym->resolve(CommonSymbol{&f, name, binding, visibility, STT_OBJECT, + objSym.getCommonAlignment(), + objSym.getCommonSize()}); + } else { + Defined newSym(&f, name, binding, visibility, type, 0, 0, nullptr); + if (canOmitFromDynSym) + newSym.exportDynamic = false; + sym->resolve(newSym); + } } template <class ELFT> void BitcodeFile::parse() { @@ -1719,10 +1717,11 @@ template <class ELFT> void BitcodeFile::parse() { .second); } - symbols.assign(obj->symbols().size(), nullptr); - for (auto it : llvm::enumerate(obj->symbols())) - symbols[it.index()] = - createBitcodeSymbol<ELFT>(keptComdats, it.value(), *this); + symbols.resize(obj->symbols().size()); + for (auto it : llvm::enumerate(obj->symbols())) { + Symbol *&sym = symbols[it.index()]; + createBitcodeSymbol<ELFT>(sym, keptComdats, it.value(), *this); + } for (auto l : obj->getDependentLibraries()) addDependentLibrary(l, this); @@ -1730,9 +1729,11 @@ template <class ELFT> void BitcodeFile::parse() { void BitcodeFile::parseLazy() { SymbolTable &symtab = *elf::symtab; - for (const lto::InputFile::Symbol &sym : obj->symbols()) - if (!sym.isUndefined()) - symtab.addSymbol(LazyObject{*this, saver.save(sym.getName())}); + symbols.resize(obj->symbols().size()); + for (auto it : llvm::enumerate(obj->symbols())) + if (!it.value().isUndefined()) + symbols[it.index()] = symtab.addSymbol( + LazyObject{*this, saver().save(it.value().getName())}); } void BinaryFile::parse() { @@ -1750,6 +1751,8 @@ void BinaryFile::parse() { if (!isAlnum(s[i])) s[i] = '_'; + llvm::StringSaver &saver = lld::saver(); + symtab->addSymbol(Defined{nullptr, saver.save(s + "_start"), STB_GLOBAL, STV_DEFAULT, STT_OBJECT, 0, 0, section}); symtab->addSymbol(Defined{nullptr, saver.save(s + "_end"), STB_GLOBAL, diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h index d622390fcade..6febea3f437d 100644 --- a/lld/ELF/InputFiles.h +++ b/lld/ELF/InputFiles.h @@ -66,7 +66,6 @@ public: enum Kind : uint8_t { ObjKind, SharedKind, - LazyObjKind, ArchiveKind, BitcodeKind, BinaryKind, @@ -287,8 +286,8 @@ private: void initializeSymbols(); void initializeJustSymbols(); - InputSectionBase *getRelocTarget(uint32_t idx, StringRef name, - const Elf_Shdr &sec); + InputSectionBase *getRelocTarget(uint32_t idx, const Elf_Shdr &sec, + uint32_t info); InputSectionBase *createInputSection(uint32_t idx, const Elf_Shdr &sec, StringRef shstrtab); @@ -408,6 +407,7 @@ inline bool isBitcode(MemoryBufferRef mb) { std::string replaceThinLTOSuffix(StringRef path); +extern SmallVector<std::unique_ptr<MemoryBuffer>> memoryBuffers; extern SmallVector<ArchiveFile *, 0> archiveFiles; extern SmallVector<BinaryFile *, 0> binaryFiles; extern SmallVector<BitcodeFile *, 0> bitcodeFiles; diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp index e1ee3def89f3..943cf18e6cf0 100644 --- a/lld/ELF/InputSection.cpp +++ b/lld/ELF/InputSection.cpp @@ -18,8 +18,7 @@ #include "SyntheticSections.h" #include "Target.h" #include "Thunks.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Compression.h" #include "llvm/Support/Endian.h" @@ -143,7 +142,7 @@ void InputSectionBase::uncompress() const { { static std::mutex mu; std::lock_guard<std::mutex> lock(mu); - uncompressedBuf = bAlloc.Allocate<char>(size); + uncompressedBuf = bAlloc().Allocate<char>(size); } if (Error e = zlib::uncompress(toStringRef(rawData), uncompressedBuf, size)) @@ -153,12 +152,6 @@ void InputSectionBase::uncompress() const { uncompressedSize = -1; } -uint64_t InputSectionBase::getOffsetInFile() const { - const uint8_t *fileStart = (const uint8_t *)file->mb.getBufferStart(); - const uint8_t *secStart = data().begin(); - return secStart - fileStart; -} - template <class ELFT> RelsOrRelas<ELFT> InputSectionBase::relsOrRelas() const { if (relSecIdx == 0) return {}; @@ -225,7 +218,8 @@ OutputSection *SectionBase::getOutputSection() { // `uncompressedSize` member and remove the header from `rawData`. template <typename ELFT> void InputSectionBase::parseCompressedHeader() { // Old-style header - if (name.startswith(".zdebug")) { + if (!(flags & SHF_COMPRESSED)) { + assert(name.startswith(".zdebug")); if (!toStringRef(rawData).startswith("ZLIB")) { error(toString(this) + ": corrupted compressed section header"); return; @@ -242,11 +236,10 @@ template <typename ELFT> void InputSectionBase::parseCompressedHeader() { // Restore the original section name. // (e.g. ".zdebug_info" -> ".debug_info") - name = saver.save("." + name.substr(2)); + name = saver().save("." + name.substr(2)); return; } - assert(flags & SHF_COMPRESSED); flags &= ~(uint64_t)SHF_COMPRESSED; // New-style header @@ -274,7 +267,6 @@ InputSection *InputSectionBase::getLinkOrderDep() const { } // Find a function symbol that encloses a given location. -template <class ELFT> Defined *InputSectionBase::getEnclosingFunction(uint64_t offset) { for (Symbol *b : file->getSymbols()) if (Defined *d = dyn_cast<Defined>(b)) @@ -285,20 +277,19 @@ Defined *InputSectionBase::getEnclosingFunction(uint64_t offset) { } // Returns an object file location string. Used to construct an error message. -template <class ELFT> std::string InputSectionBase::getLocation(uint64_t offset) { std::string secAndOffset = (name + "+0x" + Twine::utohexstr(offset) + ")").str(); // We don't have file for synthetic sections. - if (getFile<ELFT>() == nullptr) + if (file == nullptr) return (config->outputFile + ":(" + secAndOffset).str(); - std::string file = toString(getFile<ELFT>()); - if (Defined *d = getEnclosingFunction<ELFT>(offset)) - return file + ":(function " + toString(*d) + ": " + secAndOffset; + std::string filename = toString(file); + if (Defined *d = getEnclosingFunction(offset)) + return filename + ":(function " + toString(*d) + ": " + secAndOffset; - return file + ":(" + secAndOffset; + return filename + ":(" + secAndOffset; } // This function is intended to be used for constructing an error message. @@ -593,7 +584,7 @@ static Relocation *getRISCVPCRelHi20(const Symbol *sym, uint64_t addend) { InputSection *isec = cast<InputSection>(d->section); if (addend != 0) - warn("Non-zero addend in R_RISCV_PCREL_LO12 relocation to " + + warn("non-zero addend in R_RISCV_PCREL_LO12 relocation to " + isec->getObjMsg(d->value) + " is ignored"); // Relocations are sorted by offset, so we can use std::equal_range to do @@ -830,14 +821,13 @@ uint64_t InputSectionBase::getRelocTargetVA(const InputFile *file, RelType type, case R_SIZE: return sym.getSize() + a; case R_TLSDESC: - return in.got->getGlobalDynAddr(sym) + a; + return in.got->getTlsDescAddr(sym) + a; case R_TLSDESC_PC: - return in.got->getGlobalDynAddr(sym) + a - p; + return in.got->getTlsDescAddr(sym) + a - p; case R_TLSDESC_GOTPLT: - return in.got->getGlobalDynAddr(sym) + a - in.gotPlt->getVA(); + return in.got->getTlsDescAddr(sym) + a - in.gotPlt->getVA(); case R_AARCH64_TLSDESC_PAGE: - return getAArch64Page(in.got->getGlobalDynAddr(sym) + a) - - getAArch64Page(p); + return getAArch64Page(in.got->getTlsDescAddr(sym) + a) - getAArch64Page(p); case R_TLSGD_GOT: return in.got->getGlobalDynOffset(sym) + a; case R_TLSGD_GOTPLT: @@ -898,38 +888,6 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) { if (expr == R_NONE) continue; - if (expr == R_SIZE) { - target.relocateNoSym(bufLoc, type, - SignExtend64<bits>(sym.getSize() + addend)); - continue; - } - - // R_ABS/R_DTPREL and some other relocations can be used from non-SHF_ALLOC - // sections. - if (expr != R_ABS && expr != R_DTPREL && expr != R_GOTPLTREL && - expr != R_RISCV_ADD) { - std::string msg = getLocation<ELFT>(offset) + - ": has non-ABS relocation " + toString(type) + - " against symbol '" + toString(sym) + "'"; - if (expr != R_PC && expr != R_ARM_PCA) { - error(msg); - return; - } - - // If the control reaches here, we found a PC-relative relocation in a - // non-ALLOC section. Since non-ALLOC section is not loaded into memory - // at runtime, the notion of PC-relative doesn't make sense here. So, - // this is a usage error. However, GNU linkers historically accept such - // relocations without any errors and relocate them as if they were at - // address 0. For bug-compatibilty, we accept them with warnings. We - // know Steel Bank Common Lisp as of 2018 have this bug. - warn(msg); - target.relocateNoSym( - bufLoc, type, - SignExtend64<bits>(sym.getVA(addend - offset - outSecOff))); - continue; - } - if (tombstone || (isDebug && (type == target.symbolicRel || expr == R_DTPREL))) { // Resolve relocations in .debug_* referencing (discarded symbols or ICF @@ -969,7 +927,44 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) { continue; } } - target.relocateNoSym(bufLoc, type, SignExtend64<bits>(sym.getVA(addend))); + + // For a relocatable link, only tombstone values are applied. + if (config->relocatable) + continue; + + if (expr == R_SIZE) { + target.relocateNoSym(bufLoc, type, + SignExtend64<bits>(sym.getSize() + addend)); + continue; + } + + // R_ABS/R_DTPREL and some other relocations can be used from non-SHF_ALLOC + // sections. + if (expr == R_ABS || expr == R_DTPREL || expr == R_GOTPLTREL || + expr == R_RISCV_ADD) { + target.relocateNoSym(bufLoc, type, SignExtend64<bits>(sym.getVA(addend))); + continue; + } + + std::string msg = getLocation(offset) + ": has non-ABS relocation " + + toString(type) + " against symbol '" + toString(sym) + + "'"; + if (expr != R_PC && expr != R_ARM_PCA) { + error(msg); + return; + } + + // If the control reaches here, we found a PC-relative relocation in a + // non-ALLOC section. Since non-ALLOC section is not loaded into memory + // at runtime, the notion of PC-relative doesn't make sense here. So, + // this is a usage error. However, GNU linkers historically accept such + // relocations without any errors and relocate them as if they were at + // address 0. For bug-compatibilty, we accept them with warnings. We + // know Steel Bank Common Lisp as of 2018 have this bug. + warn(msg); + target.relocateNoSym( + bufLoc, type, + SignExtend64<bits>(sym.getVA(addend - offset - outSecOff))); } } @@ -1001,15 +996,15 @@ void InputSectionBase::relocate(uint8_t *buf, uint8_t *bufEnd) { } auto *sec = cast<InputSection>(this); - if (config->relocatable) { + if (config->relocatable) relocateNonAllocForRelocatable(sec, buf); - } else { - const RelsOrRelas<ELFT> rels = sec->template relsOrRelas<ELFT>(); - if (rels.areRelocsRel()) - sec->relocateNonAlloc<ELFT>(buf, rels.rels); - else - sec->relocateNonAlloc<ELFT>(buf, rels.relas); - } + // For a relocatable link, also call relocateNonAlloc() to rewrite applicable + // locations with tombstone values. + const RelsOrRelas<ELFT> rels = sec->template relsOrRelas<ELFT>(); + if (rels.areRelocsRel()) + sec->relocateNonAlloc<ELFT>(buf, rels.rels); + else + sec->relocateNonAlloc<ELFT>(buf, rels.relas); } void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) { @@ -1017,25 +1012,35 @@ void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) { const unsigned bits = config->wordsize * 8; const TargetInfo &target = *elf::target; uint64_t lastPPCRelaxedRelocOff = UINT64_C(-1); - - for (const Relocation &rel : relocations) { + AArch64Relaxer aarch64relaxer(relocations); + for (size_t i = 0, size = relocations.size(); i != size; ++i) { + const Relocation &rel = relocations[i]; if (rel.expr == R_NONE) continue; uint64_t offset = rel.offset; uint8_t *bufLoc = buf + offset; - uint64_t addrLoc = getOutputSection()->addr + offset; + uint64_t secAddr = getOutputSection()->addr; if (auto *sec = dyn_cast<InputSection>(this)) - addrLoc += sec->outSecOff; + secAddr += sec->outSecOff; + const uint64_t addrLoc = secAddr + offset; const uint64_t targetVA = SignExtend64(getRelocTargetVA(file, rel.type, rel.addend, addrLoc, - *rel.sym, rel.expr), bits); - + *rel.sym, rel.expr), + bits); switch (rel.expr) { case R_RELAX_GOT_PC: case R_RELAX_GOT_PC_NOPIC: target.relaxGot(bufLoc, rel, targetVA); break; + case R_AARCH64_GOT_PAGE_PC: + if (i + 1 < size && aarch64relaxer.tryRelaxAdrpLdr( + rel, relocations[i + 1], secAddr, buf)) { + ++i; + continue; + } + target.relocate(bufLoc, rel, targetVA); + break; case R_PPC64_RELAX_GOT_PC: { // The R_PPC64_PCREL_OPT relocation must appear immediately after // R_PPC64_GOT_PCREL34 in the relocations table at the same offset. @@ -1113,12 +1118,9 @@ void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) { // a jmp insn must be modified to shrink the jmp insn or to flip the jmp // insn. This is primarily used to relax and optimize jumps created with // basic block sections. - if (isa<InputSection>(this)) { - for (const JumpInstrMod &jumpMod : jumpInstrMods) { - uint64_t offset = jumpMod.offset; - uint8_t *bufLoc = buf + offset; - target.applyJumpInstrMod(bufLoc, jumpMod.original, jumpMod.size); - } + if (jumpInstrMod) { + target.applyJumpInstrMod(buf + jumpInstrMod->offset, jumpInstrMod->original, + jumpInstrMod->size); } } @@ -1132,7 +1134,7 @@ static void switchMorestackCallsToMorestackNonSplit( // __morestack_non_split. Symbol *moreStackNonSplit = symtab->find("__morestack_non_split"); if (!moreStackNonSplit) { - error("Mixing split-stack objects requires a definition of " + error("mixing split-stack objects requires a definition of " "__morestack_non_split"); return; } @@ -1178,11 +1180,6 @@ void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf, std::vector<Relocation *> morestackCalls; for (Relocation &rel : relocations) { - // Local symbols can't possibly be cross-calls, and should have been - // resolved long before this line. - if (rel.sym->isLocal()) - continue; - // Ignore calls into the split-stack api. if (rel.sym->getName().startswith("__morestack")) { if (rel.sym->getName().equals("__morestack")) @@ -1209,7 +1206,7 @@ void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf, if (enclosingPrologueAttempted(rel.offset, prologues)) continue; - if (Defined *f = getEnclosingFunction<ELFT>(rel.offset)) { + if (Defined *f = getEnclosingFunction(rel.offset)) { prologues.insert(f); if (target->adjustPrologueForCrossSplitStack(buf + f->value, end, f->stOther)) @@ -1227,7 +1224,7 @@ void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf, template <class ELFT> void InputSection::writeTo(uint8_t *buf) { if (auto *s = dyn_cast<SyntheticSection>(this)) { - s->writeTo(buf + outSecOff); + s->writeTo(buf); return; } @@ -1236,17 +1233,17 @@ template <class ELFT> void InputSection::writeTo(uint8_t *buf) { // If -r or --emit-relocs is given, then an InputSection // may be a relocation section. if (LLVM_UNLIKELY(type == SHT_RELA)) { - copyRelocations<ELFT>(buf + outSecOff, getDataAs<typename ELFT::Rela>()); + copyRelocations<ELFT>(buf, getDataAs<typename ELFT::Rela>()); return; } if (LLVM_UNLIKELY(type == SHT_REL)) { - copyRelocations<ELFT>(buf + outSecOff, getDataAs<typename ELFT::Rel>()); + copyRelocations<ELFT>(buf, getDataAs<typename ELFT::Rel>()); return; } // If -r is given, we may have a SHT_GROUP section. if (LLVM_UNLIKELY(type == SHT_GROUP)) { - copyShtGroup<ELFT>(buf + outSecOff); + copyShtGroup<ELFT>(buf); return; } @@ -1254,20 +1251,18 @@ template <class ELFT> void InputSection::writeTo(uint8_t *buf) { // to the buffer. if (uncompressedSize >= 0) { size_t size = uncompressedSize; - if (Error e = zlib::uncompress(toStringRef(rawData), - (char *)(buf + outSecOff), size)) + if (Error e = zlib::uncompress(toStringRef(rawData), (char *)buf, size)) fatal(toString(this) + ": uncompress failed: " + llvm::toString(std::move(e))); - uint8_t *bufEnd = buf + outSecOff + size; - relocate<ELFT>(buf + outSecOff, bufEnd); + uint8_t *bufEnd = buf + size; + relocate<ELFT>(buf, bufEnd); return; } // Copy section contents from source object file to output file // and then apply relocations. - memcpy(buf + outSecOff, data().data(), data().size()); - uint8_t *bufEnd = buf + outSecOff + data().size(); - relocate<ELFT>(buf + outSecOff, bufEnd); + memcpy(buf, rawData.data(), rawData.size()); + relocate<ELFT>(buf, buf + rawData.size()); } void InputSection::replace(InputSection *other) { @@ -1320,28 +1315,47 @@ static unsigned getReloc(IntTy begin, IntTy size, const ArrayRef<RelTy> &rels, // This function splits an input section into records and returns them. template <class ELFT> void EhInputSection::split() { const RelsOrRelas<ELFT> rels = relsOrRelas<ELFT>(); - if (rels.areRelocsRel()) - split<ELFT>(rels.rels); - else - split<ELFT>(rels.relas); + // getReloc expects the relocations to be sorted by r_offset. See the comment + // in scanRelocs. + if (rels.areRelocsRel()) { + SmallVector<typename ELFT::Rel, 0> storage; + split<ELFT>(sortRels(rels.rels, storage)); + } else { + SmallVector<typename ELFT::Rela, 0> storage; + split<ELFT>(sortRels(rels.relas, storage)); + } } template <class ELFT, class RelTy> void EhInputSection::split(ArrayRef<RelTy> rels) { - // getReloc expects the relocations to be sorted by r_offset. See the comment - // in scanRelocs. - SmallVector<RelTy, 0> storage; - rels = sortRels(rels, storage); - + ArrayRef<uint8_t> d = rawData; + const char *msg = nullptr; unsigned relI = 0; - for (size_t off = 0, end = data().size(); off != end;) { - size_t size = readEhRecordSize(this, off); - pieces.emplace_back(off, this, size, getReloc(off, size, rels, relI)); - // The empty record is the end marker. - if (size == 4) + while (!d.empty()) { + if (d.size() < 4) { + msg = "CIE/FDE too small"; break; - off += size; + } + uint64_t size = endian::read32<ELFT::TargetEndianness>(d.data()); + // If it is 0xFFFFFFFF, the next 8 bytes contain the size instead, + // but we do not support that format yet. + if (size == UINT32_MAX) { + msg = "CIE/FDE too large"; + break; + } + size += 4; + if (size > d.size()) { + msg = "CIE/FDE ends past the end of the section"; + break; + } + + uint64_t off = d.data() - rawData.data(); + pieces.emplace_back(off, this, size, getReloc(off, size, rels, relI)); + d = d.slice(size); } + if (msg) + errorOrWarn("corrupted .eh_frame: " + Twine(msg) + "\n>>> defined in " + + getObjMsg(d.data() - rawData.data())); } static size_t findNull(StringRef s, size_t entSize) { @@ -1451,11 +1465,6 @@ template InputSection::InputSection(ObjFile<ELF64LE> &, const ELF64LE::Shdr &, template InputSection::InputSection(ObjFile<ELF64BE> &, const ELF64BE::Shdr &, StringRef); -template std::string InputSectionBase::getLocation<ELF32LE>(uint64_t); -template std::string InputSectionBase::getLocation<ELF32BE>(uint64_t); -template std::string InputSectionBase::getLocation<ELF64LE>(uint64_t); -template std::string InputSectionBase::getLocation<ELF64BE>(uint64_t); - template void InputSection::writeTo<ELF32LE>(uint8_t *); template void InputSection::writeTo<ELF32BE>(uint8_t *); template void InputSection::writeTo<ELF64LE>(uint8_t *); diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h index 5319830b5d80..d7dea9d2587a 100644 --- a/lld/ELF/InputSection.h +++ b/lld/ELF/InputSection.h @@ -130,13 +130,16 @@ public: // one or two jump instructions at the end that could be relaxed to a smaller // instruction. The members below help trimming the trailing jump instruction // and shrinking a section. - unsigned bytesDropped = 0; + uint8_t bytesDropped = 0; // Whether the section needs to be padded with a NOP filler due to // deleteFallThruJmpInsn. bool nopFiller = false; - void drop_back(uint64_t num) { bytesDropped += num; } + void drop_back(unsigned num) { + assert(bytesDropped + num < 256); + bytesDropped += num; + } void push_back(uint64_t num) { assert(bytesDropped >= num); @@ -156,8 +159,6 @@ public: return rawData; } - uint64_t getOffsetInFile() const; - // Input sections are part of an output section. Special sections // like .eh_frame and merge sections are first combined into a // synthetic section that is then added to an output section. In all @@ -180,11 +181,10 @@ public: // Get the function symbol that encloses this offset from within the // section. - template <class ELFT> Defined *getEnclosingFunction(uint64_t offset); // Returns a source location string. Used to construct an error message. - template <class ELFT> std::string getLocation(uint64_t offset); + std::string getLocation(uint64_t offset); std::string getSrcMsg(const Symbol &sym, uint64_t offset); std::string getObjMsg(uint64_t offset); @@ -206,7 +206,7 @@ public: // block sections are enabled. Basic block sections creates opportunities to // relax jump instructions at basic block boundaries after reordering the // basic blocks. - SmallVector<JumpInstrMod, 0> jumpInstrMods; + JumpInstrMod *jumpInstrMod = nullptr; // A function compiled with -fsplit-stack calling a function // compiled without -fsplit-stack needs its prologue adjusted. Find @@ -324,7 +324,7 @@ public: // Splittable sections are handled as a sequence of data // rather than a single large blob of data. - std::vector<EhSectionPiece> pieces; + SmallVector<EhSectionPiece, 0> pieces; SyntheticSection *getParent() const; }; @@ -379,11 +379,7 @@ private: template <class ELFT> void copyShtGroup(uint8_t *buf); }; -#ifdef _WIN32 -static_assert(sizeof(InputSection) <= 184, "InputSection is too big"); -#else -static_assert(sizeof(InputSection) <= 176, "InputSection is too big"); -#endif +static_assert(sizeof(InputSection) <= 160, "InputSection is too big"); inline bool isDebugSection(const InputSectionBase &sec) { return (sec.flags & llvm::ELF::SHF_ALLOC) == 0 && diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index 65b943c4a54c..5b7ac6a5e925 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -207,6 +207,8 @@ BitcodeCompiler::BitcodeCompiler() { if (bitcodeFiles.empty()) return; for (Symbol *sym : symtab->symbols()) { + if (sym->isPlaceholder()) + continue; StringRef s = sym->getName(); for (StringRef prefix : {"__start_", "__stop_"}) if (s.startswith(prefix)) diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp index e8f2ce4fdf1f..bfb583453735 100644 --- a/lld/ELF/LinkerScript.cpp +++ b/lld/ELF/LinkerScript.cpp @@ -19,7 +19,7 @@ #include "SyntheticSections.h" #include "Target.h" #include "Writer.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Strings.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" @@ -64,8 +64,8 @@ static StringRef getOutputSectionName(const InputSectionBase *s) { if (InputSectionBase *rel = isec->getRelocatedSection()) { OutputSection *out = rel->getOutputSection(); if (s->type == SHT_RELA) - return saver.save(".rela" + out->name); - return saver.save(".rel" + out->name); + return saver().save(".rela" + out->name); + return saver().save(".rel" + out->name); } } @@ -135,7 +135,7 @@ uint64_t ExprValue::getSectionOffset() const { OutputSection *LinkerScript::createOutputSection(StringRef name, StringRef location) { - OutputSection *&secRef = nameToOutputSection[name]; + OutputSection *&secRef = nameToOutputSection[CachedHashStringRef(name)]; OutputSection *sec; if (secRef && secRef->location.empty()) { // There was a forward reference. @@ -150,7 +150,7 @@ OutputSection *LinkerScript::createOutputSection(StringRef name, } OutputSection *LinkerScript::getOrCreateOutputSection(StringRef name) { - OutputSection *&cmdRef = nameToOutputSection[name]; + OutputSection *&cmdRef = nameToOutputSection[CachedHashStringRef(name)]; if (!cmdRef) cmdRef = make<OutputSection>(name, SHT_PROGBITS, 0); return cmdRef; @@ -270,8 +270,8 @@ using SymbolAssignmentMap = // Collect section/value pairs of linker-script-defined symbols. This is used to // check whether symbol values converge. -static SymbolAssignmentMap getSymbolAssignmentValues( - const std::vector<SectionCommand *> §ionCommands) { +static SymbolAssignmentMap +getSymbolAssignmentValues(ArrayRef<SectionCommand *> sectionCommands) { SymbolAssignmentMap ret; for (SectionCommand *cmd : sectionCommands) { if (auto *assign = dyn_cast<SymbolAssignment>(cmd)) { @@ -306,7 +306,7 @@ getChangedSymbolAssignment(const SymbolAssignmentMap &oldValues) { // Process INSERT [AFTER|BEFORE] commands. For each command, we move the // specified output section to the designated place. void LinkerScript::processInsertCommands() { - std::vector<OutputSection *> moves; + SmallVector<OutputSection *, 0> moves; for (const InsertCommand &cmd : insertCommands) { for (StringRef name : cmd.names) { // If base is empty, it may have been discarded by @@ -486,11 +486,11 @@ static void sortInputSections(MutableArrayRef<InputSectionBase *> vec, } // Compute and remember which sections the InputSectionDescription matches. -std::vector<InputSectionBase *> +SmallVector<InputSectionBase *, 0> LinkerScript::computeInputSections(const InputSectionDescription *cmd, ArrayRef<InputSectionBase *> sections) { - std::vector<InputSectionBase *> ret; - std::vector<size_t> indexes; + SmallVector<InputSectionBase *, 0> ret; + SmallVector<size_t, 0> indexes; DenseSet<size_t> seen; auto sortByPositionThenCommandLine = [&](size_t begin, size_t end) { llvm::sort(MutableArrayRef<size_t>(indexes).slice(begin, end - begin)); @@ -561,17 +561,9 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd, } void LinkerScript::discard(InputSectionBase &s) { - if (&s == in.shStrTab || &s == mainPart->relrDyn) + if (&s == in.shStrTab.get()) error("discarding " + s.name + " section is not allowed"); - // You can discard .hash and .gnu.hash sections by linker scripts. Since - // they are synthesized sections, we need to handle them differently than - // other regular sections. - if (&s == mainPart->gnuHashTab) - mainPart->gnuHashTab = nullptr; - if (&s == mainPart->hashTab) - mainPart->hashTab = nullptr; - s.markDead(); s.parent = nullptr; for (InputSection *sec : s.dependentSections) @@ -582,21 +574,19 @@ void LinkerScript::discardSynthetic(OutputSection &outCmd) { for (Partition &part : partitions) { if (!part.armExidx || !part.armExidx->isLive()) continue; - std::vector<InputSectionBase *> secs(part.armExidx->exidxSections.begin(), - part.armExidx->exidxSections.end()); + SmallVector<InputSectionBase *, 0> secs( + part.armExidx->exidxSections.begin(), + part.armExidx->exidxSections.end()); for (SectionCommand *cmd : outCmd.commands) - if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) { - std::vector<InputSectionBase *> matches = - computeInputSections(isd, secs); - for (InputSectionBase *s : matches) + if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) + for (InputSectionBase *s : computeInputSections(isd, secs)) discard(*s); - } } } -std::vector<InputSectionBase *> +SmallVector<InputSectionBase *, 0> LinkerScript::createInputSectionList(OutputSection &outCmd) { - std::vector<InputSectionBase *> ret; + SmallVector<InputSectionBase *, 0> ret; for (SectionCommand *cmd : outCmd.commands) { if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) { @@ -612,7 +602,7 @@ LinkerScript::createInputSectionList(OutputSection &outCmd) { // Create output sections described by SECTIONS commands. void LinkerScript::processSectionCommands() { auto process = [this](OutputSection *osec) { - std::vector<InputSectionBase *> v = createInputSectionList(*osec); + SmallVector<InputSectionBase *, 0> v = createInputSectionList(*osec); // The output section name `/DISCARD/' is special. // Any input section assigned to it is discarded. @@ -656,14 +646,16 @@ void LinkerScript::processSectionCommands() { // Process OVERWRITE_SECTIONS first so that it can overwrite the main script // or orphans. - DenseMap<StringRef, OutputSection *> map; + DenseMap<CachedHashStringRef, OutputSection *> map; size_t i = 0; for (OutputSection *osec : overwriteSections) - if (process(osec) && !map.try_emplace(osec->name, osec).second) + if (process(osec) && + !map.try_emplace(CachedHashStringRef(osec->name), osec).second) warn("OVERWRITE_SECTIONS specifies duplicate " + osec->name); for (SectionCommand *&base : sectionCommands) if (auto *osec = dyn_cast<OutputSection>(base)) { - if (OutputSection *overwrite = map.lookup(osec->name)) { + if (OutputSection *overwrite = + map.lookup(CachedHashStringRef(osec->name))) { log(overwrite->location + " overwrites " + osec->name); overwrite->sectionIndex = i++; base = overwrite; @@ -830,10 +822,9 @@ addInputSec(StringMap<TinyPtrVector<OutputSection *>> &map, // Add sections that didn't match any sections command. void LinkerScript::addOrphanSections() { StringMap<TinyPtrVector<OutputSection *>> map; - std::vector<OutputSection *> v; + SmallVector<OutputSection *, 0> v; - std::function<void(InputSectionBase *)> add; - add = [&](InputSectionBase *s) { + auto add = [&](InputSectionBase *s) { if (s->isLive() && !s->parent) { orphanSections.push_back(s); @@ -849,11 +840,6 @@ void LinkerScript::addOrphanSections() { s->getOutputSection()->sectionIndex == UINT32_MAX); } } - - if (config->relocatable) - for (InputSectionBase *depSec : s->dependentSections) - if (depSec->flags & SHF_LINK_ORDER) - add(depSec); }; // For further --emit-reloc handling code we need target output section @@ -872,6 +858,10 @@ void LinkerScript::addOrphanSections() { if (auto *relIS = dyn_cast_or_null<InputSectionBase>(rel->parent)) add(relIS); add(isec); + if (config->relocatable) + for (InputSectionBase *depSec : isec->dependentSections) + if (depSec->flags & SHF_LINK_ORDER) + add(depSec); } // If no SECTIONS command was given, we should insert sections commands @@ -1113,7 +1103,7 @@ bool LinkerScript::isDiscarded(const OutputSection *sec) const { } static void maybePropagatePhdrs(OutputSection &sec, - std::vector<StringRef> &phdrs) { + SmallVector<StringRef, 0> &phdrs) { if (sec.phdrs.empty()) { // To match the bfd linker script behaviour, only propagate program // headers to sections that are allocated. @@ -1147,7 +1137,7 @@ void LinkerScript::adjustSectionsBeforeSorting() { // the previous sections. Only a few flags are needed to keep the impact low. uint64_t flags = SHF_ALLOC; - std::vector<StringRef> defPhdrs; + SmallVector<StringRef, 0> defPhdrs; for (SectionCommand *&cmd : sectionCommands) { auto *sec = dyn_cast<OutputSection>(cmd); if (!sec) @@ -1218,7 +1208,7 @@ void LinkerScript::adjustSectionsAfterSorting() { // Below is an example of such linker script: // PHDRS { seg PT_LOAD; } // SECTIONS { .aaa : { *(.aaa) } } - std::vector<StringRef> defPhdrs; + SmallVector<StringRef, 0> defPhdrs; auto firstPtLoad = llvm::find_if(phdrsCommands, [](const PhdrsCommand &cmd) { return cmd.type == PT_LOAD; }); @@ -1248,7 +1238,7 @@ static uint64_t computeBase(uint64_t min, bool allocateHeaders) { // We check if the headers fit below the first allocated section. If there isn't // enough space for these sections, we'll remove them from the PT_LOAD segment, // and we'll also remove the PT_PHDR segment. -void LinkerScript::allocateHeaders(std::vector<PhdrEntry *> &phdrs) { +void LinkerScript::allocateHeaders(SmallVector<PhdrEntry *, 0> &phdrs) { uint64_t min = std::numeric_limits<uint64_t>::max(); for (OutputSection *sec : outputSections) if (sec->flags & SHF_ALLOC) @@ -1332,8 +1322,8 @@ const Defined *LinkerScript::assignAddresses() { } // Creates program headers as instructed by PHDRS linker script command. -std::vector<PhdrEntry *> LinkerScript::createPhdrs() { - std::vector<PhdrEntry *> ret; +SmallVector<PhdrEntry *, 0> LinkerScript::createPhdrs() { + SmallVector<PhdrEntry *, 0> ret; // Process PHDRS and FILEHDR keywords because they are not // real output sections and cannot be added in the following loop. @@ -1415,8 +1405,8 @@ static Optional<size_t> getPhdrIndex(ArrayRef<PhdrsCommand> vec, // Returns indices of ELF headers containing specific section. Each index is a // zero based number of ELF header listed within PHDRS {} script block. -std::vector<size_t> LinkerScript::getPhdrIndices(OutputSection *cmd) { - std::vector<size_t> ret; +SmallVector<size_t, 0> LinkerScript::getPhdrIndices(OutputSection *cmd) { + SmallVector<size_t, 0> ret; for (StringRef s : cmd->phdrs) { if (Optional<size_t> idx = getPhdrIndex(phdrsCommands, s)) diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h index f385c8320978..d2a6f5e9acb1 100644 --- a/lld/ELF/LinkerScript.h +++ b/lld/ELF/LinkerScript.h @@ -203,20 +203,20 @@ public: // Input sections that matches at least one of SectionPatterns // will be associated with this InputSectionDescription. - std::vector<SectionPattern> sectionPatterns; + SmallVector<SectionPattern, 0> sectionPatterns; // Includes InputSections and MergeInputSections. Used temporarily during // assignment of input sections to output sections. - std::vector<InputSectionBase *> sectionBases; + SmallVector<InputSectionBase *, 0> sectionBases; // Used after the finalizeInputSections() pass. MergeInputSections have been // merged into MergeSyntheticSections. - std::vector<InputSection *> sections; + SmallVector<InputSection *, 0> sections; // Temporary record of synthetic ThunkSection instances and the pass that // they were created in. This is used to insert newly created ThunkSections // into Sections at the end of a createThunks() pass. - std::vector<std::pair<ThunkSection *, uint32_t>> thunkSections; + SmallVector<std::pair<ThunkSection *, uint32_t>, 0> thunkSections; // SectionPatterns can be filtered with the INPUT_SECTION_FLAGS command. uint64_t withFlags; @@ -244,7 +244,7 @@ struct ByteCommand : SectionCommand { }; struct InsertCommand { - std::vector<StringRef> names; + SmallVector<StringRef, 0> names; bool isAfter; StringRef where; }; @@ -271,7 +271,8 @@ class LinkerScript final { uint64_t tbssAddr = 0; }; - llvm::DenseMap<StringRef, OutputSection *> nameToOutputSection; + llvm::DenseMap<llvm::CachedHashStringRef, OutputSection *> + nameToOutputSection; void addSymbol(SymbolAssignment *cmd); void assignSymbol(SymbolAssignment *cmd, bool inSec); @@ -279,15 +280,15 @@ class LinkerScript final { void expandOutputSection(uint64_t size); void expandMemoryRegions(uint64_t size); - std::vector<InputSectionBase *> + SmallVector<InputSectionBase *, 0> computeInputSections(const InputSectionDescription *, ArrayRef<InputSectionBase *>); - std::vector<InputSectionBase *> createInputSectionList(OutputSection &cmd); + SmallVector<InputSectionBase *, 0> createInputSectionList(OutputSection &cmd); void discardSynthetic(OutputSection &); - std::vector<size_t> getPhdrIndices(OutputSection *sec); + SmallVector<size_t, 0> getPhdrIndices(OutputSection *sec); std::pair<MemoryRegion *, MemoryRegion *> findMemoryRegion(OutputSection *sec, MemoryRegion *hint); @@ -321,12 +322,12 @@ public: void adjustSectionsBeforeSorting(); void adjustSectionsAfterSorting(); - std::vector<PhdrEntry *> createPhdrs(); + SmallVector<PhdrEntry *, 0> createPhdrs(); bool needsInterpSection(); bool shouldKeep(InputSectionBase *s); const Defined *assignAddresses(); - void allocateHeaders(std::vector<PhdrEntry *> &phdrs); + void allocateHeaders(SmallVector<PhdrEntry *, 0> &phdrs); void processSectionCommands(); void processSymbolAssignments(); void declareSymbols(); @@ -337,33 +338,33 @@ public: void processInsertCommands(); // SECTIONS command list. - std::vector<SectionCommand *> sectionCommands; + SmallVector<SectionCommand *, 0> sectionCommands; // PHDRS command list. - std::vector<PhdrsCommand> phdrsCommands; + SmallVector<PhdrsCommand, 0> phdrsCommands; bool hasSectionsCommand = false; bool errorOnMissingSection = false; // List of section patterns specified with KEEP commands. They will // be kept even if they are unused and --gc-sections is specified. - std::vector<InputSectionDescription *> keptSections; + SmallVector<InputSectionDescription *, 0> keptSections; // A map from memory region name to a memory region descriptor. llvm::MapVector<llvm::StringRef, MemoryRegion *> memoryRegions; // A list of symbols referenced by the script. - std::vector<llvm::StringRef> referencedSymbols; + SmallVector<llvm::StringRef, 0> referencedSymbols; // Used to implement INSERT [AFTER|BEFORE]. Contains output sections that need // to be reordered. - std::vector<InsertCommand> insertCommands; + SmallVector<InsertCommand, 0> insertCommands; // OutputSections specified by OVERWRITE_SECTIONS. - std::vector<OutputSection *> overwriteSections; + SmallVector<OutputSection *, 0> overwriteSections; // Sections that will be warned/errored by --orphan-handling. - std::vector<const InputSectionBase *> orphanSections; + SmallVector<const InputSectionBase *, 0> orphanSections; }; extern std::unique_ptr<LinkerScript> script; diff --git a/lld/ELF/MapFile.cpp b/lld/ELF/MapFile.cpp index 1998192bfba6..c7d8967358ad 100644 --- a/lld/ELF/MapFile.cpp +++ b/lld/ELF/MapFile.cpp @@ -37,7 +37,8 @@ using namespace llvm::object; using namespace lld; using namespace lld::elf; -using SymbolMapTy = DenseMap<const SectionBase *, SmallVector<Defined *, 4>>; +using SymbolMapTy = DenseMap<const SectionBase *, + SmallVector<std::pair<Defined *, uint64_t>, 0>>; static constexpr char indent8[] = " "; // 8 spaces static constexpr char indent16[] = " "; // 16 spaces @@ -67,7 +68,7 @@ static std::vector<Defined *> getSymbols() { static SymbolMapTy getSectionSyms(ArrayRef<Defined *> syms) { SymbolMapTy ret; for (Defined *dr : syms) - ret[dr->section].push_back(dr); + ret[dr->section].emplace_back(dr, dr->getVA()); // Sort symbols by address. We want to print out symbols in the // order in the output file rather than the order they appeared @@ -76,12 +77,11 @@ static SymbolMapTy getSectionSyms(ArrayRef<Defined *> syms) { for (auto &it : ret) { // Deduplicate symbols which need a canonical PLT entry/copy relocation. set.clear(); - llvm::erase_if(it.second, - [&](Defined *sym) { return !set.insert(sym).second; }); - - llvm::stable_sort(it.second, [](Defined *a, Defined *b) { - return a->getVA() < b->getVA(); + llvm::erase_if(it.second, [&](std::pair<Defined *, uint64_t> a) { + return !set.insert(a.first).second; }); + + llvm::stable_sort(it.second, llvm::less_second()); } return ret; } @@ -91,9 +91,9 @@ static SymbolMapTy getSectionSyms(ArrayRef<Defined *> syms) { // we do that in batch using parallel-for. static DenseMap<Symbol *, std::string> getSymbolStrings(ArrayRef<Defined *> syms) { - std::vector<std::string> str(syms.size()); + auto strs = std::make_unique<std::string[]>(syms.size()); parallelForEachN(0, syms.size(), [&](size_t i) { - raw_string_ostream os(str[i]); + raw_string_ostream os(strs[i]); OutputSection *osec = syms[i]->getOutputSection(); uint64_t vma = syms[i]->getVA(); uint64_t lma = osec ? osec->getLMA() + vma - osec->getVA(0) : 0; @@ -103,7 +103,7 @@ getSymbolStrings(ArrayRef<Defined *> syms) { DenseMap<Symbol *, std::string> ret; for (size_t i = 0, e = syms.size(); i < e; ++i) - ret[syms[i]] = std::move(str[i]); + ret[syms[i]] = std::move(strs[i]); return ret; } @@ -184,7 +184,7 @@ static void writeMapFile(raw_fd_ostream &os) { writeHeader(os, isec->getVA(), osec->getLMA() + isec->outSecOff, isec->getSize(), isec->alignment); os << indent8 << toString(isec) << '\n'; - for (Symbol *sym : sectionSyms[isec]) + for (Symbol *sym : llvm::make_first_range(sectionSyms[isec])) os << symStr[sym] << '\n'; } continue; diff --git a/lld/ELF/MarkLive.cpp b/lld/ELF/MarkLive.cpp index b63f2beb9dcb..597c0684b8b2 100644 --- a/lld/ELF/MarkLive.cpp +++ b/lld/ELF/MarkLive.cpp @@ -27,7 +27,7 @@ #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Strings.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Object/ELF.h" @@ -68,8 +68,8 @@ private: SmallVector<InputSection *, 0> queue; // There are normally few input sections whose names are valid C - // identifiers, so we just store a std::vector instead of a multimap. - DenseMap<StringRef, std::vector<InputSectionBase *>> cNamedSections; + // identifiers, so we just store a SmallVector instead of a multimap. + DenseMap<StringRef, SmallVector<InputSectionBase *, 0>> cNamedSections; }; } // namespace @@ -177,11 +177,12 @@ static bool isReserved(InputSectionBase *sec) { // SHT_NOTE sections in a group are subject to garbage collection. return !sec->nextInSectionGroup; default: - // Support SHT_PROGBITS .init_array for a while - // (https://golang.org/issue/50295). + // Support SHT_PROGBITS .init_array (https://golang.org/issue/50295) and + // .init_array.N (https://github.com/rust-lang/rust/issues/92181) for a + // while. StringRef s = sec->name; - return s == ".init" || s == ".fini" || s == ".init_array" || s == ".jcr" || - s.startswith(".ctors") || s.startswith(".dtors"); + return s == ".init" || s == ".fini" || s.startswith(".init_array") || + s == ".jcr" || s.startswith(".ctors") || s.startswith(".dtors"); } } @@ -307,8 +308,8 @@ template <class ELFT> void MarkLive<ELFT>::run() { // As a workaround for glibc libc.a before 2.34 // (https://sourceware.org/PR27492), retain __libc_atexit and similar // sections regardless of zStartStopGC. - cNamedSections[saver.save("__start_" + sec->name)].push_back(sec); - cNamedSections[saver.save("__stop_" + sec->name)].push_back(sec); + cNamedSections[saver().save("__start_" + sec->name)].push_back(sec); + cNamedSections[saver().save("__stop_" + sec->name)].push_back(sec); } } diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td index bddf13a3cb42..ca9fdcde791f 100644 --- a/lld/ELF/Options.td +++ b/lld/ELF/Options.td @@ -119,10 +119,10 @@ defm call_graph_profile_sort: BB<"call-graph-profile-sort", // --chroot doesn't have a help text because it is an internal option. def chroot: Separate<["--"], "chroot">; -defm color_diagnostics: B<"color-diagnostics", +defm color_diagnostics: BB<"color-diagnostics", "Alias for --color-diagnostics=always", "Alias for --color-diagnostics=never">; -def color_diagnostics_eq: J<"color-diagnostics=">, +def color_diagnostics_eq: JJ<"color-diagnostics=">, HelpText<"Use colors in diagnostics (default: auto)">, MetaVarName<"[auto,always,never]">; @@ -344,10 +344,10 @@ defm print_symbol_order: Eq<"print-symbol-order", "Print a symbol order specified by --call-graph-ordering-file into the specified file">; def pop_state: F<"pop-state">, - HelpText<"Undo the effect of -push-state">; + HelpText<"Restore the states saved by --push-state">; def push_state: F<"push-state">, - HelpText<"Save the current state of -as-needed, -static and -whole-archive">; + HelpText<"Save the current state of --as-needed, -static and --whole-archive">; def print_map: F<"print-map">, HelpText<"Print a link map to the standard output">; @@ -395,7 +395,7 @@ def strip_all: F<"strip-all">, HelpText<"Strip all symbols. Implies --strip-debu def strip_debug: F<"strip-debug">, HelpText<"Strip debugging information">; defm symbol_ordering_file: - Eq<"symbol-ordering-file", "Layout sections to place symbols in the order specified by symbol ordering file">; + EEq<"symbol-ordering-file", "Layout sections to place symbols in the order specified by symbol ordering file">; defm sysroot: Eq<"sysroot", "Set the system root">; @@ -445,7 +445,7 @@ defm undefined_version: B<"undefined-version", "Allow unused version in version script (default)", "Report version scripts that refer undefined symbols">; -defm rsp_quoting: Eq<"rsp-quoting", "Quoting style for response files">, +defm rsp_quoting: EEq<"rsp-quoting", "Quoting style for response files">, MetaVarName<"[posix,windows]">; def v: Flag<["-"], "v">, HelpText<"Display the version number">; diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp index 4a03ac387814..c73d6e439238 100644 --- a/lld/ELF/OutputSections.cpp +++ b/lld/ELF/OutputSections.cpp @@ -15,7 +15,7 @@ #include "lld/Common/Memory.h" #include "lld/Common/Strings.h" #include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/Support/Compression.h" +#include "llvm/Config/config.h" // LLVM_ENABLE_ZLIB #include "llvm/Support/MD5.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Parallel.h" @@ -23,6 +23,9 @@ #include "llvm/Support/TimeProfiler.h" #include <regex> #include <unordered_set> +#if LLVM_ENABLE_ZLIB +#include <zlib.h> +#endif using namespace llvm; using namespace llvm::dwarf; @@ -284,38 +287,94 @@ static void fill(uint8_t *buf, size_t size, memcpy(buf + i, filler.data(), size - i); } +#if LLVM_ENABLE_ZLIB +static SmallVector<uint8_t, 0> deflateShard(ArrayRef<uint8_t> in, int level, + int flush) { + // 15 and 8 are default. windowBits=-15 is negative to generate raw deflate + // data with no zlib header or trailer. + z_stream s = {}; + deflateInit2(&s, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + s.next_in = const_cast<uint8_t *>(in.data()); + s.avail_in = in.size(); + + // Allocate a buffer of half of the input size, and grow it by 1.5x if + // insufficient. + SmallVector<uint8_t, 0> out; + size_t pos = 0; + out.resize_for_overwrite(std::max<size_t>(in.size() / 2, 64)); + do { + if (pos == out.size()) + out.resize_for_overwrite(out.size() * 3 / 2); + s.next_out = out.data() + pos; + s.avail_out = out.size() - pos; + (void)deflate(&s, flush); + pos = s.next_out - out.data(); + } while (s.avail_out == 0); + assert(s.avail_in == 0); + + out.truncate(pos); + deflateEnd(&s); + return out; +} +#endif + // Compress section contents if this section contains debug info. template <class ELFT> void OutputSection::maybeCompress() { +#if LLVM_ENABLE_ZLIB using Elf_Chdr = typename ELFT::Chdr; // Compress only DWARF debug sections. if (!config->compressDebugSections || (flags & SHF_ALLOC) || - !name.startswith(".debug_")) + !name.startswith(".debug_") || size == 0) return; llvm::TimeTraceScope timeScope("Compress debug sections"); - // Create a section header. - zDebugHeader.resize(sizeof(Elf_Chdr)); - auto *hdr = reinterpret_cast<Elf_Chdr *>(zDebugHeader.data()); - hdr->ch_type = ELFCOMPRESS_ZLIB; - hdr->ch_size = size; - hdr->ch_addralign = alignment; + // Write uncompressed data to a temporary zero-initialized buffer. + auto buf = std::make_unique<uint8_t[]>(size); + writeTo<ELFT>(buf.get()); + // We chose 1 (Z_BEST_SPEED) as the default compression level because it is + // the fastest. If -O2 is given, we use level 6 to compress debug info more by + // ~15%. We found that level 7 to 9 doesn't make much difference (~1% more + // compression) while they take significant amount of time (~2x), so level 6 + // seems enough. + const int level = config->optimize >= 2 ? 6 : Z_BEST_SPEED; - // Write section contents to a temporary buffer and compress it. - std::vector<uint8_t> buf(size); - writeTo<ELFT>(buf.data()); - // We chose 1 as the default compression level because it is the fastest. If - // -O2 is given, we use level 6 to compress debug info more by ~15%. We found - // that level 7 to 9 doesn't make much difference (~1% more compression) while - // they take significant amount of time (~2x), so level 6 seems enough. - if (Error e = zlib::compress(toStringRef(buf), compressedData, - config->optimize >= 2 ? 6 : 1)) - fatal("compress failed: " + llvm::toString(std::move(e))); + // Split input into 1-MiB shards. + constexpr size_t shardSize = 1 << 20; + const size_t numShards = (size + shardSize - 1) / shardSize; + auto shardsIn = std::make_unique<ArrayRef<uint8_t>[]>(numShards); + for (size_t i = 0, start = 0, end; start != size; ++i, start = end) { + end = std::min(start + shardSize, (size_t)size); + shardsIn[i] = makeArrayRef<uint8_t>(buf.get() + start, end - start); + } + + // Compress shards and compute Alder-32 checksums. Use Z_SYNC_FLUSH for all + // shards but the last to flush the output to a byte boundary to be + // concatenated with the next shard. + auto shardsOut = std::make_unique<SmallVector<uint8_t, 0>[]>(numShards); + auto shardsAdler = std::make_unique<uint32_t[]>(numShards); + parallelForEachN(0, numShards, [&](size_t i) { + shardsOut[i] = deflateShard(shardsIn[i], level, + i != numShards - 1 ? Z_SYNC_FLUSH : Z_FINISH); + shardsAdler[i] = adler32(1, shardsIn[i].data(), shardsIn[i].size()); + }); - // Update section headers. - size = sizeof(Elf_Chdr) + compressedData.size(); + // Update section size and combine Alder-32 checksums. + uint32_t checksum = 1; // Initial Adler-32 value + compressed.uncompressedSize = size; + size = sizeof(Elf_Chdr) + 2; // Elf_Chdir and zlib header + for (size_t i = 0; i != numShards; ++i) { + size += shardsOut[i].size(); + checksum = adler32_combine(checksum, shardsAdler[i], shardsIn[i].size()); + } + size += 4; // checksum + + compressed.shards = std::move(shardsOut); + compressed.numShards = numShards; + compressed.checksum = checksum; flags |= SHF_COMPRESSED; +#endif } static void writeInt(uint8_t *buf, uint64_t data, uint64_t size) { @@ -339,15 +398,32 @@ template <class ELFT> void OutputSection::writeTo(uint8_t *buf) { // If --compress-debug-section is specified and if this is a debug section, // we've already compressed section contents. If that's the case, // just write it down. - if (!compressedData.empty()) { - memcpy(buf, zDebugHeader.data(), zDebugHeader.size()); - memcpy(buf + zDebugHeader.size(), compressedData.data(), - compressedData.size()); + if (compressed.shards) { + auto *chdr = reinterpret_cast<typename ELFT::Chdr *>(buf); + chdr->ch_type = ELFCOMPRESS_ZLIB; + chdr->ch_size = compressed.uncompressedSize; + chdr->ch_addralign = alignment; + buf += sizeof(*chdr); + + // Compute shard offsets. + auto offsets = std::make_unique<size_t[]>(compressed.numShards); + offsets[0] = 2; // zlib header + for (size_t i = 1; i != compressed.numShards; ++i) + offsets[i] = offsets[i - 1] + compressed.shards[i - 1].size(); + + buf[0] = 0x78; // CMF + buf[1] = 0x01; // FLG: best speed + parallelForEachN(0, compressed.numShards, [&](size_t i) { + memcpy(buf + offsets[i], compressed.shards[i].data(), + compressed.shards[i].size()); + }); + + write32be(buf + (size - sizeof(*chdr) - 4), compressed.checksum); return; } // Write leading padding. - std::vector<InputSection *> sections = getInputSections(this); + SmallVector<InputSection *, 0> sections = getInputSections(*this); std::array<uint8_t, 4> filler = getFiller(); bool nonZeroFiller = read32(filler.data()) != 0; if (nonZeroFiller) @@ -355,7 +431,7 @@ template <class ELFT> void OutputSection::writeTo(uint8_t *buf) { parallelForEachN(0, sections.size(), [&](size_t i) { InputSection *isec = sections[i]; - isec->writeTo<ELFT>(buf); + isec->writeTo<ELFT>(buf + isec->outSecOff); // Fill gaps between sections. if (nonZeroFiller) { @@ -520,9 +596,9 @@ InputSection *elf::getFirstInputSection(const OutputSection *os) { return nullptr; } -std::vector<InputSection *> elf::getInputSections(const OutputSection *os) { - std::vector<InputSection *> ret; - for (SectionCommand *cmd : os->commands) +SmallVector<InputSection *, 0> elf::getInputSections(const OutputSection &os) { + SmallVector<InputSection *, 0> ret; + for (SectionCommand *cmd : os.commands) if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) ret.insert(ret.end(), isd->sections.begin(), isd->sections.end()); return ret; @@ -550,7 +626,7 @@ std::array<uint8_t, 4> OutputSection::getFiller() { void OutputSection::checkDynRelAddends(const uint8_t *bufStart) { assert(config->writeAddends && config->checkDynamicRelocs); assert(type == SHT_REL || type == SHT_RELA); - std::vector<InputSection *> sections = getInputSections(this); + SmallVector<InputSection *, 0> sections = getInputSections(*this); parallelForEachN(0, sections.size(), [&](size_t i) { // When linking with -r or --emit-relocs we might also call this function // for input .rel[a].<sec> sections which we simply pass through to the diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h index fb3eb0059909..ad656e49ceff 100644 --- a/lld/ELF/OutputSections.h +++ b/lld/ELF/OutputSections.h @@ -25,6 +25,13 @@ struct PhdrEntry; class InputSection; class InputSectionBase; +struct CompressedData { + std::unique_ptr<SmallVector<uint8_t, 0>[]> shards; + uint32_t numShards = 0; + uint32_t checksum = 0; + uint64_t uncompressedSize; +}; + // This represents a section in an output file. // It is composed of multiple InputSections. // The writer creates multiple OutputSections and assign them unique, @@ -82,8 +89,8 @@ public: Expr alignExpr; Expr lmaExpr; Expr subalignExpr; - std::vector<SectionCommand *> commands; - std::vector<StringRef> phdrs; + SmallVector<SectionCommand *, 0> commands; + SmallVector<StringRef, 0> phdrs; llvm::Optional<std::array<uint8_t, 4>> filler; ConstraintKind constraint = ConstraintKind::NoConstraint; std::string location; @@ -112,8 +119,7 @@ public: private: // Used for implementation of --compress-debug-sections option. - std::vector<uint8_t> zDebugHeader; - llvm::SmallVector<char, 0> compressedData; + CompressedData compressed; std::array<uint8_t, 4> getFiller(); }; @@ -121,7 +127,7 @@ private: int getPriority(StringRef s); InputSection *getFirstInputSection(const OutputSection *os); -std::vector<InputSection *> getInputSections(const OutputSection *os); +SmallVector<InputSection *, 0> getInputSections(const OutputSection &os); // All output sections that are handled by the linker specially are // globally accessible. Writer initializes them, so don't use them diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index cfe49007b814..074bc7150991 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -100,11 +100,11 @@ void elf::reportRangeError(uint8_t *loc, const Relocation &rel, const Twine &v, int64_t min, uint64_t max) { ErrorPlace errPlace = getErrorPlace(loc); std::string hint; - if (rel.sym && !rel.sym->isLocal()) + if (rel.sym && !rel.sym->isSection()) hint = "; references " + lld::toString(*rel.sym); if (!errPlace.srcLoc.empty()) hint += "\n>>> referenced by " + errPlace.srcLoc; - if (rel.sym && !rel.sym->isLocal()) + if (rel.sym && !rel.sym->isSection()) hint += getDefinedLocation(*rel.sym); if (errPlace.isec && errPlace.isec->name.startswith(".debug")) @@ -302,8 +302,7 @@ static void replaceWithDefined(Symbol &sym, SectionBase &sec, uint64_t value, sym.replace(Defined{sym.file, sym.getName(), sym.binding, sym.stOther, sym.type, value, size, &sec}); - sym.pltIndex = old.pltIndex; - sym.gotIndex = old.gotIndex; + sym.auxIdx = old.auxIdx; sym.verdefIndex = old.verdefIndex; sym.exportDynamic = true; sym.isUsedInRegularObj = true; @@ -404,14 +403,89 @@ static void addCopyRelSymbol(SharedSymbol &ss) { } } +// .eh_frame sections are mergeable input sections, so their input +// offsets are not linearly mapped to output section. For each input +// offset, we need to find a section piece containing the offset and +// add the piece's base address to the input offset to compute the +// output offset. That isn't cheap. +// +// This class is to speed up the offset computation. When we process +// relocations, we access offsets in the monotonically increasing +// order. So we can optimize for that access pattern. +// +// For sections other than .eh_frame, this class doesn't do anything. +namespace { +class OffsetGetter { +public: + explicit OffsetGetter(InputSectionBase &sec) { + if (auto *eh = dyn_cast<EhInputSection>(&sec)) + pieces = eh->pieces; + } + + // Translates offsets in input sections to offsets in output sections. + // Given offset must increase monotonically. We assume that Piece is + // sorted by inputOff. + uint64_t get(uint64_t off) { + if (pieces.empty()) + return off; + + while (i != pieces.size() && pieces[i].inputOff + pieces[i].size <= off) + ++i; + if (i == pieces.size()) + fatal(".eh_frame: relocation is not in any piece"); + + // Pieces must be contiguous, so there must be no holes in between. + assert(pieces[i].inputOff <= off && "Relocation not in any piece"); + + // Offset -1 means that the piece is dead (i.e. garbage collected). + if (pieces[i].outputOff == -1) + return -1; + return pieces[i].outputOff + off - pieces[i].inputOff; + } + +private: + ArrayRef<EhSectionPiece> pieces; + size_t i = 0; +}; + +// This class encapsulates states needed to scan relocations for one +// InputSectionBase. +class RelocationScanner { +public: + explicit RelocationScanner(InputSectionBase &sec) + : sec(sec), getter(sec), config(elf::config.get()), target(*elf::target) { + } + template <class ELFT, class RelTy> void scan(ArrayRef<RelTy> rels); + +private: + InputSectionBase &sec; + OffsetGetter getter; + const Configuration *const config; + const TargetInfo ⌖ + + // End of relocations, used by Mips/PPC64. + const void *end = nullptr; + + template <class RelTy> RelType getMipsN32RelType(RelTy *&rel) const; + template <class ELFT, class RelTy> + int64_t computeMipsAddend(const RelTy &rel, RelExpr expr, bool isLocal) const; + template <class ELFT, class RelTy> + int64_t computeAddend(const RelTy &rel, RelExpr expr, bool isLocal) const; + bool isStaticLinkTimeConstant(RelExpr e, RelType type, const Symbol &sym, + uint64_t relOff) const; + void processAux(RelExpr expr, RelType type, uint64_t offset, Symbol &sym, + int64_t addend) const; + template <class ELFT, class RelTy> void scanOne(RelTy *&i); +}; +} // namespace + // MIPS has an odd notion of "paired" relocations to calculate addends. // For example, if a relocation is of R_MIPS_HI16, there must be a // R_MIPS_LO16 relocation after that, and an addend is calculated using // the two relocations. template <class ELFT, class RelTy> -static int64_t computeMipsAddend(const RelTy &rel, const RelTy *end, - InputSectionBase &sec, RelExpr expr, - bool isLocal) { +int64_t RelocationScanner::computeMipsAddend(const RelTy &rel, RelExpr expr, + bool isLocal) const { if (expr == R_MIPS_GOTREL && isLocal) return sec.getFile<ELFT>()->mipsGp0; @@ -430,10 +504,10 @@ static int64_t computeMipsAddend(const RelTy &rel, const RelTy *end, // To make things worse, paired relocations might not be contiguous in // the relocation table, so we need to do linear search. *sigh* - for (const RelTy *ri = &rel; ri != end; ++ri) + for (const RelTy *ri = &rel; ri != static_cast<const RelTy *>(end); ++ri) if (ri->getType(config->isMips64EL) == pairTy && ri->getSymbol(config->isMips64EL) == symIndex) - return target->getImplicitAddend(buf + ri->r_offset, pairTy); + return target.getImplicitAddend(buf + ri->r_offset, pairTy); warn("can't find matching " + toString(pairTy) + " relocation for " + toString(type)); @@ -444,9 +518,8 @@ static int64_t computeMipsAddend(const RelTy &rel, const RelTy *end, // is in a relocation itself. If it is REL, we need to read it from an // input section. template <class ELFT, class RelTy> -static int64_t computeAddend(const RelTy &rel, const RelTy *end, - InputSectionBase &sec, RelExpr expr, - bool isLocal) { +int64_t RelocationScanner::computeAddend(const RelTy &rel, RelExpr expr, + bool isLocal) const { int64_t addend; RelType type = rel.getType(config->isMips64EL); @@ -454,13 +527,13 @@ static int64_t computeAddend(const RelTy &rel, const RelTy *end, addend = getAddend<ELFT>(rel); } else { const uint8_t *buf = sec.data().data(); - addend = target->getImplicitAddend(buf + rel.r_offset, type); + addend = target.getImplicitAddend(buf + rel.r_offset, type); } if (config->emachine == EM_PPC64 && config->isPic && type == R_PPC64_TOC) addend += getPPC64TocBase(); if (config->emachine == EM_MIPS) - addend += computeMipsAddend<ELFT>(rel, end, sec, expr, isLocal); + addend += computeMipsAddend<ELFT>(rel, expr, isLocal); return addend; } @@ -503,7 +576,7 @@ static std::string maybeReportDiscarded(Undefined &sym) { // them are known, so that some postprocessing on the list of undefined symbols // can happen before lld emits diagnostics. struct UndefinedDiag { - Symbol *sym; + Undefined *sym; struct Loc { InputSectionBase *sec; uint64_t offset; @@ -532,12 +605,12 @@ static bool canSuggestExternCForCXX(StringRef ref, StringRef def) { // Suggest an alternative spelling of an "undefined symbol" diagnostic. Returns // the suggested symbol, which is either in the symbol table, or in the same // file of sym. -template <class ELFT> static const Symbol *getAlternativeSpelling(const Undefined &sym, std::string &pre_hint, std::string &post_hint) { DenseMap<StringRef, const Symbol *> map; - if (auto *file = dyn_cast_or_null<ObjFile<ELFT>>(sym.file)) { + if (sym.file && sym.file->kind() == InputFile::ObjKind) { + auto *file = cast<ELFFileBase>(sym.file); // If sym is a symbol defined in a discarded section, maybeReportDiscarded() // will give an error. Don't suggest an alternative spelling. if (file && sym.discardedSecIdx != 0 && @@ -649,7 +722,7 @@ static const Symbol *getAlternativeSpelling(const Undefined &sym, template <class ELFT> static void reportUndefinedSymbol(const UndefinedDiag &undef, bool correctSpelling) { - Symbol &sym = *undef.sym; + Undefined &sym = *undef.sym; auto visibility = [&]() -> std::string { switch (sym.visibility) { @@ -664,7 +737,7 @@ static void reportUndefinedSymbol(const UndefinedDiag &undef, } }; - std::string msg = maybeReportDiscarded<ELFT>(cast<Undefined>(sym)); + std::string msg = maybeReportDiscarded<ELFT>(sym); if (msg.empty()) msg = "undefined " + visibility() + "symbol: " + toString(sym); @@ -690,8 +763,8 @@ static void reportUndefinedSymbol(const UndefinedDiag &undef, if (correctSpelling) { std::string pre_hint = ": ", post_hint; - if (const Symbol *corrected = getAlternativeSpelling<ELFT>( - cast<Undefined>(sym), pre_hint, post_hint)) { + if (const Symbol *corrected = + getAlternativeSpelling(sym, pre_hint, post_hint)) { msg += "\n>>> did you mean" + pre_hint + toString(*corrected) + post_hint; if (corrected->file) msg += "\n>>> defined in: " + toString(corrected->file); @@ -737,11 +810,11 @@ template <class ELFT> void elf::reportUndefinedSymbols() { // Report an undefined symbol if necessary. // Returns true if the undefined symbol will produce an error message. -static bool maybeReportUndefined(Symbol &sym, InputSectionBase &sec, +static bool maybeReportUndefined(Undefined &sym, InputSectionBase &sec, uint64_t offset) { // If versioned, issue an error (even if the symbol is weak) because we don't // know the defining filename which is required to construct a Verneed entry. - if (*sym.getVersionSuffix() == '@') { + if (sym.hasVersionSuffix) { undefs.push_back({&sym, {{&sec, offset}}, false}); return true; } @@ -761,8 +834,7 @@ static bool maybeReportUndefined(Symbol &sym, InputSectionBase &sec, // PPC32 .got2 is similar but cannot be fixed. Multiple .got2 is infeasible // because .LC0-.LTOC is not representable if the two labels are in different // .got2 - if (cast<Undefined>(sym).discardedSecIdx != 0 && - (sec.name == ".got2" || sec.name == ".toc")) + if (sym.discardedSecIdx != 0 && (sec.name == ".got2" || sec.name == ".toc")) return false; bool isWarning = @@ -777,62 +849,17 @@ static bool maybeReportUndefined(Symbol &sym, InputSectionBase &sec, // packs all relocations into the single relocation record. Here we emulate // this for the N32 ABI. Iterate over relocation with the same offset and put // theirs types into the single bit-set. -template <class RelTy> static RelType getMipsN32RelType(RelTy *&rel, RelTy *end) { +template <class RelTy> +RelType RelocationScanner::getMipsN32RelType(RelTy *&rel) const { RelType type = 0; uint64_t offset = rel->r_offset; int n = 0; - while (rel != end && rel->r_offset == offset) + while (rel != static_cast<const RelTy *>(end) && rel->r_offset == offset) type |= (rel++)->getType(config->isMips64EL) << (8 * n++); return type; } -// .eh_frame sections are mergeable input sections, so their input -// offsets are not linearly mapped to output section. For each input -// offset, we need to find a section piece containing the offset and -// add the piece's base address to the input offset to compute the -// output offset. That isn't cheap. -// -// This class is to speed up the offset computation. When we process -// relocations, we access offsets in the monotonically increasing -// order. So we can optimize for that access pattern. -// -// For sections other than .eh_frame, this class doesn't do anything. -namespace { -class OffsetGetter { -public: - explicit OffsetGetter(InputSectionBase &sec) { - if (auto *eh = dyn_cast<EhInputSection>(&sec)) - pieces = eh->pieces; - } - - // Translates offsets in input sections to offsets in output sections. - // Given offset must increase monotonically. We assume that Piece is - // sorted by inputOff. - uint64_t get(uint64_t off) { - if (pieces.empty()) - return off; - - while (i != pieces.size() && pieces[i].inputOff + pieces[i].size <= off) - ++i; - if (i == pieces.size()) - fatal(".eh_frame: relocation is not in any piece"); - - // Pieces must be contiguous, so there must be no holes in between. - assert(pieces[i].inputOff <= off && "Relocation not in any piece"); - - // Offset -1 means that the piece is dead (i.e. garbage collected). - if (pieces[i].outputOff == -1) - return -1; - return pieces[i].outputOff + off - pieces[i].inputOff; - } - -private: - ArrayRef<EhSectionPiece> pieces; - size_t i = 0; -}; -} // namespace - static void addRelativeReloc(InputSectionBase &isec, uint64_t offsetInSec, Symbol &sym, int64_t addend, RelExpr expr, RelType type) { @@ -870,7 +897,7 @@ static void addGotEntry(Symbol &sym) { // If preemptible, emit a GLOB_DAT relocation. if (sym.isPreemptible) { - mainPart->relaDyn->addReloc({target->gotRel, in.got, off, + mainPart->relaDyn->addReloc({target->gotRel, in.got.get(), off, DynamicReloc::AgainstSymbol, sym, 0, R_ABS}); return; } @@ -924,8 +951,9 @@ static bool canDefineSymbolInExecutable(Symbol &sym) { // // If this function returns false, that means we need to emit a // dynamic relocation so that the relocation will be fixed at load-time. -static bool isStaticLinkTimeConstant(RelExpr e, RelType type, const Symbol &sym, - InputSectionBase &s, uint64_t relOff) { +bool RelocationScanner::isStaticLinkTimeConstant(RelExpr e, RelType type, + const Symbol &sym, + uint64_t relOff) const { // These expressions always compute a constant if (oneof<R_GOTPLT, R_GOT_OFF, R_MIPS_GOT_LOCAL_PAGE, R_MIPS_GOTREL, R_MIPS_GOT_OFF, R_MIPS_GOT_OFF32, R_MIPS_GOT_GP_PC, @@ -937,7 +965,7 @@ static bool isStaticLinkTimeConstant(RelExpr e, RelType type, const Symbol &sym, // These never do, except if the entire file is position dependent or if // only the low bits are used. if (e == R_GOT || e == R_PLT) - return target->usesOnlyLowPageBits(type) || !config->isPic; + return target.usesOnlyLowPageBits(type) || !config->isPic; if (sym.isPreemptible) return false; @@ -957,7 +985,7 @@ static bool isStaticLinkTimeConstant(RelExpr e, RelType type, const Symbol &sym, if (!absVal && relE) return true; if (!absVal && !relE) - return target->usesOnlyLowPageBits(type); + return target.usesOnlyLowPageBits(type); assert(absVal && relE); @@ -975,7 +1003,7 @@ static bool isStaticLinkTimeConstant(RelExpr e, RelType type, const Symbol &sym, return true; error("relocation " + toString(type) + " cannot refer to absolute symbol: " + - toString(sym) + getLocation(s, sym, relOff)); + toString(sym) + getLocation(sec, sym, relOff)); return true; } @@ -992,9 +1020,8 @@ static bool isStaticLinkTimeConstant(RelExpr e, RelType type, const Symbol &sym, // sections. Given that it is ro, we will need an extra PT_LOAD. This // complicates things for the dynamic linker and means we would have to reserve // space for the extra PT_LOAD even if we end up not using it. -template <class ELFT> -static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type, - uint64_t offset, Symbol &sym, int64_t addend) { +void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset, + Symbol &sym, int64_t addend) const { // If the relocation is known to be a link-time constant, we know no dynamic // relocation will be created, pass the control to relocateAlloc() or // relocateNonAlloc() to resolve it. @@ -1009,7 +1036,7 @@ static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type, // -shared matches the spirit of its -z undefs default. -pie has freedom on // choices, and we choose dynamic relocations to be consistent with the // handling of GOT-generating relocations. - if (isStaticLinkTimeConstant(expr, type, sym, sec, offset) || + if (isStaticLinkTimeConstant(expr, type, sym, offset) || (!config->isPic && sym.isUndefWeak())) { sec.relocations.push_back({expr, type, offset, addend, &sym}); return; @@ -1017,13 +1044,13 @@ static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type, bool canWrite = (sec.flags & SHF_WRITE) || !config->zText; if (canWrite) { - RelType rel = target->getDynRel(type); - if (expr == R_GOT || (rel == target->symbolicRel && !sym.isPreemptible)) { + RelType rel = target.getDynRel(type); + if (expr == R_GOT || (rel == target.symbolicRel && !sym.isPreemptible)) { addRelativeReloc(sec, offset, sym, addend, expr, type); return; } else if (rel != 0) { - if (config->emachine == EM_MIPS && rel == target->symbolicRel) - rel = target->relativeRel; + if (config->emachine == EM_MIPS && rel == target.symbolicRel) + rel = target.relativeRel; sec.getPartition().relaDyn->addSymbolReloc(rel, sec, offset, sym, addend, type); @@ -1145,10 +1172,9 @@ static unsigned handleMipsTlsRelocation(RelType type, Symbol &sym, // symbol in TLS block. // // Returns the number of relocations processed. -template <class ELFT> -static unsigned -handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c, - typename ELFT::uint offset, int64_t addend, RelExpr expr) { +static unsigned handleTlsRelocation(RelType type, Symbol &sym, + InputSectionBase &c, uint64_t offset, + int64_t addend, RelExpr expr) { if (!sym.isTls()) return 0; @@ -1262,9 +1288,7 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c, return 0; } -template <class ELFT, class RelTy> -static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, - RelTy *start, RelTy *end) { +template <class ELFT, class RelTy> void RelocationScanner::scanOne(RelTy *&i) { const RelTy &rel = *i; uint32_t symIndex = rel.getSymbol(config->isMips64EL); Symbol &sym = sec.getFile<ELFT>()->getSymbol(symIndex); @@ -1272,32 +1296,32 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, // Deal with MIPS oddity. if (config->mipsN32Abi) { - type = getMipsN32RelType(i, end); + type = getMipsN32RelType(i); } else { type = rel.getType(config->isMips64EL); ++i; } // Get an offset in an output section this relocation is applied to. - uint64_t offset = getOffset.get(rel.r_offset); + uint64_t offset = getter.get(rel.r_offset); if (offset == uint64_t(-1)) return; // Error if the target symbol is undefined. Symbol index 0 may be used by // marker relocations, e.g. R_*_NONE and R_ARM_V4BX. Don't error on them. if (sym.isUndefined() && symIndex != 0 && - maybeReportUndefined(sym, sec, rel.r_offset)) + maybeReportUndefined(cast<Undefined>(sym), sec, offset)) return; - const uint8_t *relocatedAddr = sec.data().begin() + rel.r_offset; - RelExpr expr = target->getRelExpr(type, sym, relocatedAddr); + const uint8_t *relocatedAddr = sec.data().begin() + offset; + RelExpr expr = target.getRelExpr(type, sym, relocatedAddr); // Ignore R_*_NONE and other marker relocations. if (expr == R_NONE) return; // Read an addend. - int64_t addend = computeAddend<ELFT>(rel, end, sec, expr, sym.isLocal()); + int64_t addend = computeAddend<ELFT>(rel, expr, sym.isLocal()); if (config->emachine == EM_PPC64) { // We can separate the small code model relocations into 2 categories: @@ -1346,7 +1370,7 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, } // Process TLS relocations, including relaxing TLS relocations. Note that - // R_TPREL and R_TPREL_NEG relocations are resolved in processRelocAux. + // R_TPREL and R_TPREL_NEG relocations are resolved in processAux. if (expr == R_TPREL || expr == R_TPREL_NEG) { if (config->shared) { errorOrWarn("relocation " + toString(type) + " against " + toString(sym) + @@ -1354,8 +1378,8 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, getLocation(sec, sym, offset)); return; } - } else if (unsigned processed = handleTlsRelocation<ELFT>( - type, sym, sec, offset, addend, expr)) { + } else if (unsigned processed = + handleTlsRelocation(type, sym, sec, offset, addend, expr)) { i += (processed - 1); return; } @@ -1382,7 +1406,7 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, type == R_HEX_GD_PLT_B32_PCREL_X))) expr = fromPlt(expr); } else if (!isAbsoluteValue(sym)) { - expr = target->adjustGotPcExpr(type, addend, relocatedAddr); + expr = target.adjustGotPcExpr(type, addend, relocatedAddr); } } @@ -1413,7 +1437,7 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, sym.hasDirectReloc = true; } - processRelocAux<ELFT>(sec, expr, type, offset, sym, addend); + processAux(expr, type, offset, sym, addend); } // R_PPC64_TLSGD/R_PPC64_TLSLD is required to mark `bl __tls_get_addr` for @@ -1454,9 +1478,7 @@ static void checkPPC64TLSRelax(InputSectionBase &sec, ArrayRef<RelTy> rels) { } template <class ELFT, class RelTy> -static void scanRelocs(InputSectionBase &sec, ArrayRef<RelTy> rels) { - OffsetGetter getOffset(sec); - +void RelocationScanner::scan(ArrayRef<RelTy> rels) { // Not all relocations end up in Sec.Relocations, but a lot do. sec.relocations.reserve(rels.size()); @@ -1470,8 +1492,9 @@ static void scanRelocs(InputSectionBase &sec, ArrayRef<RelTy> rels) { if (isa<EhInputSection>(sec)) rels = sortRels(rels, storage); - for (auto i = rels.begin(), end = rels.end(); i != end;) - scanReloc<ELFT>(sec, getOffset, i, rels.begin(), end); + end = static_cast<const void *>(rels.end()); + for (auto i = rels.begin(); i != end;) + scanOne<ELFT>(i); // Sort relocations by offset for more efficient searching for // R_RISCV_PCREL_HI20 and R_PPC64_ADDR64. @@ -1484,11 +1507,12 @@ static void scanRelocs(InputSectionBase &sec, ArrayRef<RelTy> rels) { } template <class ELFT> void elf::scanRelocations(InputSectionBase &s) { + RelocationScanner scanner(s); const RelsOrRelas<ELFT> rels = s.template relsOrRelas<ELFT>(); if (rels.areRelocsRel()) - scanRelocs<ELFT>(s, rels.rels); + scanner.template scan<ELFT>(rels.rels); else - scanRelocs<ELFT>(s, rels.relas); + scanner.template scan<ELFT>(rels.relas); } static bool handleNonPreemptibleIfunc(Symbol &sym) { @@ -1545,15 +1569,17 @@ static bool handleNonPreemptibleIfunc(Symbol &sym) { // may alter section/value, so create a copy of the symbol to make // section/value fixed. auto *directSym = makeDefined(cast<Defined>(sym)); + directSym->allocateAux(); addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel, *directSym); - sym.pltIndex = directSym->pltIndex; + sym.allocateAux(); + symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx; if (sym.hasDirectReloc) { // Change the value to the IPLT and redirect all references to it. auto &d = cast<Defined>(sym); - d.section = in.iplt; - d.value = sym.pltIndex * target->ipltEntrySize; + d.section = in.iplt.get(); + d.value = d.getPltIdx() * target->ipltEntrySize; d.size = 0; // It's important to set the symbol type here so that dynamic loaders // don't try to call the PLT as if it were an ifunc resolver. @@ -1572,6 +1598,10 @@ void elf::postScanRelocations() { auto fn = [](Symbol &sym) { if (handleNonPreemptibleIfunc(sym)) return; + if (!sym.needsDynReloc()) + return; + sym.allocateAux(); + if (sym.needsGot) addGotEntry(sym); if (sym.needsPlt) @@ -1585,9 +1615,10 @@ void elf::postScanRelocations() { } else { assert(sym.isFunc() && sym.needsPlt); if (!sym.isDefined()) { - replaceWithDefined( - sym, *in.plt, - target->pltHeaderSize + target->pltEntrySize * sym.pltIndex, 0); + replaceWithDefined(sym, *in.plt, + target->pltHeaderSize + + target->pltEntrySize * sym.getPltIdx(), + 0); sym.needsCopy = true; if (config->emachine == EM_PPC) { // PPC32 canonical PLT entries are at the beginning of .glink @@ -1604,13 +1635,12 @@ void elf::postScanRelocations() { bool isLocalInExecutable = !sym.isPreemptible && !config->shared; if (sym.needsTlsDesc) { - in.got->addDynTlsEntry(sym); + in.got->addTlsDescEntry(sym); mainPart->relaDyn->addAddendOnlyRelocIfNonPreemptible( - target->tlsDescRel, *in.got, in.got->getGlobalDynOffset(sym), sym, + target->tlsDescRel, *in.got, in.got->getTlsDescOffset(sym), sym, target->tlsDescRel); } - if (sym.needsTlsGd && !sym.needsTlsDesc) { - // TODO Support mixed TLSDESC and TLS GD. + if (sym.needsTlsGd) { in.got->addDynTlsEntry(sym); uint64_t off = in.got->getGlobalDynOffset(sym); if (isLocalInExecutable) @@ -1642,8 +1672,8 @@ void elf::postScanRelocations() { in.got->relocations.push_back( {R_ADDEND, target->symbolicRel, in.got->getTlsIndexOff(), 1, &sym}); else - mainPart->relaDyn->addReloc( - {target->tlsModuleIndexRel, in.got, in.got->getTlsIndexOff()}); + mainPart->relaDyn->addReloc({target->tlsModuleIndexRel, in.got.get(), + in.got->getTlsIndexOff()}); } if (sym.needsGotDtprel) { in.got->addEntry(sym); @@ -1654,13 +1684,15 @@ void elf::postScanRelocations() { if (sym.needsTlsIe && !sym.needsTlsGdToIe) addTpOffsetGotEntry(sym); }; + + assert(symAux.empty()); for (Symbol *sym : symtab->symbols()) fn(*sym); // Local symbols may need the aforementioned non-preemptible ifunc and GOT // handling. They don't need regular PLT. for (ELFFileBase *file : objectFiles) - for (Symbol *sym : cast<ELFFileBase>(file)->getLocalSymbols()) + for (Symbol *sym : file->getLocalSymbols()) fn(*sym); } @@ -1817,7 +1849,7 @@ void ThunkCreator::mergeThunks(ArrayRef<OutputSection *> outputSections) { }); // Merge sorted vectors of Thunks and InputSections by outSecOff - std::vector<InputSection *> tmp; + SmallVector<InputSection *, 0> tmp; tmp.reserve(isd->sections.size() + newThunks.size()); std::merge(isd->sections.begin(), isd->sections.end(), @@ -2010,7 +2042,8 @@ std::pair<Thunk *, bool> ThunkCreator::getThunk(InputSection *isec, // out in the relocation addend. We compensate for the PC bias so that // an Arm and Thumb relocation to the same destination get the same keyAddend, // which is usually 0. - int64_t keyAddend = rel.addend + getPCBias(rel.type); + const int64_t pcBias = getPCBias(rel.type); + const int64_t keyAddend = rel.addend + pcBias; // We use a ((section, offset), addend) pair to find the thunk position if // possible so that we create only one thunk for aliased symbols or ICFed @@ -2029,7 +2062,7 @@ std::pair<Thunk *, bool> ThunkCreator::getThunk(InputSection *isec, if (isThunkSectionCompatible(isec, t->getThunkTargetSym()->section) && t->isCompatibleWith(*isec, rel) && target->inBranchRange(rel.type, src, - t->getThunkTargetSym()->getVA(rel.addend))) + t->getThunkTargetSym()->getVA(-pcBias))) return std::make_pair(t, false); // No existing compatible Thunk in range, create a new one @@ -2174,6 +2207,7 @@ void elf::hexagonTLSSymbolUpdate(ArrayRef<OutputSection *> outputSections) { for (Relocation &rel : isec->relocations) if (rel.sym->type == llvm::ELF::STT_TLS && rel.expr == R_PLT_PC) { if (needEntry) { + sym->allocateAux(); addPltEntry(*in.plt, *in.gotPlt, *in.relaPlt, target->pltRel, *sym); needEntry = false; diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h index c652c0a5f70f..73f6970b80f1 100644 --- a/lld/ELF/Relocations.h +++ b/lld/ELF/Relocations.h @@ -11,6 +11,7 @@ #include "lld/Common/LLVM.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" #include <map> #include <vector> @@ -117,8 +118,8 @@ struct Relocation { // jump instruction opcodes at basic block boundaries and are particularly // useful when basic block sections are enabled. struct JumpInstrMod { - JumpModType original; uint64_t offset; + JumpModType original; unsigned size; }; diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp index d3b0296acab0..7331d1156f27 100644 --- a/lld/ELF/ScriptParser.cpp +++ b/lld/ELF/ScriptParser.cpp @@ -20,7 +20,7 @@ #include "ScriptLexer.h" #include "Symbols.h" #include "Target.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" @@ -93,12 +93,12 @@ private: void readSectionAddressType(OutputSection *cmd); OutputSection *readOverlaySectionDescription(); OutputSection *readOutputSectionDescription(StringRef outSec); - std::vector<SectionCommand *> readOverlay(); - std::vector<StringRef> readOutputSectionPhdrs(); + SmallVector<SectionCommand *, 0> readOverlay(); + SmallVector<StringRef, 0> readOutputSectionPhdrs(); std::pair<uint64_t, uint64_t> readInputSectionFlags(); InputSectionDescription *readInputSectionDescription(StringRef tok); StringMatcher readFilePatterns(); - std::vector<SectionPattern> readInputSectionsList(); + SmallVector<SectionPattern, 0> readInputSectionsList(); InputSectionDescription *readInputSectionRules(StringRef filePattern, uint64_t withFlags, uint64_t withoutFlags); @@ -125,11 +125,11 @@ private: Expr readParenExpr(); // For parsing version script. - std::vector<SymbolVersion> readVersionExtern(); + SmallVector<SymbolVersion, 0> readVersionExtern(); void readAnonymousDeclaration(); void readVersionDeclaration(StringRef verStr); - std::pair<std::vector<SymbolVersion>, std::vector<SymbolVersion>> + std::pair<SmallVector<SymbolVersion, 0>, SmallVector<SymbolVersion, 0>> readSymbols(); // True if a script being read is in the --sysroot directory. @@ -181,8 +181,8 @@ static ExprValue bitOr(ExprValue a, ExprValue b) { void ScriptParser::readDynamicList() { expect("{"); - std::vector<SymbolVersion> locals; - std::vector<SymbolVersion> globals; + SmallVector<SymbolVersion, 0> locals; + SmallVector<SymbolVersion, 0> globals; std::tie(locals, globals) = readSymbols(); expect(";"); @@ -290,7 +290,7 @@ void ScriptParser::addFile(StringRef s) { SmallString<128> pathData; StringRef path = (config->sysroot + s).toStringRef(pathData); if (sys::fs::exists(path)) - driver->addFile(saver.save(path), /*withLOption=*/false); + driver->addFile(saver().save(path), /*withLOption=*/false); else setError("cannot find " + s + " inside " + config->sysroot); return; @@ -304,7 +304,7 @@ void ScriptParser::addFile(StringRef s) { if (config->sysroot.empty()) driver->addFile(s.substr(1), /*withLOption=*/false); else - driver->addFile(saver.save(config->sysroot + "/" + s.substr(1)), + driver->addFile(saver().save(config->sysroot + "/" + s.substr(1)), /*withLOption=*/false); } else if (s.startswith("-l")) { // Case 3: search in the list of library paths. @@ -327,7 +327,7 @@ void ScriptParser::addFile(StringRef s) { } else { // Finally, search in the list of library paths. if (Optional<std::string> path = findFromSearchPaths(s)) - driver->addFile(saver.save(*path), /*withLOption=*/true); + driver->addFile(saver().save(*path), /*withLOption=*/true); else setError("unable to find " + s); } @@ -519,7 +519,7 @@ void ScriptParser::readSearchDir() { // sections that use the same virtual memory range and normally would trigger // linker's sections sanity check failures. // https://sourceware.org/binutils/docs/ld/Overlay-Description.html#Overlay-Description -std::vector<SectionCommand *> ScriptParser::readOverlay() { +SmallVector<SectionCommand *, 0> ScriptParser::readOverlay() { // VA and LMA expressions are optional, though for simplicity of // implementation we assume they are not. That is what OVERLAY was designed // for first of all: to allow sections with overlapping VAs at different LMAs. @@ -529,7 +529,7 @@ std::vector<SectionCommand *> ScriptParser::readOverlay() { Expr lmaExpr = readParenExpr(); expect("{"); - std::vector<SectionCommand *> v; + SmallVector<SectionCommand *, 0> v; OutputSection *prev = nullptr; while (!errorCount() && !consume("}")) { // VA is the same for all sections. The LMAs are consecutive in memory @@ -566,7 +566,7 @@ void ScriptParser::readOverwriteSections() { void ScriptParser::readSections() { expect("{"); - std::vector<SectionCommand *> v; + SmallVector<SectionCommand *, 0> v; while (!errorCount() && !consume("}")) { StringRef tok = next(); if (tok == "OVERLAY") { @@ -597,7 +597,7 @@ void ScriptParser::readSections() { else if (!consume("BEFORE")) setError("expected AFTER/BEFORE, but got '" + next() + "'"); StringRef where = next(); - std::vector<StringRef> names; + SmallVector<StringRef, 0> names; for (SectionCommand *cmd : v) if (auto *os = dyn_cast<OutputSection>(cmd)) names.push_back(os->name); @@ -672,8 +672,8 @@ SortSectionPolicy ScriptParser::readSortKind() { // is parsed as ".foo", ".bar" with "a.o", and ".baz" with "b.o". // The semantics of that is section .foo in any file, section .bar in // any file but a.o, and section .baz in any file but b.o. -std::vector<SectionPattern> ScriptParser::readInputSectionsList() { - std::vector<SectionPattern> ret; +SmallVector<SectionPattern, 0> ScriptParser::readInputSectionsList() { + SmallVector<SectionPattern, 0> ret; while (!errorCount() && peek() != ")") { StringMatcher excludeFilePat; if (consume("EXCLUDE_FILE")) { @@ -718,7 +718,7 @@ ScriptParser::readInputSectionRules(StringRef filePattern, uint64_t withFlags, while (!errorCount() && !consume(")")) { SortSectionPolicy outer = readSortKind(); SortSectionPolicy inner = SortSectionPolicy::Default; - std::vector<SectionPattern> v; + SmallVector<SectionPattern, 0> v; if (outer != SortSectionPolicy::Default) { expect("("); inner = readSortKind(); @@ -1452,8 +1452,8 @@ Expr ScriptParser::readParenExpr() { return e; } -std::vector<StringRef> ScriptParser::readOutputSectionPhdrs() { - std::vector<StringRef> phdrs; +SmallVector<StringRef, 0> ScriptParser::readOutputSectionPhdrs() { + SmallVector<StringRef, 0> phdrs; while (!errorCount() && peek().startswith(":")) { StringRef tok = next(); phdrs.push_back((tok.size() == 1) ? next() : tok.substr(1)); @@ -1494,8 +1494,8 @@ unsigned ScriptParser::readPhdrType() { // Reads an anonymous version declaration. void ScriptParser::readAnonymousDeclaration() { - std::vector<SymbolVersion> locals; - std::vector<SymbolVersion> globals; + SmallVector<SymbolVersion, 0> locals; + SmallVector<SymbolVersion, 0> globals; std::tie(locals, globals) = readSymbols(); for (const SymbolVersion &pat : locals) config->versionDefinitions[VER_NDX_LOCAL].localPatterns.push_back(pat); @@ -1509,8 +1509,8 @@ void ScriptParser::readAnonymousDeclaration() { // e.g. "VerStr { global: foo; bar; local: *; };". void ScriptParser::readVersionDeclaration(StringRef verStr) { // Read a symbol list. - std::vector<SymbolVersion> locals; - std::vector<SymbolVersion> globals; + SmallVector<SymbolVersion, 0> locals; + SmallVector<SymbolVersion, 0> globals; std::tie(locals, globals) = readSymbols(); // Create a new version definition and add that to the global symbols. @@ -1535,11 +1535,11 @@ bool elf::hasWildcard(StringRef s) { } // Reads a list of symbols, e.g. "{ global: foo; bar; local: *; };". -std::pair<std::vector<SymbolVersion>, std::vector<SymbolVersion>> +std::pair<SmallVector<SymbolVersion, 0>, SmallVector<SymbolVersion, 0>> ScriptParser::readSymbols() { - std::vector<SymbolVersion> locals; - std::vector<SymbolVersion> globals; - std::vector<SymbolVersion> *v = &globals; + SmallVector<SymbolVersion, 0> locals; + SmallVector<SymbolVersion, 0> globals; + SmallVector<SymbolVersion, 0> *v = &globals; while (!errorCount()) { if (consume("}")) @@ -1554,7 +1554,7 @@ ScriptParser::readSymbols() { } if (consume("extern")) { - std::vector<SymbolVersion> ext = readVersionExtern(); + SmallVector<SymbolVersion, 0> ext = readVersionExtern(); v->insert(v->end(), ext.begin(), ext.end()); } else { StringRef tok = next(); @@ -1570,14 +1570,14 @@ ScriptParser::readSymbols() { // // The last semicolon is optional. E.g. this is OK: // "extern "C++" { ns::*; "f(int, double)" };" -std::vector<SymbolVersion> ScriptParser::readVersionExtern() { +SmallVector<SymbolVersion, 0> ScriptParser::readVersionExtern() { StringRef tok = next(); bool isCXX = tok == "\"C++\""; if (!isCXX && tok != "\"C\"") setError("Unknown language"); expect("{"); - std::vector<SymbolVersion> ret; + SmallVector<SymbolVersion, 0> ret; while (!errorCount() && peek() != "}") { StringRef tok = next(); ret.push_back( diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp index a12c5f22c4fe..ec425cd7e1d1 100644 --- a/lld/ELF/SymbolTable.cpp +++ b/lld/ELF/SymbolTable.cpp @@ -72,8 +72,10 @@ Symbol *SymbolTable::insert(StringRef name) { auto p = symMap.insert({CachedHashStringRef(stem), (int)symVector.size()}); if (!p.second) { Symbol *sym = symVector[p.first->second]; - if (stem.size() != name.size()) + if (stem.size() != name.size()) { sym->setName(name); + sym->hasVersionSuffix = true; + } return sym; } @@ -93,6 +95,8 @@ Symbol *SymbolTable::insert(StringRef name) { sym->referenced = false; sym->traced = false; sym->scriptDefined = false; + if (pos != StringRef::npos) + sym->hasVersionSuffix = true; sym->partition = 1; return sym; } @@ -139,12 +143,13 @@ StringMap<SmallVector<Symbol *, 0>> &SymbolTable::getDemangledSyms() { StringRef name = sym->getName(); size_t pos = name.find('@'); if (pos == std::string::npos) - demangled = demangleItanium(name); + demangled = demangle(name, config->demangle); else if (pos + 1 == name.size() || name[pos + 1] == '@') - demangled = demangleItanium(name.substr(0, pos)); + demangled = demangle(name.substr(0, pos), config->demangle); else - demangled = - (demangleItanium(name.substr(0, pos)) + name.substr(pos)).str(); + demangled = (demangle(name.substr(0, pos), config->demangle) + + name.substr(pos)) + .str(); (*demangledSyms)[demangled].push_back(sym); } } @@ -316,7 +321,8 @@ void SymbolTable::scanVersionScript() { // can contain versions in the form of <name>@<version>. // Let them parse and update their names to exclude version suffix. for (Symbol *sym : symVector) - sym->parseSymbolVersion(); + if (sym->hasVersionSuffix) + sym->parseSymbolVersion(); // isPreemptible is false at this point. To correctly compute the binding of a // Defined (which is used by includeInDynsym()), we need to know if it is diff --git a/lld/ELF/SymbolTable.h b/lld/ELF/SymbolTable.h index 84d93a3dc786..04c81f642fe0 100644 --- a/lld/ELF/SymbolTable.h +++ b/lld/ELF/SymbolTable.h @@ -32,17 +32,8 @@ namespace elf { // add*() functions, which are called by input files as they are parsed. There // is one add* function per symbol type. class SymbolTable { - struct FilterOutPlaceholder { - bool operator()(Symbol *S) const { return !S->isPlaceholder(); } - }; - using iterator = - llvm::filter_iterator<SmallVector<Symbol *, 0>::const_iterator, - FilterOutPlaceholder>; - public: - llvm::iterator_range<iterator> symbols() const { - return llvm::make_filter_range(symVector, FilterOutPlaceholder()); - } + ArrayRef<Symbol *> symbols() const { return symVector; } void wrap(Symbol *sym, Symbol *real, Symbol *wrap); @@ -57,7 +48,7 @@ public: void handleDynamicList(); // Set of .so files to not link the same shared object file more than once. - llvm::DenseMap<StringRef, SharedFile *> soNames; + llvm::DenseMap<llvm::CachedHashStringRef, SharedFile *> soNames; // Comdat groups define "link once" sections. If two comdat groups have the // same name, only one of them is linked, and the other is ignored. This map diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp index 20301497a059..d943c1996422 100644 --- a/lld/ELF/Symbols.cpp +++ b/lld/ELF/Symbols.cpp @@ -26,16 +26,9 @@ using namespace llvm::ELF; using namespace lld; using namespace lld::elf; -// Returns a symbol for an error message. -static std::string demangle(StringRef symName) { - if (elf::config->demangle) - return demangleItanium(symName); - return std::string(symName); -} - std::string lld::toString(const elf::Symbol &sym) { StringRef name = sym.getName(); - std::string ret = demangle(name); + std::string ret = demangle(name, config->demangle); const char *suffix = sym.getVersionSuffix(); if (*suffix == '@') @@ -44,7 +37,7 @@ std::string lld::toString(const elf::Symbol &sym) { } std::string lld::toELFString(const Archive::Symbol &b) { - return demangle(b.getName()); + return demangle(b.getName(), config->demangle); } Defined *ElfSym::bss; @@ -66,6 +59,7 @@ DenseMap<const Symbol *, std::pair<const InputFile *, const InputFile *>> elf::backwardReferences; SmallVector<std::tuple<std::string, const InputFile *, const Symbol &>, 0> elf::whyExtract; +SmallVector<SymbolAux, 0> elf::symAux; static uint64_t getSymVA(const Symbol &sym, int64_t addend) { switch (sym.kind()) { @@ -140,8 +134,7 @@ static uint64_t getSymVA(const Symbol &sym, int64_t addend) { return 0; case Symbol::LazyArchiveKind: case Symbol::LazyObjectKind: - assert(sym.isUsedInRegularObj && "lazy symbol reached writer"); - return 0; + llvm_unreachable("lazy symbol reached writer"); case Symbol::CommonKind: llvm_unreachable("common symbol reached writer"); case Symbol::PlaceholderKind: @@ -161,7 +154,7 @@ uint64_t Symbol::getGotVA() const { } uint64_t Symbol::getGotOffset() const { - return gotIndex * target->gotEntrySize; + return getGotIdx() * target->gotEntrySize; } uint64_t Symbol::getGotPltVA() const { @@ -172,15 +165,15 @@ uint64_t Symbol::getGotPltVA() const { uint64_t Symbol::getGotPltOffset() const { if (isInIplt) - return pltIndex * target->gotEntrySize; - return (pltIndex + target->gotPltHeaderEntriesNum) * target->gotEntrySize; + return getPltIdx() * target->gotEntrySize; + return (getPltIdx() + target->gotPltHeaderEntriesNum) * target->gotEntrySize; } uint64_t Symbol::getPltVA() const { uint64_t outVA = isInIplt - ? in.iplt->getVA() + pltIndex * target->ipltEntrySize + ? in.iplt->getVA() + getPltIdx() * target->ipltEntrySize : in.plt->getVA() + in.plt->headerSize + - pltIndex * target->pltEntrySize; + getPltIdx() * target->pltEntrySize; // While linking microMIPS code PLT code are always microMIPS // code. Set the less-significant bit to track that fact. @@ -216,12 +209,13 @@ void Symbol::parseSymbolVersion() { if (pos == StringRef::npos) return; StringRef verstr = s.substr(pos + 1); - if (verstr.empty()) - return; // Truncate the symbol name so that it doesn't include the version string. nameSize = pos; + if (verstr.empty()) + return; + // If this is not in this DSO, it is not a definition. if (!isDefined()) return; @@ -274,19 +268,15 @@ MemoryBufferRef LazyArchive::getMemberBuffer() { } uint8_t Symbol::computeBinding() const { - if (config->relocatable) - return binding; if ((visibility != STV_DEFAULT && visibility != STV_PROTECTED) || - (versionId == VER_NDX_LOCAL && !isLazy())) + versionId == VER_NDX_LOCAL) return STB_LOCAL; - if (!config->gnuUnique && binding == STB_GNU_UNIQUE) + if (binding == STB_GNU_UNIQUE && !config->gnuUnique) return STB_GLOBAL; return binding; } bool Symbol::includeInDynsym() const { - if (!config->hasDynSymTab) - return false; if (computeBinding() == STB_LOCAL) return false; if (!isDefined() && !isCommon()) @@ -294,7 +284,7 @@ bool Symbol::includeInDynsym() const { // expects undefined weak symbols not to exist in .dynsym, e.g. // __pthread_mutex_lock reference in _dl_add_to_namespace_list, // __pthread_initialize_minimal reference in csu/libc-start.c. - return !(config->noDynamicLinker && isUndefWeak()); + return !(isUndefWeak() && config->noDynamicLinker); return exportDynamic || inDynamicList; } @@ -354,7 +344,7 @@ void elf::maybeWarnUnorderableSymbol(const Symbol *sym) { // Returns true if a symbol can be replaced at load-time by a symbol // with the same name defined in other ELF executable or DSO. bool elf::computeIsPreemptible(const Symbol &sym) { - assert(!sym.isLocal()); + assert(!sym.isLocal() || sym.isPlaceholder()); // Only symbols with default visibility that appear in dynsym can be // preempted. Symbols with protected visibility cannot be preempted. diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h index 27c36eedce80..cd5d4b280e79 100644 --- a/lld/ELF/Symbols.h +++ b/lld/ELF/Symbols.h @@ -42,20 +42,17 @@ class SharedSymbol; class Symbol; class Undefined; -// This is a StringRef-like container that doesn't run strlen(). -// -// ELF string tables contain a lot of null-terminated strings. Most of them -// are not necessary for the linker because they are names of local symbols, -// and the linker doesn't use local symbol names for name resolution. So, we -// use this class to represents strings read from string tables. -struct StringRefZ { - StringRefZ(const char *s) : data(s), size(-1) {} - StringRefZ(StringRef s) : data(s.data()), size(s.size()) {} - - const char *data; - const uint32_t size; +// Some index properties of a symbol are stored separately in this auxiliary +// struct to decrease sizeof(SymbolUnion) in the majority of cases. +struct SymbolAux { + uint32_t gotIdx = -1; + uint32_t pltIdx = -1; + uint32_t tlsDescIdx = -1; + uint32_t tlsGdIdx = -1; }; +extern SmallVector<SymbolAux, 0> symAux; + // The base class for real symbol classes. class Symbol { public: @@ -76,14 +73,14 @@ public: protected: const char *nameData; - mutable uint32_t nameSize; + // 32-bit size saves space. + uint32_t nameSize; public: + // A symAux index used to access GOT/PLT entry indexes. This is allocated in + // postScanRelocations(). + uint32_t auxIdx = -1; uint32_t dynsymIndex = 0; - uint32_t gotIndex = -1; - uint32_t pltIndex = -1; - - uint32_t globalDynIndex = -1; // This field is a index to the symbol's version definition. uint16_t verdefIndex = -1; @@ -144,6 +141,9 @@ public: // True if this symbol is specified by --trace-symbol option. uint8_t traced : 1; + // True if the name contains '@'. + uint8_t hasVersionSuffix : 1; + inline void replace(const Symbol &newSym); bool includeInDynsym() const; @@ -166,11 +166,7 @@ public: // all input files have been added. bool isUndefWeak() const { return isWeak() && isUndefined(); } - StringRef getName() const { - if (nameSize == (uint32_t)-1) - nameSize = strlen(nameData); - return {nameData, nameSize}; - } + StringRef getName() const { return {nameData, nameSize}; } void setName(StringRef s) { nameData = s.data(); @@ -183,13 +179,23 @@ public: // // For @@, the name has been truncated by insert(). For @, the name has been // truncated by Symbol::parseSymbolVersion(). - const char *getVersionSuffix() const { - (void)getName(); - return nameData + nameSize; + const char *getVersionSuffix() const { return nameData + nameSize; } + + uint32_t getGotIdx() const { + return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].gotIdx; + } + uint32_t getPltIdx() const { + return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].pltIdx; + } + uint32_t getTlsDescIdx() const { + return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].tlsDescIdx; + } + uint32_t getTlsGdIdx() const { + return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].tlsGdIdx; } - bool isInGot() const { return gotIndex != -1U; } - bool isInPlt() const { return pltIndex != -1U; } + bool isInGot() const { return getGotIdx() != uint32_t(-1); } + bool isInPlt() const { return getPltIdx() != uint32_t(-1); } uint64_t getVA(int64_t addend = 0) const; @@ -240,16 +246,18 @@ private: inline size_t getSymbolSize() const; protected: - Symbol(Kind k, InputFile *file, StringRefZ name, uint8_t binding, + Symbol(Kind k, InputFile *file, StringRef name, uint8_t binding, uint8_t stOther, uint8_t type) - : file(file), nameData(name.data), nameSize(name.size), binding(binding), - type(type), stOther(stOther), symbolKind(k), visibility(stOther & 3), + : file(file), nameData(name.data()), nameSize(name.size()), + binding(binding), type(type), stOther(stOther), symbolKind(k), + visibility(stOther & 3), isUsedInRegularObj(!file || file->kind() == InputFile::ObjKind), exportDynamic(isExportDynamic(k, visibility)), inDynamicList(false), - canInline(false), referenced(false), traced(false), isInIplt(false), - gotInIgot(false), isPreemptible(false), used(!config->gcSections), - folded(false), needsTocRestore(false), scriptDefined(false), - needsCopy(false), needsGot(false), needsPlt(false), needsTlsDesc(false), + canInline(false), referenced(false), traced(false), + hasVersionSuffix(false), isInIplt(false), gotInIgot(false), + isPreemptible(false), used(!config->gcSections), folded(false), + needsTocRestore(false), scriptDefined(false), needsCopy(false), + needsGot(false), needsPlt(false), needsTlsDesc(false), needsTlsGd(false), needsTlsGdToIe(false), needsTlsLd(false), needsGotDtprel(false), needsTlsIe(false), hasDirectReloc(false) {} @@ -297,6 +305,16 @@ public: uint8_t needsTlsIe : 1; uint8_t hasDirectReloc : 1; + bool needsDynReloc() const { + return needsCopy || needsGot || needsPlt || needsTlsDesc || needsTlsGd || + needsTlsGdToIe || needsTlsLd || needsGotDtprel || needsTlsIe; + } + void allocateAux() { + assert(auxIdx == uint32_t(-1)); + auxIdx = symAux.size(); + symAux.emplace_back(); + } + // The partition whose dynamic symbol table contains this symbol's definition. uint8_t partition = 1; @@ -311,7 +329,7 @@ public: // Represents a symbol that is defined in the current output file. class Defined : public Symbol { public: - Defined(InputFile *file, StringRefZ name, uint8_t binding, uint8_t stOther, + Defined(InputFile *file, StringRef name, uint8_t binding, uint8_t stOther, uint8_t type, uint64_t value, uint64_t size, SectionBase *section) : Symbol(DefinedKind, file, name, binding, stOther, type), value(value), size(size), section(section) {} @@ -346,7 +364,7 @@ public: // section. (Therefore, the later passes don't see any CommonSymbols.) class CommonSymbol : public Symbol { public: - CommonSymbol(InputFile *file, StringRefZ name, uint8_t binding, + CommonSymbol(InputFile *file, StringRef name, uint8_t binding, uint8_t stOther, uint8_t type, uint64_t alignment, uint64_t size) : Symbol(CommonKind, file, name, binding, stOther, type), alignment(alignment), size(size) {} @@ -359,7 +377,7 @@ public: class Undefined : public Symbol { public: - Undefined(InputFile *file, StringRefZ name, uint8_t binding, uint8_t stOther, + Undefined(InputFile *file, StringRef name, uint8_t binding, uint8_t stOther, uint8_t type, uint32_t discardedSecIdx = 0) : Symbol(UndefinedKind, file, name, binding, stOther, type), discardedSecIdx(discardedSecIdx) {} @@ -500,9 +518,9 @@ union SymbolUnion { }; // It is important to keep the size of SymbolUnion small for performance and -// memory usage reasons. 80 bytes is a soft limit based on the size of Defined +// memory usage reasons. 72 bytes is a soft limit based on the size of Defined // on a 64-bit system. -static_assert(sizeof(SymbolUnion) <= 80, "SymbolUnion too large"); +static_assert(sizeof(SymbolUnion) <= 72, "SymbolUnion too large"); template <typename T> struct AssertSymbol { static_assert(std::is_trivially_destructible<T>(), @@ -575,6 +593,7 @@ void Symbol::replace(const Symbol &newSym) { canInline = old.canInline; referenced = old.referenced; traced = old.traced; + hasVersionSuffix = old.hasVersionSuffix; isPreemptible = old.isPreemptible; scriptDefined = old.scriptDefined; partition = old.partition; diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index e480118f5ae9..f125e3f0a51a 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -22,9 +22,8 @@ #include "Symbols.h" #include "Target.h" #include "Writer.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/DWARF.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" #include "lld/Common/Strings.h" #include "lld/Common/Version.h" #include "llvm/ADT/SetOperations.h" @@ -73,7 +72,7 @@ static ArrayRef<uint8_t> getVersion() { // This is only for testing. StringRef s = getenv("LLD_VERSION"); if (s.empty()) - s = saver.save(Twine("Linker: ") + getLLDVersion()); + s = saver().save(Twine("Linker: ") + getLLDVersion()); // +1 to include the terminating '\0'. return {(const uint8_t *)s.data(), s.size() + 1}; @@ -171,7 +170,7 @@ MipsOptionsSection<ELFT> *MipsOptionsSection<ELFT>::create() { if (!ELFT::Is64Bits) return nullptr; - std::vector<InputSectionBase *> sections; + SmallVector<InputSectionBase *, 0> sections; for (InputSectionBase *sec : inputSections) if (sec->type == SHT_MIPS_OPTIONS) sections.push_back(sec); @@ -228,7 +227,7 @@ MipsReginfoSection<ELFT> *MipsReginfoSection<ELFT>::create() { if (ELFT::Is64Bits) return nullptr; - std::vector<InputSectionBase *> sections; + SmallVector<InputSectionBase *, 0> sections; for (InputSectionBase *sec : inputSections) if (sec->type == SHT_MIPS_REGINFO) sections.push_back(sec); @@ -255,7 +254,7 @@ MipsReginfoSection<ELFT> *MipsReginfoSection<ELFT>::create() { InputSection *elf::createInterpSection() { // StringSaver guarantees that the returned string ends with '\0'. - StringRef s = saver.save(config->dynamicLinker); + StringRef s = saver().save(config->dynamicLinker); ArrayRef<uint8_t> contents = {(const uint8_t *)s.data(), s.size() + 1}; return make<InputSection>(nullptr, SHF_ALLOC, SHT_PROGBITS, 1, contents, @@ -496,9 +495,8 @@ static void writeCieFde(uint8_t *buf, ArrayRef<uint8_t> d) { memcpy(buf, d.data(), d.size()); size_t aligned = alignTo(d.size(), config->wordsize); - - // Zero-clear trailing padding if it exists. - memset(buf + d.size(), 0, aligned - d.size()); + assert(std::all_of(buf + d.size(), buf + aligned, + [](uint8_t c) { return c == 0; })); // Fix the size field. -4 since size does not include the size field itself. write32(buf, aligned - 4); @@ -551,9 +549,9 @@ void EhFrameSection::finalizeContents() { // Returns data for .eh_frame_hdr. .eh_frame_hdr is a binary search table // to get an FDE from an address to which FDE is applied. This function // returns a list of such pairs. -std::vector<EhFrameSection::FdeData> EhFrameSection::getFdeData() const { +SmallVector<EhFrameSection::FdeData, 0> EhFrameSection::getFdeData() const { uint8_t *buf = Out::bufferStart + getParent()->offset + outSecOff; - std::vector<FdeData> ret; + SmallVector<FdeData, 0> ret; uint64_t va = getPartition().ehFrameHdr->getVA(); for (CieRecord *rec : cieRecords) { @@ -650,14 +648,20 @@ GotSection::GotSection() } void GotSection::addEntry(Symbol &sym) { - sym.gotIndex = numEntries; - ++numEntries; + assert(sym.auxIdx == symAux.size() - 1); + symAux.back().gotIdx = numEntries++; +} + +bool GotSection::addTlsDescEntry(Symbol &sym) { + assert(sym.auxIdx == symAux.size() - 1); + symAux.back().tlsDescIdx = numEntries; + numEntries += 2; + return true; } bool GotSection::addDynTlsEntry(Symbol &sym) { - if (sym.globalDynIndex != -1U) - return false; - sym.globalDynIndex = numEntries; + assert(sym.auxIdx == symAux.size() - 1); + symAux.back().tlsGdIdx = numEntries; // Global Dynamic TLS entries take two GOT slots. numEntries += 2; return true; @@ -673,12 +677,20 @@ bool GotSection::addTlsIndex() { return true; } +uint32_t GotSection::getTlsDescOffset(const Symbol &sym) const { + return sym.getTlsDescIdx() * config->wordsize; +} + +uint64_t GotSection::getTlsDescAddr(const Symbol &sym) const { + return getVA() + getTlsDescOffset(sym); +} + uint64_t GotSection::getGlobalDynAddr(const Symbol &b) const { - return this->getVA() + b.globalDynIndex * config->wordsize; + return this->getVA() + b.getTlsGdIdx() * config->wordsize; } uint64_t GotSection::getGlobalDynOffset(const Symbol &b) const { - return b.globalDynIndex * config->wordsize; + return b.getTlsGdIdx() * config->wordsize; } void GotSection::finalizeContents() { @@ -972,12 +984,18 @@ void MipsGotSection::build() { } } - // Update Symbol::gotIndex field to use this + // Update SymbolAux::gotIdx field to use this // value later in the `sortMipsSymbols` function. - for (auto &p : primGot->global) - p.first->gotIndex = p.second; - for (auto &p : primGot->relocs) - p.first->gotIndex = p.second; + for (auto &p : primGot->global) { + if (p.first->auxIdx == uint32_t(-1)) + p.first->allocateAux(); + symAux.back().gotIdx = p.second; + } + for (auto &p : primGot->relocs) { + if (p.first->auxIdx == uint32_t(-1)) + p.first->allocateAux(); + symAux.back().gotIdx = p.second; + } // Create dynamic relocations. for (FileGot &got : gots) { @@ -1145,7 +1163,8 @@ GotPltSection::GotPltSection() } void GotPltSection::addEntry(Symbol &sym) { - assert(sym.pltIndex == entries.size()); + assert(sym.auxIdx == symAux.size() - 1 && + symAux.back().pltIdx == entries.size()); entries.push_back(&sym); } @@ -1190,7 +1209,7 @@ IgotPltSection::IgotPltSection() target->gotEntrySize, getIgotPltName()) {} void IgotPltSection::addEntry(Symbol &sym) { - assert(sym.pltIndex == entries.size()); + assert(symAux.back().pltIdx == entries.size()); entries.push_back(&sym); } @@ -1218,7 +1237,7 @@ StringTableSection::StringTableSection(StringRef name, bool dynamic) // them with some other string that happens to be the same. unsigned StringTableSection::addString(StringRef s, bool hashIt) { if (hashIt) { - auto r = stringMap.insert(std::make_pair(s, this->size)); + auto r = stringMap.try_emplace(CachedHashStringRef(s), size); if (!r.second) return r.first->second; } @@ -1265,11 +1284,11 @@ DynamicSection<ELFT>::DynamicSection() // .rela.dyn // // DT_RELASZ is the total size of the included sections. -static uint64_t addRelaSz(RelocationBaseSection *relaDyn) { - size_t size = relaDyn->getSize(); - if (in.relaIplt->getParent() == relaDyn->getParent()) +static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) { + size_t size = relaDyn.getSize(); + if (in.relaIplt->getParent() == relaDyn.getParent()) size += in.relaIplt->getSize(); - if (in.relaPlt->getParent() == relaDyn->getParent()) + if (in.relaPlt->getParent() == relaDyn.getParent()) size += in.relaPlt->getSize(); return size; } @@ -1375,7 +1394,8 @@ DynamicSection<ELFT>::computeContents() { (in.relaIplt->isNeeded() && part.relaDyn->getParent() == in.relaIplt->getParent())) { addInSec(part.relaDyn->dynamicTag, *part.relaDyn); - entries.emplace_back(part.relaDyn->sizeDynamicTag, addRelaSz(part.relaDyn)); + entries.emplace_back(part.relaDyn->sizeDynamicTag, + addRelaSz(*part.relaDyn)); bool isRela = config->isRela; addInt(isRela ? DT_RELAENT : DT_RELENT, @@ -1390,7 +1410,8 @@ DynamicSection<ELFT>::computeContents() { addInt(isRela ? DT_RELACOUNT : DT_RELCOUNT, numRelativeRels); } } - if (part.relrDyn && !part.relrDyn->relocs.empty()) { + if (part.relrDyn && part.relrDyn->getParent() && + !part.relrDyn->relocs.empty()) { addInSec(config->useAndroidRelrTags ? DT_ANDROID_RELR : DT_RELR, *part.relrDyn); addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRSZ : DT_RELRSZ, @@ -1441,9 +1462,9 @@ DynamicSection<ELFT>::computeContents() { addInt(DT_STRSZ, part.dynStrTab->getSize()); if (!config->zText) addInt(DT_TEXTREL, 0); - if (part.gnuHashTab) + if (part.gnuHashTab && part.gnuHashTab->getParent()) addInSec(DT_GNU_HASH, *part.gnuHashTab); - if (part.hashTab) + if (part.hashTab && part.hashTab->getParent()) addInSec(DT_HASH, *part.hashTab); if (isMain) { @@ -1626,7 +1647,7 @@ void RelocationBaseSection::addReloc(const DynamicReloc &reloc) { } void RelocationBaseSection::finalizeContents() { - SymbolTableBaseSection *symTab = getPartition().dynSymTab; + SymbolTableBaseSection *symTab = getPartition().dynSymTab.get(); // When linking glibc statically, .rel{,a}.plt contains R_*_IRELATIVE // relocations due to IFUNC (e.g. strcpy). sh_link will be set to 0 in that @@ -1636,11 +1657,11 @@ void RelocationBaseSection::finalizeContents() { else getParent()->link = 0; - if (in.relaPlt == this && in.gotPlt->getParent()) { + if (in.relaPlt.get() == this && in.gotPlt->getParent()) { getParent()->flags |= ELF::SHF_INFO_LINK; getParent()->info = in.gotPlt->getParent()->sectionIndex; } - if (in.relaIplt == this && in.igotPlt->getParent()) { + if (in.relaIplt.get() == this && in.igotPlt->getParent()) { getParent()->flags |= ELF::SHF_INFO_LINK; getParent()->info = in.igotPlt->getParent()->sectionIndex; } @@ -1677,7 +1698,7 @@ RelocationSection<ELFT>::RelocationSection(StringRef name, bool sort) } template <class ELFT> void RelocationSection<ELFT>::writeTo(uint8_t *buf) { - SymbolTableBaseSection *symTab = getPartition().dynSymTab; + SymbolTableBaseSection *symTab = getPartition().dynSymTab.get(); parallelForEach(relocs, [symTab](DynamicReloc &rel) { rel.computeRaw(symTab); }); @@ -1686,9 +1707,13 @@ template <class ELFT> void RelocationSection<ELFT>::writeTo(uint8_t *buf) { // is to make results easier to read. if (sort) { const RelType relativeRel = target->relativeRel; - parallelSort(relocs, [&](const DynamicReloc &a, const DynamicReloc &b) { - return std::make_tuple(a.type != relativeRel, a.r_sym, a.r_offset) < - std::make_tuple(b.type != relativeRel, b.r_sym, b.r_offset); + auto nonRelative = + llvm::partition(relocs, [=](auto &r) { return r.type == relativeRel; }); + parallelSort(relocs.begin(), nonRelative, + [&](auto &a, auto &b) { return a.r_offset < b.r_offset; }); + // Non-relative relocations are few, so don't bother with parallelSort. + std::sort(nonRelative, relocs.end(), [&](auto &a, auto &b) { + return std::tie(a.r_sym, a.r_offset) < std::tie(b.r_sym, b.r_offset); }); } @@ -1772,8 +1797,8 @@ bool AndroidPackedRelocationSection<ELFT>::updateAllocSize() { for (const DynamicReloc &rel : relocs) { Elf_Rela r; r.r_offset = rel.getOffset(); - r.setSymbolAndType(rel.getSymIndex(getPartition().dynSymTab), rel.type, - false); + r.setSymbolAndType(rel.getSymIndex(getPartition().dynSymTab.get()), + rel.type, false); if (config->isRela) r.r_addend = rel.computeAddend(); @@ -1999,42 +2024,30 @@ template <class ELFT> bool RelrSection<ELFT>::updateAllocSize() { const size_t nBits = wordsize * 8 - 1; // Get offsets for all relative relocations and sort them. - std::vector<uint64_t> offsets; - for (const RelativeReloc &rel : relocs) - offsets.push_back(rel.getOffset()); - llvm::sort(offsets); + std::unique_ptr<uint64_t[]> offsets(new uint64_t[relocs.size()]); + for (auto it : llvm::enumerate(relocs)) + offsets[it.index()] = it.value().getOffset(); + std::sort(offsets.get(), offsets.get() + relocs.size()); // For each leading relocation, find following ones that can be folded // as a bitmap and fold them. - for (size_t i = 0, e = offsets.size(); i < e;) { + for (size_t i = 0, e = relocs.size(); i != e;) { // Add a leading relocation. relrRelocs.push_back(Elf_Relr(offsets[i])); uint64_t base = offsets[i] + wordsize; ++i; // Find foldable relocations to construct bitmaps. - while (i < e) { + for (;;) { uint64_t bitmap = 0; - - while (i < e) { - uint64_t delta = offsets[i] - base; - - // If it is too far, it cannot be folded. - if (delta >= nBits * wordsize) - break; - - // If it is not a multiple of wordsize away, it cannot be folded. - if (delta % wordsize) + for (; i != e; ++i) { + uint64_t d = offsets[i] - base; + if (d >= nBits * wordsize || d % wordsize) break; - - // Fold it. - bitmap |= 1ULL << (delta / wordsize); - ++i; + bitmap |= uint64_t(1) << (d / wordsize); } - if (!bitmap) break; - relrRelocs.push_back(Elf_Relr((bitmap << 1) | 1)); base += nBits * wordsize; } @@ -2068,7 +2081,7 @@ static bool sortMipsSymbols(const SymbolTableEntry &l, // Sort entries related to non-local preemptible symbols by GOT indexes. // All other entries go to the beginning of a dynsym in arbitrary order. if (l.sym->isInGot() && r.sym->isInGot()) - return l.sym->gotIndex < r.sym->gotIndex; + return l.sym->getGotIdx() < r.sym->getGotIdx(); if (!l.sym->isInGot() && !r.sym->isInGot()) return false; return !l.sym->isInGot(); @@ -2099,7 +2112,7 @@ void SymbolTableBaseSection::finalizeContents() { // Only the main partition's dynsym indexes are stored in the symbols // themselves. All other partitions use a lookup table. - if (this == mainPart->dynSymTab) { + if (this == mainPart->dynSymTab.get()) { size_t i = 0; for (const SymbolTableEntry &s : symbols) s.sym->dynsymIndex = ++i; @@ -2116,9 +2129,8 @@ void SymbolTableBaseSection::finalizeContents() { void SymbolTableBaseSection::sortSymTabSymbols() { // Move all local symbols before global symbols. auto e = std::stable_partition( - symbols.begin(), symbols.end(), [](const SymbolTableEntry &s) { - return s.sym->isLocal() || s.sym->computeBinding() == STB_LOCAL; - }); + symbols.begin(), symbols.end(), + [](const SymbolTableEntry &s) { return s.sym->isLocal(); }); size_t numLocals = e - symbols.begin(); getParent()->info = numLocals + 1; @@ -2127,12 +2139,12 @@ void SymbolTableBaseSection::sortSymTabSymbols() { // symbols, they are already naturally placed first in each group. That // happens because STT_FILE is always the first symbol in the object and hence // precede all other local symbols we add for a file. - MapVector<InputFile *, std::vector<SymbolTableEntry>> arr; + MapVector<InputFile *, SmallVector<SymbolTableEntry, 0>> arr; for (const SymbolTableEntry &s : llvm::make_range(symbols.begin(), e)) arr[s.sym->file].push_back(s); auto i = symbols.begin(); - for (std::pair<InputFile *, std::vector<SymbolTableEntry>> &p : arr) + for (auto &p : arr) for (SymbolTableEntry &entry : p.second) *i++ = entry; } @@ -2146,7 +2158,7 @@ void SymbolTableBaseSection::addSymbol(Symbol *b) { } size_t SymbolTableBaseSection::getSymbolIndex(Symbol *sym) { - if (this == mainPart->dynSymTab) + if (this == mainPart->dynSymTab.get()) return sym->dynsymIndex; // Initializes symbol lookup tables lazily. This is used only for -r, @@ -2183,8 +2195,6 @@ static BssSection *getCommonSec(Symbol *sym) { } static uint32_t getSymSectionIndex(Symbol *sym) { - if (getCommonSec(sym)) - return SHN_COMMON; assert(!(sym->needsCopy && sym->isObject())); if (!isa<Defined>(sym) || sym->needsCopy) return SHN_UNDEF; @@ -2197,7 +2207,6 @@ static uint32_t getSymSectionIndex(Symbol *sym) { // Write the internal symbol table contents to the output symbol table. template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) { // The first entry is a null entry as per the ELF spec. - memset(buf, 0, sizeof(Elf_Sym)); buf += sizeof(Elf_Sym); auto *eSym = reinterpret_cast<Elf_Sym *>(buf); @@ -2206,14 +2215,10 @@ template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) { Symbol *sym = ent.sym; bool isDefinedHere = type == SHT_SYMTAB || sym->partition == partition; - // Set st_info and st_other. - eSym->st_other = 0; - if (sym->isLocal()) { - eSym->setBindingAndType(STB_LOCAL, sym->type); - } else { - eSym->setBindingAndType(sym->computeBinding(), sym->type); - eSym->setVisibility(sym->visibility); - } + // Set st_name, st_info and st_other. + eSym->st_name = ent.strTabOffset; + eSym->setBindingAndType(sym->binding, sym->type); + eSym->st_other = sym->visibility; // The 3 most significant bits of st_other are used by OpenPOWER ABI. // See getPPC64GlobalEntryToLocalEntryOffset() for more details. @@ -2224,30 +2229,29 @@ template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) { else if (config->emachine == EM_AARCH64) eSym->st_other |= sym->stOther & STO_AARCH64_VARIANT_PCS; - eSym->st_name = ent.strTabOffset; - if (isDefinedHere) - eSym->st_shndx = getSymSectionIndex(ent.sym); - else - eSym->st_shndx = 0; - - // Copy symbol size if it is a defined symbol. st_size is not significant - // for undefined symbols, so whether copying it or not is up to us if that's - // the case. We'll leave it as zero because by not setting a value, we can - // get the exact same outputs for two sets of input files that differ only - // in undefined symbol size in DSOs. - if (eSym->st_shndx == SHN_UNDEF || !isDefinedHere) - eSym->st_size = 0; - else - eSym->st_size = sym->getSize(); - - // st_value is usually an address of a symbol, but that has a special - // meaning for uninstantiated common symbols (--no-define-common). - if (BssSection *commonSec = getCommonSec(ent.sym)) + if (BssSection *commonSec = getCommonSec(sym)) { + // st_value is usually an address of a symbol, but that has a special + // meaning for uninstantiated common symbols (--no-define-common). + eSym->st_shndx = SHN_COMMON; eSym->st_value = commonSec->alignment; - else if (isDefinedHere) - eSym->st_value = sym->getVA(); - else - eSym->st_value = 0; + eSym->st_size = cast<Defined>(sym)->size; + } else { + const uint32_t shndx = getSymSectionIndex(sym); + if (isDefinedHere) { + eSym->st_shndx = shndx; + eSym->st_value = sym->getVA(); + // Copy symbol size if it is a defined symbol. st_size is not + // significant for undefined symbols, so whether copying it or not is up + // to us if that's the case. We'll leave it as zero because by not + // setting a value, we can get the exact same outputs for two sets of + // input files that differ only in undefined symbol size in DSOs. + eSym->st_size = shndx != SHN_UNDEF ? cast<Defined>(sym)->size : 0; + } else { + eSym->st_shndx = 0; + eSym->st_value = 0; + eSym->st_size = 0; + } + } ++eSym; } @@ -2298,7 +2302,7 @@ void SymtabShndxSection::writeTo(uint8_t *buf) { // we need to write actual index, otherwise, we must write SHN_UNDEF(0). buf += 4; // Ignore .symtab[0] entry. for (const SymbolTableEntry &entry : in.symTab->getSymbols()) { - if (getSymSectionIndex(entry.sym) == SHN_XINDEX) + if (!getCommonSec(entry.sym) && getSymSectionIndex(entry.sym) == SHN_XINDEX) write32(buf, entry.sym->getOutputSection()->sectionIndex); buf += 4; } @@ -2379,11 +2383,6 @@ void GnuHashTableSection::finalizeContents() { } void GnuHashTableSection::writeTo(uint8_t *buf) { - // The output buffer is not guaranteed to be zero-cleared because we pre- - // fill executable sections with trap instructions. This is a precaution - // for that case, which happens only when --no-rosegment is given. - memset(buf, 0, size); - // Write a header. write32(buf, nBuckets); write32(buf + 4, getPartition().dynSymTab->getNumSymbols() - symbols.size()); @@ -2466,8 +2465,9 @@ void GnuHashTableSection::addSymbols(SmallVectorImpl<SymbolTableEntry> &v) { symbols.push_back({b, ent.strTabOffset, hash, bucketIdx}); } - llvm::stable_sort(symbols, [](const Entry &l, const Entry &r) { - return l.bucketIdx < r.bucketIdx; + llvm::sort(symbols, [](const Entry &l, const Entry &r) { + return std::tie(l.bucketIdx, l.strTabOffset) < + std::tie(r.bucketIdx, r.strTabOffset); }); v.erase(mid, v.end()); @@ -2481,7 +2481,7 @@ HashTableSection::HashTableSection() } void HashTableSection::finalizeContents() { - SymbolTableBaseSection *symTab = getPartition().dynSymTab; + SymbolTableBaseSection *symTab = getPartition().dynSymTab.get(); if (OutputSection *sec = symTab->getParent()) getParent()->link = sec->sectionIndex; @@ -2495,11 +2495,7 @@ void HashTableSection::finalizeContents() { } void HashTableSection::writeTo(uint8_t *buf) { - SymbolTableBaseSection *symTab = getPartition().dynSymTab; - - // See comment in GnuHashTableSection::writeTo. - memset(buf, 0, size); - + SymbolTableBaseSection *symTab = getPartition().dynSymTab.get(); unsigned numSymbols = symTab->getNumSymbols(); uint32_t *p = reinterpret_cast<uint32_t *>(buf); @@ -2553,7 +2549,8 @@ void PltSection::writeTo(uint8_t *buf) { } void PltSection::addEntry(Symbol &sym) { - sym.pltIndex = entries.size(); + assert(sym.auxIdx == symAux.size() - 1); + symAux.back().pltIdx = entries.size(); entries.push_back(&sym); } @@ -2599,7 +2596,8 @@ size_t IpltSection::getSize() const { } void IpltSection::addEntry(Symbol &sym) { - sym.pltIndex = entries.size(); + assert(sym.auxIdx == symAux.size() - 1); + symAux.back().pltIdx = entries.size(); entries.push_back(&sym); } @@ -2808,7 +2806,7 @@ createSymbols(ArrayRef<std::vector<GdbIndexSection::NameAttrEntry>> nameAttrs, // For each chunk, compute the number of compilation units preceding it. uint32_t cuIdx = 0; - std::vector<uint32_t> cuIdxs(chunks.size()); + std::unique_ptr<uint32_t[]> cuIdxs(new uint32_t[chunks.size()]); for (uint32_t i = 0, e = chunks.size(); i != e; ++i) { cuIdxs[i] = cuIdx; cuIdx += chunks[i].compilationUnits.size(); @@ -2824,11 +2822,13 @@ createSymbols(ArrayRef<std::vector<GdbIndexSection::NameAttrEntry>> nameAttrs, numShards)); // A sharded map to uniquify symbols by name. - std::vector<DenseMap<CachedHashStringRef, size_t>> map(numShards); + auto map = + std::make_unique<DenseMap<CachedHashStringRef, size_t>[]>(numShards); size_t shift = 32 - countTrailingZeros(numShards); // Instantiate GdbSymbols while uniqufying them by name. - std::vector<std::vector<GdbSymbol>> symbols(numShards); + auto symbols = std::make_unique<std::vector<GdbSymbol>[]>(numShards); + parallelForEachN(0, concurrency, [&](size_t threadId) { uint32_t i = 0; for (ArrayRef<NameAttrEntry> entries : nameAttrs) { @@ -2852,14 +2852,15 @@ createSymbols(ArrayRef<std::vector<GdbIndexSection::NameAttrEntry>> nameAttrs, }); size_t numSymbols = 0; - for (ArrayRef<GdbSymbol> v : symbols) + for (ArrayRef<GdbSymbol> v : makeArrayRef(symbols.get(), numShards)) numSymbols += v.size(); // The return type is a flattened vector, so we'll copy each vector // contents to Ret. std::vector<GdbSymbol> ret; ret.reserve(numSymbols); - for (std::vector<GdbSymbol> &vec : symbols) + for (std::vector<GdbSymbol> &vec : + makeMutableArrayRef(symbols.get(), numShards)) for (GdbSymbol &sym : vec) ret.push_back(std::move(sym)); @@ -3019,8 +3020,7 @@ void EhFrameHeader::writeTo(uint8_t *buf) { void EhFrameHeader::write() { uint8_t *buf = Out::bufferStart + getParent()->offset + outSecOff; using FdeData = EhFrameSection::FdeData; - - std::vector<FdeData> fdes = getPartition().ehFrame->getFdeData(); + SmallVector<FdeData, 0> fdes = getPartition().ehFrame->getFdeData(); buf[0] = 1; buf[1] = DW_EH_PE_pcrel | DW_EH_PE_sdata4; @@ -3708,10 +3708,6 @@ static uint8_t getAbiVersion() { } template <typename ELFT> void elf::writeEhdr(uint8_t *buf, Partition &part) { - // For executable segments, the trap instructions are written before writing - // the header. Setting Elf header bytes to zero ensures that any unused bytes - // in header are zero-cleared, instead of having trap instructions. - memset(buf, 0, sizeof(typename ELFT::Ehdr)); memcpy(buf, "\177ELF", 4); auto *eHdr = reinterpret_cast<typename ELFT::Ehdr *>(buf); @@ -3799,8 +3795,9 @@ void PartitionIndexSection::writeTo(uint8_t *buf) { write32(buf, mainPart->dynStrTab->getVA() + partitions[i].nameStrTab - va); write32(buf + 4, partitions[i].elfHeader->getVA() - (va + 4)); - SyntheticSection *next = - i == partitions.size() - 1 ? in.partEnd : partitions[i + 1].elfHeader; + SyntheticSection *next = i == partitions.size() - 1 + ? in.partEnd.get() + : partitions[i + 1].elfHeader.get(); write32(buf + 8, next->getVA() - partitions[i].elfHeader->getVA()); va += 12; @@ -3808,6 +3805,30 @@ void PartitionIndexSection::writeTo(uint8_t *buf) { } } +void InStruct::reset() { + attributes.reset(); + bss.reset(); + bssRelRo.reset(); + got.reset(); + gotPlt.reset(); + igotPlt.reset(); + ppc64LongBranchTarget.reset(); + mipsGot.reset(); + mipsRldMap.reset(); + partEnd.reset(); + partIndex.reset(); + plt.reset(); + iplt.reset(); + ppc32Got2.reset(); + ibtPlt.reset(); + relaPlt.reset(); + relaIplt.reset(); + shStrTab.reset(); + strTab.reset(); + symTab.reset(); + symTabShndx.reset(); +} + InStruct elf::in; std::vector<Partition> elf::partitions; diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h index c35e19cf2fb4..9f4073048ce5 100644 --- a/lld/ELF/SyntheticSections.h +++ b/lld/ELF/SyntheticSections.h @@ -34,7 +34,6 @@ namespace elf { class Defined; struct PhdrEntry; class SymbolTableBaseSection; -class VersionNeedBaseSection; class SyntheticSection : public InputSection { public: @@ -87,7 +86,7 @@ public: uint32_t fdeVARel; }; - std::vector<FdeData> getFdeData() const; + SmallVector<FdeData, 0> getFdeData() const; ArrayRef<CieRecord *> getCieRecords() const { return cieRecords; } template <class ELFT> void iterateFDEWithLSDA(llvm::function_ref<void(InputSection &)> fn); @@ -130,8 +129,11 @@ public: void writeTo(uint8_t *buf) override; void addEntry(Symbol &sym); + bool addTlsDescEntry(Symbol &sym); bool addDynTlsEntry(Symbol &sym); bool addTlsIndex(); + uint32_t getTlsDescOffset(const Symbol &sym) const; + uint64_t getTlsDescAddr(const Symbol &sym) const; uint64_t getGlobalDynAddr(const Symbol &b) const; uint64_t getGlobalDynOffset(const Symbol &b) const; @@ -419,7 +421,7 @@ private: uint64_t size = 0; - llvm::DenseMap<StringRef, unsigned> stringMap; + llvm::DenseMap<llvm::CachedHashStringRef, unsigned> stringMap; SmallVector<StringRef, 0> strings; }; @@ -1203,24 +1205,24 @@ struct Partition { StringRef name; uint64_t nameStrTab; - SyntheticSection *elfHeader; - SyntheticSection *programHeaders; - std::vector<PhdrEntry *> phdrs; + std::unique_ptr<SyntheticSection> elfHeader; + std::unique_ptr<SyntheticSection> programHeaders; + SmallVector<PhdrEntry *, 0> phdrs; - ARMExidxSyntheticSection *armExidx; - BuildIdSection *buildId; - SyntheticSection *dynamic; - StringTableSection *dynStrTab; - SymbolTableBaseSection *dynSymTab; - EhFrameHeader *ehFrameHdr; - EhFrameSection *ehFrame; - GnuHashTableSection *gnuHashTab; - HashTableSection *hashTab; - RelocationBaseSection *relaDyn; - RelrBaseSection *relrDyn; - VersionDefinitionSection *verDef; - SyntheticSection *verNeed; - VersionTableSection *verSym; + std::unique_ptr<ARMExidxSyntheticSection> armExidx; + std::unique_ptr<BuildIdSection> buildId; + std::unique_ptr<SyntheticSection> dynamic; + std::unique_ptr<StringTableSection> dynStrTab; + std::unique_ptr<SymbolTableBaseSection> dynSymTab; + std::unique_ptr<EhFrameHeader> ehFrameHdr; + std::unique_ptr<EhFrameSection> ehFrame; + std::unique_ptr<GnuHashTableSection> gnuHashTab; + std::unique_ptr<HashTableSection> hashTab; + std::unique_ptr<RelocationBaseSection> relaDyn; + std::unique_ptr<RelrBaseSection> relrDyn; + std::unique_ptr<VersionDefinitionSection> verDef; + std::unique_ptr<SyntheticSection> verNeed; + std::unique_ptr<VersionTableSection> verSym; unsigned getNumber() const { return this - &partitions[0] + 1; } }; @@ -1235,27 +1237,29 @@ inline Partition &SectionBase::getPartition() const { // Linker generated sections which can be used as inputs and are not specific to // a partition. struct InStruct { - InputSection *attributes; - BssSection *bss; - BssSection *bssRelRo; - GotSection *got; - GotPltSection *gotPlt; - IgotPltSection *igotPlt; - PPC64LongBranchTargetSection *ppc64LongBranchTarget; - MipsGotSection *mipsGot; - MipsRldMapSection *mipsRldMap; - SyntheticSection *partEnd; - SyntheticSection *partIndex; - PltSection *plt; - IpltSection *iplt; - PPC32Got2Section *ppc32Got2; - IBTPltSection *ibtPlt; - RelocationBaseSection *relaPlt; - RelocationBaseSection *relaIplt; - StringTableSection *shStrTab; - StringTableSection *strTab; - SymbolTableBaseSection *symTab; - SymtabShndxSection *symTabShndx; + std::unique_ptr<InputSection> attributes; + std::unique_ptr<BssSection> bss; + std::unique_ptr<BssSection> bssRelRo; + std::unique_ptr<GotSection> got; + std::unique_ptr<GotPltSection> gotPlt; + std::unique_ptr<IgotPltSection> igotPlt; + std::unique_ptr<PPC64LongBranchTargetSection> ppc64LongBranchTarget; + std::unique_ptr<MipsGotSection> mipsGot; + std::unique_ptr<MipsRldMapSection> mipsRldMap; + std::unique_ptr<SyntheticSection> partEnd; + std::unique_ptr<SyntheticSection> partIndex; + std::unique_ptr<PltSection> plt; + std::unique_ptr<IpltSection> iplt; + std::unique_ptr<PPC32Got2Section> ppc32Got2; + std::unique_ptr<IBTPltSection> ibtPlt; + std::unique_ptr<RelocationBaseSection> relaPlt; + std::unique_ptr<RelocationBaseSection> relaIplt; + std::unique_ptr<StringTableSection> shStrTab; + std::unique_ptr<StringTableSection> strTab; + std::unique_ptr<SymbolTableBaseSection> symTab; + std::unique_ptr<SymtabShndxSection> symTabShndx; + + void reset(); }; extern InStruct in; diff --git a/lld/ELF/Target.cpp b/lld/ELF/Target.cpp index 88d3006f9a2d..f0e7ebfc64df 100644 --- a/lld/ELF/Target.cpp +++ b/lld/ELF/Target.cpp @@ -107,7 +107,7 @@ template <class ELFT> static ErrorPlace getErrPlace(const uint8_t *loc) { continue; } if (isecLoc <= loc && loc < isecLoc + isec->getSize()) { - auto objLoc = isec->template getLocation<ELFT>(loc - isecLoc); + std::string objLoc = isec->getLocation(loc - isecLoc); // Return object file location and source file location. // TODO: Refactor getSrcMsg not to take a variable. Undefined dummy(nullptr, "", STB_LOCAL, 0, 0); diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h index e0e97301ca98..f7b947ec3aa2 100644 --- a/lld/ELF/Target.h +++ b/lld/ELF/Target.h @@ -221,6 +221,16 @@ void addPPC64SaveRestore(); uint64_t getPPC64TocBase(); uint64_t getAArch64Page(uint64_t expr); +class AArch64Relaxer { + bool safeToRelaxAdrpLdr = true; + +public: + explicit AArch64Relaxer(ArrayRef<Relocation> relocs); + + bool tryRelaxAdrpLdr(const Relocation &adrpRel, const Relocation &ldrRel, + uint64_t secAddr, uint8_t *buf) const; +}; + extern const TargetInfo *target; TargetInfo *getTarget(); diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp index 38de4db191f4..ae740810acb5 100644 --- a/lld/ELF/Thunks.cpp +++ b/lld/ELF/Thunks.cpp @@ -27,8 +27,7 @@ #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Endian.h" @@ -434,7 +433,7 @@ void AArch64ABSLongThunk::writeTo(uint8_t *buf) { } void AArch64ABSLongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__AArch64AbsLongThunk_" + destination.getName()), + addSymbol(saver().save("__AArch64AbsLongThunk_" + destination.getName()), STT_FUNC, 0, isec); addSymbol("$x", STT_NOTYPE, 0, isec); addSymbol("$d", STT_NOTYPE, 8, isec); @@ -460,8 +459,8 @@ void AArch64ADRPThunk::writeTo(uint8_t *buf) { } void AArch64ADRPThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__AArch64ADRPThunk_" + destination.getName()), STT_FUNC, - 0, isec); + addSymbol(saver().save("__AArch64ADRPThunk_" + destination.getName()), + STT_FUNC, 0, isec); addSymbol("$x", STT_NOTYPE, 0, isec); } @@ -560,7 +559,7 @@ void ARMV7ABSLongThunk::writeLong(uint8_t *buf) { } void ARMV7ABSLongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__ARMv7ABSLongThunk_" + destination.getName()), + addSymbol(saver().save("__ARMv7ABSLongThunk_" + destination.getName()), STT_FUNC, 0, isec); addSymbol("$a", STT_NOTYPE, 0, isec); } @@ -578,7 +577,7 @@ void ThumbV7ABSLongThunk::writeLong(uint8_t *buf) { } void ThumbV7ABSLongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__Thumbv7ABSLongThunk_" + destination.getName()), + addSymbol(saver().save("__Thumbv7ABSLongThunk_" + destination.getName()), STT_FUNC, 1, isec); addSymbol("$t", STT_NOTYPE, 0, isec); } @@ -599,8 +598,8 @@ void ARMV7PILongThunk::writeLong(uint8_t *buf) { } void ARMV7PILongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__ARMV7PILongThunk_" + destination.getName()), STT_FUNC, - 0, isec); + addSymbol(saver().save("__ARMV7PILongThunk_" + destination.getName()), + STT_FUNC, 0, isec); addSymbol("$a", STT_NOTYPE, 0, isec); } @@ -620,7 +619,7 @@ void ThumbV7PILongThunk::writeLong(uint8_t *buf) { } void ThumbV7PILongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__ThumbV7PILongThunk_" + destination.getName()), + addSymbol(saver().save("__ThumbV7PILongThunk_" + destination.getName()), STT_FUNC, 1, isec); addSymbol("$t", STT_NOTYPE, 0, isec); } @@ -635,7 +634,7 @@ void ARMV5ABSLongThunk::writeLong(uint8_t *buf) { } void ARMV5ABSLongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__ARMv5ABSLongThunk_" + destination.getName()), + addSymbol(saver().save("__ARMv5ABSLongThunk_" + destination.getName()), STT_FUNC, 0, isec); addSymbol("$a", STT_NOTYPE, 0, isec); addSymbol("$d", STT_NOTYPE, 4, isec); @@ -661,8 +660,8 @@ void ARMV5PILongThunk::writeLong(uint8_t *buf) { } void ARMV5PILongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__ARMV5PILongThunk_" + destination.getName()), STT_FUNC, - 0, isec); + addSymbol(saver().save("__ARMV5PILongThunk_" + destination.getName()), + STT_FUNC, 0, isec); addSymbol("$a", STT_NOTYPE, 0, isec); addSymbol("$d", STT_NOTYPE, 12, isec); } @@ -691,7 +690,7 @@ void ThumbV6MABSLongThunk::writeLong(uint8_t *buf) { } void ThumbV6MABSLongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__Thumbv6MABSLongThunk_" + destination.getName()), + addSymbol(saver().save("__Thumbv6MABSLongThunk_" + destination.getName()), STT_FUNC, 1, isec); addSymbol("$t", STT_NOTYPE, 0, isec); addSymbol("$d", STT_NOTYPE, 8, isec); @@ -717,7 +716,7 @@ void ThumbV6MPILongThunk::writeLong(uint8_t *buf) { } void ThumbV6MPILongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__Thumbv6MPILongThunk_" + destination.getName()), + addSymbol(saver().save("__Thumbv6MPILongThunk_" + destination.getName()), STT_FUNC, 1, isec); addSymbol("$t", STT_NOTYPE, 0, isec); addSymbol("$d", STT_NOTYPE, 12, isec); @@ -735,7 +734,7 @@ void MipsThunk::writeTo(uint8_t *buf) { } void MipsThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__LA25Thunk_" + destination.getName()), STT_FUNC, 0, + addSymbol(saver().save("__LA25Thunk_" + destination.getName()), STT_FUNC, 0, isec); } @@ -758,8 +757,9 @@ void MicroMipsThunk::writeTo(uint8_t *buf) { } void MicroMipsThunk::addSymbols(ThunkSection &isec) { - Defined *d = addSymbol( - saver.save("__microLA25Thunk_" + destination.getName()), STT_FUNC, 0, isec); + Defined *d = + addSymbol(saver().save("__microLA25Thunk_" + destination.getName()), + STT_FUNC, 0, isec); d->stOther |= STO_MIPS_MICROMIPS; } @@ -782,8 +782,9 @@ void MicroMipsR6Thunk::writeTo(uint8_t *buf) { } void MicroMipsR6Thunk::addSymbols(ThunkSection &isec) { - Defined *d = addSymbol( - saver.save("__microLA25Thunk_" + destination.getName()), STT_FUNC, 0, isec); + Defined *d = + addSymbol(saver().save("__microLA25Thunk_" + destination.getName()), + STT_FUNC, 0, isec); d->stOther |= STO_MIPS_MICROMIPS; } @@ -843,7 +844,7 @@ void PPC32PltCallStub::addSymbols(ThunkSection &isec) { else os << ".plt_pic32."; os << destination.getName(); - addSymbol(saver.save(os.str()), STT_FUNC, 0, isec); + addSymbol(saver().save(os.str()), STT_FUNC, 0, isec); } bool PPC32PltCallStub::isCompatibleWith(const InputSection &isec, @@ -852,7 +853,7 @@ bool PPC32PltCallStub::isCompatibleWith(const InputSection &isec, } void PPC32LongThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__LongThunk_" + destination.getName()), STT_FUNC, 0, + addSymbol(saver().save("__LongThunk_" + destination.getName()), STT_FUNC, 0, isec); } @@ -896,8 +897,8 @@ void PPC64PltCallStub::writeTo(uint8_t *buf) { } void PPC64PltCallStub::addSymbols(ThunkSection &isec) { - Defined *s = addSymbol(saver.save("__plt_" + destination.getName()), STT_FUNC, - 0, isec); + Defined *s = addSymbol(saver().save("__plt_" + destination.getName()), + STT_FUNC, 0, isec); s->needsTocRestore = true; s->file = destination.file; } @@ -947,7 +948,7 @@ void PPC64R2SaveStub::writeTo(uint8_t *buf) { } void PPC64R2SaveStub::addSymbols(ThunkSection &isec) { - Defined *s = addSymbol(saver.save("__toc_save_" + destination.getName()), + Defined *s = addSymbol(saver().save("__toc_save_" + destination.getName()), STT_FUNC, 0, isec); s->needsTocRestore = true; } @@ -983,7 +984,7 @@ void PPC64R12SetupStub::writeTo(uint8_t *buf) { } void PPC64R12SetupStub::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__gep_setup_" + destination.getName()), STT_FUNC, 0, + addSymbol(saver().save("__gep_setup_" + destination.getName()), STT_FUNC, 0, isec); } @@ -1019,7 +1020,7 @@ void PPC64PCRelPLTStub::writeTo(uint8_t *buf) { } void PPC64PCRelPLTStub::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__plt_pcrel_" + destination.getName()), STT_FUNC, 0, + addSymbol(saver().save("__plt_pcrel_" + destination.getName()), STT_FUNC, 0, isec); } @@ -1035,7 +1036,7 @@ void PPC64LongBranchThunk::writeTo(uint8_t *buf) { } void PPC64LongBranchThunk::addSymbols(ThunkSection &isec) { - addSymbol(saver.save("__long_branch_" + destination.getName()), STT_FUNC, 0, + addSymbol(saver().save("__long_branch_" + destination.getName()), STT_FUNC, 0, isec); } diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 497e56886b72..69fcad390d61 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -20,8 +20,8 @@ #include "SyntheticSections.h" #include "Target.h" #include "lld/Common/Arrays.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Filesystem.h" -#include "lld/Common/Memory.h" #include "lld/Common/Strings.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSwitch.h" @@ -55,7 +55,6 @@ public: private: void copyLocalSymbols(); void addSectionSymbols(); - void forEachRelSec(llvm::function_ref<void(InputSectionBase &)> fn); void sortSections(); void resolveShfLinkOrder(); void finalizeAddressDependentContent(); @@ -65,7 +64,7 @@ private: void checkExecuteOnly(); void setReservedSymbolSections(); - std::vector<PhdrEntry *> createPhdrs(Partition &part); + SmallVector<PhdrEntry *, 0> createPhdrs(Partition &part); void addPhdrForSection(Partition &part, unsigned shType, unsigned pType, unsigned pFlags); void assignFileOffsets(); @@ -100,7 +99,7 @@ template <class ELFT> void elf::writeResult() { Writer<ELFT>().run(); } -static void removeEmptyPTLoad(std::vector<PhdrEntry *> &phdrs) { +static void removeEmptyPTLoad(SmallVector<PhdrEntry *, 0> &phdrs) { auto it = std::stable_partition( phdrs.begin(), phdrs.end(), [&](const PhdrEntry *p) { if (p->p_type != PT_LOAD) @@ -121,7 +120,7 @@ static void removeEmptyPTLoad(std::vector<PhdrEntry *> &phdrs) { } void elf::copySectionsIntoPartitions() { - std::vector<InputSectionBase *> newSections; + SmallVector<InputSectionBase *, 0> newSections; for (unsigned part = 2; part != partitions.size() + 1; ++part) { for (InputSectionBase *s : inputSections) { if (!(s->flags & SHF_ALLOC) || !s->isLive()) @@ -299,18 +298,18 @@ template <class ELFT> void elf::createSyntheticSections() { auto add = [](SyntheticSection &sec) { inputSections.push_back(&sec); }; - in.shStrTab = make<StringTableSection>(".shstrtab", false); + in.shStrTab = std::make_unique<StringTableSection>(".shstrtab", false); Out::programHeaders = make<OutputSection>("", 0, SHF_ALLOC); Out::programHeaders->alignment = config->wordsize; if (config->strip != StripPolicy::All) { - in.strTab = make<StringTableSection>(".strtab", false); - in.symTab = make<SymbolTableSection<ELFT>>(*in.strTab); - in.symTabShndx = make<SymtabShndxSection>(); + in.strTab = std::make_unique<StringTableSection>(".strtab", false); + in.symTab = std::make_unique<SymbolTableSection<ELFT>>(*in.strTab); + in.symTabShndx = std::make_unique<SymtabShndxSection>(); } - in.bss = make<BssSection>(".bss", 0, 1); + in.bss = std::make_unique<BssSection>(".bss", 0, 1); add(*in.bss); // If there is a SECTIONS command and a .data.rel.ro section name use name @@ -318,14 +317,14 @@ template <class ELFT> void elf::createSyntheticSections() { // This makes sure our relro is contiguous. bool hasDataRelRo = script->hasSectionsCommand && findSection(".data.rel.ro", 0); - in.bssRelRo = - make<BssSection>(hasDataRelRo ? ".data.rel.ro.bss" : ".bss.rel.ro", 0, 1); + in.bssRelRo = std::make_unique<BssSection>( + hasDataRelRo ? ".data.rel.ro.bss" : ".bss.rel.ro", 0, 1); add(*in.bssRelRo); // Add MIPS-specific sections. if (config->emachine == EM_MIPS) { if (!config->shared && config->hasDynSymTab) { - in.mipsRldMap = make<MipsRldMapSection>(); + in.mipsRldMap = std::make_unique<MipsRldMapSection>(); add(*in.mipsRldMap); } if (auto *sec = MipsAbiFlagsSection<ELFT>::create()) @@ -345,49 +344,52 @@ template <class ELFT> void elf::createSyntheticSections() { }; if (!part.name.empty()) { - part.elfHeader = make<PartitionElfHeaderSection<ELFT>>(); + part.elfHeader = std::make_unique<PartitionElfHeaderSection<ELFT>>(); part.elfHeader->name = part.name; add(*part.elfHeader); - part.programHeaders = make<PartitionProgramHeadersSection<ELFT>>(); + part.programHeaders = + std::make_unique<PartitionProgramHeadersSection<ELFT>>(); add(*part.programHeaders); } if (config->buildId != BuildIdKind::None) { - part.buildId = make<BuildIdSection>(); + part.buildId = std::make_unique<BuildIdSection>(); add(*part.buildId); } - part.dynStrTab = make<StringTableSection>(".dynstr", true); - part.dynSymTab = make<SymbolTableSection<ELFT>>(*part.dynStrTab); - part.dynamic = make<DynamicSection<ELFT>>(); + part.dynStrTab = std::make_unique<StringTableSection>(".dynstr", true); + part.dynSymTab = + std::make_unique<SymbolTableSection<ELFT>>(*part.dynStrTab); + part.dynamic = std::make_unique<DynamicSection<ELFT>>(); if (config->androidPackDynRelocs) - part.relaDyn = make<AndroidPackedRelocationSection<ELFT>>(relaDynName); - else part.relaDyn = - make<RelocationSection<ELFT>>(relaDynName, config->zCombreloc); + std::make_unique<AndroidPackedRelocationSection<ELFT>>(relaDynName); + else + part.relaDyn = std::make_unique<RelocationSection<ELFT>>( + relaDynName, config->zCombreloc); if (config->hasDynSymTab) { add(*part.dynSymTab); - part.verSym = make<VersionTableSection>(); + part.verSym = std::make_unique<VersionTableSection>(); add(*part.verSym); if (!namedVersionDefs().empty()) { - part.verDef = make<VersionDefinitionSection>(); + part.verDef = std::make_unique<VersionDefinitionSection>(); add(*part.verDef); } - part.verNeed = make<VersionNeedSection<ELFT>>(); + part.verNeed = std::make_unique<VersionNeedSection<ELFT>>(); add(*part.verNeed); if (config->gnuHash) { - part.gnuHashTab = make<GnuHashTableSection>(); + part.gnuHashTab = std::make_unique<GnuHashTableSection>(); add(*part.gnuHashTab); } if (config->sysvHash) { - part.hashTab = make<HashTableSection>(); + part.hashTab = std::make_unique<HashTableSection>(); add(*part.hashTab); } @@ -397,23 +399,23 @@ template <class ELFT> void elf::createSyntheticSections() { } if (config->relrPackDynRelocs) { - part.relrDyn = make<RelrSection<ELFT>>(); + part.relrDyn = std::make_unique<RelrSection<ELFT>>(); add(*part.relrDyn); } if (!config->relocatable) { if (config->ehFrameHdr) { - part.ehFrameHdr = make<EhFrameHeader>(); + part.ehFrameHdr = std::make_unique<EhFrameHeader>(); add(*part.ehFrameHdr); } - part.ehFrame = make<EhFrameSection>(); + part.ehFrame = std::make_unique<EhFrameSection>(); add(*part.ehFrame); } if (config->emachine == EM_ARM && !config->relocatable) { // The ARMExidxsyntheticsection replaces all the individual .ARM.exidx // InputSections. - part.armExidx = make<ARMExidxSyntheticSection>(); + part.armExidx = std::make_unique<ARMExidxSyntheticSection>(); add(*part.armExidx); } } @@ -422,13 +424,14 @@ template <class ELFT> void elf::createSyntheticSections() { // Create the partition end marker. This needs to be in partition number 255 // so that it is sorted after all other partitions. It also has other // special handling (see createPhdrs() and combineEhSections()). - in.partEnd = make<BssSection>(".part.end", config->maxPageSize, 1); + in.partEnd = + std::make_unique<BssSection>(".part.end", config->maxPageSize, 1); in.partEnd->partition = 255; add(*in.partEnd); - in.partIndex = make<PartitionIndexSection>(); - addOptionalRegular("__part_index_begin", in.partIndex, 0); - addOptionalRegular("__part_index_end", in.partIndex, + in.partIndex = std::make_unique<PartitionIndexSection>(); + addOptionalRegular("__part_index_begin", in.partIndex.get(), 0); + addOptionalRegular("__part_index_end", in.partIndex.get(), in.partIndex->getSize()); add(*in.partIndex); } @@ -436,26 +439,26 @@ template <class ELFT> void elf::createSyntheticSections() { // Add .got. MIPS' .got is so different from the other archs, // it has its own class. if (config->emachine == EM_MIPS) { - in.mipsGot = make<MipsGotSection>(); + in.mipsGot = std::make_unique<MipsGotSection>(); add(*in.mipsGot); } else { - in.got = make<GotSection>(); + in.got = std::make_unique<GotSection>(); add(*in.got); } if (config->emachine == EM_PPC) { - in.ppc32Got2 = make<PPC32Got2Section>(); + in.ppc32Got2 = std::make_unique<PPC32Got2Section>(); add(*in.ppc32Got2); } if (config->emachine == EM_PPC64) { - in.ppc64LongBranchTarget = make<PPC64LongBranchTargetSection>(); + in.ppc64LongBranchTarget = std::make_unique<PPC64LongBranchTargetSection>(); add(*in.ppc64LongBranchTarget); } - in.gotPlt = make<GotPltSection>(); + in.gotPlt = std::make_unique<GotPltSection>(); add(*in.gotPlt); - in.igotPlt = make<IgotPltSection>(); + in.igotPlt = std::make_unique<IgotPltSection>(); add(*in.igotPlt); // _GLOBAL_OFFSET_TABLE_ is defined relative to either .got.plt or .got. Treat @@ -472,7 +475,7 @@ template <class ELFT> void elf::createSyntheticSections() { // We always need to add rel[a].plt to output if it has entries. // Even for static linking it can contain R_[*]_IRELATIVE relocations. - in.relaPlt = make<RelocationSection<ELFT>>( + in.relaPlt = std::make_unique<RelocationSection<ELFT>>( config->isRela ? ".rela.plt" : ".rel.plt", /*sort=*/false); add(*in.relaPlt); @@ -482,21 +485,23 @@ template <class ELFT> void elf::createSyntheticSections() { // that would cause a section type mismatch. However, because the Android // dynamic loader reads .rel.plt after .rel.dyn, we can get the desired // behaviour by placing the iplt section in .rel.plt. - in.relaIplt = make<RelocationSection<ELFT>>( + in.relaIplt = std::make_unique<RelocationSection<ELFT>>( config->androidPackDynRelocs ? in.relaPlt->name : relaDynName, /*sort=*/false); add(*in.relaIplt); if ((config->emachine == EM_386 || config->emachine == EM_X86_64) && (config->andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)) { - in.ibtPlt = make<IBTPltSection>(); + in.ibtPlt = std::make_unique<IBTPltSection>(); add(*in.ibtPlt); } - in.plt = config->emachine == EM_PPC ? make<PPC32GlinkSection>() - : make<PltSection>(); + if (config->emachine == EM_PPC) + in.plt = std::make_unique<PPC32GlinkSection>(); + else + in.plt = std::make_unique<PltSection>(); add(*in.plt); - in.iplt = make<IpltSection>(); + in.iplt = std::make_unique<IpltSection>(); add(*in.iplt); if (config->andFeatures) @@ -532,8 +537,6 @@ template <class ELFT> void Writer<ELFT>::run() { // finalizeSections does that. finalizeSections(); checkExecuteOnly(); - if (errorCount()) - return; // If --compressed-debug-sections is specified, compress .debug_* sections. // Do it right now because it changes the size of output sections. @@ -557,9 +560,6 @@ template <class ELFT> void Writer<ELFT>::run() { for (Partition &part : partitions) setPhdrs(part); - if (config->relocatable) - for (OutputSection *sec : outputSections) - sec->addr = 0; // Handle --print-map(-M)/--Map, --why-extract=, --cref and // --print-archive-stats=. Dump them before checkSections() because the files @@ -1030,26 +1030,6 @@ template <class ELFT> void Writer<ELFT>::addRelIpltSymbols() { Out::elfHeader, 0, STV_HIDDEN); } -template <class ELFT> -void Writer<ELFT>::forEachRelSec( - llvm::function_ref<void(InputSectionBase &)> fn) { - // Scan all relocations. Each relocation goes through a series - // of tests to determine if it needs special treatment, such as - // creating GOT, PLT, copy relocations, etc. - // Note that relocations for non-alloc sections are directly - // processed by InputSection::relocateNonAlloc. - for (InputSectionBase *isec : inputSections) - if (isec->isLive() && isa<InputSection>(isec) && (isec->flags & SHF_ALLOC)) - fn(*isec); - for (Partition &part : partitions) { - for (EhInputSection *es : part.ehFrame->sections) - fn(*es); - if (part.armExidx && part.armExidx->isLive()) - for (InputSection *ex : part.armExidx->exidxSections) - fn(*ex); - } -} - // This function generates assignments for predefined symbols (e.g. _end or // _etext) and inserts them into the commands sequence to be processed at the // appropriate time. This ensures that the value is going to be correct by the @@ -1059,17 +1039,17 @@ template <class ELFT> void Writer<ELFT>::setReservedSymbolSections() { if (ElfSym::globalOffsetTable) { // The _GLOBAL_OFFSET_TABLE_ symbol is defined by target convention usually // to the start of the .got or .got.plt section. - InputSection *gotSection = in.gotPlt; + InputSection *sec = in.gotPlt.get(); if (!target->gotBaseSymInGotPlt) - gotSection = in.mipsGot ? cast<InputSection>(in.mipsGot) - : cast<InputSection>(in.got); - ElfSym::globalOffsetTable->section = gotSection; + sec = in.mipsGot.get() ? cast<InputSection>(in.mipsGot.get()) + : cast<InputSection>(in.got.get()); + ElfSym::globalOffsetTable->section = sec; } // .rela_iplt_{start,end} mark the start and the end of in.relaIplt. if (ElfSym::relaIpltStart && in.relaIplt->isNeeded()) { - ElfSym::relaIpltStart->section = in.relaIplt; - ElfSym::relaIpltEnd->section = in.relaIplt; + ElfSym::relaIpltStart->section = in.relaIplt.get(); + ElfSym::relaIpltEnd->section = in.relaIplt.get(); ElfSym::relaIpltEnd->value = in.relaIplt->getSize(); } @@ -1170,9 +1150,9 @@ static bool shouldSkip(SectionCommand *cmd) { // We want to place orphan sections so that they share as much // characteristics with their neighbors as possible. For example, if // both are rw, or both are tls. -static std::vector<SectionCommand *>::iterator -findOrphanPos(std::vector<SectionCommand *>::iterator b, - std::vector<SectionCommand *>::iterator e) { +static SmallVectorImpl<SectionCommand *>::iterator +findOrphanPos(SmallVectorImpl<SectionCommand *>::iterator b, + SmallVectorImpl<SectionCommand *>::iterator e) { OutputSection *sec = cast<OutputSection>(*e); // Find the first element that has as close a rank as possible. @@ -1209,9 +1189,9 @@ findOrphanPos(std::vector<SectionCommand *>::iterator b, auto *os = dyn_cast<OutputSection>(cmd); return os && os->hasInputSections; }; - auto j = std::find_if(llvm::make_reverse_iterator(i), - llvm::make_reverse_iterator(b), - isOutputSecWithInputSections); + auto j = + std::find_if(std::make_reverse_iterator(i), std::make_reverse_iterator(b), + isOutputSecWithInputSections); i = j.base(); // As a special case, if the orphan section is the last section, put @@ -1284,14 +1264,14 @@ static DenseMap<const InputSectionBase *, int> buildSectionOrder() { // Build a map from symbols to their priorities. Symbols that didn't // appear in the symbol ordering file have the lowest priority 0. // All explicitly mentioned symbols have negative (higher) priorities. - DenseMap<StringRef, SymbolOrderEntry> symbolOrder; + DenseMap<CachedHashStringRef, SymbolOrderEntry> symbolOrder; int priority = -config->symbolOrderingFile.size(); for (StringRef s : config->symbolOrderingFile) - symbolOrder.insert({s, {priority++, false}}); + symbolOrder.insert({CachedHashStringRef(s), {priority++, false}}); // Build a map from sections to their priorities. auto addSym = [&](Symbol &sym) { - auto it = symbolOrder.find(sym.getName()); + auto it = symbolOrder.find(CachedHashStringRef(sym.getName())); if (it == symbolOrder.end()) return; SymbolOrderEntry &ent = it->second; @@ -1310,20 +1290,16 @@ static DenseMap<const InputSectionBase *, int> buildSectionOrder() { // We want both global and local symbols. We get the global ones from the // symbol table and iterate the object files for the local ones. for (Symbol *sym : symtab->symbols()) - if (!sym->isLazy()) - addSym(*sym); + addSym(*sym); for (ELFFileBase *file : objectFiles) - for (Symbol *sym : file->getSymbols()) { - if (!sym->isLocal()) - break; + for (Symbol *sym : file->getLocalSymbols()) addSym(*sym); - } if (config->warnSymbolOrdering) for (auto orderEntry : symbolOrder) if (!orderEntry.second.present) - warn("symbol ordering file: no such symbol: " + orderEntry.first); + warn("symbol ordering file: no such symbol: " + orderEntry.first.val()); return sectionOrder; } @@ -1332,8 +1308,8 @@ static DenseMap<const InputSectionBase *, int> buildSectionOrder() { static void sortISDBySectionOrder(InputSectionDescription *isd, const DenseMap<const InputSectionBase *, int> &order) { - std::vector<InputSection *> unorderedSections; - std::vector<std::pair<InputSection *, int>> orderedSections; + SmallVector<InputSection *, 0> unorderedSections; + SmallVector<std::pair<InputSection *, int>, 0> orderedSections; uint64_t unorderedSize = 0; for (InputSection *isec : isd->sections) { @@ -1646,7 +1622,7 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() { // can assign Virtual Addresses to OutputSections that are not monotonically // increasing. for (Partition &part : partitions) - finalizeSynthetic(part.armExidx); + finalizeSynthetic(part.armExidx.get()); resolveShfLinkOrder(); // Converts call x@GDPLT to call __tls_get_addr @@ -1699,6 +1675,10 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() { } } + if (config->relocatable) + for (OutputSection *sec : outputSections) + sec->addr = 0; + // If addrExpr is set, the address may not be a multiple of the alignment. // Warn because this is error-prone. for (SectionCommand *cmd : script->sectionCommands) @@ -1766,21 +1746,19 @@ template <class ELFT> void Writer<ELFT>::optimizeBasicBlockJumps() { // jump to the following section as it is not required. // 2. If there are two consecutive jump instructions, it checks // if they can be flipped and one can be deleted. - for (OutputSection *os : outputSections) { - if (!(os->flags & SHF_EXECINSTR)) + for (OutputSection *osec : outputSections) { + if (!(osec->flags & SHF_EXECINSTR)) continue; - std::vector<InputSection *> sections = getInputSections(os); - std::vector<unsigned> result(sections.size()); + SmallVector<InputSection *, 0> sections = getInputSections(*osec); + size_t numDeleted = 0; // Delete all fall through jump instructions. Also, check if two // consecutive jump instructions can be flipped so that a fall // through jmp instruction can be deleted. - parallelForEachN(0, sections.size(), [&](size_t i) { + for (size_t i = 0, e = sections.size(); i != e; ++i) { InputSection *next = i + 1 < sections.size() ? sections[i + 1] : nullptr; - InputSection &is = *sections[i]; - result[i] = - target->deleteFallThruJmpInsn(is, is.getFile<ELFT>(), next) ? 1 : 0; - }); - size_t numDeleted = std::count(result.begin(), result.end(), 1); + InputSection &sec = *sections[i]; + numDeleted += target->deleteFallThruJmpInsn(sec, sec.file, next); + } if (numDeleted > 0) { script->assignAddresses(); LLVM_DEBUG(llvm::dbgs() @@ -1790,11 +1768,9 @@ template <class ELFT> void Writer<ELFT>::optimizeBasicBlockJumps() { fixSymbolsAfterShrinking(); - for (OutputSection *os : outputSections) { - std::vector<InputSection *> sections = getInputSections(os); - for (InputSection *is : sections) + for (OutputSection *osec : outputSections) + for (InputSection *is : getInputSections(*osec)) is->trim(); - } } // In order to allow users to manipulate linker-synthesized sections, @@ -1866,9 +1842,9 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { // Even the author of gold doesn't remember why gold behaves that way. // https://sourceware.org/ml/binutils/2002-03/msg00360.html if (mainPart->dynamic->parent) - symtab->addSymbol(Defined{/*file=*/nullptr, "_DYNAMIC", STB_WEAK, - STV_HIDDEN, STT_NOTYPE, - /*value=*/0, /*size=*/0, mainPart->dynamic}); + symtab->addSymbol( + Defined{/*file=*/nullptr, "_DYNAMIC", STB_WEAK, STV_HIDDEN, STT_NOTYPE, + /*value=*/0, /*size=*/0, mainPart->dynamic.get()}); // Define __rel[a]_iplt_{start,end} symbols if needed. addRelIpltSymbols(); @@ -1911,11 +1887,14 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { // pieces. The relocation scan uses those pieces, so this has to be // earlier. for (Partition &part : partitions) - finalizeSynthetic(part.ehFrame); + finalizeSynthetic(part.ehFrame.get()); } - for (Symbol *sym : symtab->symbols()) - sym->isPreemptible = computeIsPreemptible(*sym); + if (config->hasDynSymTab) { + parallelForEach(symtab->symbols(), [](Symbol *sym) { + sym->isPreemptible = computeIsPreemptible(*sym); + }); + } // Change values of linker-script-defined symbols from placeholders (assigned // by declareSymbols) to actual definitions. @@ -1929,7 +1908,21 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { // a linker-script-defined symbol is absolute. ppc64noTocRelax.clear(); if (!config->relocatable) { - forEachRelSec(scanRelocations<ELFT>); + // Scan all relocations. Each relocation goes through a series of tests to + // determine if it needs special treatment, such as creating GOT, PLT, + // copy relocations, etc. Note that relocations for non-alloc sections are + // directly processed by InputSection::relocateNonAlloc. + for (InputSectionBase *sec : inputSections) + if (sec->isLive() && isa<InputSection>(sec) && (sec->flags & SHF_ALLOC)) + scanRelocations<ELFT>(*sec); + for (Partition &part : partitions) { + for (EhInputSection *sec : part.ehFrame->sections) + scanRelocations<ELFT>(*sec); + if (part.armExidx && part.armExidx->isLive()) + for (InputSection *sec : part.armExidx->exidxSections) + scanRelocations<ELFT>(*sec); + } + reportUndefinedSymbols<ELFT>(); postScanRelocations(); } @@ -1955,7 +1948,7 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { for (SharedFile *file : sharedFiles) { bool allNeededIsKnown = llvm::all_of(file->dtNeeded, [&](StringRef needed) { - return symtab->soNames.count(needed); + return symtab->soNames.count(CachedHashStringRef(needed)); }); if (!allNeededIsKnown) continue; @@ -1973,6 +1966,8 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { for (Symbol *sym : symtab->symbols()) { if (!sym->isUsedInRegularObj || !includeInSymtab(*sym)) continue; + if (!config->relocatable) + sym->binding = sym->computeBinding(); if (in.symTab) in.symTab->addSymbol(sym); @@ -1997,10 +1992,6 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { } } - // Do not proceed if there was an undefined symbol. - if (errorCount()) - return; - if (in.mipsGot) in.mipsGot->build(); @@ -2076,35 +2067,35 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { { llvm::TimeTraceScope timeScope("Finalize synthetic sections"); - finalizeSynthetic(in.bss); - finalizeSynthetic(in.bssRelRo); - finalizeSynthetic(in.symTabShndx); - finalizeSynthetic(in.shStrTab); - finalizeSynthetic(in.strTab); - finalizeSynthetic(in.got); - finalizeSynthetic(in.mipsGot); - finalizeSynthetic(in.igotPlt); - finalizeSynthetic(in.gotPlt); - finalizeSynthetic(in.relaIplt); - finalizeSynthetic(in.relaPlt); - finalizeSynthetic(in.plt); - finalizeSynthetic(in.iplt); - finalizeSynthetic(in.ppc32Got2); - finalizeSynthetic(in.partIndex); + finalizeSynthetic(in.bss.get()); + finalizeSynthetic(in.bssRelRo.get()); + finalizeSynthetic(in.symTabShndx.get()); + finalizeSynthetic(in.shStrTab.get()); + finalizeSynthetic(in.strTab.get()); + finalizeSynthetic(in.got.get()); + finalizeSynthetic(in.mipsGot.get()); + finalizeSynthetic(in.igotPlt.get()); + finalizeSynthetic(in.gotPlt.get()); + finalizeSynthetic(in.relaIplt.get()); + finalizeSynthetic(in.relaPlt.get()); + finalizeSynthetic(in.plt.get()); + finalizeSynthetic(in.iplt.get()); + finalizeSynthetic(in.ppc32Got2.get()); + finalizeSynthetic(in.partIndex.get()); // Dynamic section must be the last one in this list and dynamic // symbol table section (dynSymTab) must be the first one. for (Partition &part : partitions) { - finalizeSynthetic(part.dynSymTab); - finalizeSynthetic(part.gnuHashTab); - finalizeSynthetic(part.hashTab); - finalizeSynthetic(part.verDef); - finalizeSynthetic(part.relaDyn); - finalizeSynthetic(part.relrDyn); - finalizeSynthetic(part.ehFrameHdr); - finalizeSynthetic(part.verSym); - finalizeSynthetic(part.verNeed); - finalizeSynthetic(part.dynamic); + finalizeSynthetic(part.dynSymTab.get()); + finalizeSynthetic(part.gnuHashTab.get()); + finalizeSynthetic(part.hashTab.get()); + finalizeSynthetic(part.verDef.get()); + finalizeSynthetic(part.relaDyn.get()); + finalizeSynthetic(part.relrDyn.get()); + finalizeSynthetic(part.ehFrameHdr.get()); + finalizeSynthetic(part.verSym.get()); + finalizeSynthetic(part.verNeed.get()); + finalizeSynthetic(part.dynamic.get()); } } @@ -2133,6 +2124,8 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { // sometimes using forward symbol declarations. We want to set the correct // values. They also might change after adding the thunks. finalizeAddressDependentContent(); + + // All information needed for OutputSection part of Map file is available. if (errorCount()) return; @@ -2140,8 +2133,8 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { llvm::TimeTraceScope timeScope("Finalize synthetic sections"); // finalizeAddressDependentContent may have added local symbols to the // static symbol table. - finalizeSynthetic(in.symTab); - finalizeSynthetic(in.ppc64LongBranchTarget); + finalizeSynthetic(in.symTab.get()); + finalizeSynthetic(in.ppc64LongBranchTarget.get()); } // Relaxation to delete inter-basic block jumps created by basic block @@ -2164,12 +2157,13 @@ template <class ELFT> void Writer<ELFT>::checkExecuteOnly() { if (!config->executeOnly) return; - for (OutputSection *os : outputSections) - if (os->flags & SHF_EXECINSTR) - for (InputSection *isec : getInputSections(os)) + for (OutputSection *osec : outputSections) + if (osec->flags & SHF_EXECINSTR) + for (InputSection *isec : getInputSections(*osec)) if (!(isec->flags & SHF_EXECINSTR)) - error("cannot place " + toString(isec) + " into " + toString(os->name) + - ": -execute-only does not support intermingling data and code"); + error("cannot place " + toString(isec) + " into " + + toString(osec->name) + + ": --execute-only does not support intermingling data and code"); } // The linker is expected to define SECNAME_start and SECNAME_end @@ -2223,9 +2217,9 @@ void Writer<ELFT>::addStartStopSymbols(OutputSection *sec) { StringRef s = sec->name; if (!isValidCIdentifier(s)) return; - addOptionalRegular(saver.save("__start_" + s), sec, 0, + addOptionalRegular(saver().save("__start_" + s), sec, 0, config->zStartStopVisibility); - addOptionalRegular(saver.save("__stop_" + s), sec, -1, + addOptionalRegular(saver().save("__stop_" + s), sec, -1, config->zStartStopVisibility); } @@ -2258,8 +2252,8 @@ static uint64_t computeFlags(uint64_t flags) { // Decide which program headers to create and which sections to include in each // one. template <class ELFT> -std::vector<PhdrEntry *> Writer<ELFT>::createPhdrs(Partition &part) { - std::vector<PhdrEntry *> ret; +SmallVector<PhdrEntry *, 0> Writer<ELFT>::createPhdrs(Partition &part) { + SmallVector<PhdrEntry *, 0> ret; auto addHdr = [&](unsigned type, unsigned flags) -> PhdrEntry * { ret.push_back(make<PhdrEntry>(type, flags)); return ret.back(); @@ -2858,8 +2852,8 @@ template <class ELFT> void Writer<ELFT>::writeTrapInstr() { // Fill the last page. for (PhdrEntry *p : part.phdrs) if (p->p_type == PT_LOAD && (p->p_flags & PF_X)) - fillTrap(Out::bufferStart + alignDown(p->firstSec->offset + p->p_filesz, - config->maxPageSize), + fillTrap(Out::bufferStart + + alignDown(p->firstSec->offset + p->p_filesz, 4), Out::bufferStart + alignTo(p->firstSec->offset + p->p_filesz, config->maxPageSize)); @@ -2909,15 +2903,16 @@ computeHash(llvm::MutableArrayRef<uint8_t> hashBuf, llvm::ArrayRef<uint8_t> data, std::function<void(uint8_t *dest, ArrayRef<uint8_t> arr)> hashFn) { std::vector<ArrayRef<uint8_t>> chunks = split(data, 1024 * 1024); - std::vector<uint8_t> hashes(chunks.size() * hashBuf.size()); + const size_t hashesSize = chunks.size() * hashBuf.size(); + std::unique_ptr<uint8_t[]> hashes(new uint8_t[hashesSize]); // Compute hash values. parallelForEachN(0, chunks.size(), [&](size_t i) { - hashFn(hashes.data() + i * hashBuf.size(), chunks[i]); + hashFn(hashes.get() + i * hashBuf.size(), chunks[i]); }); // Write to the final output buffer. - hashFn(hashBuf.data(), hashes); + hashFn(hashBuf.data(), makeArrayRef(hashes.get(), hashesSize)); } template <class ELFT> void Writer<ELFT>::writeBuildId() { @@ -2932,34 +2927,35 @@ template <class ELFT> void Writer<ELFT>::writeBuildId() { // Compute a hash of all sections of the output file. size_t hashSize = mainPart->buildId->hashSize; - std::vector<uint8_t> buildId(hashSize); - llvm::ArrayRef<uint8_t> buf{Out::bufferStart, size_t(fileSize)}; + std::unique_ptr<uint8_t[]> buildId(new uint8_t[hashSize]); + MutableArrayRef<uint8_t> output(buildId.get(), hashSize); + llvm::ArrayRef<uint8_t> input{Out::bufferStart, size_t(fileSize)}; switch (config->buildId) { case BuildIdKind::Fast: - computeHash(buildId, buf, [](uint8_t *dest, ArrayRef<uint8_t> arr) { + computeHash(output, input, [](uint8_t *dest, ArrayRef<uint8_t> arr) { write64le(dest, xxHash64(arr)); }); break; case BuildIdKind::Md5: - computeHash(buildId, buf, [&](uint8_t *dest, ArrayRef<uint8_t> arr) { + computeHash(output, input, [&](uint8_t *dest, ArrayRef<uint8_t> arr) { memcpy(dest, MD5::hash(arr).data(), hashSize); }); break; case BuildIdKind::Sha1: - computeHash(buildId, buf, [&](uint8_t *dest, ArrayRef<uint8_t> arr) { + computeHash(output, input, [&](uint8_t *dest, ArrayRef<uint8_t> arr) { memcpy(dest, SHA1::hash(arr).data(), hashSize); }); break; case BuildIdKind::Uuid: - if (auto ec = llvm::getRandomBytes(buildId.data(), hashSize)) + if (auto ec = llvm::getRandomBytes(buildId.get(), hashSize)) error("entropy source failure: " + ec.message()); break; default: llvm_unreachable("unknown BuildIdKind"); } for (Partition &part : partitions) - part.buildId->writeBuildId(buildId); + part.buildId->writeBuildId(output); } template void elf::createSyntheticSections<ELF32LE>(); diff --git a/lld/MachO/Arch/ARM.cpp b/lld/MachO/Arch/ARM.cpp index 42c7b8930868..4dda94d7b0f3 100644 --- a/lld/MachO/Arch/ARM.cpp +++ b/lld/MachO/Arch/ARM.cpp @@ -116,7 +116,7 @@ void ARM::relocateOne(uint8_t *loc, const Reloc &r, uint64_t value, return; } else if (isBlx && !defined->thumb) { Bitfield::set<Cond>(base, 0xe); // unconditional BL - Bitfield::set<BitfieldFlag<24>>(base, 1); + Bitfield::set<BitfieldFlag<24>>(base, true); isBlx = false; } } else { diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp index 9f71e81b073a..4fae93469b5f 100644 --- a/lld/MachO/ConcatOutputSection.cpp +++ b/lld/MachO/ConcatOutputSection.cpp @@ -13,8 +13,7 @@ #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/TimeProfiler.h" @@ -244,7 +243,7 @@ void ConcatOutputSection::finalize() { // contains several branch instructions in succession, then the distance // from the current position to the position where the thunks are inserted // grows. So leave room for a bunch of thunks. - unsigned slop = 100 * thunkSize; + unsigned slop = 256 * thunkSize; while (finalIdx < endIdx && isecAddr + inputs[finalIdx]->getSize() < isecVA + forwardBranchRange - slop) finalizeOne(inputs[finalIdx++]); @@ -322,8 +321,8 @@ void ConcatOutputSection::finalize() { // get written are happy. thunkInfo.isec->live = true; - StringRef thunkName = saver.save(funcSym->getName() + ".thunk." + - std::to_string(thunkInfo.sequence++)); + StringRef thunkName = saver().save(funcSym->getName() + ".thunk." + + std::to_string(thunkInfo.sequence++)); r.referent = thunkInfo.sym = symtab->addDefined( thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, /*size=*/thunkSize, /*isWeakDef=*/false, /*isPrivateExtern=*/true, diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h index 42528185e57c..e6849a39d03a 100644 --- a/lld/MachO/Config.h +++ b/lld/MachO/Config.h @@ -12,6 +12,7 @@ #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/BinaryFormat/MachO.h" @@ -27,6 +28,7 @@ namespace lld { namespace macho { +class InputSection; class Symbol; struct SymbolPriorityEntry; @@ -169,6 +171,12 @@ struct Configuration { std::vector<SegmentProtection> segmentProtections; llvm::DenseMap<llvm::StringRef, SymbolPriorityEntry> priorities; + llvm::MapVector<std::pair<const InputSection *, const InputSection *>, + uint64_t> + callGraphProfile; + bool callGraphProfileSort = false; + llvm::StringRef printSymbolOrder; + SectionRenameMap sectionRenameMap; SegmentRenameMap segmentRenameMap; @@ -181,7 +189,7 @@ struct Configuration { llvm::MachO::Architecture arch() const { return platformInfo.target.Arch; } - llvm::MachO::PlatformKind platform() const { + llvm::MachO::PlatformType platform() const { return platformInfo.target.Platform; } }; @@ -207,7 +215,7 @@ enum class ForceLoad { No, // Never load the archive, regardless of other flags }; -extern Configuration *config; +extern std::unique_ptr<Configuration> config; } // namespace macho } // namespace lld diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index e9d65d3c73f2..e4c9f4dd6024 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -15,6 +15,7 @@ #include "ObjC.h" #include "OutputSection.h" #include "OutputSegment.h" +#include "SectionPriorities.h" #include "SymbolTable.h" #include "Symbols.h" #include "SyntheticSections.h" @@ -59,8 +60,8 @@ using namespace llvm::sys; using namespace lld; using namespace lld::macho; -Configuration *macho::config; -DependencyTracker *macho::depTracker; +std::unique_ptr<Configuration> macho::config; +std::unique_ptr<DependencyTracker> macho::depTracker; static HeaderFileType getOutputType(const InputArgList &args) { // TODO: -r, -dylinker, -preload... @@ -128,7 +129,7 @@ static Optional<StringRef> findFramework(StringRef name) { // only append suffix if realpath() succeeds Twine suffixed = location + suffix; if (fs::exists(suffixed)) - return resolvedFrameworks[key] = saver.save(suffixed.str()); + return resolvedFrameworks[key] = saver().save(suffixed.str()); } // Suffix lookup failed, fall through to the no-suffix case. } @@ -165,7 +166,7 @@ getSearchPaths(unsigned optionCode, InputArgList &args, path::append(buffer, path); // Do not warn about paths that are computed via the syslib roots if (fs::is_directory(buffer)) { - paths.push_back(saver.save(buffer.str())); + paths.push_back(saver().save(buffer.str())); found = true; } } @@ -183,7 +184,7 @@ getSearchPaths(unsigned optionCode, InputArgList &args, SmallString<261> buffer(root); path::append(buffer, path); if (fs::is_directory(buffer)) - paths.push_back(saver.save(buffer.str())); + paths.push_back(saver().save(buffer.str())); } } return paths; @@ -249,7 +250,8 @@ static llvm::CachePruningPolicy getLTOCachePolicy(InputArgList &args) { static DenseMap<StringRef, ArchiveFile *> loadedArchives; static InputFile *addFile(StringRef path, ForceLoad forceLoadArchive, - bool isExplicit = true, bool isBundleLoader = false) { + bool isLazy = false, bool isExplicit = true, + bool isBundleLoader = false) { Optional<MemoryBufferRef> buffer = readFile(path); if (!buffer) return nullptr; @@ -319,7 +321,7 @@ static InputFile *addFile(StringRef path, ForceLoad forceLoadArchive, break; } case file_magic::macho_object: - newFile = make<ObjFile>(mbref, getModTime(path), ""); + newFile = make<ObjFile>(mbref, getModTime(path), "", isLazy); break; case file_magic::macho_dynamically_linked_shared_lib: case file_magic::macho_dynamically_linked_shared_lib_stub: @@ -331,7 +333,7 @@ static InputFile *addFile(StringRef path, ForceLoad forceLoadArchive, } break; case file_magic::bitcode: - newFile = make<BitcodeFile>(mbref, "", 0); + newFile = make<BitcodeFile>(mbref, "", 0, isLazy); break; case file_magic::macho_executable: case file_magic::macho_bundle: @@ -346,9 +348,20 @@ static InputFile *addFile(StringRef path, ForceLoad forceLoadArchive, error(path + ": unhandled file type"); } if (newFile && !isa<DylibFile>(newFile)) { + if ((isa<ObjFile>(newFile) || isa<BitcodeFile>(newFile)) && newFile->lazy && + config->forceLoadObjC) { + for (Symbol *sym : newFile->symbols) + if (sym && sym->getName().startswith(objc::klass)) { + extract(*newFile, "-ObjC"); + break; + } + if (newFile->lazy && hasObjCSection(mbref)) + extract(*newFile, "-ObjC"); + } + // printArchiveMemberLoad() prints both .a and .o names, so no need to - // print the .a name here. - if (config->printEachFile && magic != file_magic::archive) + // print the .a name here. Similarly skip lazy files. + if (config->printEachFile && magic != file_magic::archive && !isLazy) message(toString(newFile)); inputFiles.insert(newFile); } @@ -360,7 +373,7 @@ static void addLibrary(StringRef name, bool isNeeded, bool isWeak, ForceLoad forceLoadArchive) { if (Optional<StringRef> path = findLibrary(name)) { if (auto *dylibFile = dyn_cast_or_null<DylibFile>( - addFile(*path, forceLoadArchive, isExplicit))) { + addFile(*path, forceLoadArchive, /*isLazy=*/false, isExplicit))) { if (isNeeded) dylibFile->forceNeeded = true; if (isWeak) @@ -380,7 +393,7 @@ static void addFramework(StringRef name, bool isNeeded, bool isWeak, ForceLoad forceLoadArchive) { if (Optional<StringRef> path = findFramework(name)) { if (auto *dylibFile = dyn_cast_or_null<DylibFile>( - addFile(*path, forceLoadArchive, isExplicit))) { + addFile(*path, forceLoadArchive, /*isLazy=*/false, isExplicit))) { if (isNeeded) dylibFile->forceNeeded = true; if (isWeak) @@ -425,13 +438,13 @@ void macho::parseLCLinkerOption(InputFile *f, unsigned argc, StringRef data) { } } -static void addFileList(StringRef path) { +static void addFileList(StringRef path, bool isLazy) { Optional<MemoryBufferRef> buffer = readFile(path); if (!buffer) return; MemoryBufferRef mbref = *buffer; for (StringRef path : args::getLines(mbref)) - addFile(rerootPath(path), ForceLoad::Default); + addFile(rerootPath(path), ForceLoad::Default, isLazy); } // An order file has one entry per line, in the following format: @@ -448,60 +461,6 @@ static void addFileList(StringRef path) { // entry (the one nearest to the front of the list.) // // The file can also have line comments that start with '#'. -static void parseOrderFile(StringRef path) { - Optional<MemoryBufferRef> buffer = readFile(path); - if (!buffer) { - error("Could not read order file at " + path); - return; - } - - MemoryBufferRef mbref = *buffer; - size_t priority = std::numeric_limits<size_t>::max(); - for (StringRef line : args::getLines(mbref)) { - StringRef objectFile, symbol; - line = line.take_until([](char c) { return c == '#'; }); // ignore comments - line = line.ltrim(); - - CPUType cpuType = StringSwitch<CPUType>(line) - .StartsWith("i386:", CPU_TYPE_I386) - .StartsWith("x86_64:", CPU_TYPE_X86_64) - .StartsWith("arm:", CPU_TYPE_ARM) - .StartsWith("arm64:", CPU_TYPE_ARM64) - .StartsWith("ppc:", CPU_TYPE_POWERPC) - .StartsWith("ppc64:", CPU_TYPE_POWERPC64) - .Default(CPU_TYPE_ANY); - - if (cpuType != CPU_TYPE_ANY && cpuType != target->cpuType) - continue; - - // Drop the CPU type as well as the colon - if (cpuType != CPU_TYPE_ANY) - line = line.drop_until([](char c) { return c == ':'; }).drop_front(); - - constexpr std::array<StringRef, 2> fileEnds = {".o:", ".o):"}; - for (StringRef fileEnd : fileEnds) { - size_t pos = line.find(fileEnd); - if (pos != StringRef::npos) { - // Split the string around the colon - objectFile = line.take_front(pos + fileEnd.size() - 1); - line = line.drop_front(pos + fileEnd.size()); - break; - } - } - symbol = line.trim(); - - if (!symbol.empty()) { - SymbolPriorityEntry &entry = config->priorities[symbol]; - if (!objectFile.empty()) - entry.objectFiles.insert(std::make_pair(objectFile, priority)); - else - entry.anyObjectFile = std::max(entry.anyObjectFile, priority); - } - - --priority; - } -} - // We expect sub-library names of the form "libfoo", which will match a dylib // with a path of .*/libfoo.{dylib, tbd}. // XXX ld64 seems to ignore the extension entirely when matching sub-libraries; @@ -545,7 +504,8 @@ static void compileBitcodeFiles() { auto *lto = make<BitcodeCompiler>(); for (InputFile *file : inputFiles) if (auto *bitcodeFile = dyn_cast<BitcodeFile>(file)) - lto->add(*bitcodeFile); + if (!file->lazy) + lto->add(*bitcodeFile); for (ObjFile *file : lto->compile()) inputFiles.insert(file); @@ -629,11 +589,11 @@ static std::string lowerDash(StringRef s) { } // Has the side-effect of setting Config::platformInfo. -static PlatformKind parsePlatformVersion(const ArgList &args) { +static PlatformType parsePlatformVersion(const ArgList &args) { const Arg *arg = args.getLastArg(OPT_platform_version); if (!arg) { error("must specify -platform_version"); - return PlatformKind::unknown; + return PLATFORM_UNKNOWN; } StringRef platformStr = arg->getValue(0); @@ -641,20 +601,20 @@ static PlatformKind parsePlatformVersion(const ArgList &args) { StringRef sdkVersionStr = arg->getValue(2); // TODO(compnerd) see if we can generate this case list via XMACROS - PlatformKind platform = - StringSwitch<PlatformKind>(lowerDash(platformStr)) - .Cases("macos", "1", PlatformKind::macOS) - .Cases("ios", "2", PlatformKind::iOS) - .Cases("tvos", "3", PlatformKind::tvOS) - .Cases("watchos", "4", PlatformKind::watchOS) - .Cases("bridgeos", "5", PlatformKind::bridgeOS) - .Cases("mac-catalyst", "6", PlatformKind::macCatalyst) - .Cases("ios-simulator", "7", PlatformKind::iOSSimulator) - .Cases("tvos-simulator", "8", PlatformKind::tvOSSimulator) - .Cases("watchos-simulator", "9", PlatformKind::watchOSSimulator) - .Cases("driverkit", "10", PlatformKind::driverKit) - .Default(PlatformKind::unknown); - if (platform == PlatformKind::unknown) + PlatformType platform = + StringSwitch<PlatformType>(lowerDash(platformStr)) + .Cases("macos", "1", PLATFORM_MACOS) + .Cases("ios", "2", PLATFORM_IOS) + .Cases("tvos", "3", PLATFORM_TVOS) + .Cases("watchos", "4", PLATFORM_WATCHOS) + .Cases("bridgeos", "5", PLATFORM_BRIDGEOS) + .Cases("mac-catalyst", "6", PLATFORM_MACCATALYST) + .Cases("ios-simulator", "7", PLATFORM_IOSSIMULATOR) + .Cases("tvos-simulator", "8", PLATFORM_TVOSSIMULATOR) + .Cases("watchos-simulator", "9", PLATFORM_WATCHOSSIMULATOR) + .Cases("driverkit", "10", PLATFORM_DRIVERKIT) + .Default(PLATFORM_UNKNOWN); + if (platform == PLATFORM_UNKNOWN) error(Twine("malformed platform: ") + platformStr); // TODO: check validity of version strings, which varies by platform // NOTE: ld64 accepts version strings with 5 components @@ -675,7 +635,7 @@ static TargetInfo *createTargetInfo(InputArgList &args) { return nullptr; } - PlatformKind platform = parsePlatformVersion(args); + PlatformType platform = parsePlatformVersion(args); config->platformInfo.target = MachO::Target(getArchitectureFromName(archName), platform); @@ -861,27 +821,27 @@ static std::vector<SectionAlign> parseSectAlign(const opt::InputArgList &args) { return sectAligns; } -PlatformKind macho::removeSimulator(PlatformKind platform) { +PlatformType macho::removeSimulator(PlatformType platform) { switch (platform) { - case PlatformKind::iOSSimulator: - return PlatformKind::iOS; - case PlatformKind::tvOSSimulator: - return PlatformKind::tvOS; - case PlatformKind::watchOSSimulator: - return PlatformKind::watchOS; + case PLATFORM_IOSSIMULATOR: + return PLATFORM_IOS; + case PLATFORM_TVOSSIMULATOR: + return PLATFORM_TVOS; + case PLATFORM_WATCHOSSIMULATOR: + return PLATFORM_WATCHOS; default: return platform; } } static bool dataConstDefault(const InputArgList &args) { - static const std::vector<std::pair<PlatformKind, VersionTuple>> minVersion = { - {PlatformKind::macOS, VersionTuple(10, 15)}, - {PlatformKind::iOS, VersionTuple(13, 0)}, - {PlatformKind::tvOS, VersionTuple(13, 0)}, - {PlatformKind::watchOS, VersionTuple(6, 0)}, - {PlatformKind::bridgeOS, VersionTuple(4, 0)}}; - PlatformKind platform = removeSimulator(config->platformInfo.target.Platform); + static const std::vector<std::pair<PlatformType, VersionTuple>> minVersion = { + {PLATFORM_MACOS, VersionTuple(10, 15)}, + {PLATFORM_IOS, VersionTuple(13, 0)}, + {PLATFORM_TVOS, VersionTuple(13, 0)}, + {PLATFORM_WATCHOS, VersionTuple(6, 0)}, + {PLATFORM_BRIDGEOS, VersionTuple(4, 0)}}; + PlatformType platform = removeSimulator(config->platformInfo.target.Platform); auto it = llvm::find_if(minVersion, [&](const auto &p) { return p.first == platform; }); if (it != minVersion.end()) @@ -962,6 +922,7 @@ static void createFiles(const InputArgList &args) { TimeTraceScope timeScope("Load input files"); // This loop should be reserved for options whose exact ordering matters. // Other options should be handled via filtered() and/or getLastArg(). + bool isLazy = false; for (const Arg *arg : args) { const Option &opt = arg->getOption(); warnIfDeprecatedOption(opt); @@ -969,7 +930,7 @@ static void createFiles(const InputArgList &args) { switch (opt.getID()) { case OPT_INPUT: - addFile(rerootPath(arg->getValue()), ForceLoad::Default); + addFile(rerootPath(arg->getValue()), ForceLoad::Default, isLazy); break; case OPT_needed_library: if (auto *dylibFile = dyn_cast_or_null<DylibFile>( @@ -989,7 +950,7 @@ static void createFiles(const InputArgList &args) { dylibFile->forceWeakImport = true; break; case OPT_filelist: - addFileList(arg->getValue()); + addFileList(arg->getValue(), isLazy); break; case OPT_force_load: addFile(rerootPath(arg->getValue()), ForceLoad::Yes); @@ -1011,6 +972,16 @@ static void createFiles(const InputArgList &args) { opt.getID() == OPT_reexport_framework, /*isExplicit=*/true, ForceLoad::Default); break; + case OPT_start_lib: + if (isLazy) + error("nested --start-lib"); + isLazy = true; + break; + case OPT_end_lib: + if (!isLazy) + error("stray --end-lib"); + isLazy = false; + break; default: break; } @@ -1083,14 +1054,14 @@ static void referenceStubBinder() { symtab->addUndefined("dyld_stub_binder", /*file=*/nullptr, /*isWeak=*/false); } -bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, - raw_ostream &stdoutOS, raw_ostream &stderrOS) { - lld::stdoutOS = &stdoutOS; - lld::stderrOS = &stderrOS; - - errorHandler().cleanupCallback = []() { - freeArena(); +bool macho::link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, + bool disableOutput) { + // This driver-specific context will be freed later by lldMain(). + auto *ctx = new CommonLinkerContext; + ctx->e.initialize(stdoutOS, stderrOS, exitEarly, disableOutput); + ctx->e.cleanupCallback = []() { resolvedFrameworks.clear(); resolvedLibraries.clear(); cachedReads.clear(); @@ -1111,17 +1082,15 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, InputFile::resetIdCount(); }; - errorHandler().logName = args::getFilenameWithoutExe(argsArr[0]); - stderrOS.enable_colors(stderrOS.has_colors()); + ctx->e.logName = args::getFilenameWithoutExe(argsArr[0]); MachOOptTable parser; InputArgList args = parser.parse(argsArr.slice(1)); - errorHandler().errorLimitExceededMsg = - "too many errors emitted, stopping now " - "(use --error-limit=0 to see all errors)"; - errorHandler().errorLimit = args::getInteger(args, OPT_error_limit_eq, 20); - errorHandler().verbose = args.hasArg(OPT_verbose); + ctx->e.errorLimitExceededMsg = "too many errors emitted, stopping now " + "(use --error-limit=0 to see all errors)"; + ctx->e.errorLimit = args::getInteger(args, OPT_error_limit_eq, 20); + ctx->e.verbose = args.hasArg(OPT_verbose); if (args.hasArg(OPT_help_hidden)) { parser.printHelp(argsArr[0], /*showHidden=*/true); @@ -1136,11 +1105,11 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, return true; } - config = make<Configuration>(); - symtab = make<SymbolTable>(); + config = std::make_unique<Configuration>(); + symtab = std::make_unique<SymbolTable>(); target = createTargetInfo(args); - depTracker = - make<DependencyTracker>(args.getLastArgValue(OPT_dependency_info)); + depTracker = std::make_unique<DependencyTracker>( + args.getLastArgValue(OPT_dependency_info)); if (errorCount()) return false; @@ -1165,7 +1134,7 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, // these are meaningful for our text based stripping if (config->osoPrefix.equals(".") || config->osoPrefix.endswith(sep)) expanded += sep; - config->osoPrefix = saver.save(expanded.str()); + config->osoPrefix = saver().save(expanded.str()); } } @@ -1228,7 +1197,8 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, if (const Arg *arg = args.getLastArg(OPT_bundle_loader)) { if (config->outputType != MH_BUNDLE) error("-bundle_loader can only be used with MachO bundle output"); - addFile(arg->getValue(), ForceLoad::Default, /*isExplicit=*/false, + addFile(arg->getValue(), ForceLoad::Default, /*isLazy=*/false, + /*isExplicit=*/false, /*isBundleLoader=*/true); } if (const Arg *arg = args.getLastArg(OPT_umbrella)) { @@ -1246,7 +1216,7 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, config->thinLTOCacheDir = args.getLastArgValue(OPT_cache_path_lto); config->thinLTOCachePolicy = getLTOCachePolicy(args); config->runtimePaths = args::getStrings(args, OPT_rpath); - config->allLoad = args.hasArg(OPT_all_load); + config->allLoad = args.hasFlag(OPT_all_load, OPT_noall_load, false); config->archMultiple = args.hasArg(OPT_arch_multiple); config->applicationExtension = args.hasFlag( OPT_application_extension, OPT_no_application_extension, false); @@ -1262,16 +1232,20 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, config->emitDataInCodeInfo = args.hasFlag(OPT_data_in_code_info, OPT_no_data_in_code_info, true); config->icfLevel = getICFLevel(args); - config->dedupLiterals = args.hasArg(OPT_deduplicate_literals) || - config->icfLevel != ICFLevel::none; + config->dedupLiterals = + args.hasFlag(OPT_deduplicate_literals, OPT_icf_eq, false) || + config->icfLevel != ICFLevel::none; config->warnDylibInstallName = args.hasFlag( OPT_warn_dylib_install_name, OPT_no_warn_dylib_install_name, false); + config->callGraphProfileSort = args.hasFlag( + OPT_call_graph_profile_sort, OPT_no_call_graph_profile_sort, true); + config->printSymbolOrder = args.getLastArgValue(OPT_print_symbol_order); // FIXME: Add a commandline flag for this too. config->zeroModTime = getenv("ZERO_AR_DATE"); - std::array<PlatformKind, 3> encryptablePlatforms{ - PlatformKind::iOS, PlatformKind::watchOS, PlatformKind::tvOS}; + std::array<PlatformType, 3> encryptablePlatforms{ + PLATFORM_IOS, PLATFORM_WATCHOS, PLATFORM_TVOS}; config->emitEncryptionInfo = args.hasFlag(OPT_encryptable, OPT_no_encryption, is_contained(encryptablePlatforms, config->platform())); @@ -1386,7 +1360,7 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, config->adhocCodesign = args.hasFlag( OPT_adhoc_codesign, OPT_no_adhoc_codesign, (config->arch() == AK_arm64 || config->arch() == AK_arm64e) && - config->platform() == PlatformKind::macOS); + config->platform() == PLATFORM_MACOS); if (args.hasArg(OPT_v)) { message(getLLDVersion(), lld::errs()); @@ -1448,7 +1422,7 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, // Parse LTO options. if (const Arg *arg = args.getLastArg(OPT_mcpu)) - parseClangOption(saver.save("-mcpu=" + StringRef(arg->getValue())), + parseClangOption(saver().save("-mcpu=" + StringRef(arg->getValue())), arg->getSpelling()); for (const Arg *arg : args.filtered(OPT_mllvm)) @@ -1458,8 +1432,10 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, replaceCommonSymbols(); StringRef orderFile = args.getLastArgValue(OPT_order_file); - if (!orderFile.empty()) + if (!orderFile.empty()) { parseOrderFile(orderFile); + config->callGraphProfileSort = false; + } referenceStubBinder(); @@ -1509,6 +1485,8 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, } gatherInputSections(); + if (config->callGraphProfileSort) + extractCallGraphProfile(); if (config->deadStrip) markLive(); @@ -1535,11 +1513,5 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly, timeTraceProfilerCleanup(); } - - if (canExitEarly) - exitLld(errorCount() ? 1 : 0); - - bool ret = errorCount() == 0; - errorHandler().reset(); - return ret; + return errorCount() == 0; } diff --git a/lld/MachO/Driver.h b/lld/MachO/Driver.h index 4a970ac8a084..c2933344e611 100644 --- a/lld/MachO/Driver.h +++ b/lld/MachO/Driver.h @@ -13,19 +13,13 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/MachO.h" #include "llvm/Option/OptTable.h" #include "llvm/Support/MemoryBuffer.h" #include <set> #include <type_traits> -namespace llvm { -namespace MachO { -class InterfaceFile; -enum class PlatformKind : unsigned; -} // namespace MachO -} // namespace llvm - namespace lld { namespace macho { @@ -74,7 +68,7 @@ uint32_t getModTime(llvm::StringRef path); void printArchiveMemberLoad(StringRef reason, const InputFile *); // Map simulator platforms to their underlying device platform. -llvm::MachO::PlatformKind removeSimulator(llvm::MachO::PlatformKind platform); +llvm::MachO::PlatformType removeSimulator(llvm::MachO::PlatformType platform); // Helper class to export dependency info. class DependencyTracker { @@ -115,7 +109,7 @@ private: std::set<std::string> notFounds; }; -extern DependencyTracker *depTracker; +extern std::unique_ptr<DependencyTracker> depTracker; } // namespace macho } // namespace lld diff --git a/lld/MachO/DriverUtils.cpp b/lld/MachO/DriverUtils.cpp index 3c5440544614..83940b54486f 100644 --- a/lld/MachO/DriverUtils.cpp +++ b/lld/MachO/DriverUtils.cpp @@ -13,8 +13,7 @@ #include "Target.h" #include "lld/Common/Args.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Reproduce.h" #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/DenseMap.h" @@ -82,7 +81,7 @@ InputArgList MachOOptTable::parse(ArrayRef<const char *> argv) { // Expand response files (arguments in the form of @<filename>) // and then parse the argument again. - cl::ExpandResponseFiles(saver, cl::TokenizeGNUCommandLine, vec); + cl::ExpandResponseFiles(saver(), cl::TokenizeGNUCommandLine, vec); InputArgList args = ParseArgs(vec, missingIndex, missingCount); // Handle -fatal_warnings early since it converts missing argument warnings @@ -191,12 +190,12 @@ Optional<StringRef> macho::resolveDylibPath(StringRef dylibPath) { bool tbdExists = fs::exists(tbdPath); searchedDylib(tbdPath, tbdExists); if (tbdExists) - return saver.save(tbdPath.str()); + return saver().save(tbdPath.str()); bool dylibExists = fs::exists(dylibPath); searchedDylib(dylibPath, dylibExists); if (dylibExists) - return saver.save(dylibPath); + return saver().save(dylibPath); return {}; } @@ -261,7 +260,7 @@ macho::findPathCombination(const Twine &name, bool exists = fs::exists(location); searchedDylib(location, exists); if (exists) - return saver.save(location.str()); + return saver().save(location.str()); } } return {}; diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp index c596b8c1ff32..bbeb2dc09bf0 100644 --- a/lld/MachO/InputFiles.cpp +++ b/lld/MachO/InputFiles.cpp @@ -56,17 +56,18 @@ #include "SyntheticSections.h" #include "Target.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/DWARF.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" #include "lld/Common/Reproduce.h" #include "llvm/ADT/iterator.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/LTO/LTO.h" +#include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/Endian.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/TarWriter.h" +#include "llvm/Support/TimeProfiler.h" #include "llvm/TextAPI/Architecture.h" #include "llvm/TextAPI/InterfaceFile.h" @@ -114,7 +115,7 @@ static std::vector<PlatformInfo> getPlatformInfos(const InputFile *input) { std::vector<PlatformInfo> platformInfos; for (auto *cmd : findCommands<build_version_command>(hdr, LC_BUILD_VERSION)) { PlatformInfo info; - info.target.Platform = static_cast<PlatformKind>(cmd->platform); + info.target.Platform = static_cast<PlatformType>(cmd->platform); info.minimum = decodeVersion(cmd->minos); platformInfos.emplace_back(std::move(info)); } @@ -124,16 +125,16 @@ static std::vector<PlatformInfo> getPlatformInfos(const InputFile *input) { PlatformInfo info; switch (cmd->cmd) { case LC_VERSION_MIN_MACOSX: - info.target.Platform = PlatformKind::macOS; + info.target.Platform = PLATFORM_MACOS; break; case LC_VERSION_MIN_IPHONEOS: - info.target.Platform = PlatformKind::iOS; + info.target.Platform = PLATFORM_IOS; break; case LC_VERSION_MIN_TVOS: - info.target.Platform = PlatformKind::tvOS; + info.target.Platform = PLATFORM_TVOS; break; case LC_VERSION_MIN_WATCHOS: - info.target.Platform = PlatformKind::watchOS; + info.target.Platform = PLATFORM_WATCHOS; break; } info.minimum = decodeVersion(cmd->version); @@ -208,6 +209,8 @@ Optional<MemoryBufferRef> macho::readFile(StringRef path) { return cachedReads[key] = mbref; } + llvm::BumpPtrAllocator &bAlloc = lld::bAlloc(); + // Object files and archive files may be fat files, which contain multiple // real files for different CPU ISAs. Here, we search for a file that matches // with the current link target and returns it as a MemoryBufferRef. @@ -239,7 +242,7 @@ Optional<MemoryBufferRef> macho::readFile(StringRef path) { } InputFile::InputFile(Kind kind, const InterfaceFile &interface) - : id(idCount++), fileKind(kind), name(saver.save(interface.getPath())) {} + : id(idCount++), fileKind(kind), name(saver().save(interface.getPath())) {} // Some sections comprise of fixed-size records, so instead of splitting them at // symbol boundaries, we split them based on size. Records are distinct from @@ -325,6 +328,25 @@ void ObjFile::parseSections(ArrayRef<SectionHeader> sectionHeaders) { if (name == section_names::compactUnwind) compactUnwindSection = §ions.back(); } else if (segname == segment_names::llvm) { + if (name == "__cg_profile" && config->callGraphProfileSort) { + TimeTraceScope timeScope("Parsing call graph section"); + BinaryStreamReader reader(data, support::little); + while (!reader.empty()) { + uint32_t fromIndex, toIndex; + uint64_t count; + if (Error err = reader.readInteger(fromIndex)) + fatal(toString(this) + ": Expected 32-bit integer"); + if (Error err = reader.readInteger(toIndex)) + fatal(toString(this) + ": Expected 32-bit integer"); + if (Error err = reader.readInteger(count)) + fatal(toString(this) + ": Expected 64-bit integer"); + callGraph.emplace_back(); + CallGraphEntry &entry = callGraph.back(); + entry.fromIndex = fromIndex; + entry.toIndex = toIndex; + entry.count = count; + } + } // ld64 does not appear to emit contents from sections within the __LLVM // segment. Symbols within those sections point to bitcode metadata // instead of actual symbols. Global symbols within those sections could @@ -815,13 +837,21 @@ OpaqueFile::OpaqueFile(MemoryBufferRef mb, StringRef segName, sections.back().subsections.push_back({0, isec}); } -ObjFile::ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName) - : InputFile(ObjKind, mb), modTime(modTime) { +ObjFile::ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName, + bool lazy) + : InputFile(ObjKind, mb, lazy), modTime(modTime) { this->archiveName = std::string(archiveName); - if (target->wordSize == 8) - parse<LP64>(); - else - parse<ILP32>(); + if (lazy) { + if (target->wordSize == 8) + parseLazy<LP64>(); + else + parseLazy<ILP32>(); + } else { + if (target->wordSize == 8) + parse<LP64>(); + else + parse<ILP32>(); + } } template <class LP> void ObjFile::parse() { @@ -883,6 +913,32 @@ template <class LP> void ObjFile::parse() { registerCompactUnwind(); } +template <class LP> void ObjFile::parseLazy() { + using Header = typename LP::mach_header; + using NList = typename LP::nlist; + + auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart()); + auto *hdr = reinterpret_cast<const Header *>(mb.getBufferStart()); + const load_command *cmd = findCommand(hdr, LC_SYMTAB); + if (!cmd) + return; + auto *c = reinterpret_cast<const symtab_command *>(cmd); + ArrayRef<NList> nList(reinterpret_cast<const NList *>(buf + c->symoff), + c->nsyms); + const char *strtab = reinterpret_cast<const char *>(buf) + c->stroff; + symbols.resize(nList.size()); + for (auto it : llvm::enumerate(nList)) { + const NList &sym = it.value(); + if ((sym.n_type & N_EXT) && !isUndef(sym)) { + // TODO: Bound checking + StringRef name = strtab + sym.n_strx; + symbols[it.index()] = symtab->addLazyObject(name, *this); + if (!lazy) + break; + } + } +} + void ObjFile::parseDebugInfo() { std::unique_ptr<DwarfObject> dObj = DwarfObject::create(this); if (!dObj) @@ -1156,7 +1212,7 @@ DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella, // Find all the $ld$* symbols to process first. parseTrie(buf + c->export_off, c->export_size, [&](const Twine &name, uint64_t flags) { - StringRef savedName = saver.save(name); + StringRef savedName = saver().save(name); if (handleLDSymbol(savedName)) return; entries.push_back({savedName, flags}); @@ -1230,7 +1286,7 @@ DylibFile::DylibFile(const InterfaceFile &interface, DylibFile *umbrella, umbrella = this; this->umbrella = umbrella; - installName = saver.save(interface.getInstallName()); + installName = saver().save(interface.getInstallName()); compatibilityVersion = interface.getCompatibilityVersion().rawValue(); currentVersion = interface.getCurrentVersion().rawValue(); @@ -1249,7 +1305,7 @@ DylibFile::DylibFile(const InterfaceFile &interface, DylibFile *umbrella, exportingFile = isImplicitlyLinked(installName) ? this : umbrella; auto addSymbol = [&](const Twine &name) -> void { - StringRef savedName = saver.save(name); + StringRef savedName = saver().save(name); if (exportingFile->hiddenSymbols.contains(CachedHashStringRef(savedName))) return; @@ -1368,7 +1424,7 @@ void DylibFile::handleLDPreviousSymbol(StringRef name, StringRef originalName) { config->platformInfo.minimum >= end) return; - this->installName = saver.save(installName); + this->installName = saver().save(installName); if (!compatVersion.empty()) { VersionTuple cVersion; @@ -1390,7 +1446,7 @@ void DylibFile::handleLDInstallNameSymbol(StringRef name, if (!condition.consume_front("os") || version.tryParse(condition)) warn("failed to parse os version, symbol '" + originalName + "' ignored"); else if (version == config->platformInfo.minimum) - this->installName = saver.save(installName); + this->installName = saver().save(installName); } void DylibFile::handleLDHideSymbol(StringRef name, StringRef originalName) { @@ -1426,7 +1482,7 @@ ArchiveFile::ArchiveFile(std::unique_ptr<object::Archive> &&f) void ArchiveFile::addLazySymbols() { for (const object::Archive::Symbol &sym : file->symbols()) - symtab->addLazy(sym.getName(), this, sym); + symtab->addLazyArchive(sym.getName(), this, sym); } static Expected<InputFile *> loadArchiveMember(MemoryBufferRef mb, @@ -1495,7 +1551,7 @@ void ArchiveFile::fetch(const object::Archive::Symbol &sym) { static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym, BitcodeFile &file) { - StringRef name = saver.save(objSym.getName()); + StringRef name = saver().save(objSym.getName()); if (objSym.isUndefined()) return symtab->addUndefined(name, &file, /*isWeakRef=*/objSym.isWeak()); @@ -1527,8 +1583,8 @@ static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym, } BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName, - uint64_t offsetInArchive) - : InputFile(BitcodeKind, mb) { + uint64_t offsetInArchive, bool lazy) + : InputFile(BitcodeKind, mb, lazy) { this->archiveName = std::string(archiveName); std::string path = mb.getBufferIdentifier().str(); // ThinLTO assumes that all MemoryBufferRefs given to it have a unique @@ -1537,19 +1593,55 @@ BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName, // So, we append the archive name to disambiguate two members with the same // name from multiple different archives, and offset within the archive to // disambiguate two members of the same name from a single archive. - MemoryBufferRef mbref( - mb.getBuffer(), - saver.save(archiveName.empty() ? path - : archiveName + sys::path::filename(path) + - utostr(offsetInArchive))); + MemoryBufferRef mbref(mb.getBuffer(), + saver().save(archiveName.empty() + ? path + : archiveName + + sys::path::filename(path) + + utostr(offsetInArchive))); obj = check(lto::InputFile::create(mbref)); + if (lazy) + parseLazy(); + else + parse(); +} +void BitcodeFile::parse() { // Convert LTO Symbols to LLD Symbols in order to perform resolution. The // "winning" symbol will then be marked as Prevailing at LTO compilation // time. + symbols.clear(); for (const lto::InputFile::Symbol &objSym : obj->symbols()) symbols.push_back(createBitcodeSymbol(objSym, *this)); } +void BitcodeFile::parseLazy() { + symbols.resize(obj->symbols().size()); + for (auto it : llvm::enumerate(obj->symbols())) { + const lto::InputFile::Symbol &objSym = it.value(); + if (!objSym.isUndefined()) { + symbols[it.index()] = + symtab->addLazyObject(saver().save(objSym.getName()), *this); + if (!lazy) + break; + } + } +} + +void macho::extract(InputFile &file, StringRef reason) { + assert(file.lazy); + file.lazy = false; + printArchiveMemberLoad(reason, &file); + if (auto *bitcode = dyn_cast<BitcodeFile>(&file)) { + bitcode->parse(); + } else { + auto &f = cast<ObjFile>(file); + if (target->wordSize == 8) + f.parse<LP64>(); + else + f.parse<ILP32>(); + } +} + template void ObjFile::parse<LP64>(); diff --git a/lld/MachO/InputFiles.h b/lld/MachO/InputFiles.h index 36da70011170..6a4a4fdb43b6 100644 --- a/lld/MachO/InputFiles.h +++ b/lld/MachO/InputFiles.h @@ -65,6 +65,16 @@ struct Section { Section(uint64_t addr) : address(addr){}; }; +// Represents a call graph profile edge. +struct CallGraphEntry { + // The index of the caller in the symbol table. + uint32_t fromIndex; + // The index of the callee in the symbol table. + uint32_t toIndex; + // Number of calls from callee to caller in the profile. + uint64_t count; +}; + class InputFile { public: enum Kind { @@ -84,16 +94,21 @@ public: std::vector<Symbol *> symbols; std::vector<Section> sections; - // Provides an easy way to sort InputFiles deterministically. - const int id; // If not empty, this stores the name of the archive containing this file. // We use this string for creating error messages. std::string archiveName; + // Provides an easy way to sort InputFiles deterministically. + const int id; + + // True if this is a lazy ObjFile or BitcodeFile. + bool lazy = false; + protected: - InputFile(Kind kind, MemoryBufferRef mb) - : mb(mb), id(idCount++), fileKind(kind), name(mb.getBufferIdentifier()) {} + InputFile(Kind kind, MemoryBufferRef mb, bool lazy = false) + : mb(mb), id(idCount++), lazy(lazy), fileKind(kind), + name(mb.getBufferIdentifier()) {} InputFile(Kind, const llvm::MachO::InterfaceFile &); @@ -107,19 +122,22 @@ private: // .o file class ObjFile final : public InputFile { public: - ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName); + ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName, + bool lazy = false); ArrayRef<llvm::MachO::data_in_code_entry> getDataInCode() const; + template <class LP> void parse(); static bool classof(const InputFile *f) { return f->kind() == ObjKind; } llvm::DWARFUnit *compileUnit = nullptr; const uint32_t modTime; std::vector<ConcatInputSection *> debugSections; + std::vector<CallGraphEntry> callGraph; private: Section *compactUnwindSection = nullptr; - template <class LP> void parse(); + template <class LP> void parseLazy(); template <class SectionHeader> void parseSections(ArrayRef<SectionHeader>); template <class LP> void parseSymbols(ArrayRef<typename LP::section> sectionHeaders, @@ -218,10 +236,14 @@ private: class BitcodeFile final : public InputFile { public: explicit BitcodeFile(MemoryBufferRef mb, StringRef archiveName, - uint64_t offsetInArchive); + uint64_t offsetInArchive, bool lazy = false); static bool classof(const InputFile *f) { return f->kind() == BitcodeKind; } + void parse(); std::unique_ptr<llvm::lto::InputFile> obj; + +private: + void parseLazy(); }; extern llvm::SetVector<InputFile *> inputFiles; @@ -229,6 +251,8 @@ extern llvm::DenseMap<llvm::CachedHashStringRef, MemoryBufferRef> cachedReads; llvm::Optional<MemoryBufferRef> readFile(StringRef path); +void extract(InputFile &file, StringRef reason); + namespace detail { template <class CommandType, class... Types> diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index fa137223c426..5535e871c539 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -53,6 +53,7 @@ public: virtual bool isLive(uint64_t off) const = 0; virtual void markLive(uint64_t off) = 0; virtual InputSection *canonical() { return this; } + virtual const InputSection *canonical() const { return this; } OutputSection *parent = nullptr; @@ -124,6 +125,9 @@ public: ConcatInputSection *canonical() override { return replacement ? replacement : this; } + const InputSection *canonical() const override { + return replacement ? replacement : this; + } static bool classof(const InputSection *isec) { return isec->kind() == ConcatKind; @@ -234,7 +238,9 @@ public: bool isLive(uint64_t off) const override { return live[off >> power2LiteralSize]; } - void markLive(uint64_t off) override { live[off >> power2LiteralSize] = 1; } + void markLive(uint64_t off) override { + live[off >> power2LiteralSize] = true; + } static bool classof(const InputSection *isec) { return isec->kind() == WordLiteralKind; diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp index c71ea33d2896..fd49a09229d1 100644 --- a/lld/MachO/LTO.cpp +++ b/lld/MachO/LTO.cpp @@ -14,7 +14,7 @@ #include "Target.h" #include "lld/Common/Args.h" -#include "lld/Common/ErrorHandler.h" +#include "lld/Common/CommonLinkerContext.h" #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" #include "llvm/LTO/Config.h" @@ -148,7 +148,7 @@ std::vector<ObjFile *> BitcodeCompiler::compile() { modTime = getModTime(filePath); } ret.push_back(make<ObjFile>( - MemoryBufferRef(buf[i], saver.save(filePath.str())), modTime, "")); + MemoryBufferRef(buf[i], saver().save(filePath.str())), modTime, "")); } for (std::unique_ptr<MemoryBuffer> &file : files) if (file) diff --git a/lld/MachO/MapFile.cpp b/lld/MachO/MapFile.cpp index 79471eecbd52..93abea2ed08b 100644 --- a/lld/MachO/MapFile.cpp +++ b/lld/MachO/MapFile.cpp @@ -40,26 +40,6 @@ using namespace llvm::sys; using namespace lld; using namespace lld::macho; -using SymbolMapTy = DenseMap<const InputSection *, SmallVector<Defined *, 4>>; - -// Returns a map from sections to their symbols. -static SymbolMapTy getSectionSyms(ArrayRef<Defined *> syms) { - SymbolMapTy ret; - for (Defined *dr : syms) - ret[dr->isec].push_back(dr); - - // Sort symbols by address. We want to print out symbols in the order they - // appear in the output file rather than the order they appeared in the input - // files. - for (auto &it : ret) - parallelSort( - it.second.begin(), it.second.end(), [](Defined *a, Defined *b) { - return a->getVA() != b->getVA() ? a->getVA() < b->getVA() - : a->getName() < b->getName(); - }); - return ret; -} - // Returns a list of all symbols that we want to print out. static std::vector<Defined *> getSymbols() { std::vector<Defined *> v; @@ -126,7 +106,10 @@ void macho::writeMapFile() { // Collect symbol info that we want to print out. std::vector<Defined *> syms = getSymbols(); - SymbolMapTy sectionSyms = getSectionSyms(syms); + parallelSort(syms.begin(), syms.end(), [](Defined *a, Defined *b) { + return a->getVA() != b->getVA() ? a->getVA() < b->getVA() + : a->getName() < b->getName(); + }); DenseMap<Symbol *, std::string> symStr = getSymbolStrings(syms); // Dump table of sections @@ -144,15 +127,9 @@ void macho::writeMapFile() { // Dump table of symbols os << "# Symbols:\n"; os << "# Address\t File Name\n"; - for (InputSection *isec : inputSections) { - auto symsIt = sectionSyms.find(isec); - assert(!shouldOmitFromOutput(isec) || (symsIt == sectionSyms.end())); - if (symsIt == sectionSyms.end()) - continue; - for (Symbol *sym : symsIt->second) { - os << format("0x%08llX\t[%3u] %s\n", sym->getVA(), - readerToFileOrdinal[sym->getFile()], symStr[sym].c_str()); - } + for (Symbol *sym : syms) { + os << format("0x%08llX\t[%3u] %s\n", sym->getVA(), + readerToFileOrdinal[sym->getFile()], symStr[sym].c_str()); } // TODO: when we implement -dead_strip, we should dump dead stripped symbols diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td index 7cd4d01a0a5a..3d1d97641d71 100644 --- a/lld/MachO/Options.td +++ b/lld/MachO/Options.td @@ -73,12 +73,25 @@ def thinlto_cache_policy: Joined<["--"], "thinlto-cache-policy=">, Group<grp_lld>; def O : JoinedOrSeparate<["-"], "O">, HelpText<"Optimize output file size">; +def start_lib: Flag<["--"], "start-lib">, + HelpText<"Start a grouping of objects that should be treated as if they were together in an archive">; +def end_lib: Flag<["--"], "end-lib">, + HelpText<"End a grouping of objects that should be treated as if they were together in an archive">; def no_warn_dylib_install_name: Joined<["--"], "no-warn-dylib-install-name">, HelpText<"Do not warn on -install-name if -dylib is not passed (default)">, Group<grp_lld>; def warn_dylib_install_name: Joined<["--"], "warn-dylib-install-name">, HelpText<"Warn on -install-name if -dylib is not passed">, Group<grp_lld>; +def call_graph_profile_sort: Flag<["--"], "call-graph-profile-sort">, + HelpText<"Reorder sections with call graph profile (default)">, + Group<grp_lld>; +def no_call_graph_profile_sort : Flag<["--"], "no-call-graph-profile-sort">, + HelpText<"Do not reorder sections with call graph profile">, + Group<grp_lld>; +def print_symbol_order: Joined<["--"], "print-symbol-order=">, + HelpText<"Print a symbol order specified by --call-graph-profile-sort into the specified file">, + Group<grp_lld>; // This is a complete Options.td compiled from Apple's ld(1) manpage // dated 2018-03-07 and cross checked with ld64 source code in repo @@ -214,6 +227,9 @@ def F : JoinedOrSeparate<["-"], "F">, def all_load : Flag<["-"], "all_load">, HelpText<"Load all members of all static archive libraries">, Group<grp_libs>; +def noall_load : Flag<["-"], "noall_load">, + HelpText<"Don't load all static members from archives, this is the default, this negates -all_load">, + Group<grp_libs>; def ObjC : Flag<["-"], "ObjC">, HelpText<"Load all members of static archives that are an Objective-C class or category.">, Group<grp_libs>; @@ -980,10 +996,6 @@ def no_dead_strip_inits_and_terms : Flag<["-"], "no_dead_strip_inits_and_terms"> HelpText<"Unnecessary option: initialization and termination are roots of the dead strip graph, so never dead stripped">, Flags<[HelpHidden]>, Group<grp_deprecated>; -def noall_load : Flag<["-"], "noall_load">, - HelpText<"Unnecessary option: this is already the default">, - Flags<[HelpHidden]>, - Group<grp_deprecated>; def grp_obsolete : OptionGroup<"obsolete">, HelpText<"OBSOLETE">; diff --git a/lld/MachO/SectionPriorities.cpp b/lld/MachO/SectionPriorities.cpp new file mode 100644 index 000000000000..35510d7338e8 --- /dev/null +++ b/lld/MachO/SectionPriorities.cpp @@ -0,0 +1,379 @@ +//===- SectionPriorities.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This is based on the ELF port, see ELF/CallGraphSort.cpp for the details +/// about the algorithm. +/// +//===----------------------------------------------------------------------===// + +#include "SectionPriorities.h" +#include "Config.h" +#include "InputFiles.h" +#include "Symbols.h" +#include "Target.h" +#include "lld/Common/Args.h" +#include "lld/Common/CommonLinkerContext.h" +#include "lld/Common/ErrorHandler.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/Optional.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/TimeProfiler.h" +#include "llvm/Support/raw_ostream.h" +#include <numeric> + +using namespace llvm; +using namespace llvm::MachO; +using namespace llvm::sys; +using namespace lld; +using namespace lld::macho; + +namespace { +struct Edge { + int from; + uint64_t weight; +}; + +struct Cluster { + Cluster(int sec, size_t s) : next(sec), prev(sec), size(s) {} + + double getDensity() const { + if (size == 0) + return 0; + return double(weight) / double(size); + } + + int next; + int prev; + uint64_t size; + uint64_t weight = 0; + uint64_t initialWeight = 0; + Edge bestPred = {-1, 0}; +}; + +class CallGraphSort { +public: + CallGraphSort(); + + DenseMap<const InputSection *, size_t> run(); + +private: + std::vector<Cluster> clusters; + std::vector<const InputSection *> sections; +}; +// Maximum amount the combined cluster density can be worse than the original +// cluster to consider merging. +constexpr int MAX_DENSITY_DEGRADATION = 8; +} // end anonymous namespace + +using SectionPair = std::pair<const InputSection *, const InputSection *>; + +// Take the edge list in config->callGraphProfile, resolve symbol names to +// Symbols, and generate a graph between InputSections with the provided +// weights. +CallGraphSort::CallGraphSort() { + MapVector<SectionPair, uint64_t> &profile = config->callGraphProfile; + DenseMap<const InputSection *, int> secToCluster; + + auto getOrCreateCluster = [&](const InputSection *isec) -> int { + auto res = secToCluster.try_emplace(isec, clusters.size()); + if (res.second) { + sections.push_back(isec); + clusters.emplace_back(clusters.size(), isec->getSize()); + } + return res.first->second; + }; + + // Create the graph + for (std::pair<SectionPair, uint64_t> &c : profile) { + const auto fromSec = c.first.first->canonical(); + const auto toSec = c.first.second->canonical(); + uint64_t weight = c.second; + // Ignore edges between input sections belonging to different output + // sections. This is done because otherwise we would end up with clusters + // containing input sections that can't actually be placed adjacently in the + // output. This messes with the cluster size and density calculations. We + // would also end up moving input sections in other output sections without + // moving them closer to what calls them. + if (fromSec->parent != toSec->parent) + continue; + + int from = getOrCreateCluster(fromSec); + int to = getOrCreateCluster(toSec); + + clusters[to].weight += weight; + + if (from == to) + continue; + + // Remember the best edge. + Cluster &toC = clusters[to]; + if (toC.bestPred.from == -1 || toC.bestPred.weight < weight) { + toC.bestPred.from = from; + toC.bestPred.weight = weight; + } + } + for (Cluster &c : clusters) + c.initialWeight = c.weight; +} + +// It's bad to merge clusters which would degrade the density too much. +static bool isNewDensityBad(Cluster &a, Cluster &b) { + double newDensity = double(a.weight + b.weight) / double(a.size + b.size); + return newDensity < a.getDensity() / MAX_DENSITY_DEGRADATION; +} + +// Find the leader of V's belonged cluster (represented as an equivalence +// class). We apply union-find path-halving technique (simple to implement) in +// the meantime as it decreases depths and the time complexity. +static int getLeader(std::vector<int> &leaders, int v) { + while (leaders[v] != v) { + leaders[v] = leaders[leaders[v]]; + v = leaders[v]; + } + return v; +} + +static void mergeClusters(std::vector<Cluster> &cs, Cluster &into, int intoIdx, + Cluster &from, int fromIdx) { + int tail1 = into.prev, tail2 = from.prev; + into.prev = tail2; + cs[tail2].next = intoIdx; + from.prev = tail1; + cs[tail1].next = fromIdx; + into.size += from.size; + into.weight += from.weight; + from.size = 0; + from.weight = 0; +} + +// Group InputSections into clusters using the Call-Chain Clustering heuristic +// then sort the clusters by density. +DenseMap<const InputSection *, size_t> CallGraphSort::run() { + const uint64_t maxClusterSize = target->getPageSize(); + + // Cluster indices sorted by density. + std::vector<int> sorted(clusters.size()); + // For union-find. + std::vector<int> leaders(clusters.size()); + + std::iota(leaders.begin(), leaders.end(), 0); + std::iota(sorted.begin(), sorted.end(), 0); + + llvm::stable_sort(sorted, [&](int a, int b) { + return clusters[a].getDensity() > clusters[b].getDensity(); + }); + + for (int l : sorted) { + // The cluster index is the same as the index of its leader here because + // clusters[L] has not been merged into another cluster yet. + Cluster &c = clusters[l]; + + // Don't consider merging if the edge is unlikely. + if (c.bestPred.from == -1 || c.bestPred.weight * 10 <= c.initialWeight) + continue; + + int predL = getLeader(leaders, c.bestPred.from); + // Already in the same cluster. + if (l == predL) + continue; + + Cluster *predC = &clusters[predL]; + if (c.size + predC->size > maxClusterSize) + continue; + + if (isNewDensityBad(*predC, c)) + continue; + + leaders[l] = predL; + mergeClusters(clusters, *predC, predL, c, l); + } + // Sort remaining non-empty clusters by density. + sorted.clear(); + for (int i = 0, e = (int)clusters.size(); i != e; ++i) + if (clusters[i].size > 0) + sorted.push_back(i); + llvm::stable_sort(sorted, [&](int a, int b) { + return clusters[a].getDensity() > clusters[b].getDensity(); + }); + + DenseMap<const InputSection *, size_t> orderMap; + + // Sections will be sorted by decreasing order. Absent sections will have + // priority 0 and be placed at the end of sections. + // NB: This is opposite from COFF/ELF to be compatible with the existing + // order-file code. + int curOrder = clusters.size(); + for (int leader : sorted) { + for (int i = leader;;) { + orderMap[sections[i]] = curOrder--; + i = clusters[i].next; + if (i == leader) + break; + } + } + if (!config->printSymbolOrder.empty()) { + std::error_code ec; + raw_fd_ostream os(config->printSymbolOrder, ec, sys::fs::OF_None); + if (ec) { + error("cannot open " + config->printSymbolOrder + ": " + ec.message()); + return orderMap; + } + // Print the symbols ordered by C3, in the order of decreasing curOrder + // Instead of sorting all the orderMap, just repeat the loops above. + for (int leader : sorted) + for (int i = leader;;) { + const InputSection *isec = sections[i]; + // Search all the symbols in the file of the section + // and find out a Defined symbol with name that is within the + // section. + for (Symbol *sym : isec->getFile()->symbols) { + if (auto *d = dyn_cast_or_null<Defined>(sym)) { + if (d->isec == isec) + os << sym->getName() << "\n"; + } + } + i = clusters[i].next; + if (i == leader) + break; + } + } + + return orderMap; +} + +static size_t getSymbolPriority(const SymbolPriorityEntry &entry, + const InputFile *f) { + // We don't use toString(InputFile *) here because it returns the full path + // for object files, and we only want the basename. + StringRef filename; + if (f->archiveName.empty()) + filename = path::filename(f->getName()); + else + filename = saver().save(path::filename(f->archiveName) + "(" + + path::filename(f->getName()) + ")"); + return std::max(entry.objectFiles.lookup(filename), entry.anyObjectFile); +} + +void macho::extractCallGraphProfile() { + TimeTraceScope timeScope("Extract call graph profile"); + for (const InputFile *file : inputFiles) { + auto *obj = dyn_cast_or_null<ObjFile>(file); + if (!obj) + continue; + for (const CallGraphEntry &entry : obj->callGraph) { + assert(entry.fromIndex < obj->symbols.size() && + entry.toIndex < obj->symbols.size()); + auto *fromSym = dyn_cast_or_null<Defined>(obj->symbols[entry.fromIndex]); + auto *toSym = dyn_cast_or_null<Defined>(obj->symbols[entry.toIndex]); + + if (!fromSym || !toSym) + continue; + config->callGraphProfile[{fromSym->isec, toSym->isec}] += entry.count; + } + } +} + +void macho::parseOrderFile(StringRef path) { + Optional<MemoryBufferRef> buffer = readFile(path); + if (!buffer) { + error("Could not read order file at " + path); + return; + } + + MemoryBufferRef mbref = *buffer; + size_t priority = std::numeric_limits<size_t>::max(); + for (StringRef line : args::getLines(mbref)) { + StringRef objectFile, symbol; + line = line.take_until([](char c) { return c == '#'; }); // ignore comments + line = line.ltrim(); + + CPUType cpuType = StringSwitch<CPUType>(line) + .StartsWith("i386:", CPU_TYPE_I386) + .StartsWith("x86_64:", CPU_TYPE_X86_64) + .StartsWith("arm:", CPU_TYPE_ARM) + .StartsWith("arm64:", CPU_TYPE_ARM64) + .StartsWith("ppc:", CPU_TYPE_POWERPC) + .StartsWith("ppc64:", CPU_TYPE_POWERPC64) + .Default(CPU_TYPE_ANY); + + if (cpuType != CPU_TYPE_ANY && cpuType != target->cpuType) + continue; + + // Drop the CPU type as well as the colon + if (cpuType != CPU_TYPE_ANY) + line = line.drop_until([](char c) { return c == ':'; }).drop_front(); + + constexpr std::array<StringRef, 2> fileEnds = {".o:", ".o):"}; + for (StringRef fileEnd : fileEnds) { + size_t pos = line.find(fileEnd); + if (pos != StringRef::npos) { + // Split the string around the colon + objectFile = line.take_front(pos + fileEnd.size() - 1); + line = line.drop_front(pos + fileEnd.size()); + break; + } + } + symbol = line.trim(); + + if (!symbol.empty()) { + SymbolPriorityEntry &entry = config->priorities[symbol]; + if (!objectFile.empty()) + entry.objectFiles.insert(std::make_pair(objectFile, priority)); + else + entry.anyObjectFile = std::max(entry.anyObjectFile, priority); + } + + --priority; + } +} + +// Sort sections by the profile data provided by __LLVM,__cg_profile sections. +// +// This first builds a call graph based on the profile data then merges sections +// according to the C³ heuristic. All clusters are then sorted by a density +// metric to further improve locality. +static DenseMap<const InputSection *, size_t> computeCallGraphProfileOrder() { + TimeTraceScope timeScope("Call graph profile sort"); + return CallGraphSort().run(); +} + +// Each section gets assigned the priority of the highest-priority symbol it +// contains. +DenseMap<const InputSection *, size_t> macho::buildInputSectionPriorities() { + if (config->callGraphProfileSort) + return computeCallGraphProfileOrder(); + DenseMap<const InputSection *, size_t> sectionPriorities; + + if (config->priorities.empty()) + return sectionPriorities; + + auto addSym = [&](Defined &sym) { + if (sym.isAbsolute()) + return; + + auto it = config->priorities.find(sym.getName()); + if (it == config->priorities.end()) + return; + + SymbolPriorityEntry &entry = it->second; + size_t &priority = sectionPriorities[sym.isec]; + priority = + std::max(priority, getSymbolPriority(entry, sym.isec->getFile())); + }; + + // TODO: Make sure this handles weak symbols correctly. + for (const InputFile *file : inputFiles) { + if (isa<ObjFile>(file)) + for (Symbol *sym : file->symbols) + if (auto *d = dyn_cast_or_null<Defined>(sym)) + addSym(*d); + } + + return sectionPriorities; +} diff --git a/lld/MachO/SectionPriorities.h b/lld/MachO/SectionPriorities.h new file mode 100644 index 000000000000..9cc4eff958cd --- /dev/null +++ b/lld/MachO/SectionPriorities.h @@ -0,0 +1,55 @@ +//===- SectionPriorities.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLD_MACHO_SECTION_PRIORITIES_H +#define LLD_MACHO_SECTION_PRIORITIES_H + +#include "InputSection.h" +#include "llvm/ADT/DenseMap.h" + +namespace lld { +namespace macho { + +// Reads every input section's call graph profile, and combines them into +// config->callGraphProfile. If an order file is present, any edges where one +// or both of the vertices are specified in the order file are discarded. +void extractCallGraphProfile(); + +// Reads the order file at `path` into config->priorities. +// +// An order file has one entry per line, in the following format: +// +// <cpu>:<object file>:<symbol name> +// +// <cpu> and <object file> are optional. If not specified, then that entry +// matches any symbol of that name. Parsing this format is not quite +// straightforward because the symbol name itself can contain colons, so when +// encountering a colon, we consider the preceding characters to decide if it +// can be a valid CPU type or file path. +// +// If a symbol is matched by multiple entries, then it takes the lowest-ordered +// entry (the one nearest to the front of the list.) +// +// The file can also have line comments that start with '#'. +void parseOrderFile(StringRef path); + +// Returns layout priorities for some or all input sections. Sections are laid +// out in decreasing order; that is, a higher priority section will be closer +// to the beginning of its output section. +// +// If either an order file or a call graph profile are present, this is used +// as the source of priorities. If both are present, the order file takes +// precedence. If neither is present, an empty map is returned. +// +// Each section gets assigned the priority of the highest-priority symbol it +// contains. +llvm::DenseMap<const InputSection *, size_t> buildInputSectionPriorities(); +} // namespace macho +} // namespace lld + +#endif diff --git a/lld/MachO/SymbolTable.cpp b/lld/MachO/SymbolTable.cpp index cec717e0cd31..c7017c8a000d 100644 --- a/lld/MachO/SymbolTable.cpp +++ b/lld/MachO/SymbolTable.cpp @@ -113,8 +113,10 @@ Symbol *SymbolTable::addUndefined(StringRef name, InputFile *file, if (wasInserted) replaceSymbol<Undefined>(s, name, file, refState); - else if (auto *lazy = dyn_cast<LazySymbol>(s)) + else if (auto *lazy = dyn_cast<LazyArchive>(s)) lazy->fetchArchiveMember(); + else if (isa<LazyObject>(s)) + extract(*s->getFile(), s->getName()); else if (auto *dynsym = dyn_cast<DylibSymbol>(s)) dynsym->reference(refState); else if (auto *undefined = dyn_cast<Undefined>(s)) @@ -178,14 +180,14 @@ Symbol *SymbolTable::addDynamicLookup(StringRef name) { return addDylib(name, /*file=*/nullptr, /*isWeakDef=*/false, /*isTlv=*/false); } -Symbol *SymbolTable::addLazy(StringRef name, ArchiveFile *file, - const object::Archive::Symbol &sym) { +Symbol *SymbolTable::addLazyArchive(StringRef name, ArchiveFile *file, + const object::Archive::Symbol &sym) { Symbol *s; bool wasInserted; std::tie(s, wasInserted) = insert(name, file); if (wasInserted) { - replaceSymbol<LazySymbol>(s, file, sym); + replaceSymbol<LazyArchive>(s, file, sym); } else if (isa<Undefined>(s)) { file->fetch(sym); } else if (auto *dysym = dyn_cast<DylibSymbol>(s)) { @@ -193,7 +195,27 @@ Symbol *SymbolTable::addLazy(StringRef name, ArchiveFile *file, if (dysym->getRefState() != RefState::Unreferenced) file->fetch(sym); else - replaceSymbol<LazySymbol>(s, file, sym); + replaceSymbol<LazyArchive>(s, file, sym); + } + } + return s; +} + +Symbol *SymbolTable::addLazyObject(StringRef name, InputFile &file) { + Symbol *s; + bool wasInserted; + std::tie(s, wasInserted) = insert(name, &file); + + if (wasInserted) { + replaceSymbol<LazyObject>(s, file, name); + } else if (isa<Undefined>(s)) { + extract(file, name); + } else if (auto *dysym = dyn_cast<DylibSymbol>(s)) { + if (dysym->isWeakDef()) { + if (dysym->getRefState() != RefState::Unreferenced) + extract(file, name); + else + replaceSymbol<LazyObject>(s, file, name); } } return s; @@ -319,4 +341,4 @@ void lld::macho::treatUndefinedSymbol(const Undefined &sym, StringRef source) { } } -SymbolTable *macho::symtab; +std::unique_ptr<SymbolTable> macho::symtab; diff --git a/lld/MachO/SymbolTable.h b/lld/MachO/SymbolTable.h index 625f78aa6141..5f844170efe0 100644 --- a/lld/MachO/SymbolTable.h +++ b/lld/MachO/SymbolTable.h @@ -51,8 +51,9 @@ public: Symbol *addDylib(StringRef name, DylibFile *file, bool isWeakDef, bool isTlv); Symbol *addDynamicLookup(StringRef name); - Symbol *addLazy(StringRef name, ArchiveFile *file, - const llvm::object::Archive::Symbol &sym); + Symbol *addLazyArchive(StringRef name, ArchiveFile *file, + const llvm::object::Archive::Symbol &sym); + Symbol *addLazyObject(StringRef name, InputFile &file); Defined *addSynthetic(StringRef name, InputSection *, uint64_t value, bool isPrivateExtern, bool includeInSymtab, @@ -70,7 +71,7 @@ private: void treatUndefinedSymbol(const Undefined &, StringRef source = ""); -extern SymbolTable *symtab; +extern std::unique_ptr<SymbolTable> symtab; } // namespace macho } // namespace lld diff --git a/lld/MachO/Symbols.cpp b/lld/MachO/Symbols.cpp index bb6d073dcf30..7bf8c6dd56d6 100644 --- a/lld/MachO/Symbols.cpp +++ b/lld/MachO/Symbols.cpp @@ -9,6 +9,7 @@ #include "Symbols.h" #include "InputFiles.h" #include "SyntheticSections.h" +#include "lld/Common/Strings.h" using namespace llvm; using namespace lld; @@ -27,17 +28,12 @@ static_assert(sizeof(void *) != 8 || sizeof(Defined) == 80, static_assert(sizeof(SymbolUnion) == sizeof(Defined), "Defined should be the largest Symbol kind"); -// Returns a symbol for an error message. -static std::string demangle(StringRef symName) { - if (config->demangle) - return demangleItanium(symName); - return std::string(symName); +std::string lld::toString(const Symbol &sym) { + return demangle(sym.getName(), config->demangle); } -std::string lld::toString(const Symbol &sym) { return demangle(sym.getName()); } - std::string lld::toMachOString(const object::Archive::Symbol &b) { - return demangle(b.getName()); + return demangle(b.getName(), config->demangle); } uint64_t Symbol::getStubVA() const { return in.stubs->getVA(stubsIndex); } @@ -105,4 +101,4 @@ uint64_t DylibSymbol::getVA() const { return isInStubs() ? getStubVA() : Symbol::getVA(); } -void LazySymbol::fetchArchiveMember() { getFile()->fetch(sym); } +void LazyArchive::fetchArchiveMember() { getFile()->fetch(sym); } diff --git a/lld/MachO/Symbols.h b/lld/MachO/Symbols.h index d1182a0a2d32..b3d86b0d3cad 100644 --- a/lld/MachO/Symbols.h +++ b/lld/MachO/Symbols.h @@ -37,7 +37,8 @@ public: UndefinedKind, CommonKind, DylibKind, - LazyKind, + LazyArchiveKind, + LazyObjectKind, }; virtual ~Symbol() {} @@ -51,6 +52,9 @@ public: } bool isLive() const { return used; } + bool isLazy() const { + return symbolKind == LazyArchiveKind || symbolKind == LazyObjectKind; + } virtual uint64_t getVA() const { return 0; } @@ -280,26 +284,39 @@ private: const bool tlv : 1; }; -class LazySymbol : public Symbol { +class LazyArchive : public Symbol { public: - LazySymbol(ArchiveFile *file, const llvm::object::Archive::Symbol &sym) - : Symbol(LazyKind, sym.getName(), file), sym(sym) {} + LazyArchive(ArchiveFile *file, const llvm::object::Archive::Symbol &sym) + : Symbol(LazyArchiveKind, sym.getName(), file), sym(sym) {} ArchiveFile *getFile() const { return cast<ArchiveFile>(file); } void fetchArchiveMember(); - static bool classof(const Symbol *s) { return s->kind() == LazyKind; } + static bool classof(const Symbol *s) { return s->kind() == LazyArchiveKind; } private: const llvm::object::Archive::Symbol sym; }; +// A defined symbol in an ObjFile/BitcodeFile surrounded by --start-lib and +// --end-lib. +class LazyObject : public Symbol { +public: + LazyObject(InputFile &file, StringRef name) + : Symbol(LazyObjectKind, name, &file) { + isUsedInRegularObj = false; + } + + static bool classof(const Symbol *s) { return s->kind() == LazyObjectKind; } +}; + union SymbolUnion { alignas(Defined) char a[sizeof(Defined)]; alignas(Undefined) char b[sizeof(Undefined)]; alignas(CommonSymbol) char c[sizeof(CommonSymbol)]; alignas(DylibSymbol) char d[sizeof(DylibSymbol)]; - alignas(LazySymbol) char e[sizeof(LazySymbol)]; + alignas(LazyArchive) char e[sizeof(LazyArchive)]; + alignas(LazyObject) char f[sizeof(LazyObject)]; }; template <typename T, typename... ArgT> diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index b64a9db485c5..3fe551f5684b 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -16,8 +16,7 @@ #include "SymbolTable.h" #include "Symbols.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/EndianStream.h" @@ -85,7 +84,7 @@ static uint32_t cpuSubtype() { if (config->outputType == MH_EXECUTE && !config->staticLink && target->cpuSubtype == CPU_SUBTYPE_X86_64_ALL && - config->platform() == PlatformKind::macOS && + config->platform() == PLATFORM_MACOS && config->platformInfo.minimum >= VersionTuple(10, 5)) subtype |= CPU_SUBTYPE_LIB64; @@ -834,7 +833,7 @@ void SymtabSection::emitBeginSourceStab(DWARFUnit *compileUnit) { if (!dir.endswith(sep)) dir += sep; stab.strx = stringTableSection.addString( - saver.save(dir + compileUnit->getUnitDIE().getShortName())); + saver().save(dir + compileUnit->getUnitDIE().getShortName())); stabs.emplace_back(std::move(stab)); } @@ -856,7 +855,7 @@ void SymtabSection::emitObjectFileStab(ObjFile *file) { if (!file->archiveName.empty()) path.append({"(", file->getName(), ")"}); - StringRef adjustedPath = saver.save(path.str()); + StringRef adjustedPath = saver().save(path.str()); adjustedPath.consume_front(config->osoPrefix); stab.strx = stringTableSection.addString(adjustedPath); @@ -1283,7 +1282,10 @@ void BitcodeBundleSection::finalize() { using namespace llvm::sys::fs; CHECK_EC(createTemporaryFile("bitcode-bundle", "xar", xarPath)); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" xar_t xar(xar_open(xarPath.data(), O_RDWR)); +#pragma clang diagnostic pop if (!xar) fatal("failed to open XAR temporary file at " + xarPath); CHECK_EC(xar_opt_set(xar, XAR_OPT_COMPRESSION, XAR_OPT_VAL_NONE)); @@ -1379,26 +1381,15 @@ DeduplicatedCStringSection::DeduplicatedCStringSection() void DeduplicatedCStringSection::finalizeContents() { // Add all string pieces to the string table builder to create section // contents. - for (const CStringInputSection *isec : inputs) + for (CStringInputSection *isec : inputs) { for (size_t i = 0, e = isec->pieces.size(); i != e; ++i) if (isec->pieces[i].live) - builder.add(isec->getCachedHashStringRef(i)); + isec->pieces[i].outSecOff = + builder.add(isec->getCachedHashStringRef(i)); + isec->isFinal = true; + } - // Fix the string table content. After this, the contents will never change. builder.finalizeInOrder(); - - // finalize() fixed tail-optimized strings, so we can now get - // offsets of strings. Get an offset for each string and save it - // to a corresponding SectionPiece for easy access. - for (CStringInputSection *isec : inputs) { - for (size_t i = 0, e = isec->pieces.size(); i != e; ++i) { - if (!isec->pieces[i].live) - continue; - isec->pieces[i].outSecOff = - builder.getOffset(isec->getCachedHashStringRef(i)); - isec->isFinal = true; - } - } } // This section is actually emitted as __TEXT,__const by ld64, but clang may @@ -1474,7 +1465,7 @@ void WordLiteralSection::writeTo(uint8_t *buf) const { void macho::createSyntheticSymbols() { auto addHeaderSymbol = [](const char *name) { symtab->addSynthetic(name, in.header->isec, /*value=*/0, - /*privateExtern=*/true, /*includeInSymtab=*/false, + /*isPrivateExtern=*/true, /*includeInSymtab=*/false, /*referencedDynamically=*/false); }; @@ -1487,11 +1478,11 @@ void macho::createSyntheticSymbols() { // Otherwise, it's an absolute symbol. if (config->isPic) symtab->addSynthetic("__mh_execute_header", in.header->isec, /*value=*/0, - /*privateExtern=*/false, /*includeInSymtab=*/true, + /*isPrivateExtern=*/false, /*includeInSymtab=*/true, /*referencedDynamically=*/true); else symtab->addSynthetic("__mh_execute_header", /*isec=*/nullptr, /*value=*/0, - /*privateExtern=*/false, /*includeInSymtab=*/true, + /*isPrivateExtern=*/false, /*includeInSymtab=*/true, /*referencedDynamically=*/true); break; diff --git a/lld/MachO/UnwindInfoSection.cpp b/lld/MachO/UnwindInfoSection.cpp index d28c7a33ff36..49af2f6ad9a8 100644 --- a/lld/MachO/UnwindInfoSection.cpp +++ b/lld/MachO/UnwindInfoSection.cpp @@ -226,7 +226,7 @@ void UnwindInfoSectionImpl<Ptr>::prepareRelocations(ConcatInputSection *isec) { // (See discussions/alternatives already considered on D107533) if (!defined->isExternal()) if (Symbol *sym = symtab->find(defined->getName())) - if (sym->kind() != Symbol::LazyKind) + if (!sym->isLazy()) r.referent = s = sym; } if (auto *undefined = dyn_cast<Undefined>(s)) { diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp index 8903f0189ef9..2c0794e08ae3 100644 --- a/lld/MachO/Writer.cpp +++ b/lld/MachO/Writer.cpp @@ -14,6 +14,7 @@ #include "MapFile.h" #include "OutputSection.h" #include "OutputSegment.h" +#include "SectionPriorities.h" #include "SymbolTable.h" #include "Symbols.h" #include "SyntheticSections.h" @@ -21,8 +22,7 @@ #include "UnwindInfoSection.h" #include "lld/Common/Arrays.h" -#include "lld/Common/ErrorHandler.h" -#include "lld/Common/Memory.h" +#include "lld/Common/CommonLinkerContext.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/LEB128.h" @@ -415,19 +415,19 @@ public: void writeTo(uint8_t *buf) const override { auto *c = reinterpret_cast<version_min_command *>(buf); switch (platformInfo.target.Platform) { - case PlatformKind::macOS: + case PLATFORM_MACOS: c->cmd = LC_VERSION_MIN_MACOSX; break; - case PlatformKind::iOS: - case PlatformKind::iOSSimulator: + case PLATFORM_IOS: + case PLATFORM_IOSSIMULATOR: c->cmd = LC_VERSION_MIN_IPHONEOS; break; - case PlatformKind::tvOS: - case PlatformKind::tvOSSimulator: + case PLATFORM_TVOS: + case PLATFORM_TVOSSIMULATOR: c->cmd = LC_VERSION_MIN_TVOS; break; - case PlatformKind::watchOS: - case PlatformKind::watchOSSimulator: + case PLATFORM_WATCHOS: + case PLATFORM_WATCHOSSIMULATOR: c->cmd = LC_VERSION_MIN_WATCHOS; break; default: @@ -610,7 +610,7 @@ static bool needsBinding(const Symbol *sym) { } static void prepareSymbolRelocation(Symbol *sym, const InputSection *isec, - const Reloc &r) { + const lld::macho::Reloc &r) { assert(sym->isLive()); const RelocAttrs &relocAttrs = target->getRelocAttrs(r.type); @@ -643,7 +643,7 @@ void Writer::scanRelocations() { continue; for (auto it = isec->relocs.begin(); it != isec->relocs.end(); ++it) { - Reloc &r = *it; + lld::macho::Reloc &r = *it; if (target->hasAttr(r.type, RelocAttrBits::SUBTRAHEND)) { // Skip over the following UNSIGNED relocation -- it's just there as the // minuend, and doesn't have the usual UNSIGNED semantics. We don't want @@ -709,14 +709,14 @@ void Writer::scanSymbols() { // TODO: ld64 enforces the old load commands in a few other cases. static bool useLCBuildVersion(const PlatformInfo &platformInfo) { - static const std::vector<std::pair<PlatformKind, VersionTuple>> minVersion = { - {PlatformKind::macOS, VersionTuple(10, 14)}, - {PlatformKind::iOS, VersionTuple(12, 0)}, - {PlatformKind::iOSSimulator, VersionTuple(13, 0)}, - {PlatformKind::tvOS, VersionTuple(12, 0)}, - {PlatformKind::tvOSSimulator, VersionTuple(13, 0)}, - {PlatformKind::watchOS, VersionTuple(5, 0)}, - {PlatformKind::watchOSSimulator, VersionTuple(6, 0)}}; + static const std::vector<std::pair<PlatformType, VersionTuple>> minVersion = { + {PLATFORM_MACOS, VersionTuple(10, 14)}, + {PLATFORM_IOS, VersionTuple(12, 0)}, + {PLATFORM_IOSSIMULATOR, VersionTuple(13, 0)}, + {PLATFORM_TVOS, VersionTuple(12, 0)}, + {PLATFORM_TVOSSIMULATOR, VersionTuple(13, 0)}, + {PLATFORM_WATCHOS, VersionTuple(5, 0)}, + {PLATFORM_WATCHOSSIMULATOR, VersionTuple(6, 0)}}; auto it = llvm::find_if(minVersion, [&](const auto &p) { return p.first == platformInfo.target.Platform; }); @@ -849,52 +849,6 @@ template <class LP> void Writer::createLoadCommands() { : 0)); } -static size_t getSymbolPriority(const SymbolPriorityEntry &entry, - const InputFile *f) { - // We don't use toString(InputFile *) here because it returns the full path - // for object files, and we only want the basename. - StringRef filename; - if (f->archiveName.empty()) - filename = path::filename(f->getName()); - else - filename = saver.save(path::filename(f->archiveName) + "(" + - path::filename(f->getName()) + ")"); - return std::max(entry.objectFiles.lookup(filename), entry.anyObjectFile); -} - -// Each section gets assigned the priority of the highest-priority symbol it -// contains. -static DenseMap<const InputSection *, size_t> buildInputSectionPriorities() { - DenseMap<const InputSection *, size_t> sectionPriorities; - - if (config->priorities.empty()) - return sectionPriorities; - - auto addSym = [&](Defined &sym) { - if (sym.isAbsolute()) - return; - - auto it = config->priorities.find(sym.getName()); - if (it == config->priorities.end()) - return; - - SymbolPriorityEntry &entry = it->second; - size_t &priority = sectionPriorities[sym.isec]; - priority = - std::max(priority, getSymbolPriority(entry, sym.isec->getFile())); - }; - - // TODO: Make sure this handles weak symbols correctly. - for (const InputFile *file : inputFiles) { - if (isa<ObjFile>(file)) - for (Symbol *sym : file->symbols) - if (auto *d = dyn_cast_or_null<Defined>(sym)) - addSym(*d); - } - - return sectionPriorities; -} - // Sorting only can happen once all outputs have been collected. Here we sort // segments, output sections within each segment, and input sections within each // output segment. @@ -908,13 +862,28 @@ static void sortSegmentsAndSections() { uint32_t sectionIndex = 0; for (OutputSegment *seg : outputSegments) { seg->sortOutputSections(); + // References from thread-local variable sections are treated as offsets + // relative to the start of the thread-local data memory area, which + // is initialized via copying all the TLV data sections (which are all + // contiguous). If later data sections require a greater alignment than + // earlier ones, the offsets of data within those sections won't be + // guaranteed to aligned unless we normalize alignments. We therefore use + // the largest alignment for all TLV data sections. + uint32_t tlvAlign = 0; + for (const OutputSection *osec : seg->getSections()) + if (isThreadLocalData(osec->flags) && osec->align > tlvAlign) + tlvAlign = osec->align; + for (OutputSection *osec : seg->getSections()) { // Now that the output sections are sorted, assign the final // output section indices. if (!osec->isHidden()) osec->index = ++sectionIndex; - if (!firstTLVDataSection && isThreadLocalData(osec->flags)) - firstTLVDataSection = osec; + if (isThreadLocalData(osec->flags)) { + if (!firstTLVDataSection) + firstTLVDataSection = osec; + osec->align = tlvAlign; + } if (!isecPriorities.empty()) { if (auto *merged = dyn_cast<ConcatOutputSection>(osec)) { @@ -1108,7 +1077,7 @@ void Writer::writeUuid() { threadFutures.reserve(chunks.size()); for (size_t i = 0; i < chunks.size(); ++i) threadFutures.emplace_back(threadPool.async( - [&](size_t i) { hashes[i] = xxHash64(chunks[i]); }, i)); + [&](size_t j) { hashes[j] = xxHash64(chunks[j]); }, i)); for (std::shared_future<void> &future : threadFutures) future.wait(); @@ -1160,7 +1129,13 @@ template <class LP> void Writer::run() { sortSegmentsAndSections(); createLoadCommands<LP>(); finalizeAddresses(); - threadPool.async(writeMapFile); + threadPool.async([&] { + if (LLVM_ENABLE_THREADS && config->timeTraceEnabled) + timeTraceProfilerInitialize(config->timeTraceGranularity, "writeMapFile"); + writeMapFile(); + if (LLVM_ENABLE_THREADS && config->timeTraceEnabled) + timeTraceProfilerFinishThread(); + }); finalizeLinkEditSegment(); writeOutputFile(); } @@ -1192,7 +1167,7 @@ void macho::createSyntheticSections() { // This section contains space for just a single word, and will be used by // dyld to cache an address to the image loader it uses. - uint8_t *arr = bAlloc.Allocate<uint8_t>(target->wordSize); + uint8_t *arr = bAlloc().Allocate<uint8_t>(target->wordSize); memset(arr, 0, target->wordSize); in.imageLoaderCache = make<ConcatInputSection>( segment_names::data, section_names::data, /*file=*/nullptr, diff --git a/lld/MachO/ld64-vs-lld.rst b/lld/MachO/ld64-vs-lld.rst index 14fb72a2ac8f..b31b4bb0ea89 100644 --- a/lld/MachO/ld64-vs-lld.rst +++ b/lld/MachO/ld64-vs-lld.rst @@ -4,6 +4,22 @@ LD64 vs LLD-MACHO This doc lists all significant deliberate differences in behavior between LD64 and LLD-MachO. +String literal deduplication +**************************** +LD64 always deduplicates string literals. LLD only does it when the `--icf=` or +the `--deduplicate-literals` flag is passed. Omitting deduplication by default +ensures that our link is as fast as possible. However, it may also break some +programs which have (incorrectly) relied on string deduplication always +occurring. In particular, programs which compare string literals via pointer +equality must be fixed to use value equality instead. + +``-no_deduplicate`` Flag +********************** +- LD64: + * This turns off ICF (deduplication pass) in the linker. +- LLD + * This turns off ICF and string merging in the linker. + ObjC symbols treatment ********************** There are differences in how LLD and LD64 handle ObjC symbols loaded from archives. diff --git a/lld/docs/_templates/indexsidebar.html b/lld/docs/_templates/indexsidebar.html index 588be9309bde..f9ecb724153d 100644 --- a/lld/docs/_templates/indexsidebar.html +++ b/lld/docs/_templates/indexsidebar.html @@ -1,4 +1,9 @@ <h3>Bugs</h3> -<p>lld bugs should be reported at the - LLVM <a href="https://bugs.llvm.org/">Bugzilla</a>.</p> +<p> +To report bugs, please visit +<a href="https://github.com/llvm/llvm-project/labels/lld:COFF">PE/COFF</a>, +<a href="https://github.com/llvm/llvm-project/labels/lld:ELF">ELF</a>, +<a href="https://github.com/llvm/llvm-project/labels/lld:MachO">Mach-O</a>, or +<a href="https://github.com/llvm/llvm-project/labels/lld:wasm">WebAssembly</a>. +</p> diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1 index 04f0982b4ced..da43cf0ef7ab 100644 --- a/lld/docs/ld.lld.1 +++ b/lld/docs/ld.lld.1 @@ -486,7 +486,7 @@ Save the current state of and .Fl -whole-archive. .It Fl -pop-state -Undo the effect of +Restore the states saved by .Fl -push-state. .It Fl -relocatable , Fl r Create relocatable object file. diff --git a/lld/include/lld/Common/CommonLinkerContext.h b/lld/include/lld/Common/CommonLinkerContext.h new file mode 100644 index 000000000000..3954d38ded63 --- /dev/null +++ b/lld/include/lld/Common/CommonLinkerContext.h @@ -0,0 +1,65 @@ +//===- CommonLinkerContext.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Entry point for all global state in lldCommon. The objective is for LLD to be +// used "as a library" in a thread-safe manner. +// +// Instead of program-wide globals or function-local statics, we prefer +// aggregating all "global" states into a heap-based structure +// (CommonLinkerContext). This also achieves deterministic initialization & +// shutdown for all "global" states. +// +//===----------------------------------------------------------------------===// + +#ifndef LLD_COMMON_COMMONLINKINGCONTEXT_H +#define LLD_COMMON_COMMONLINKINGCONTEXT_H + +#include "lld/Common/ErrorHandler.h" +#include "lld/Common/Memory.h" +#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/Support/StringSaver.h" + +namespace llvm { +class raw_ostream; +} // namespace llvm + +namespace lld { +struct SpecificAllocBase; +class CommonLinkerContext { +public: + CommonLinkerContext(); + virtual ~CommonLinkerContext(); + + static void destroy(); + + llvm::BumpPtrAllocator bAlloc; + llvm::StringSaver saver{bAlloc}; + llvm::DenseMap<void *, SpecificAllocBase *> instances; + + ErrorHandler e; + +private: + llvm::codegen::RegisterCodeGenFlags cgf; +}; + +// Retrieve the global state. Currently only one state can exist per process, +// but in the future we plan on supporting an arbitrary number of LLD instances +// in a single process. +CommonLinkerContext &commonContext(); + +template <typename T = CommonLinkerContext> T &context() { + return static_cast<T &>(commonContext()); +} + +bool hasContext(); + +inline llvm::StringSaver &saver() { return context().saver; } +inline llvm::BumpPtrAllocator &bAlloc() { return context().bAlloc; } +} // namespace lld + +#endif diff --git a/lld/include/lld/Common/Driver.h b/lld/include/lld/Common/Driver.h index 0e505a16463e..91cb91b9f808 100644 --- a/lld/include/lld/Common/Driver.h +++ b/lld/include/lld/Common/Driver.h @@ -9,6 +9,7 @@ #ifndef LLD_COMMON_DRIVER_H #define LLD_COMMON_DRIVER_H +#include "lld/Common/CommonLinkerContext.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/raw_ostream.h" @@ -28,28 +29,28 @@ SafeReturn safeLldMain(int argc, const char **argv, llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS); namespace coff { -bool link(llvm::ArrayRef<const char *> args, bool canExitEarly, - llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS); +bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput); } namespace mingw { -bool link(llvm::ArrayRef<const char *> args, bool canExitEarly, - llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS); +bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput); } namespace elf { -bool link(llvm::ArrayRef<const char *> args, bool canExitEarly, - llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS); +bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput); } namespace macho { -bool link(llvm::ArrayRef<const char *> args, bool canExitEarly, - llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS); +bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput); } namespace wasm { -bool link(llvm::ArrayRef<const char *> args, bool canExitEarly, - llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS); +bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS, + llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput); } } diff --git a/lld/include/lld/Common/ErrorHandler.h b/lld/include/lld/Common/ErrorHandler.h index d95a2537c1f2..ce077290d60b 100644 --- a/lld/include/lld/Common/ErrorHandler.h +++ b/lld/include/lld/Common/ErrorHandler.h @@ -73,6 +73,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileOutputBuffer.h" +#include <mutex> namespace llvm { class DiagnosticInfo; @@ -81,11 +82,6 @@ class raw_ostream; namespace lld { -// We wrap stdout and stderr so that you can pass alternative stdout/stderr as -// arguments to lld::*::link() functions. -extern llvm::raw_ostream *stdoutOS; -extern llvm::raw_ostream *stderrOS; - llvm::raw_ostream &outs(); llvm::raw_ostream &errs(); @@ -93,6 +89,11 @@ enum class ErrorTag { LibNotFound, SymbolNotFound }; class ErrorHandler { public: + ~ErrorHandler(); + + void initialize(llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS, + bool exitEarly, bool disableOutput); + uint64_t errorCount = 0; uint64_t errorLimit = 20; StringRef errorLimitExceededMsg = "too many errors emitted, stopping now"; @@ -112,11 +113,9 @@ public: void message(const Twine &msg, llvm::raw_ostream &s); void warn(const Twine &msg); - void reset() { - if (cleanupCallback) - cleanupCallback(); - *this = ErrorHandler(); - } + raw_ostream &outs(); + raw_ostream &errs(); + void flushStreams(); std::unique_ptr<llvm::FileOutputBuffer> outputBuffer; @@ -126,6 +125,19 @@ private: std::string getLocation(const Twine &msg); void reportDiagnostic(StringRef location, Colors c, StringRef diagKind, const Twine &msg); + + // We want to separate multi-line messages with a newline. `sep` is "\n" + // if the last messages was multi-line. Otherwise "". + llvm::StringRef sep; + + // We wrap stdout and stderr so that you can pass alternative stdout/stderr as + // arguments to lld::*::link() functions. Since lld::outs() or lld::errs() can + // be indirectly called from multiple threads, we protect them using a mutex. + // In the future, we plan on supporting several concurent linker contexts, + // which explains why the mutex is not a global but part of this context. + std::mutex mu; + llvm::raw_ostream *stdoutOS{}; + llvm::raw_ostream *stderrOS{}; }; /// Returns the default error handler. diff --git a/lld/include/lld/Common/Memory.h b/lld/include/lld/Common/Memory.h index f516a327cfb2..0b2f474c3013 100644 --- a/lld/include/lld/Common/Memory.h +++ b/lld/include/lld/Common/Memory.h @@ -22,42 +22,41 @@ #define LLD_COMMON_MEMORY_H #include "llvm/Support/Allocator.h" -#include "llvm/Support/StringSaver.h" -#include <vector> namespace lld { - -// Use this arena if your object doesn't have a destructor. -extern llvm::BumpPtrAllocator bAlloc; -extern llvm::StringSaver saver; - -void freeArena(); - -// These two classes are hack to keep track of all -// SpecificBumpPtrAllocator instances. +// A base class only used by the CommonLinkerContext to keep track of the +// SpecificAlloc<> instances. struct SpecificAllocBase { - SpecificAllocBase() { instances.push_back(this); } virtual ~SpecificAllocBase() = default; - virtual void reset() = 0; - static std::vector<SpecificAllocBase *> instances; + static SpecificAllocBase *getOrCreate(void *tag, size_t size, size_t align, + SpecificAllocBase *(&creator)(void *)); }; +// An arena of specific types T, created on-demand. template <class T> struct SpecificAlloc : public SpecificAllocBase { - void reset() override { alloc.DestroyAll(); } + static SpecificAllocBase *create(void *storage) { + return new (storage) SpecificAlloc<T>(); + } llvm::SpecificBumpPtrAllocator<T> alloc; + static int tag; }; -// Use a static local for these singletons so they are only registered if an -// object of this instance is ever constructed. Otherwise we will create and -// register ELF allocators for COFF and the reverse. +// The address of this static member is only used as a key in +// CommonLinkerContext::instances. Its value does not matter. +template <class T> int SpecificAlloc<T>::tag = 0; + +// Creates the arena on-demand on the first call; or returns it, if it was +// already created. template <typename T> inline llvm::SpecificBumpPtrAllocator<T> &getSpecificAllocSingleton() { - static SpecificAlloc<T> instance; - return instance.alloc; + SpecificAllocBase *instance = SpecificAllocBase::getOrCreate( + &SpecificAlloc<T>::tag, sizeof(SpecificAlloc<T>), + alignof(SpecificAlloc<T>), SpecificAlloc<T>::create); + return ((SpecificAlloc<T> *)instance)->alloc; } -// Use this arena if your object has a destructor. -// Your destructor will be invoked from freeArena(). +// Creates new instances of T off a (almost) contiguous arena/object pool. The +// instances are destroyed whenever lldMain() goes out of scope. template <typename T, typename... U> T *make(U &&... args) { return new (getSpecificAllocSingleton<T>().Allocate()) T(std::forward<U>(args)...); diff --git a/lld/include/lld/Common/Strings.h b/lld/include/lld/Common/Strings.h index 71126f615017..ece801892767 100644 --- a/lld/include/lld/Common/Strings.h +++ b/lld/include/lld/Common/Strings.h @@ -12,14 +12,19 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Demangle/Demangle.h" #include "llvm/Support/GlobPattern.h" #include <string> #include <vector> namespace lld { -// Returns a demangled C++ symbol name. If Name is not a mangled -// name, it returns name. -std::string demangleItanium(llvm::StringRef name); +// Returns a demangled symbol name. If Name is not a mangled name, it returns +// name. +inline std::string demangle(llvm::StringRef symName, bool shouldDemangle) { + if (shouldDemangle) + return llvm::demangle(symName.str().c_str()); + return std::string(symName); +} std::vector<uint8_t> parseHex(llvm::StringRef s); bool isValidCIdentifier(llvm::StringRef s); diff --git a/lld/include/lld/Core/LinkingContext.h b/lld/include/lld/Core/LinkingContext.h index e090ff990231..091369e14319 100644 --- a/lld/include/lld/Core/LinkingContext.h +++ b/lld/include/lld/Core/LinkingContext.h @@ -9,6 +9,7 @@ #ifndef LLD_CORE_LINKING_CONTEXT_H #define LLD_CORE_LINKING_CONTEXT_H +#include "lld/Common/CommonLinkerContext.h" #include "lld/Core/Node.h" #include "lld/Core/Reader.h" #include "llvm/ADT/ArrayRef.h" @@ -34,7 +35,7 @@ class SharedLibraryFile; /// The base class LinkingContext contains the options needed by core linking. /// Subclasses of LinkingContext have additional options needed by specific /// Writers. -class LinkingContext { +class LinkingContext : public CommonLinkerContext { public: virtual ~LinkingContext(); diff --git a/lld/tools/lld/lld.cpp b/lld/tools/lld/lld.cpp index cad97f2153c2..0a6439fff2a2 100644 --- a/lld/tools/lld/lld.cpp +++ b/lld/tools/lld/lld.cpp @@ -87,6 +87,8 @@ static bool isPETarget(std::vector<const char *> &v) { // Expand response files (arguments in the form of @<filename>) // to allow detecting the -m argument from arguments in them. SmallVector<const char *, 256> expandedArgs(v.data(), v.data() + v.size()); + BumpPtrAllocator a; + StringSaver saver(a); cl::ExpandResponseFiles(saver, getDefaultQuotingStyle(), expandedArgs); for (auto it = expandedArgs.begin(); it + 1 != expandedArgs.end(); ++it) { if (StringRef(*it) != "-m") @@ -134,27 +136,42 @@ static Flavor parseFlavor(std::vector<const char *> &v) { return parseProgname(arg0); } +bool inTestOutputDisabled = false; + /// Universal linker main(). This linker emulates the gnu, darwin, or /// windows linker based on the argv[0] or -flavor option. static int lldMain(int argc, const char **argv, llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS, bool exitEarly = true) { std::vector<const char *> args(argv, argv + argc); - switch (parseFlavor(args)) { - case Gnu: - if (isPETarget(args)) - return !mingw::link(args, exitEarly, stdoutOS, stderrOS); - return !elf::link(args, exitEarly, stdoutOS, stderrOS); - case WinLink: - return !coff::link(args, exitEarly, stdoutOS, stderrOS); - case Darwin: - return !macho::link(args, exitEarly, stdoutOS, stderrOS); - case Wasm: - return !lld::wasm::link(args, exitEarly, stdoutOS, stderrOS); - default: - die("lld is a generic driver.\n" - "Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld" - " (WebAssembly) instead"); - } + auto link = [&args]() { + Flavor f = parseFlavor(args); + if (f == Gnu && isPETarget(args)) + return mingw::link; + else if (f == Gnu) + return elf::link; + else if (f == WinLink) + return coff::link; + else if (f == Darwin) + return macho::link; + else if (f == Wasm) + return lld::wasm::link; + else + die("lld is a generic driver.\n" + "Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld" + " (WebAssembly) instead"); + }; + // Run the driver. If an error occurs, false will be returned. + bool r = link()(args, stdoutOS, stderrOS, exitEarly, inTestOutputDisabled); + + // Call exit() if we can to avoid calling destructors. + if (exitEarly) + exitLld(!r ? 1 : 0); + + // Delete the global context and clear the global context pointer, so that it + // cannot be accessed anymore. + CommonLinkerContext::destroy(); + + return !r ? 1 : 0; } // Similar to lldMain except that exceptions are caught. @@ -176,7 +193,7 @@ SafeReturn lld::safeLldMain(int argc, const char **argv, // Cleanup memory and reset everything back in pristine condition. This path // is only taken when LLD is in test, or when it is used as a library. llvm::CrashRecoveryContext crc; - if (!crc.RunSafely([&]() { errorHandler().reset(); })) { + if (!crc.RunSafely([&]() { CommonLinkerContext::destroy(); })) { // The memory is corrupted beyond any possible recovery. return {r, /*canRunAgain=*/false}; } @@ -207,8 +224,7 @@ int main(int argc, const char **argv) { for (unsigned i = inTestVerbosity(); i > 0; --i) { // Disable stdout/stderr for all iterations but the last one. - if (i != 1) - errorHandler().disableOutput = true; + inTestOutputDisabled = (i != 1); // Execute one iteration. auto r = safeLldMain(argc, argv, llvm::outs(), llvm::errs()); |