diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/Tooling/DependencyScanning')
4 files changed, 158 insertions, 90 deletions
diff --git a/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index b4d5a29ca695..b1b87e7fa573 100644 --- a/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -106,7 +106,8 @@ DependencyScanningFilesystemSharedCache:: // sharding gives a performance edge by reducing the lock contention. // FIXME: A better heuristic might also consider the OS to account for // the different cost of lock contention on different OSes. - NumShards = std::max(2u, llvm::hardware_concurrency() / 4); + NumShards = + std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4); CacheShards = std::make_unique<CacheShard[]>(NumShards); } diff --git a/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index f643c538f8f9..16040c2f4626 100644 --- a/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -8,24 +8,25 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Frontend/Utils.h" -#include "llvm/Support/JSON.h" - -static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { - std::vector<llvm::StringRef> Strings; - for (auto &&I : Set) - Strings.push_back(I.getKey()); - std::sort(Strings.begin(), Strings.end()); - return llvm::json::Array(Strings); -} namespace clang{ namespace tooling{ namespace dependencies{ +std::vector<std::string> FullDependencies::getAdditionalCommandLine( + std::function<StringRef(ClangModuleDep)> LookupPCMPath, + std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const { + std::vector<std::string> Ret = AdditionalNonPathCommandLine; + + dependencies::detail::appendCommonModuleArguments( + ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret); + + return Ret; +} + DependencyScanningTool::DependencyScanningTool( DependencyScanningService &Service) - : Format(Service.getFormat()), Worker(Service) { -} + : Worker(Service) {} llvm::Expected<std::string> DependencyScanningTool::getDependencyFile( const tooling::CompilationDatabase &Compilations, StringRef CWD) { @@ -36,7 +37,7 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile( StringRef File) override { if (!this->Opts) this->Opts = std::make_unique<DependencyOutputOptions>(Opts); - Dependencies.push_back(File); + Dependencies.push_back(std::string(File)); } void handleModuleDependency(ModuleDeps MD) override { @@ -75,11 +76,36 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile( std::vector<std::string> Dependencies; }; + // We expect a single command here because if a source file occurs multiple + // times in the original CDB, then `computeDependencies` would run the + // `DependencyScanningAction` once for every time the input occured in the + // CDB. Instead we split up the CDB into single command chunks to avoid this + // behavior. + assert(Compilations.getAllCompileCommands().size() == 1 && + "Expected a compilation database with a single command!"); + std::string Input = Compilations.getAllCompileCommands().front().Filename; + + MakeDependencyPrinterConsumer Consumer; + auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + std::string Output; + Consumer.printDependencies(Output); + return Output; +} + +llvm::Expected<FullDependenciesResult> +DependencyScanningTool::getFullDependencies( + const tooling::CompilationDatabase &Compilations, StringRef CWD, + const llvm::StringSet<> &AlreadySeen) { class FullDependencyPrinterConsumer : public DependencyConsumer { public: + FullDependencyPrinterConsumer(const llvm::StringSet<> &AlreadySeen) + : AlreadySeen(AlreadySeen) {} + void handleFileDependency(const DependencyOutputOptions &Opts, StringRef File) override { - Dependencies.push_back(File); + Dependencies.push_back(std::string(File)); } void handleModuleDependency(ModuleDeps MD) override { @@ -90,55 +116,41 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile( ContextHash = std::move(Hash); } - void printDependencies(std::string &S, StringRef MainFile) { - // Sort the modules by name to get a deterministic order. - std::vector<StringRef> Modules; - for (auto &&Dep : ClangModuleDeps) - Modules.push_back(Dep.first); - std::sort(Modules.begin(), Modules.end()); + FullDependenciesResult getFullDependencies() const { + FullDependencies FD; - llvm::raw_string_ostream OS(S); + FD.ContextHash = std::move(ContextHash); - using namespace llvm::json; + FD.FileDeps.assign(Dependencies.begin(), Dependencies.end()); - Array Imports; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; + for (auto &&M : ClangModuleDeps) { + auto &MD = M.second; if (MD.ImportedByMainFile) - Imports.push_back(MD.ModuleName); + FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash}); } - Array Mods; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; - Object Mod{ - {"name", MD.ModuleName}, - {"file-deps", toJSONSorted(MD.FileDeps)}, - {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, - {"clang-modulemap-file", MD.ClangModuleMapFile}, - }; - Mods.push_back(std::move(Mod)); - } + FullDependenciesResult FDR; - Object O{ - {"input-file", MainFile}, - {"clang-context-hash", ContextHash}, - {"file-deps", Dependencies}, - {"clang-module-deps", std::move(Imports)}, - {"clang-modules", std::move(Mods)}, - }; + for (auto &&M : ClangModuleDeps) { + // TODO: Avoid handleModuleDependency even being called for modules + // we've already seen. + if (AlreadySeen.count(M.first)) + continue; + FDR.DiscoveredModules.push_back(std::move(M.second)); + } - S = llvm::formatv("{0:2},\n", Value(std::move(O))).str(); - return; + FDR.FullDeps = std::move(FD); + return FDR; } private: std::vector<std::string> Dependencies; std::unordered_map<std::string, ModuleDeps> ClangModuleDeps; std::string ContextHash; + std::vector<std::string> OutputPaths; + const llvm::StringSet<> &AlreadySeen; }; - // We expect a single command here because if a source file occurs multiple // times in the original CDB, then `computeDependencies` would run the // `DependencyScanningAction` once for every time the input occured in the @@ -147,26 +159,13 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile( assert(Compilations.getAllCompileCommands().size() == 1 && "Expected a compilation database with a single command!"); std::string Input = Compilations.getAllCompileCommands().front().Filename; - - if (Format == ScanningOutputFormat::Make) { - MakeDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output); - return Output; - } else { - FullDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output, Input); - return Output; - } + + FullDependencyPrinterConsumer Consumer(AlreadySeen); + llvm::Error Result = + Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + return Consumer.getFullDependencies(); } } // end namespace dependencies diff --git a/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index edf2cf8bd70f..32bbc578d2db 100644 --- a/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -118,7 +118,7 @@ public: .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings; } - FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory; + FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory); Compiler.setFileManager(FileMgr); Compiler.createSourceManager(*FileMgr); @@ -142,12 +142,17 @@ public: Consumer)); break; case ScanningOutputFormat::Full: - Compiler.addDependencyCollector( - std::make_shared<ModuleDepCollector>(Compiler, Consumer)); + Compiler.addDependencyCollector(std::make_shared<ModuleDepCollector>( + std::move(Opts), Compiler, Consumer)); break; } - Consumer.handleContextHash(Compiler.getInvocation().getModuleHash()); + // Consider different header search and diagnostic options to create + // different modules. This avoids the unsound aliasing of module PCMs. + // + // TODO: Implement diagnostic bucketing and header search pruning to reduce + // the impact of strict context hashing. + Compiler.getHeaderSearchOpts().ModulesStrictContextHash = true; auto Action = std::make_unique<PreprocessOnlyAction>(); const bool Result = Compiler.ExecuteAction(*Action); diff --git a/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 422940047f2d..4f6eff799f22 100644 --- a/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/contrib/llvm-project/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -17,12 +17,60 @@ using namespace clang; using namespace tooling; using namespace dependencies; +std::vector<std::string> ModuleDeps::getFullCommandLine( + std::function<StringRef(ClangModuleDep)> LookupPCMPath, + std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const { + std::vector<std::string> Ret = NonPathCommandLine; + + // TODO: Build full command line. That also means capturing the original + // command line into NonPathCommandLine. + + dependencies::detail::appendCommonModuleArguments( + ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret); + + return Ret; +} + +void dependencies::detail::appendCommonModuleArguments( + llvm::ArrayRef<ClangModuleDep> Modules, + std::function<StringRef(ClangModuleDep)> LookupPCMPath, + std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps, + std::vector<std::string> &Result) { + llvm::StringSet<> AlreadyAdded; + + std::function<void(llvm::ArrayRef<ClangModuleDep>)> AddArgs = + [&](llvm::ArrayRef<ClangModuleDep> Modules) { + for (const ClangModuleDep &CMD : Modules) { + if (!AlreadyAdded.insert(CMD.ModuleName + CMD.ContextHash).second) + continue; + const ModuleDeps &M = LookupModuleDeps(CMD); + // Depth first traversal. + AddArgs(M.ClangModuleDeps); + Result.push_back(("-fmodule-file=" + LookupPCMPath(CMD)).str()); + if (!M.ClangModuleMapFile.empty()) { + Result.push_back("-fmodule-map-file=" + M.ClangModuleMapFile); + } + } + }; + + Result.push_back("-fno-implicit-modules"); + Result.push_back("-fno-implicit-module-maps"); + AddArgs(Modules); +} + void ModuleDepCollectorPP::FileChanged(SourceLocation Loc, FileChangeReason Reason, SrcMgr::CharacteristicKind FileType, FileID PrevFID) { if (Reason != PPCallbacks::EnterFile) return; + + // This has to be delayed as the context hash can change at the start of + // `CompilerInstance::ExecuteAction`. + if (MDC.ContextHash.empty()) { + MDC.ContextHash = Instance.getInvocation().getModuleHash(); + MDC.Consumer.handleContextHash(MDC.ContextHash); + } SourceManager &SM = Instance.getSourceManager(); @@ -37,7 +85,7 @@ void ModuleDepCollectorPP::FileChanged(SourceLocation Loc, StringRef FileName = llvm::sys::path::remove_leading_dotslash(File->getName()); - MDC.MainDeps.push_back(FileName); + MDC.MainDeps.push_back(std::string(FileName)); } void ModuleDepCollectorPP::InclusionDirective( @@ -48,9 +96,18 @@ void ModuleDepCollectorPP::InclusionDirective( if (!File && !Imported) { // This is a non-modular include that HeaderSearch failed to find. Add it // here as `FileChanged` will never see it. - MDC.MainDeps.push_back(FileName); + MDC.MainDeps.push_back(std::string(FileName)); } + handleImport(Imported); +} +void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc, + ModuleIdPath Path, + const Module *Imported) { + handleImport(Imported); +} + +void ModuleDepCollectorPP::handleImport(const Module *Imported) { if (!Imported) return; @@ -61,8 +118,8 @@ void ModuleDepCollectorPP::InclusionDirective( void ModuleDepCollectorPP::EndOfMainFile() { FileID MainFileID = Instance.getSourceManager().getMainFileID(); - MDC.MainFile = - Instance.getSourceManager().getFileEntryForID(MainFileID)->getName(); + MDC.MainFile = std::string( + Instance.getSourceManager().getFileEntryForID(MainFileID)->getName()); for (const Module *M : DirectDeps) { handleTopLevelModule(M); @@ -71,9 +128,8 @@ void ModuleDepCollectorPP::EndOfMainFile() { for (auto &&I : MDC.Deps) MDC.Consumer.handleModuleDependency(I.second); - DependencyOutputOptions Opts; for (auto &&I : MDC.MainDeps) - MDC.Consumer.handleFileDependency(Opts, I); + MDC.Consumer.handleFileDependency(*MDC.Opts, I); } void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { @@ -92,9 +148,9 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { .getModuleMap() .getContainingModuleMapFile(M); - MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : ""; + MD.ClangModuleMapFile = std::string(ModuleMap ? ModuleMap->getName() : ""); MD.ModuleName = M->getFullModuleName(); - MD.ModulePCMPath = M->getASTFile()->getName(); + MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName()); MD.ContextHash = MDC.ContextHash; serialization::ModuleFile *MF = MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile()); @@ -103,30 +159,37 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.FileDeps.insert(IF.getFile()->getName()); }); - addAllSubmoduleDeps(M, MD); + llvm::DenseSet<const Module *> AddedModules; + addAllSubmoduleDeps(M, MD, AddedModules); } -void ModuleDepCollectorPP::addAllSubmoduleDeps(const Module *M, - ModuleDeps &MD) { - addModuleDep(M, MD); +void ModuleDepCollectorPP::addAllSubmoduleDeps( + const Module *M, ModuleDeps &MD, + llvm::DenseSet<const Module *> &AddedModules) { + addModuleDep(M, MD, AddedModules); for (const Module *SubM : M->submodules()) - addAllSubmoduleDeps(SubM, MD); + addAllSubmoduleDeps(SubM, MD, AddedModules); } -void ModuleDepCollectorPP::addModuleDep(const Module *M, ModuleDeps &MD) { +void ModuleDepCollectorPP::addModuleDep( + const Module *M, ModuleDeps &MD, + llvm::DenseSet<const Module *> &AddedModules) { for (const Module *Import : M->Imports) { if (Import->getTopLevelModule() != M->getTopLevelModule()) { - MD.ClangModuleDeps.insert(Import->getTopLevelModuleName()); + if (AddedModules.insert(Import->getTopLevelModule()).second) + MD.ClangModuleDeps.push_back( + {std::string(Import->getTopLevelModuleName()), + Instance.getInvocation().getModuleHash()}); handleTopLevelModule(Import->getTopLevelModule()); } } } -ModuleDepCollector::ModuleDepCollector(CompilerInstance &I, - DependencyConsumer &C) - : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()) { -} +ModuleDepCollector::ModuleDepCollector( + std::unique_ptr<DependencyOutputOptions> Opts, CompilerInstance &I, + DependencyConsumer &C) + : Instance(I), Consumer(C), Opts(std::move(Opts)) {} void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) { PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(Instance, *this)); |