diff options
Diffstat (limited to 'lib/Tooling/DependencyScanning')
4 files changed, 420 insertions, 44 deletions
diff --git a/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp new file mode 100644 index 0000000000000..7436c72563277 --- /dev/null +++ b/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -0,0 +1,234 @@ +//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/Lex/DependencyDirectivesSourceMinimizer.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Threading.h" + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( + StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) { + // Load the file and its content from the file system. + llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile = + FS.openFileForRead(Filename); + if (!MaybeFile) + return MaybeFile.getError(); + llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status(); + if (!Stat) + return Stat.getError(); + + llvm::vfs::File &F = **MaybeFile; + llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer = + F.getBuffer(Stat->getName()); + if (!MaybeBuffer) + return MaybeBuffer.getError(); + + llvm::SmallString<1024> MinimizedFileContents; + // Minimize the file down to directives that might affect the dependencies. + const auto &Buffer = *MaybeBuffer; + SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens; + if (!Minimize || minimizeSourceToDependencyDirectives( + Buffer->getBuffer(), MinimizedFileContents, Tokens)) { + // Use the original file unless requested otherwise, or + // if the minimization failed. + // FIXME: Propage the diagnostic if desired by the client. + CachedFileSystemEntry Result; + Result.MaybeStat = std::move(*Stat); + Result.Contents.reserve(Buffer->getBufferSize() + 1); + Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd()); + // Implicitly null terminate the contents for Clang's lexer. + Result.Contents.push_back('\0'); + Result.Contents.pop_back(); + return Result; + } + + CachedFileSystemEntry Result; + size_t Size = MinimizedFileContents.size(); + Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(), + Stat->getLastModificationTime(), + Stat->getUser(), Stat->getGroup(), Size, + Stat->getType(), Stat->getPermissions()); + // The contents produced by the minimizer must be null terminated. + assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' && + "not null terminated contents"); + // Even though there's an implicit null terminator in the minimized contents, + // we want to temporarily make it explicit. This will ensure that the + // std::move will preserve it even if it needs to do a copy if the + // SmallString still has the small capacity. + MinimizedFileContents.push_back('\0'); + Result.Contents = std::move(MinimizedFileContents); + // Now make the null terminator implicit again, so that Clang's lexer can find + // it right where the buffer ends. + Result.Contents.pop_back(); + + // Compute the skipped PP ranges that speedup skipping over inactive + // preprocessor blocks. + llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32> + SkippedRanges; + minimize_source_to_dependency_directives::computeSkippedRanges(Tokens, + SkippedRanges); + PreprocessorSkippedRangeMapping Mapping; + for (const auto &Range : SkippedRanges) { + if (Range.Length < 16) { + // Ignore small ranges as non-profitable. + // FIXME: This is a heuristic, its worth investigating the tradeoffs + // when it should be applied. + continue; + } + Mapping[Range.Offset] = Range.Length; + } + Result.PPSkippedRangeMapping = std::move(Mapping); + + return Result; +} + +CachedFileSystemEntry +CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) { + assert(Stat.isDirectory() && "not a directory!"); + auto Result = CachedFileSystemEntry(); + Result.MaybeStat = std::move(Stat); + return Result; +} + +DependencyScanningFilesystemSharedCache:: + DependencyScanningFilesystemSharedCache() { + // This heuristic was chosen using a empirical testing on a + // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache + // sharding gives a performance edge by reducing the lock contention. + // FIXME: A better heuristic might also consider the OS to account for + // the different cost of lock contention on different OSes. + NumShards = std::max(2u, llvm::hardware_concurrency() / 4); + CacheShards = std::make_unique<CacheShard[]>(NumShards); +} + +/// Returns a cache entry for the corresponding key. +/// +/// A new cache entry is created if the key is not in the cache. This is a +/// thread safe call. +DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & +DependencyScanningFilesystemSharedCache::get(StringRef Key) { + CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards]; + std::unique_lock<std::mutex> LockGuard(Shard.CacheLock); + auto It = Shard.Cache.try_emplace(Key); + return It.first->getValue(); +} + +llvm::ErrorOr<const CachedFileSystemEntry *> +DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( + const StringRef Filename) { + if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) { + return Entry; + } + + // FIXME: Handle PCM/PCH files. + // FIXME: Handle module map files. + + bool KeepOriginalSource = IgnoredFiles.count(Filename); + DependencyScanningFilesystemSharedCache::SharedFileSystemEntry + &SharedCacheEntry = SharedCache.get(Filename); + const CachedFileSystemEntry *Result; + { + std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock); + CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; + + if (!CacheEntry.isValid()) { + llvm::vfs::FileSystem &FS = getUnderlyingFS(); + auto MaybeStatus = FS.status(Filename); + if (!MaybeStatus) + CacheEntry = CachedFileSystemEntry(MaybeStatus.getError()); + else if (MaybeStatus->isDirectory()) + CacheEntry = CachedFileSystemEntry::createDirectoryEntry( + std::move(*MaybeStatus)); + else + CacheEntry = CachedFileSystemEntry::createFileEntry( + Filename, FS, !KeepOriginalSource); + } + + Result = &CacheEntry; + } + + // Store the result in the local cache. + setCachedEntry(Filename, Result); + return Result; +} + +llvm::ErrorOr<llvm::vfs::Status> +DependencyScanningWorkerFilesystem::status(const Twine &Path) { + SmallString<256> OwnedFilename; + StringRef Filename = Path.toStringRef(OwnedFilename); + const llvm::ErrorOr<const CachedFileSystemEntry *> Result = + getOrCreateFileSystemEntry(Filename); + if (!Result) + return Result.getError(); + return (*Result)->getStatus(); +} + +namespace { + +/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using +/// this subclass. +class MinimizedVFSFile final : public llvm::vfs::File { +public: + MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer, + llvm::vfs::Status Stat) + : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} + + llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; } + + const llvm::MemoryBuffer *getBufferPtr() const { return Buffer.get(); } + + llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> + getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, + bool IsVolatile) override { + return std::move(Buffer); + } + + std::error_code close() override { return {}; } + +private: + std::unique_ptr<llvm::MemoryBuffer> Buffer; + llvm::vfs::Status Stat; +}; + +llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> +createFile(const CachedFileSystemEntry *Entry, + ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) { + if (Entry->isDirectory()) + return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( + std::make_error_code(std::errc::is_a_directory)); + llvm::ErrorOr<StringRef> Contents = Entry->getContents(); + if (!Contents) + return Contents.getError(); + auto Result = std::make_unique<MinimizedVFSFile>( + llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), + /*RequiresNullTerminator=*/false), + *Entry->getStatus()); + if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings) + (*PPSkipMappings)[Result->getBufferPtr()] = + &Entry->getPPSkippedRangeMapping(); + return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( + std::unique_ptr<llvm::vfs::File>(std::move(Result))); +} + +} // end anonymous namespace + +llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> +DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) { + SmallString<256> OwnedFilename; + StringRef Filename = Path.toStringRef(OwnedFilename); + + const llvm::ErrorOr<const CachedFileSystemEntry *> Result = + getOrCreateFileSystemEntry(Filename); + if (!Result) + return Result.getError(); + return createFile(Result.get(), PPSkipMappings); +} diff --git a/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/lib/Tooling/DependencyScanning/DependencyScanningService.cpp new file mode 100644 index 0000000000000..e5cebe381000a --- /dev/null +++ b/lib/Tooling/DependencyScanning/DependencyScanningService.cpp @@ -0,0 +1,19 @@ +//===- DependencyScanningService.cpp - clang-scan-deps service ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +DependencyScanningService::DependencyScanningService(ScanningMode Mode, + bool ReuseFileManager, + bool SkipExcludedPPRanges) + : Mode(Mode), ReuseFileManager(ReuseFileManager), + SkipExcludedPPRanges(SkipExcludedPPRanges) {} diff --git a/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp new file mode 100644 index 0000000000000..82b3ae806c65d --- /dev/null +++ b/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -0,0 +1,71 @@ +//===- DependencyScanningTool.cpp - clang-scan-deps service ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "clang/Frontend/Utils.h" + +namespace clang{ +namespace tooling{ +namespace dependencies{ + +DependencyScanningTool::DependencyScanningTool(DependencyScanningService &Service, +const tooling::CompilationDatabase &Compilations) : Worker(Service), Compilations(Compilations) {} + +llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(const std::string &Input, + StringRef CWD) { + /// Prints out all of the gathered dependencies into a string. + class DependencyPrinterConsumer : public DependencyConsumer { + public: + void handleFileDependency(const DependencyOutputOptions &Opts, + StringRef File) override { + if (!this->Opts) + this->Opts = std::make_unique<DependencyOutputOptions>(Opts); + Dependencies.push_back(File); + } + + void printDependencies(std::string &S) { + if (!Opts) + return; + + class DependencyPrinter : public DependencyFileGenerator { + public: + DependencyPrinter(DependencyOutputOptions &Opts, + ArrayRef<std::string> Dependencies) + : DependencyFileGenerator(Opts) { + for (const auto &Dep : Dependencies) + addDependency(Dep); + } + + void printDependencies(std::string &S) { + llvm::raw_string_ostream OS(S); + outputDependencyFile(OS); + } + }; + + DependencyPrinter Generator(*Opts, Dependencies); + Generator.printDependencies(S); + } + + private: + std::unique_ptr<DependencyOutputOptions> Opts; + std::vector<std::string> Dependencies; + }; + + DependencyPrinterConsumer Consumer; + auto Result = + Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + std::string Output; + Consumer.printDependencies(Output); + return Output; +} + +} // end namespace dependencies +} // end namespace tooling +} // end namespace clang diff --git a/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 4868f26637960..f382c202f8c2a 100644 --- a/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -8,9 +8,12 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Frontend/Utils.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/Tooling.h" using namespace clang; @@ -19,21 +22,25 @@ using namespace dependencies; namespace { -/// Prints out all of the gathered dependencies into a string. -class DependencyPrinter : public DependencyFileGenerator { +/// Forwards the gatherered dependencies to the consumer. +class DependencyConsumerForwarder : public DependencyFileGenerator { public: - DependencyPrinter(std::unique_ptr<DependencyOutputOptions> Opts, - std::string &S) - : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), S(S) {} + DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts, + DependencyConsumer &C) + : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {} void finishedMainFile(DiagnosticsEngine &Diags) override { - llvm::raw_string_ostream OS(S); - outputDependencyFile(OS); + llvm::SmallString<256> CanonPath; + for (const auto &File : getDependencies()) { + CanonPath = File; + llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true); + C.handleFileDependency(*Opts, CanonPath); + } } private: std::unique_ptr<DependencyOutputOptions> Opts; - std::string &S; + DependencyConsumer &C; }; /// A proxy file system that doesn't call `chdir` when changing the working @@ -62,10 +69,12 @@ private: /// dependency scanning for the given compiler invocation. class DependencyScanningAction : public tooling::ToolAction { public: - DependencyScanningAction(StringRef WorkingDirectory, - std::string &DependencyFileContents) - : WorkingDirectory(WorkingDirectory), - DependencyFileContents(DependencyFileContents) {} + DependencyScanningAction( + StringRef WorkingDirectory, DependencyConsumer &Consumer, + llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS, + ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) + : WorkingDirectory(WorkingDirectory), Consumer(Consumer), + DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings) {} bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation, FileManager *FileMgr, @@ -74,8 +83,6 @@ public: // Create a compiler instance to handle the actual work. CompilerInstance Compiler(std::move(PCHContainerOps)); Compiler.setInvocation(std::move(Invocation)); - FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory; - Compiler.setFileManager(FileMgr); // Don't print 'X warnings and Y errors generated'. Compiler.getDiagnosticOpts().ShowCarets = false; @@ -84,6 +91,32 @@ public: if (!Compiler.hasDiagnostics()) return false; + // Use the dependency scanning optimized file system if we can. + if (DepFS) { + const CompilerInvocation &CI = Compiler.getInvocation(); + // Add any filenames that were explicity passed in the build settings and + // that might be opened, as we want to ensure we don't run source + // minimization on them. + DepFS->IgnoredFiles.clear(); + for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) + DepFS->IgnoredFiles.insert(Entry.Path); + for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles) + DepFS->IgnoredFiles.insert(Entry); + + // Support for virtual file system overlays on top of the caching + // filesystem. + FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation( + CI, Compiler.getDiagnostics(), DepFS)); + + // Pass the skip mappings which should speed up excluded conditional block + // skipping in the preprocessor. + if (PPSkipMappings) + Compiler.getPreprocessorOpts() + .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings; + } + + FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory; + Compiler.setFileManager(FileMgr); Compiler.createSourceManager(*FileMgr); // Create the dependency collector that will collect the produced @@ -93,57 +126,76 @@ public: // invocation to the collector. The options in the invocation are reset, // which ensures that the compiler won't create new dependency collectors, // and thus won't write out the extra '.d' files to disk. - auto Opts = llvm::make_unique<DependencyOutputOptions>( + auto Opts = std::make_unique<DependencyOutputOptions>( std::move(Compiler.getInvocation().getDependencyOutputOpts())); // We need at least one -MT equivalent for the generator to work. if (Opts->Targets.empty()) Opts->Targets = {"clang-scan-deps dependency"}; - Compiler.addDependencyCollector(std::make_shared<DependencyPrinter>( - std::move(Opts), DependencyFileContents)); + Compiler.addDependencyCollector( + std::make_shared<DependencyConsumerForwarder>(std::move(Opts), + Consumer)); - auto Action = llvm::make_unique<PreprocessOnlyAction>(); + auto Action = std::make_unique<PreprocessOnlyAction>(); const bool Result = Compiler.ExecuteAction(*Action); - FileMgr->clearStatCache(); + if (!DepFS) + FileMgr->clearStatCache(); return Result; } private: StringRef WorkingDirectory; - /// The dependency file will be written to this string. - std::string &DependencyFileContents; + DependencyConsumer &Consumer; + llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS; + ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings; }; } // end anonymous namespace -DependencyScanningWorker::DependencyScanningWorker() { +DependencyScanningWorker::DependencyScanningWorker( + DependencyScanningService &Service) { DiagOpts = new DiagnosticOptions(); PCHContainerOps = std::make_shared<PCHContainerOperations>(); - /// FIXME: Use the shared file system from the service for fast scanning - /// mode. - WorkerFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem()); + RealFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem()); + if (Service.canSkipExcludedPPRanges()) + PPSkipMappings = + std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>(); + if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing) + DepFS = new DependencyScanningWorkerFilesystem( + Service.getSharedCache(), RealFS, PPSkipMappings.get()); + if (Service.canReuseFileManager()) + Files = new FileManager(FileSystemOptions(), RealFS); } -llvm::Expected<std::string> -DependencyScanningWorker::getDependencyFile(const std::string &Input, - StringRef WorkingDirectory, - const CompilationDatabase &CDB) { +static llvm::Error runWithDiags( + DiagnosticOptions *DiagOpts, + llvm::function_ref<bool(DiagnosticConsumer &DC)> BodyShouldSucceed) { // Capture the emitted diagnostics and report them to the client // in the case of a failure. std::string DiagnosticOutput; llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); - TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.get()); - - WorkerFS->setCurrentWorkingDirectory(WorkingDirectory); - tooling::ClangTool Tool(CDB, Input, PCHContainerOps, WorkerFS); - Tool.clearArgumentsAdjusters(); - Tool.setRestoreWorkingDir(false); - Tool.setPrintErrorMessage(false); - Tool.setDiagnosticConsumer(&DiagPrinter); - std::string Output; - DependencyScanningAction Action(WorkingDirectory, Output); - if (Tool.run(&Action)) { - return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(), - llvm::inconvertibleErrorCode()); - } - return Output; + TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts); + + if (BodyShouldSucceed(DiagPrinter)) + return llvm::Error::success(); + return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(), + llvm::inconvertibleErrorCode()); +} + +llvm::Error DependencyScanningWorker::computeDependencies( + const std::string &Input, StringRef WorkingDirectory, + const CompilationDatabase &CDB, DependencyConsumer &Consumer) { + RealFS->setCurrentWorkingDirectory(WorkingDirectory); + return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) { + /// Create the tool that uses the underlying file system to ensure that any + /// file system requests that are made by the driver do not go through the + /// dependency scanning filesystem. + tooling::ClangTool Tool(CDB, Input, PCHContainerOps, RealFS, Files); + Tool.clearArgumentsAdjusters(); + Tool.setRestoreWorkingDir(false); + Tool.setPrintErrorMessage(false); + Tool.setDiagnosticConsumer(&DC); + DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS, + PPSkipMappings.get()); + return !Tool.run(&Action); + }); } |