diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2020-07-26 19:36:28 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2020-07-26 19:36:28 +0000 |
commit | cfca06d7963fa0909f90483b42a6d7d194d01e08 (patch) | |
tree | 209fb2a2d68f8f277793fc8df46c753d31bc853b /compiler-rt/lib/fuzzer | |
parent | 706b4fc47bbc608932d3b491ae19a3b9cde9497b (diff) |
Notes
Diffstat (limited to 'compiler-rt/lib/fuzzer')
31 files changed, 625 insertions, 274 deletions
diff --git a/compiler-rt/lib/fuzzer/FuzzerBuiltins.h b/compiler-rt/lib/fuzzer/FuzzerBuiltins.h index 5f1ccef8a9ca..4c0ada82662d 100644 --- a/compiler-rt/lib/fuzzer/FuzzerBuiltins.h +++ b/compiler-rt/lib/fuzzer/FuzzerBuiltins.h @@ -11,7 +11,7 @@ #ifndef LLVM_FUZZER_BUILTINS_H #define LLVM_FUZZER_BUILTINS_H -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if !LIBFUZZER_MSVC #include <cstdint> diff --git a/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h b/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h index bc65c60098be..c5bec9787d8e 100644 --- a/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h +++ b/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h @@ -12,7 +12,7 @@ #ifndef LLVM_FUZZER_BUILTINS_MSVC_H #define LLVM_FUZZER_BUILTINS_MSVC_H -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_MSVC #include <intrin.h> diff --git a/compiler-rt/lib/fuzzer/FuzzerCorpus.h b/compiler-rt/lib/fuzzer/FuzzerCorpus.h index 6a95ef3a8e64..54d1e09ec6df 100644 --- a/compiler-rt/lib/fuzzer/FuzzerCorpus.h +++ b/compiler-rt/lib/fuzzer/FuzzerCorpus.h @@ -38,12 +38,102 @@ struct InputInfo { bool HasFocusFunction = false; Vector<uint32_t> UniqFeatureSet; Vector<uint8_t> DataFlowTraceForFocusFunction; + // Power schedule. + bool NeedsEnergyUpdate = false; + double Energy = 0.0; + size_t SumIncidence = 0; + Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs; + + // Delete feature Idx and its frequency from FeatureFreqs. + bool DeleteFeatureFreq(uint32_t Idx) { + if (FeatureFreqs.empty()) + return false; + + // Binary search over local feature frequencies sorted by index. + auto Lower = std::lower_bound(FeatureFreqs.begin(), FeatureFreqs.end(), + std::pair<uint32_t, uint16_t>(Idx, 0)); + + if (Lower != FeatureFreqs.end() && Lower->first == Idx) { + FeatureFreqs.erase(Lower); + return true; + } + return false; + } + + // Assign more energy to a high-entropy seed, i.e., that reveals more + // information about the globally rare features in the neighborhood + // of the seed. Since we do not know the entropy of a seed that has + // never been executed we assign fresh seeds maximum entropy and + // let II->Energy approach the true entropy from above. + void UpdateEnergy(size_t GlobalNumberOfFeatures) { + Energy = 0.0; + SumIncidence = 0; + + // Apply add-one smoothing to locally discovered features. + for (auto F : FeatureFreqs) { + size_t LocalIncidence = F.second + 1; + Energy -= LocalIncidence * logl(LocalIncidence); + SumIncidence += LocalIncidence; + } + + // Apply add-one smoothing to locally undiscovered features. + // PreciseEnergy -= 0; // since logl(1.0) == 0) + SumIncidence += (GlobalNumberOfFeatures - FeatureFreqs.size()); + + // Add a single locally abundant feature apply add-one smoothing. + size_t AbdIncidence = NumExecutedMutations + 1; + Energy -= AbdIncidence * logl(AbdIncidence); + SumIncidence += AbdIncidence; + + // Normalize. + if (SumIncidence != 0) + Energy = (Energy / SumIncidence) + logl(SumIncidence); + } + + // Increment the frequency of the feature Idx. + void UpdateFeatureFrequency(uint32_t Idx) { + NeedsEnergyUpdate = true; + + // The local feature frequencies is an ordered vector of pairs. + // If there are no local feature frequencies, push_back preserves order. + // Set the feature frequency for feature Idx32 to 1. + if (FeatureFreqs.empty()) { + FeatureFreqs.push_back(std::pair<uint32_t, uint16_t>(Idx, 1)); + return; + } + + // Binary search over local feature frequencies sorted by index. + auto Lower = std::lower_bound(FeatureFreqs.begin(), FeatureFreqs.end(), + std::pair<uint32_t, uint16_t>(Idx, 0)); + + // If feature Idx32 already exists, increment its frequency. + // Otherwise, insert a new pair right after the next lower index. + if (Lower != FeatureFreqs.end() && Lower->first == Idx) { + Lower->second++; + } else { + FeatureFreqs.insert(Lower, std::pair<uint32_t, uint16_t>(Idx, 1)); + } + } +}; + +struct EntropicOptions { + bool Enabled; + size_t NumberOfRarestFeatures; + size_t FeatureFrequencyThreshold; }; class InputCorpus { - static const size_t kFeatureSetSize = 1 << 21; - public: - InputCorpus(const std::string &OutputCorpus) : OutputCorpus(OutputCorpus) { + static const uint32_t kFeatureSetSize = 1 << 21; + static const uint8_t kMaxMutationFactor = 20; + static const size_t kSparseEnergyUpdates = 100; + + size_t NumExecutedMutations = 0; + + EntropicOptions Entropic; + +public: + InputCorpus(const std::string &OutputCorpus, EntropicOptions Entropic) + : Entropic(Entropic), OutputCorpus(OutputCorpus) { memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature)); memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature)); } @@ -70,6 +160,7 @@ class InputCorpus { Res = std::max(Res, II->U.size()); return Res; } + void IncrementNumExecutedMutations() { NumExecutedMutations++; } size_t NumInputsThatTouchFocusFunction() { return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) { @@ -99,6 +190,10 @@ class InputCorpus { II.MayDeleteFile = MayDeleteFile; II.UniqFeatureSet = FeatureSet; II.HasFocusFunction = HasFocusFunction; + // Assign maximal energy to the new seed. + II.Energy = RareFeatures.empty() ? 1.0 : log(RareFeatures.size()); + II.SumIncidence = RareFeatures.size(); + II.NeedsEnergyUpdate = false; std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end()); ComputeSHA1(U.data(), U.size(), II.Sha1); auto Sha1Str = Sha1ToString(II.Sha1); @@ -111,7 +206,7 @@ class InputCorpus { // But if we don't, we'll use the DFT of its base input. if (II.DataFlowTraceForFocusFunction.empty() && BaseII) II.DataFlowTraceForFocusFunction = BaseII->DataFlowTraceForFocusFunction; - UpdateCorpusDistribution(); + DistributionNeedsUpdate = true; PrintCorpus(); // ValidateFeatureSet(); return &II; @@ -162,7 +257,7 @@ class InputCorpus { Hashes.insert(Sha1ToString(II->Sha1)); II->U = U; II->Reduced = true; - UpdateCorpusDistribution(); + DistributionNeedsUpdate = true; } bool HasUnit(const Unit &U) { return Hashes.count(Hash(U)); } @@ -175,6 +270,7 @@ class InputCorpus { // Returns an index of random unit from the corpus to mutate. size_t ChooseUnitIdxToMutate(Random &Rand) { + UpdateCorpusDistribution(Rand); size_t Idx = static_cast<size_t>(CorpusDistribution(Rand)); assert(Idx < Inputs.size()); return Idx; @@ -210,10 +306,65 @@ class InputCorpus { InputInfo &II = *Inputs[Idx]; DeleteFile(II); Unit().swap(II.U); + II.Energy = 0.0; + II.NeedsEnergyUpdate = false; + DistributionNeedsUpdate = true; if (FeatureDebug) Printf("EVICTED %zd\n", Idx); } + void AddRareFeature(uint32_t Idx) { + // Maintain *at least* TopXRarestFeatures many rare features + // and all features with a frequency below ConsideredRare. + // Remove all other features. + while (RareFeatures.size() > Entropic.NumberOfRarestFeatures && + FreqOfMostAbundantRareFeature > Entropic.FeatureFrequencyThreshold) { + + // Find most and second most abbundant feature. + uint32_t MostAbundantRareFeatureIndices[2] = {RareFeatures[0], + RareFeatures[0]}; + size_t Delete = 0; + for (size_t i = 0; i < RareFeatures.size(); i++) { + uint32_t Idx2 = RareFeatures[i]; + if (GlobalFeatureFreqs[Idx2] >= + GlobalFeatureFreqs[MostAbundantRareFeatureIndices[0]]) { + MostAbundantRareFeatureIndices[1] = MostAbundantRareFeatureIndices[0]; + MostAbundantRareFeatureIndices[0] = Idx2; + Delete = i; + } + } + + // Remove most abundant rare feature. + RareFeatures[Delete] = RareFeatures.back(); + RareFeatures.pop_back(); + + for (auto II : Inputs) { + if (II->DeleteFeatureFreq(MostAbundantRareFeatureIndices[0])) + II->NeedsEnergyUpdate = true; + } + + // Set 2nd most abundant as the new most abundant feature count. + FreqOfMostAbundantRareFeature = + GlobalFeatureFreqs[MostAbundantRareFeatureIndices[1]]; + } + + // Add rare feature, handle collisions, and update energy. + RareFeatures.push_back(Idx); + GlobalFeatureFreqs[Idx] = 0; + for (auto II : Inputs) { + II->DeleteFeatureFreq(Idx); + + // Apply add-one smoothing to this locally undiscovered feature. + // Zero energy seeds will never be fuzzed and remain zero energy. + if (II->Energy > 0.0) { + II->SumIncidence += 1; + II->Energy += logl(II->SumIncidence) / II->SumIncidence; + } + } + + DistributionNeedsUpdate = true; + } + bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) { assert(NewSize); Idx = Idx % kFeatureSetSize; @@ -228,6 +379,8 @@ class InputCorpus { DeleteInput(OldIdx); } else { NumAddedFeatures++; + if (Entropic.Enabled) + AddRareFeature((uint32_t)Idx); } NumUpdatedFeatures++; if (FeatureDebug) @@ -239,6 +392,30 @@ class InputCorpus { return false; } + // Increment frequency of feature Idx globally and locally. + void UpdateFeatureFrequency(InputInfo *II, size_t Idx) { + uint32_t Idx32 = Idx % kFeatureSetSize; + + // Saturated increment. + if (GlobalFeatureFreqs[Idx32] == 0xFFFF) + return; + uint16_t Freq = GlobalFeatureFreqs[Idx32]++; + + // Skip if abundant. + if (Freq > FreqOfMostAbundantRareFeature || + std::find(RareFeatures.begin(), RareFeatures.end(), Idx32) == + RareFeatures.end()) + return; + + // Update global frequencies. + if (Freq == FreqOfMostAbundantRareFeature) + FreqOfMostAbundantRareFeature++; + + // Update local frequencies. + if (II) + II->UpdateFeatureFrequency(Idx32); + } + size_t NumFeatures() const { return NumAddedFeatures; } size_t NumFeatureUpdates() const { return NumUpdatedFeatures; } @@ -265,19 +442,60 @@ private: // Updates the probability distribution for the units in the corpus. // Must be called whenever the corpus or unit weights are changed. // - // Hypothesis: units added to the corpus last are more interesting. - // - // Hypothesis: inputs with infrequent features are more interesting. - void UpdateCorpusDistribution() { + // Hypothesis: inputs that maximize information about globally rare features + // are interesting. + void UpdateCorpusDistribution(Random &Rand) { + // Skip update if no seeds or rare features were added/deleted. + // Sparse updates for local change of feature frequencies, + // i.e., randomly do not skip. + if (!DistributionNeedsUpdate && + (!Entropic.Enabled || Rand(kSparseEnergyUpdates))) + return; + + DistributionNeedsUpdate = false; + size_t N = Inputs.size(); assert(N); Intervals.resize(N + 1); Weights.resize(N); std::iota(Intervals.begin(), Intervals.end(), 0); - for (size_t i = 0; i < N; i++) - Weights[i] = Inputs[i]->NumFeatures - ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1) - : 0.; + + bool VanillaSchedule = true; + if (Entropic.Enabled) { + for (auto II : Inputs) { + if (II->NeedsEnergyUpdate && II->Energy != 0.0) { + II->NeedsEnergyUpdate = false; + II->UpdateEnergy(RareFeatures.size()); + } + } + + for (size_t i = 0; i < N; i++) { + + if (Inputs[i]->NumFeatures == 0) { + // If the seed doesn't represent any features, assign zero energy. + Weights[i] = 0.; + } else if (Inputs[i]->NumExecutedMutations / kMaxMutationFactor > + NumExecutedMutations / Inputs.size()) { + // If the seed was fuzzed a lot more than average, assign zero energy. + Weights[i] = 0.; + } else { + // Otherwise, simply assign the computed energy. + Weights[i] = Inputs[i]->Energy; + } + + // If energy for all seeds is zero, fall back to vanilla schedule. + if (Weights[i] > 0.0) + VanillaSchedule = false; + } + } + + if (VanillaSchedule) { + for (size_t i = 0; i < N; i++) + Weights[i] = Inputs[i]->NumFeatures + ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1) + : 0.; + } + if (FeatureDebug) { for (size_t i = 0; i < N; i++) Printf("%zd ", Inputs[i]->NumFeatures); @@ -302,6 +520,11 @@ private: uint32_t InputSizesPerFeature[kFeatureSetSize]; uint32_t SmallestElementPerFeature[kFeatureSetSize]; + bool DistributionNeedsUpdate = true; + uint16_t FreqOfMostAbundantRareFeature = 0; + uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {}; + Vector<uint32_t> RareFeatures; + std::string OutputCorpus; }; diff --git a/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp b/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp index 99ff918f7c6c..48df8e668604 100644 --- a/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp @@ -248,6 +248,11 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, const Vector<SizedFile> &CorporaFiles) { Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n", DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size()); + if (CorporaFiles.empty()) { + Printf("ERROR: can't collect data flow without corpus provided."); + return 1; + } + static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0"; putenv(DFSanEnv); MkDir(DirPath); diff --git a/compiler-rt/lib/fuzzer/FuzzerDefs.h b/compiler-rt/lib/fuzzer/FuzzerDefs.h index 5793e86aa804..1a2752af2f4d 100644 --- a/compiler-rt/lib/fuzzer/FuzzerDefs.h +++ b/compiler-rt/lib/fuzzer/FuzzerDefs.h @@ -21,156 +21,6 @@ #include <vector> -// Platform detection. -#ifdef __linux__ -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_FUCHSIA 0 -#define LIBFUZZER_LINUX 1 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 -#define LIBFUZZER_WINDOWS 0 -#define LIBFUZZER_EMSCRIPTEN 0 -#elif __APPLE__ -#define LIBFUZZER_APPLE 1 -#define LIBFUZZER_FUCHSIA 0 -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 -#define LIBFUZZER_WINDOWS 0 -#define LIBFUZZER_EMSCRIPTEN 0 -#elif __NetBSD__ -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_FUCHSIA 0 -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_NETBSD 1 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 -#define LIBFUZZER_WINDOWS 0 -#define LIBFUZZER_EMSCRIPTEN 0 -#elif __FreeBSD__ -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_FUCHSIA 0 -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 1 -#define LIBFUZZER_OPENBSD 0 -#define LIBFUZZER_WINDOWS 0 -#define LIBFUZZER_EMSCRIPTEN 0 -#elif __OpenBSD__ -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_FUCHSIA 0 -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 1 -#define LIBFUZZER_WINDOWS 0 -#define LIBFUZZER_EMSCRIPTEN 0 -#elif _WIN32 -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_FUCHSIA 0 -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 -#define LIBFUZZER_WINDOWS 1 -#define LIBFUZZER_EMSCRIPTEN 0 -#elif __Fuchsia__ -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_FUCHSIA 1 -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 -#define LIBFUZZER_WINDOWS 0 -#define LIBFUZZER_EMSCRIPTEN 0 -#elif __EMSCRIPTEN__ -#define LIBFUZZER_APPLE 0 -#define LIBFUZZER_FUCHSIA 0 -#define LIBFUZZER_LINUX 0 -#define LIBFUZZER_NETBSD 0 -#define LIBFUZZER_FREEBSD 0 -#define LIBFUZZER_OPENBSD 0 -#define LIBFUZZER_WINDOWS 0 -#define LIBFUZZER_EMSCRIPTEN 1 -#else -#error "Support for your platform has not been implemented" -#endif - -#if defined(_MSC_VER) && !defined(__clang__) -// MSVC compiler is being used. -#define LIBFUZZER_MSVC 1 -#else -#define LIBFUZZER_MSVC 0 -#endif - -#ifndef __has_attribute -# define __has_attribute(x) 0 -#endif - -#define LIBFUZZER_POSIX \ - (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || \ - LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) - -#ifdef __x86_64 -# if __has_attribute(target) -# define ATTRIBUTE_TARGET_POPCNT __attribute__((target("popcnt"))) -# else -# define ATTRIBUTE_TARGET_POPCNT -# endif -#else -# define ATTRIBUTE_TARGET_POPCNT -#endif - - -#ifdef __clang__ // avoid gcc warning. -# if __has_attribute(no_sanitize) -# define ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) -# else -# define ATTRIBUTE_NO_SANITIZE_MEMORY -# endif -# define ALWAYS_INLINE __attribute__((always_inline)) -#else -# define ATTRIBUTE_NO_SANITIZE_MEMORY -# define ALWAYS_INLINE -#endif // __clang__ - -#if LIBFUZZER_WINDOWS -#define ATTRIBUTE_NO_SANITIZE_ADDRESS -#else -#define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) -#endif - -#if LIBFUZZER_WINDOWS -#define ATTRIBUTE_ALIGNED(X) __declspec(align(X)) -#define ATTRIBUTE_INTERFACE __declspec(dllexport) -// This is used for __sancov_lowest_stack which is needed for -// -fsanitize-coverage=stack-depth. That feature is not yet available on -// Windows, so make the symbol static to avoid linking errors. -#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC static -#define ATTRIBUTE_NOINLINE __declspec(noinline) -#else -#define ATTRIBUTE_ALIGNED(X) __attribute__((aligned(X))) -#define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) -#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ - ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local - -#define ATTRIBUTE_NOINLINE __attribute__((noinline)) -#endif - -#if defined(__has_feature) -# if __has_feature(address_sanitizer) -# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_ADDRESS -# elif __has_feature(memory_sanitizer) -# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_MEMORY -# else -# define ATTRIBUTE_NO_SANITIZE_ALL -# endif -#else -# define ATTRIBUTE_NO_SANITIZE_ALL -#endif - namespace fuzzer { template <class T> T Min(T a, T b) { return a < b ? a : b; } diff --git a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp index dd3cab0ee8d2..a847c76e292d 100644 --- a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -16,6 +16,7 @@ #include "FuzzerInternal.h" #include "FuzzerMerge.h" #include "FuzzerMutate.h" +#include "FuzzerPlatform.h" #include "FuzzerRandom.h" #include "FuzzerTracePC.h" #include <algorithm> @@ -195,8 +196,11 @@ static void ParseFlags(const Vector<std::string> &Args, } // Disable len_control by default, if LLVMFuzzerCustomMutator is used. - if (EF->LLVMFuzzerCustomMutator) + if (EF->LLVMFuzzerCustomMutator) { Flags.len_control = 0; + Printf("INFO: found LLVMFuzzerCustomMutator (%p). " + "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); + } Inputs = new Vector<std::string>; for (size_t A = 1; A < Args.size(); A++) { @@ -303,8 +307,7 @@ static bool AllInputsAreFiles() { return true; } -static std::string GetDedupTokenFromFile(const std::string &Path) { - auto S = FileToString(Path); +static std::string GetDedupTokenFromCmdOutput(const std::string &S) { auto Beg = S.find("DEDUP_TOKEN:"); if (Beg == std::string::npos) return ""; @@ -329,10 +332,9 @@ int CleanseCrashInput(const Vector<std::string> &Args, assert(Cmd.hasArgument(InputFilePath)); Cmd.removeArgument(InputFilePath); - auto LogFilePath = TempPath(".txt"); - auto TmpFilePath = TempPath(".repro"); + auto TmpFilePath = TempPath("CleanseCrashInput", ".repro"); Cmd.addArgument(TmpFilePath); - Cmd.setOutputFile(LogFilePath); + Cmd.setOutputFile(getDevNull()); Cmd.combineOutAndErr(); std::string CurrentFilePath = InputFilePath; @@ -367,7 +369,6 @@ int CleanseCrashInput(const Vector<std::string> &Args, } if (!Changed) break; } - RemoveFile(LogFilePath); return 0; } @@ -390,8 +391,6 @@ int MinimizeCrashInput(const Vector<std::string> &Args, BaseCmd.addFlag("max_total_time", "600"); } - auto LogFilePath = TempPath(".txt"); - BaseCmd.setOutputFile(LogFilePath); BaseCmd.combineOutAndErr(); std::string CurrentFilePath = InputFilePath; @@ -403,17 +402,17 @@ int MinimizeCrashInput(const Vector<std::string> &Args, Command Cmd(BaseCmd); Cmd.addArgument(CurrentFilePath); - std::string CommandLine = Cmd.toString(); - Printf("CRASH_MIN: executing: %s\n", CommandLine.c_str()); - int ExitCode = ExecuteCommand(Cmd); - if (ExitCode == 0) { + Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str()); + std::string CmdOutput; + bool Success = ExecuteCommand(Cmd, &CmdOutput); + if (Success) { Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); exit(1); } Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " "it further\n", CurrentFilePath.c_str(), U.size()); - auto DedupToken1 = GetDedupTokenFromFile(LogFilePath); + auto DedupToken1 = GetDedupTokenFromCmdOutput(CmdOutput); if (!DedupToken1.empty()) Printf("CRASH_MIN: DedupToken1: %s\n", DedupToken1.c_str()); @@ -423,11 +422,11 @@ int MinimizeCrashInput(const Vector<std::string> &Args, : Options.ArtifactPrefix + "minimized-from-" + Hash(U); Cmd.addFlag("minimize_crash_internal_step", "1"); Cmd.addFlag("exact_artifact_path", ArtifactPath); - CommandLine = Cmd.toString(); - Printf("CRASH_MIN: executing: %s\n", CommandLine.c_str()); - ExitCode = ExecuteCommand(Cmd); - CopyFileToErr(LogFilePath); - if (ExitCode == 0) { + Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str()); + CmdOutput.clear(); + Success = ExecuteCommand(Cmd, &CmdOutput); + Printf("%s", CmdOutput.c_str()); + if (Success) { if (Flags.exact_artifact_path) { CurrentFilePath = Flags.exact_artifact_path; WriteToFile(U, CurrentFilePath); @@ -436,7 +435,7 @@ int MinimizeCrashInput(const Vector<std::string> &Args, CurrentFilePath.c_str(), U.size()); break; } - auto DedupToken2 = GetDedupTokenFromFile(LogFilePath); + auto DedupToken2 = GetDedupTokenFromCmdOutput(CmdOutput); if (!DedupToken2.empty()) Printf("CRASH_MIN: DedupToken2: %s\n", DedupToken2.c_str()); @@ -453,7 +452,6 @@ int MinimizeCrashInput(const Vector<std::string> &Args, CurrentFilePath = ArtifactPath; Printf("*********************************\n"); } - RemoveFile(LogFilePath); return 0; } @@ -488,7 +486,7 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, std::sort(OldCorpus.begin(), OldCorpus.end()); std::sort(NewCorpus.begin(), NewCorpus.end()); - std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath(".txt"); + std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); Vector<std::string> NewFiles; Set<uint32_t> NewFeatures, NewCov; CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, @@ -711,6 +709,26 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.CollectDataFlow = Flags.collect_data_flow; if (Flags.stop_file) Options.StopFile = Flags.stop_file; + Options.Entropic = Flags.entropic; + Options.EntropicFeatureFrequencyThreshold = + (size_t)Flags.entropic_feature_frequency_threshold; + Options.EntropicNumberOfRarestFeatures = + (size_t)Flags.entropic_number_of_rarest_features; + if (Options.Entropic) { + if (!Options.FocusFunction.empty()) { + Printf("ERROR: The parameters `--entropic` and `--focus_function` cannot " + "be used together.\n"); + exit(1); + } + Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", + Options.EntropicFeatureFrequencyThreshold, + Options.EntropicNumberOfRarestFeatures); + } + struct EntropicOptions Entropic; + Entropic.Enabled = Options.Entropic; + Entropic.FeatureFrequencyThreshold = + Options.EntropicFeatureFrequencyThreshold; + Entropic.NumberOfRarestFeatures = Options.EntropicNumberOfRarestFeatures; unsigned Seed = Flags.seed; // Initialize Seed. @@ -731,7 +749,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Random Rand(Seed); auto *MD = new MutationDispatcher(Rand, Options); - auto *Corpus = new InputCorpus(Options.OutputCorpus); + auto *Corpus = new InputCorpus(Options.OutputCorpus, Entropic); auto *F = new Fuzzer(Callback, *Corpus, *MD, Options); for (auto &U: Dictionary) diff --git a/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp b/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp index dcd713459487..95233d2a10d3 100644 --- a/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp @@ -10,7 +10,7 @@ // requires that clients of LibFuzzer pass ``--export-dynamic`` to the linker. // That is a complication we don't wish to expose to clients right now. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_APPLE #include "FuzzerExtFunctions.h" diff --git a/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp b/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp index d56dab36c646..24ddc57d47d6 100644 --- a/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp @@ -11,7 +11,7 @@ // weak symbols to be undefined. That is a complication we don't want to expose // to clients right now. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA || \ LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN diff --git a/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp b/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp index 55efe8f80e90..688bad1d51ca 100644 --- a/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp @@ -9,7 +9,7 @@ // compiled with MSVC. Uses weak aliases when compiled with clang. Unfortunately // the method each compiler supports is not supported by the other. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_WINDOWS #include "FuzzerExtFunctions.h" diff --git a/compiler-rt/lib/fuzzer/FuzzerExtraCounters.cpp b/compiler-rt/lib/fuzzer/FuzzerExtraCounters.cpp index b2face778203..d36beba1b1ba 100644 --- a/compiler-rt/lib/fuzzer/FuzzerExtraCounters.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerExtraCounters.cpp @@ -8,7 +8,8 @@ // Extra coverage counters defined by user code. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" +#include <cstdint> #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ LIBFUZZER_OPENBSD || LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN diff --git a/compiler-rt/lib/fuzzer/FuzzerFlags.def b/compiler-rt/lib/fuzzer/FuzzerFlags.def index a67415743032..832224a705d2 100644 --- a/compiler-rt/lib/fuzzer/FuzzerFlags.def +++ b/compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -22,7 +22,7 @@ FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is used.") FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " "to use as an additional seed corpus. Alternatively, an \"@\" followed by " - "the name of a file containing the comma-seperated list.") + "the name of a file containing the comma-separated list.") FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") FUZZER_FLAG_INT(mutate_depth, 5, "Apply this number of consecutive mutations to each input.") @@ -153,6 +153,14 @@ FUZZER_FLAG_STRING(focus_function, "Experimental. " "Fuzzing will focus on inputs that trigger calls to this function. " "If -focus_function=auto and -data_flow_trace is used, libFuzzer " "will choose the focus functions automatically.") +FUZZER_FLAG_INT(entropic, 0, "Experimental. Enables entropic power schedule.") +FUZZER_FLAG_INT(entropic_feature_frequency_threshold, 0xFF, "Experimental. If " + "entropic is enabled, all features which are observed less often than " + "the specified value are considered as rare.") +FUZZER_FLAG_INT(entropic_number_of_rarest_features, 100, "Experimental. If " + "entropic is enabled, we keep track of the frequencies only for the " + "Top-X least abundant features (union features that are considered as " + "rare).") FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) diff --git a/compiler-rt/lib/fuzzer/FuzzerFork.cpp b/compiler-rt/lib/fuzzer/FuzzerFork.cpp index 95ed36551463..d9e6b79443e0 100644 --- a/compiler-rt/lib/fuzzer/FuzzerFork.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerFork.cpp @@ -297,7 +297,7 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, for (auto &Dir : CorpusDirs) GetSizedFilesFromDir(Dir, &SeedFiles); std::sort(SeedFiles.begin(), SeedFiles.end()); - Env.TempDir = TempPath(".dir"); + Env.TempDir = TempPath("FuzzWithFork", ".dir"); Env.DFTDir = DirPlusFile(Env.TempDir, "DFT"); RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. MkDir(Env.TempDir); diff --git a/compiler-rt/lib/fuzzer/FuzzerIO.cpp b/compiler-rt/lib/fuzzer/FuzzerIO.cpp index f0708164be87..cbb1dbe1b86d 100644 --- a/compiler-rt/lib/fuzzer/FuzzerIO.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerIO.cpp @@ -151,9 +151,9 @@ void RmDirRecursive(const std::string &Dir) { [](const std::string &Path) { RemoveFile(Path); }); } -std::string TempPath(const char *Extension) { - return DirPlusFile(TmpDir(), - "libFuzzerTemp." + std::to_string(GetPid()) + Extension); +std::string TempPath(const char *Prefix, const char *Extension) { + return DirPlusFile(TmpDir(), std::string("libFuzzerTemp.") + Prefix + + std::to_string(GetPid()) + Extension); } } // namespace fuzzer diff --git a/compiler-rt/lib/fuzzer/FuzzerIO.h b/compiler-rt/lib/fuzzer/FuzzerIO.h index ae8dd24e373c..6e4368b971fa 100644 --- a/compiler-rt/lib/fuzzer/FuzzerIO.h +++ b/compiler-rt/lib/fuzzer/FuzzerIO.h @@ -42,7 +42,7 @@ std::string DirName(const std::string &FileName); // Returns path to a TmpDir. std::string TmpDir(); -std::string TempPath(const char *Extension); +std::string TempPath(const char *Prefix, const char *Extension); bool IsInterestingCoverageFile(const std::string &FileName); diff --git a/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp b/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp index fcd9b8d8b9c7..aac85b08727a 100644 --- a/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // IO functions implementation using Posix API. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA #include "FuzzerExtFunctions.h" diff --git a/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp b/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp index 56757aa09a37..651283a551cf 100644 --- a/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // IO functions implementation for Windows. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_WINDOWS #include "FuzzerExtFunctions.h" diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp index 451a4c173167..02db6d27b0a3 100644 --- a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp @@ -12,6 +12,7 @@ #include "FuzzerIO.h" #include "FuzzerInternal.h" #include "FuzzerMutate.h" +#include "FuzzerPlatform.h" #include "FuzzerRandom.h" #include "FuzzerTracePC.h" #include <algorithm> @@ -256,7 +257,7 @@ void Fuzzer::ExitCallback() { void Fuzzer::MaybeExitGracefully() { if (!F->GracefulExitRequested) return; Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); - RmDirRecursive(TempPath(".dir")); + RmDirRecursive(TempPath("FuzzWithFork", ".dir")); F->PrintFinalStats(); _Exit(0); } @@ -265,7 +266,7 @@ void Fuzzer::InterruptCallback() { Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); PrintFinalStats(); ScopedDisableMsanInterceptorChecks S; // RmDirRecursive may call opendir(). - RmDirRecursive(TempPath(".dir")); + RmDirRecursive(TempPath("FuzzWithFork", ".dir")); // Stop right now, don't perform any at-exit actions. _Exit(Options.InterruptExitCode); } @@ -475,6 +476,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, TPC.CollectFeatures([&](size_t Feature) { if (Corpus.AddFeature(Feature, Size, Options.Shrink)) UniqFeatureSetTmp.push_back(Feature); + if (Options.Entropic) + Corpus.UpdateFeatureFrequency(II, Feature); if (Options.ReduceInputs && II) if (std::binary_search(II->UniqFeatureSet.begin(), II->UniqFeatureSet.end(), Feature)) @@ -693,6 +696,7 @@ void Fuzzer::MutateAndTestOne() { assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); Size = NewSize; II.NumExecutedMutations++; + Corpus.IncrementNumExecutedMutations(); bool FoundUniqFeatures = false; bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, @@ -706,6 +710,8 @@ void Fuzzer::MutateAndTestOne() { if (Options.ReduceDepth && !FoundUniqFeatures) break; } + + II.NeedsEnergyUpdate = true; } void Fuzzer::PurgeAllocator() { @@ -770,12 +776,14 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { } PrintStats("INITED"); - if (!Options.FocusFunction.empty()) + if (!Options.FocusFunction.empty()) { Printf("INFO: %zd/%zd inputs touch the focus function\n", Corpus.NumInputsThatTouchFocusFunction(), Corpus.size()); - if (!Options.DataFlowTrace.empty()) - Printf("INFO: %zd/%zd inputs have the Data Flow Trace\n", - Corpus.NumInputsWithDataFlowTrace(), Corpus.size()); + if (!Options.DataFlowTrace.empty()) + Printf("INFO: %zd/%zd inputs have the Data Flow Trace\n", + Corpus.NumInputsWithDataFlowTrace(), + Corpus.NumInputsThatTouchFocusFunction()); + } if (Corpus.empty() && Options.MaxNumberOfRuns) { Printf("ERROR: no interesting inputs were found. " diff --git a/compiler-rt/lib/fuzzer/FuzzerMain.cpp b/compiler-rt/lib/fuzzer/FuzzerMain.cpp index 771a34aed316..75f2f8e75c9b 100644 --- a/compiler-rt/lib/fuzzer/FuzzerMain.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerMain.cpp @@ -9,6 +9,7 @@ //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" +#include "FuzzerPlatform.h" extern "C" { // This function should be defined by the user. diff --git a/compiler-rt/lib/fuzzer/FuzzerMerge.h b/compiler-rt/lib/fuzzer/FuzzerMerge.h index c14dd589e62d..e0c6bc539bdb 100644 --- a/compiler-rt/lib/fuzzer/FuzzerMerge.h +++ b/compiler-rt/lib/fuzzer/FuzzerMerge.h @@ -13,7 +13,7 @@ // The process should tolerate the crashes, OOMs, leaks, etc. // // Algorithm: -// The outter process collects the set of files and writes their names +// The outer process collects the set of files and writes their names // into a temporary "control" file, then repeatedly launches the inner // process until all inputs are processed. // The outer process does not actually execute the target code. @@ -22,13 +22,14 @@ // and b) the last processed input. Then it starts processing the inputs one // by one. Before processing every input it writes one line to control file: // STARTED INPUT_ID INPUT_SIZE -// After processing an input it write another line: -// DONE INPUT_ID Feature1 Feature2 Feature3 ... +// After processing an input it writes the following lines: +// FT INPUT_ID Feature1 Feature2 Feature3 ... +// COV INPUT_ID Coverage1 Coverage2 Coverage3 ... // If a crash happens while processing an input the last line in the control // file will be "STARTED INPUT_ID" and so the next process will know // where to resume. // -// Once all inputs are processed by the innner process(es) the outer process +// Once all inputs are processed by the inner process(es) the outer process // reads the control files and does the merge based entirely on the contents // of control file. // It uses a single pass greedy algorithm choosing first the smallest inputs diff --git a/compiler-rt/lib/fuzzer/FuzzerOptions.h b/compiler-rt/lib/fuzzer/FuzzerOptions.h index beecc980380b..9d975bd61fe7 100644 --- a/compiler-rt/lib/fuzzer/FuzzerOptions.h +++ b/compiler-rt/lib/fuzzer/FuzzerOptions.h @@ -44,6 +44,9 @@ struct FuzzingOptions { size_t MaxNumberOfRuns = -1L; int ReportSlowUnits = 10; bool OnlyASCII = false; + bool Entropic = false; + size_t EntropicFeatureFrequencyThreshold = 0xFF; + size_t EntropicNumberOfRarestFeatures = 100; std::string OutputCorpus; std::string ArtifactPrefix = "./"; std::string ExactArtifactPath; diff --git a/compiler-rt/lib/fuzzer/FuzzerPlatform.h b/compiler-rt/lib/fuzzer/FuzzerPlatform.h new file mode 100644 index 000000000000..8befdb882cc6 --- /dev/null +++ b/compiler-rt/lib/fuzzer/FuzzerPlatform.h @@ -0,0 +1,163 @@ +//===-- FuzzerPlatform.h --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Common platform macros. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_PLATFORM_H +#define LLVM_FUZZER_PLATFORM_H + +// Platform detection. +#ifdef __linux__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 1 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __APPLE__ +#define LIBFUZZER_APPLE 1 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __NetBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 1 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __FreeBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 1 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __OpenBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 1 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif _WIN32 +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 1 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __Fuchsia__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 1 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __EMSCRIPTEN__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 1 +#else +#error "Support for your platform has not been implemented" +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC compiler is being used. +#define LIBFUZZER_MSVC 1 +#else +#define LIBFUZZER_MSVC 0 +#endif + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#define LIBFUZZER_POSIX \ + (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || \ + LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) + +#ifdef __x86_64 +#if __has_attribute(target) +#define ATTRIBUTE_TARGET_POPCNT __attribute__((target("popcnt"))) +#else +#define ATTRIBUTE_TARGET_POPCNT +#endif +#else +#define ATTRIBUTE_TARGET_POPCNT +#endif + +#ifdef __clang__ // avoid gcc warning. +#if __has_attribute(no_sanitize) +#define ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) +#else +#define ATTRIBUTE_NO_SANITIZE_MEMORY +#endif +#define ALWAYS_INLINE __attribute__((always_inline)) +#else +#define ATTRIBUTE_NO_SANITIZE_MEMORY +#define ALWAYS_INLINE +#endif // __clang__ + +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_NO_SANITIZE_ADDRESS +#else +#define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#endif + +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_ALIGNED(X) __declspec(align(X)) +#define ATTRIBUTE_INTERFACE __declspec(dllexport) +// This is used for __sancov_lowest_stack which is needed for +// -fsanitize-coverage=stack-depth. That feature is not yet available on +// Windows, so make the symbol static to avoid linking errors. +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC static +#define ATTRIBUTE_NOINLINE __declspec(noinline) +#else +#define ATTRIBUTE_ALIGNED(X) __attribute__((aligned(X))) +#define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ + ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local + +#define ATTRIBUTE_NOINLINE __attribute__((noinline)) +#endif + +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_ADDRESS +#elif __has_feature(memory_sanitizer) +#define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_MEMORY +#else +#define ATTRIBUTE_NO_SANITIZE_ALL +#endif +#else +#define ATTRIBUTE_NO_SANITIZE_ALL +#endif + +#endif // LLVM_FUZZER_PLATFORM_H diff --git a/compiler-rt/lib/fuzzer/FuzzerSHA1.cpp b/compiler-rt/lib/fuzzer/FuzzerSHA1.cpp index 43e5e78cd787..2005dc700305 100644 --- a/compiler-rt/lib/fuzzer/FuzzerSHA1.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerSHA1.cpp @@ -17,6 +17,7 @@ #include "FuzzerSHA1.h" #include "FuzzerDefs.h" +#include "FuzzerPlatform.h" /* This code is public-domain - it is based on libcrypt * placed in the public domain by Wei Dai and other contributors. diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp index f03be7a39502..b2ca7693e540 100644 --- a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp @@ -19,6 +19,7 @@ #include "FuzzerDictionary.h" #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" +#include "FuzzerPlatform.h" #include "FuzzerUtil.h" #include "FuzzerValueBitMap.h" #include <set> @@ -240,7 +241,9 @@ void TracePC::IterateCoveredFunctions(CallBack CB) { void TracePC::SetFocusFunction(const std::string &FuncName) { // This function should be called once. assert(!FocusFunctionCounterPtr); - if (FuncName.empty()) + // "auto" is not a valid function name. If this function is called with "auto" + // that means the auto focus functionality failed. + if (FuncName.empty() || FuncName == "auto") return; for (size_t M = 0; M < NumModules; M++) { auto &PCTE = ModulePCTable[M]; @@ -256,6 +259,10 @@ void TracePC::SetFocusFunction(const std::string &FuncName) { return; } } + + Printf("ERROR: Failed to set focus function. Make sure the function name is " + "valid (%s) and symbolization is enabled.\n", FuncName.c_str()); + exit(1); } bool TracePC::ObservedFocusFunction() { diff --git a/compiler-rt/lib/fuzzer/FuzzerUtil.cpp b/compiler-rt/lib/fuzzer/FuzzerUtil.cpp index 7aa84a1faad7..7eecb68d0729 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtil.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtil.cpp @@ -151,32 +151,37 @@ bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) { return true; } +// Code duplicated (and tested) in llvm/include/llvm/Support/Base64.h std::string Base64(const Unit &U) { static const char Table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789+/"; - std::string Res; - size_t i; - for (i = 0; i + 2 < U.size(); i += 3) { - uint32_t x = (U[i] << 16) + (U[i + 1] << 8) + U[i + 2]; - Res += Table[(x >> 18) & 63]; - Res += Table[(x >> 12) & 63]; - Res += Table[(x >> 6) & 63]; - Res += Table[x & 63]; + std::string Buffer; + Buffer.resize(((U.size() + 2) / 3) * 4); + + size_t i = 0, j = 0; + for (size_t n = U.size() / 3 * 3; i < n; i += 3, j += 4) { + uint32_t x = ((unsigned char)U[i] << 16) | ((unsigned char)U[i + 1] << 8) | + (unsigned char)U[i + 2]; + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = Table[(x >> 6) & 63]; + Buffer[j + 3] = Table[x & 63]; } if (i + 1 == U.size()) { - uint32_t x = (U[i] << 16); - Res += Table[(x >> 18) & 63]; - Res += Table[(x >> 12) & 63]; - Res += "=="; + uint32_t x = ((unsigned char)U[i] << 16); + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = '='; + Buffer[j + 3] = '='; } else if (i + 2 == U.size()) { - uint32_t x = (U[i] << 16) + (U[i + 1] << 8); - Res += Table[(x >> 18) & 63]; - Res += Table[(x >> 12) & 63]; - Res += Table[(x >> 6) & 63]; - Res += "="; + uint32_t x = ((unsigned char)U[i] << 16) | ((unsigned char)U[i + 1] << 8); + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = Table[(x >> 6) & 63]; + Buffer[j + 3] = '='; } - return Res; + return Buffer; } static std::mutex SymbolizeMutex; diff --git a/compiler-rt/lib/fuzzer/FuzzerUtil.h b/compiler-rt/lib/fuzzer/FuzzerUtil.h index 00ea6550646f..4ae35838306d 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtil.h +++ b/compiler-rt/lib/fuzzer/FuzzerUtil.h @@ -57,8 +57,11 @@ unsigned long GetPid(); size_t GetPeakRSSMb(); int ExecuteCommand(const Command &Cmd); +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput); +// Fuchsia does not have popen/pclose. FILE *OpenProcessPipe(const char *Command, const char *Mode); +int CloseProcessPipe(FILE *F); const void *SearchMemory(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilDarwin.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilDarwin.cpp index d449bc248f09..a5bed658a446 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtilDarwin.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtilDarwin.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // Misc utils for Darwin. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_APPLE #include "FuzzerCommand.h" #include "FuzzerIO.h" diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp index bde9f68d62aa..190fb7866649 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // Misc utils implementation using Fuchsia/Zircon APIs. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_FUCHSIA @@ -34,6 +34,8 @@ #include <zircon/syscalls/object.h> #include <zircon/types.h> +#include <vector> + namespace fuzzer { // Given that Fuchsia doesn't have the POSIX signals that libFuzzer was written @@ -424,6 +426,17 @@ RunOnDestruction<Fn> at_scope_exit(Fn fn) { return RunOnDestruction<Fn>(fn); } +static fdio_spawn_action_t clone_fd_action(int localFd, int targetFd) { + return { + .action = FDIO_SPAWN_ACTION_CLONE_FD, + .fd = + { + .local_fd = localFd, + .target_fd = targetFd, + }, + }; +} + int ExecuteCommand(const Command &Cmd) { zx_status_t rc; @@ -442,17 +455,26 @@ int ExecuteCommand(const Command &Cmd) { // so write the log file(s) there. // However, we don't want to apply this logic for absolute paths. int FdOut = STDOUT_FILENO; + bool discardStdout = false; + bool discardStderr = false; + if (Cmd.hasOutputFile()) { std::string Path = Cmd.getOutputFile(); - bool IsAbsolutePath = Path.length() > 1 && Path[0] == '/'; - if (!IsAbsolutePath && Cmd.hasFlag("artifact_prefix")) - Path = Cmd.getFlagValue("artifact_prefix") + "/" + Path; - - FdOut = open(Path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); - if (FdOut == -1) { - Printf("libFuzzer: failed to open %s: %s\n", Path.c_str(), - strerror(errno)); - return ZX_ERR_IO; + if (Path == getDevNull()) { + // On Fuchsia, there's no "/dev/null" like-file, so we + // just don't copy the FDs into the spawned process. + discardStdout = true; + } else { + bool IsAbsolutePath = Path.length() > 1 && Path[0] == '/'; + if (!IsAbsolutePath && Cmd.hasFlag("artifact_prefix")) + Path = Cmd.getFlagValue("artifact_prefix") + "/" + Path; + + FdOut = open(Path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); + if (FdOut == -1) { + Printf("libFuzzer: failed to open %s: %s\n", Path.c_str(), + strerror(errno)); + return ZX_ERR_IO; + } } } auto CloseFdOut = at_scope_exit([FdOut]() { @@ -462,43 +484,29 @@ int ExecuteCommand(const Command &Cmd) { // Determine stderr int FdErr = STDERR_FILENO; - if (Cmd.isOutAndErrCombined()) + if (Cmd.isOutAndErrCombined()) { FdErr = FdOut; + if (discardStdout) + discardStderr = true; + } // Clone the file descriptors into the new process - fdio_spawn_action_t SpawnAction[] = { - { - .action = FDIO_SPAWN_ACTION_CLONE_FD, - .fd = - { - .local_fd = STDIN_FILENO, - .target_fd = STDIN_FILENO, - }, - }, - { - .action = FDIO_SPAWN_ACTION_CLONE_FD, - .fd = - { - .local_fd = FdOut, - .target_fd = STDOUT_FILENO, - }, - }, - { - .action = FDIO_SPAWN_ACTION_CLONE_FD, - .fd = - { - .local_fd = FdErr, - .target_fd = STDERR_FILENO, - }, - }, - }; + std::vector<fdio_spawn_action_t> SpawnActions; + SpawnActions.push_back(clone_fd_action(STDIN_FILENO, STDIN_FILENO)); + + if (!discardStdout) + SpawnActions.push_back(clone_fd_action(FdOut, STDOUT_FILENO)); + if (!discardStderr) + SpawnActions.push_back(clone_fd_action(FdErr, STDERR_FILENO)); // Start the process. char ErrorMsg[FDIO_SPAWN_ERR_MSG_MAX_LENGTH]; zx_handle_t ProcessHandle = ZX_HANDLE_INVALID; - rc = fdio_spawn_etc( - ZX_HANDLE_INVALID, FDIO_SPAWN_CLONE_ALL & (~FDIO_SPAWN_CLONE_STDIO), - Argv[0], Argv.get(), nullptr, 3, SpawnAction, &ProcessHandle, ErrorMsg); + rc = fdio_spawn_etc(ZX_HANDLE_INVALID, + FDIO_SPAWN_CLONE_ALL & (~FDIO_SPAWN_CLONE_STDIO), Argv[0], + Argv.get(), nullptr, SpawnActions.size(), + SpawnActions.data(), &ProcessHandle, ErrorMsg); + if (rc != ZX_OK) { Printf("libFuzzer: failed to launch '%s': %s, %s\n", Argv[0], ErrorMsg, _zx_status_get_string(rc)); @@ -525,6 +533,16 @@ int ExecuteCommand(const Command &Cmd) { return Info.return_code; } +bool ExecuteCommand(const Command &BaseCmd, std::string *CmdOutput) { + auto LogFilePath = TempPath("SimPopenOut", ".txt"); + Command Cmd(BaseCmd); + Cmd.setOutputFile(LogFilePath); + int Ret = ExecuteCommand(Cmd); + *CmdOutput = FileToString(LogFilePath); + RemoveFile(LogFilePath); + return Ret == 0; +} + const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, size_t PattLen) { return memmem(Data, DataLen, Patt, PattLen); diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp index 993023e70393..95490b992e0b 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // Misc utils for Linux. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN #include "FuzzerCommand.h" diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp index 8048e6a8afd2..fc57b724db10 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // Misc utils implementation using Posix API. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_POSIX #include "FuzzerIO.h" #include "FuzzerInternal.h" @@ -86,6 +86,20 @@ static void SetSigaction(int signum, } } +// Return true on success, false otherwise. +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput) { + FILE *Pipe = popen(Cmd.toString().c_str(), "r"); + if (!Pipe) + return false; + + if (CmdOutput) { + char TmpBuffer[128]; + while (fgets(TmpBuffer, sizeof(TmpBuffer), Pipe)) + CmdOutput->append(TmpBuffer); + } + return pclose(Pipe) == 0; +} + void SetTimer(int Seconds) { struct itimerval T { {Seconds, 0}, { Seconds, 0 } @@ -149,6 +163,10 @@ FILE *OpenProcessPipe(const char *Command, const char *Mode) { return popen(Command, Mode); } +int CloseProcessPipe(FILE *F) { + return pclose(F); +} + const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, size_t PattLen) { return memmem(Data, DataLen, Patt, PattLen); diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp index 527e7dbd1cf6..6c693e3d7eea 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // Misc utils implementation for Windows. //===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" #if LIBFUZZER_WINDOWS #include "FuzzerCommand.h" #include "FuzzerIO.h" @@ -152,11 +152,28 @@ FILE *OpenProcessPipe(const char *Command, const char *Mode) { return _popen(Command, Mode); } +int CloseProcessPipe(FILE *F) { + return _pclose(F); +} + int ExecuteCommand(const Command &Cmd) { std::string CmdLine = Cmd.toString(); return system(CmdLine.c_str()); } +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput) { + FILE *Pipe = _popen(Cmd.toString().c_str(), "r"); + if (!Pipe) + return false; + + if (CmdOutput) { + char TmpBuffer[128]; + while (fgets(TmpBuffer, sizeof(TmpBuffer), Pipe)) + CmdOutput->append(TmpBuffer); + } + return _pclose(Pipe) == 0; +} + const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, size_t PattLen) { // TODO: make this implementation more efficient. diff --git a/compiler-rt/lib/fuzzer/FuzzerValueBitMap.h b/compiler-rt/lib/fuzzer/FuzzerValueBitMap.h index bc039f1df27f..ddbfe200af9c 100644 --- a/compiler-rt/lib/fuzzer/FuzzerValueBitMap.h +++ b/compiler-rt/lib/fuzzer/FuzzerValueBitMap.h @@ -11,7 +11,8 @@ #ifndef LLVM_FUZZER_VALUE_BIT_MAP_H #define LLVM_FUZZER_VALUE_BIT_MAP_H -#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" +#include <cstdint> namespace fuzzer { |