diff options
Diffstat (limited to 'lib/Fuzzer')
-rw-r--r-- | lib/Fuzzer/CMakeLists.txt | 1 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerCorpus.h | 40 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerDriver.cpp | 20 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerFlags.def | 3 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerInternal.h | 14 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerLoop.cpp | 30 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerMerge.cpp | 8 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerMutate.cpp | 23 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerMutate.h | 6 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerTracePC.cpp | 83 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerTracePC.h | 23 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerTraceState.cpp | 181 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerUtil.cpp | 7 | ||||
-rw-r--r-- | lib/Fuzzer/FuzzerUtil.h | 10 | ||||
-rw-r--r-- | lib/Fuzzer/afl/afl_driver.cpp | 4 | ||||
-rw-r--r-- | lib/Fuzzer/test/CMakeLists.txt | 3 | ||||
-rw-r--r-- | lib/Fuzzer/test/FlagsTest.cpp | 32 | ||||
-rw-r--r-- | lib/Fuzzer/test/FuzzerUnittest.cpp | 31 | ||||
-rw-r--r-- | lib/Fuzzer/test/fuzzer-flags.test | 17 | ||||
-rw-r--r-- | lib/Fuzzer/test/fuzzer-traces-hooks.test | 2 | ||||
-rw-r--r-- | lib/Fuzzer/test/reduce_inputs.test | 3 |
21 files changed, 234 insertions, 307 deletions
diff --git a/lib/Fuzzer/CMakeLists.txt b/lib/Fuzzer/CMakeLists.txt index fa743c280e86..bc744890b997 100644 --- a/lib/Fuzzer/CMakeLists.txt +++ b/lib/Fuzzer/CMakeLists.txt @@ -46,7 +46,6 @@ if ( LLVM_USE_SANITIZE_COVERAGE OR CMAKE_SYSTEM_NAME MATCHES "Darwin|Linux" ) FuzzerShmemPosix.cpp FuzzerShmemWindows.cpp FuzzerTracePC.cpp - FuzzerTraceState.cpp FuzzerUtil.cpp FuzzerUtilDarwin.cpp FuzzerUtilLinux.cpp diff --git a/lib/Fuzzer/FuzzerCorpus.h b/lib/Fuzzer/FuzzerCorpus.h index 218ae5b6ac4d..bae0aea78f13 100644 --- a/lib/Fuzzer/FuzzerCorpus.h +++ b/lib/Fuzzer/FuzzerCorpus.h @@ -34,7 +34,8 @@ struct InputInfo { size_t NumExecutedMutations = 0; size_t NumSuccessfullMutations = 0; bool MayDeleteFile = false; - std::vector<uint32_t> FeatureSet; + bool Reduced = false; + std::vector<uint32_t> UniqFeatureSet; }; class InputCorpus { @@ -79,7 +80,8 @@ class InputCorpus { II.U = U; II.NumFeatures = NumFeatures; II.MayDeleteFile = MayDeleteFile; - II.FeatureSet = FeatureSet; + II.UniqFeatureSet = FeatureSet; + std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end()); ComputeSHA1(U.data(), U.size(), II.Sha1); Hashes.insert(Sha1ToString(II.Sha1)); UpdateCorpusDistribution(); @@ -117,34 +119,21 @@ class InputCorpus { Printf("%s sz=%zd ", Sha1ToString(II->Sha1).c_str(), II->U.size()); PrintUnit(II->U); Printf(" "); - PrintFeatureSet(II->FeatureSet); + PrintFeatureSet(II->UniqFeatureSet); Printf("\n"); } i++; } } - // If FeatureSet is that same as in II, replace II->U with {Data,Size}. - bool TryToReplace(InputInfo *II, const uint8_t *Data, size_t Size, - const std::vector<uint32_t> &FeatureSet) { - if (II->U.size() > Size && II->FeatureSet.size() && - II->FeatureSet == FeatureSet) { - if (FeatureDebug) - Printf("Replace: %zd => %zd\n", II->U.size(), Size); - Replace(II, {Data, Data + Size}); - PrintCorpus(); - return true; - } - return false; - } - void Replace(InputInfo *II, const Unit &U) { - assert(II->U.size()); + assert(II->U.size() > U.size()); Hashes.erase(Sha1ToString(II->Sha1)); DeleteFile(*II); ComputeSHA1(U.data(), U.size(), II->Sha1); Hashes.insert(Sha1ToString(II->Sha1)); II->U = U; + II->Reduced = true; } bool HasUnit(const Unit &U) { return Hashes.count(Hash(U)); } @@ -198,7 +187,7 @@ class InputCorpus { Printf("EVICTED %zd\n", Idx); } - void AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) { + bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) { assert(NewSize); Idx = Idx % kFeatureSetSize; uint32_t OldSize = GetFeature(Idx); @@ -218,8 +207,9 @@ class InputCorpus { Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize); SmallestElementPerFeature[Idx] = Inputs.size(); InputSizesPerFeature[Idx] = NewSize; - CountingFeatures = true; + return true; } + return false; } size_t NumFeatures() const { return NumAddedFeatures; } @@ -238,7 +228,6 @@ private: size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } void ValidateFeatureSet() { - if (!CountingFeatures) return; if (FeatureDebug) PrintFeatureSet(); for (size_t Idx = 0; Idx < kFeatureSetSize; Idx++) @@ -256,14 +245,12 @@ private: // Must be called whenever the corpus or unit weights are changed. void UpdateCorpusDistribution() { size_t N = Inputs.size(); + assert(N); Intervals.resize(N + 1); Weights.resize(N); std::iota(Intervals.begin(), Intervals.end(), 0); - if (CountingFeatures) - for (size_t i = 0; i < N; i++) - Weights[i] = Inputs[i]->NumFeatures * (i + 1); - else - std::iota(Weights.begin(), Weights.end(), 1); + for (size_t i = 0; i < N; i++) + Weights[i] = Inputs[i]->NumFeatures * (i + 1); CorpusDistribution = std::piecewise_constant_distribution<double>( Intervals.begin(), Intervals.end(), Weights.begin()); } @@ -275,7 +262,6 @@ private: std::unordered_set<std::string> Hashes; std::vector<InputInfo*> Inputs; - bool CountingFeatures = false; size_t NumAddedFeatures = 0; size_t NumUpdatedFeatures = 0; uint32_t InputSizesPerFeature[kFeatureSetSize]; diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp index 87968893853e..fd8cab38a7bb 100644 --- a/lib/Fuzzer/FuzzerDriver.cpp +++ b/lib/Fuzzer/FuzzerDriver.cpp @@ -186,7 +186,11 @@ static void ParseFlags(const std::vector<std::string> &Args) { } Inputs = new std::vector<std::string>; for (size_t A = 1; A < Args.size(); A++) { - if (ParseOneFlag(Args[A].c_str())) continue; + if (ParseOneFlag(Args[A].c_str())) { + if (Flags.ignore_remaining_args) + break; + continue; + } Inputs->push_back(Args[A]); } } @@ -356,16 +360,17 @@ int MinimizeCrashInput(const std::vector<std::string> &Args, exit(1); } std::string InputFilePath = Inputs->at(0); - std::string BaseCmd = - CloneArgsWithoutX(Args, "minimize_crash", "exact_artifact_path"); - auto InputPos = BaseCmd.find(" " + InputFilePath + " "); + auto BaseCmd = SplitBefore( + "-ignore_remaining_args=1", + CloneArgsWithoutX(Args, "minimize_crash", "exact_artifact_path")); + auto InputPos = BaseCmd.first.find(" " + InputFilePath + " "); assert(InputPos != std::string::npos); - BaseCmd.erase(InputPos, InputFilePath.size() + 1); + BaseCmd.first.erase(InputPos, InputFilePath.size() + 1); if (Flags.runs <= 0 && Flags.max_total_time == 0) { Printf("INFO: you need to specify -runs=N or " "-max_total_time=N with -minimize_crash=1\n" "INFO: defaulting to -max_total_time=600\n"); - BaseCmd += " -max_total_time=600"; + BaseCmd.first += " -max_total_time=600"; } auto LogFilePath = DirPlusFile( @@ -378,7 +383,8 @@ int MinimizeCrashInput(const std::vector<std::string> &Args, Printf("CRASH_MIN: minimizing crash input: '%s' (%zd bytes)\n", CurrentFilePath.c_str(), U.size()); - auto Cmd = BaseCmd + " " + CurrentFilePath + LogFileRedirect; + auto Cmd = BaseCmd.first + " " + CurrentFilePath + LogFileRedirect + " " + + BaseCmd.second; Printf("CRASH_MIN: executing: %s\n", Cmd.c_str()); int ExitCode = ExecuteCommand(Cmd); diff --git a/lib/Fuzzer/FuzzerFlags.def b/lib/Fuzzer/FuzzerFlags.def index 5e70cbad3cf1..526805705b20 100644 --- a/lib/Fuzzer/FuzzerFlags.def +++ b/lib/Fuzzer/FuzzerFlags.def @@ -121,6 +121,9 @@ FUZZER_FLAG_STRING(exit_on_src_pos, "Exit if a newly found PC originates" FUZZER_FLAG_STRING(exit_on_item, "Exit if an item with a given sha1 sum" " was added to the corpus. " "Used primarily for testing libFuzzer itself.") +FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " + "after this one. Useful for fuzzers that need to do their own " + "argument parsing.") FUZZER_FLAG_STRING(run_equivalence_server, "Experimental") FUZZER_FLAG_STRING(use_equivalence_server, "Experimental") diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h index a732f895375e..3fc3fe004cef 100644 --- a/lib/Fuzzer/FuzzerInternal.h +++ b/lib/Fuzzer/FuzzerInternal.h @@ -38,7 +38,6 @@ public: void Loop(); void MinimizeCrashLoop(const Unit &U); void ShuffleAndMinimize(UnitVector *V); - void InitializeTraceState(); void RereadOutputCorpus(size_t MaxSize); size_t secondsSinceProcessStartUp() { @@ -100,19 +99,10 @@ private: void WriteToOutputCorpus(const Unit &U); void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix); void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0); - void PrintStatusForNewUnit(const Unit &U); + void PrintStatusForNewUnit(const Unit &U, const char *Text); void ShuffleCorpus(UnitVector *V); void CheckExitOnSrcPosOrItem(); - // Trace-based fuzzing: we run a unit with some kind of tracing - // enabled and record potentially useful mutations. Then - // We apply these mutations one by one to the unit and run it again. - - // Start tracing; forget all previously proposed mutations. - void StartTraceRecording(); - // Stop tracing. - void StopTraceRecording(); - static void StaticDeathCallback(); void DumpCurrentUnit(const char *Prefix); void DeathCallback(); @@ -142,7 +132,7 @@ private: size_t MaxInputLen = 0; size_t MaxMutationLen = 0; - std::vector<uint32_t> FeatureSetTmp; + std::vector<uint32_t> UniqFeatureSetTmp; // Need to know our own thread. static thread_local bool IsMyThread; diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp index 6816f3af8a6f..8ac7a847aef7 100644 --- a/lib/Fuzzer/FuzzerLoop.cpp +++ b/lib/Fuzzer/FuzzerLoop.cpp @@ -114,7 +114,6 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, : CB(CB), Corpus(Corpus), MD(MD), Options(Options) { if (EF->__sanitizer_set_death_callback) EF->__sanitizer_set_death_callback(StaticDeathCallback); - InitializeTraceState(); assert(!F); F = this; TPC.ResetMaps(); @@ -403,22 +402,29 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, ExecuteCallback(Data, Size); - FeatureSetTmp.clear(); + UniqFeatureSetTmp.clear(); + size_t FoundUniqFeaturesOfII = 0; size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); TPC.CollectFeatures([&](size_t Feature) { - Corpus.AddFeature(Feature, Size, Options.Shrink); - if (Options.ReduceInputs) - FeatureSetTmp.push_back(Feature); + if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + UniqFeatureSetTmp.push_back(Feature); + if (Options.ReduceInputs && II) + if (std::binary_search(II->UniqFeatureSet.begin(), + II->UniqFeatureSet.end(), Feature)) + FoundUniqFeaturesOfII++; }); PrintPulseAndReportSlowInput(Data, Size); size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; if (NumNewFeatures) { Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile, - FeatureSetTmp); + UniqFeatureSetTmp); CheckExitOnSrcPosOrItem(); return true; } - if (II && Corpus.TryToReplace(II, Data, Size, FeatureSetTmp)) { + if (II && FoundUniqFeaturesOfII && + FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && + II->U.size() > Size) { + Corpus.Replace(II, {Data, Data + Size}); CheckExitOnSrcPosOrItem(); return true; } @@ -501,10 +507,10 @@ void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) { Printf("Base64: %s\n", Base64(U).c_str()); } -void Fuzzer::PrintStatusForNewUnit(const Unit &U) { +void Fuzzer::PrintStatusForNewUnit(const Unit &U, const char *Text) { if (!Options.PrintNEW) return; - PrintStats("NEW ", ""); + PrintStats(Text, ""); if (Options.Verbosity) { Printf(" L: %zd ", U.size()); MD.PrintMutationSequence(); @@ -515,7 +521,8 @@ void Fuzzer::PrintStatusForNewUnit(const Unit &U) { void Fuzzer::ReportNewCoverage(InputInfo *II, const Unit &U) { II->NumSuccessfullMutations++; MD.RecordSuccessfulMutationSequence(); - PrintStatusForNewUnit(U); + PrintStatusForNewUnit(U, II->Reduced ? "REDUCE" : + "NEW "); WriteToOutputCorpus(U); NumberOfNewUnitsAdded++; TPC.PrintNewPCs(); @@ -600,13 +607,10 @@ void Fuzzer::MutateAndTestOne() { assert(NewSize > 0 && "Mutator returned empty unit"); assert(NewSize <= CurrentMaxMutationLen && "Mutator return overisized unit"); Size = NewSize; - if (i == 0) - StartTraceRecording(); II.NumExecutedMutations++; if (RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II)) ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); - StopTraceRecording(); TryDetectingAMemoryLeak(CurrentUnitData, Size, /*DuringInitialCorpusExecution*/ false); } diff --git a/lib/Fuzzer/FuzzerMerge.cpp b/lib/Fuzzer/FuzzerMerge.cpp index 612f4bbb28f2..616c0999aa39 100644 --- a/lib/Fuzzer/FuzzerMerge.cpp +++ b/lib/Fuzzer/FuzzerMerge.cpp @@ -241,7 +241,6 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { return true; }); // Show stats. - TotalNumberOfRuns++; if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) PrintStats("pulse "); // Write the post-run marker and the coverage. @@ -286,12 +285,13 @@ void Fuzzer::CrashResistantMerge(const std::vector<std::string> &Args, // Execute the inner process untill it passes. // Every inner process should execute at least one input. - std::string BaseCmd = CloneArgsWithoutX(Args, "keep-all-flags"); + auto BaseCmd = SplitBefore("-ignore_remaining_args=1", + CloneArgsWithoutX(Args, "keep-all-flags")); bool Success = false; for (size_t i = 1; i <= AllFiles.size(); i++) { Printf("MERGE-OUTER: attempt %zd\n", i); - auto ExitCode = - ExecuteCommand(BaseCmd + " -merge_control_file=" + CFPath); + auto ExitCode = ExecuteCommand(BaseCmd.first + " -merge_control_file=" + + CFPath + " " + BaseCmd.second); if (!ExitCode) { Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", i); Success = true; diff --git a/lib/Fuzzer/FuzzerMutate.cpp b/lib/Fuzzer/FuzzerMutate.cpp index 53cb9027e455..5998ef9d3193 100644 --- a/lib/Fuzzer/FuzzerMutate.cpp +++ b/lib/Fuzzer/FuzzerMutate.cpp @@ -43,8 +43,6 @@ MutationDispatcher::MutationDispatcher(Random &Rand, {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, "ManualDict"}, - {&MutationDispatcher::Mutate_AddWordFromTemporaryAutoDictionary, - "TempAutoDict"}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, "PersAutoDict"}, }); @@ -165,11 +163,6 @@ size_t MutationDispatcher::Mutate_AddWordFromManualDictionary(uint8_t *Data, return AddWordFromDictionary(ManualDictionary, Data, Size, MaxSize); } -size_t MutationDispatcher::Mutate_AddWordFromTemporaryAutoDictionary( - uint8_t *Data, size_t Size, size_t MaxSize) { - return AddWordFromDictionary(TempAutoDictionary, Data, Size, MaxSize); -} - size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size, size_t MaxSize, DictionaryEntry &DE) { @@ -251,7 +244,7 @@ size_t MutationDispatcher::Mutate_AddWordFromTORC( uint8_t *Data, size_t Size, size_t MaxSize) { Word W; DictionaryEntry DE; - switch (Rand(3)) { + switch (Rand(4)) { case 0: { auto X = TPC.TORC8.Get(Rand.Rand()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); @@ -267,6 +260,10 @@ size_t MutationDispatcher::Mutate_AddWordFromTORC( auto X = TPC.TORCW.Get(Rand.Rand()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; + case 3: if (Options.UseMemmem) { + auto X = TPC.MMT.Get(Rand.Rand()); + DE = DictionaryEntry(X); + } break; default: assert(0); } @@ -533,14 +530,4 @@ void MutationDispatcher::AddWordToManualDictionary(const Word &W) { {W, std::numeric_limits<size_t>::max()}); } -void MutationDispatcher::AddWordToAutoDictionary(DictionaryEntry DE) { - static const size_t kMaxAutoDictSize = 1 << 14; - if (TempAutoDictionary.size() >= kMaxAutoDictSize) return; - TempAutoDictionary.push_back(DE); -} - -void MutationDispatcher::ClearAutoDictionary() { - TempAutoDictionary.clear(); -} - } // namespace fuzzer diff --git a/lib/Fuzzer/FuzzerMutate.h b/lib/Fuzzer/FuzzerMutate.h index 8c8fb3fd74c7..84b04c0dbf3e 100644 --- a/lib/Fuzzer/FuzzerMutate.h +++ b/lib/Fuzzer/FuzzerMutate.h @@ -52,10 +52,6 @@ public: size_t Mutate_AddWordFromManualDictionary(uint8_t *Data, size_t Size, size_t MaxSize); - /// Mutates data by adding a word from the temporary automatic dictionary. - size_t Mutate_AddWordFromTemporaryAutoDictionary(uint8_t *Data, size_t Size, - size_t MaxSize); - /// Mutates data by adding a word from the TORC. size_t Mutate_AddWordFromTORC(uint8_t *Data, size_t Size, size_t MaxSize); @@ -84,8 +80,6 @@ public: void AddWordToManualDictionary(const Word &W); - void AddWordToAutoDictionary(DictionaryEntry DE); - void ClearAutoDictionary(); void PrintRecommendedDictionary(); void SetCorpus(const InputCorpus *Corpus) { this->Corpus = Corpus; } diff --git a/lib/Fuzzer/FuzzerTracePC.cpp b/lib/Fuzzer/FuzzerTracePC.cpp index 6f5c7be41062..ced0a2133340 100644 --- a/lib/Fuzzer/FuzzerTracePC.cpp +++ b/lib/Fuzzer/FuzzerTracePC.cpp @@ -37,6 +37,8 @@ namespace fuzzer { TracePC TPC; +int ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr; + uint8_t *TracePC::Counters() const { return __sancov_trace_pc_guard_8bit_counters; } @@ -293,6 +295,20 @@ void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { ValueProfileMap.AddValue(Idx); } +static size_t InternalStrnlen(const char *S, size_t MaxLen) { + size_t Len = 0; + for (; Len < MaxLen && S[Len]; Len++) {} + return Len; +} + +// Finds min of (strlen(S1), strlen(S2)). +// Needed bacause one of these strings may actually be non-zero terminated. +static size_t InternalStrnlen2(const char *S1, const char *S2) { + size_t Len = 0; + for (; S1[Len] && S2[Len]; Len++) {} + return Len; +} + } // namespace fuzzer extern "C" { @@ -415,4 +431,71 @@ void __sanitizer_cov_trace_gep(uintptr_t Idx) { uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); fuzzer::TPC.HandleCmp(PC, Idx, (uintptr_t)0); } + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1, + const void *s2, size_t n, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + if (result == 0) return; // No reason to mutate. + if (n <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/false); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1, + const char *s2, size_t n, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + if (result == 0) return; // No reason to mutate. + size_t Len1 = fuzzer::InternalStrnlen(s1, n); + size_t Len2 = fuzzer::InternalStrnlen(s2, n); + n = std::min(n, Len1); + n = std::min(n, Len2); + if (n <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/true); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1, + const char *s2, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + if (result == 0) return; // No reason to mutate. + size_t N = fuzzer::InternalStrnlen2(s1, s2); + if (N <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, N, /*StopAtZero*/true); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + return __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, + const char *s2, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + return __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, + const char *s2, char *result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, + const char *s2, char *result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1, + const void *s2, size_t len2, void *result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), len2); +} } // extern "C" diff --git a/lib/Fuzzer/FuzzerTracePC.h b/lib/Fuzzer/FuzzerTracePC.h index 5ec8c590b4df..b36c4f54306c 100644 --- a/lib/Fuzzer/FuzzerTracePC.h +++ b/lib/Fuzzer/FuzzerTracePC.h @@ -45,6 +45,28 @@ struct TableOfRecentCompares { Pair Table[kSize]; }; +template <size_t kSizeT> +struct MemMemTable { + static const size_t kSize = kSizeT; + Word MemMemWords[kSize]; + Word EmptyWord; + + void Add(const uint8_t *Data, size_t Size) { + if (Size <= 2) return; + Size = std::min(Size, Word::GetMaxSize()); + size_t Idx = SimpleFastHash(Data, Size) % kSize; + MemMemWords[Idx].Set(Data, Size); + } + const Word &Get(size_t Idx) { + for (size_t i = 0; i < kSize; i++) { + const Word &W = MemMemWords[(Idx + i) % kSize]; + if (W.size()) return W; + } + EmptyWord.Set(nullptr, 0); + return EmptyWord; + } +}; + class TracePC { public: static const size_t kNumPCs = 1 << 21; @@ -81,6 +103,7 @@ class TracePC { TableOfRecentCompares<uint32_t, 32> TORC4; TableOfRecentCompares<uint64_t, 32> TORC8; TableOfRecentCompares<Word, 32> TORCW; + MemMemTable<1024> MMT; void PrintNewPCs(); void InitializePrintNewPCs(); diff --git a/lib/Fuzzer/FuzzerTraceState.cpp b/lib/Fuzzer/FuzzerTraceState.cpp deleted file mode 100644 index 8670e2ad6727..000000000000 --- a/lib/Fuzzer/FuzzerTraceState.cpp +++ /dev/null @@ -1,181 +0,0 @@ -//===- FuzzerTraceState.cpp - Trace-based fuzzer mutator ------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Data tracing. -//===----------------------------------------------------------------------===// - -#include "FuzzerDictionary.h" -#include "FuzzerIO.h" -#include "FuzzerInternal.h" -#include "FuzzerMutate.h" -#include "FuzzerTracePC.h" -#include <algorithm> -#include <cstring> -#include <map> -#include <set> -#include <thread> - -namespace fuzzer { - -// Declared as static globals for faster checks inside the hooks. -static bool RecordingMemmem = false; - -int ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr; - -class TraceState { -public: - TraceState(MutationDispatcher &MD, const FuzzingOptions &Options, - const Fuzzer *F) - : MD(MD), Options(Options), F(F) {} - - void StartTraceRecording() { - if (!Options.UseMemmem) - return; - RecordingMemmem = true; - InterestingWords.clear(); - MD.ClearAutoDictionary(); - } - - void StopTraceRecording() { - if (!RecordingMemmem) - return; - for (auto &W : InterestingWords) - MD.AddWordToAutoDictionary({W}); - } - - void AddInterestingWord(const uint8_t *Data, size_t Size) { - if (!RecordingMemmem || !F->InFuzzingThread()) return; - if (Size <= 1) return; - Size = std::min(Size, Word::GetMaxSize()); - Word W(Data, Size); - InterestingWords.insert(W); - } - - private: - - // TODO: std::set is too inefficient, need to have a custom DS here. - std::set<Word> InterestingWords; - MutationDispatcher &MD; - const FuzzingOptions Options; - const Fuzzer *F; -}; - -static TraceState *TS; - -void Fuzzer::StartTraceRecording() { - if (!TS) return; - TS->StartTraceRecording(); -} - -void Fuzzer::StopTraceRecording() { - if (!TS) return; - TS->StopTraceRecording(); -} - -void Fuzzer::InitializeTraceState() { - if (!Options.UseMemmem) return; - TS = new TraceState(MD, Options, this); -} - -static size_t InternalStrnlen(const char *S, size_t MaxLen) { - size_t Len = 0; - for (; Len < MaxLen && S[Len]; Len++) {} - return Len; -} - -// Finds min of (strlen(S1), strlen(S2)). -// Needed bacause one of these strings may actually be non-zero terminated. -static size_t InternalStrnlen2(const char *S1, const char *S2) { - size_t Len = 0; - for (; S1[Len] && S2[Len]; Len++) {} - return Len; -} - -} // namespace fuzzer - -using fuzzer::TS; - -extern "C" { - -// We may need to avoid defining weak hooks to stay compatible with older clang. -#ifndef LLVM_FUZZER_DEFINES_SANITIZER_WEAK_HOOOKS -# define LLVM_FUZZER_DEFINES_SANITIZER_WEAK_HOOOKS 1 -#endif - -#if LLVM_FUZZER_DEFINES_SANITIZER_WEAK_HOOOKS - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1, - const void *s2, size_t n, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - if (result == 0) return; // No reason to mutate. - if (n <= 1) return; // Not interesting. - fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/false); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1, - const char *s2, size_t n, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - if (result == 0) return; // No reason to mutate. - size_t Len1 = fuzzer::InternalStrnlen(s1, n); - size_t Len2 = fuzzer::InternalStrnlen(s2, n); - n = std::min(n, Len1); - n = std::min(n, Len2); - if (n <= 1) return; // Not interesting. - fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/true); -} - - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1, - const char *s2, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - if (result == 0) return; // No reason to mutate. - size_t N = fuzzer::InternalStrnlen2(s1, s2); - if (N <= 1) return; // Not interesting. - fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, N, /*StopAtZero*/true); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, - const char *s2, size_t n, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - return __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, - const char *s2, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - return __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, - const char *s2, char *result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - TS->AddInterestingWord(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, - const char *s2, char *result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - TS->AddInterestingWord(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1, - const void *s2, size_t len2, void *result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - TS->AddInterestingWord(reinterpret_cast<const uint8_t *>(s2), len2); -} - -#endif // LLVM_FUZZER_DEFINES_SANITIZER_WEAK_HOOOKS -} // extern "C" diff --git a/lib/Fuzzer/FuzzerUtil.cpp b/lib/Fuzzer/FuzzerUtil.cpp index f5fd3a85187c..2d95f40e46a1 100644 --- a/lib/Fuzzer/FuzzerUtil.cpp +++ b/lib/Fuzzer/FuzzerUtil.cpp @@ -215,4 +215,11 @@ bool ExecuteCommandAndReadOutput(const std::string &Command, std::string *Out) { return true; } +size_t SimpleFastHash(const uint8_t *Data, size_t Size) { + size_t Res = 0; + for (size_t i = 0; i < Size; i++) + Res = Res * 11 + Data[i]; + return Res; +} + } // namespace fuzzer diff --git a/lib/Fuzzer/FuzzerUtil.h b/lib/Fuzzer/FuzzerUtil.h index f84fd9ef0fce..62d6e61dcf17 100644 --- a/lib/Fuzzer/FuzzerUtil.h +++ b/lib/Fuzzer/FuzzerUtil.h @@ -67,10 +67,20 @@ inline std::string CloneArgsWithoutX(const std::vector<std::string> &Args, return CloneArgsWithoutX(Args, X, X); } +inline std::pair<std::string, std::string> SplitBefore(std::string X, + std::string S) { + auto Pos = S.find(X); + if (Pos == std::string::npos) + return std::make_pair(S, ""); + return std::make_pair(S.substr(0, Pos), S.substr(Pos)); +} + std::string DisassembleCmd(const std::string &FileName); std::string SearchRegexCmd(const std::string &Regex); +size_t SimpleFastHash(const uint8_t *Data, size_t Size); + } // namespace fuzzer #endif // LLVM_FUZZER_UTIL_H diff --git a/lib/Fuzzer/afl/afl_driver.cpp b/lib/Fuzzer/afl/afl_driver.cpp index d0521bdfdd67..15bceb896e17 100644 --- a/lib/Fuzzer/afl/afl_driver.cpp +++ b/lib/Fuzzer/afl/afl_driver.cpp @@ -22,8 +22,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; } EOF -# Build your target with -fsanitize-coverage=trace-pc using fresh clang. -clang -g -fsanitize-coverage=trace-pc test_fuzzer.cc -c +# Build your target with -fsanitize-coverage=trace-pc-guard using fresh clang. +clang -g -fsanitize-coverage=trace-pc-guard test_fuzzer.cc -c # Build afl-llvm-rt.o.c from the AFL distribution. clang -c -w $AFL_HOME/llvm_mode/afl-llvm-rt.o.c # Build this file, link it with afl-llvm-rt.o.o and the target code. diff --git a/lib/Fuzzer/test/CMakeLists.txt b/lib/Fuzzer/test/CMakeLists.txt index 30566bdc87ae..43aea2b7a186 100644 --- a/lib/Fuzzer/test/CMakeLists.txt +++ b/lib/Fuzzer/test/CMakeLists.txt @@ -90,6 +90,7 @@ set(Tests EmptyTest EquivalenceATest EquivalenceBTest + FlagsTest FourIndependentBranchesTest FullCoverageSetTest InitializeTest @@ -272,5 +273,5 @@ add_lit_testsuite(check-fuzzer "Running Fuzzer tests" # Don't add dependencies on Windows. The linker step would fail on Windows, # since cmake will use link.exe for linking and won't include compiler-rt libs. if(NOT MSVC) - add_dependencies(check-fuzzer FileCheck sancov not llvm-symbolizer) + add_dependencies(check-fuzzer FileCheck sancov not) endif() diff --git a/lib/Fuzzer/test/FlagsTest.cpp b/lib/Fuzzer/test/FlagsTest.cpp new file mode 100644 index 000000000000..ac64b9d48df5 --- /dev/null +++ b/lib/Fuzzer/test/FlagsTest.cpp @@ -0,0 +1,32 @@ +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. + +// Parse some flags +#include <string> +#include <vector> + +static std::vector<std::string> Flags; + +extern "C" int LLVMFuzzerInitialize(int *Argc, char ***Argv) { + // Parse --flags and anything after -ignore_remaining_args=1 is passed. + int I = 1; + while (I < *Argc) { + std::string S((*Argv)[I++]); + if (S == "-ignore_remaining_args=1") + break; + if (S.substr(0, 2) == "--") + Flags.push_back(S); + } + while (I < *Argc) + Flags.push_back(std::string((*Argv)[I++])); + + return 0; +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + fprintf(stderr, "BINGO "); + for (auto Flag : Flags) + fprintf(stderr, "%s ", Flag.c_str()); + fprintf(stderr, "\n"); + exit(0); +} diff --git a/lib/Fuzzer/test/FuzzerUnittest.cpp b/lib/Fuzzer/test/FuzzerUnittest.cpp index 1053c28527bf..eba2663029b2 100644 --- a/lib/Fuzzer/test/FuzzerUnittest.cpp +++ b/lib/Fuzzer/test/FuzzerUnittest.cpp @@ -424,35 +424,6 @@ TEST(FuzzerMutate, AddWordFromDictionary2) { TestAddWordFromDictionary(&MutationDispatcher::Mutate, 1 << 15); } -void TestAddWordFromDictionaryWithHint(Mutator M, int NumIter) { - std::unique_ptr<ExternalFunctions> t(new ExternalFunctions()); - fuzzer::EF = t.get(); - Random Rand(0); - MutationDispatcher MD(Rand, {}); - uint8_t W[] = {0xAA, 0xBB, 0xCC, 0xDD, 0xFF, 0xEE, 0xEF}; - size_t PosHint = 7777; - MD.AddWordToAutoDictionary({Word(W, sizeof(W)), PosHint}); - int FoundMask = 0; - for (int i = 0; i < NumIter; i++) { - uint8_t T[10000]; - memset(T, 0, sizeof(T)); - size_t NewSize = (MD.*M)(T, 9000, 10000); - if (NewSize >= PosHint + sizeof(W) && - !memcmp(W, T + PosHint, sizeof(W))) - FoundMask = 1; - } - EXPECT_EQ(FoundMask, 1); -} - -TEST(FuzzerMutate, AddWordFromDictionaryWithHint1) { - TestAddWordFromDictionaryWithHint( - &MutationDispatcher::Mutate_AddWordFromTemporaryAutoDictionary, 1 << 5); -} - -TEST(FuzzerMutate, AddWordFromDictionaryWithHint2) { - TestAddWordFromDictionaryWithHint(&MutationDispatcher::Mutate, 1 << 10); -} - void TestChangeASCIIInteger(Mutator M, int NumIter) { std::unique_ptr<ExternalFunctions> t(new ExternalFunctions()); fuzzer::EF = t.get(); @@ -593,7 +564,7 @@ TEST(Corpus, Distribution) { size_t N = 10; size_t TriesPerUnit = 1<<16; for (size_t i = 0; i < N; i++) - C->AddToCorpus(Unit{ static_cast<uint8_t>(i) }, 0, false, {}); + C->AddToCorpus(Unit{ static_cast<uint8_t>(i) }, 1, false, {}); std::vector<size_t> Hist(N); for (size_t i = 0; i < N * TriesPerUnit; i++) { diff --git a/lib/Fuzzer/test/fuzzer-flags.test b/lib/Fuzzer/test/fuzzer-flags.test index 76ea27705750..976da2906d7c 100644 --- a/lib/Fuzzer/test/fuzzer-flags.test +++ b/lib/Fuzzer/test/fuzzer-flags.test @@ -1,10 +1,21 @@ -RUN: LLVMFuzzer-SimpleTest -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR +# Does not work on windows for unknown reason. +UNSUPPORTED: windows + +RUN: LLVMFuzzer-FlagsTest -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR FOO_BAR: WARNING: unrecognized flag '-foo_bar=1'; use -help=1 to list all flags FOO_BAR: BINGO -RUN: LLVMFuzzer-SimpleTest -runs=10 --max_len=100 2>&1 | FileCheck %s --check-prefix=DASH_DASH +RUN: LLVMFuzzer-FlagsTest -runs=10 --max_len=100 2>&1 | FileCheck %s --check-prefix=DASH_DASH DASH_DASH: WARNING: did you mean '-max_len=100' (single dash)? DASH_DASH: INFO: A corpus is not provided, starting from an empty corpus -RUN: LLVMFuzzer-SimpleTest -help=1 2>&1 | FileCheck %s --check-prefix=NO_INTERNAL +RUN: LLVMFuzzer-FlagsTest -help=1 2>&1 | FileCheck %s --check-prefix=NO_INTERNAL NO_INTERNAL-NOT: internal flag + +RUN: LLVMFuzzer-FlagsTest --foo-bar -runs=10 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU +PASSTHRU: BINGO --foo-bar --baz -help=1 test + +RUN: mkdir -p %t/T0 %t/T1 +RUN: touch %t/T1/empty +RUN: LLVMFuzzer-FlagsTest --foo-bar -merge=1 %t/T0 %t/T1 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU-MERGE +PASSTHRU-MERGE: BINGO --foo-bar --baz -help=1 test diff --git a/lib/Fuzzer/test/fuzzer-traces-hooks.test b/lib/Fuzzer/test/fuzzer-traces-hooks.test index f93a8b7199e2..77ca4b47bd01 100644 --- a/lib/Fuzzer/test/fuzzer-traces-hooks.test +++ b/lib/Fuzzer/test/fuzzer-traces-hooks.test @@ -10,7 +10,7 @@ RUN: not LLVMFuzzer-StrstrTest -seed=1 -runs=2000000 2>&1 | File RUN: not LLVMFuzzer-Memcmp64BytesTest -seed=1 -runs=1000000 2>&1 | FileCheck %s -RUN: LLVMFuzzer-RepeatedMemcmp -seed=11 -runs=100000 2>&1 | FileCheck %s --check-prefix=RECOMMENDED_DICT +RUN: LLVMFuzzer-RepeatedMemcmp -seed=11 -runs=100000 -max_len=20 2>&1 | FileCheck %s --check-prefix=RECOMMENDED_DICT RECOMMENDED_DICT:###### Recommended dictionary. ###### RECOMMENDED_DICT-DAG: "foo" RECOMMENDED_DICT-DAG: "bar" diff --git a/lib/Fuzzer/test/reduce_inputs.test b/lib/Fuzzer/test/reduce_inputs.test index a4a5c57123d3..5ce4440788f4 100644 --- a/lib/Fuzzer/test/reduce_inputs.test +++ b/lib/Fuzzer/test/reduce_inputs.test @@ -9,5 +9,6 @@ CHECK: INFO: found item with checksum '0eb8e4ed029b774d80f2b66408203801cb982a60' RUN: LLVMFuzzer-ShrinkControlFlowSimpleTest -runs=0 %t/C 2>&1 | FileCheck %s --check-prefix=COUNT COUNT: READ units: 3 - +# a bit longer test +RUN: LLVMFuzzer-ShrinkControlFlowTest -exit_on_item=0eb8e4ed029b774d80f2b66408203801cb982a60 -seed=1 -reduce_inputs=1 -runs=1000000 2>&1 | FileCheck %s |