diff options
Diffstat (limited to 'scripts/afl.py')
| -rwxr-xr-x | scripts/afl.py | 245 |
1 files changed, 245 insertions, 0 deletions
diff --git a/scripts/afl.py b/scripts/afl.py new file mode 100755 index 000000000000..c4312ce84f83 --- /dev/null +++ b/scripts/afl.py @@ -0,0 +1,245 @@ +#! /usr/bin/python3 -B +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2018-2021 Gavin D. Howard and contributors. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import os +import sys +import shutil +import subprocess + + +# Print the usage and exit with an error. +def usage(): + print("usage: {} [--asan] dir [results_dir [exe options...]]".format(script)) + print(" The valid values for dir are: 'bc1', 'bc2', 'bc3', and 'dc'.") + sys.exit(1) + + +# Check for a crash. +# @param exebase The calculator that crashed. +# @param out The file to copy the crash file to. +# @param error The error code (negative). +# @param file The crash file. +# @param type The type of run that caused the crash. This is just a string +# that would make sense to the user. +# @param test The contents of the crash file, or which line caused the crash +# for a run through stdin. +def check_crash(exebase, out, error, file, type, test): + if error < 0: + print("\n{} crashed ({}) on {}:\n".format(exebase, -error, type)) + print(" {}".format(test)) + print("\nCopying to \"{}\"".format(out)) + shutil.copy2(file, out) + print("\nexiting...") + sys.exit(error) + + +# Runs a test. This function is used to ensure that if a test times out, it is +# discarded. Otherwise, some tests result in incredibly long runtimes. We need +# to ignore those. +# +# @param cmd The command to run. +# @param exebase The calculator to test. +# @param tout The timeout to use. +# @param indata The data to push through stdin for the test. +# @param out The file to copy the test file to if it causes a crash. +# @param file The test file. +# @param type The type of test. This is just a string that would make sense +# to the user. +# @param test The test. It could be an entire file, or just one line. +# @param environ The environment to run the command under. +def run_test(cmd, exebase, tout, indata, out, file, type, test, environ=None): + try: + p = subprocess.run(cmd, timeout=tout, input=indata, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=environ) + check_crash(exebase, out, p.returncode, file, type, test) + except subprocess.TimeoutExpired: + print("\n {} timed out. Continuing...\n".format(exebase)) + + +# Creates and runs a test. This basically just takes a file, runs it through the +# appropriate calculator as a whole file, then runs it through the calculator +# using stdin. +# @param file The file to test. +# @param tout The timeout to use. +# @param environ The environment to run under. +def create_test(file, tout, environ=None): + + print(" {}".format(file)) + + base = os.path.basename(file) + + if base == "README.txt": + return + + with open(file, "rb") as f: + lines = f.readlines() + + print(" Running whole file...") + + run_test(exe + [ file ], exebase, tout, halt.encode(), out, file, "file", file, environ) + + print(" Running file through stdin...") + + with open(file, "rb") as f: + content = f.read() + + run_test(exe, exebase, tout, content, out, file, + "running {} through stdin".format(file), file, environ) + + +# Get the children of a directory. +# @param dir The directory to get the children of. +# @param get_files True if files should be gotten, false if directories should +# be gotten. +def get_children(dir, get_files): + dirs = [] + with os.scandir(dir) as it: + for entry in it: + if not entry.name.startswith('.') and \ + ((entry.is_dir() and not get_files) or \ + (entry.is_file() and get_files)): + dirs.append(entry.name) + dirs.sort() + return dirs + + +# Returns the correct executable name for the directory under test. +# @param d The directory under test. +def exe_name(d): + return "bc" if d == "bc1" or d == "bc2" or d == "bc3" else "dc" + + +# Housekeeping. +script = sys.argv[0] +scriptdir = os.path.dirname(script) + +# Must run this script alone. +if __name__ != "__main__": + usage() + +timeout = 2.5 + +if len(sys.argv) < 2: + usage() + +idx = 1 + +exedir = sys.argv[idx] + +asan = (exedir == "--asan") + +# We could possibly run under ASan. See later for what that means. +if asan: + idx += 1 + if len(sys.argv) < idx + 1: + usage() + exedir = sys.argv[idx] + +print("exedir: {}".format(exedir)) + +# Grab the correct directory of AFL++ results. +if len(sys.argv) >= idx + 2: + resultsdir = sys.argv[idx + 1] +else: + if exedir == "bc1": + resultsdir = scriptdir + "/../tests/fuzzing/bc_outputs1" + elif exedir == "bc2": + resultsdir = scriptdir + "/../tests/fuzzing/bc_outputs2" + elif exedir == "bc3": + resultsdir = scriptdir + "/../tests/fuzzing/bc_outputs3" + elif exedir == "dc": + resultsdir = scriptdir + "/../tests/fuzzing/dc_outputs" + else: + raise ValueError("exedir must be either bc1, bc2, bc3, or dc"); + +print("resultsdir: {}".format(resultsdir)) + +# More command-line processing. +if len(sys.argv) >= idx + 3: + exe = sys.argv[idx + 2] +else: + exe = scriptdir + "/../bin/" + exe_name(exedir) + +exebase = os.path.basename(exe) + + +# Use the correct options. +if exebase == "bc": + halt = "halt\n" + options = "-lq" + seed = ["-e", "seed = 1280937142.20981723890730892738902938071028973408912703984712093", "-f-" ] +else: + halt = "q\n" + options = "-x" + seed = ["-e", "1280937142.20981723890730892738902938071028973408912703984712093j", "-f-" ] + +# More command-line processing. +if len(sys.argv) >= idx + 4: + exe = [ exe, sys.argv[idx + 3:], options ] + seed +else: + exe = [ exe, options ] + seed +for i in range(4, len(sys.argv)): + exe.append(sys.argv[i]) + +out = scriptdir + "/../.test.txt" + +print(os.path.realpath(os.getcwd())) + +dirs = get_children(resultsdir, False) + +# Set the correct ASAN_OPTIONS. +if asan: + env = os.environ.copy() + env['ASAN_OPTIONS'] = 'abort_on_error=1:allocator_may_return_null=1' + +for d in dirs: + + d = resultsdir + "/" + d + + print(d) + + # Check the crash files. + files = get_children(d + "/crashes/", True) + + for file in files: + file = d + "/crashes/" + file + create_test(file, timeout) + + # If we are running under ASan, we want to check all files. Otherwise, skip. + if not asan: + continue + + # Check all of the test cases found by AFL++. + files = get_children(d + "/queue/", True) + + for file in files: + file = d + "/queue/" + file + create_test(file, timeout * 2, env) + +print("Done") |