diff options
Diffstat (limited to 'lib/sanitizer_common/scripts')
| -rwxr-xr-x | lib/sanitizer_common/scripts/check_lint.sh | 63 | ||||
| -rwxr-xr-x | lib/sanitizer_common/scripts/cpplint.py | 4 | ||||
| -rwxr-xr-x | lib/sanitizer_common/scripts/gen_dynamic_list.py | 14 | ||||
| -rwxr-xr-x | lib/sanitizer_common/scripts/litlint.py | 72 | ||||
| -rwxr-xr-x | lib/sanitizer_common/scripts/litlint_test.py | 23 | ||||
| -rwxr-xr-x | lib/sanitizer_common/scripts/sancov.py | 105 |
6 files changed, 245 insertions, 36 deletions
diff --git a/lib/sanitizer_common/scripts/check_lint.sh b/lib/sanitizer_common/scripts/check_lint.sh index 5f1bd4ba4316e..267273d977940 100755 --- a/lib/sanitizer_common/scripts/check_lint.sh +++ b/lib/sanitizer_common/scripts/check_lint.sh @@ -1,16 +1,18 @@ -#!/bin/bash +#!/bin/sh SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # Guess path to LLVM_CHECKOUT if not provided -if [ "${LLVM_CHECKOUT}" == "" ]; then +if [ "${LLVM_CHECKOUT}" = "" ]; then LLVM_CHECKOUT="${SCRIPT_DIR}/../../../../../" fi -# Cpplint setup +# python tools setup CPPLINT=${SCRIPT_DIR}/cpplint.py +LITLINT=${SCRIPT_DIR}/litlint.py if [ "${PYTHON_EXECUTABLE}" != "" ]; then CPPLINT="${PYTHON_EXECUTABLE} ${CPPLINT}" + LITLINT="${PYTHON_EXECUTABLE} ${LITLINT}" fi # Filters @@ -19,7 +21,7 @@ LLVM_LINT_FILTER=-,+whitespace COMMON_LINT_FILTER=-build/include,-build/header_guard,-legal/copyright,-whitespace/comments,-readability/casting,\ -build/namespaces ASAN_RTL_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/int -ASAN_TEST_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/sizeof,-runtime/int,-runtime/printf +ASAN_TEST_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/sizeof,-runtime/int,-runtime/printf,-runtime/threadsafe_fn ASAN_LIT_TEST_LINT_FILTER=${ASAN_TEST_LINT_FILTER},-whitespace/line_length TSAN_RTL_LINT_FILTER=${COMMON_LINT_FILTER} TSAN_TEST_LINT_FILTER=${TSAN_RTL_LINT_FILTER},-runtime/threadsafe_fn,-runtime/int @@ -27,7 +29,8 @@ TSAN_LIT_TEST_LINT_FILTER=${TSAN_TEST_LINT_FILTER},-whitespace/line_length MSAN_RTL_LINT_FILTER=${COMMON_LINT_FILTER} LSAN_RTL_LINT_FILTER=${COMMON_LINT_FILTER} LSAN_LIT_TEST_LINT_FILTER=${LSAN_RTL_LINT_FILTER},-whitespace/line_length -COMMON_RTL_INC_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/int,-runtime/sizeof,-runtime/printf +DFSAN_RTL_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/int,-runtime/printf,-runtime/references,-readability/function +COMMON_RTL_INC_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/int,-runtime/sizeof,-runtime/printf,-readability/fn_size SANITIZER_INCLUDES_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/int MKTEMP="mktemp -q /tmp/tmp.XXXXXXXXXX" cd ${LLVM_CHECKOUT} @@ -44,51 +47,67 @@ run_lint() { cat $TASK_LOG | grep -v "Done processing" | grep -v "Total errors found" \ | grep -v "Skipping input" >> $ERROR_LOG fi - if [[ "${SILENT}" != "1" ]]; then + if [ "${SILENT}" != "1" ]; then cat $TASK_LOG fi + ${LITLINT} "$@" 2>>$ERROR_LOG } run_lint ${LLVM_LINT_FILTER} --filter=${LLVM_LINT_FILTER} \ - lib/Transforms/Instrumentation/*Sanitizer.cpp \ - lib/Transforms/Utils/SpecialCaseList.cpp & + lib/Transforms/Instrumentation/*Sanitizer.cpp & -COMPILER_RT=projects/compiler-rt +if [ "${COMPILER_RT}" = "" ]; then + COMPILER_RT=projects/compiler-rt +fi +LIT_TESTS=${COMPILER_RT}/test # Headers SANITIZER_INCLUDES=${COMPILER_RT}/include/sanitizer run_lint ${SANITIZER_INCLUDES_LINT_FILTER} ${SANITIZER_INCLUDES}/*.h & # Sanitizer_common COMMON_RTL=${COMPILER_RT}/lib/sanitizer_common -run_lint ${COMMON_RTL_INC_LINT_FILTER} ${COMMON_RTL}/*.{cc,h} \ +run_lint ${COMMON_RTL_INC_LINT_FILTER} ${COMMON_RTL}/*.cc \ + ${COMMON_RTL}/*.h \ ${COMMON_RTL}/tests/*.cc & # Interception INTERCEPTION=${COMPILER_RT}/lib/interception -run_lint ${ASAN_RTL_LINT_FILTER} ${INTERCEPTION}/*.{cc,h} & +run_lint ${ASAN_RTL_LINT_FILTER} ${INTERCEPTION}/*.cc \ + ${INTERCEPTION}/*.h & # ASan ASAN_RTL=${COMPILER_RT}/lib/asan -run_lint ${ASAN_RTL_LINT_FILTER} ${ASAN_RTL}/*.{cc,h} & -run_lint ${ASAN_TEST_LINT_FILTER} ${ASAN_RTL}/tests/*.{cc,h} & -run_lint ${ASAN_LIT_TEST_LINT_FILTER} ${ASAN_RTL}/lit_tests/*/*.cc & +run_lint ${ASAN_RTL_LINT_FILTER} ${ASAN_RTL}/*.cc \ + ${ASAN_RTL}/*.h & +run_lint ${ASAN_TEST_LINT_FILTER} ${ASAN_RTL}/tests/*.cc \ + ${ASAN_RTL}/tests/*.h & +run_lint ${ASAN_LIT_TEST_LINT_FILTER} ${LIT_TESTS}/asan/*/*.cc & # TSan TSAN_RTL=${COMPILER_RT}/lib/tsan -run_lint ${TSAN_RTL_LINT_FILTER} ${TSAN_RTL}/rtl/*.{cc,h} & -run_lint ${TSAN_TEST_LINT_FILTER} ${TSAN_RTL}/tests/rtl/*.{cc,h} \ +run_lint ${TSAN_RTL_LINT_FILTER} ${TSAN_RTL}/rtl/*.cc \ + ${TSAN_RTL}/rtl/*.h & +run_lint ${TSAN_TEST_LINT_FILTER} ${TSAN_RTL}/tests/rtl/*.cc \ + ${TSAN_RTL}/tests/rtl/*.h \ ${TSAN_RTL}/tests/unit/*.cc & -run_lint ${TSAN_LIT_TEST_LINT_FILTER} ${TSAN_RTL}/lit_tests/*.cc & +run_lint ${TSAN_LIT_TEST_LINT_FILTER} ${LIT_TESTS}/tsan/*.cc & # MSan MSAN_RTL=${COMPILER_RT}/lib/msan -run_lint ${MSAN_RTL_LINT_FILTER} ${MSAN_RTL}/*.{cc,h} & +run_lint ${MSAN_RTL_LINT_FILTER} ${MSAN_RTL}/*.cc \ + ${MSAN_RTL}/*.h & # LSan LSAN_RTL=${COMPILER_RT}/lib/lsan -run_lint ${LSAN_RTL_LINT_FILTER} ${LSAN_RTL}/*.{cc,h} \ - ${LSAN_RTL}/tests/*.{cc,h} & -run_lint ${LSAN_LIT_TEST_LINT_FILTER} ${LSAN_RTL}/lit_tests/*/*.cc & +run_lint ${LSAN_RTL_LINT_FILTER} ${LSAN_RTL}/*.cc \ + ${LSAN_RTL}/*.h & +run_lint ${LSAN_LIT_TEST_LINT_FILTER} ${LIT_TESTS}/lsan/*/*.cc & + +# DFSan +DFSAN_RTL=${COMPILER_RT}/lib/dfsan +run_lint ${DFSAN_RTL_LINT_FILTER} ${DFSAN_RTL}/*.cc \ + ${DFSAN_RTL}/*.h & +${DFSAN_RTL}/scripts/check_custom_wrappers.sh >> $ERROR_LOG # Misc files FILES=${COMMON_RTL}/*.inc @@ -106,7 +125,7 @@ for temp in $TMPFILES; do rm -f $temp done -if [[ -s $ERROR_LOG ]]; then +if [ -s $ERROR_LOG ]; then cat $ERROR_LOG exit 1 fi diff --git a/lib/sanitizer_common/scripts/cpplint.py b/lib/sanitizer_common/scripts/cpplint.py index a8c9f6784f2d6..742459af172f2 100755 --- a/lib/sanitizer_common/scripts/cpplint.py +++ b/lib/sanitizer_common/scripts/cpplint.py @@ -3634,7 +3634,7 @@ def UpdateIncludeState(filename, include_state, io=codecs): io: The io factory to use to read the file. Provided for testability. Returns: - True if a header was succesfully added. False otherwise. + True if a header was successfully added. False otherwise. """ headerfile = None try: @@ -3706,7 +3706,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, # Let's copy the include_state so it is only messed up within this function. include_state = include_state.copy() - # Did we find the header for this file (if any) and succesfully load it? + # Did we find the header for this file (if any) and successfully load it? header_found = False # Use the absolute path so that matching works properly. diff --git a/lib/sanitizer_common/scripts/gen_dynamic_list.py b/lib/sanitizer_common/scripts/gen_dynamic_list.py index 32ba9226911e8..7bab230650cbe 100755 --- a/lib/sanitizer_common/scripts/gen_dynamic_list.py +++ b/lib/sanitizer_common/scripts/gen_dynamic_list.py @@ -35,12 +35,16 @@ def get_global_functions(library): functions = [] nm_proc = subprocess.Popen(['nm', library], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - nm_out = nm_proc.communicate()[0].split('\n') + nm_out = nm_proc.communicate()[0].decode().split('\n') if nm_proc.returncode != 0: raise subprocess.CalledProcessError(nm_proc.returncode, 'nm') + func_symbols = ['T', 'W'] + # On PowerPC, nm prints function descriptors from .data section. + if os.uname()[4] in ["powerpc", "ppc64"]: + func_symbols += ['D'] for line in nm_out: cols = line.split(' ') - if (len(cols) == 3 and cols[1] in ('T', 'W')) : + if len(cols) == 3 and cols[1] in func_symbols : functions.append(cols[2]) return functions @@ -75,11 +79,11 @@ def main(argv): for line in f: result.append(line.rstrip()) # Print the resulting list in the format recognized by ld. - print '{' + print('{') result.sort() for f in result: - print ' ' + f + ';' - print '};' + print(' ' + f + ';') + print('};') if __name__ == '__main__': main(sys.argv) diff --git a/lib/sanitizer_common/scripts/litlint.py b/lib/sanitizer_common/scripts/litlint.py new file mode 100755 index 0000000000000..1e78448b63d4d --- /dev/null +++ b/lib/sanitizer_common/scripts/litlint.py @@ -0,0 +1,72 @@ +#!/usr/bin/python +# +# litlint +# +# Ensure RUN commands in lit tests are free of common errors. +# +# If any errors are detected, litlint returns a nonzero exit code. +# + +import optparse +import re +import sys + +# Compile regex once for all files +runRegex = re.compile(r'(?<!-o)(?<!%run) %t\s') + +def LintLine(s): + """ Validate a line + + Args: + s: str, the line to validate + + Returns: + Returns an error message and a 1-based column number if an error was + detected, otherwise (None, None). + """ + + # Check that RUN command can be executed with an emulator + m = runRegex.search(s) + if m: + start, end = m.span() + return ('missing %run before %t', start + 2) + + # No errors + return (None, None) + + +def LintFile(p): + """ Check that each RUN command can be executed with an emulator + + Args: + p: str, valid path to a file + + Returns: + The number of errors detected. + """ + errs = 0 + with open(p, 'r') as f: + for i, s in enumerate(f.readlines(), start=1): + msg, col = LintLine(s) + if msg != None: + errs += 1 + errorMsg = 'litlint: {}:{}:{}: error: {}.\n{}{}\n' + arrow = (col-1) * ' ' + '^' + sys.stderr.write(errorMsg.format(p, i, col, msg, s, arrow)) + return errs + + +if __name__ == "__main__": + # Parse args + parser = optparse.OptionParser() + parser.add_option('--filter') # ignored + (options, filenames) = parser.parse_args() + + # Lint each file + errs = 0 + for p in filenames: + errs += LintFile(p) + + # If errors, return nonzero + if errs > 0: + sys.exit(1) diff --git a/lib/sanitizer_common/scripts/litlint_test.py b/lib/sanitizer_common/scripts/litlint_test.py new file mode 100755 index 0000000000000..3ce482d704442 --- /dev/null +++ b/lib/sanitizer_common/scripts/litlint_test.py @@ -0,0 +1,23 @@ +#!/usr/bin/python + +# Tests for litlint.py +# +# Usage: python litlint_test.py +# +# Returns nonzero if any test fails + +import litlint +import unittest + +class TestLintLine(unittest.TestCase): + def test_missing_run(self): + f = litlint.LintLine + self.assertEqual(f(' %t '), ('missing %run before %t', 2)) + self.assertEqual(f(' %t\n'), ('missing %run before %t', 2)) + self.assertEqual(f(' %t.so '), (None, None)) + self.assertEqual(f(' %t.o '), (None, None)) + self.assertEqual(f('%run %t '), (None, None)) + self.assertEqual(f('-o %t '), (None, None)) + +if __name__ == '__main__': + unittest.main() diff --git a/lib/sanitizer_common/scripts/sancov.py b/lib/sanitizer_common/scripts/sancov.py index aa791bc4eb018..476953015280c 100755 --- a/lib/sanitizer_common/scripts/sancov.py +++ b/lib/sanitizer_common/scripts/sancov.py @@ -4,23 +4,27 @@ # We need to merge these integers into a set and then # either print them (as hex) or dump them into another file. import array +import struct import sys +import bisect +import os.path prog_name = ""; def Usage(): print >> sys.stderr, "Usage: \n" + \ " " + prog_name + " merge file1 [file2 ...] > output\n" \ - " " + prog_name + " print file1 [file2 ...]\n" + " " + prog_name + " print file1 [file2 ...]\n" \ + " " + prog_name + " unpack file1 [file2 ...]\n" \ + " " + prog_name + " rawunpack file1 [file2 ...]\n" exit(1) def ReadOneFile(path): - f = open(path, mode="rb") - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - s = set(array.array('I', f.read(size))) - f.close() + with open(path, mode="rb") as f: + f.seek(0, 2) + size = f.tell() + f.seek(0, 0) + s = set(array.array('I', f.read(size))) print >>sys.stderr, "%s: read %d PCs from %s" % (prog_name, size / 4, path) return s @@ -44,6 +48,89 @@ def MergeAndPrint(files): a = array.array('I', s) a.tofile(sys.stdout) + +def UnpackOneFile(path): + with open(path, mode="rb") as f: + print >> sys.stderr, "%s: unpacking %s" % (prog_name, path) + while True: + header = f.read(12) + if not header: return + if len(header) < 12: + break + pid, module_length, blob_size = struct.unpack('iII', header) + module = f.read(module_length) + blob = f.read(blob_size) + assert(len(module) == module_length) + assert(len(blob) == blob_size) + extracted_file = "%s.%d.sancov" % (module, pid) + print >> sys.stderr, "%s: extracting %s" % \ + (prog_name, extracted_file) + # The packed file may contain multiple blobs for the same pid/module + # pair. Append to the end of the file instead of overwriting. + with open(extracted_file, 'ab') as f2: + f2.write(blob) + # fail + raise Exception('Error reading file %s' % path) + + +def Unpack(files): + for f in files: + UnpackOneFile(f) + +def UnpackOneRawFile(path, map_path): + mem_map = [] + with open(map_path, mode="rt") as f_map: + print >> sys.stderr, "%s: reading map %s" % (prog_name, map_path) + bits = int(f_map.readline()) + for line in f_map: + parts = line.rstrip().split() + mem_map.append((int(parts[0], 16), + int(parts[1], 16), + int(parts[2], 16), + ' '.join(parts[3:]))) + mem_map.sort(key=lambda m : m[0]) + mem_map_keys = [m[0] for m in mem_map] + + with open(path, mode="rb") as f: + print >> sys.stderr, "%s: unpacking %s" % (prog_name, path) + + f.seek(0, 2) + size = f.tell() + f.seek(0, 0) + if bits == 64: + typecode = 'L' + else: + typecode = 'I' + pcs = array.array(typecode, f.read(size)) + mem_map_pcs = [[] for i in range(0, len(mem_map))] + + for pc in pcs: + if pc == 0: continue + map_idx = bisect.bisect(mem_map_keys, pc) - 1 + (start, end, base, module_path) = mem_map[map_idx] + assert pc >= start + if pc >= end: + print >> sys.stderr, "warning: %s: pc %x outside of any known mapping" % (prog_name, pc) + continue + mem_map_pcs[map_idx].append(pc - base) + + for ((start, end, base, module_path), pc_list) in zip(mem_map, mem_map_pcs): + if len(pc_list) == 0: continue + assert path.endswith('.sancov.raw') + dst_path = module_path + '.' + os.path.basename(path)[:-4] + print "writing %d PCs to %s" % (len(pc_list), dst_path) + arr = array.array('I') + arr.fromlist(sorted(pc_list)) + with open(dst_path, 'ab') as f2: + arr.tofile(f2) + +def RawUnpack(files): + for f in files: + if not f.endswith('.sancov.raw'): + raise Exception('Unexpected raw file name %s' % f) + f_map = f[:-3] + 'map' + UnpackOneRawFile(f, f_map) + if __name__ == '__main__': prog_name = sys.argv[0] if len(sys.argv) <= 2: @@ -52,5 +139,9 @@ if __name__ == '__main__': PrintFiles(sys.argv[2:]) elif sys.argv[1] == "merge": MergeAndPrint(sys.argv[2:]) + elif sys.argv[1] == "unpack": + Unpack(sys.argv[2:]) + elif sys.argv[1] == "rawunpack": + RawUnpack(sys.argv[2:]) else: Usage() |