diff options
Diffstat (limited to 'utils/extract_symbols.py')
| -rwxr-xr-x | utils/extract_symbols.py | 496 |
1 files changed, 496 insertions, 0 deletions
diff --git a/utils/extract_symbols.py b/utils/extract_symbols.py new file mode 100755 index 0000000000000..9f467a7d0556b --- /dev/null +++ b/utils/extract_symbols.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python + +"""A tool for extracting a list of symbols to export + +When exporting symbols from a dll or exe we either need to mark the symbols in +the source code as __declspec(dllexport) or supply a list of symbols to the +linker. This program automates the latter by inspecting the symbol tables of a +list of link inputs and deciding which of those symbols need to be exported. + +We can't just export all the defined symbols, as there's a limit of 65535 +exported symbols and in clang we go way over that, particularly in a debug +build. Therefore a large part of the work is pruning symbols either which can't +be imported, or which we think are things that have definitions in public header +files (i.e. template instantiations) and we would get defined in the thing +importing these symbols anyway. +""" + +from __future__ import print_function +import sys +import re +import os +import subprocess +import multiprocessing +import argparse + +# Define functions which extract a list of symbols from a library using several +# different tools. We use subprocess.Popen and yield a symbol at a time instead +# of using subprocess.check_output and returning a list as, especially on +# Windows, waiting for the entire output to be ready can take a significant +# amount of time. + +def dumpbin_get_symbols(lib): + process = subprocess.Popen(['dumpbin','/symbols',lib], bufsize=1, + stdout=subprocess.PIPE, stdin=subprocess.PIPE, + universal_newlines=True) + process.stdin.close() + for line in process.stdout: + # Look for external symbols that are defined in some section + match = re.match("^.+SECT.+External\s+\|\s+(\S+).*$", line) + if match: + yield match.group(1) + process.wait() + +def nm_get_symbols(lib): + process = subprocess.Popen(['nm',lib], bufsize=1, + stdout=subprocess.PIPE, stdin=subprocess.PIPE, + universal_newlines=True) + process.stdin.close() + for line in process.stdout: + # Look for external symbols that are defined in some section + match = re.match("^\S+\s+[BDGRSTVW]\s+(\S+)$", line) + if match: + yield match.group(1) + process.wait() + +def readobj_get_symbols(lib): + process = subprocess.Popen(['llvm-readobj','-symbols',lib], bufsize=1, + stdout=subprocess.PIPE, stdin=subprocess.PIPE, + universal_newlines=True) + process.stdin.close() + for line in process.stdout: + # When looking through the output of llvm-readobj we expect to see Name, + # Section, then StorageClass, so record Name and Section when we see + # them and decide if this is a defined external symbol when we see + # StorageClass. + match = re.search('Name: (\S+)', line) + if match: + name = match.group(1) + match = re.search('Section: (\S+)', line) + if match: + section = match.group(1) + match = re.search('StorageClass: (\S+)', line) + if match: + storageclass = match.group(1) + if section != 'IMAGE_SYM_ABSOLUTE' and \ + section != 'IMAGE_SYM_UNDEFINED' and \ + storageclass == 'External': + yield name + process.wait() + +# Define functions which determine if the target is 32-bit Windows (as that's +# where calling convention name decoration happens). + +def dumpbin_is_32bit_windows(lib): + # dumpbin /headers can output a huge amount of data (>100MB in a debug + # build) so we read only up to the 'machine' line then close the output. + process = subprocess.Popen(['dumpbin','/headers',lib], bufsize=1, + stdout=subprocess.PIPE, stdin=subprocess.PIPE, + universal_newlines=True) + process.stdin.close() + retval = False + for line in process.stdout: + match = re.match('.+machine \((\S+)\)', line) + if match: + retval = (match.group(1) == 'x86') + break + process.stdout.close() + process.wait() + return retval + +def objdump_is_32bit_windows(lib): + output = subprocess.check_output(['objdump','-f',lib], + universal_newlines=True) + for line in output: + match = re.match('.+file format (\S+)', line) + if match: + return (match.group(1) == 'pe-i386') + return False + +def readobj_is_32bit_windows(lib): + output = subprocess.check_output(['llvm-readobj','-file-headers',lib], + universal_newlines=True) + for line in output: + match = re.match('Format: (\S+)', line) + if match: + return (match.group(1) == 'COFF-i386') + return False + +# MSVC mangles names to ?<identifier_mangling>@<type_mangling>. By examining the +# identifier/type mangling we can decide which symbols could possibly be +# required and which we can discard. +def should_keep_microsoft_symbol(symbol, calling_convention_decoration): + # Keep unmangled (i.e. extern "C") names + if not '?' in symbol: + if calling_convention_decoration: + # Remove calling convention decoration from names + match = re.match('[_@]([^@]+)', symbol) + if match: + return match.group(1) + return symbol + # Function template instantiations start with ?$, discard them as it's + # assumed that the definition is public + elif symbol.startswith('??$'): + return None + # Deleting destructors start with ?_G or ?_E and can be discarded because + # link.exe gives you a warning telling you they can't be exported if you + # don't + elif symbol.startswith('??_G') or symbol.startswith('??_E'): + return None + # Constructors (?0) and destructors (?1) of templates (?$) are assumed to be + # defined in headers and not required to be kept + elif symbol.startswith('??0?$') or symbol.startswith('??1?$'): + return None + # An anonymous namespace is mangled as ?A(maybe hex number)@. Any symbol + # that mentions an anonymous namespace can be discarded, as the anonymous + # namespace doesn't exist outside of that translation unit. + elif re.search('\?A(0x\w+)?@', symbol): + return None + # Keep mangled llvm:: and clang:: function symbols. How we detect these is a + # bit of a mess and imprecise, but that avoids having to completely demangle + # the symbol name. The outermost namespace is at the end of the identifier + # mangling, and the identifier mangling is followed by the type mangling, so + # we look for (llvm|clang)@@ followed by something that looks like a + # function type mangling. To spot a function type we use (this is derived + # from clang/lib/AST/MicrosoftMangle.cpp): + # <function-type> ::= <function-class> <this-cvr-qualifiers> + # <calling-convention> <return-type> + # <argument-list> <throw-spec> + # <function-class> ::= [A-Z] + # <this-cvr-qualifiers> ::= [A-Z0-9_]* + # <calling-convention> ::= [A-JQ] + # <return-type> ::= .+ + # <argument-list> ::= X (void) + # ::= .+@ (list of types) + # ::= .*Z (list of types, varargs) + # <throw-spec> ::= exceptions are not allowed + elif re.search('(llvm|clang)@@[A-Z][A-Z0-9_]*[A-JQ].+(X|.+@|.*Z)$', symbol): + return symbol + return None + +# Itanium manglings are of the form _Z<identifier_mangling><type_mangling>. We +# demangle the identifier mangling to identify symbols that can be safely +# discarded. +def should_keep_itanium_symbol(symbol, calling_convention_decoration): + # Start by removing any calling convention decoration (which we expect to + # see on all symbols, even mangled C++ symbols) + if calling_convention_decoration and symbol.startswith('_'): + symbol = symbol[1:] + # Keep unmangled names + if not symbol.startswith('_') and not symbol.startswith('.'): + return symbol + # Discard manglings that aren't nested names + match = re.match('_Z(T[VTIS])?(N.+)', symbol) + if not match: + return None + # Demangle the name. If the name is too complex then we don't need to keep + # it, but it the demangling fails then keep the symbol just in case. + try: + names, _ = parse_itanium_nested_name(match.group(2)) + except TooComplexName: + return None + if not names: + return symbol + # Constructors and destructors of templates classes are assumed to be + # defined in headers and not required to be kept + if re.match('[CD][123]', names[-1][0]) and names[-2][1]: + return None + # Discard function template instantiations as it's assumed that the + # definition is public + elif names[-1][1]: + return None + # Keep llvm:: and clang:: names + elif names[0][0] == '4llvm' or names[0][0] == '5clang': + return symbol + # Discard everything else + else: + return None + +# Certain kinds of complex manglings we assume cannot be part of a public +# interface, and we handle them by raising an exception. +class TooComplexName(Exception): + pass + +# Parse an itanium mangled name from the start of a string and return a +# (name, rest of string) pair. +def parse_itanium_name(arg): + # Check for a normal name + match = re.match('(\d+)(.+)', arg) + if match: + n = int(match.group(1)) + name = match.group(1)+match.group(2)[:n] + rest = match.group(2)[n:] + return name, rest + # Check for constructor/destructor names + match = re.match('([CD][123])(.+)', arg) + if match: + return match.group(1), match.group(2) + # Assume that a sequence of characters that doesn't end a nesting is an + # operator (this is very imprecise, but appears to be good enough) + match = re.match('([^E]+)(.+)', arg) + if match: + return match.group(1), match.group(2) + # Anything else: we can't handle it + return None, arg + +# Parse an itanium mangled template argument list from the start of a string +# and throw it away, returning the rest of the string. +def skip_itanium_template(arg): + # A template argument list starts with I + assert arg.startswith('I'), arg + tmp = arg[1:] + while tmp: + # Check for names + match = re.match('(\d+)(.+)', tmp) + if match: + n = int(match.group(1)) + tmp = match.group(2)[n:] + continue + # Check for substitutions + match = re.match('S[A-Z0-9]*_(.+)', tmp) + if match: + tmp = match.group(1) + # Start of a template + elif tmp.startswith('I'): + tmp = skip_itanium_template(tmp) + # Start of a nested name + elif tmp.startswith('N'): + _, tmp = parse_itanium_nested_name(tmp) + # Start of an expression: assume that it's too complicated + elif tmp.startswith('L') or tmp.startswith('X'): + raise TooComplexName + # End of the template + elif tmp.startswith('E'): + return tmp[1:] + # Something else: probably a type, skip it + else: + tmp = tmp[1:] + return None + +# Parse an itanium mangled nested name and transform it into a list of pairs of +# (name, is_template), returning (list, rest of string). +def parse_itanium_nested_name(arg): + # A nested name starts with N + assert arg.startswith('N'), arg + ret = [] + + # Skip past the N, and possibly a substitution + match = re.match('NS[A-Z0-9]*_(.+)', arg) + if match: + tmp = match.group(1) + else: + tmp = arg[1:] + + # Skip past CV-qualifiers and ref qualifiers + match = re.match('[rVKRO]*(.+)', tmp); + if match: + tmp = match.group(1) + + # Repeatedly parse names from the string until we reach the end of the + # nested name + while tmp: + # An E ends the nested name + if tmp.startswith('E'): + return ret, tmp[1:] + # Parse a name + name_part, tmp = parse_itanium_name(tmp) + if not name_part: + # If we failed then we don't know how to demangle this + return None, None + is_template = False + # If this name is a template record that, then skip the template + # arguments + if tmp.startswith('I'): + tmp = skip_itanium_template(tmp) + is_template = True + # Add the name to the list + ret.append((name_part, is_template)) + + # If we get here then something went wrong + return None, None + +def extract_symbols(arg): + get_symbols, should_keep_symbol, calling_convention_decoration, lib = arg + symbols = dict() + for symbol in get_symbols(lib): + symbol = should_keep_symbol(symbol, calling_convention_decoration) + if symbol: + symbols[symbol] = 1 + symbols.setdefault(symbol,0) + return symbols + +if __name__ == '__main__': + tool_exes = ['dumpbin','nm','objdump','llvm-readobj'] + parser = argparse.ArgumentParser( + description='Extract symbols to export from libraries') + parser.add_argument('--mangling', choices=['itanium','microsoft'], + required=True, help='expected symbol mangling scheme') + parser.add_argument('--tools', choices=tool_exes, nargs='*', + help='tools to use to extract symbols and determine the' + ' target') + parser.add_argument('libs', metavar='lib', type=str, nargs='+', + help='libraries to extract symbols from') + parser.add_argument('-o', metavar='file', type=str, help='output to file') + args = parser.parse_args() + + # Determine the function to use to get the list of symbols from the inputs, + # and the function to use to determine if the target is 32-bit windows. + tools = { 'dumpbin' : (dumpbin_get_symbols, dumpbin_is_32bit_windows), + 'nm' : (nm_get_symbols, None), + 'objdump' : (None, objdump_is_32bit_windows), + 'llvm-readobj' : (readobj_get_symbols, readobj_is_32bit_windows) } + get_symbols = None + is_32bit_windows = None + # If we have a tools argument then use that for the list of tools to check + if args.tools: + tool_exes = args.tools + # Find a tool to use by trying each in turn until we find one that exists + # (subprocess.call will throw OSError when the program does not exist) + get_symbols = None + for exe in tool_exes: + try: + # Close std streams as we don't want any output and we don't + # want the process to wait for something on stdin. + p = subprocess.Popen([exe], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + universal_newlines=True) + p.stdout.close() + p.stderr.close() + p.stdin.close() + p.wait() + # Keep going until we have a tool to use for both get_symbols and + # is_32bit_windows + if not get_symbols: + get_symbols = tools[exe][0] + if not is_32bit_windows: + is_32bit_windows = tools[exe][1] + if get_symbols and is_32bit_windows: + break + except OSError: + continue + if not get_symbols: + print("Couldn't find a program to read symbols with", file=sys.stderr) + exit(1) + if not is_32bit_windows: + print("Couldn't find a program to determing the target", file=sys.stderr) + exit(1) + + # How we determine which symbols to keep and which to discard depends on + # the mangling scheme + if args.mangling == 'microsoft': + should_keep_symbol = should_keep_microsoft_symbol + else: + should_keep_symbol = should_keep_itanium_symbol + + # Get the list of libraries to extract symbols from + libs = list() + for lib in args.libs: + # When invoked by cmake the arguments are the cmake target names of the + # libraries, so we need to add .lib/.a to the end and maybe lib to the + # start to get the filename. Also allow objects. + suffixes = ['.lib','.a','.obj','.o'] + if not any([lib.endswith(s) for s in suffixes]): + for s in suffixes: + if os.path.exists(lib+s): + lib = lib+s + break + if os.path.exists('lib'+lib+s): + lib = 'lib'+lib+s + break + if not any([lib.endswith(s) for s in suffixes]): + print("Don't know what to do with argument "+lib, file=sys.stderr) + exit(1) + libs.append(lib) + + # Check if calling convention decoration is used by inspecting the first + # library in the list + calling_convention_decoration = is_32bit_windows(libs[0]) + + # Extract symbols from libraries in parallel. This is a huge time saver when + # doing a debug build, as there are hundreds of thousands of symbols in each + # library. + pool = multiprocessing.Pool() + try: + # Only one argument can be passed to the mapping function, and we can't + # use a lambda or local function definition as that doesn't work on + # windows, so create a list of tuples which duplicates the arguments + # that are the same in all calls. + vals = [(get_symbols, should_keep_symbol, calling_convention_decoration, x) for x in libs] + # Do an async map then wait for the result to make sure that + # KeyboardInterrupt gets caught correctly (see + # http://bugs.python.org/issue8296) + result = pool.map_async(extract_symbols, vals) + pool.close() + libs_symbols = result.get(3600) + except KeyboardInterrupt: + # On Ctrl-C terminate everything and exit + pool.terminate() + pool.join() + exit(1) + + # Merge everything into a single dict + symbols = dict() + for this_lib_symbols in libs_symbols: + for k,v in list(this_lib_symbols.items()): + symbols[k] = v + symbols.setdefault(k,0) + + # Count instances of member functions of template classes, and map the + # symbol name to the function+class. We do this under the assumption that if + # a member function of a template class is instantiated many times it's + # probably declared in a public header file. + template_function_count = dict() + template_function_mapping = dict() + template_function_count[""] = 0 + for k in symbols: + name = None + if args.mangling == 'microsoft': + # Member functions of templates start with + # ?<fn_name>@?$<class_name>@, so we map to <fn_name>@?$<class_name>. + # As manglings go from the innermost scope to the outermost scope + # this means: + # * When we have a function member of a subclass of a template + # class then <fn_name> will actually contain the mangling of + # both the subclass and the function member. This is fine. + # * When we have a function member of a template subclass of a + # (possibly template) class then it's the innermost template + # subclass that becomes <class_name>. This should be OK so long + # as we don't have multiple classes with a template subclass of + # the same name. + match = re.search("^\?(\??\w+\@\?\$\w+)\@", k) + if match: + name = match.group(1) + else: + # Find member functions of templates by demangling the name and + # checking if the second-to-last name in the list is a template. + match = re.match('_Z(T[VTIS])?(N.+)', k) + if match: + try: + names, _ = parse_itanium_nested_name(match.group(2)) + if names and names[-2][1]: + name = ''.join([x for x,_ in names]) + except TooComplexName: + # Manglings that are too complex should already have been + # filtered out, but if we happen to somehow see one here + # just leave it as-is. + pass + if name: + old_count = template_function_count.setdefault(name,0) + template_function_count[name] = old_count + 1 + template_function_mapping[k] = name + else: + template_function_mapping[k] = "" + + # Print symbols which both: + # * Appear in exactly one input, as symbols defined in multiple + # objects/libraries are assumed to have public definitions. + # * Aren't instances of member functions of templates which have been + # instantiated 100 times or more, which are assumed to have public + # definitions. (100 is an arbitrary guess here.) + if args.o: + outfile = open(args.o,'w') + else: + outfile = sys.stdout + for k,v in list(symbols.items()): + template_count = template_function_count[template_function_mapping[k]] + if v == 1 and template_count < 100: + print(k, file=outfile) |