diff options
Diffstat (limited to 'scripts/swig_bot_lib')
| -rw-r--r-- | scripts/swig_bot_lib/__init__.py | 0 | ||||
| -rw-r--r-- | scripts/swig_bot_lib/client.py | 205 | ||||
| -rw-r--r-- | scripts/swig_bot_lib/local.py | 131 | ||||
| -rw-r--r-- | scripts/swig_bot_lib/remote.py | 39 | ||||
| -rw-r--r-- | scripts/swig_bot_lib/server.py | 138 |
5 files changed, 513 insertions, 0 deletions
diff --git a/scripts/swig_bot_lib/__init__.py b/scripts/swig_bot_lib/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/scripts/swig_bot_lib/__init__.py diff --git a/scripts/swig_bot_lib/client.py b/scripts/swig_bot_lib/client.py new file mode 100644 index 000000000000..9bf55b42b666 --- /dev/null +++ b/scripts/swig_bot_lib/client.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python + +""" +SWIG generation client. Supports both local and remote generation of SWIG +bindings for multiple languages. +""" + +# Future imports +from __future__ import absolute_import +from __future__ import print_function + +# Python modules +import argparse +import io +import logging +import os +import socket +import struct +import sys + +# LLDB modules +import use_lldb_suite +from lldbsuite.support import fs +from lldbsuite.support import sockutil + +# package imports +from . import local +from . import remote + +default_ip = "127.0.0.1" +default_port = 8537 + +def add_subparser_args(parser): + """Returns options processed from the provided command line. + + @param args the command line to process. + """ + + # A custom action used by the --local command line option. It can be + # used with either 0 or 1 argument. If used with 0 arguments, it + # searches for a copy of swig located on the physical machine. If + # used with 1 argument, the argument is the path to a swig executable. + class FindLocalSwigAction(argparse.Action): + def __init__(self, option_strings, dest, **kwargs): + super(FindLocalSwigAction, self).__init__( + option_strings, dest, nargs='?', **kwargs) + def __call__(self, parser, namespace, values, option_string=None): + swig_exe = None + if values is None: + swig_exe = fs.find_executable('swig') + else: + swig_exe = values + setattr(namespace, self.dest, os.path.normpath(swig_exe)) + + # A custom action used by the --remote command line option. It can be + # used with either 0 or 1 arguments. If used with 0 arguments it chooses + # a default connection string. If used with one argument it is a string + # of the form `ip_address[:port]`. If the port is unspecified, the + # default port is used. + class RemoteIpAction(argparse.Action): + def __init__(self, option_strings, dest, **kwargs): + super(RemoteIpAction, self).__init__( + option_strings, dest, nargs='?', **kwargs) + def __call__(self, parser, namespace, values, option_string=None): + ip_port = None + if values is None: + ip_port = (default_ip, default_port) + else: + result = values.split(':') + if len(result)==1: + ip_port = (result[0], default_port) + elif len(result)==2: + ip_port = (result[0], int(result[1])) + else: + raise ValueError("Invalid connection string") + setattr(namespace, self.dest, ip_port) + + parser.add_argument( + "--local", + action=FindLocalSwigAction, + dest="swig_executable", + help=( + "Run the copy of swig at the specified location, or search PATH" + "if the location is omitted")) + + parser.add_argument( + "--remote", + action=RemoteIpAction, + help=( + "Use the given connection string to connect to a remote " + "generation service")) + + parser.add_argument( + "--src-root", + required=True, + help="The root folder of the LLDB source tree.") + + parser.add_argument( + "--target-dir", + default=os.getcwd(), + help=( + "Specifies the build dir where the language binding " + "should be placed")) + + parser.add_argument( + "--language", + dest="languages", + action="append", + help="Specifies the language to generate bindings for") + +def finalize_subparser_options(options): + if options.languages is None: + options.languages = ['python'] + + if options.remote is None and options.swig_executable is None: + logging.error("Must specify either --local or --remote") + sys.exit(-3) + + return options + +def establish_remote_connection(ip_port): + logging.debug("Creating socket...") + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + logging.info("Connecting to server {} on port {}" + .format(ip_port[0], ip_port[1])) + s.connect(ip_port) + logging.info("Connection established...") + return s + +def transmit_request(connection, packed_input): + logging.info("Sending {} bytes of compressed data." + .format(len(packed_input))) + connection.sendall(struct.pack("!I", len(packed_input))) + connection.sendall(packed_input) + logging.info("Awaiting response.") + response_len = struct.unpack("!I", sockutil.recvall(connection, 4))[0] + logging.debug("Expecting {} byte response".format(response_len)) + response = sockutil.recvall(connection, response_len) + return response + +def handle_response(options, connection, response): + logging.debug("Received {} byte response.".format(len(response))) + logging.debug("Creating output directory {}" + .format(options.target_dir)) + os.makedirs(options.target_dir, exist_ok=True) + + logging.info("Unpacking response archive into {}" + .format(options.target_dir)) + local.unpack_archive(options.target_dir, response) + response_file_path = os.path.normpath( + os.path.join(options.target_dir, "swig_output.json")) + if not os.path.isfile(response_file_path): + logging.error("Response file '{}' does not exist." + .format(response_file_path)) + return + try: + response = remote.deserialize_response_status( + io.open(response_file_path)) + if response[0] != 0: + logging.error("An error occurred during generation. Status={}" + .format(response[0])) + logging.error(response[1]) + else: + logging.info("SWIG generation successful.") + if len(response[1]) > 0: + logging.info(response[1]) + finally: + os.unlink(response_file_path) + +def run(options): + if options.remote is None: + logging.info("swig bot client using local swig installation at '{}'" + .format(options.swig_executable)) + if not os.path.isfile(options.swig_executable): + logging.error("Swig executable '{}' does not exist." + .format(options.swig_executable)) + config = local.LocalConfig() + config.languages = options.languages + config.src_root = options.src_root + config.target_dir = options.target_dir + config.swig_executable = options.swig_executable + local.generate(config) + else: + logging.info("swig bot client using remote generation with server '{}'" + .format(options.remote)) + connection = None + try: + config = remote.generate_config(options.languages) + logging.debug("Generated config json {}".format(config)) + inputs = [("include/lldb", ".h"), + ("include/lldb/API", ".h"), + ("scripts", ".swig"), + ("scripts/Python", ".swig"), + ("scripts/interface", ".i")] + zip_data = io.BytesIO() + packed_input = local.pack_archive(zip_data, options.src_root, inputs) + logging.info("(null) -> config.json") + packed_input.writestr("config.json", config) + packed_input.close() + connection = establish_remote_connection(options.remote) + response = transmit_request(connection, zip_data.getvalue()) + handle_response(options, connection, response) + finally: + if connection is not None: + connection.close()
\ No newline at end of file diff --git a/scripts/swig_bot_lib/local.py b/scripts/swig_bot_lib/local.py new file mode 100644 index 000000000000..7cca0b3cabbb --- /dev/null +++ b/scripts/swig_bot_lib/local.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +""" +Shared functionality used by `client` and `server` when generating or preparing +to generate SWIG on the local machine. +""" + +# Future imports +from __future__ import absolute_import +from __future__ import print_function + +# Python modules +import argparse +import imp +import io +import logging +import os +import subprocess +import sys +import tempfile +import zipfile + +# LLDB modules +import use_lldb_suite + +# Package imports +from lldbsuite.support import fs + +class LocalConfig(object): + src_root = None + target_dir = None + languages = None + swig_executable = None + +def pack_archive(bytes_io, src_root, filters): + logging.info("Creating input file package...") + zip_file = None + try: + # It's possible that a custom-built interpreter will not have the + # standard zlib module. If so, we can only store, not compress. By + # try to compress since we usually have a standard Python distribution. + zip_file = zipfile.ZipFile(bytes_io, mode='w', + compression=zipfile.ZIP_DEFLATED) + except RuntimeError: + zip_file = zipfile.ZipFile(bytes_io, mode='w', + compression=zipfile.ZIP_STORED) + archive_entries = [] + if filters is not None: + def filter_func(t): + subfolder = t[0] + ext = t[1] + full_path = os.path.normpath(os.path.join(src_root, subfolder)) + candidates = [os.path.normpath(os.path.join(full_path, f)) + for f in os.listdir(full_path)] + actual = filter( + lambda f : os.path.isfile(f) and os.path.splitext(f)[1] == ext, + candidates) + return (subfolder, map(lambda f : os.path.basename(f), actual)) + archive_entries = map(filter_func, filters) + else: + for (root, dirs, files) in os.walk(src_root): + logging.debug("Adding files {} from directory {} to output package" + .format(files, root)) + if len(files) > 0: + rel_root = os.path.relpath(root, src_root) + archive_entries.append((rel_root, files)) + + archive_entries = list(archive_entries) + for entry in archive_entries: + subfolder = entry[0] + files = list(entry[1]) + for file in files: + rel_path = os.path.normpath(os.path.join(subfolder, file)) + full_path = os.path.join(src_root, rel_path) + logging.info("{} -> {}".format(full_path, rel_path)) + zip_file.write(full_path, rel_path) + + return zip_file + +def unpack_archive(folder, archive_bytes): + zip_data = io.BytesIO(archive_bytes) + logging.debug("Opening zip archive...") + zip_file = zipfile.ZipFile(zip_data, mode='r') + zip_file.extractall(folder) + zip_file.close() + +def generate(options): + include_folder = os.path.join(options.src_root, "include") + in_file = os.path.join(options.src_root, "scripts", "lldb.swig") + include_folder = os.path.normcase(include_folder) + + for lang in options.languages: + lang = lang.lower() + out_dir = os.path.join(options.target_dir, lang.title()) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + out_file = os.path.join(out_dir, "LLDBWrap{}.cpp".format(lang.title())) + swig_command = [ + options.swig_executable, + "-c++", + ] + swig_command.append("-" + lang) + if lang == "python": + swig_command.append("-threads") + + swig_command.extend([ + "-I" + include_folder, + "-D__STDC_LIMIT_MACROS", + "-D__STDC_CONSTANT_MACROS", + "-outdir", out_dir, + "-o", out_file, + in_file + ]) + + logging.info("generating swig {} bindings into {}" + .format(lang, out_dir)) + logging.debug("swig command line: {}".format(swig_command)) + try: + # Execute swig + swig_output = subprocess.check_output( + swig_command, stderr=subprocess.STDOUT, universal_newlines=True) + + logging.info("swig generation succeeded") + if swig_output is not None and len(swig_output) > 0: + logging.info("swig output: %s", swig_output) + return (0, swig_output) + except subprocess.CalledProcessError as e: + logging.error("An error occurred executing swig. returncode={}" + .format(e.returncode)) + logging.error(e.output) + return (e.returncode, e.output)
\ No newline at end of file diff --git a/scripts/swig_bot_lib/remote.py b/scripts/swig_bot_lib/remote.py new file mode 100644 index 000000000000..590a873d6270 --- /dev/null +++ b/scripts/swig_bot_lib/remote.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +""" +Shared functionality used by `client` and `server` when dealing with +remote transmission +""" + +# Future imports +from __future__ import absolute_import +from __future__ import print_function + +# Python modules +import json +import logging +import os +import socket +import struct +import sys + +# LLDB modules +import use_lldb_suite + +def generate_config(languages): + config = {"languages": languages} + return json.dumps(config) + +def parse_config(json_reader): + json_data = json_reader.read() + options_dict = json.loads(json_data) + return options_dict + +def serialize_response_status(status): + status = {"retcode": status[0], "output": status[1]} + return json.dumps(status) + +def deserialize_response_status(json_reader): + json_data = json_reader.read() + response_dict = json.loads(json_data) + return (response_dict["retcode"], response_dict["output"]) diff --git a/scripts/swig_bot_lib/server.py b/scripts/swig_bot_lib/server.py new file mode 100644 index 000000000000..cc25cee4d4b1 --- /dev/null +++ b/scripts/swig_bot_lib/server.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +""" +SWIG generation server. Listens for connections from swig generation clients +and runs swig in the requested fashion, sending back the results. +""" + +# Future imports +from __future__ import absolute_import +from __future__ import print_function + +# Python modules +import argparse +import io +import logging +import os +import select +import shutil +import socket +import struct +import sys +import tempfile +import traceback + +# LLDB modules +import use_lldb_suite +from lldbsuite.support import fs +from lldbsuite.support import sockutil + +# package imports +from . import local +from . import remote + +default_port = 8537 + +def add_subparser_args(parser): + parser.add_argument( + "--port", + action="store", + default=default_port, + help=("The local port to bind to")) + + parser.add_argument( + "--swig-executable", + action="store", + default=fs.find_executable("swig"), + dest="swig_executable") + +def finalize_subparser_options(options): + pass + +def initialize_listening_socket(options): + logging.debug("Creating socket...") + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + logging.info("Binding to ip address '', port {}".format(options.port)) + s.bind(('', options.port)) + + logging.debug("Putting socket in listen mode...") + s.listen() + return s + +def accept_once(sock, options): + logging.debug("Waiting for connection...") + while True: + rlist, wlist, xlist = select.select([sock], [], [], 0.5) + if not rlist: + continue + + client, addr = sock.accept() + logging.info("Received connection from {}".format(addr)) + data_size = struct.unpack("!I", sockutil.recvall(client, 4))[0] + logging.debug("Expecting {} bytes of data from client" + .format(data_size)) + data = sockutil.recvall(client, data_size) + logging.info("Received {} bytes of data from client" + .format(len(data))) + + pack_location = None + try: + tempfolder = os.path.join(tempfile.gettempdir(), "swig-bot") + os.makedirs(tempfolder, exist_ok=True) + + pack_location = tempfile.mkdtemp(dir=tempfolder) + logging.debug("Extracting archive to {}".format(pack_location)) + + local.unpack_archive(pack_location, data) + logging.debug("Successfully unpacked archive...") + + config_file = os.path.normpath(os.path.join(pack_location, + "config.json")) + parsed_config = remote.parse_config(io.open(config_file)) + config = local.LocalConfig() + config.languages = parsed_config["languages"] + config.swig_executable = options.swig_executable + config.src_root = pack_location + config.target_dir = os.path.normpath( + os.path.join(config.src_root, "output")) + logging.info( + "Running swig. languages={}, swig={}, src_root={}, target={}" + .format(config.languages, config.swig_executable, + config.src_root, config.target_dir)) + + status = local.generate(config) + logging.debug("Finished running swig. Packaging up files {}" + .format(os.listdir(config.target_dir))) + zip_data = io.BytesIO() + zip_file = local.pack_archive(zip_data, config.target_dir, None) + response_status = remote.serialize_response_status(status) + logging.debug("Sending response status {}".format(response_status)) + logging.info("(swig output) -> swig_output.json") + zip_file.writestr("swig_output.json", response_status) + + zip_file.close() + response_data = zip_data.getvalue() + logging.info("Sending {} byte response".format(len(response_data))) + client.sendall(struct.pack("!I", len(response_data))) + client.sendall(response_data) + finally: + if pack_location is not None: + logging.debug("Removing temporary folder {}" + .format(pack_location)) + shutil.rmtree(pack_location) + +def accept_loop(sock, options): + while True: + try: + accept_once(sock, options) + except Exception as e: + error = traceback.format_exc() + logging.error("An error occurred while processing the connection.") + logging.error(error) + +def run(options): + print(options) + sock = initialize_listening_socket(options) + accept_loop(sock, options) + return options |