diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2017-01-02 19:26:05 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2017-01-02 19:26:05 +0000 |
commit | 14f1b3e8826ce43b978db93a62d1166055db5394 (patch) | |
tree | 0a00ad8d3498783fe0193f3b656bca17c4c8697d /packages/Python/lldbsuite/test_event/formatter | |
parent | 4ee8c119c71a06dcad1e0fecc8c675e480e59337 (diff) |
Notes
Diffstat (limited to 'packages/Python/lldbsuite/test_event/formatter')
5 files changed, 266 insertions, 70 deletions
diff --git a/packages/Python/lldbsuite/test_event/formatter/__init__.py b/packages/Python/lldbsuite/test_event/formatter/__init__.py index 556370ebb9dee..2481e326e9468 100644 --- a/packages/Python/lldbsuite/test_event/formatter/__init__.py +++ b/packages/Python/lldbsuite/test_event/formatter/__init__.py @@ -76,7 +76,8 @@ def create_results_formatter(config): # we lose the test result info. read_bytes = sock.recv(1) if read_bytes is None or (len(read_bytes) < 1) or (read_bytes != b'*'): - raise Exception("listening socket did not respond with ack byte: response={}".format(read_bytes)) + raise Exception( + "listening socket did not respond with ack byte: response={}".format(read_bytes)) return sock, lambda: socket_closer(sock) diff --git a/packages/Python/lldbsuite/test_event/formatter/curses.py b/packages/Python/lldbsuite/test_event/formatter/curses.py index 4e89fa9bf01e7..f415575ded8ac 100644 --- a/packages/Python/lldbsuite/test_event/formatter/curses.py +++ b/packages/Python/lldbsuite/test_event/formatter/curses.py @@ -38,7 +38,10 @@ class Curses(results_formatter.ResultsFormatter): self.results = list() try: self.main_window = lldbcurses.intialize_curses() - self.main_window.add_key_action('\t', self.main_window.select_next_first_responder, "Switch between views that can respond to keyboard input") + self.main_window.add_key_action( + '\t', + self.main_window.select_next_first_responder, + "Switch between views that can respond to keyboard input") self.main_window.refresh() self.job_panel = None self.results_panel = None @@ -87,10 +90,15 @@ class Curses(results_formatter.ResultsFormatter): selected_idx = self.results_panel.get_selected_idx() if selected_idx >= 0 and selected_idx < len(self.results): if self.info_panel is None: - info_frame = self.results_panel.get_contained_rect(top_inset=10, left_inset=10, right_inset=10, height=30) - self.info_panel = lldbcurses.BoxedPanel(info_frame, "Result Details") - # Add a key action for any key that will hide this panel when any key is pressed - self.info_panel.add_key_action(-1, self.hide_info_panel, 'Hide the info panel') + info_frame = self.results_panel.get_contained_rect( + top_inset=10, left_inset=10, right_inset=10, height=30) + self.info_panel = lldbcurses.BoxedPanel( + info_frame, "Result Details") + # Add a key action for any key that will hide this panel when + # any key is pressed + self.info_panel.add_key_action(-1, + self.hide_info_panel, + 'Hide the info panel') self.info_panel.top() else: self.info_panel.show() @@ -98,9 +106,15 @@ class Curses(results_formatter.ResultsFormatter): self.main_window.push_first_responder(self.info_panel) test_start = self.results[selected_idx][0] test_result = self.results[selected_idx][1] - self.info_panel.set_line(0, "File: %s" % (test_start['test_filename'])) - self.info_panel.set_line(1, "Test: %s.%s" % (test_start['test_class'], test_start['test_name'])) - self.info_panel.set_line(2, "Time: %s" % (test_result['elapsed_time'])) + self.info_panel.set_line( + 0, "File: %s" % + (test_start['test_filename'])) + self.info_panel.set_line( + 1, "Test: %s.%s" % + (test_start['test_class'], test_start['test_name'])) + self.info_panel.set_line( + 2, "Time: %s" % + (test_result['elapsed_time'])) self.info_panel.set_line(3, "Status: %s" % (test_result['status'])) def hide_info_panel(self): @@ -110,7 +124,8 @@ class Curses(results_formatter.ResultsFormatter): def toggle_status(self, status): if status: - # Toggle showing and hiding results whose status matches "status" in "Results" window + # Toggle showing and hiding results whose status matches "status" + # in "Results" window if status in self.hide_status_list: self.hide_status_list.remove(status) else: @@ -127,7 +142,13 @@ class Curses(results_formatter.ResultsFormatter): if status in self.hide_status_list: continue name = test_result['test_class'] + '.' + test_result['test_name'] - self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status, test_result), test_result['elapsed_time'], name)) + self.results_panel.append_line( + '%s (%6.2f sec) %s' % + (self.status_to_short_str( + status, + test_result), + test_result['elapsed_time'], + name)) if update: self.main_window.refresh() @@ -145,9 +166,14 @@ class Curses(results_formatter.ResultsFormatter): #print(str(test_event), file=self.events_file) event = test_event['event'] if self.status_panel: - self.status_panel.update_status('time', str(datetime.timedelta(seconds=math.floor(time.time() - self.start_time)))) + self.status_panel.update_status( + 'time', str( + datetime.timedelta( + seconds=math.floor( + time.time() - self.start_time)))) if event == 'test_start': - name = test_event['test_class'] + '.' + test_event['test_name'] + name = test_event['test_class'] + \ + '.' + test_event['test_name'] self.job_tests[worker_index] = test_event if 'pid' in test_event: line = 'pid: %5d ' % (test_event['pid']) + name @@ -163,14 +189,20 @@ class Curses(results_formatter.ResultsFormatter): else: line = '' self.job_panel.set_line(worker_index, line) - name = test_event['test_class'] + '.' + test_event['test_name'] - elapsed_time = test_event['event_time'] - self.job_tests[worker_index]['event_time'] - if not status in self.hide_status_list: - self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status, test_event), elapsed_time, name)) + name = test_event['test_class'] + \ + '.' + test_event['test_name'] + elapsed_time = test_event[ + 'event_time'] - self.job_tests[worker_index]['event_time'] + if status not in self.hide_status_list: + self.results_panel.append_line( + '%s (%6.2f sec) %s' % + (self.status_to_short_str( + status, test_event), elapsed_time, name)) self.main_window.refresh() # Append the result pairs test_event['elapsed_time'] = elapsed_time - self.results.append([self.job_tests[worker_index], test_event]) + self.results.append( + [self.job_tests[worker_index], test_event]) self.job_tests[worker_index] = '' elif event == 'job_begin': self.jobs[worker_index] = test_event @@ -185,40 +217,121 @@ class Curses(results_formatter.ResultsFormatter): elif event == 'initialize': self.initialize_event = test_event num_jobs = test_event['worker_count'] - job_frame = self.main_window.get_contained_rect(height=num_jobs+2) - results_frame = self.main_window.get_contained_rect(top_inset=num_jobs+2, bottom_inset=1) - status_frame = self.main_window.get_contained_rect(height=1, top_inset=self.main_window.get_size().h-1) - self.job_panel = lldbcurses.BoxedPanel(frame=job_frame, title="Jobs") - self.results_panel = lldbcurses.BoxedPanel(frame=results_frame, title="Results") - - self.results_panel.add_key_action(curses.KEY_UP, self.results_panel.select_prev , "Select the previous list entry") - self.results_panel.add_key_action(curses.KEY_DOWN, self.results_panel.select_next , "Select the next list entry") - self.results_panel.add_key_action(curses.KEY_HOME, self.results_panel.scroll_begin , "Scroll to the start of the list") - self.results_panel.add_key_action(curses.KEY_END, self.results_panel.scroll_end , "Scroll to the end of the list") - self.results_panel.add_key_action(curses.KEY_ENTER, self.show_info_panel , "Display info for the selected result item") - self.results_panel.add_key_action('.', lambda : self.toggle_status(EventBuilder.STATUS_SUCCESS) , "Toggle showing/hiding tests whose status is 'success'") - self.results_panel.add_key_action('e', lambda : self.toggle_status(EventBuilder.STATUS_ERROR) , "Toggle showing/hiding tests whose status is 'error'") - self.results_panel.add_key_action('f', lambda : self.toggle_status(EventBuilder.STATUS_FAILURE) , "Toggle showing/hiding tests whose status is 'failure'") - self.results_panel.add_key_action('s', lambda : self.toggle_status(EventBuilder.STATUS_SKIP) , "Toggle showing/hiding tests whose status is 'skip'") - self.results_panel.add_key_action('x', lambda : self.toggle_status(EventBuilder.STATUS_EXPECTED_FAILURE) , "Toggle showing/hiding tests whose status is 'expected_failure'") - self.results_panel.add_key_action('?', lambda : self.toggle_status(EventBuilder.STATUS_UNEXPECTED_SUCCESS), "Toggle showing/hiding tests whose status is 'unexpected_success'") - self.status_panel = lldbcurses.StatusPanel(frame=status_frame) + job_frame = self.main_window.get_contained_rect( + height=num_jobs + 2) + results_frame = self.main_window.get_contained_rect( + top_inset=num_jobs + 2, bottom_inset=1) + status_frame = self.main_window.get_contained_rect( + height=1, top_inset=self.main_window.get_size().h - 1) + self.job_panel = lldbcurses.BoxedPanel( + frame=job_frame, title="Jobs") + self.results_panel = lldbcurses.BoxedPanel( + frame=results_frame, title="Results") + + self.results_panel.add_key_action( + curses.KEY_UP, + self.results_panel.select_prev, + "Select the previous list entry") + self.results_panel.add_key_action( + curses.KEY_DOWN, self.results_panel.select_next, "Select the next list entry") + self.results_panel.add_key_action( + curses.KEY_HOME, + self.results_panel.scroll_begin, + "Scroll to the start of the list") + self.results_panel.add_key_action( + curses.KEY_END, self.results_panel.scroll_end, "Scroll to the end of the list") + self.results_panel.add_key_action( + curses.KEY_ENTER, + self.show_info_panel, + "Display info for the selected result item") + self.results_panel.add_key_action( + '.', + lambda: self.toggle_status( + EventBuilder.STATUS_SUCCESS), + "Toggle showing/hiding tests whose status is 'success'") + self.results_panel.add_key_action( + 'e', + lambda: self.toggle_status( + EventBuilder.STATUS_ERROR), + "Toggle showing/hiding tests whose status is 'error'") + self.results_panel.add_key_action( + 'f', + lambda: self.toggle_status( + EventBuilder.STATUS_FAILURE), + "Toggle showing/hiding tests whose status is 'failure'") + self.results_panel.add_key_action('s', lambda: self.toggle_status( + EventBuilder.STATUS_SKIP), "Toggle showing/hiding tests whose status is 'skip'") + self.results_panel.add_key_action( + 'x', + lambda: self.toggle_status( + EventBuilder.STATUS_EXPECTED_FAILURE), + "Toggle showing/hiding tests whose status is 'expected_failure'") + self.results_panel.add_key_action( + '?', + lambda: self.toggle_status( + EventBuilder.STATUS_UNEXPECTED_SUCCESS), + "Toggle showing/hiding tests whose status is 'unexpected_success'") + self.status_panel = lldbcurses.StatusPanel( + frame=status_frame) self.main_window.add_child(self.job_panel) self.main_window.add_child(self.results_panel) self.main_window.add_child(self.status_panel) - self.main_window.set_first_responder(self.results_panel) - - self.status_panel.add_status_item(name="time", title="Elapsed", format="%s", width=20, value="0:00:00", update=False) - self.status_panel.add_status_item(name=EventBuilder.STATUS_SUCCESS, title="Success", format="%u", width=20, value=0, update=False) - self.status_panel.add_status_item(name=EventBuilder.STATUS_FAILURE, title="Failure", format="%u", width=20, value=0, update=False) - self.status_panel.add_status_item(name=EventBuilder.STATUS_ERROR, title="Error", format="%u", width=20, value=0, update=False) - self.status_panel.add_status_item(name=EventBuilder.STATUS_SKIP, title="Skipped", format="%u", width=20, value=0, update=True) - self.status_panel.add_status_item(name=EventBuilder.STATUS_EXPECTED_FAILURE, title="Expected Failure", format="%u", width=30, value=0, update=False) - self.status_panel.add_status_item(name=EventBuilder.STATUS_UNEXPECTED_SUCCESS, title="Unexpected Success", format="%u", width=30, value=0, update=False) + self.main_window.set_first_responder( + self.results_panel) + + self.status_panel.add_status_item( + name="time", + title="Elapsed", + format="%s", + width=20, + value="0:00:00", + update=False) + self.status_panel.add_status_item( + name=EventBuilder.STATUS_SUCCESS, + title="Success", + format="%u", + width=20, + value=0, + update=False) + self.status_panel.add_status_item( + name=EventBuilder.STATUS_FAILURE, + title="Failure", + format="%u", + width=20, + value=0, + update=False) + self.status_panel.add_status_item( + name=EventBuilder.STATUS_ERROR, + title="Error", + format="%u", + width=20, + value=0, + update=False) + self.status_panel.add_status_item( + name=EventBuilder.STATUS_SKIP, + title="Skipped", + format="%u", + width=20, + value=0, + update=True) + self.status_panel.add_status_item( + name=EventBuilder.STATUS_EXPECTED_FAILURE, + title="Expected Failure", + format="%u", + width=30, + value=0, + update=False) + self.status_panel.add_status_item( + name=EventBuilder.STATUS_UNEXPECTED_SUCCESS, + title="Unexpected Success", + format="%u", + width=30, + value=0, + update=False) self.main_window.refresh() elif event == 'terminate': - #self.main_window.key_event_loop() + # self.main_window.key_event_loop() lldbcurses.terminate_curses() check_for_one_key = False self.using_terminal = False diff --git a/packages/Python/lldbsuite/test_event/formatter/pickled.py b/packages/Python/lldbsuite/test_event/formatter/pickled.py index 6d800f6c8baac..588614e2f7b42 100644 --- a/packages/Python/lldbsuite/test_event/formatter/pickled.py +++ b/packages/Python/lldbsuite/test_event/formatter/pickled.py @@ -31,21 +31,29 @@ class RawPickledFormatter(ResultsFormatter): return parser class StreamSerializer(object): + @staticmethod def serialize(test_event, out_file): - # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes} + # Send it as + # {serialized_length_of_serialized_bytes}{serialized_bytes} import struct msg = cPickle.dumps(test_event) packet = struct.pack("!I%ds" % len(msg), len(msg), msg) out_file.send(packet) class BlockSerializer(object): + @staticmethod def serialize(test_event, out_file): cPickle.dump(test_event, out_file) def __init__(self, out_file, options, file_is_stream): - super(RawPickledFormatter, self).__init__(out_file, options, file_is_stream) + super( + RawPickledFormatter, + self).__init__( + out_file, + options, + file_is_stream) self.pid = os.getpid() if file_is_stream: self.serializer = self.StreamSerializer() diff --git a/packages/Python/lldbsuite/test_event/formatter/results_formatter.py b/packages/Python/lldbsuite/test_event/formatter/results_formatter.py index 3bf389b972665..35a9fa8d5d564 100644 --- a/packages/Python/lldbsuite/test_event/formatter/results_formatter.py +++ b/packages/Python/lldbsuite/test_event/formatter/results_formatter.py @@ -14,6 +14,7 @@ from __future__ import absolute_import # System modules import argparse import os +import re import sys import threading @@ -27,6 +28,9 @@ from ..event_builder import EventBuilder import lldbsuite +FILE_LEVEL_KEY_RE = re.compile(r"^(.+\.py)[^.:]*$") + + class ResultsFormatter(object): """Provides interface to formatting test results out to a file-like object. @@ -207,6 +211,26 @@ class ResultsFormatter(object): component_count += 1 return key + @classmethod + def _is_file_level_issue(cls, key, event): + """Returns whether a given key represents a file-level event. + + @param cls this class. Unused, but following PEP8 for + preferring @classmethod over @staticmethod. + + @param key the key for the issue being tested. + + @param event the event for the issue being tested. + + @return True when the given key (as made by _make_key()) + represents an event that is at the test file level (i.e. + it isn't scoped to a test class or method). + """ + if key is None: + return False + else: + return FILE_LEVEL_KEY_RE.match(key) is not None + def _mark_test_as_expected_failure(self, test_result_event): key = self._make_key(test_result_event) if key is not None: @@ -321,8 +345,8 @@ class ResultsFormatter(object): # after this check below since this check may rewrite # the event type if event_type == EventBuilder.TYPE_JOB_RESULT: - # Possibly convert the job status (timeout, exceptional exit) - # to an appropriate test_result event. + # Possibly convert the job status (timeout, + # exceptional exit) # to an appropriate test_result event. self._maybe_remap_job_result_event(test_event) event_type = test_event.get("event", "") @@ -335,21 +359,22 @@ class ResultsFormatter(object): if event_type == "terminate": self.terminate_called = True elif event_type in EventBuilder.RESULT_TYPES: - # Keep track of event counts per test/job result status type. - # The only job (i.e. inferior process) results that make it - # here are ones that cannot be remapped to the most recently - # started test for the given worker index. + # Keep track of event counts per test/job result status + # type. The only job (i.e. inferior process) results that + # make it here are ones that cannot be remapped to the most + # recently started test for the given worker index. status = test_event["status"] self.result_status_counts[status] += 1 - # Clear the most recently started test for the related worker. + # Clear the most recently started test for the related + # worker. worker_index = test_event.get("worker_index", None) if worker_index is not None: self.started_tests_by_worker.pop(worker_index, None) if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES: # A test/job status value in any of those status values - # causes a testrun failure. If such a test fails, check - # whether it can be rerun. If it can be rerun, add it + # causes a testrun failure. If such a test fails, check + # whether it can be rerun. If it can be rerun, add it # to the rerun job. self._maybe_add_test_to_rerun_list(test_event) @@ -360,14 +385,13 @@ class ResultsFormatter(object): "failed to find test filename for " "test event {}".format(test_event)) - # Save the most recent test event for the test key. - # This allows a second test phase to overwrite the most - # recent result for the test key (unique per method). - # We do final reporting at the end, so we'll report based - # on final results. - # We do this so that a re-run caused by, perhaps, the need - # to run a low-load, single-worker test run can have the final - # run's results to always be used. + # Save the most recent test event for the test key. This + # allows a second test phase to overwrite the most recent + # result for the test key (unique per method). We do final + # reporting at the end, so we'll report based on final + # results. We do this so that a re-run caused by, perhaps, + # the need to run a low-load, single-worker test run can + # have the final run's results to always be used. if test_key in self.result_events: # We are replacing the result of something that was # already counted by the base class. Remove the double @@ -393,7 +417,8 @@ class ResultsFormatter(object): elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE: self._mark_test_for_rerun_eligibility(test_event) - elif event_type == EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE: + elif (event_type == + EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE): self._mark_test_as_expected_failure(test_event) def set_expected_timeouts_by_basename(self, basenames): @@ -688,7 +713,7 @@ class ResultsFormatter(object): # prevent buildbots from thinking it is an issue when scanning # for TIMEOUT. "Expected Timeout", True, "EXPECTED TIME-OUT"] - ] + ] # Partition all the events by test result status result_events_by_status = self._partition_results_by_status( @@ -715,3 +740,50 @@ class ResultsFormatter(object): for key, event in events_by_key: out_file.write("key: {}\n".format(key)) out_file.write("event: {}\n".format(event)) + + def clear_file_level_issues(self, tests_for_rerun, out_file): + """Clear file-charged issues in any of the test rerun files. + + @param tests_for_rerun the list of test-dir-relative paths that have + functions that require rerunning. This is the test list + returned by the results_formatter at the end of the previous run. + + @return the number of file-level issues that were cleared. + """ + if tests_for_rerun is None: + return 0 + + cleared_file_level_issues = 0 + # Find the unique set of files that are covered by the given tests + # that are to be rerun. We derive the files that are eligible for + # having their markers cleared, because we support running in a mode + # where only flaky tests are eligible for rerun. If the file-level + # issue occurred in a file that was not marked as flaky, then we + # shouldn't be clearing the event here. + basename_set = set() + for test_file_relpath in tests_for_rerun: + basename_set.add(os.path.basename(test_file_relpath)) + + # Find all the keys for file-level events that are considered + # test issues. + file_level_issues = [(key, event) + for key, event in self.result_events.items() + if ResultsFormatter._is_file_level_issue( + key, event) + and event.get("status", "") in + EventBuilder.TESTRUN_ERROR_STATUS_VALUES] + + # Now remove any file-level error for the given test base name. + for key, event in file_level_issues: + # If the given file base name is in the rerun set, then we + # clear that entry from the result set. + if os.path.basename(key) in basename_set: + self.result_events.pop(key, None) + cleared_file_level_issues += 1 + if out_file is not None: + out_file.write( + "clearing file-level issue for file {} " + "(issue type: {})" + .format(key, event.get("status", "<unset-status>"))) + + return cleared_file_level_issues diff --git a/packages/Python/lldbsuite/test_event/formatter/xunit.py b/packages/Python/lldbsuite/test_event/formatter/xunit.py index b3682f8fb107d..d3ea8677f0a6e 100644 --- a/packages/Python/lldbsuite/test_event/formatter/xunit.py +++ b/packages/Python/lldbsuite/test_event/formatter/xunit.py @@ -180,7 +180,7 @@ class XunitFormatter(ResultsFormatter): "unexpected_successes": [], "expected_failures": [], "all": [] - } + } self.status_handlers = { EventBuilder.STATUS_SUCCESS: self._handle_success, @@ -197,9 +197,11 @@ class XunitFormatter(ResultsFormatter): self._handle_exceptional_exit, EventBuilder.STATUS_TIMEOUT: self._handle_timeout - } + } - RESULT_TYPES = {EventBuilder.TYPE_TEST_RESULT, EventBuilder.TYPE_JOB_RESULT} + RESULT_TYPES = { + EventBuilder.TYPE_TEST_RESULT, + EventBuilder.TYPE_JOB_RESULT} def handle_event(self, test_event): super(XunitFormatter, self).handle_event(test_event) |