diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2017-01-02 19:26:05 +0000 | 
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2017-01-02 19:26:05 +0000 | 
| commit | 14f1b3e8826ce43b978db93a62d1166055db5394 (patch) | |
| tree | 0a00ad8d3498783fe0193f3b656bca17c4c8697d /packages/Python/lldbsuite/test_event/formatter | |
| parent | 4ee8c119c71a06dcad1e0fecc8c675e480e59337 (diff) | |
Notes
Diffstat (limited to 'packages/Python/lldbsuite/test_event/formatter')
5 files changed, 266 insertions, 70 deletions
diff --git a/packages/Python/lldbsuite/test_event/formatter/__init__.py b/packages/Python/lldbsuite/test_event/formatter/__init__.py index 556370ebb9de..2481e326e946 100644 --- a/packages/Python/lldbsuite/test_event/formatter/__init__.py +++ b/packages/Python/lldbsuite/test_event/formatter/__init__.py @@ -76,7 +76,8 @@ def create_results_formatter(config):          # we lose the test result info.          read_bytes = sock.recv(1)          if read_bytes is None or (len(read_bytes) < 1) or (read_bytes != b'*'): -            raise Exception("listening socket did not respond with ack byte: response={}".format(read_bytes)) +            raise Exception( +                "listening socket did not respond with ack byte: response={}".format(read_bytes))          return sock, lambda: socket_closer(sock) diff --git a/packages/Python/lldbsuite/test_event/formatter/curses.py b/packages/Python/lldbsuite/test_event/formatter/curses.py index 4e89fa9bf01e..f415575ded8a 100644 --- a/packages/Python/lldbsuite/test_event/formatter/curses.py +++ b/packages/Python/lldbsuite/test_event/formatter/curses.py @@ -38,7 +38,10 @@ class Curses(results_formatter.ResultsFormatter):          self.results = list()          try:              self.main_window = lldbcurses.intialize_curses() -            self.main_window.add_key_action('\t', self.main_window.select_next_first_responder, "Switch between views that can respond to keyboard input") +            self.main_window.add_key_action( +                '\t', +                self.main_window.select_next_first_responder, +                "Switch between views that can respond to keyboard input")              self.main_window.refresh()              self.job_panel = None              self.results_panel = None @@ -87,10 +90,15 @@ class Curses(results_formatter.ResultsFormatter):          selected_idx = self.results_panel.get_selected_idx()          if selected_idx >= 0 and selected_idx < len(self.results):              if self.info_panel is None: -                info_frame = self.results_panel.get_contained_rect(top_inset=10, left_inset=10, right_inset=10, height=30) -                self.info_panel = lldbcurses.BoxedPanel(info_frame, "Result Details") -                # Add a key action for any key that will hide this panel when any key is pressed -                self.info_panel.add_key_action(-1, self.hide_info_panel, 'Hide the info panel') +                info_frame = self.results_panel.get_contained_rect( +                    top_inset=10, left_inset=10, right_inset=10, height=30) +                self.info_panel = lldbcurses.BoxedPanel( +                    info_frame, "Result Details") +                # Add a key action for any key that will hide this panel when +                # any key is pressed +                self.info_panel.add_key_action(-1, +                                               self.hide_info_panel, +                                               'Hide the info panel')                  self.info_panel.top()              else:                  self.info_panel.show() @@ -98,9 +106,15 @@ class Curses(results_formatter.ResultsFormatter):              self.main_window.push_first_responder(self.info_panel)              test_start = self.results[selected_idx][0]              test_result = self.results[selected_idx][1] -            self.info_panel.set_line(0, "File: %s" % (test_start['test_filename'])) -            self.info_panel.set_line(1, "Test: %s.%s" % (test_start['test_class'], test_start['test_name'])) -            self.info_panel.set_line(2, "Time: %s" % (test_result['elapsed_time'])) +            self.info_panel.set_line( +                0, "File: %s" % +                (test_start['test_filename'])) +            self.info_panel.set_line( +                1, "Test: %s.%s" % +                (test_start['test_class'], test_start['test_name'])) +            self.info_panel.set_line( +                2, "Time: %s" % +                (test_result['elapsed_time']))              self.info_panel.set_line(3, "Status: %s" % (test_result['status']))      def hide_info_panel(self): @@ -110,7 +124,8 @@ class Curses(results_formatter.ResultsFormatter):      def toggle_status(self, status):          if status: -            # Toggle showing and hiding results whose status matches "status" in "Results" window +            # Toggle showing and hiding results whose status matches "status" +            # in "Results" window              if status in self.hide_status_list:                  self.hide_status_list.remove(status)              else: @@ -127,7 +142,13 @@ class Curses(results_formatter.ResultsFormatter):              if status in self.hide_status_list:                  continue              name = test_result['test_class'] + '.' + test_result['test_name'] -            self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status, test_result), test_result['elapsed_time'], name)) +            self.results_panel.append_line( +                '%s (%6.2f sec) %s' % +                (self.status_to_short_str( +                    status, +                    test_result), +                    test_result['elapsed_time'], +                    name))          if update:              self.main_window.refresh() @@ -145,9 +166,14 @@ class Curses(results_formatter.ResultsFormatter):                      #print(str(test_event), file=self.events_file)                      event = test_event['event']                      if self.status_panel: -                        self.status_panel.update_status('time', str(datetime.timedelta(seconds=math.floor(time.time() - self.start_time)))) +                        self.status_panel.update_status( +                            'time', str( +                                datetime.timedelta( +                                    seconds=math.floor( +                                        time.time() - self.start_time))))                      if event == 'test_start': -                        name = test_event['test_class'] + '.' + test_event['test_name'] +                        name = test_event['test_class'] + \ +                            '.' + test_event['test_name']                          self.job_tests[worker_index] = test_event                          if 'pid' in test_event:                              line = 'pid: %5d ' % (test_event['pid']) + name @@ -163,14 +189,20 @@ class Curses(results_formatter.ResultsFormatter):                          else:                              line = ''                          self.job_panel.set_line(worker_index, line) -                        name = test_event['test_class'] + '.' + test_event['test_name'] -                        elapsed_time = test_event['event_time'] - self.job_tests[worker_index]['event_time'] -                        if not status in self.hide_status_list: -                            self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status, test_event), elapsed_time, name)) +                        name = test_event['test_class'] + \ +                            '.' + test_event['test_name'] +                        elapsed_time = test_event[ +                            'event_time'] - self.job_tests[worker_index]['event_time'] +                        if status not in self.hide_status_list: +                            self.results_panel.append_line( +                                '%s (%6.2f sec) %s' % +                                (self.status_to_short_str( +                                    status, test_event), elapsed_time, name))                          self.main_window.refresh()                          # Append the result pairs                          test_event['elapsed_time'] = elapsed_time -                        self.results.append([self.job_tests[worker_index], test_event]) +                        self.results.append( +                            [self.job_tests[worker_index], test_event])                          self.job_tests[worker_index] = ''                      elif event == 'job_begin':                          self.jobs[worker_index] = test_event @@ -185,40 +217,121 @@ class Curses(results_formatter.ResultsFormatter):                      elif event == 'initialize':                          self.initialize_event = test_event                          num_jobs = test_event['worker_count'] -                        job_frame = self.main_window.get_contained_rect(height=num_jobs+2) -                        results_frame = self.main_window.get_contained_rect(top_inset=num_jobs+2, bottom_inset=1) -                        status_frame = self.main_window.get_contained_rect(height=1, top_inset=self.main_window.get_size().h-1) -                        self.job_panel = lldbcurses.BoxedPanel(frame=job_frame, title="Jobs") -                        self.results_panel = lldbcurses.BoxedPanel(frame=results_frame, title="Results") - -                        self.results_panel.add_key_action(curses.KEY_UP,    self.results_panel.select_prev      , "Select the previous list entry") -                        self.results_panel.add_key_action(curses.KEY_DOWN,  self.results_panel.select_next      , "Select the next list entry") -                        self.results_panel.add_key_action(curses.KEY_HOME,  self.results_panel.scroll_begin     , "Scroll to the start of the list") -                        self.results_panel.add_key_action(curses.KEY_END,   self.results_panel.scroll_end       , "Scroll to the end of the list") -                        self.results_panel.add_key_action(curses.KEY_ENTER, self.show_info_panel                , "Display info for the selected result item") -                        self.results_panel.add_key_action('.', lambda : self.toggle_status(EventBuilder.STATUS_SUCCESS)           , "Toggle showing/hiding tests whose status is 'success'") -                        self.results_panel.add_key_action('e', lambda : self.toggle_status(EventBuilder.STATUS_ERROR)             , "Toggle showing/hiding tests whose status is 'error'") -                        self.results_panel.add_key_action('f', lambda : self.toggle_status(EventBuilder.STATUS_FAILURE)           , "Toggle showing/hiding tests whose status is 'failure'") -                        self.results_panel.add_key_action('s', lambda : self.toggle_status(EventBuilder.STATUS_SKIP)              , "Toggle showing/hiding tests whose status is 'skip'") -                        self.results_panel.add_key_action('x', lambda : self.toggle_status(EventBuilder.STATUS_EXPECTED_FAILURE)  , "Toggle showing/hiding tests whose status is 'expected_failure'") -                        self.results_panel.add_key_action('?', lambda : self.toggle_status(EventBuilder.STATUS_UNEXPECTED_SUCCESS), "Toggle showing/hiding tests whose status is 'unexpected_success'") -                        self.status_panel = lldbcurses.StatusPanel(frame=status_frame) +                        job_frame = self.main_window.get_contained_rect( +                            height=num_jobs + 2) +                        results_frame = self.main_window.get_contained_rect( +                            top_inset=num_jobs + 2, bottom_inset=1) +                        status_frame = self.main_window.get_contained_rect( +                            height=1, top_inset=self.main_window.get_size().h - 1) +                        self.job_panel = lldbcurses.BoxedPanel( +                            frame=job_frame, title="Jobs") +                        self.results_panel = lldbcurses.BoxedPanel( +                            frame=results_frame, title="Results") + +                        self.results_panel.add_key_action( +                            curses.KEY_UP, +                            self.results_panel.select_prev, +                            "Select the previous list entry") +                        self.results_panel.add_key_action( +                            curses.KEY_DOWN, self.results_panel.select_next, "Select the next list entry") +                        self.results_panel.add_key_action( +                            curses.KEY_HOME, +                            self.results_panel.scroll_begin, +                            "Scroll to the start of the list") +                        self.results_panel.add_key_action( +                            curses.KEY_END, self.results_panel.scroll_end, "Scroll to the end of the list") +                        self.results_panel.add_key_action( +                            curses.KEY_ENTER, +                            self.show_info_panel, +                            "Display info for the selected result item") +                        self.results_panel.add_key_action( +                            '.', +                            lambda: self.toggle_status( +                                EventBuilder.STATUS_SUCCESS), +                            "Toggle showing/hiding tests whose status is 'success'") +                        self.results_panel.add_key_action( +                            'e', +                            lambda: self.toggle_status( +                                EventBuilder.STATUS_ERROR), +                            "Toggle showing/hiding tests whose status is 'error'") +                        self.results_panel.add_key_action( +                            'f', +                            lambda: self.toggle_status( +                                EventBuilder.STATUS_FAILURE), +                            "Toggle showing/hiding tests whose status is 'failure'") +                        self.results_panel.add_key_action('s', lambda: self.toggle_status( +                            EventBuilder.STATUS_SKIP), "Toggle showing/hiding tests whose status is 'skip'") +                        self.results_panel.add_key_action( +                            'x', +                            lambda: self.toggle_status( +                                EventBuilder.STATUS_EXPECTED_FAILURE), +                            "Toggle showing/hiding tests whose status is 'expected_failure'") +                        self.results_panel.add_key_action( +                            '?', +                            lambda: self.toggle_status( +                                EventBuilder.STATUS_UNEXPECTED_SUCCESS), +                            "Toggle showing/hiding tests whose status is 'unexpected_success'") +                        self.status_panel = lldbcurses.StatusPanel( +                            frame=status_frame)                          self.main_window.add_child(self.job_panel)                          self.main_window.add_child(self.results_panel)                          self.main_window.add_child(self.status_panel) -                        self.main_window.set_first_responder(self.results_panel) - -                        self.status_panel.add_status_item(name="time", title="Elapsed", format="%s", width=20, value="0:00:00", update=False) -                        self.status_panel.add_status_item(name=EventBuilder.STATUS_SUCCESS, title="Success", format="%u", width=20, value=0, update=False) -                        self.status_panel.add_status_item(name=EventBuilder.STATUS_FAILURE, title="Failure", format="%u", width=20, value=0, update=False) -                        self.status_panel.add_status_item(name=EventBuilder.STATUS_ERROR, title="Error", format="%u", width=20, value=0, update=False) -                        self.status_panel.add_status_item(name=EventBuilder.STATUS_SKIP, title="Skipped", format="%u", width=20, value=0, update=True) -                        self.status_panel.add_status_item(name=EventBuilder.STATUS_EXPECTED_FAILURE, title="Expected Failure", format="%u", width=30, value=0, update=False) -                        self.status_panel.add_status_item(name=EventBuilder.STATUS_UNEXPECTED_SUCCESS, title="Unexpected Success", format="%u", width=30, value=0, update=False) +                        self.main_window.set_first_responder( +                            self.results_panel) + +                        self.status_panel.add_status_item( +                            name="time", +                            title="Elapsed", +                            format="%s", +                            width=20, +                            value="0:00:00", +                            update=False) +                        self.status_panel.add_status_item( +                            name=EventBuilder.STATUS_SUCCESS, +                            title="Success", +                            format="%u", +                            width=20, +                            value=0, +                            update=False) +                        self.status_panel.add_status_item( +                            name=EventBuilder.STATUS_FAILURE, +                            title="Failure", +                            format="%u", +                            width=20, +                            value=0, +                            update=False) +                        self.status_panel.add_status_item( +                            name=EventBuilder.STATUS_ERROR, +                            title="Error", +                            format="%u", +                            width=20, +                            value=0, +                            update=False) +                        self.status_panel.add_status_item( +                            name=EventBuilder.STATUS_SKIP, +                            title="Skipped", +                            format="%u", +                            width=20, +                            value=0, +                            update=True) +                        self.status_panel.add_status_item( +                            name=EventBuilder.STATUS_EXPECTED_FAILURE, +                            title="Expected Failure", +                            format="%u", +                            width=30, +                            value=0, +                            update=False) +                        self.status_panel.add_status_item( +                            name=EventBuilder.STATUS_UNEXPECTED_SUCCESS, +                            title="Unexpected Success", +                            format="%u", +                            width=30, +                            value=0, +                            update=False)                          self.main_window.refresh()                      elif event == 'terminate': -                        #self.main_window.key_event_loop() +                        # self.main_window.key_event_loop()                          lldbcurses.terminate_curses()                          check_for_one_key = False                          self.using_terminal = False diff --git a/packages/Python/lldbsuite/test_event/formatter/pickled.py b/packages/Python/lldbsuite/test_event/formatter/pickled.py index 6d800f6c8baa..588614e2f7b4 100644 --- a/packages/Python/lldbsuite/test_event/formatter/pickled.py +++ b/packages/Python/lldbsuite/test_event/formatter/pickled.py @@ -31,21 +31,29 @@ class RawPickledFormatter(ResultsFormatter):          return parser      class StreamSerializer(object): +          @staticmethod          def serialize(test_event, out_file): -            # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes} +            # Send it as +            # {serialized_length_of_serialized_bytes}{serialized_bytes}              import struct              msg = cPickle.dumps(test_event)              packet = struct.pack("!I%ds" % len(msg), len(msg), msg)              out_file.send(packet)      class BlockSerializer(object): +          @staticmethod          def serialize(test_event, out_file):              cPickle.dump(test_event, out_file)      def __init__(self, out_file, options, file_is_stream): -        super(RawPickledFormatter, self).__init__(out_file, options, file_is_stream) +        super( +            RawPickledFormatter, +            self).__init__( +            out_file, +            options, +            file_is_stream)          self.pid = os.getpid()          if file_is_stream:              self.serializer = self.StreamSerializer() diff --git a/packages/Python/lldbsuite/test_event/formatter/results_formatter.py b/packages/Python/lldbsuite/test_event/formatter/results_formatter.py index 3bf389b97266..35a9fa8d5d56 100644 --- a/packages/Python/lldbsuite/test_event/formatter/results_formatter.py +++ b/packages/Python/lldbsuite/test_event/formatter/results_formatter.py @@ -14,6 +14,7 @@ from __future__ import absolute_import  # System modules  import argparse  import os +import re  import sys  import threading @@ -27,6 +28,9 @@ from ..event_builder import EventBuilder  import lldbsuite +FILE_LEVEL_KEY_RE = re.compile(r"^(.+\.py)[^.:]*$") + +  class ResultsFormatter(object):      """Provides interface to formatting test results out to a file-like object. @@ -207,6 +211,26 @@ class ResultsFormatter(object):              component_count += 1          return key +    @classmethod +    def _is_file_level_issue(cls, key, event): +        """Returns whether a given key represents a file-level event. + +        @param cls this class.  Unused, but following PEP8 for +        preferring @classmethod over @staticmethod. + +        @param key the key for the issue being tested. + +        @param event the event for the issue being tested. + +        @return True when the given key (as made by _make_key()) +        represents an event that is at the test file level (i.e. +        it isn't scoped to a test class or method). +        """ +        if key is None: +            return False +        else: +            return FILE_LEVEL_KEY_RE.match(key) is not None +      def _mark_test_as_expected_failure(self, test_result_event):          key = self._make_key(test_result_event)          if key is not None: @@ -321,8 +345,8 @@ class ResultsFormatter(object):                  # after this check below since this check may rewrite                  # the event type                  if event_type == EventBuilder.TYPE_JOB_RESULT: -                    # Possibly convert the job status (timeout, exceptional exit) -                    # to an appropriate test_result event. +                    # Possibly convert the job status (timeout, +                    # exceptional exit) # to an appropriate test_result event.                      self._maybe_remap_job_result_event(test_event)                      event_type = test_event.get("event", "") @@ -335,21 +359,22 @@ class ResultsFormatter(object):                  if event_type == "terminate":                      self.terminate_called = True                  elif event_type in EventBuilder.RESULT_TYPES: -                    # Keep track of event counts per test/job result status type. -                    # The only job (i.e. inferior process) results that make it -                    # here are ones that cannot be remapped to the most recently -                    # started test for the given worker index. +                    # Keep track of event counts per test/job result status +                    # type. The only job (i.e. inferior process) results that +                    # make it here are ones that cannot be remapped to the most +                    # recently started test for the given worker index.                      status = test_event["status"]                      self.result_status_counts[status] += 1 -                    # Clear the most recently started test for the related worker. +                    # Clear the most recently started test for the related +                    # worker.                      worker_index = test_event.get("worker_index", None)                      if worker_index is not None:                          self.started_tests_by_worker.pop(worker_index, None)                      if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES:                          # A test/job status value in any of those status values -                        # causes a testrun failure.  If such a test fails, check -                        # whether it can be rerun.  If it can be rerun, add it +                        # causes a testrun failure. If such a test fails, check +                        # whether it can be rerun. If it can be rerun, add it                          # to the rerun job.                          self._maybe_add_test_to_rerun_list(test_event) @@ -360,14 +385,13 @@ class ResultsFormatter(object):                              "failed to find test filename for "                              "test event {}".format(test_event)) -                    # Save the most recent test event for the test key. -                    # This allows a second test phase to overwrite the most -                    # recent result for the test key (unique per method). -                    # We do final reporting at the end, so we'll report based -                    # on final results. -                    # We do this so that a re-run caused by, perhaps, the need -                    # to run a low-load, single-worker test run can have the final -                    # run's results to always be used. +                    # Save the most recent test event for the test key. This +                    # allows a second test phase to overwrite the most recent +                    # result for the test key (unique per method). We do final +                    # reporting at the end, so we'll report based on final +                    # results. We do this so that a re-run caused by, perhaps, +                    # the need to run a low-load, single-worker test run can +                    # have the final run's results to always be used.                      if test_key in self.result_events:                          # We are replacing the result of something that was                          # already counted by the base class.  Remove the double @@ -393,7 +417,8 @@ class ResultsFormatter(object):                  elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE:                      self._mark_test_for_rerun_eligibility(test_event) -                elif event_type == EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE: +                elif (event_type == +                      EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE):                      self._mark_test_as_expected_failure(test_event)      def set_expected_timeouts_by_basename(self, basenames): @@ -688,7 +713,7 @@ class ResultsFormatter(object):               # prevent buildbots from thinking it is an issue when scanning               # for TIMEOUT.               "Expected Timeout", True, "EXPECTED TIME-OUT"] -            ] +        ]          # Partition all the events by test result status          result_events_by_status = self._partition_results_by_status( @@ -715,3 +740,50 @@ class ResultsFormatter(object):                  for key, event in events_by_key:                      out_file.write("key:   {}\n".format(key))                      out_file.write("event: {}\n".format(event)) + +    def clear_file_level_issues(self, tests_for_rerun, out_file): +        """Clear file-charged issues in any of the test rerun files. + +        @param tests_for_rerun the list of test-dir-relative paths that have +        functions that require rerunning.  This is the test list +        returned by the results_formatter at the end of the previous run. + +        @return the number of file-level issues that were cleared. +        """ +        if tests_for_rerun is None: +            return 0 + +        cleared_file_level_issues = 0 +        # Find the unique set of files that are covered by the given tests +        # that are to be rerun.  We derive the files that are eligible for +        # having their markers cleared, because we support running in a mode +        # where only flaky tests are eligible for rerun.  If the file-level +        # issue occurred in a file that was not marked as flaky, then we +        # shouldn't be clearing the event here. +        basename_set = set() +        for test_file_relpath in tests_for_rerun: +            basename_set.add(os.path.basename(test_file_relpath)) + +        # Find all the keys for file-level events that are considered +        # test issues. +        file_level_issues = [(key, event) +                             for key, event in self.result_events.items() +                             if ResultsFormatter._is_file_level_issue( +                                     key, event) +                             and event.get("status", "") in +                             EventBuilder.TESTRUN_ERROR_STATUS_VALUES] + +        # Now remove any file-level error for the given test base name. +        for key, event in file_level_issues: +            # If the given file base name is in the rerun set, then we +            # clear that entry from the result set. +            if os.path.basename(key) in basename_set: +                self.result_events.pop(key, None) +                cleared_file_level_issues += 1 +                if out_file is not None: +                    out_file.write( +                        "clearing file-level issue for file {} " +                        "(issue type: {})" +                        .format(key, event.get("status", "<unset-status>"))) + +        return cleared_file_level_issues diff --git a/packages/Python/lldbsuite/test_event/formatter/xunit.py b/packages/Python/lldbsuite/test_event/formatter/xunit.py index b3682f8fb107..d3ea8677f0a6 100644 --- a/packages/Python/lldbsuite/test_event/formatter/xunit.py +++ b/packages/Python/lldbsuite/test_event/formatter/xunit.py @@ -180,7 +180,7 @@ class XunitFormatter(ResultsFormatter):              "unexpected_successes": [],              "expected_failures": [],              "all": [] -            } +        }          self.status_handlers = {              EventBuilder.STATUS_SUCCESS: self._handle_success, @@ -197,9 +197,11 @@ class XunitFormatter(ResultsFormatter):                  self._handle_exceptional_exit,              EventBuilder.STATUS_TIMEOUT:                  self._handle_timeout -            } +        } -    RESULT_TYPES = {EventBuilder.TYPE_TEST_RESULT, EventBuilder.TYPE_JOB_RESULT} +    RESULT_TYPES = { +        EventBuilder.TYPE_TEST_RESULT, +        EventBuilder.TYPE_JOB_RESULT}      def handle_event(self, test_event):          super(XunitFormatter, self).handle_event(test_event)  | 
