diff options
Diffstat (limited to 'utils/opt-viewer')
| -rwxr-xr-x | utils/opt-viewer/opt-diff.py | 70 | ||||
| -rwxr-xr-x | utils/opt-viewer/opt-stats.py | 56 | ||||
| -rwxr-xr-x | utils/opt-viewer/opt-viewer.py | 321 | ||||
| -rw-r--r-- | utils/opt-viewer/optrecord.py | 214 | ||||
| -rw-r--r-- | utils/opt-viewer/style.css | 6 | 
5 files changed, 464 insertions, 203 deletions
| diff --git a/utils/opt-viewer/opt-diff.py b/utils/opt-viewer/opt-diff.py new file mode 100755 index 000000000000..8c377860653e --- /dev/null +++ b/utils/opt-viewer/opt-diff.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python2.7 + +from __future__ import print_function + +desc = '''Generate the difference of two YAML files into a new YAML file (works on +pair of directories too).  A new attribute 'Added' is set to True or False +depending whether the entry is added or removed from the first input to the +next. + +The tools requires PyYAML.''' + +import yaml +# Try to use the C parser. +try: +    from yaml import CLoader as Loader +except ImportError: +    from yaml import Loader + +import optrecord +import argparse +from collections import defaultdict +from multiprocessing import cpu_count, Pool +import os, os.path +import fnmatch + +def find_files(dir_or_file): +    if os.path.isfile(dir_or_file): +        return [dir_or_file] + +    all = [] +    for dir, subdirs, files in os.walk(dir_or_file): +        for file in files: +            if fnmatch.fnmatch(file, "*.opt.yaml"): +                all.append( os.path.join(dir, file)) +    return all + +if __name__ == '__main__': +    parser = argparse.ArgumentParser(description=desc) +    parser.add_argument('yaml_dir_or_file_1') +    parser.add_argument('yaml_dir_or_file_2') +    parser.add_argument( +        '--jobs', +        '-j', +        default=cpu_count(), +        type=int, +        help='Max job count (defaults to current CPU count)') +    parser.add_argument('--output', '-o', default='diff.opt.yaml') +    args = parser.parse_args() + +    if args.jobs == 1: +        pmap = map +    else: +        pool = Pool(processes=args.jobs) +        pmap = pool.map + +    files1 = find_files(args.yaml_dir_or_file_1) +    files2 = find_files(args.yaml_dir_or_file_2) + +    all_remarks1, _, _ = optrecord.gather_results(pmap, files1) +    all_remarks2, _, _ = optrecord.gather_results(pmap, files2) + +    added = set(all_remarks2.values()) - set(all_remarks1.values()) +    removed = set(all_remarks1.values()) - set(all_remarks2.values()) + +    for r in added: +        r.Added = True +    for r in removed: +        r.Added = False +    stream = file(args.output, 'w') +    yaml.dump_all(added | removed, stream) diff --git a/utils/opt-viewer/opt-stats.py b/utils/opt-viewer/opt-stats.py new file mode 100755 index 000000000000..b22a052a737a --- /dev/null +++ b/utils/opt-viewer/opt-stats.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python2.7 + +from __future__ import print_function + +desc = '''Generate statistics about optimization records from the YAML files +generated with -fsave-optimization-record and -fdiagnostics-show-hotness. + +The tools requires PyYAML and Pygments Python packages.''' + +import optrecord +import argparse +import operator +from collections import defaultdict +from multiprocessing import cpu_count, Pool + +if __name__ == '__main__': +    parser = argparse.ArgumentParser(description=desc) +    parser.add_argument('yaml_files', nargs='+') +    parser.add_argument( +        '--jobs', +        '-j', +        default=cpu_count(), +        type=int, +        help='Max job count (defaults to current CPU count)') +    args = parser.parse_args() + +    if len(args.yaml_files) == 0: +        parser.print_help() +        sys.exit(1) + +    if args.jobs == 1: +        pmap = map +    else: +        pool = Pool(processes=args.jobs) +        pmap = pool.map + +    all_remarks, file_remarks, _ = optrecord.gather_results(pmap, args.yaml_files) + +    bypass = defaultdict(int) +    byname = defaultdict(int) +    for r in all_remarks.itervalues(): +        bypass[r.Pass] += 1 +        byname[r.Pass + "/" + r.Name] += 1 + +    total = len(all_remarks) +    print("{:24s} {:10d}\n".format("Total number of remarks", total)) + +    print("Top 10 remarks by pass:") +    for (passname, count) in sorted(bypass.items(), key=operator.itemgetter(1), +                                    reverse=True)[:10]: +        print("  {:30s} {:2.0f}%". format(passname, count * 100. / total)) + +    print("\nTop 10 remarks:") +    for (name, count) in sorted(byname.items(), key=operator.itemgetter(1), +                                reverse=True)[:10]: +        print("  {:30s} {:2.0f}%". format(name, count * 100. / total)) diff --git a/utils/opt-viewer/opt-viewer.py b/utils/opt-viewer/opt-viewer.py index c936597475c5..a14aee5f298d 100755 --- a/utils/opt-viewer/opt-viewer.py +++ b/utils/opt-viewer/opt-viewer.py @@ -5,167 +5,41 @@ from __future__ import print_function  desc = '''Generate HTML output to visualize optimization records from the YAML files  generated with -fsave-optimization-record and -fdiagnostics-show-hotness. -The tools requires PyYAML and Pygments Python packages. +The tools requires PyYAML and Pygments Python packages.''' -For faster parsing, you may want to use libYAML with PyYAML.''' - -import yaml -# Try to use the C parser. -try: -    from yaml import CLoader as Loader -except ImportError: -    from yaml import Loader +import optrecord +import functools +from multiprocessing import Pool +from multiprocessing import Lock, cpu_count +import errno  import argparse  import os.path  import re -import subprocess  import shutil  from pygments import highlight  from pygments.lexers.c_cpp import CppLexer  from pygments.formatters import HtmlFormatter +import cgi -parser = argparse.ArgumentParser(description=desc) -parser.add_argument('yaml_files', nargs='+') -parser.add_argument('output_dir') -parser.add_argument('-source-dir', '-s', default='', help='set source directory') -args = parser.parse_args() - -p = subprocess.Popen(['c++filt', '-n'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - -def demangle(name): -    p.stdin.write(name + '\n') -    return p.stdout.readline().rstrip() - - -class Remark(yaml.YAMLObject): -    max_hotness = 0 - -    # Work-around for http://pyyaml.org/ticket/154. -    yaml_loader = Loader - -    @classmethod -    def should_display_hotness(cls): -        # If max_hotness is 0 at the end, we assume hotness information is -        # missing and no relative hotness information is displayed -        return cls.max_hotness != 0 - -    # Map function names to their source location for function where inlining happened -    caller_loc = dict() - -    def __getattr__(self, name): -        # If hotness is missing, assume 0 -        if name == 'Hotness': -            return 0 -        raise AttributeError - -    @property -    def File(self): -        return self.DebugLoc['File'] - -    @property -    def Line(self): -        return int(self.DebugLoc['Line']) - -    @property -    def Column(self): -        return self.DebugLoc['Column'] - -    @property -    def DebugLocString(self): -        return "{}:{}:{}".format(self.File, self.Line, self.Column) - -    @property -    def DemangledFunctionName(self): -        return demangle(self.Function) - -    @classmethod -    def make_link(cls, File, Line): -        return "{}#L{}".format(SourceFileRenderer.html_file_name(File), Line) - -    @property -    def Link(self): -        return Remark.make_link(self.File, self.Line) - -    def getArgString(self, mapping): -        mapping = mapping.copy() -        dl = mapping.get('DebugLoc') -        if dl: -            del mapping['DebugLoc'] - -        assert(len(mapping) == 1) -        (key, value) = mapping.items()[0] - -        if key == 'Caller' or key == 'Callee': -            value = demangle(value) - -        if dl and key != 'Caller': -            return "<a href={}>{}</a>".format( -                Remark.make_link(dl['File'], dl['Line']), value) -        else: -            return value - -    @property -    def message(self): -        # Args is a list of mappings (dictionaries) -        values = [self.getArgString(mapping) for mapping in self.Args] -        return "".join(values) - -    @property -    def RelativeHotness(self): -        if Remark.should_display_hotness(): -            return "{}%".format(int(round(self.Hotness * 100 / Remark.max_hotness))) -        else: -            return '' - -    @property -    def key(self): -        return (self.__class__, self.Pass, self.Name, self.File, self.Line, self.Column, self.message) - - -class Analysis(Remark): -    yaml_tag = '!Analysis' - -    @property -    def color(self): -        return "white" - - -class AnalysisFPCommute(Analysis): -    yaml_tag = '!AnalysisFPCommute' - - -class AnalysisAliasing(Analysis): -    yaml_tag = '!AnalysisAliasing' - - -class Passed(Remark): -    yaml_tag = '!Passed' - -    @property -    def color(self): -        return "green" - - -class Missed(Remark): -    yaml_tag = '!Missed' - -    @property -    def color(self): -        return "red" +# This allows passing the global context to the child processes. +class Context: +    def __init__(self, caller_loc = dict()): +       # Map function names to their source location for function where inlining happened +       self.caller_loc = caller_loc +context = Context()  class SourceFileRenderer: -    def __init__(self, filename): +    def __init__(self, source_dir, output_dir, filename):          existing_filename = None          if os.path.exists(filename):              existing_filename = filename          else: -            fn = os.path.join(args.source_dir, filename) +            fn = os.path.join(source_dir, filename)              if os.path.exists(fn):                  existing_filename = fn -        self.stream = open(os.path.join(args.output_dir, SourceFileRenderer.html_file_name(filename)), 'w') +        self.stream = open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w')          if existing_filename:              self.source_stream = open(existing_filename)          else: @@ -176,35 +50,47 @@ class SourceFileRenderer:  </html>              '''.format(filename), file=self.stream) -        self.html_formatter = HtmlFormatter() -        self.cpp_lexer = CppLexer() +        self.html_formatter = HtmlFormatter(encoding='utf-8') +        self.cpp_lexer = CppLexer(stripnl=False) -    def render_source_line(self, linenum, line): -        html_line = highlight(line, self.cpp_lexer, self.html_formatter) -        print(''' +    def render_source_lines(self, stream, line_remarks): +        file_text = stream.read() +        html_highlighted = highlight(file_text, self.cpp_lexer, self.html_formatter) + +        # Take off the header and footer, these must be +        #   reapplied line-wise, within the page structure +        html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '') +        html_highlighted = html_highlighted.replace('</pre></div>', '') + +        for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1): +            print('''  <tr>  <td><a name=\"L{linenum}\">{linenum}</a></td>  <td></td>  <td></td> -<td>{html_line}</td> +<td><div class="highlight"><pre>{html_line}</pre></div></td>  </tr>'''.format(**locals()), file=self.stream) +            for remark in line_remarks.get(linenum, []): +                self.render_inline_remarks(remark, html_line) +      def render_inline_remarks(self, r, line):          inlining_context = r.DemangledFunctionName -        dl = Remark.caller_loc.get(r.Function) +        dl = context.caller_loc.get(r.Function)          if dl: -            link = Remark.make_link(dl['File'], dl['Line'] - 2) +            link = optrecord.make_link(dl['File'], dl['Line'] - 2)              inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())          # Column is the number of characters *including* tabs, keep those and          # replace everything else with spaces. -        indent = line[:r.Column - 1] +        indent = line[:max(r.Column, 1) - 1]          indent = re.sub('\S', ' ', indent) +          print('''  <tr>  <td></td>  <td>{r.RelativeHotness}</td> -<td class=\"column-entry-{r.color}\">{r.Pass}</td> +<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>  <td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\"> {r.message} </span></td>  <td class=\"column-entry-yellow\">{inlining_context}</td>  </tr>'''.format(**locals()), file=self.stream) @@ -228,31 +114,26 @@ class SourceFileRenderer:  <td>Source</td>  <td>Inline Context</td>  </tr>''', file=self.stream) -        for (linenum, line) in enumerate(self.source_stream.readlines(), start=1): -            self.render_source_line(linenum, line) -            for remark in line_remarks.get(linenum, []): -                self.render_inline_remarks(remark, line) +        self.render_source_lines(self.source_stream, line_remarks) +          print('''  </table>  </body>  </html>''', file=self.stream) -    @classmethod -    def html_file_name(cls, filename): -        return filename.replace('/', '_') + ".html" -  class IndexRenderer: -    def __init__(self): -        self.stream = open(os.path.join(args.output_dir, 'index.html'), 'w') +    def __init__(self, output_dir): +        self.stream = open(os.path.join(output_dir, 'index.html'), 'w') -    def render_entry(self, r): +    def render_entry(self, r, odd): +        escaped_name = cgi.escape(r.DemangledFunctionName)          print('''  <tr> -<td><a href={r.Link}>{r.DebugLocString}</a></td> -<td>{r.RelativeHotness}</td> -<td>{r.DemangledFunctionName}</td> -<td class=\"column-entry-{r.color}\">{r.Pass}</td> +<td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td> +<td class=\"column-entry-{odd}\">{r.RelativeHotness}</td> +<td class=\"column-entry-{odd}\">{escaped_name}</td> +<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>  </tr>'''.format(**locals()), file=self.stream)      def render(self, all_remarks): @@ -270,49 +151,83 @@ class IndexRenderer:  <td>Function</td>  <td>Pass</td>  </tr>''', file=self.stream) -        for remark in all_remarks: -            self.render_entry(remark) +        for i, remark in enumerate(all_remarks): +            self.render_entry(remark, i % 2)          print('''  </table>  </body>  </html>''', file=self.stream) -all_remarks = dict() -file_remarks = dict() +def _render_file(source_dir, output_dir, ctx, entry): +    global context +    context = ctx +    filename, remarks = entry +    SourceFileRenderer(source_dir, output_dir, filename).render(remarks) -for input_file in args.yaml_files: -    f = open(input_file) -    docs = yaml.load_all(f, Loader=Loader) -    for remark in docs: -        # Avoid remarks withoug debug location or if they are duplicated -        if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: -            continue -        all_remarks[remark.key] = remark -        file_remarks.setdefault(remark.File, dict()).setdefault(remark.Line, []).append(remark) +def map_remarks(all_remarks): +    # Set up a map between function names and their source location for +    # function where inlining happened +    for remark in all_remarks.itervalues(): +        if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined": +            for arg in remark.Args: +                caller = arg.get('Caller') +                if caller: +                    context.caller_loc[caller] = arg['DebugLoc'] -        Remark.max_hotness = max(Remark.max_hotness, remark.Hotness) -# Set up a map between function names and their source location for function where inlining happened -for remark in all_remarks.itervalues(): -    if type(remark) == Passed and remark.Pass == "inline" and remark.Name == "Inlined": -        for arg in remark.Args: -            caller = arg.get('Caller') -            if caller: -                    Remark.caller_loc[caller] = arg['DebugLoc'] - -if Remark.should_display_hotness(): -    sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: r.Hotness, reverse=True) -else: -    sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: (r.File, r.Line, r.Column)) - -if not os.path.exists(args.output_dir): -    os.mkdir(args.output_dir) - -for (filename, remarks) in file_remarks.iteritems(): -    SourceFileRenderer(filename).render(remarks) - -IndexRenderer().render(sorted_remarks) - -shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), "style.css"), args.output_dir) +def generate_report(pmap, all_remarks, file_remarks, source_dir, output_dir, should_display_hotness): +    try: +        os.makedirs(output_dir) +    except OSError as e: +        if e.errno == errno.EEXIST and os.path.isdir(output_dir): +            pass +        else: +            raise + +    _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context) +    pmap(_render_file_bound, file_remarks.items()) + +    if should_display_hotness: +        sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.__dict__), reverse=True) +    else: +        sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: (r.File, r.Line, r.Column, r.__dict__)) +    IndexRenderer(args.output_dir).render(sorted_remarks) + +    shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), +            "style.css"), output_dir) + + +if __name__ == '__main__': +    parser = argparse.ArgumentParser(description=desc) +    parser.add_argument('yaml_files', nargs='+') +    parser.add_argument('output_dir') +    parser.add_argument( +        '--jobs', +        '-j', +        default=cpu_count(), +        type=int, +        help='Max job count (defaults to current CPU count)') +    parser.add_argument( +        '-source-dir', +        '-s', +        default='', +        help='set source directory') +    args = parser.parse_args() + +    if len(args.yaml_files) == 0: +        parser.print_help() +        sys.exit(1) + +    if args.jobs == 1: +        pmap = map +    else: +        pool = Pool(processes=args.jobs) +        pmap = pool.map + +    all_remarks, file_remarks, should_display_hotness = optrecord.gather_results(pmap, args.yaml_files) + +    map_remarks(all_remarks) + +    generate_report(pmap, all_remarks, file_remarks, args.source_dir, args.output_dir, should_display_hotness) diff --git a/utils/opt-viewer/optrecord.py b/utils/opt-viewer/optrecord.py new file mode 100644 index 000000000000..3dc77e9db019 --- /dev/null +++ b/utils/opt-viewer/optrecord.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python2.7 + +from __future__ import print_function + +import yaml +# Try to use the C parser. +try: +    from yaml import CLoader as Loader +except ImportError: +    print("For faster parsing, you may want to install libYAML for PyYAML") +    from yaml import Loader + +import functools +from collections import defaultdict +import itertools +from multiprocessing import Pool +from multiprocessing import Lock, cpu_count +import cgi +import subprocess + +import traceback + +p = subprocess.Popen(['c++filt', '-n'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) +p_lock = Lock() + + +def demangle(name): +    with p_lock: +        p.stdin.write(name + '\n') +        return p.stdout.readline().rstrip() + +def html_file_name(filename): +    return filename.replace('/', '_') + ".html" + +def make_link(File, Line): +    return "{}#L{}".format(html_file_name(File), Line) + + +class Remark(yaml.YAMLObject): +    # Work-around for http://pyyaml.org/ticket/154. +    yaml_loader = Loader + +    def initmissing(self): +        if not hasattr(self, 'Hotness'): +            self.Hotness = 0 +        if not hasattr(self, 'Args'): +            self.Args = [] + +    @property +    def File(self): +        return self.DebugLoc['File'] + +    @property +    def Line(self): +        return int(self.DebugLoc['Line']) + +    @property +    def Column(self): +        return self.DebugLoc['Column'] + +    @property +    def DebugLocString(self): +        return "{}:{}:{}".format(self.File, self.Line, self.Column) + +    @property +    def DemangledFunctionName(self): +        return demangle(self.Function) + +    @property +    def Link(self): +        return make_link(self.File, self.Line) + +    def getArgString(self, mapping): +        mapping = mapping.copy() +        dl = mapping.get('DebugLoc') +        if dl: +            del mapping['DebugLoc'] + +        assert(len(mapping) == 1) +        (key, value) = mapping.items()[0] + +        if key == 'Caller' or key == 'Callee': +            value = cgi.escape(demangle(value)) + +        if dl and key != 'Caller': +            return "<a href={}>{}</a>".format( +                make_link(dl['File'], dl['Line']), value) +        else: +            return value + +    def getDiffPrefix(self): +        if hasattr(self, 'Added'): +            if self.Added: +                return '+' +            else: +                return '-' +        return '' + +    @property +    def PassWithDiffPrefix(self): +        return self.getDiffPrefix() + self.Pass + +    @property +    def message(self): +        # Args is a list of mappings (dictionaries) +        values = [self.getArgString(mapping) for mapping in self.Args] +        return "".join(values) + +    @property +    def RelativeHotness(self): +        if self.max_hotness: +            return "{}%".format(int(round(self.Hotness * 100 / self.max_hotness))) +        else: +            return '' + +    @property +    def key(self): +        k = (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, self.Line, self.Column, self.Function) +        for arg in self.Args: +            for (key, value) in arg.iteritems(): +                if type(value) is dict: +                    value = tuple(value.items()) +                k += (key, value) +        return k + +    def __hash__(self): +        return hash(self.key) + +    def __eq__(self, other): +        return self.key == other.key + +    def __repr__(self): +        return str(self.key) + + +class Analysis(Remark): +    yaml_tag = '!Analysis' + +    @property +    def color(self): +        return "white" + + +class AnalysisFPCommute(Analysis): +    yaml_tag = '!AnalysisFPCommute' + + +class AnalysisAliasing(Analysis): +    yaml_tag = '!AnalysisAliasing' + + +class Passed(Remark): +    yaml_tag = '!Passed' + +    @property +    def color(self): +        return "green" + + +class Missed(Remark): +    yaml_tag = '!Missed' + +    @property +    def color(self): +        return "red" + + +def get_remarks(input_file): +    max_hotness = 0 +    all_remarks = dict() +    file_remarks = defaultdict(functools.partial(defaultdict, list)) + +    with open(input_file) as f: +        docs = yaml.load_all(f, Loader=Loader) +        for remark in docs: +            remark.initmissing() +            # Avoid remarks withoug debug location or if they are duplicated +            if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: +                continue +            all_remarks[remark.key] = remark + +            file_remarks[remark.File][remark.Line].append(remark) + +            # If we're reading a back a diff yaml file, max_hotness is already +            # captured which may actually be less than the max hotness found +            # in the file. +            if hasattr(remark, 'max_hotness'): +                max_hotness = remark.max_hotness +            max_hotness = max(max_hotness, remark.Hotness) + +    return max_hotness, all_remarks, file_remarks + + +def gather_results(pmap, filenames): +    remarks = pmap(get_remarks, filenames) +    max_hotness = max(entry[0] for entry in remarks) + +    def merge_file_remarks(file_remarks_job, all_remarks, merged): +        for filename, d in file_remarks_job.iteritems(): +            for line, remarks in d.iteritems(): +                for remark in remarks: +                    # Bring max_hotness into the remarks so that +                    # RelativeHotness does not depend on an external global. +                    remark.max_hotness = max_hotness +                    if remark.key not in all_remarks: +                        merged[filename][line].append(remark) + +    all_remarks = dict() +    file_remarks = defaultdict(functools.partial(defaultdict, list)) +    for _, all_remarks_job, file_remarks_job in remarks: +        merge_file_remarks(file_remarks_job, all_remarks, file_remarks) +        all_remarks.update(all_remarks_job) + +    return all_remarks, file_remarks, max_hotness != 0 diff --git a/utils/opt-viewer/style.css b/utils/opt-viewer/style.css index 2ef244a15718..595c3e46847d 100644 --- a/utils/opt-viewer/style.css +++ b/utils/opt-viewer/style.css @@ -62,6 +62,12 @@ table {    text-align: left;    background-color: #ffe1a6;  } +.column-entry-0 { +  background-color: #ffffff; +} +.column-entry-1 { +  background-color: #eeeeee; +}  .line-number {    text-align: right;    color: #aaa; | 
