diff options
Diffstat (limited to 'utils/analyzer/SATestBuild.py')
| -rw-r--r-- | utils/analyzer/SATestBuild.py | 272 |
1 files changed, 204 insertions, 68 deletions
diff --git a/utils/analyzer/SATestBuild.py b/utils/analyzer/SATestBuild.py index 60c8796e338fd..4da025aa53b3a 100644 --- a/utils/analyzer/SATestBuild.py +++ b/utils/analyzer/SATestBuild.py @@ -45,32 +45,55 @@ variable. It should contain a comma separated list. import CmpRuns import SATestUtils -import os +from subprocess import CalledProcessError, check_call +import argparse import csv -import sys import glob +import logging import math +import multiprocessing +import os +import plistlib import shutil +import sys +import threading import time -import plistlib -import argparse -from subprocess import check_call, CalledProcessError -import multiprocessing +import Queue #------------------------------------------------------------------------------ # Helper functions. #------------------------------------------------------------------------------ +Local = threading.local() +Local.stdout = sys.stdout +Local.stderr = sys.stderr +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s:%(levelname)s:%(name)s: %(message)s') -sys.stdout = SATestUtils.flushfile(sys.stdout) +class StreamToLogger(object): + def __init__(self, logger, log_level=logging.INFO): + self.logger = logger + self.log_level = log_level + + def write(self, buf): + # Rstrip in order not to write an extra newline. + self.logger.log(self.log_level, buf.rstrip()) + + def flush(self): + pass + + def fileno(self): + return 0 def getProjectMapPath(): ProjectMapPath = os.path.join(os.path.abspath(os.curdir), ProjectMapFile) if not os.path.exists(ProjectMapPath): - print "Error: Cannot find the Project Map file " + ProjectMapPath +\ - "\nRunning script for the wrong directory?" + Local.stdout.write("Error: Cannot find the Project Map file " + + ProjectMapPath + + "\nRunning script for the wrong directory?\n") sys.exit(1) return ProjectMapPath @@ -100,7 +123,7 @@ if not Clang: sys.exit(1) # Number of jobs. -Jobs = int(math.ceil(multiprocessing.cpu_count() * 0.75)) +MaxJobs = int(math.ceil(multiprocessing.cpu_count() * 0.75)) # Project map stores info about all the "registered" projects. ProjectMapFile = "projectMap.csv" @@ -113,6 +136,9 @@ CleanupScript = "cleanup_run_static_analyzer.sh" # This is a file containing commands for scan-build. BuildScript = "run_static_analyzer.cmd" +# A comment in a build script which disables wrapping. +NoPrefixCmd = "#NOPREFIX" + # The log file name. LogFolderName = "Logs" BuildLogName = "run_static_analyzer.log" @@ -157,7 +183,7 @@ Checkers = ",".join([ "nullability" ]) -Verbose = 1 +Verbose = 0 #------------------------------------------------------------------------------ # Test harness logic. @@ -170,7 +196,8 @@ def runCleanupScript(Dir, PBuildLogFile): """ Cwd = os.path.join(Dir, PatchedSourceDirName) ScriptPath = os.path.join(Dir, CleanupScript) - SATestUtils.runScript(ScriptPath, PBuildLogFile, Cwd) + SATestUtils.runScript(ScriptPath, PBuildLogFile, Cwd, + Stdout=Local.stdout, Stderr=Local.stderr) def runDownloadScript(Dir, PBuildLogFile): @@ -178,7 +205,8 @@ def runDownloadScript(Dir, PBuildLogFile): Run the script to download the project, if it exists. """ ScriptPath = os.path.join(Dir, DownloadScript) - SATestUtils.runScript(ScriptPath, PBuildLogFile, Dir) + SATestUtils.runScript(ScriptPath, PBuildLogFile, Dir, + Stdout=Local.stdout, Stderr=Local.stderr) def downloadAndPatch(Dir, PBuildLogFile): @@ -192,8 +220,8 @@ def downloadAndPatch(Dir, PBuildLogFile): if not os.path.exists(CachedSourceDirPath): runDownloadScript(Dir, PBuildLogFile) if not os.path.exists(CachedSourceDirPath): - print "Error: '%s' not found after download." % ( - CachedSourceDirPath) + Local.stderr.write("Error: '%s' not found after download.\n" % ( + CachedSourceDirPath)) exit(1) PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) @@ -211,10 +239,10 @@ def applyPatch(Dir, PBuildLogFile): PatchfilePath = os.path.join(Dir, PatchfileName) PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) if not os.path.exists(PatchfilePath): - print " No local patches." + Local.stdout.write(" No local patches.\n") return - print " Applying patch." + Local.stdout.write(" Applying patch.\n") try: check_call("patch -p1 < '%s'" % (PatchfilePath), cwd=PatchedSourceDirPath, @@ -222,7 +250,8 @@ def applyPatch(Dir, PBuildLogFile): stdout=PBuildLogFile, shell=True) except: - print "Error: Patch failed. See %s for details." % (PBuildLogFile.name) + Local.stderr.write("Error: Patch failed. See %s for details.\n" % ( + PBuildLogFile.name)) sys.exit(1) @@ -233,7 +262,8 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile): """ BuildScriptPath = os.path.join(Dir, BuildScript) if not os.path.exists(BuildScriptPath): - print "Error: build script is not defined: %s" % BuildScriptPath + Local.stderr.write( + "Error: build script is not defined: %s\n" % BuildScriptPath) sys.exit(1) AllCheckers = Checkers @@ -247,9 +277,18 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile): SBOptions += "-plist-html -o '%s' " % SBOutputDir SBOptions += "-enable-checker " + AllCheckers + " " SBOptions += "--keep-empty " + AnalyzerConfig = [ + ("stable-report-filename", "true"), + ("serialize-stats", "true"), + ] + + SBOptions += "-analyzer-config '%s' " % ( + ",".join("%s=%s" % (key, value) for (key, value) in AnalyzerConfig)) + # Always use ccc-analyze to ensure that we can locate the failures # directory. SBOptions += "--override-compiler " + ExtraEnv = {} try: SBCommandFile = open(BuildScriptPath, "r") SBPrefix = "scan-build " + SBOptions + " " @@ -257,23 +296,35 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile): Command = Command.strip() if len(Command) == 0: continue + + # Custom analyzer invocation specified by project. + # Communicate required information using environment variables + # instead. + if Command == NoPrefixCmd: + SBPrefix = "" + ExtraEnv['OUTPUT'] = SBOutputDir + ExtraEnv['CC'] = Clang + continue + # If using 'make', auto imply a -jX argument # to speed up analysis. xcodebuild will # automatically use the maximum number of cores. if (Command.startswith("make ") or Command == "make") and \ "-j" not in Command: - Command += " -j%d" % Jobs + Command += " -j%d" % MaxJobs SBCommand = SBPrefix + Command + if Verbose == 1: - print " Executing: %s" % (SBCommand,) + Local.stdout.write(" Executing: %s\n" % (SBCommand,)) check_call(SBCommand, cwd=SBCwd, stderr=PBuildLogFile, stdout=PBuildLogFile, + env=dict(os.environ, **ExtraEnv), shell=True) except CalledProcessError: - print "Error: scan-build failed. Its output was: " + Local.stderr.write("Error: scan-build failed. Its output was: \n") PBuildLogFile.seek(0) - shutil.copyfileobj(PBuildLogFile, sys.stdout) + shutil.copyfileobj(PBuildLogFile, Local.stderr) sys.exit(1) @@ -282,8 +333,9 @@ def runAnalyzePreprocessed(Dir, SBOutputDir, Mode): Run analysis on a set of preprocessed files. """ if os.path.exists(os.path.join(Dir, BuildScript)): - print "Error: The preprocessed files project should not contain %s" % \ - BuildScript + Local.stderr.write( + "Error: The preprocessed files project should not contain %s\n" % ( + BuildScript)) raise Exception() CmdPrefix = Clang + " -cc1 " @@ -313,7 +365,8 @@ def runAnalyzePreprocessed(Dir, SBOutputDir, Mode): if SATestUtils.hasNoExtension(FileName): continue if not SATestUtils.isValidSingleInputFile(FileName): - print "Error: Invalid single input file %s." % (FullFileName,) + Local.stderr.write( + "Error: Invalid single input file %s.\n" % (FullFileName,)) raise Exception() # Build and call the analyzer command. @@ -322,14 +375,15 @@ def runAnalyzePreprocessed(Dir, SBOutputDir, Mode): LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b") try: if Verbose == 1: - print " Executing: %s" % (Command,) + Local.stdout.write(" Executing: %s\n" % (Command,)) check_call(Command, cwd=Dir, stderr=LogFile, stdout=LogFile, shell=True) except CalledProcessError, e: - print "Error: Analyzes of %s failed. See %s for details." \ - "Error code %d." % ( - FullFileName, LogFile.name, e.returncode) + Local.stderr.write("Error: Analyzes of %s failed. " + "See %s for details." + "Error code %d.\n" % ( + FullFileName, LogFile.name, e.returncode)) Failed = True finally: LogFile.close() @@ -349,7 +403,7 @@ def removeLogFile(SBOutputDir): if (os.path.exists(BuildLogPath)): RmCommand = "rm '%s'" % BuildLogPath if Verbose == 1: - print " Executing: %s" % (RmCommand,) + Local.stdout.write(" Executing: %s\n" % (RmCommand,)) check_call(RmCommand, shell=True) @@ -357,8 +411,8 @@ def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild): TBegin = time.time() BuildLogPath = getBuildLogPath(SBOutputDir) - print "Log file: %s" % (BuildLogPath,) - print "Output directory: %s" % (SBOutputDir, ) + Local.stdout.write("Log file: %s\n" % (BuildLogPath,)) + Local.stdout.write("Output directory: %s\n" % (SBOutputDir, )) removeLogFile(SBOutputDir) @@ -366,8 +420,9 @@ def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild): if (os.path.exists(SBOutputDir)): RmCommand = "rm -r '%s'" % SBOutputDir if Verbose == 1: - print " Executing: %s" % (RmCommand,) - check_call(RmCommand, shell=True) + Local.stdout.write(" Executing: %s\n" % (RmCommand,)) + check_call(RmCommand, shell=True, stdout=Local.stdout, + stderr=Local.stderr) assert(not os.path.exists(SBOutputDir)) os.makedirs(os.path.join(SBOutputDir, LogFolderName)) @@ -384,8 +439,9 @@ def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild): runCleanupScript(Dir, PBuildLogFile) normalizeReferenceResults(Dir, SBOutputDir, ProjectBuildMode) - print "Build complete (time: %.2f). See the log for more details: %s" % \ - ((time.time() - TBegin), BuildLogPath) + Local.stdout.write("Build complete (time: %.2f). " + "See the log for more details: %s\n" % ( + (time.time() - TBegin), BuildLogPath)) def normalizeReferenceResults(Dir, SBOutputDir, ProjectBuildMode): @@ -456,14 +512,16 @@ def checkBuild(SBOutputDir): CleanUpEmptyPlists(SBOutputDir) CleanUpEmptyFolders(SBOutputDir) Plists = glob.glob(SBOutputDir + "/*/*.plist") - print "Number of bug reports (non-empty plist files) produced: %d" %\ - len(Plists) + Local.stdout.write( + "Number of bug reports (non-empty plist files) produced: %d\n" % + len(Plists)) return - print "Error: analysis failed." - print "Total of %d failures discovered." % TotalFailed + Local.stderr.write("Error: analysis failed.\n") + Local.stderr.write("Total of %d failures discovered.\n" % TotalFailed) if TotalFailed > NumOfFailuresInSummary: - print "See the first %d below.\n" % NumOfFailuresInSummary + Local.stderr.write( + "See the first %d below.\n" % NumOfFailuresInSummary) # TODO: Add a line "See the results folder for more." Idx = 0 @@ -471,9 +529,9 @@ def checkBuild(SBOutputDir): if Idx >= NumOfFailuresInSummary: break Idx += 1 - print "\n-- Error #%d -----------\n" % Idx + Local.stderr.write("\n-- Error #%d -----------\n" % Idx) with open(FailLogPathI, "r") as FailLogI: - shutil.copyfileobj(FailLogI, sys.stdout) + shutil.copyfileobj(FailLogI, Local.stdout) sys.exit(1) @@ -526,25 +584,30 @@ def runCmpResults(Dir, Strictness=0): assert(RefDir != NewDir) if Verbose == 1: - print " Comparing Results: %s %s" % (RefDir, NewDir) + Local.stdout.write(" Comparing Results: %s %s\n" % ( + RefDir, NewDir)) PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) - Opts = CmpRuns.CmpOptions(rootA="", rootB=PatchedSourceDirPath) + Opts, Args = CmpRuns.generate_option_parser().parse_args( + ["--rootA", "", "--rootB", PatchedSourceDirPath]) # Scan the results, delete empty plist files. NumDiffs, ReportsInRef, ReportsInNew = \ - CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False) + CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, + deleteEmpty=False, + Stdout=Local.stdout) if (NumDiffs > 0): - print "Warning: %s differences in diagnostics." % NumDiffs + Local.stdout.write("Warning: %s differences in diagnostics.\n" + % NumDiffs) if Strictness >= 2 and NumDiffs > 0: - print "Error: Diffs found in strict mode (2)." + Local.stdout.write("Error: Diffs found in strict mode (2).\n") TestsPassed = False elif Strictness >= 1 and ReportsInRef != ReportsInNew: - print "Error: The number of results are different in "\ - "strict mode (1)." + Local.stdout.write("Error: The number of results are different " + + " strict mode (1).\n") TestsPassed = False - print "Diagnostic comparison complete (time: %.2f)." % ( - time.time() - TBegin) + Local.stdout.write("Diagnostic comparison complete (time: %.2f).\n" % ( + time.time() - TBegin)) return TestsPassed @@ -564,19 +627,49 @@ def cleanupReferenceResults(SBOutputDir): removeLogFile(SBOutputDir) +class TestProjectThread(threading.Thread): + def __init__(self, TasksQueue, ResultsDiffer, FailureFlag): + """ + :param ResultsDiffer: Used to signify that results differ from + the canonical ones. + :param FailureFlag: Used to signify a failure during the run. + """ + self.TasksQueue = TasksQueue + self.ResultsDiffer = ResultsDiffer + self.FailureFlag = FailureFlag + super(TestProjectThread, self).__init__() + + # Needed to gracefully handle interrupts with Ctrl-C + self.daemon = True + + def run(self): + while not self.TasksQueue.empty(): + try: + ProjArgs = self.TasksQueue.get() + Logger = logging.getLogger(ProjArgs[0]) + Local.stdout = StreamToLogger(Logger, logging.INFO) + Local.stderr = StreamToLogger(Logger, logging.ERROR) + if not testProject(*ProjArgs): + self.ResultsDiffer.set() + self.TasksQueue.task_done() + except: + self.FailureFlag.set() + raise + + def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0): """ Test a given project. :return TestsPassed: Whether tests have passed according to the :param Strictness: criteria. """ - print " \n\n--- Building project %s" % (ID,) + Local.stdout.write(" \n\n--- Building project %s\n" % (ID,)) TBegin = time.time() Dir = getProjectDir(ID) if Verbose == 1: - print " Build directory: %s." % (Dir,) + Local.stdout.write(" Build directory: %s.\n" % (Dir,)) # Set the build results directory. RelOutputDir = getSBOutputDirName(IsReferenceBuild) @@ -592,8 +685,8 @@ def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0): else: TestsPassed = runCmpResults(Dir, Strictness) - print "Completed tests for project %s (time: %.2f)." % \ - (ID, (time.time() - TBegin)) + Local.stdout.write("Completed tests for project %s (time: %.2f).\n" % ( + ID, (time.time() - TBegin))) return TestsPassed @@ -626,17 +719,62 @@ def validateProjectFile(PMapFile): " (single file), 1 (project), or 2(single file c++11)." raise Exception() +def singleThreadedTestAll(ProjectsToTest): + """ + Run all projects. + :return: whether tests have passed. + """ + Success = True + for ProjArgs in ProjectsToTest: + Success &= testProject(*ProjArgs) + return Success + +def multiThreadedTestAll(ProjectsToTest, Jobs): + """ + Run each project in a separate thread. + + This is OK despite GIL, as testing is blocked + on launching external processes. + + :return: whether tests have passed. + """ + TasksQueue = Queue.Queue() + + for ProjArgs in ProjectsToTest: + TasksQueue.put(ProjArgs) + + ResultsDiffer = threading.Event() + FailureFlag = threading.Event() + + for i in range(Jobs): + T = TestProjectThread(TasksQueue, ResultsDiffer, FailureFlag) + T.start() + + # Required to handle Ctrl-C gracefully. + while TasksQueue.unfinished_tasks: + time.sleep(0.1) # Seconds. + if FailureFlag.is_set(): + Local.stderr.write("Test runner crashed\n") + sys.exit(1) + return not ResultsDiffer.is_set() + + +def testAll(Args): + ProjectsToTest = [] -def testAll(IsReferenceBuild=False, Strictness=0): - TestsPassed = True with projectFileHandler() as PMapFile: validateProjectFile(PMapFile) # Test the projects. for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile): - TestsPassed &= testProject( - ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness) - return TestsPassed + ProjectsToTest.append((ProjName, + int(ProjBuildMode), + Args.regenerate, + Args.strictness)) + if Args.jobs <= 1: + return singleThreadedTestAll(ProjectsToTest) + else: + return multiThreadedTestAll(ProjectsToTest, Args.jobs) if __name__ == '__main__': @@ -650,14 +788,12 @@ if __name__ == '__main__': reference. Default is 0.') Parser.add_argument('-r', dest='regenerate', action='store_true', default=False, help='Regenerate reference output.') + Parser.add_argument('-j', '--jobs', dest='jobs', type=int, + default=0, + help='Number of projects to test concurrently') Args = Parser.parse_args() - IsReference = False - Strictness = Args.strictness - if Args.regenerate: - IsReference = True - - TestsPassed = testAll(IsReference, Strictness) + TestsPassed = testAll(Args) if not TestsPassed: print "ERROR: Tests failed." sys.exit(42) |