diff options
Diffstat (limited to 'utils/analyzer/SATestBuild.py')
| -rw-r--r-- | utils/analyzer/SATestBuild.py | 262 |
1 files changed, 215 insertions, 47 deletions
diff --git a/utils/analyzer/SATestBuild.py b/utils/analyzer/SATestBuild.py index c2209cc..3fccb9a 100644 --- a/utils/analyzer/SATestBuild.py +++ b/utils/analyzer/SATestBuild.py @@ -29,7 +29,7 @@ The files which should be kept around for failure investigations: Assumptions (TODO: shouldn't need to assume these.): The script is being run from the Repository Directory. - The compiler for scan-build is in the PATH. + The compiler for scan-build and scan-build are in the PATH. export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH For more logging, set the env variables: @@ -45,18 +45,19 @@ import glob import shutil import time import plistlib -from subprocess import check_call +from subprocess import check_call, CalledProcessError # Project map stores info about all the "registered" projects. ProjectMapFile = "projectMap.csv" # Names of the project specific scripts. # The script that needs to be executed before the build can start. -PreprocessScript = "pre_run_static_analyzer.sh" +CleanupScript = "cleanup_run_static_analyzer.sh" # This is a file containing commands for scan-build. BuildScript = "run_static_analyzer.cmd" # The log file name. +LogFolderName = "Logs" BuildLogName = "run_static_analyzer.log" # Summary file - contains the summary of the failures. Ex: This info can be be # displayed when buildbot detects a build failure. @@ -69,8 +70,24 @@ DiffsSummaryFileName = "diffs.txt" SBOutputDirName = "ScanBuildResults" SBOutputDirReferencePrefix = "Ref" +# The list of checkers used during analyzes. +# Currently, consists of all the non experimental checkers. +Checkers="experimental.security.taint,core,deadcode,cplusplus,security,unix,osx,cocoa" + Verbose = 1 +IsReferenceBuild = False + +# Make sure we flush the output after every print statement. +class flushfile(object): + def __init__(self, f): + self.f = f + def write(self, x): + self.f.write(x) + self.f.flush() + +sys.stdout = flushfile(sys.stdout) + def getProjectMapPath(): ProjectMapPath = os.path.join(os.path.abspath(os.curdir), ProjectMapFile) @@ -83,9 +100,15 @@ def getProjectMapPath(): def getProjectDir(ID): return os.path.join(os.path.abspath(os.curdir), ID) +def getSBOutputDirName() : + if IsReferenceBuild == True : + return SBOutputDirReferencePrefix + SBOutputDirName + else : + return SBOutputDirName + # Run pre-processing script if any. -def runPreProcessingScript(Dir, PBuildLogFile): - ScriptPath = os.path.join(Dir, PreprocessScript) +def runCleanupScript(Dir, PBuildLogFile): + ScriptPath = os.path.join(Dir, CleanupScript) if os.path.exists(ScriptPath): try: if Verbose == 1: @@ -109,8 +132,8 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile): if not os.path.exists(BuildScriptPath): print "Error: build script is not defined: %s" % BuildScriptPath sys.exit(-1) - SBOptions = "-plist -o " + SBOutputDir + " " - SBOptions += "-enable-checker core,deadcode.DeadStores" + SBOptions = "-plist-html -o " + SBOutputDir + " " + SBOptions += "-enable-checker " + Checkers + " " try: SBCommandFile = open(BuildScriptPath, "r") SBPrefix = "scan-build " + SBOptions + " " @@ -124,34 +147,107 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile): except: print "Error: scan-build failed. See ",PBuildLogFile.name,\ " for details." - sys.exit(-1) + raise + +def hasNoExtension(FileName): + (Root, Ext) = os.path.splitext(FileName) + if ((Ext == "")) : + return True + return False + +def isValidSingleInputFile(FileName): + (Root, Ext) = os.path.splitext(FileName) + if ((Ext == ".i") | (Ext == ".ii") | + (Ext == ".c") | (Ext == ".cpp") | + (Ext == ".m") | (Ext == "")) : + return True + return False + +# Run analysis on a set of preprocessed files. +def runAnalyzePreprocessed(Dir, SBOutputDir): + if os.path.exists(os.path.join(Dir, BuildScript)): + print "Error: The preprocessed files project should not contain %s" % \ + BuildScript + raise Exception() + + CmdPrefix = "clang -cc1 -analyze -analyzer-output=plist -w " + CmdPrefix += "-analyzer-checker=" + Checkers +" -fcxx-exceptions -fblocks " + + PlistPath = os.path.join(Dir, SBOutputDir, "date") + FailPath = os.path.join(PlistPath, "failures"); + os.makedirs(FailPath); + + for FullFileName in glob.glob(Dir + "/*"): + FileName = os.path.basename(FullFileName) + Failed = False + + # Only run the analyzes on supported files. + if (hasNoExtension(FileName)): + continue + if (isValidSingleInputFile(FileName) == False): + print "Error: Invalid single input file %s." % (FullFileName,) + raise Exception() + + # Build and call the analyzer command. + OutputOption = "-o " + os.path.join(PlistPath, FileName) + ".plist " + Command = CmdPrefix + OutputOption + os.path.join(Dir, FileName) + LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b") + try: + if Verbose == 1: + print " Executing: %s" % (Command,) + check_call(Command, cwd = Dir, stderr=LogFile, + stdout=LogFile, + shell=True) + except CalledProcessError, e: + print "Error: Analyzes of %s failed. See %s for details." \ + "Error code %d." % \ + (FullFileName, LogFile.name, e.returncode) + Failed = True + finally: + LogFile.close() + + # If command did not fail, erase the log file. + if Failed == False: + os.remove(LogFile.name); -def buildProject(Dir, SBOutputDir): +def buildProject(Dir, SBOutputDir, IsScanBuild): TBegin = time.time() - BuildLogPath = os.path.join(Dir, BuildLogName) + BuildLogPath = os.path.join(SBOutputDir, LogFolderName, BuildLogName) print "Log file: %s" % (BuildLogPath,) - + print "Output directory: %s" %(SBOutputDir, ) + # Clean up the log file. if (os.path.exists(BuildLogPath)) : RmCommand = "rm " + BuildLogPath if Verbose == 1: - print " Executing: %s." % (RmCommand,) + print " Executing: %s" % (RmCommand,) check_call(RmCommand, shell=True) + + # Clean up scan build results. + if (os.path.exists(SBOutputDir)) : + RmCommand = "rm -r " + SBOutputDir + if Verbose == 1: + print " Executing: %s" % (RmCommand,) + check_call(RmCommand, shell=True) + assert(not os.path.exists(SBOutputDir)) + os.makedirs(os.path.join(SBOutputDir, LogFolderName)) # Open the log file. PBuildLogFile = open(BuildLogPath, "wb+") - try: - # Clean up scan build results. - if (os.path.exists(SBOutputDir)) : - RmCommand = "rm -r " + SBOutputDir - if Verbose == 1: - print " Executing: %s" % (RmCommand,) - check_call(RmCommand, stderr=PBuildLogFile, - stdout=PBuildLogFile, shell=True) - runPreProcessingScript(Dir, PBuildLogFile) - runScanBuild(Dir, SBOutputDir, PBuildLogFile) + # Build and analyze the project. + try: + runCleanupScript(Dir, PBuildLogFile) + + if IsScanBuild: + runScanBuild(Dir, SBOutputDir, PBuildLogFile) + else: + runAnalyzePreprocessed(Dir, SBOutputDir) + + if IsReferenceBuild : + runCleanupScript(Dir, PBuildLogFile) + finally: PBuildLogFile.close() @@ -185,9 +281,9 @@ def checkBuild(SBOutputDir): return; # Create summary file to display when the build fails. - SummaryPath = os.path.join(SBOutputDir, FailuresSummaryFileName); + SummaryPath = os.path.join(SBOutputDir, LogFolderName, FailuresSummaryFileName) if (Verbose > 0): - print " Creating the failures summary file %s." % (SummaryPath,) + print " Creating the failures summary file %s" % (SummaryPath,) SummaryLog = open(SummaryPath, "w+") try: @@ -212,8 +308,7 @@ def checkBuild(SBOutputDir): finally: SummaryLog.close() - print "Error: Scan-build failed. See ", \ - os.path.join(SBOutputDir, FailuresSummaryFileName) + print "Error: analysis failed. See ", SummaryPath sys.exit(-1) # Auxiliary object to discard stdout. @@ -231,6 +326,11 @@ def runCmpResults(Dir): # We have to go one level down the directory tree. RefList = glob.glob(RefDir + "/*") NewList = glob.glob(NewDir + "/*") + + # Log folders are also located in the results dir, so ignore them. + RefList.remove(os.path.join(RefDir, LogFolderName)) + NewList.remove(os.path.join(NewDir, LogFolderName)) + if len(RefList) == 0 or len(NewList) == 0: return False assert(len(RefList) == len(NewList)) @@ -243,7 +343,7 @@ def runCmpResults(Dir): NewList.sort() # Iterate and find the differences. - HaveDiffs = False + NumDiffs = 0 PairList = zip(RefList, NewList) for P in PairList: RefDir = P[0] @@ -259,16 +359,47 @@ def runCmpResults(Dir): OLD_STDOUT = sys.stdout sys.stdout = Discarder() # Scan the results, delete empty plist files. - HaveDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False) + NumDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False) sys.stdout = OLD_STDOUT - if HaveDiffs: - print "Warning: difference in diagnostics. See %s" % (DiffsPath,) - HaveDiffs=True + if (NumDiffs > 0) : + print "Warning: %r differences in diagnostics. See %s" % \ + (NumDiffs, DiffsPath,) print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin) - return HaveDiffs + return (NumDiffs > 0) + +def updateSVN(Mode, ProjectsMap): + try: + ProjectsMap.seek(0) + for I in csv.reader(ProjectsMap): + ProjName = I[0] + Path = os.path.join(ProjName, getSBOutputDirName()) + + if Mode == "delete": + Command = "svn delete %s" % (Path,) + else: + Command = "svn add %s" % (Path,) + + if Verbose == 1: + print " Executing: %s" % (Command,) + check_call(Command, shell=True) + + if Mode == "delete": + CommitCommand = "svn commit -m \"[analyzer tests] Remove " \ + "reference results.\"" + else: + CommitCommand = "svn commit -m \"[analyzer tests] Add new " \ + "reference results.\"" + if Verbose == 1: + print " Executing: %s" % (CommitCommand,) + check_call(CommitCommand, shell=True) + except: + print "Error: SVN update failed." + sys.exit(-1) + +def testProject(ID, IsScanBuild, Dir=None): + print " \n\n--- Building project %s" % (ID,) -def testProject(ID, IsReferenceBuild, Dir=None): TBegin = time.time() if Dir is None : @@ -277,13 +408,10 @@ def testProject(ID, IsReferenceBuild, Dir=None): print " Build directory: %s." % (Dir,) # Set the build results directory. - if IsReferenceBuild == True : - SBOutputDir = os.path.join(Dir, SBOutputDirReferencePrefix + \ - SBOutputDirName) - else : - SBOutputDir = os.path.join(Dir, SBOutputDirName) - - buildProject(Dir, SBOutputDir) + RelOutputDir = getSBOutputDirName() + SBOutputDir = os.path.join(Dir, RelOutputDir) + + buildProject(Dir, SBOutputDir, IsScanBuild) checkBuild(SBOutputDir) @@ -293,15 +421,55 @@ def testProject(ID, IsReferenceBuild, Dir=None): print "Completed tests for project %s (time: %.2f)." % \ (ID, (time.time()-TBegin)) -def testAll(IsReferenceBuild=False): +def testAll(InIsReferenceBuild = False, UpdateSVN = False): + global IsReferenceBuild + IsReferenceBuild = InIsReferenceBuild + PMapFile = open(getProjectMapPath(), "rb") - try: - PMapReader = csv.reader(PMapFile) - for I in PMapReader: - print " --- Building project %s" % (I[0],) - testProject(I[0], IsReferenceBuild) + try: + # Validate the input. + for I in csv.reader(PMapFile): + if (len(I) != 2) : + print "Error: Rows in the ProjectMapFile should have 3 entries." + raise Exception() + if (not ((I[1] == "1") | (I[1] == "0"))): + print "Error: Second entry in the ProjectMapFile should be 0 or 1." + raise Exception() + + # When we are regenerating the reference results, we might need to + # update svn. Remove reference results from SVN. + if UpdateSVN == True: + assert(InIsReferenceBuild == True); + updateSVN("delete", PMapFile); + + # Test the projects. + PMapFile.seek(0) + for I in csv.reader(PMapFile): + testProject(I[0], int(I[1])) + + # Add reference results to SVN. + if UpdateSVN == True: + updateSVN("add", PMapFile); + + except: + print "Error occurred. Premature termination." + raise finally: PMapFile.close() if __name__ == '__main__': - testAll() + IsReference = False + UpdateSVN = False + if len(sys.argv) >= 2: + if sys.argv[1] == "-r": + IsReference = True + elif sys.argv[1] == "-rs": + IsReference = True + UpdateSVN = True + else: + print >> sys.stderr, 'Usage: ', sys.argv[0],\ + '[-r|-rs]' \ + 'Use -r to regenerate reference output' \ + 'Use -rs to regenerate reference output and update svn' + + testAll(IsReference, UpdateSVN) |
