#!/usr/bin/env python## Given a previous good compile narrow down miscompiles.# Expects two directories named "before" and "after" each containing a set of# assembly or object files where the "after" version is assumed to be broken.# You also have to provide a script called "link_test". It is called with a list# of files which should be linked together and result tested. "link_test" should# returns with exitcode 0 if the linking and testing succeeded.## abtest.py operates by taking all files from the "before" directory and# in each step replacing one of them with a file from the "bad" directory.## Additionally you can perform the same steps with a single .s file. In this# mode functions are identified by " -- Begin function FunctionName" and# " -- End function" markers. The abtest.py then takes all# function from the file in the "before" directory and replaces one function# with the corresponding function from the "bad" file in each step.## Example usage to identify miscompiled files:# 1. Create a link_test script, make it executable. Simple Example:# clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM"# 2. Run the script to figure out which files are miscompiled:# > ./abtest.py # somefile.s: ok# someotherfile.s: skipped: same content# anotherfile.s: failed: './link_test' exitcode != 0# ...# Example usage to identify miscompiled functions inside a file:# 3. Run the tests on a single file (assuming before/file.s and# after/file.s exist)# > ./abtest.py file.s# funcname1 [0/XX]: ok# funcname2 [1/XX]: ok# funcname3 [2/XX]: skipped: same content# funcname4 [3/XX]: failed: './link_test' exitcode != 0# ...fromfnmatchimportfilterfromsysimportstderrimportargparseimportfilecmpimportosimportsubprocessimportsysLINKTEST="./link_test"ESCAPE="\033[%sm"BOLD=ESCAPE%"1"RED=ESCAPE%"31"NORMAL=ESCAPE%"0"FAILED=RED+"failed"+NORMALdeffind(dir,file_filter=None):files=[walkdir[0]+"/"+fileforwalkdirinos.walk(dir)forfileinwalkdir[2]]iffile_filter!=None:files=filter(files,file_filter)returnfilesdeferror(message):stderr.write("Error: %s\n"%(message,))defwarn(message):stderr.write("Warning: %s\n"%(message,))defextract_functions(file):functions=[]in_function=Noneforlineinopen(file):marker=line.find(" -- Begin function ")ifmarker!=-1:ifin_function!=None:warn("Missing end of function %s"%(in_function,))funcname=line[marker+19:-1]in_function=funcnametext=linecontinuemarker=line.find(" -- End function")ifmarker!=-1:text+=linefunctions.append((in_function,text))in_function=Nonecontinueifin_function!=None:text+=linereturnfunctionsdefreplace_function(file,function,replacement,dest):out=open(dest,"w")skip=Falsefound=Falsein_function=Noneforlineinopen(file):marker=line.find(" -- Begin function ")ifmarker!=-1:ifin_function!=None:warn("Missing end of function %s"%(in_function,))funcname=line[marker+19:-1]in_function=funcnameifin_function==function:out.write(replacement)skip=Trueelse:marker=line.find(" -- End function")ifmarker!=-1:in_function=Noneifskip:skip=Falsecontinueifnotskip:out.write(line)defannounce_test(name):stderr.write("%s%s%s: "%(BOLD,name,NORMAL))stderr.flush()defannounce_result(result,info):stderr.write(result)ifinfo!="":stderr.write(": %s"%info)stderr.write("\n")stderr.flush()deftestrun(files):linkline="%s%s"%(LINKTEST," ".join(files),)res=subprocess.call(linkline,shell=True)ifres!=0:announce_result(FAILED,"'%s' exitcode != 0"%LINKTEST)returnFalseelse:announce_result("ok","")returnTruedefcheck_files():"""Check files mode"""foriinrange(0,len(NO_PREFIX)):f=NO_PREFIX[i]b=baddir+"/"+fifbnotinBAD_FILES:warn("There is no corresponding file to '%s' in %s" \
%(gooddir+"/"+f,baddir))continueannounce_test(f+" [%s/%s]"%(i+1,len(NO_PREFIX)))# combine files (everything from good except f)testfiles=[]skip=FalseforcinNO_PREFIX:badfile=baddir+"/"+cgoodfile=gooddir+"/"+cifc==f:testfiles.append(badfile)iffilecmp.cmp(goodfile,badfile):announce_result("skipped","same content")skip=Truebreakelse:testfiles.append(goodfile)ifskip:continuetestrun(testfiles)defcheck_functions_in_file(base,goodfile,badfile):functions=extract_functions(goodfile)iflen(functions)==0:warn("Couldn't find any function in %s, missing annotations?"%(goodfile,))returnbadfunctions=dict(extract_functions(badfile))iflen(functions)==0:warn("Couldn't find any function in %s, missing annotations?"%(badfile,))returnCOMBINED="/tmp/combined.s"i=0for(func,func_text)infunctions:announce_test(func+" [%s/%s]"%(i+1,len(functions)))i+=1iffuncnotinbadfunctions:warn("Function '%s' missing from bad file"%func)continueifbadfunctions[func]==func_text:announce_result("skipped","same content")continuereplace_function(goodfile,func,badfunctions[func],COMBINED)testfiles=[]forcinNO_PREFIX:ifc==base:testfiles.append(COMBINED)continuetestfiles.append(gooddir+"/"+c)testrun(testfiles)parser=argparse.ArgumentParser()parser.add_argument('--a',dest='dir_a',default='before')parser.add_argument('--b',dest='dir_b',default='after')parser.add_argument('--insane',help='Skip sanity check',action='store_true')parser.add_argument('file',metavar='file',nargs='?')config=parser.parse_args()gooddir=config.dir_abaddir=config.dir_bBAD_FILES=find(baddir,"*")GOOD_FILES=find(gooddir,"*")NO_PREFIX=sorted([x[len(gooddir)+1:]forxinGOOD_FILES])# "Checking whether build environment is sane ..."ifnotconfig.insane:announce_test("sanity check")ifnotos.access(LINKTEST,os.X_OK):error("Expect '%s' to be present and executable"%(LINKTEST,))exit(1)res=testrun(GOOD_FILES)ifnotres:# "build environment is grinning and holding a spatula. Guess not."linkline="%s%s"%(LINKTEST," ".join(GOOD_FILES),)stderr.write("\n%s\n\n"%linkline)stderr.write("Returned with exitcode != 0\n")sys.exit(1)ifconfig.fileisnotNone:# File exchange modegoodfile=gooddir+"/"+config.filebadfile=baddir+"/"+config.filecheck_functions_in_file(config.file,goodfile,badfile)else:# Function exchange modecheck_files()