#!/usr/bin/env python## __COPYRIGHT__## runtest.py - wrapper script for running SCons tests## This script mainly exists to set PYTHONPATH to the right list of# directories to test the SCons modules.## By default, it directly uses the modules in the local tree:# ./src/ (source files we ship) and ./QMTest/ (other modules we don't).## HOWEVER, now that SCons has Repository support, we don't have# Aegis copy all of the files into the local tree. So if you're# using Aegis and want to run tests by hand using this script, you# must "aecp ." the entire source tree into your local directory# structure. When you're done with your change, you can then# "aecpu -unch ." to un-copy any files that you haven't changed.## When any -p option is specified, this script assumes it's in a# directory in which a build has been performed, and sets PYTHONPATH# so that it *only* references the modules that have unpacked from# the specified built package, to test whether the packages are good.## Options:## -a Run all tests; does a virtual 'find' for# all SCons tests under the current directory.## --aegis Print test results to an output file (specified# by the -o option) in the format expected by# aetest(5). This is intended for use in the# batch_test_command field in the Aegis project# config file.## -d Debug. Runs the script under the Python# debugger (pdb.py) so you don't have to# muck with PYTHONPATH yourself.## -f file Only execute the tests listed in the specified# file.## -h Print the help and exit.## -l List available tests and exit.## -n No execute, just print command lines.## -o file Print test results to the specified file.# The --aegis and --xml options specify the# output format.## -P Python Use the specified Python interpreter.## -p package Test against the specified package.## --passed In the final summary, also report which tests# passed. The default is to only report tests# which failed or returned NO RESULT.## -q Quiet. By default, runtest.py prints the# command line it will execute before# executing it. This suppresses that print.## --sp The Aegis search path.## --spe The Aegis executable search path.## -t Print the execution time of each test.## -X The scons "script" is an executable; don't# feed it to Python.## -x scons The scons script to use for tests.## --xml Print test results to an output file (specified# by the -o option) in an SCons-specific XML format.# This is (will be) used for reporting results back# to a central SCons test monitoring infrastructure.## (Note: There used to be a -v option that specified the SCons# version to be tested, when we were installing in a version-specific# library directory. If we ever resurrect that as the default, then# you can find the appropriate code in the 0.04 version of this script,# rather than reinventing that wheel.)#importgetoptimportglobimportosimportos.pathimportpopen2importreimportstatimportstringimportsysimporttimeifnothasattr(os,'WEXITSTATUS'):os.WEXITSTATUS=lambdax:xcwd=os.getcwd()all=0baseline=0builddir=os.path.join(cwd,'build')debug=''execute_tests=1format=Nonelist_only=Noneprintcommand=1package=Noneprint_passed_summary=Nonescons=Nonescons_exec=Noneoutputfile=Nonetestlistfile=Noneversion=''print_times=Nonepython=Nonesp=Nonespe=Nonehelpstr="""\Usage: runtest.py [OPTIONS] [TEST ...]Options: -a, --all Run all tests. --aegis Print results in Aegis format. -b BASE, --baseline BASE Run test scripts against baseline BASE. --builddir DIR Directory in which packages were built. -d, --debug Run test scripts under the Python debugger. -f FILE, --file FILE Run tests in specified FILE. -h, --help Print this message and exit. -l, --list List available tests and exit. -n, --no-exec No execute, just print command lines. --noqmtest Execute tests directly, not using QMTest. -o FILE, --output FILE Print test results to FILE. -P Python Use the specified Python interpreter. -p PACKAGE, --package PACKAGE Test against the specified PACKAGE: deb Debian local-tar-gz .tar.gz standalone package local-zip .zip standalone package rpm Red Hat src-tar-gz .tar.gz source package src-zip .zip source package tar-gz .tar.gz distribution zip .zip distribution --passed Summarize which tests passed. --qmtest Run using the QMTest harness. -q, --quiet Don't print the test being executed. --sp PATH The Aegis search path. --spe PATH The Aegis executable search path. -t, --time Print test execution time. -v version Specify the SCons version. --verbose=LEVEL Set verbose level: 1 = print executed commands, 2 = print commands and non-zero output, 3 = print commands and all output. -X Test script is executable, don't feed to Python. -x SCRIPT, --exec SCRIPT Test SCRIPT. --xml Print results in SCons XML format."""opts,args=getopt.getopt(sys.argv[1:],"ab:df:hlno:P:p:qv:Xx:t",['all','aegis','baseline=','builddir=','debug','file=','help','list','no-exec','noqmtest','output=','package=','passed','python=','qmtest','quiet','spe=','version=','exec=','time','verbose=','xml'])foro,ainopts:ifoin['-a','--all']:all=1elifoin['-b','--baseline']:baseline=aelifoin['--builddir']:builddir=aifnotos.path.isabs(builddir):builddir=os.path.normpath(os.path.join(cwd,builddir))elifoin['-d','--debug']:fordirinsys.path:pdb=os.path.join(dir,'pdb.py')ifos.path.exists(pdb):debug=pdbbreakelifoin['-f','--file']:ifnotos.path.isabs(a):a=os.path.join(cwd,a)testlistfile=aelifoin['-h','--help']:printhelpstrsys.exit(0)elifoin['-l','--list']:list_only=1elifoin['-n','--no-exec']:execute_tests=Noneelifoin['--noqmtest']:qmtest=Noneelifoin['-o','--output']:ifa!='-'andnotos.path.isabs(a):a=os.path.join(cwd,a)outputfile=aelifoin['-p','--package']:package=aelifoin['--passed']:print_passed_summary=1elifoin['-P','--python']:python=aelifoin['--qmtest']:qmtest='qmtest.py'elifoin['-q','--quiet']:printcommand=0elifoin['--sp']:sp=string.split(a,os.pathsep)elifoin['--spe']:spe=string.split(a,os.pathsep)elifoin['-t','--time']:print_times=1elifoin['--verbose']:os.environ['TESTCMD_VERBOSE']=aelifoin['-v','--version']:version=aelifoin['-X']:scons_exec=1elifoin['-x','--exec']:scons=aelifoin['--aegis','--xml']:format=oifnotargsandnotallandnottestlistfile:sys.stderr.write("""\runtest.py: No tests were specified. List one or more tests on the command line, use the -f option to specify a file containing a list of tests, or use the -a option to find and run all tests.""")sys.exit(1)ifsys.platformin('win32','cygwin'):defwhereis(file):pathext=['']+string.split(os.environ['PATHEXT'])fordirinstring.split(os.environ['PATH'],os.pathsep):f=os.path.join(dir,file)forextinpathext:fext=f+extifos.path.isfile(fext):returnfextreturnNoneelse:defwhereis(file):fordirinstring.split(os.environ['PATH'],os.pathsep):f=os.path.join(dir,file)ifos.path.isfile(f):try:st=os.stat(f)exceptOSError:continueifstat.S_IMODE(st[stat.ST_MODE])&0111:returnfreturnNonetry:qmtestexceptNameError:q='qmtest.py'qmtest=whereis(q)ifqmtest:qmtest=qelse:sys.stderr.write('Warning: %s not found on $PATH, assuming --noqmtest option.\n'%q)sys.stderr.flush()aegis=whereis('aegis')ifformat=='--aegis'andaegis:change=os.popen("aesub '$c' 2>/dev/null","r").read()ifchange:ifspisNone:paths=os.popen("aesub '$sp' 2>/dev/null","r").read()[:-1]sp=string.split(paths,os.pathsep)ifspeisNone:spe=os.popen("aesub '$spe' 2>/dev/null","r").read()[:-1]spe=string.split(spe,os.pathsep)else:aegis=NoneifspisNone:sp=[]ifspeisNone:spe=[]sp.append(builddir)sp.append(cwd)#_ws=re.compile('\s')defescape(s):if_ws.search(s):s='"'+s+'"'returns# Set up lowest-common-denominator spawning of a process on both Windows# and non-Windows systems that works all the way back to Python 1.5.2.try:os.spawnvexceptAttributeError:defspawn_it(command_args):pid=os.fork()ifpid==0:os.execv(command_args[0],command_args)else:pid,status=os.waitpid(pid,0)returnstatus>>8else:defspawn_it(command_args):command=command_args[0]command_args=map(escape,command_args)returnos.spawnv(os.P_WAIT,command,command_args)classBase:def__init__(self,path,spe=None):self.path=pathself.abspath=os.path.abspath(path)ifspe:fordirinspe:f=os.path.join(dir,path)ifos.path.isfile(f):self.abspath=fbreakself.status=NoneclassSystemExecutor(Base):defexecute(self):s=spawn_it(self.command_args)self.status=sifs<0ors>2:sys.stdout.write("Unexpected exit status %d\n"%s)try:popen2.Popen3exceptAttributeError:classPopenExecutor(Base):defexecute(self):(tochild,fromchild,childerr)=os.popen3(self.command_str)tochild.close()self.stderr=childerr.read()self.stdout=fromchild.read()fromchild.close()self.status=childerr.close()ifnotself.status:self.status=0else:classPopenExecutor(Base):defexecute(self):p=popen2.Popen3(self.command_str,1)p.tochild.close()self.stdout=p.fromchild.read()self.stderr=p.childerr.read()self.status=p.wait()classAegis(SystemExecutor):defheader(self,f):f.write('test_result = [\n')defwrite(self,f):f.write(' { file_name = "%s";\n'%self.path)f.write(' exit_status = %d; },\n'%self.status)deffooter(self,f):f.write('];\n')classXML(PopenExecutor):defheader(self,f):f.write(' <results>\n')defwrite(self,f):f.write(' <test>\n')f.write(' <file_name>%s</file_name>\n'%self.path)f.write(' <command_line>%s</command_line>\n'%self.command_str)f.write(' <exit_status>%s</exit_status>\n'%self.status)f.write(' <stdout>%s</stdout>\n'%self.stdout)f.write(' <stderr>%s</stderr>\n'%self.stderr)f.write(' <time>%.1f</time>\n'%self.test_time)f.write(' </test>\n')deffooter(self,f):f.write(' <time>%.1f</time>\n'%self.total_time)f.write(' </results>\n')format_class={None:SystemExecutor,'--aegis':Aegis,'--xml':XML,}Test=format_class[format]ifpackage:dir={'deb':'usr','local-tar-gz':None,'local-zip':None,'rpm':'usr','src-tar-gz':'','src-zip':'','tar-gz':'','zip':'',}# The hard-coded "python2.1" here is the library directory# name on Debian systems, not an executable, so it's all right.lib={'deb':os.path.join('python2.1','site-packages')}ifnotdir.has_key(package):sys.stderr.write("Unknown package '%s'\n"%package)sys.exit(2)test_dir=os.path.join(builddir,'test-%s'%package)ifdir[package]isNone:scons_script_dir=test_dirglobs=glob.glob(os.path.join(test_dir,'scons-local-*'))ifnotglobs:sys.stderr.write("No `scons-local-*' dir in `%s'\n"%test_dir)sys.exit(2)scons_lib_dir=Nonepythonpath_dir=globs[len(globs)-1]elifsys.platform=='win32':scons_script_dir=os.path.join(test_dir,dir[package],'Scripts')scons_lib_dir=os.path.join(test_dir,dir[package])pythonpath_dir=scons_lib_direlse:scons_script_dir=os.path.join(test_dir,dir[package],'bin')l=lib.get(package,'scons')scons_lib_dir=os.path.join(test_dir,dir[package],'lib',l)pythonpath_dir=scons_lib_dirscons_runtest_dir=builddirelse:sd=Noneld=None# XXX: Logic like the following will be necessary once# we fix runtest.py to run tests within an Aegis change# without symlinks back to the baseline(s).##if spe:# if not scons:# for dir in spe:# d = os.path.join(dir, 'src', 'script')# f = os.path.join(d, 'scons.py')# if os.path.isfile(f):# sd = d# scons = f# spe = map(lambda x: os.path.join(x, 'src', 'engine'), spe)# ld = string.join(spe, os.pathsep)ifnotbaselineorbaseline=='.':base=cwdelifbaseline=='-':# Tentative code for fetching information directly from the# QMTest context file.##import qm.common#import qm.test.context#qm.rc.Load("test")#context = qm.test.context.Context()#context.Read('context')url=Nonesvn_info=os.popen("svn info 2>&1","r").read()match=re.search('URL: (.*)',svn_info)ifmatch:url=match.group(1)ifnoturl:sys.stderr.write('runtest.py: could not find a URL:\n')sys.stderr.write(svn_info)sys.exit(1)importtempfilebase=tempfile.mkdtemp(prefix='runtest-tmp-')command='cd %s && svn co -q %s'%(base,url)base=os.path.join(base,os.path.split(url)[1])ifprintcommand:printcommandifexecute_tests:os.system(command)else:base=baselinescons_runtest_dir=basescons_script_dir=sdoros.path.join(base,'src','script')scons_lib_dir=ldoros.path.join(base,'src','engine')pythonpath_dir=scons_lib_dirifscons:# Let the version of SCons that the -x option pointed to find# its own modules.os.environ['SCONS']=sconselifscons_lib_dir:# Because SCons is really aggressive about finding its modules,# it sometimes finds SCons modules elsewhere on the system.# This forces SCons to use the modules that are being tested.os.environ['SCONS_LIB_DIR']=scons_lib_dirifscons_exec:os.environ['SCONS_EXEC']='1'os.environ['SCONS_RUNTEST_DIR']=scons_runtest_diros.environ['SCONS_SCRIPT_DIR']=scons_script_diros.environ['SCONS_CWD']=cwdos.environ['SCONS_VERSION']=versionold_pythonpath=os.environ.get('PYTHONPATH')pythonpaths=[pythonpath_dir]fordirinsp:ifformat=='--aegis':q=os.path.join(dir,'build','QMTest')else:q=os.path.join(dir,'QMTest')pythonpaths.append(q)os.environ['SCONS_SOURCE_PATH_EXECUTABLE']=string.join(spe,os.pathsep)os.environ['PYTHONPATH']=string.join(pythonpaths,os.pathsep)ifold_pythonpath:os.environ['PYTHONPATH']=os.environ['PYTHONPATH']+ \
os.pathsep+ \
old_pythonpathtests=[]ifargs:ifspe:forainargs:ifos.path.isabs(a):tests.extend(glob.glob(a))else:fordirinspe:x=os.path.join(dir,a)globs=glob.glob(x)ifglobs:tests.extend(globs)breakelse:forainargs:tests.extend(glob.glob(a))eliftestlistfile:tests=open(testlistfile,'r').readlines()tests=filter(lambdax:x[0]!='#',tests)tests=map(lambdax:x[:-1],tests)elifallandnotqmtest:# Find all of the SCons functional tests in the local directory# tree. This is anything under the 'src' subdirectory that ends# with 'Tests.py', or any Python script (*.py) under the 'test'# subdirectory.## Note that there are some tests under 'src' that *begin* with# 'test_', but they're packaging and installation tests, not# functional tests, so we don't execute them by default. (They can# still be executed by hand, though, and are routinely executed# by the Aegis packaging build to make sure that we're building# things correctly.)tdict={}deffind_Tests_py(tdict,dirname,names):forninfilter(lambdan:n[-8:]=="Tests.py",names):t=os.path.join(dirname,n)ifnottdict.has_key(t):tdict[t]=1os.path.walk('src',find_Tests_py,tdict)deffind_py(tdict,dirname,names):forninfilter(lambdan:n[-3:]==".py",names):t=os.path.join(dirname,n)ifnottdict.has_key(t):tdict[t]=1os.path.walk('test',find_py,tdict)ifformat=='--aegis'andaegis:cmd="aegis -list -unf pf 2>/dev/null"forlineinos.popen(cmd,"r").readlines():a=string.split(line)ifa[0]=="test"andnottdict.has_key(a[-1]):tdict[a[-1]]=Test(a[-1],spe)cmd="aegis -list -unf cf 2>/dev/null"forlineinos.popen(cmd,"r").readlines():a=string.split(line)ifa[0]=="test":ifa[1]=="remove":deltdict[a[-1]]elifnottdict.has_key(a[-1]):tdict[a[-1]]=Test(a[-1],spe)tests=tdict.keys()tests.sort()ifqmtest:ifbaseline:aegis_result_stream='scons_tdb.AegisBaselineStream'qmr_file='baseline.qmr'else:aegis_result_stream='scons_tdb.AegisChangeStream'qmr_file='results.qmr'ifprint_times:aegis_result_stream=aegis_result_stream+"(print_time='1')"qmtest_args=[qmtest,]ifformat=='--aegis':dir=builddirifnotos.path.isdir(dir):dir=cwdqmtest_args.extend(['-D',dir])qmtest_args.extend(['run','--output %s'%qmr_file,'--format none','--result-stream="%s"'%aegis_result_stream,])ifpython:qmtest_args.append('--context python="%s"'%python)ifoutputfile:ifformat=='--xml':rsclass='scons_tdb.SConsXMLResultStream'else:rsclass='scons_tdb.AegisBatchStream'qof="r'"+outputfile+"'"rs='--result-stream="%s(filename=%s)"'%(rsclass,qof)qmtest_args.append(rs)ifformat=='--aegis':tests=map(lambdax:string.replace(x,cwd+os.sep,''),tests)else:os.environ['SCONS']=os.path.join(cwd,'src','script','scons.py')cmd=string.join(qmtest_args+tests,' ')ifprintcommand:sys.stdout.write(cmd+'\n')sys.stdout.flush()status=0ifexecute_tests:status=os.WEXITSTATUS(os.system(cmd))sys.exit(status)#try:# os.chdir(scons_script_dir)#except OSError:# passtests=map(Test,tests)classUnbuffered:def__init__(self,file):self.file=filedefwrite(self,arg):self.file.write(arg)self.file.flush()def__getattr__(self,attr):returngetattr(self.file,attr)sys.stdout=Unbuffered(sys.stdout)iflist_only:fortintests:sys.stdout.write(t.abspath+"\n")sys.exit(0)#ifnotpython:ifos.name=='java':python=os.path.join(sys.prefix,'jython')else:python=sys.executable# time.clock() is the suggested interface for doing benchmarking timings,# but time.time() does a better job on Linux systems, so let that be# the non-Windows default.ifsys.platform=='win32':time_func=time.clockelse:time_func=time.timeifprint_times:print_time_func=lambdafmt,time:sys.stdout.write(fmt%time)else:print_time_func=lambdafmt,time:Nonetotal_start_time=time_func()fortintests:t.command_args=[python,'-tt']ifdebug:t.command_args.append(debug)t.command_args.append(t.abspath)t.command_str=string.join(map(escape,t.command_args)," ")ifprintcommand:sys.stdout.write(t.command_str+"\n")test_start_time=time_func()ifexecute_tests:t.execute()t.test_time=time_func()-test_start_timeprint_time_func("Test execution time: %.1f seconds\n",t.test_time)iflen(tests)>0:tests[0].total_time=time_func()-total_start_timeprint_time_func("Total execution time for all tests: %.1f seconds\n",tests[0].total_time)passed=filter(lambdat:t.status==0,tests)fail=filter(lambdat:t.status==1,tests)no_result=filter(lambdat:t.status==2,tests)iflen(tests)!=1andexecute_tests:ifpassedandprint_passed_summary:iflen(passed)==1:sys.stdout.write("\nPassed the following test:\n")else:sys.stdout.write("\nPassed the following %d tests:\n"%len(passed))paths=map(lambdax:x.path,passed)sys.stdout.write("\t"+string.join(paths,"\n\t")+"\n")iffail:iflen(fail)==1:sys.stdout.write("\nFailed the following test:\n")else:sys.stdout.write("\nFailed the following %d tests:\n"%len(fail))paths=map(lambdax:x.path,fail)sys.stdout.write("\t"+string.join(paths,"\n\t")+"\n")ifno_result:iflen(no_result)==1:sys.stdout.write("\nNO RESULT from the following test:\n")else:sys.stdout.write("\nNO RESULT from the following %d tests:\n"%len(no_result))paths=map(lambdax:x.path,no_result)sys.stdout.write("\t"+string.join(paths,"\n\t")+"\n")ifoutputfile:ifoutputfile=='-':f=sys.stdoutelse:f=open(outputfile,'w')tests[0].header(f)#f.write("test_result = [\n")fortintests:t.write(f)tests[0].footer(f)#f.write("];\n")ifoutputfile!='-':f.close()ifformat=='--aegis':sys.exit(0)eliflen(fail):sys.exit(1)eliflen(no_result):sys.exit(2)else:sys.exit(0)