# This Source Code Form is subject to the terms of the Mozilla Public# License, v. 2.0. If a copy of the MPL was not distributed with this# file, You can obtain one at http://mozilla.org/MPL/2.0/.from__future__importabsolute_import,print_function,unicode_literalsimportargparseimportcopyimportloggingimportreimportshlexfromcollectionsimportdefaultdictlogger=logging.getLogger(__name__)# The build type aliases are very cryptic and only used in try flags these are# mappings from the single char alias to a longer more recognizable form.BUILD_TYPE_ALIASES={'o':'opt','d':'debug'}# consider anything in this whitelist of kinds to be governed by -b/-pBUILD_KINDS=set(['build','artifact-build','hazard','l10n','valgrind','spidermonkey',])# mapping from shortcut name (usable with -u) to a boolean function identifying# matching test namesdefalias_prefix(prefix):returnlambdaname:name.startswith(prefix)defalias_contains(infix):returnlambdaname:infixinnamedefalias_matches(pattern):pattern=re.compile(pattern)returnlambdaname:pattern.match(name)UNITTEST_ALIASES={# Aliases specify shorthands that can be used in try syntax. The shorthand# is the dictionary key, with the value representing a pattern for matching# unittest_try_names.## Note that alias expansion is performed in the absence of any chunk# prefixes. For example, the first example above would replace "foo-7"# with "foobar-7". Note that a few aliases allowed chunks to be specified# without a leading `-`, for example 'mochitest-dt1'. That's no longer# supported.'cppunit':alias_prefix('cppunit'),'crashtest':alias_prefix('crashtest'),'crashtest-e10s':alias_prefix('crashtest-e10s'),'e10s':alias_contains('e10s'),'firefox-ui-functional':alias_prefix('firefox-ui-functional'),'firefox-ui-functional-e10s':alias_prefix('firefox-ui-functional-e10s'),'gaia-js-integration':alias_contains('gaia-js-integration'),'gtest':alias_prefix('gtest'),'jittest':alias_prefix('jittest'),'jittests':alias_prefix('jittest'),'jsreftest':alias_prefix('jsreftest'),'jsreftest-e10s':alias_prefix('jsreftest-e10s'),'marionette':alias_prefix('marionette'),'marionette-e10s':alias_prefix('marionette-e10s'),'mochitest':alias_prefix('mochitest'),'mochitests':alias_prefix('mochitest'),'mochitest-e10s':alias_prefix('mochitest-e10s'),'mochitests-e10s':alias_prefix('mochitest-e10s'),'mochitest-debug':alias_prefix('mochitest-debug-'),'mochitest-a11y':alias_contains('mochitest-a11y'),'mochitest-bc':alias_prefix('mochitest-browser-chrome'),'mochitest-e10s-bc':alias_prefix('mochitest-browser-chrome-e10s'),'mochitest-browser-chrome':alias_prefix('mochitest-browser-chrome'),'mochitest-e10s-browser-chrome':alias_prefix('mochitest-browser-chrome-e10s'),'mochitest-chrome':alias_contains('mochitest-chrome'),'mochitest-dt':alias_prefix('mochitest-devtools-chrome'),'mochitest-e10s-dt':alias_prefix('mochitest-devtools-chrome-e10s'),'mochitest-gl':alias_prefix('mochitest-webgl'),'mochitest-gl-e10s':alias_prefix('mochitest-webgl-e10s'),'mochitest-gpu':alias_prefix('mochitest-gpu'),'mochitest-gpu-e10s':alias_prefix('mochitest-gpu-e10s'),'mochitest-media':alias_prefix('mochitest-media'),'mochitest-media-e10s':alias_prefix('mochitest-media-e10s'),'mochitest-vg':alias_prefix('mochitest-valgrind'),'reftest':alias_matches(r'^(plain-)?reftest.*$'),'reftest-no-accel':alias_matches(r'^(plain-)?reftest-no-accel.*$'),'reftests':alias_matches(r'^(plain-)?reftest.*$'),'reftests-e10s':alias_matches(r'^(plain-)?reftest-e10s.*$'),'reftest-gpu':alias_matches(r'^(plain-)?reftest-gpu.*$'),'robocop':alias_prefix('robocop'),'web-platform-test':alias_prefix('web-platform-tests'),'web-platform-tests':alias_prefix('web-platform-tests'),'web-platform-tests-e10s':alias_prefix('web-platform-tests-e10s'),'web-platform-tests-reftests':alias_prefix('web-platform-tests-reftests'),'web-platform-tests-reftests-e10s':alias_prefix('web-platform-tests-reftests-e10s'),'web-platform-tests-wdspec':alias_prefix('web-platform-tests-wdspec'),'web-platform-tests-wdspec-e10s':alias_prefix('web-platform-tests-wdspec-e10s'),'xpcshell':alias_prefix('xpcshell'),}# unittest platforms can be specified by substring of the "pretty name", which# is basically the old Buildbot builder name. This dict has {pretty name,# [test_platforms]} translations, This includes only the most commonly-used# substrings. It is OK to add new test platforms to various shorthands here;# if you add a new Linux64 test platform for instance, people will expect that# their previous methods of requesting "all linux64 tests" will include this# new platform, and they shouldn't have to explicitly spell out the new platform# every time for such cases.## Note that the test platforms here are only the prefix up to the `/`.UNITTEST_PLATFORM_PRETTY_NAMES={'Ubuntu':['linux32','linux64','linux64-asan','linux64-stylo-sequential'],'x64':['linux64','linux64-asan','linux64-stylo-sequential'],'Android 4.3 Emulator':['android-em-4.3-arm7-api-16'],'Android 4.3 Emulator PGO':['android-em-4-3-armv7-api16-pgo'],'Android 7.0 Moto G5 32bit':['android-hw-g5-7.0-arm7-api-16'],'Android 8.0 Google Pixel 2 32bit':['android-hw-p2-8.0-arm7-api-16'],'Android 8.0 Google Pixel 2 64bit':['android-hw-p2-8.0-android-aarch64'],'10.10':['macosx1010-64'],'10.14':['macosx1014-64'],# other commonly-used substrings for platforms not yet supported with# in-tree taskgraphs:# '10.10.5': [..TODO..],# '10.6': [..TODO..],# '10.8': [..TODO..],# 'Android 2.3 API9': [..TODO..],'Windows 7':['windows7-32'],'Windows 7 VM':['windows7-32-vm'],'Windows 8':['windows8-64'],'Windows 10':['windows10-64'],# 'Windows XP': [..TODO..],# 'win32': [..TODO..],# 'win64': [..TODO..],}TEST_CHUNK_SUFFIX=re.compile('(.*)-([0-9]+)$')defescape_whitespace_in_brackets(input_str):''' In tests you may restrict them by platform [] inside of the brackets whitespace may occur this is typically invalid shell syntax so we escape it with backslash sequences . '''result=""in_brackets=Falseforcharininput_str:ifchar=='[':in_brackets=Trueresult+=charcontinueifchar==']':in_brackets=Falseresult+=charcontinueifchar==' 'andin_brackets:result+='\ 'continueresult+=charreturnresultdefsplit_try_msg(message):try:try_idx=message.index('try:')exceptValueError:return[]message=message[try_idx:].split('\n')[0]# shlex used to ensure we split correctly when giving values to argparse.returnshlex.split(escape_whitespace_in_brackets(message))defparse_message(message):parts=split_try_msg(message)# Argument parser based on try flag flagsparser=argparse.ArgumentParser()parser.add_argument('-b','--build',dest='build_types')parser.add_argument('-p','--platform',nargs='?',dest='platforms',const='all',default='all')parser.add_argument('-u','--unittests',nargs='?',dest='unittests',const='all',default='all')parser.add_argument('-t','--talos',nargs='?',dest='talos',const='all',default='none')parser.add_argument('-r','--raptor',nargs='?',dest='raptor',const='all',default='none')parser.add_argument('-i','--interactive',dest='interactive',action='store_true',default=False)parser.add_argument('-e','--all-emails',dest='notifications',action='store_const',const='all')parser.add_argument('-f','--failure-emails',dest='notifications',action='store_const',const='failure')parser.add_argument('-j','--job',dest='jobs',action='append')parser.add_argument('--rebuild-talos',dest='talos_trigger_tests',action='store',type=int,default=1)parser.add_argument('--rebuild-raptor',dest='raptor_trigger_tests',action='store',type=int,default=1)parser.add_argument('--setenv',dest='env',action='append')parser.add_argument('--geckoProfile',dest='profile',action='store_true')parser.add_argument('--tag',dest='tag',action='store',default=None)parser.add_argument('--no-retry',dest='no_retry',action='store_true')parser.add_argument('--include-nightly',dest='include_nightly',action='store_true')parser.add_argument('--artifact',dest='artifact',action='store_true')# While we are transitioning from BB to TC, we want to push jobs to tc-worker# machines but not overload machines with every try push. Therefore, we add# this temporary option to be able to push jobs to tc-worker.parser.add_argument('-w','--taskcluster-worker',dest='taskcluster_worker',action='store_true',default=False)# In order to run test jobs multiple timesparser.add_argument('--rebuild',dest='trigger_tests',type=int,default=1)args,_=parser.parse_known_args(parts)returnvars(args)classTryOptionSyntax(object):def__init__(self,parameters,full_task_graph,graph_config):""" Apply the try options in parameters. The resulting object has attributes: - build_types: a list containing zero or more of 'opt' and 'debug' - platforms: a list of selected platform names, or None for all - unittests: a list of tests, of the form given below, or None for all - jobs: a list of requested job names, or None for all - trigger_tests: the number of times tests should be triggered (--rebuild) - interactive: true if --interactive - notifications: either None if no notifications or one of 'all' or 'failure' - talos_trigger_tests: the number of time talos tests should be triggered (--rebuild-talos) - env: additional environment variables (ENV=value) - profile: run talos in profile mode - tag: restrict tests to the specified tag - no_retry: do not retry failed jobs The unittests and talos lists contain dictionaries of the form: { 'test': '<suite name>', 'platforms': [..platform names..], # to limit to only certain platforms 'only_chunks': set([..chunk numbers..]), # to limit only to certain chunks } """self.graph_config=graph_configself.jobs=[]self.build_types=[]self.platforms=[]self.unittests=[]self.talos=[]self.raptor=[]self.trigger_tests=0self.interactive=Falseself.notifications=Noneself.talos_trigger_tests=0self.raptor_trigger_tests=0self.env=[]self.profile=Falseself.tag=Noneself.no_retry=Falseself.artifact=Falseoptions=parameters['try_options']ifnotoptions:returnNoneself.jobs=self.parse_jobs(options['jobs'])self.build_types=self.parse_build_types(options['build_types'],full_task_graph)self.platforms=self.parse_platforms(options['platforms'],full_task_graph)self.unittests=self.parse_test_option("unittest_try_name",options['unittests'],full_task_graph)self.talos=self.parse_test_option("talos_try_name",options['talos'],full_task_graph)self.raptor=self.parse_test_option("raptor_try_name",options['raptor'],full_task_graph)self.trigger_tests=options['trigger_tests']self.interactive=options['interactive']self.notifications=options['notifications']self.talos_trigger_tests=options['talos_trigger_tests']self.raptor_trigger_tests=options['raptor_trigger_tests']self.env=options['env']self.profile=options['profile']self.tag=options['tag']self.no_retry=options['no_retry']self.artifact=options['artifact']self.include_nightly=options['include_nightly']self.test_tiers=self.generate_test_tiers(full_task_graph)defgenerate_test_tiers(self,full_task_graph):retval=defaultdict(set)fortinfull_task_graph.tasks.itervalues():ift.attributes.get('kind')=='test':try:tier=t.task['extra']['treeherder']['tier']name=t.attributes.get('unittest_try_name')retval[name].add(tier)exceptKeyError:passreturnretvaldefparse_jobs(self,jobs_arg):ifnotjobs_argorjobs_arg==['none']:return[]# default is `-j none`ifjobs_arg==['all']:returnNoneexpanded=[]forjobinjobs_arg:expanded.extend(j.strip()forjinjob.split(','))returnexpandeddefparse_build_types(self,build_types_arg,full_task_graph):ifbuild_types_argisNone:build_types_arg=[]build_types=filter(None,[BUILD_TYPE_ALIASES.get(build_type)forbuild_typeinbuild_types_arg])all_types=set(t.attributes['build_type']fortinfull_task_graph.tasks.itervalues()if'build_type'int.attributes)bad_types=set(build_types)-all_typesifbad_types:raiseException("Unknown build type(s) [%s] specified for try"%','.join(bad_types))returnbuild_typesdefparse_platforms(self,platform_arg,full_task_graph):ifplatform_arg=='all':returnNoneRIDEALONG_BUILDS=self.graph_config['try']['ridealong-builds']results=[]forbuildinplatform_arg.split(','):results.append(build)ifbuildin('macosx64',):results.append('macosx64-shippable')logger.info("adding macosx64-shippable for try syntax using macosx64.")ifbuildinRIDEALONG_BUILDS:results.extend(RIDEALONG_BUILDS[build])logger.info("platform %s triggers ridealong builds %s"%(build,', '.join(RIDEALONG_BUILDS[build])))test_platforms=set(t.attributes['test_platform']fortinfull_task_graph.tasks.itervalues()if'test_platform'int.attributes)build_platforms=set(t.attributes['build_platform']fortinfull_task_graph.tasks.itervalues()if'build_platform'int.attributes)all_platforms=test_platforms|build_platformsbad_platforms=set(results)-all_platformsifbad_platforms:raiseException("Unknown platform(s) [%s] specified for try"%','.join(bad_platforms))returnresultsdefparse_test_option(self,attr_name,test_arg,full_task_graph):''' Parse a unittest (-u) or talos (-t) option, in the context of a full task graph containing available `unittest_try_name` or `talos_try_name` attributes. There are three cases: - test_arg is == 'none' (meaning an empty list) - test_arg is == 'all' (meaning use the list of jobs for that job type) - test_arg is comma string which needs to be parsed '''# Empty job list case...iftest_argisNoneortest_arg=='none':return[]all_platforms=set(t.attributes['test_platform'].split('/')[0]fortinfull_task_graph.tasks.itervalues()if'test_platform'int.attributes)tests=self.parse_test_opts(test_arg,all_platforms)ifnottests:return[]all_tests=set(t.attributes[attr_name]fortinfull_task_graph.tasks.itervalues()ifattr_nameint.attributes)# Special case where tests is 'all' and must be expandediftests[0]['test']=='all':results=[]all_entry=tests[0]fortestinall_tests:entry={'test':test}# If there are platform restrictions copy them across the list.if'platforms'inall_entry:entry['platforms']=list(all_entry['platforms'])results.append(entry)returnself.parse_test_chunks(all_tests,results)else:returnself.parse_test_chunks(all_tests,tests)defparse_test_opts(self,input_str,all_platforms):''' Parse `testspec,testspec,..`, where each testspec is a test name optionally followed by a list of test platforms or negated platforms in `[]`. No brackets indicates that tests should run on all platforms for which builds are available. If testspecs are provided, then each is treated, from left to right, as an instruction to include or (if negated) exclude a set of test platforms. A single spec may expand to multiple test platforms via UNITTEST_PLATFORM_PRETTY_NAMES. If the first test spec is negated, processing begins with the full set of available test platforms; otherwise, processing begins with an empty set of test platforms. '''# Final results which we will return.tests=[]cur_test={}token=''in_platforms=Falsedefnormalize_platforms():if'platforms'notincur_test:return# if the first spec is a negation, start with all platformsifcur_test['platforms'][0][0]=='-':platforms=all_platforms.copy()else:platforms=[]forplatformincur_test['platforms']:ifplatform[0]=='-':platforms=[pforpinplatformsifp!=platform[1:]]else:platforms.append(platform)cur_test['platforms']=platformsdefadd_test(value):normalize_platforms()cur_test['test']=value.strip()tests.insert(0,cur_test)defadd_platform(value):platform=value.strip()ifplatform[0]=='-':negated=Trueplatform=platform[1:]else:negated=Falseplatforms=UNITTEST_PLATFORM_PRETTY_NAMES.get(platform,[platform])ifnegated:platforms=["-"+pforpinplatforms]cur_test['platforms']=platforms+cur_test.get('platforms',[])# This might be somewhat confusing but we parse the string _backwards_ so# there is no ambiguity over what state we are in.forcharinreversed(input_str):# , indicates exiting a stateifchar==',':# Exit a particular platform.ifin_platforms:add_platform(token)# Exit a particular test.else:add_test(token)cur_test={}# Token must always be reset after we exit a statetoken=''elifchar=='[':# Exiting platform state entering test state.add_platform(token)token=''in_platforms=Falseelifchar==']':# Entering platform state.in_platforms=Trueelse:# Accumulator.token=char+token# Handle any left over tokens.iftoken:add_test(token)returntestsdefhandle_alias(self,test,all_tests):''' Expand a test if its name refers to an alias, returning a list of test dictionaries cloned from the first (to maintain any metadata). '''iftest['test']notinUNITTEST_ALIASES:return[test]alias=UNITTEST_ALIASES[test['test']]defmktest(name):newtest=copy.deepcopy(test)newtest['test']=namereturnnewtestdefexprmatch(alias):return[tfortinall_testsifalias(t)]return[mktest(t)fortinexprmatch(alias)]defparse_test_chunks(self,all_tests,tests):''' Test flags may include parameters to narrow down the number of chunks in a given push. We don't model 1 chunk = 1 job in taskcluster so we must check each test flag to see if it is actually specifying a chunk. '''results=[]seen_chunks={}fortestintests:matches=TEST_CHUNK_SUFFIX.match(test['test'])ifmatches:name=matches.group(1)chunk=matches.group(2)ifnameinseen_chunks:seen_chunks[name].add(chunk)else:seen_chunks[name]={chunk}test['test']=nametest['only_chunks']=seen_chunks[name]results.append(test)else:results.extend(self.handle_alias(test,all_tests))# uniquify the results over the test namesresults={test['test']:testfortestinresults}.values()returnresultsdeffind_all_attribute_suffixes(self,graph,prefix):rv=set()fortingraph.tasks.itervalues():foraint.attributes:ifa.startswith(prefix):rv.add(a[len(prefix):])returnsorted(rv)deftask_matches(self,task):attr=task.attributes.getdefcheck_run_on_projects():ifattr('nightly')andnotself.include_nightly:returnFalsereturnset(['try','all'])&set(attr('run_on_projects',[]))# Don't schedule code coverage when try option syntax is usedif'ccov'inattr('build_platform',[]):returnFalse# Don't schedule tasks for windows10-aarch64 unless try fuzzy is usedif'windows10-aarch64'inattr("test_platform",""):returnFalse# Don't schedule android-hw tests when try option syntax is usedif'android-hw'intask.label:returnFalsedefmatch_test(try_spec,attr_name):run_by_default=Trueifattr('build_type')notinself.build_types:returnFalseifself.platformsisnotNone:ifattr('build_platform')notinself.platforms:returnFalseelse:ifnotcheck_run_on_projects():run_by_default=Falseiftry_specisNone:returnrun_by_default# TODO: optimize this search a bitfortestintry_spec:ifattr(attr_name)==test['test']:breakelse:returnFalseif'only_chunks'intestandattr('test_chunk')notintest['only_chunks']:returnFalsetier=task.task['extra']['treeherder']['tier']if'platforms'intest:if'all'intest['platforms']:returnTrueplatform=attr('test_platform','').split('/')[0]# Platforms can be forced by syntax like "-u xpcshell[Windows 8]"returnplatformintest['platforms']eliftier!=1:# Require tier 2/3 tests to be specifically enabled if there# are other platforms that run this test suite as tier 1name=attr('unittest_try_name')test_tiers=self.test_tiers.get(name)if1notintest_tiers:logger.debug("not skipping tier {} test without explicit inclusion: {}; ""it is configured to run on tiers {}".format(tier,task.label,test_tiers))returnTrueelse:logger.debug("skipping mixed tier {} (of {}) test without explicit inclusion: {}".format(tier,test_tiers,task.label))returnFalseelifrun_by_default:returncheck_run_on_projects()else:returnFalseifattr('job_try_name'):# Beware the subtle distinction between [] and None for self.jobs and self.platforms.# They will be [] if there was no try syntax, and None if try syntax was detected but# they remained unspecified.ifself.jobsisnotNone:returnattr('job_try_name')inself.jobs# User specified `-j all`ifself.platformsisnotNoneandattr('build_platform')notinself.platforms:returnFalse# honor -p for jobs governed by a platform# "all" means "everything with `try` in run_on_projects"returncheck_run_on_projects()elifattr('kind')=='test':returnmatch_test(self.unittests,'unittest_try_name') \ormatch_test(self.talos,'talos_try_name') \ormatch_test(self.raptor,'raptor_try_name')elifattr('kind')inBUILD_KINDS:ifattr('build_type')notinself.build_types:returnFalseelifself.platformsisNone:# for "-p all", look for try in the 'run_on_projects' attributereturncheck_run_on_projects()else:ifattr('build_platform')notinself.platforms:returnFalsereturnTrueelse:returnFalsedef__str__(self):defnone_for_all(list):iflistisNone:return'<all>'return', '.join(str(e)foreinlist)return"\n".join(["build_types: "+", ".join(self.build_types),"platforms: "+none_for_all(self.platforms),"unittests: "+none_for_all(self.unittests),"talos: "+none_for_all(self.talos),"raptor"+none_for_all(self.raptor),"jobs: "+none_for_all(self.jobs),"trigger_tests: "+str(self.trigger_tests),"interactive: "+str(self.interactive),"notifications: "+str(self.notifications),"talos_trigger_tests: "+str(self.talos_trigger_tests),"raptor_trigger_tests: "+str(self.raptor_trigger_tests),"env: "+str(self.env),"profile: "+str(self.profile),"tag: "+str(self.tag),"no_retry: "+str(self.no_retry),"artifact: "+str(self.artifact),])