1# 2# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 The SCons Foundation 3# 4# Permission is hereby granted, free of charge, to any person obtaining 5# a copy of this software and associated documentation files (the 6# "Software"), to deal in the Software without restriction, including 7# without limitation the rights to use, copy, modify, merge, publish, 8# distribute, sublicense, and/or sell copies of the Software, and to 9# permit persons to whom the Software is furnished to do so, subject to 10# the following conditions: 11# 12# The above copyright notice and this permission notice shall be included 13# in all copies or substantial portions of the Software. 14# 15# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 16# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 17# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 23__doc__=""" 24Generic Taskmaster module for the SCons build engine. 25 26This module contains the primary interface(s) between a wrapping user 27interface and the SCons build engine. There are two key classes here: 28 29 Taskmaster 30 This is the main engine for walking the dependency graph and 31 calling things to decide what does or doesn't need to be built. 32 33 Task 34 This is the base class for allowing a wrapping interface to 35 decide what does or doesn't actually need to be done. The 36 intention is for a wrapping interface to subclass this as 37 appropriate for different types of behavior it may need. 38 39 The canonical example is the SCons native Python interface, 40 which has Task subclasses that handle its specific behavior, 41 like printing "`foo' is up to date" when a top-level target 42 doesn't need to be built, and handling the -c option by removing 43 targets as its "build" action. There is also a separate subclass 44 for suppressing this output when the -q option is used. 45 46 The Taskmaster instantiates a Task object for each (set of) 47 target(s) that it decides need to be evaluated and/or built. 48""" 49 50__revision__="src/engine/SCons/Taskmaster.py issue-2856:2676:d23b7a2f45e8 2012/08/05 15:38:28 garyo" 51 52fromitertoolsimportchain 53importoperator 54importsys 55importtraceback 56 57importSCons.Errors 58importSCons.Node 59importSCons.Warnings 60 61StateString=SCons.Node.StateString 62NODE_NO_STATE=SCons.Node.no_state 63NODE_PENDING=SCons.Node.pending 64NODE_EXECUTING=SCons.Node.executing 65NODE_UP_TO_DATE=SCons.Node.up_to_date 66NODE_EXECUTED=SCons.Node.executed 67NODE_FAILED=SCons.Node.failed 68 69print_prepare=0# set by option --debug=prepare 70 71# A subsystem for recording stats about how different Nodes are handled by 72# the main Taskmaster loop. There's no external control here (no need for 73# a --debug= option); enable it by changing the value of CollectStats. 74 75CollectStats=None 76

78""" 79 A simple class for holding statistics about the disposition of a 80 Node by the Taskmaster. If we're collecting statistics, each Node 81 processed by the Taskmaster gets one of these attached, in which case 82 the Taskmaster records its decision each time it processes the Node. 83 (Ideally, that's just once per Node.) 84 """

115""" 116 Default SCons build engine task. 117 118 This controls the interaction of the actual building of node 119 and the rest of the engine. 120 121 This is expected to handle all of the normally-customizable 122 aspects of controlling a build, so any given application 123 *should* be able to do what it wants by sub-classing this 124 class and overriding methods as appropriate. If an application 125 needs to customze something by sub-classing Taskmaster (or 126 some other build engine class), we should first try to migrate 127 that functionality into this class. 128 129 Note that it's generally a good idea for sub-classes to call 130 these methods explicitly to update state, etc., rather than 131 roll their own interaction with Taskmaster from scratch. 132 """

145""" 146 Hook to allow the calling interface to display a message. 147 148 This hook gets called as part of preparing a task for execution 149 (that is, a Node to be built). As part of figuring out what Node 150 should be built next, the actually target list may be altered, 151 along with a message describing the alteration. The calling 152 interface can subclass Task and provide a concrete implementation 153 of this method to see those messages. 154 """ 155pass

158""" 159 Called just before the task is executed. 160 161 This is mainly intended to give the target Nodes a chance to 162 unlink underlying files and make all necessary directories before 163 the Action is actually called to build the targets. 164 """ 165globalprint_prepare 166T=self.tm.trace 167ifT:T.write(self.trace_message(u'Task.prepare()',self.node)) 168 169# Now that it's the appropriate time, give the TaskMaster a 170# chance to raise any exceptions it encountered while preparing 171# this task. 172self.exception_raise() 173 174ifself.tm.message: 175self.display(self.tm.message) 176self.tm.message=None 177 178# Let the targets take care of any necessary preparations. 179# This includes verifying that all of the necessary sources 180# and dependencies exist, removing the target file(s), etc. 181# 182# As of April 2008, the get_executor().prepare() method makes 183# sure that all of the aggregate sources necessary to build this 184# Task's target(s) exist in one up-front check. The individual 185# target t.prepare() methods check that each target's explicit 186# or implicit dependencies exists, and also initialize the 187# .sconsign info. 188executor=self.targets[0].get_executor() 189executor.prepare() 190fortinexecutor.get_action_targets(): 191ifprint_prepare: 192print"Preparing target %s..."%t 193forsint.side_effects: 194print"...with side-effect %s..."%s 195t.prepare() 196forsint.side_effects: 197ifprint_prepare: 198print"...Preparing side-effect %s..."%s 199s.prepare()

207# TODO(deprecate): "return True" is the old default behavior; 208# change it to NotImplementedError (after running through the 209# Deprecation Cycle) so the desired behavior is explicitly 210# determined by which concrete subclass is used. 211#raise NotImplementedError 212msg=('Taskmaster.Task is an abstract base class; instead of\n' 213'\tusing it directly, ' 214'derive from it and override the abstract methods.') 215SCons.Warnings.warn(SCons.Warnings.TaskmasterNeedsExecuteWarning,msg) 216returnTrue

280""" 281 Called when the task has been successfully executed and 282 the Taskmaster instance wants to call the Node's callback 283 methods. 284 285 This may have been a do-nothing operation (to preserve build 286 order), so we must check the node's state before deciding whether 287 it was "built", in which case we call the appropriate Node method. 288 In any event, we always call "visited()", which will handle any 289 post-visit actions that must take place regardless of whether 290 or not the target was an actual built target or a source Node. 291 """ 292T=self.tm.trace 293ifT:T.write(self.trace_message('Task.executed_with_callbacks()', 294self.node)) 295 296fortinself.targets: 297ift.get_state()==NODE_EXECUTING: 298forside_effectint.side_effects: 299side_effect.set_state(NODE_NO_STATE) 300t.set_state(NODE_EXECUTED) 301ifnott.cached: 302t.push_to_cache() 303t.built() 304t.visited()

309""" 310 Default action when a task fails: stop the build. 311 312 Note: Although this function is normally invoked on nodes in 313 the executing state, it might also be invoked on up-to-date 314 nodes when using Configure(). 315 """ 316self.fail_stop()

319""" 320 Explicit stop-the-build failure. 321 322 This sets failure status on the target nodes and all of 323 their dependent parent nodes. 324 325 Note: Although this function is normally invoked on nodes in 326 the executing state, it might also be invoked on up-to-date 327 nodes when using Configure(). 328 """ 329T=self.tm.trace 330ifT:T.write(self.trace_message('Task.failed_stop()',self.node)) 331 332# Invoke will_not_build() to clean-up the pending children 333# list. 334self.tm.will_not_build(self.targets,lambdan:n.set_state(NODE_FAILED)) 335 336# Tell the taskmaster to not start any new tasks 337self.tm.stop() 338 339# We're stopping because of a build failure, but give the 340# calling Task class a chance to postprocess() the top-level 341# target under which the build failure occurred. 342self.targets=[self.tm.current_top] 343self.top=1

346""" 347 Explicit continue-the-build failure. 348 349 This sets failure status on the target nodes and all of 350 their dependent parent nodes. 351 352 Note: Although this function is normally invoked on nodes in 353 the executing state, it might also be invoked on up-to-date 354 nodes when using Configure(). 355 """ 356T=self.tm.trace 357ifT:T.write(self.trace_message('Task.failed_continue()',self.node)) 358 359self.tm.will_not_build(self.targets,lambdan:n.set_state(NODE_FAILED))

421""" 422 Post-processes a task after it's been executed. 423 424 This examines all the targets just built (or not, we don't care 425 if the build was successful, or even if there was no build 426 because everything was up-to-date) to see if they have any 427 waiting parent Nodes, or Nodes waiting on a common side effect, 428 that can be put back on the candidates list. 429 """ 430T=self.tm.trace 431ifT:T.write(self.trace_message(u'Task.postprocess()',self.node)) 432 433# We may have built multiple targets, some of which may have 434# common parents waiting for this build. Count up how many 435# targets each parent was waiting for so we can subtract the 436# values later, and so we *don't* put waiting side-effect Nodes 437# back on the candidates list if the Node is also a waiting 438# parent. 439 440targets=set(self.targets) 441 442pending_children=self.tm.pending_children 443parents={} 444fortintargets: 445# A node can only be in the pending_children set if it has 446# some waiting_parents. 447ift.waiting_parents: 448ifT:T.write(self.trace_message(u'Task.postprocess()', 449t, 450'removing')) 451pending_children.discard(t) 452forpint.waiting_parents: 453parents[p]=parents.get(p,0)+1 454 455fortintargets: 456forsint.side_effects: 457ifs.get_state()==NODE_EXECUTING: 458s.set_state(NODE_NO_STATE) 459forpins.waiting_parents: 460parents[p]=parents.get(p,0)+1 461forpins.waiting_s_e: 462ifp.ref_count==0: 463self.tm.candidates.append(p) 464 465forp,subtractinparents.items(): 466p.ref_count=p.ref_count-subtract 467ifT:T.write(self.trace_message(u'Task.postprocess()', 468p, 469'adjusted parent ref count')) 470ifp.ref_count==0: 471self.tm.candidates.append(p) 472 473fortintargets: 474t.postprocess()

475 476# Exception handling subsystem. 477# 478# Exceptions that occur while walking the DAG or examining Nodes 479# must be raised, but must be raised at an appropriate time and in 480# a controlled manner so we can, if necessary, recover gracefully, 481# possibly write out signature information for Nodes we've updated, 482# etc. This is done by having the Taskmaster tell us about the 483# exception, and letting 484

492""" 493 Clears any recorded exception. 494 495 This also changes the "exception_raise" attribute to point 496 to the appropriate do-nothing method. 497 """ 498self.exception=(None,None,None) 499self.exception_raise=self._no_exception_to_raise

502""" 503 Records an exception to be raised at the appropriate time. 504 505 This also changes the "exception_raise" attribute to point 506 to the method that will, in fact 507 """ 508ifnotexception: 509exception=sys.exc_info() 510self.exception=exception 511self.exception_raise=self._exception_raise

531""" 532 Always returns True (indicating this Task should always 533 be executed). 534 535 Subclasses that need this behavior (as opposed to the default 536 of only executing Nodes that are out of date w.r.t. their 537 dependencies) can use this as follows: 538 539 class MyTaskSubclass(SCons.Taskmaster.Task): 540 needs_execute = SCons.Taskmaster.Task.execute_always 541 """ 542returnTrue

546""" 547 Returns True (indicating this Task should be executed) if this 548 Task's target state indicates it needs executing, which has 549 already been determined by an earlier up-to-date check. 550 """ 551returnself.targets[0].get_state()==SCons.Node.executing

590""" 591 Returns the next candidate Node for (potential) evaluation. 592 593 The candidate list (really a stack) initially consists of all of 594 the top-level (command line) targets provided when the Taskmaster 595 was initialized. While we walk the DAG, visiting Nodes, all the 596 children that haven't finished processing get pushed on to the 597 candidate list. Each child can then be popped and examined in 598 turn for whether *their* children are all up-to-date, in which 599 case a Task will be created for their actual evaluation and 600 potential building. 601 602 Here is where we also allow candidate Nodes to alter the list of 603 Nodes that should be examined. This is used, for example, when 604 invoking SCons in a source directory. A source directory Node can 605 return its corresponding build directory Node, essentially saying, 606 "Hey, you really need to build this thing over here instead." 607 """ 608try: 609returnself.candidates.pop() 610exceptIndexError: 611pass 612try: 613node=self.top_targets_left.pop() 614exceptIndexError: 615returnNone 616self.current_top=node 617alt,message=node.alter_targets() 618ifalt: 619self.message=message 620self.candidates.append(node) 621self.candidates.extend(self.order(alt)) 622node=self.candidates.pop() 623returnnode

626""" 627 Stops Taskmaster processing by not returning a next candidate. 628 629 Note that we have to clean-up the Taskmaster candidate list 630 because the cycle detection depends on the fact all nodes have 631 been processed somehow. 632 """ 633whileself.candidates: 634candidates=self.candidates 635self.candidates=[] 636self.will_not_build(candidates) 637returnNone

640""" 641 Validate the content of the pending_children set. Assert if an 642 internal error is found. 643 644 This function is used strictly for debugging the taskmaster by 645 checking that no invariants are violated. It is not used in 646 normal operation. 647 648 The pending_children set is used to detect cycles in the 649 dependency graph. We call a "pending child" a child that is 650 found in the "pending" state when checking the dependencies of 651 its parent node. 652 653 A pending child can occur when the Taskmaster completes a loop 654 through a cycle. For example, lets imagine a graph made of 655 three node (A, B and C) making a cycle. The evaluation starts 656 at node A. The taskmaster first consider whether node A's 657 child B is up-to-date. Then, recursively, node B needs to 658 check whether node C is up-to-date. This leaves us with a 659 dependency graph looking like: 660 661 Next candidate \ 662 \ 663 Node A (Pending) --> Node B(Pending) --> Node C (NoState) 664 ^ | 665 | | 666 +-------------------------------------+ 667 668 Now, when the Taskmaster examines the Node C's child Node A, 669 it finds that Node A is in the "pending" state. Therefore, 670 Node A is a pending child of node C. 671 672 Pending children indicate that the Taskmaster has potentially 673 loop back through a cycle. We say potentially because it could 674 also occur when a DAG is evaluated in parallel. For example, 675 consider the following graph: 676 677 678 Node A (Pending) --> Node B(Pending) --> Node C (Pending) --> ... 679 | ^ 680 | | 681 +----------> Node D (NoState) --------+ 682 / 683 Next candidate / 684 685 The Taskmaster first evaluates the nodes A, B, and C and 686 starts building some children of node C. Assuming, that the 687 maximum parallel level has not been reached, the Taskmaster 688 will examine Node D. It will find that Node C is a pending 689 child of Node D. 690 691 In summary, evaluating a graph with a cycle will always 692 involve a pending child at one point. A pending child might 693 indicate either a cycle or a diamond-shaped DAG. Only a 694 fraction of the nodes ends-up being a "pending child" of 695 another node. This keeps the pending_children set small in 696 practice. 697 698 We can differentiate between the two cases if we wait until 699 the end of the build. At this point, all the pending children 700 nodes due to a diamond-shaped DAG will have been properly 701 built (or will have failed to build). But, the pending 702 children involved in a cycle will still be in the pending 703 state. 704 705 The taskmaster removes nodes from the pending_children set as 706 soon as a pending_children node moves out of the pending 707 state. This also helps to keep the pending_children set small. 708 """ 709 710forninself.pending_children: 711assertn.statein(NODE_PENDING,NODE_EXECUTING), \ 712(str(n),StateString[n.state]) 713assertlen(n.waiting_parents)!=0,(str(n),len(n.waiting_parents)) 714forpinn.waiting_parents: 715assertp.ref_count>0,(str(n),str(p),p.ref_count)

727""" 728 Finds the next node that is ready to be built. 729 730 This is *the* main guts of the DAG walk. We loop through the 731 list of candidates, looking for something that has no un-built 732 children (i.e., that is a leaf Node or has dependencies that are 733 all leaf Nodes or up-to-date). Candidate Nodes are re-scanned 734 (both the target Node itself and its sources, which are always 735 scanned in the context of a given target) to discover implicit 736 dependencies. A Node that must wait for some children to be 737 built will be put back on the candidates list after the children 738 have finished building. A Node that has been put back on the 739 candidates list in this way may have itself (or its sources) 740 re-scanned, in order to handle generated header files (e.g.) and 741 the implicit dependencies therein. 742 743 Note that this method does not do any signature calculation or 744 up-to-date check itself. All of that is handled by the Task 745 class. This is purely concerned with the dependency graph walk. 746 """ 747 748self.ready_exc=None 749 750T=self.trace 751ifT:T.write(u'\n'+self.trace_message('Looking for a node to evaluate')) 752 753whileTrue: 754node=self.next_candidate() 755ifnodeisNone: 756ifT:T.write(self.trace_message('No candidate anymore.')+u'\n') 757returnNone 758 759node=node.disambiguate() 760state=node.get_state() 761 762# For debugging only: 763# 764# try: 765# self._validate_pending_children() 766# except: 767# self.ready_exc = sys.exc_info() 768# return node 769 770ifCollectStats: 771ifnothasattr(node,'stats'): 772node.stats=Stats() 773StatsNodes.append(node) 774S=node.stats 775S.considered=S.considered+1 776else: 777S=None 778 779ifT:T.write(self.trace_message(u' Considering node %s and its children:'%self.trace_node(node))) 780 781ifstate==NODE_NO_STATE: 782# Mark this node as being on the execution stack: 783node.set_state(NODE_PENDING) 784elifstate>NODE_PENDING: 785# Skip this node if it has already been evaluated: 786ifS:S.already_handled=S.already_handled+1 787ifT:T.write(self.trace_message(u' already handled (executed)')) 788continue 789 790executor=node.get_executor() 791 792try: 793children=executor.get_all_children() 794exceptSystemExit: 795exc_value=sys.exc_info()[1] 796e=SCons.Errors.ExplicitExit(node,exc_value.code) 797self.ready_exc=(SCons.Errors.ExplicitExit,e) 798ifT:T.write(self.trace_message(' SystemExit')) 799returnnode 800exceptException,e: 801# We had a problem just trying to figure out the 802# children (like a child couldn't be linked in to a 803# VariantDir, or a Scanner threw something). Arrange to 804# raise the exception when the Task is "executed." 805self.ready_exc=sys.exc_info() 806ifS:S.problem=S.problem+1 807ifT:T.write(self.trace_message(' exception %s while scanning children.\n'%e)) 808returnnode 809 810children_not_visited=[] 811children_pending=set() 812children_not_ready=[] 813children_failed=False 814 815forchildinchain(executor.get_all_prerequisites(),children): 816childstate=child.get_state() 817 818ifT:T.write(self.trace_message(u' '+self.trace_node(child))) 819 820ifchildstate==NODE_NO_STATE: 821children_not_visited.append(child) 822elifchildstate==NODE_PENDING: 823children_pending.add(child) 824elifchildstate==NODE_FAILED: 825children_failed=True 826 827ifchildstate<=NODE_EXECUTING: 828children_not_ready.append(child) 829 830 831# These nodes have not even been visited yet. Add 832# them to the list so that on some next pass we can 833# take a stab at evaluating them (or their children). 834children_not_visited.reverse() 835self.candidates.extend(self.order(children_not_visited)) 836#if T and children_not_visited: 837# T.write(self.trace_message(' adding to candidates: %s' % map(str, children_not_visited))) 838# T.write(self.trace_message(' candidates now: %s\n' % map(str, self.candidates))) 839 840# Skip this node if any of its children have failed. 841# 842# This catches the case where we're descending a top-level 843# target and one of our children failed while trying to be 844# built by a *previous* descent of an earlier top-level 845# target. 846# 847# It can also occur if a node is reused in multiple 848# targets. One first descends though the one of the 849# target, the next time occurs through the other target. 850# 851# Note that we can only have failed_children if the 852# --keep-going flag was used, because without it the build 853# will stop before diving in the other branch. 854# 855# Note that even if one of the children fails, we still 856# added the other children to the list of candidate nodes 857# to keep on building (--keep-going). 858ifchildren_failed: 859forninexecutor.get_action_targets(): 860n.set_state(NODE_FAILED) 861 862ifS:S.child_failed=S.child_failed+1 863ifT:T.write(self.trace_message('****** %s\n'%self.trace_node(node))) 864continue 865 866ifchildren_not_ready: 867forchildinchildren_not_ready: 868# We're waiting on one or more derived targets 869# that have not yet finished building. 870ifS:S.not_built=S.not_built+1 871 872# Add this node to the waiting parents lists of 873# anything we're waiting on, with a reference 874# count so we can be put back on the list for 875# re-evaluation when they've all finished. 876node.ref_count=node.ref_count+child.add_to_waiting_parents(node) 877ifT:T.write(self.trace_message(u' adjusted ref count: %s, child %s'% 878(self.trace_node(node),repr(str(child))))) 879 880ifT: 881forpcinchildren_pending: 882T.write(self.trace_message(' adding %s to the pending children set\n'% 883self.trace_node(pc))) 884self.pending_children=self.pending_children|children_pending 885 886continue 887 888# Skip this node if it has side-effects that are 889# currently being built: 890wait_side_effects=False 891forseinexecutor.get_action_side_effects(): 892ifse.get_state()==NODE_EXECUTING: 893se.add_to_waiting_s_e(node) 894wait_side_effects=True 895 896ifwait_side_effects: 897ifS:S.side_effects=S.side_effects+1 898continue 899 900# The default when we've gotten through all of the checks above: 901# this node is ready to be built. 902ifS:S.build=S.build+1 903ifT:T.write(self.trace_message(u'Evaluating %s\n'% 904self.trace_node(node))) 905 906# For debugging only: 907# 908# try: 909# self._validate_pending_children() 910# except: 911# self.ready_exc = sys.exc_info() 912# return node 913 914returnnode 915 916returnNone

919""" 920 Returns the next task to be executed. 921 922 This simply asks for the next Node to be evaluated, and then wraps 923 it in the specific Task subclass with which we were initialized. 924 """ 925node=self._find_next_ready_node() 926 927ifnodeisNone: 928returnNone 929 930tlist=node.get_executor().get_all_targets() 931 932task=self.tasker(self,tlist,nodeinself.original_top,node) 933try: 934task.make_ready() 935except: 936# We had a problem just trying to get this task ready (like 937# a child couldn't be linked in to a VariantDir when deciding 938# whether this node is current). Arrange to raise the 939# exception when the Task is "executed." 940self.ready_exc=sys.exc_info() 941 942ifself.ready_exc: 943task.exception_set(self.ready_exc) 944 945self.ready_exc=None 946 947returntask

1000"""1001 Check for dependency cycles.1002 """1003ifnotself.pending_children:1004return10051006nclist=[(n,find_cycle([n],set()))forninself.pending_children]10071008genuine_cycles=[1009nodefornode,cycleinnclist1010ifcycleornode.get_state()!=NODE_EXECUTED1011]1012ifnotgenuine_cycles:1013# All of the "cycles" found were single nodes in EXECUTED state,1014# which is to say, they really weren't cycles. Just return.1015return10161017desc='Found dependency cycle(s):\n'1018fornode,cycleinnclist:1019ifcycle:1020desc=desc+" "+" -> ".join(map(str,cycle))+"\n"1021else:1022desc=desc+ \ 1023" Internal Error: no cycle found for node %s (%s) in state %s\n"% \ 1024(node,repr(node),StateString[node.get_state()])10251026raiseSCons.Errors.UserError(desc)