Navigation

Source code for tornado.gen

"""``tornado.gen`` is a generator-based interface to make it easier towork in an asynchronous environment. Code using the ``gen`` moduleis technically asynchronous, but it is written as a single generatorinstead of a collection of separate functions.For example, the following asynchronous handler:: class AsyncHandler(RequestHandler): @asynchronous def get(self): http_client = AsyncHTTPClient() http_client.fetch("http://example.com", callback=self.on_fetch) def on_fetch(self, response): do_something_with_response(response) self.render("template.html")could be written with ``gen`` as:: class GenAsyncHandler(RequestHandler): @asynchronous @gen.engine def get(self): http_client = AsyncHTTPClient() response = yield gen.Task(http_client.fetch, "http://example.com") do_something_with_response(response) self.render("template.html")`Task` works with any function that takes a ``callback`` keywordargument. You can also yield a list of ``Tasks``, which will bestarted at the same time and run in parallel; a list of results willbe returned when they are all finished:: def get(self): http_client = AsyncHTTPClient() response1, response2 = yield [gen.Task(http_client.fetch, url1), gen.Task(http_client.fetch, url2)]For more complicated interfaces, `Task` can be split into two parts:`Callback` and `Wait`:: class GenAsyncHandler2(RequestHandler): @asynchronous @gen.engine def get(self): http_client = AsyncHTTPClient() http_client.fetch("http://example.com", callback=(yield gen.Callback("key")) response = yield gen.Wait("key") do_something_with_response(response) self.render("template.html")The ``key`` argument to `Callback` and `Wait` allows for multipleasynchronous operations to be started at different times and proceedin parallel: yield several callbacks with different keys, then waitfor them once all the async operations have started.The result of a `Wait` or `Task` yield expression depends on how the callbackwas run. If it was called with no arguments, the result is ``None``. Ifit was called with one argument, the result is that argument. If it wascalled with more than one argument or any keyword arguments, the resultis an `Arguments` object, which is a named tuple ``(args, kwargs)``."""importfunctoolsimportoperatorimportsysimporttypesclassKeyReuseError(Exception):passclassUnknownKeyError(Exception):passclassLeakedCallbackError(Exception):passclassBadYieldError(Exception):pass

[docs]defengine(func):"""Decorator for asynchronous generators. Any generator that yields objects from this module must be wrapped in this decorator. The decorator only works on functions that are already asynchronous. For `~tornado.web.RequestHandler` ``get``/``post``/etc methods, this means that both the `tornado.gen.engine` and `tornado.web.asynchronous` decorators must be used (in either order). In most other cases, it means that it doesn't make sense to use ``gen.engine`` on functions that don't already take a callback argument. """@functools.wraps(func)defwrapper(*args,**kwargs):gen=func(*args,**kwargs)ifisinstance(gen,types.GeneratorType):Runner(gen).run()returnassertgenisNone,gen# no yield, so we're donereturnwrapper

classYieldPoint(object):"""Base class for objects that may be yielded from the generator."""defstart(self,runner):"""Called by the runner after the generator has yielded. No other methods will be called on this object before ``start``. """raiseNotImplementedError()defis_ready(self):"""Called by the runner to determine whether to resume the generator. Returns a boolean; may be called more than once. """raiseNotImplementedError()defget_result(self):"""Returns the value to use as the result of the yield expression. This method will only be called once, and only after `is_ready` has returned true. """raiseNotImplementedError()

[docs]classCallback(YieldPoint):"""Returns a callable object that will allow a matching `Wait` to proceed. The key may be any value suitable for use as a dictionary key, and is used to match ``Callbacks`` to their corresponding ``Waits``. The key must be unique among outstanding callbacks within a single run of the generator function, but may be reused across different runs of the same function (so constants generally work fine). The callback may be called with zero or one arguments; if an argument is given it will be returned by `Wait`. """def__init__(self,key):self.key=keydefstart(self,runner):self.runner=runnerrunner.register_callback(self.key)defis_ready(self):returnTruedefget_result(self):returnself.runner.result_callback(self.key)

[docs]classWait(YieldPoint):"""Returns the argument passed to the result of a previous `Callback`."""def__init__(self,key):self.key=keydefstart(self,runner):self.runner=runnerdefis_ready(self):returnself.runner.is_ready(self.key)defget_result(self):returnself.runner.pop_result(self.key)

[docs]classWaitAll(YieldPoint):"""Returns the results of multiple previous `Callbacks`. The argument is a sequence of `Callback` keys, and the result is a list of results in the same order. `WaitAll` is equivalent to yielding a list of `Wait` objects. """def__init__(self,keys):self.keys=keysdefstart(self,runner):self.runner=runnerdefis_ready(self):returnall(self.runner.is_ready(key)forkeyinself.keys)defget_result(self):return[self.runner.pop_result(key)forkeyinself.keys]

[docs]classTask(YieldPoint):"""Runs a single asynchronous operation. Takes a function (and optional additional arguments) and runs it with those arguments plus a ``callback`` keyword argument. The argument passed to the callback is returned as the result of the yield expression. A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique key generated automatically):: result = yield gen.Task(func, args) func(args, callback=(yield gen.Callback(key))) result = yield gen.Wait(key) """def__init__(self,func,*args,**kwargs):assert"callback"notinkwargsself.args=argsself.kwargs=kwargsself.func=funcdefstart(self,runner):self.runner=runnerself.key=object()runner.register_callback(self.key)self.kwargs["callback"]=runner.result_callback(self.key)self.func(*self.args,**self.kwargs)defis_ready(self):returnself.runner.is_ready(self.key)defget_result(self):returnself.runner.pop_result(self.key)

classMulti(YieldPoint):"""Runs multiple asynchronous operations in parallel. Takes a list of ``Tasks`` or other ``YieldPoints`` and returns a list of their responses. It is not necessary to call `Multi` explicitly, since the engine will do so automatically when the generator yields a list of ``YieldPoints``. """def__init__(self,children):assertall(isinstance(i,YieldPoint)foriinchildren)self.children=childrendefstart(self,runner):foriinself.children:i.start(runner)defis_ready(self):returnall(i.is_ready()foriinself.children)defget_result(self):return[i.get_result()foriinself.children]class_NullYieldPoint(YieldPoint):defstart(self,runner):passdefis_ready(self):returnTruedefget_result(self):returnNoneclassRunner(object):"""Internal implementation of `tornado.gen.engine`. Maintains information about pending callbacks and their results. """def__init__(self,gen):self.gen=genself.yield_point=_NullYieldPoint()self.pending_callbacks=set()self.results={}self.running=Falseself.finished=Falseself.exc_info=Nonedefregister_callback(self,key):"""Adds ``key`` to the list of callbacks."""ifkeyinself.pending_callbacks:raiseKeyReuseError("key %r is already pending"%key)self.pending_callbacks.add(key)defis_ready(self,key):"""Returns true if a result is available for ``key``."""ifkeynotinself.pending_callbacks:raiseUnknownKeyError("key %r is not pending"%key)returnkeyinself.resultsdefset_result(self,key,result):"""Sets the result for ``key`` and attempts to resume the generator."""self.results[key]=resultself.run()defpop_result(self,key):"""Returns the result for ``key`` and unregisters it."""self.pending_callbacks.remove(key)returnself.results.pop(key)defrun(self):"""Starts or resumes the generator, running until it reaches a yield point that is not ready. """ifself.runningorself.finished:returntry:self.running=TruewhileTrue:ifself.exc_infoisNone:try:ifnotself.yield_point.is_ready():returnnext=self.yield_point.get_result()exceptException:self.exc_info=sys.exc_info()try:ifself.exc_infoisnotNone:exc_info=self.exc_infoself.exc_info=Noneyielded=self.gen.throw(*exc_info)else:yielded=self.gen.send(next)exceptStopIteration:self.finished=Trueifself.pending_callbacks:raiseLeakedCallbackError("finished without waiting for callbacks %r"%self.pending_callbacks)returnexceptException:self.finished=Trueraiseifisinstance(yielded,list):yielded=Multi(yielded)ifisinstance(yielded,YieldPoint):self.yield_point=yieldedself.yield_point.start(self)else:self.exc_info=(BadYieldError("yielded unknown object %r"%yielded),)finally:self.running=Falsedefresult_callback(self,key):definner(*args,**kwargs):ifkwargsorlen(args)>1:result=Arguments(args,kwargs)elifargs:result=args[0]else:result=Noneself.set_result(key,result)returninner# in python 2.6+ this could be a collections.namedtuple

[docs]classArguments(tuple):"""The result of a yield expression whose callback had more than one argument (or keyword arguments). The `Arguments` object can be used as a tuple ``(args, kwargs)`` or an object with attributes ``args`` and ``kwargs``. """__slots__=()def__new__(cls,args,kwargs):returntuple.__new__(cls,(args,kwargs))args=property(operator.itemgetter(0))kwargs=property(operator.itemgetter(1))