Source code for tornado.platform.twisted

# Author: Ovidiu Predescu# Date: July 2011## Licensed under the Apache License, Version 2.0 (the "License"); you may# not use this file except in compliance with the License. You may obtain# a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the# License for the specific language governing permissions and limitations# under the License."""Bridges between the Twisted reactor and Tornado IOLoop.This module lets you run applications and libraries written forTwisted in a Tornado application. It can be used in two modes,depending on which library's underlying event loop you want to use.This module has been tested with Twisted versions 11.0.0 and newer."""from__future__importabsolute_import,division,print_function,with_statementimportdatetimeimportfunctoolsimportnumbersimportsocketimportsysimporttwisted.internet.abstract# type: ignorefromtwisted.internet.deferimportDeferred# type: ignorefromtwisted.internet.posixbaseimportPosixReactorBase# type: ignorefromtwisted.internet.interfacesimportIReactorFDSet,IDelayedCall,IReactorTime,IReadDescriptor,IWriteDescriptor# type: ignorefromtwisted.pythonimportfailure,log# type: ignorefromtwisted.internetimporterror# type: ignoreimporttwisted.names.cache# type: ignoreimporttwisted.names.client# type: ignoreimporttwisted.names.hosts# type: ignoreimporttwisted.names.resolve# type: ignorefromzope.interfaceimportimplementer# type: ignorefromtornado.concurrentimportFuturefromtornado.escapeimportutf8fromtornadoimportgenimporttornado.ioloopfromtornado.logimportapp_logfromtornado.netutilimportResolverfromtornado.stack_contextimportNullContext,wrapfromtornado.ioloopimportIOLoopfromtornado.utilimporttimedelta_to_seconds@implementer(IDelayedCall)classTornadoDelayedCall(object):"""DelayedCall object for Tornado."""def__init__(self,reactor,seconds,f,*args,**kw):self._reactor=reactorself._func=functools.partial(f,*args,**kw)self._time=self._reactor.seconds()+secondsself._timeout=self._reactor._io_loop.add_timeout(self._time,self._called)self._active=Truedef_called(self):self._active=Falseself._reactor._removeDelayedCall(self)try:self._func()except:app_log.error("_called caught exception",exc_info=True)defgetTime(self):returnself._timedefcancel(self):self._active=Falseself._reactor._io_loop.remove_timeout(self._timeout)self._reactor._removeDelayedCall(self)defdelay(self,seconds):self._reactor._io_loop.remove_timeout(self._timeout)self._time+=secondsself._timeout=self._reactor._io_loop.add_timeout(self._time,self._called)defreset(self,seconds):self._reactor._io_loop.remove_timeout(self._timeout)self._time=self._reactor.seconds()+secondsself._timeout=self._reactor._io_loop.add_timeout(self._time,self._called)defactive(self):returnself._active@implementer(IReactorTime,IReactorFDSet)

[docs]classTornadoReactor(PosixReactorBase):"""Twisted reactor built on the Tornado IOLoop. `TornadoReactor` implements the Twisted reactor interface on top of the Tornado IOLoop. To use it, simply call `install` at the beginning of the application:: import tornado.platform.twisted tornado.platform.twisted.install() from twisted.internet import reactor When the app is ready to start, call ``IOLoop.current().start()`` instead of ``reactor.run()``. It is also possible to create a non-global reactor by calling ``tornado.platform.twisted.TornadoReactor(io_loop)``. However, if the `.IOLoop` and reactor are to be short-lived (such as those used in unit tests), additional cleanup may be required. Specifically, it is recommended to call:: reactor.fireSystemEvent('shutdown') reactor.disconnectAll() before closing the `.IOLoop`. .. versionchanged:: 4.1 The ``io_loop`` argument is deprecated. """def__init__(self,io_loop=None):ifnotio_loop:io_loop=tornado.ioloop.IOLoop.current()self._io_loop=io_loopself._readers={}# map of reader objects to fdself._writers={}# map of writer objects to fdself._fds={}# a map of fd to a (reader, writer) tupleself._delayedCalls={}PosixReactorBase.__init__(self)self.addSystemEventTrigger('during','shutdown',self.crash)# IOLoop.start() bypasses some of the reactor initialization.# Fire off the necessary events if they weren't already triggered# by reactor.run().defstart_if_necessary():ifnotself._started:self.fireSystemEvent('startup')self._io_loop.add_callback(start_if_necessary)# IReactorTimedefseconds(self):returnself._io_loop.time()defcallLater(self,seconds,f,*args,**kw):dc=TornadoDelayedCall(self,seconds,f,*args,**kw)self._delayedCalls[dc]=TruereturndcdefgetDelayedCalls(self):return[xforxinself._delayedCallsifx._active]def_removeDelayedCall(self,dc):ifdcinself._delayedCalls:delself._delayedCalls[dc]# IReactorThreadsdefcallFromThread(self,f,*args,**kw):assertcallable(f),"%s is not callable"%fwithNullContext():# This NullContext is mainly for an edge case when running# TwistedIOLoop on top of a TornadoReactor.# TwistedIOLoop.add_callback uses reactor.callFromThread and# should not pick up additional StackContexts along the way.self._io_loop.add_callback(f,*args,**kw)# We don't need the waker code from the super class, Tornado uses# its own waker.definstallWaker(self):passdefwakeUp(self):pass# IReactorFDSetdef_invoke_callback(self,fd,events):iffdnotinself._fds:return(reader,writer)=self._fds[fd]ifreader:err=Noneifreader.fileno()==-1:err=error.ConnectionLost()elifevents&IOLoop.READ:err=log.callWithLogger(reader,reader.doRead)iferrisNoneandevents&IOLoop.ERROR:err=error.ConnectionLost()iferrisnotNone:self.removeReader(reader)reader.readConnectionLost(failure.Failure(err))ifwriter:err=Noneifwriter.fileno()==-1:err=error.ConnectionLost()elifevents&IOLoop.WRITE:err=log.callWithLogger(writer,writer.doWrite)iferrisNoneandevents&IOLoop.ERROR:err=error.ConnectionLost()iferrisnotNone:self.removeWriter(writer)writer.writeConnectionLost(failure.Failure(err))defaddReader(self,reader):ifreaderinself._readers:# Don't add the reader if it's already therereturnfd=reader.fileno()self._readers[reader]=fdiffdinself._fds:(_,writer)=self._fds[fd]self._fds[fd]=(reader,writer)ifwriter:# We already registered this fd for write events,# update it for read events as well.self._io_loop.update_handler(fd,IOLoop.READ|IOLoop.WRITE)else:withNullContext():self._fds[fd]=(reader,None)self._io_loop.add_handler(fd,self._invoke_callback,IOLoop.READ)defaddWriter(self,writer):ifwriterinself._writers:returnfd=writer.fileno()self._writers[writer]=fdiffdinself._fds:(reader,_)=self._fds[fd]self._fds[fd]=(reader,writer)ifreader:# We already registered this fd for read events,# update it for write events as well.self._io_loop.update_handler(fd,IOLoop.READ|IOLoop.WRITE)else:withNullContext():self._fds[fd]=(None,writer)self._io_loop.add_handler(fd,self._invoke_callback,IOLoop.WRITE)defremoveReader(self,reader):ifreaderinself._readers:fd=self._readers.pop(reader)(_,writer)=self._fds[fd]ifwriter:# We have a writer so we need to update the IOLoop for# write events only.self._fds[fd]=(None,writer)self._io_loop.update_handler(fd,IOLoop.WRITE)else:# Since we have no writer registered, we remove the# entry from _fds and unregister the handler from the# IOLoopdelself._fds[fd]self._io_loop.remove_handler(fd)defremoveWriter(self,writer):ifwriterinself._writers:fd=self._writers.pop(writer)(reader,_)=self._fds[fd]ifreader:# We have a reader so we need to update the IOLoop for# read events only.self._fds[fd]=(reader,None)self._io_loop.update_handler(fd,IOLoop.READ)else:# Since we have no reader registered, we remove the# entry from the _fds and unregister the handler from# the IOLoop.delself._fds[fd]self._io_loop.remove_handler(fd)defremoveAll(self):returnself._removeAll(self._readers,self._writers)defgetReaders(self):returnself._readers.keys()defgetWriters(self):returnself._writers.keys()# The following functions are mainly used in twisted-style test cases;# it is expected that most users of the TornadoReactor will call# IOLoop.start() instead of Reactor.run().defstop(self):PosixReactorBase.stop(self)fire_shutdown=functools.partial(self.fireSystemEvent,"shutdown")self._io_loop.add_callback(fire_shutdown)defcrash(self):PosixReactorBase.crash(self)self._io_loop.stop()defdoIteration(self,delay):raiseNotImplementedError("doIteration")defmainLoop(self):# Since this class is intended to be used in applications# where the top-level event loop is ``io_loop.start()`` rather# than ``reactor.run()``, it is implemented a little# differently than other Twisted reactors. We override# ``mainLoop`` instead of ``doIteration`` and must implement# timed call functionality on top of `.IOLoop.add_timeout`# rather than using the implementation in# ``PosixReactorBase``.self._io_loop.start()

class_TestReactor(TornadoReactor):"""Subclass of TornadoReactor for use in unittests. This can't go in the test.py file because of import-order dependencies with the Twisted reactor test builder. """def__init__(self):# always use a new ioloopsuper(_TestReactor,self).__init__(IOLoop())deflistenTCP(self,port,factory,backlog=50,interface=''):# default to localhost to avoid firewall prompts on the macifnotinterface:interface='127.0.0.1'returnsuper(_TestReactor,self).listenTCP(port,factory,backlog=backlog,interface=interface)deflistenUDP(self,port,protocol,interface='',maxPacketSize=8192):ifnotinterface:interface='127.0.0.1'returnsuper(_TestReactor,self).listenUDP(port,protocol,interface=interface,maxPacketSize=maxPacketSize)

[docs]definstall(io_loop=None):"""Install this package as the default Twisted reactor. ``install()`` must be called very early in the startup process, before most other twisted-related imports. Conversely, because it initializes the `.IOLoop`, it cannot be called before `.fork_processes` or multi-process `~.TCPServer.start`. These conflicting requirements make it difficult to use `.TornadoReactor` in multi-process mode, and an external process manager such as ``supervisord`` is recommended instead. .. versionchanged:: 4.1 The ``io_loop`` argument is deprecated. """ifnotio_loop:io_loop=tornado.ioloop.IOLoop.current()reactor=TornadoReactor(io_loop)fromtwisted.internet.mainimportinstallReactor# type: ignoreinstallReactor(reactor)returnreactor

[docs]classTwistedIOLoop(tornado.ioloop.IOLoop):"""IOLoop implementation that runs on Twisted. `TwistedIOLoop` implements the Tornado IOLoop interface on top of the Twisted reactor. Recommended usage:: from tornado.platform.twisted import TwistedIOLoop from twisted.internet import reactor TwistedIOLoop().install() # Set up your tornado application as usual using `IOLoop.instance` reactor.run() Uses the global Twisted reactor by default. To create multiple ``TwistedIOLoops`` in the same process, you must pass a unique reactor when constructing each one. Not compatible with `tornado.process.Subprocess.set_exit_callback` because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict with each other. See also :meth:`tornado.ioloop.IOLoop.install` for general notes on installing alternative IOLoops. """definitialize(self,reactor=None,**kwargs):super(TwistedIOLoop,self).initialize(**kwargs)ifreactorisNone:importtwisted.internet.reactor# type: ignorereactor=twisted.internet.reactorself.reactor=reactorself.fds={}defclose(self,all_fds=False):fds=self.fdsself.reactor.removeAll()forcinself.reactor.getDelayedCalls():c.cancel()ifall_fds:forfdinfds.values():self.close_fd(fd.fileobj)defadd_handler(self,fd,handler,events):iffdinself.fds:raiseValueError('fd %s added twice'%fd)fd,fileobj=self.split_fd(fd)self.fds[fd]=_FD(fd,fileobj,wrap(handler))ifevents&tornado.ioloop.IOLoop.READ:self.fds[fd].reading=Trueself.reactor.addReader(self.fds[fd])ifevents&tornado.ioloop.IOLoop.WRITE:self.fds[fd].writing=Trueself.reactor.addWriter(self.fds[fd])defupdate_handler(self,fd,events):fd,fileobj=self.split_fd(fd)ifevents&tornado.ioloop.IOLoop.READ:ifnotself.fds[fd].reading:self.fds[fd].reading=Trueself.reactor.addReader(self.fds[fd])else:ifself.fds[fd].reading:self.fds[fd].reading=Falseself.reactor.removeReader(self.fds[fd])ifevents&tornado.ioloop.IOLoop.WRITE:ifnotself.fds[fd].writing:self.fds[fd].writing=Trueself.reactor.addWriter(self.fds[fd])else:ifself.fds[fd].writing:self.fds[fd].writing=Falseself.reactor.removeWriter(self.fds[fd])defremove_handler(self,fd):fd,fileobj=self.split_fd(fd)iffdnotinself.fds:returnself.fds[fd].lost=Trueifself.fds[fd].reading:self.reactor.removeReader(self.fds[fd])ifself.fds[fd].writing:self.reactor.removeWriter(self.fds[fd])delself.fds[fd]defstart(self):old_current=IOLoop.current(instance=False)try:self._setup_logging()self.make_current()self.reactor.run()finally:ifold_currentisNone:IOLoop.clear_current()else:old_current.make_current()defstop(self):self.reactor.crash()defadd_timeout(self,deadline,callback,*args,**kwargs):# This method could be simplified (since tornado 4.0) by# overriding call_at instead of add_timeout, but we leave it# for now as a test of backwards-compatibility.ifisinstance(deadline,numbers.Real):delay=max(deadline-self.time(),0)elifisinstance(deadline,datetime.timedelta):delay=timedelta_to_seconds(deadline)else:raiseTypeError("Unsupported deadline %r")returnself.reactor.callLater(delay,self._run_callback,functools.partial(wrap(callback),*args,**kwargs))defremove_timeout(self,timeout):iftimeout.active():timeout.cancel()defadd_callback(self,callback,*args,**kwargs):self.reactor.callFromThread(self._run_callback,functools.partial(wrap(callback),*args,**kwargs))defadd_callback_from_signal(self,callback,*args,**kwargs):self.add_callback(callback,*args,**kwargs)

[docs]classTwistedResolver(Resolver):"""Twisted-based asynchronous resolver. This is a non-blocking and non-threaded resolver. It is recommended only when threads cannot be used, since it has limitations compared to the standard ``getaddrinfo``-based `~tornado.netutil.Resolver` and `~tornado.netutil.ThreadedResolver`. Specifically, it returns at most one result, and arguments other than ``host`` and ``family`` are ignored. It may fail to resolve when ``family`` is not ``socket.AF_UNSPEC``. Requires Twisted 12.1 or newer. .. versionchanged:: 4.1 The ``io_loop`` argument is deprecated. """definitialize(self,io_loop=None):self.io_loop=io_looporIOLoop.current()# partial copy of twisted.names.client.createResolver, which doesn't# allow for a reactor to be passed in.self.reactor=tornado.platform.twisted.TornadoReactor(io_loop)host_resolver=twisted.names.hosts.Resolver('/etc/hosts')cache_resolver=twisted.names.cache.CacheResolver(reactor=self.reactor)real_resolver=twisted.names.client.Resolver('/etc/resolv.conf',reactor=self.reactor)self.resolver=twisted.names.resolve.ResolverChain([host_resolver,cache_resolver,real_resolver])@gen.coroutinedefresolve(self,host,port,family=0):# getHostByName doesn't accept IP addresses, so if the input# looks like an IP address just return it immediately.iftwisted.internet.abstract.isIPAddress(host):resolved=hostresolved_family=socket.AF_INETeliftwisted.internet.abstract.isIPv6Address(host):resolved=hostresolved_family=socket.AF_INET6else:deferred=self.resolver.getHostByName(utf8(host))resolved=yieldgen.Task(deferred.addBoth)ifisinstance(resolved,failure.Failure):try:resolved.raiseException()excepttwisted.names.error.DomainErrorase:raiseIOError(e)eliftwisted.internet.abstract.isIPAddress(resolved):resolved_family=socket.AF_INETeliftwisted.internet.abstract.isIPv6Address(resolved):resolved_family=socket.AF_INET6else:resolved_family=socket.AF_UNSPECiffamily!=socket.AF_UNSPECandfamily!=resolved_family:raiseException('Requested socket family %d but got %d'%(family,resolved_family))result=[(resolved_family,(resolved,port)),]raisegen.Return(result)

ifhasattr(gen.convert_yielded,'register'):@gen.convert_yielded.register(Deferred)# type: ignoredef_(d):f=Future()deferrback(failure):try:failure.raiseException()# Should never happen, but just in caseraiseException("errback called without error")except:f.set_exc_info(sys.exc_info())d.addCallbacks(f.set_result,errback)returnf