Source code for tornado.iostream

#!/usr/bin/env python## Copyright 2009 Facebook## Licensed under the Apache License, Version 2.0 (the "License"); you may# not use this file except in compliance with the License. You may obtain# a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the# License for the specific language governing permissions and limitations# under the License."""Utility classes to write to and read from non-blocking files and sockets.Contents:* `BaseIOStream`: Generic interface for reading and writing.* `IOStream`: Implementation of BaseIOStream using non-blocking sockets.* `SSLIOStream`: SSL-aware version of IOStream.* `PipeIOStream`: Pipe-based IOStream implementation."""from__future__importabsolute_import,division,print_function,with_statementimportcollectionsimporterrnoimportnumbersimportosimportsocketimportsysimportrefromtornado.concurrentimportTracebackFuturefromtornadoimportioloopfromtornado.logimportgen_log,app_logfromtornado.netutilimportssl_wrap_socket,ssl_match_hostname,SSLCertificateError,_client_ssl_defaults,_server_ssl_defaultsfromtornadoimportstack_contextfromtornado.utilimporterrno_from_exceptiontry:fromtornado.platform.posiximport_set_nonblockingexceptImportError:_set_nonblocking=Nonetry:importsslexceptImportError:# ssl is not available on Google App Enginessl=None# These errnos indicate that a non-blocking operation must be retried# at a later time. On most platforms they're the same value, but on# some they differ._ERRNO_WOULDBLOCK=(errno.EWOULDBLOCK,errno.EAGAIN)ifhasattr(errno,"WSAEWOULDBLOCK"):_ERRNO_WOULDBLOCK+=(errno.WSAEWOULDBLOCK,)# type: ignore# These errnos indicate that a connection has been abruptly terminated.# They should be caught and handled less noisily than other errors._ERRNO_CONNRESET=(errno.ECONNRESET,errno.ECONNABORTED,errno.EPIPE,errno.ETIMEDOUT)ifhasattr(errno,"WSAECONNRESET"):_ERRNO_CONNRESET+=(errno.WSAECONNRESET,errno.WSAECONNABORTED,errno.WSAETIMEDOUT)# type: ignoreifsys.platform=='darwin':# OSX appears to have a race condition that causes send(2) to return# EPROTOTYPE if called while a socket is being torn down:# http://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/# Since the socket is being closed anyway, treat this as an ECONNRESET# instead of an unexpected error._ERRNO_CONNRESET+=(errno.EPROTOTYPE,)# type: ignore# More non-portable errnos:_ERRNO_INPROGRESS=(errno.EINPROGRESS,)ifhasattr(errno,"WSAEINPROGRESS"):_ERRNO_INPROGRESS+=(errno.WSAEINPROGRESS,)# type: ignore

[docs]classStreamClosedError(IOError):"""Exception raised by `IOStream` methods when the stream is closed. Note that the close callback is scheduled to run *after* other callbacks on the stream (to allow for buffered data to be processed), so you may see this error before you see the close callback. The ``real_error`` attribute contains the underlying error that caused the stream to close (if any). .. versionchanged:: 4.3 Added the ``real_error`` attribute. """def__init__(self,real_error=None):super(StreamClosedError,self).__init__('Stream is closed')self.real_error=real_error

[docs]classUnsatisfiableReadError(Exception):"""Exception raised when a read cannot be satisfied. Raised by ``read_until`` and ``read_until_regex`` with a ``max_bytes`` argument. """pass

[docs]classStreamBufferFullError(Exception):"""Exception raised by `IOStream` methods when the buffer is full. """

[docs]classBaseIOStream(object):"""A utility class to write to and read from a non-blocking file or socket. We support a non-blocking ``write()`` and a family of ``read_*()`` methods. All of the methods take an optional ``callback`` argument and return a `.Future` only if no callback is given. When the operation completes, the callback will be run or the `.Future` will resolve with the data read (or ``None`` for ``write()``). All outstanding ``Futures`` will resolve with a `StreamClosedError` when the stream is closed; users of the callback interface will be notified via `.BaseIOStream.set_close_callback` instead. When a stream is closed due to an error, the IOStream's ``error`` attribute contains the exception object. Subclasses must implement `fileno`, `close_fd`, `write_to_fd`, `read_from_fd`, and optionally `get_fd_error`. """def__init__(self,io_loop=None,max_buffer_size=None,read_chunk_size=None,max_write_buffer_size=None):"""`BaseIOStream` constructor. :arg io_loop: The `.IOLoop` to use; defaults to `.IOLoop.current`. Deprecated since Tornado 4.1. :arg max_buffer_size: Maximum amount of incoming data to buffer; defaults to 100MB. :arg read_chunk_size: Amount of data to read at one time from the underlying transport; defaults to 64KB. :arg max_write_buffer_size: Amount of outgoing data to buffer; defaults to unlimited. .. versionchanged:: 4.0 Add the ``max_write_buffer_size`` parameter. Changed default ``read_chunk_size`` to 64KB. """self.io_loop=io_looporioloop.IOLoop.current()self.max_buffer_size=max_buffer_sizeor104857600# A chunk size that is too close to max_buffer_size can cause# spurious failures.self.read_chunk_size=min(read_chunk_sizeor65536,self.max_buffer_size//2)self.max_write_buffer_size=max_write_buffer_sizeself.error=Noneself._read_buffer=collections.deque()self._write_buffer=collections.deque()self._read_buffer_size=0self._write_buffer_size=0self._write_buffer_frozen=Falseself._read_delimiter=Noneself._read_regex=Noneself._read_max_bytes=Noneself._read_bytes=Noneself._read_partial=Falseself._read_until_close=Falseself._read_callback=Noneself._read_future=Noneself._streaming_callback=Noneself._write_callback=Noneself._write_future=Noneself._close_callback=Noneself._connect_callback=Noneself._connect_future=None# _ssl_connect_future should be defined in SSLIOStream# but it's here so we can clean it up in maybe_run_close_callback.# TODO: refactor that so subclasses can add additional futures# to be cancelled.self._ssl_connect_future=Noneself._connecting=Falseself._state=Noneself._pending_callbacks=0self._closed=False

[docs]deffileno(self):"""Returns the file descriptor for this stream."""raiseNotImplementedError()

[docs]defclose_fd(self):"""Closes the file underlying this stream. ``close_fd`` is called by `BaseIOStream` and should not be called elsewhere; other users should call `close` instead. """raiseNotImplementedError()

[docs]defwrite_to_fd(self,data):"""Attempts to write ``data`` to the underlying file. Returns the number of bytes written. """raiseNotImplementedError()

[docs]defread_from_fd(self):"""Attempts to read from the underlying file. Returns ``None`` if there was nothing to read (the socket returned `~errno.EWOULDBLOCK` or equivalent), otherwise returns the data. When possible, should return no more than ``self.read_chunk_size`` bytes at a time. """raiseNotImplementedError()

[docs]defget_fd_error(self):"""Returns information about any error on the underlying file. This method is called after the `.IOLoop` has signaled an error on the file descriptor, and should return an Exception (such as `socket.error` with additional information, or None if no such information is available. """returnNone

[docs]defread_until_regex(self,regex,callback=None,max_bytes=None):"""Asynchronously read until we have matched the given regex. The result includes the data that matches the regex and anything that came before it. If a callback is given, it will be run with the data as an argument; if not, this method returns a `.Future`. If ``max_bytes`` is not None, the connection will be closed if more than ``max_bytes`` bytes have been read and the regex is not satisfied. .. versionchanged:: 4.0 Added the ``max_bytes`` argument. The ``callback`` argument is now optional and a `.Future` will be returned if it is omitted. """future=self._set_read_callback(callback)self._read_regex=re.compile(regex)self._read_max_bytes=max_bytestry:self._try_inline_read()exceptUnsatisfiableReadErrorase:# Handle this the same way as in _handle_events.gen_log.info("Unsatisfiable read, closing connection: %s"%e)self.close(exc_info=True)returnfutureexcept:iffutureisnotNone:# Ensure that the future doesn't log an error because its# failure was never examined.future.add_done_callback(lambdaf:f.exception())raisereturnfuture

[docs]defread_until(self,delimiter,callback=None,max_bytes=None):"""Asynchronously read until we have found the given delimiter. The result includes all the data read including the delimiter. If a callback is given, it will be run with the data as an argument; if not, this method returns a `.Future`. If ``max_bytes`` is not None, the connection will be closed if more than ``max_bytes`` bytes have been read and the delimiter is not found. .. versionchanged:: 4.0 Added the ``max_bytes`` argument. The ``callback`` argument is now optional and a `.Future` will be returned if it is omitted. """future=self._set_read_callback(callback)self._read_delimiter=delimiterself._read_max_bytes=max_bytestry:self._try_inline_read()exceptUnsatisfiableReadErrorase:# Handle this the same way as in _handle_events.gen_log.info("Unsatisfiable read, closing connection: %s"%e)self.close(exc_info=True)returnfutureexcept:iffutureisnotNone:future.add_done_callback(lambdaf:f.exception())raisereturnfuture

[docs]defread_bytes(self,num_bytes,callback=None,streaming_callback=None,partial=False):"""Asynchronously read a number of bytes. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the final result will be empty. Otherwise, the result is all the data that was read. If a callback is given, it will be run with the data as an argument; if not, this method returns a `.Future`. If ``partial`` is true, the callback is run as soon as we have any bytes to return (but never more than ``num_bytes``) .. versionchanged:: 4.0 Added the ``partial`` argument. The callback argument is now optional and a `.Future` will be returned if it is omitted. """future=self._set_read_callback(callback)assertisinstance(num_bytes,numbers.Integral)self._read_bytes=num_bytesself._read_partial=partialself._streaming_callback=stack_context.wrap(streaming_callback)try:self._try_inline_read()except:iffutureisnotNone:future.add_done_callback(lambdaf:f.exception())raisereturnfuture

[docs]defread_until_close(self,callback=None,streaming_callback=None):"""Asynchronously reads all data from the socket until it is closed. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the final result will be empty. Otherwise, the result is all the data that was read. If a callback is given, it will be run with the data as an argument; if not, this method returns a `.Future`. Note that if a ``streaming_callback`` is used, data will be read from the socket as quickly as it becomes available; there is no way to apply backpressure or cancel the reads. If flow control or cancellation are desired, use a loop with `read_bytes(partial=True) <.read_bytes>` instead. .. versionchanged:: 4.0 The callback argument is now optional and a `.Future` will be returned if it is omitted. """future=self._set_read_callback(callback)self._streaming_callback=stack_context.wrap(streaming_callback)ifself.closed():ifself._streaming_callbackisnotNone:self._run_read_callback(self._read_buffer_size,True)self._run_read_callback(self._read_buffer_size,False)returnfutureself._read_until_close=Truetry:self._try_inline_read()except:iffutureisnotNone:future.add_done_callback(lambdaf:f.exception())raisereturnfuture

[docs]defwrite(self,data,callback=None):"""Asynchronously write the given data to this stream. If ``callback`` is given, we call it when all of the buffered write data has been successfully written to the stream. If there was previously buffered write data and an old write callback, that callback is simply overwritten with this new callback. If no ``callback`` is given, this method returns a `.Future` that resolves (with a result of ``None``) when the write has been completed. If `write` is called again before that `.Future` has resolved, the previous future will be orphaned and will never resolve. .. versionchanged:: 4.0 Now returns a `.Future` if no callback is given. """assertisinstance(data,bytes)self._check_closed()# We use bool(_write_buffer) as a proxy for write_buffer_size>0,# so never put empty strings in the buffer.ifdata:if(self.max_write_buffer_sizeisnotNoneandself._write_buffer_size+len(data)>self.max_write_buffer_size):raiseStreamBufferFullError("Reached maximum write buffer size")# Break up large contiguous strings before inserting them in the# write buffer, so we don't have to recopy the entire thing# as we slice off pieces to send to the socket.WRITE_BUFFER_CHUNK_SIZE=128*1024foriinrange(0,len(data),WRITE_BUFFER_CHUNK_SIZE):self._write_buffer.append(data[i:i+WRITE_BUFFER_CHUNK_SIZE])self._write_buffer_size+=len(data)ifcallbackisnotNone:self._write_callback=stack_context.wrap(callback)future=Noneelse:future=self._write_future=TracebackFuture()future.add_done_callback(lambdaf:f.exception())ifnotself._connecting:self._handle_write()ifself._write_buffer:self._add_io_state(self.io_loop.WRITE)self._maybe_add_error_listener()returnfuture

[docs]defset_close_callback(self,callback):"""Call the given callback when the stream is closed. This is not necessary for applications that use the `.Future` interface; all outstanding ``Futures`` will resolve with a `StreamClosedError` when the stream is closed. """self._close_callback=stack_context.wrap(callback)self._maybe_add_error_listener()

[docs]defclose(self,exc_info=False):"""Close this stream. If ``exc_info`` is true, set the ``error`` attribute to the current exception from `sys.exc_info` (or if ``exc_info`` is a tuple, use that instead of `sys.exc_info`). """ifnotself.closed():ifexc_info:ifnotisinstance(exc_info,tuple):exc_info=sys.exc_info()ifany(exc_info):self.error=exc_info[1]ifself._read_until_close:if(self._streaming_callbackisnotNoneandself._read_buffer_size):self._run_read_callback(self._read_buffer_size,True)self._read_until_close=Falseself._run_read_callback(self._read_buffer_size,False)ifself._stateisnotNone:self.io_loop.remove_handler(self.fileno())self._state=Noneself.close_fd()self._closed=Trueself._maybe_run_close_callback()

def_maybe_run_close_callback(self):# If there are pending callbacks, don't run the close callback# until they're done (see _maybe_add_error_handler)ifself.closed()andself._pending_callbacks==0:futures=[]ifself._read_futureisnotNone:futures.append(self._read_future)self._read_future=Noneifself._write_futureisnotNone:futures.append(self._write_future)self._write_future=Noneifself._connect_futureisnotNone:futures.append(self._connect_future)self._connect_future=Noneifself._ssl_connect_futureisnotNone:futures.append(self._ssl_connect_future)self._ssl_connect_future=Noneforfutureinfutures:future.set_exception(StreamClosedError(real_error=self.error))ifself._close_callbackisnotNone:cb=self._close_callbackself._close_callback=Noneself._run_callback(cb)# Delete any unfinished callbacks to break up reference cycles.self._read_callback=self._write_callback=None# Clear the buffers so they can be cleared immediately even# if the IOStream object is kept alive by a reference cycle.# TODO: Clear the read buffer too; it currently breaks some tests.self._write_buffer=None

[docs]defreading(self):"""Returns true if we are currently reading from the stream."""returnself._read_callbackisnotNoneorself._read_futureisnotNone

[docs]defwriting(self):"""Returns true if we are currently writing to the stream."""returnbool(self._write_buffer)

[docs]defclosed(self):"""Returns true if the stream has been closed."""returnself._closed

[docs]defset_nodelay(self,value):"""Sets the no-delay flag for this stream. By default, data written to TCP streams may be held for a time to make the most efficient use of bandwidth (according to Nagle's algorithm). The no-delay flag requests that data be written as soon as possible, even if doing so would consume additional bandwidth. This flag is currently defined only for TCP-based ``IOStreams``. .. versionadded:: 3.1 """pass

def_handle_events(self,fd,events):ifself.closed():gen_log.warning("Got events for closed stream %s",fd)returntry:ifself._connecting:# Most IOLoops will report a write failed connect# with the WRITE event, but SelectIOLoop reports a# READ as well so we must check for connecting before# either.self._handle_connect()ifself.closed():returnifevents&self.io_loop.READ:self._handle_read()ifself.closed():returnifevents&self.io_loop.WRITE:self._handle_write()ifself.closed():returnifevents&self.io_loop.ERROR:self.error=self.get_fd_error()# We may have queued up a user callback in _handle_read or# _handle_write, so don't close the IOStream until those# callbacks have had a chance to run.self.io_loop.add_callback(self.close)returnstate=self.io_loop.ERRORifself.reading():state|=self.io_loop.READifself.writing():state|=self.io_loop.WRITEifstate==self.io_loop.ERRORandself._read_buffer_size==0:# If the connection is idle, listen for reads too so# we can tell if the connection is closed. If there is# data in the read buffer we won't run the close callback# yet anyway, so we don't need to listen in this case.state|=self.io_loop.READifstate!=self._state:assertself._stateisnotNone, \
"shouldn't happen: _handle_events without self._state"self._state=stateself.io_loop.update_handler(self.fileno(),self._state)exceptUnsatisfiableReadErrorase:gen_log.info("Unsatisfiable read, closing connection: %s"%e)self.close(exc_info=True)exceptException:gen_log.error("Uncaught exception, closing connection.",exc_info=True)self.close(exc_info=True)raisedef_run_callback(self,callback,*args):defwrapper():self._pending_callbacks-=1try:returncallback(*args)exceptException:app_log.error("Uncaught exception, closing connection.",exc_info=True)# Close the socket on an uncaught exception from a user callback# (It would eventually get closed when the socket object is# gc'd, but we don't want to rely on gc happening before we# run out of file descriptors)self.close(exc_info=True)# Re-raise the exception so that IOLoop.handle_callback_exception# can see it and log the errorraisefinally:self._maybe_add_error_listener()# We schedule callbacks to be run on the next IOLoop iteration# rather than running them directly for several reasons:# * Prevents unbounded stack growth when a callback calls an# IOLoop operation that immediately runs another callback# * Provides a predictable execution context for e.g.# non-reentrant mutexes# * Ensures that the try/except in wrapper() is run outside# of the application's StackContextswithstack_context.NullContext():# stack_context was already captured in callback, we don't need to# capture it again for IOStream's wrapper. This is especially# important if the callback was pre-wrapped before entry to# IOStream (as in HTTPConnection._header_callback), as we could# capture and leak the wrong context here.self._pending_callbacks+=1self.io_loop.add_callback(wrapper)def_read_to_buffer_loop(self):# This method is called from _handle_read and _try_inline_read.try:ifself._read_bytesisnotNone:target_bytes=self._read_byteselifself._read_max_bytesisnotNone:target_bytes=self._read_max_byteselifself.reading():# For read_until without max_bytes, or# read_until_close, read as much as we can before# scanning for the delimiter.target_bytes=Noneelse:target_bytes=0next_find_pos=0# Pretend to have a pending callback so that an EOF in# _read_to_buffer doesn't trigger an immediate close# callback. At the end of this method we'll either# establish a real pending callback via# _read_from_buffer or run the close callback.## We need two try statements here so that# pending_callbacks is decremented before the `except`# clause below (which calls `close` and does need to# trigger the callback)self._pending_callbacks+=1whilenotself.closed():# Read from the socket until we get EWOULDBLOCK or equivalent.# SSL sockets do some internal buffering, and if the data is# sitting in the SSL object's buffer select() and friends# can't see it; the only way to find out if it's there is to# try to read it.ifself._read_to_buffer()==0:breakself._run_streaming_callback()# If we've read all the bytes we can use, break out of# this loop. We can't just call read_from_buffer here# because of subtle interactions with the# pending_callback and error_listener mechanisms.## If we've reached target_bytes, we know we're done.if(target_bytesisnotNoneandself._read_buffer_size>=target_bytes):break# Otherwise, we need to call the more expensive find_read_pos.# It's inefficient to do this on every read, so instead# do it on the first read and whenever the read buffer# size has doubled.ifself._read_buffer_size>=next_find_pos:pos=self._find_read_pos()ifposisnotNone:returnposnext_find_pos=self._read_buffer_size*2returnself._find_read_pos()finally:self._pending_callbacks-=1def_handle_read(self):try:pos=self._read_to_buffer_loop()exceptUnsatisfiableReadError:raiseexceptExceptionase:gen_log.warning("error on read: %s"%e)self.close(exc_info=True)returnifposisnotNone:self._read_from_buffer(pos)returnelse:self._maybe_run_close_callback()def_set_read_callback(self,callback):assertself._read_callbackisNone,"Already reading"assertself._read_futureisNone,"Already reading"ifcallbackisnotNone:self._read_callback=stack_context.wrap(callback)else:self._read_future=TracebackFuture()returnself._read_futuredef_run_read_callback(self,size,streaming):ifstreaming:callback=self._streaming_callbackelse:callback=self._read_callbackself._read_callback=self._streaming_callback=Noneifself._read_futureisnotNone:assertcallbackisNonefuture=self._read_futureself._read_future=Nonefuture.set_result(self._consume(size))ifcallbackisnotNone:assert(self._read_futureisNone)orstreamingself._run_callback(callback,self._consume(size))else:# If we scheduled a callback, we will add the error listener# afterwards. If we didn't, we have to do it now.self._maybe_add_error_listener()def_try_inline_read(self):"""Attempt to complete the current read operation from buffered data. If the read can be completed without blocking, schedules the read callback on the next IOLoop iteration; otherwise starts listening for reads on the socket. """# See if we've already got the data from a previous readself._run_streaming_callback()pos=self._find_read_pos()ifposisnotNone:self._read_from_buffer(pos)returnself._check_closed()try:pos=self._read_to_buffer_loop()exceptException:# If there was an in _read_to_buffer, we called close() already,# but couldn't run the close callback because of _pending_callbacks.# Before we escape from this function, run the close callback if# applicable.self._maybe_run_close_callback()raiseifposisnotNone:self._read_from_buffer(pos)return# We couldn't satisfy the read inline, so either close the stream# or listen for new data.ifself.closed():self._maybe_run_close_callback()else:self._add_io_state(ioloop.IOLoop.READ)def_read_to_buffer(self):"""Reads from the socket and appends the result to the read buffer. Returns the number of bytes read. Returns 0 if there is nothing to read (i.e. the read returns EWOULDBLOCK or equivalent). On error closes the socket and raises an exception. """whileTrue:try:chunk=self.read_from_fd()except(socket.error,IOError,OSError)ase:iferrno_from_exception(e)==errno.EINTR:continue# ssl.SSLError is a subclass of socket.errorifself._is_connreset(e):# Treat ECONNRESET as a connection close rather than# an error to minimize log spam (the exception will# be available on self.error for apps that care).self.close(exc_info=True)returnself.close(exc_info=True)raisebreakifchunkisNone:return0self._read_buffer.append(chunk)self._read_buffer_size+=len(chunk)ifself._read_buffer_size>self.max_buffer_size:gen_log.error("Reached maximum read buffer size")self.close()raiseStreamBufferFullError("Reached maximum read buffer size")returnlen(chunk)def_run_streaming_callback(self):ifself._streaming_callbackisnotNoneandself._read_buffer_size:bytes_to_consume=self._read_buffer_sizeifself._read_bytesisnotNone:bytes_to_consume=min(self._read_bytes,bytes_to_consume)self._read_bytes-=bytes_to_consumeself._run_read_callback(bytes_to_consume,True)def_read_from_buffer(self,pos):"""Attempts to complete the currently-pending read from the buffer. The argument is either a position in the read buffer or None, as returned by _find_read_pos. """self._read_bytes=self._read_delimiter=self._read_regex=Noneself._read_partial=Falseself._run_read_callback(pos,False)def_find_read_pos(self):"""Attempts to find a position in the read buffer that satisfies the currently-pending read. Returns a position in the buffer if the current read can be satisfied, or None if it cannot. """if(self._read_bytesisnotNoneand(self._read_buffer_size>=self._read_bytesor(self._read_partialandself._read_buffer_size>0))):num_bytes=min(self._read_bytes,self._read_buffer_size)returnnum_byteselifself._read_delimiterisnotNone:# Multi-byte delimiters (e.g. '\r\n') may straddle two# chunks in the read buffer, so we can't easily find them# without collapsing the buffer. However, since protocols# using delimited reads (as opposed to reads of a known# length) tend to be "line" oriented, the delimiter is likely# to be in the first few chunks. Merge the buffer gradually# since large merges are relatively expensive and get undone in# _consume().ifself._read_buffer:whileTrue:loc=self._read_buffer[0].find(self._read_delimiter)ifloc!=-1:delimiter_len=len(self._read_delimiter)self._check_max_bytes(self._read_delimiter,loc+delimiter_len)returnloc+delimiter_leniflen(self._read_buffer)==1:break_double_prefix(self._read_buffer)self._check_max_bytes(self._read_delimiter,len(self._read_buffer[0]))elifself._read_regexisnotNone:ifself._read_buffer:whileTrue:m=self._read_regex.search(self._read_buffer[0])ifmisnotNone:self._check_max_bytes(self._read_regex,m.end())returnm.end()iflen(self._read_buffer)==1:break_double_prefix(self._read_buffer)self._check_max_bytes(self._read_regex,len(self._read_buffer[0]))returnNonedef_check_max_bytes(self,delimiter,size):if(self._read_max_bytesisnotNoneandsize>self._read_max_bytes):raiseUnsatisfiableReadError("delimiter %r not found within %d bytes"%(delimiter,self._read_max_bytes))def_handle_write(self):whileself._write_buffer:try:ifnotself._write_buffer_frozen:# On windows, socket.send blows up if given a# write buffer that's too large, instead of just# returning the number of bytes it was able to# process. Therefore we must not call socket.send# with more than 128KB at a time._merge_prefix(self._write_buffer,128*1024)num_bytes=self.write_to_fd(self._write_buffer[0])ifnum_bytes==0:# With OpenSSL, if we couldn't write the entire buffer,# the very same string object must be used on the# next call to send. Therefore we suppress# merging the write buffer after an incomplete send.# A cleaner solution would be to set# SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER, but this is# not yet accessible from python# (http://bugs.python.org/issue8240)self._write_buffer_frozen=Truebreakself._write_buffer_frozen=False_merge_prefix(self._write_buffer,num_bytes)self._write_buffer.popleft()self._write_buffer_size-=num_bytesexcept(socket.error,IOError,OSError)ase:ife.args[0]in_ERRNO_WOULDBLOCK:self._write_buffer_frozen=Truebreakelse:ifnotself._is_connreset(e):# Broken pipe errors are usually caused by connection# reset, and its better to not log EPIPE errors to# minimize log spamgen_log.warning("Write error on %s: %s",self.fileno(),e)self.close(exc_info=True)returnifnotself._write_buffer:ifself._write_callback:callback=self._write_callbackself._write_callback=Noneself._run_callback(callback)ifself._write_future:future=self._write_futureself._write_future=Nonefuture.set_result(None)def_consume(self,loc):ifloc==0:returnb""_merge_prefix(self._read_buffer,loc)self._read_buffer_size-=locreturnself._read_buffer.popleft()def_check_closed(self):ifself.closed():raiseStreamClosedError(real_error=self.error)def_maybe_add_error_listener(self):# This method is part of an optimization: to detect a connection that# is closed when we're not actively reading or writing, we must listen# for read events. However, it is inefficient to do this when the# connection is first established because we are going to read or write# immediately anyway. Instead, we insert checks at various times to# see if the connection is idle and add the read listener then.ifself._pending_callbacks!=0:returnifself._stateisNoneorself._state==ioloop.IOLoop.ERROR:ifself.closed():self._maybe_run_close_callback()elif(self._read_buffer_size==0andself._close_callbackisnotNone):self._add_io_state(ioloop.IOLoop.READ)def_add_io_state(self,state):"""Adds `state` (IOLoop.{READ,WRITE} flags) to our event handler. Implementation notes: Reads and writes have a fast path and a slow path. The fast path reads synchronously from socket buffers, while the slow path uses `_add_io_state` to schedule an IOLoop callback. Note that in both cases, the callback is run asynchronously with `_run_callback`. To detect closed connections, we must have called `_add_io_state` at some point, but we want to delay this as much as possible so we don't have to set an `IOLoop.ERROR` listener that will be overwritten by the next slow-path operation. As long as there are callbacks scheduled for fast-path ops, those callbacks may do more reads. If a sequence of fast-path ops do not end in a slow-path op, (e.g. for an @asynchronous long-poll request), we must add the error handler. This is done in `_run_callback` and `write` (since the write callback is optional so we can have a fast-path write with no `_run_callback`) """ifself.closed():# connection has been closed, so there can be no future eventsreturnifself._stateisNone:self._state=ioloop.IOLoop.ERROR|statewithstack_context.NullContext():self.io_loop.add_handler(self.fileno(),self._handle_events,self._state)elifnotself._state&state:self._state=self._state|stateself.io_loop.update_handler(self.fileno(),self._state)def_is_connreset(self,exc):"""Return true if exc is ECONNRESET or equivalent. May be overridden in subclasses. """return(isinstance(exc,(socket.error,IOError))anderrno_from_exception(exc)in_ERRNO_CONNRESET)

[docs]defconnect(self,address,callback=None,server_hostname=None):"""Connects the socket to a remote address without blocking. May only be called if the socket passed to the constructor was not previously connected. The address parameter is in the same format as for `socket.connect <socket.socket.connect>` for the type of socket passed to the IOStream constructor, e.g. an ``(ip, port)`` tuple. Hostnames are accepted here, but will be resolved synchronously and block the IOLoop. If you have a hostname instead of an IP address, the `.TCPClient` class is recommended instead of calling this method directly. `.TCPClient` will do asynchronous DNS resolution and handle both IPv4 and IPv6. If ``callback`` is specified, it will be called with no arguments when the connection is completed; if not this method returns a `.Future` (whose result after a successful connection will be the stream itself). In SSL mode, the ``server_hostname`` parameter will be used for certificate validation (unless disabled in the ``ssl_options``) and SNI (if supported; requires Python 2.7.9+). Note that it is safe to call `IOStream.write <BaseIOStream.write>` while the connection is pending, in which case the data will be written as soon as the connection is ready. Calling `IOStream` read methods before the socket is connected works on some platforms but is non-portable. .. versionchanged:: 4.0 If no callback is given, returns a `.Future`. .. versionchanged:: 4.2 SSL certificates are validated by default; pass ``ssl_options=dict(cert_reqs=ssl.CERT_NONE)`` or a suitably-configured `ssl.SSLContext` to the `SSLIOStream` constructor to disable. """self._connecting=TrueifcallbackisnotNone:self._connect_callback=stack_context.wrap(callback)future=Noneelse:future=self._connect_future=TracebackFuture()try:self.socket.connect(address)exceptsocket.errorase:# In non-blocking mode we expect connect() to raise an# exception with EINPROGRESS or EWOULDBLOCK.## On freebsd, other errors such as ECONNREFUSED may be# returned immediately when attempting to connect to# localhost, so handle them the same way as an error# reported later in _handle_connect.if(errno_from_exception(e)notin_ERRNO_INPROGRESSanderrno_from_exception(e)notin_ERRNO_WOULDBLOCK):iffutureisNone:gen_log.warning("Connect error on fd %s: %s",self.socket.fileno(),e)self.close(exc_info=True)returnfutureself._add_io_state(self.io_loop.WRITE)returnfuture

[docs]defstart_tls(self,server_side,ssl_options=None,server_hostname=None):"""Convert this `IOStream` to an `SSLIOStream`. This enables protocols that begin in clear-text mode and switch to SSL after some initial negotiation (such as the ``STARTTLS`` extension to SMTP and IMAP). This method cannot be used if there are outstanding reads or writes on the stream, or if there is any data in the IOStream's buffer (data in the operating system's socket buffer is allowed). This means it must generally be used immediately after reading or writing the last clear-text data. It can also be used immediately after connecting, before any reads or writes. The ``ssl_options`` argument may be either an `ssl.SSLContext` object or a dictionary of keyword arguments for the `ssl.wrap_socket` function. The ``server_hostname`` argument will be used for certificate validation unless disabled in the ``ssl_options``. This method returns a `.Future` whose result is the new `SSLIOStream`. After this method has been called, any other operation on the original stream is undefined. If a close callback is defined on this stream, it will be transferred to the new stream. .. versionadded:: 4.0 .. versionchanged:: 4.2 SSL certificates are validated by default; pass ``ssl_options=dict(cert_reqs=ssl.CERT_NONE)`` or a suitably-configured `ssl.SSLContext` to disable. """if(self._read_callbackorself._read_futureorself._write_callbackorself._write_futureorself._connect_callbackorself._connect_futureorself._pending_callbacksorself._closedorself._read_bufferorself._write_buffer):raiseValueError("IOStream is not idle; cannot convert to SSL")ifssl_optionsisNone:ifserver_side:ssl_options=_server_ssl_defaultselse:ssl_options=_client_ssl_defaultssocket=self.socketself.io_loop.remove_handler(socket)self.socket=Nonesocket=ssl_wrap_socket(socket,ssl_options,server_hostname=server_hostname,server_side=server_side,do_handshake_on_connect=False)orig_close_callback=self._close_callbackself._close_callback=Nonefuture=TracebackFuture()ssl_stream=SSLIOStream(socket,ssl_options=ssl_options,io_loop=self.io_loop)# Wrap the original close callback so we can fail our Future as well.# If we had an "unwrap" counterpart to this method we would need# to restore the original callback after our Future resolves# so that repeated wrap/unwrap calls don't build up layers.defclose_callback():ifnotfuture.done():# Note that unlike most Futures returned by IOStream,# this one passes the underlying error through directly# instead of wrapping everything in a StreamClosedError# with a real_error attribute. This is because once the# connection is established it's more helpful to raise# the SSLError directly than to hide it behind a# StreamClosedError (and the client is expecting SSL# issues rather than network issues since this method is# named start_tls).future.set_exception(ssl_stream.errororStreamClosedError())iforig_close_callbackisnotNone:orig_close_callback()ssl_stream.set_close_callback(close_callback)ssl_stream._ssl_connect_callback=lambda:future.set_result(ssl_stream)ssl_stream.max_buffer_size=self.max_buffer_sizessl_stream.read_chunk_size=self.read_chunk_sizereturnfuture

def_handle_connect(self):err=self.socket.getsockopt(socket.SOL_SOCKET,socket.SO_ERROR)iferr!=0:self.error=socket.error(err,os.strerror(err))# IOLoop implementations may vary: some of them return# an error state before the socket becomes writable, so# in that case a connection failure would be handled by the# error path in _handle_events instead of here.ifself._connect_futureisNone:gen_log.warning("Connect error on fd %s: %s",self.socket.fileno(),errno.errorcode[err])self.close()returnifself._connect_callbackisnotNone:callback=self._connect_callbackself._connect_callback=Noneself._run_callback(callback)ifself._connect_futureisnotNone:future=self._connect_futureself._connect_future=Nonefuture.set_result(self)self._connecting=Falsedefset_nodelay(self,value):if(self.socketisnotNoneandself.socket.familyin(socket.AF_INET,socket.AF_INET6)):try:self.socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1ifvalueelse0)exceptsocket.errorase:# Sometimes setsockopt will fail if the socket is closed# at the wrong time. This can happen with HTTPServer# resetting the value to false between requests.ife.errno!=errno.EINVALandnotself._is_connreset(e):raise

[docs]classSSLIOStream(IOStream):"""A utility class to write to and read from a non-blocking SSL socket. If the socket passed to the constructor is already connected, it should be wrapped with:: ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs) before constructing the `SSLIOStream`. Unconnected sockets will be wrapped when `IOStream.connect` is finished. """def__init__(self,*args,**kwargs):"""The ``ssl_options`` keyword argument may either be an `ssl.SSLContext` object or a dictionary of keywords arguments for `ssl.wrap_socket` """self._ssl_options=kwargs.pop('ssl_options',_client_ssl_defaults)super(SSLIOStream,self).__init__(*args,**kwargs)self._ssl_accepting=Trueself._handshake_reading=Falseself._handshake_writing=Falseself._ssl_connect_callback=Noneself._server_hostname=None# If the socket is already connected, attempt to start the handshake.try:self.socket.getpeername()exceptsocket.error:passelse:# Indirectly start the handshake, which will run on the next# IOLoop iteration and then the real IO state will be set in# _handle_events.self._add_io_state(self.io_loop.WRITE)defreading(self):returnself._handshake_readingorsuper(SSLIOStream,self).reading()defwriting(self):returnself._handshake_writingorsuper(SSLIOStream,self).writing()def_do_ssl_handshake(self):# Based on code from test_ssl.py in the python stdlibtry:self._handshake_reading=Falseself._handshake_writing=Falseself.socket.do_handshake()exceptssl.SSLErroraserr:iferr.args[0]==ssl.SSL_ERROR_WANT_READ:self._handshake_reading=Truereturneliferr.args[0]==ssl.SSL_ERROR_WANT_WRITE:self._handshake_writing=Truereturneliferr.args[0]in(ssl.SSL_ERROR_EOF,ssl.SSL_ERROR_ZERO_RETURN):returnself.close(exc_info=True)eliferr.args[0]==ssl.SSL_ERROR_SSL:try:peer=self.socket.getpeername()exceptException:peer='(not connected)'gen_log.warning("SSL Error on %s%s: %s",self.socket.fileno(),peer,err)returnself.close(exc_info=True)raiseexceptsocket.erroraserr:# Some port scans (e.g. nmap in -sT mode) have been known# to cause do_handshake to raise EBADF and ENOTCONN, so make# those errors quiet as well.# https://groups.google.com/forum/?fromgroups#!topic/python-tornado/ApucKJat1_0if(self._is_connreset(err)orerr.args[0]in(errno.EBADF,errno.ENOTCONN)):returnself.close(exc_info=True)raiseexceptAttributeError:# On Linux, if the connection was reset before the call to# wrap_socket, do_handshake will fail with an# AttributeError.returnself.close(exc_info=True)else:self._ssl_accepting=Falseifnotself._verify_cert(self.socket.getpeercert()):self.close()returnself._run_ssl_connect_callback()def_run_ssl_connect_callback(self):ifself._ssl_connect_callbackisnotNone:callback=self._ssl_connect_callbackself._ssl_connect_callback=Noneself._run_callback(callback)ifself._ssl_connect_futureisnotNone:future=self._ssl_connect_futureself._ssl_connect_future=Nonefuture.set_result(self)def_verify_cert(self,peercert):"""Returns True if peercert is valid according to the configured validation mode and hostname. The ssl handshake already tested the certificate for a valid CA signature; the only thing that remains is to check the hostname. """ifisinstance(self._ssl_options,dict):verify_mode=self._ssl_options.get('cert_reqs',ssl.CERT_NONE)elifisinstance(self._ssl_options,ssl.SSLContext):verify_mode=self._ssl_options.verify_modeassertverify_modein(ssl.CERT_NONE,ssl.CERT_REQUIRED,ssl.CERT_OPTIONAL)ifverify_mode==ssl.CERT_NONEorself._server_hostnameisNone:returnTruecert=self.socket.getpeercert()ifcertisNoneandverify_mode==ssl.CERT_REQUIRED:gen_log.warning("No SSL certificate given")returnFalsetry:ssl_match_hostname(peercert,self._server_hostname)exceptSSLCertificateErrorase:gen_log.warning("Invalid SSL certificate: %s"%e)returnFalseelse:returnTruedef_handle_read(self):ifself._ssl_accepting:self._do_ssl_handshake()returnsuper(SSLIOStream,self)._handle_read()def_handle_write(self):ifself._ssl_accepting:self._do_ssl_handshake()returnsuper(SSLIOStream,self)._handle_write()defconnect(self,address,callback=None,server_hostname=None):self._server_hostname=server_hostname# Pass a dummy callback to super.connect(), which is slightly# more efficient than letting it return a Future we ignore.super(SSLIOStream,self).connect(address,callback=lambda:None)returnself.wait_for_handshake(callback)def_handle_connect(self):# Call the superclass method to check for errors.super(SSLIOStream,self)._handle_connect()ifself.closed():return# When the connection is complete, wrap the socket for SSL# traffic. Note that we do this by overriding _handle_connect# instead of by passing a callback to super().connect because# user callbacks are enqueued asynchronously on the IOLoop,# but since _handle_events calls _handle_connect immediately# followed by _handle_write we need this to be synchronous.## The IOLoop will get confused if we swap out self.socket while the# fd is registered, so remove it now and re-register after# wrap_socket().self.io_loop.remove_handler(self.socket)old_state=self._stateself._state=Noneself.socket=ssl_wrap_socket(self.socket,self._ssl_options,server_hostname=self._server_hostname,do_handshake_on_connect=False)self._add_io_state(old_state)

[docs]defwait_for_handshake(self,callback=None):"""Wait for the initial SSL handshake to complete. If a ``callback`` is given, it will be called with no arguments once the handshake is complete; otherwise this method returns a `.Future` which will resolve to the stream itself after the handshake is complete. Once the handshake is complete, information such as the peer's certificate and NPN/ALPN selections may be accessed on ``self.socket``. This method is intended for use on server-side streams or after using `IOStream.start_tls`; it should not be used with `IOStream.connect` (which already waits for the handshake to complete). It may only be called once per stream. .. versionadded:: 4.2 """if(self._ssl_connect_callbackisnotNoneorself._ssl_connect_futureisnotNone):raiseRuntimeError("Already waiting")ifcallbackisnotNone:self._ssl_connect_callback=stack_context.wrap(callback)future=Noneelse:future=self._ssl_connect_future=TracebackFuture()ifnotself._ssl_accepting:self._run_ssl_connect_callback()returnfuture

defwrite_to_fd(self,data):try:returnself.socket.send(data)exceptssl.SSLErrorase:ife.args[0]==ssl.SSL_ERROR_WANT_WRITE:# In Python 3.5+, SSLSocket.send raises a WANT_WRITE error if# the socket is not writeable; we need to transform this into# an EWOULDBLOCK socket.error or a zero return value,# either of which will be recognized by the caller of this# method. Prior to Python 3.5, an unwriteable socket would# simply return 0 bytes written.return0raisedefread_from_fd(self):ifself._ssl_accepting:# If the handshake hasn't finished yet, there can't be anything# to read (attempting to read may or may not raise an exception# depending on the SSL version)returnNonetry:# SSLSocket objects have both a read() and recv() method,# while regular sockets only have recv().# The recv() method blocks (at least in python 2.6) if it is# called when there is nothing to read, so we have to use# read() instead.chunk=self.socket.read(self.read_chunk_size)exceptssl.SSLErrorase:# SSLError is a subclass of socket.error, so this except# block must come first.ife.args[0]==ssl.SSL_ERROR_WANT_READ:returnNoneelse:raiseexceptsocket.errorase:ife.args[0]in_ERRNO_WOULDBLOCK:returnNoneelse:raiseifnotchunk:self.close()returnNonereturnchunkdef_is_connreset(self,e):ifisinstance(e,ssl.SSLError)ande.args[0]==ssl.SSL_ERROR_EOF:returnTruereturnsuper(SSLIOStream,self)._is_connreset(e)

[docs]classPipeIOStream(BaseIOStream):"""Pipe-based `IOStream` implementation. The constructor takes an integer file descriptor (such as one returned by `os.pipe`) rather than an open file object. Pipes are generally one-way, so a `PipeIOStream` can be used for reading or writing but not both. """def__init__(self,fd,*args,**kwargs):self.fd=fd_set_nonblocking(fd)super(PipeIOStream,self).__init__(*args,**kwargs)deffileno(self):returnself.fddefclose_fd(self):os.close(self.fd)defwrite_to_fd(self,data):returnos.write(self.fd,data)defread_from_fd(self):try:chunk=os.read(self.fd,self.read_chunk_size)except(IOError,OSError)ase:iferrno_from_exception(e)in_ERRNO_WOULDBLOCK:returnNoneeliferrno_from_exception(e)==errno.EBADF:# If the writing half of a pipe is closed, select will# report it as readable but reads will fail with EBADF.self.close(exc_info=True)returnNoneelse:raiseifnotchunk:self.close()returnNonereturnchunk

def_double_prefix(deque):"""Grow by doubling, but don't split the second chunk just because the first one is small. """new_len=max(len(deque[0])*2,(len(deque[0])+len(deque[1])))_merge_prefix(deque,new_len)def_merge_prefix(deque,size):"""Replace the first entries in a deque of strings with a single string of up to size bytes. >>> d = collections.deque(['abc', 'de', 'fghi', 'j']) >>> _merge_prefix(d, 5); print(d) deque(['abcde', 'fghi', 'j']) Strings will be split as necessary to reach the desired size. >>> _merge_prefix(d, 7); print(d) deque(['abcdefg', 'hi', 'j']) >>> _merge_prefix(d, 3); print(d) deque(['abc', 'defg', 'hi', 'j']) >>> _merge_prefix(d, 100); print(d) deque(['abcdefghij']) """iflen(deque)==1andlen(deque[0])<=size:returnprefix=[]remaining=sizewhiledequeandremaining>0:chunk=deque.popleft()iflen(chunk)>remaining:deque.appendleft(chunk[remaining:])chunk=chunk[:remaining]prefix.append(chunk)remaining-=len(chunk)# This data structure normally just contains byte strings, but# the unittest gets messy if it doesn't use the default str() type,# so do the merge based on the type of data that's actually present.ifprefix:deque.appendleft(type(prefix[0])().join(prefix))ifnotdeque:deque.appendleft(b"")defdoctests():importdoctestreturndoctest.DocTestSuite()