Navigation

Source code for tornado.httpclient

"""Blocking and non-blocking HTTP client interfaces.This module defines a common interface shared by two implementations,`simple_httpclient` and `curl_httpclient`. Applications may eitherinstantiate their chosen implementation class directly or use the`AsyncHTTPClient` class from this module, which selects an implementationthat can be overridden with the `AsyncHTTPClient.configure` method.The default implementation is `simple_httpclient`, and this is expectedto be suitable for most users' needs. However, some applications may wishto switch to `curl_httpclient` for reasons such as the following:* `curl_httpclient` is more likely to be compatible with sites that are not-quite-compliant with the HTTP spec, or sites that use little-exercised features of HTTP.* `simple_httpclient` only supports SSL on Python 2.6 and above.* `curl_httpclient` is faster* `curl_httpclient` was the default prior to Tornado 2.0.Note that if you are using `curl_httpclient`, it is highly recommended thatyou use a recent version of ``libcurl`` and ``pycurl``. Currently the minimumsupported version is 7.18.2, and the recommended version is 7.21.1 or newer."""importcalendarimportemail.utilsimporthttplibimportosimporttimeimportweakreffromtornado.escapeimportutf8fromtornadoimporthttputilfromtornado.ioloopimportIOLoopfromtornado.utilimportimport_object,bytes_type

[docs]deffetch(self,request,**kwargs):"""Executes a request, returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` If an error occurs during the fetch, we raise an `HTTPError`. """defcallback(response):self._response=responseself._io_loop.stop()self._async_client.fetch(request,callback,**kwargs)self._io_loop.start()response=self._responseself._response=Noneresponse.rethrow()returnresponse

[docs]classAsyncHTTPClient(object):"""An non-blocking HTTP client. Example usage:: import ioloop def handle_request(response): if response.error: print "Error:", response.error else: print response.body ioloop.IOLoop.instance().stop() http_client = httpclient.AsyncHTTPClient() http_client.fetch("http://www.google.com/", handle_request) ioloop.IOLoop.instance().start() The constructor for this class is magic in several respects: It actually creates an instance of an implementation-specific subclass, and instances are reused as a kind of pseudo-singleton (one per IOLoop). The keyword argument force_instance=True can be used to suppress this singleton behavior. Constructor arguments other than io_loop and force_instance are deprecated. The implementation subclass as well as arguments to its constructor can be set with the static method configure() """_async_clients=weakref.WeakKeyDictionary()_impl_class=None_impl_kwargs=Nonedef__new__(cls,io_loop=None,max_clients=10,force_instance=False,**kwargs):io_loop=io_looporIOLoop.instance()ifio_loopincls._async_clientsandnotforce_instance:returncls._async_clients[io_loop]else:ifclsisAsyncHTTPClient:ifcls._impl_classisNone:fromtornado.simple_httpclientimportSimpleAsyncHTTPClientAsyncHTTPClient._impl_class=SimpleAsyncHTTPClientimpl=cls._impl_classelse:impl=clsinstance=super(AsyncHTTPClient,cls).__new__(impl)args={}ifcls._impl_kwargs:args.update(cls._impl_kwargs)args.update(kwargs)instance.initialize(io_loop,max_clients,**args)ifnotforce_instance:cls._async_clients[io_loop]=instancereturninstance

[docs]defclose(self):"""Destroys this http client, freeing any file descriptors used. Not needed in normal use, but may be helpful in unittests that create and destroy http clients. No other methods may be called on the AsyncHTTPClient after close(). """ifself._async_clients[self.io_loop]isself:delself._async_clients[self.io_loop]

[docs]deffetch(self,request,callback,**kwargs):"""Executes a request, calling callback with an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` If an error occurs during the fetch, the HTTPResponse given to the callback has a non-None error attribute that contains the exception encountered during the request. You can call response.rethrow() to throw the exception (if any) in the callback. """raiseNotImplementedError()

@staticmethod

[docs]defconfigure(impl,**kwargs):"""Configures the AsyncHTTPClient subclass to use. AsyncHTTPClient() actually creates an instance of a subclass. This method may be called with either a class object or the fully-qualified name of such a class (or None to use the default, SimpleAsyncHTTPClient) If additional keyword arguments are given, they will be passed to the constructor of each subclass instance created. The keyword argument max_clients determines the maximum number of simultaneous fetch() operations that can execute in parallel on each IOLoop. Additional arguments may be supported depending on the implementation class in use. Example:: AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") """ifisinstance(impl,(unicode,bytes_type)):impl=import_object(impl)ifimplisnotNoneandnotissubclass(impl,AsyncHTTPClient):raiseValueError("Invalid AsyncHTTPClient implementation")AsyncHTTPClient._impl_class=implAsyncHTTPClient._impl_kwargs=kwargs

[docs]classHTTPRequest(object):"""HTTP client request object."""def__init__(self,url,method="GET",headers=None,body=None,auth_username=None,auth_password=None,connect_timeout=20.0,request_timeout=20.0,if_modified_since=None,follow_redirects=True,max_redirects=5,user_agent=None,use_gzip=True,network_interface=None,streaming_callback=None,header_callback=None,prepare_curl_callback=None,proxy_host=None,proxy_port=None,proxy_username=None,proxy_password='',allow_nonstandard_methods=False,validate_cert=True,ca_certs=None,allow_ipv6=None):"""Creates an `HTTPRequest`. All parameters except `url` are optional. :arg string url: URL to fetch :arg string method: HTTP method, e.g. "GET" or "POST" :arg headers: Additional HTTP headers to pass on the request :type headers: `~tornado.httputil.HTTPHeaders` or `dict` :arg string auth_username: Username for HTTP "Basic" authentication :arg string auth_password: Password for HTTP "Basic" authentication :arg float connect_timeout: Timeout for initial connection in seconds :arg float request_timeout: Timeout for entire request in seconds :arg datetime if_modified_since: Timestamp for ``If-Modified-Since`` header :arg bool follow_redirects: Should redirects be followed automatically or return the 3xx response? :arg int max_redirects: Limit for `follow_redirects` :arg string user_agent: String to send as ``User-Agent`` header :arg bool use_gzip: Request gzip encoding from the server :arg string network_interface: Network interface to use for request :arg callable streaming_callback: If set, `streaming_callback` will be run with each chunk of data as it is received, and `~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in the final response. :arg callable header_callback: If set, `header_callback` will be run with each header line as it is received, and `~HTTPResponse.headers` will be empty in the final response. :arg callable prepare_curl_callback: If set, will be called with a `pycurl.Curl` object to allow the application to make additional `setopt` calls. :arg string proxy_host: HTTP proxy hostname. To use proxies, `proxy_host` and `proxy_port` must be set; `proxy_username` and `proxy_pass` are optional. Proxies are currently only support with `curl_httpclient`. :arg int proxy_port: HTTP proxy port :arg string proxy_username: HTTP proxy username :arg string proxy_password: HTTP proxy password :arg bool allow_nonstandard_methods: Allow unknown values for `method` argument? :arg bool validate_cert: For HTTPS requests, validate the server's certificate? :arg string ca_certs: filename of CA certificates in PEM format, or None to use defaults. Note that in `curl_httpclient`, if any request uses a custom `ca_certs` file, they all must (they don't have to all use the same `ca_certs`, but it's not possible to mix requests with ca_certs and requests that use the defaults. :arg bool allow_ipv6: Use IPv6 when available? Default is false in `simple_httpclient` and true in `curl_httpclient` """ifheadersisNone:headers=httputil.HTTPHeaders()ifif_modified_since:timestamp=calendar.timegm(if_modified_since.utctimetuple())headers["If-Modified-Since"]=email.utils.formatdate(timestamp,localtime=False,usegmt=True)self.proxy_host=proxy_hostself.proxy_port=proxy_portself.proxy_username=proxy_usernameself.proxy_password=proxy_passwordself.url=urlself.method=methodself.headers=headersself.body=utf8(body)self.auth_username=auth_usernameself.auth_password=auth_passwordself.connect_timeout=connect_timeoutself.request_timeout=request_timeoutself.follow_redirects=follow_redirectsself.max_redirects=max_redirectsself.user_agent=user_agentself.use_gzip=use_gzipself.network_interface=network_interfaceself.streaming_callback=streaming_callbackself.header_callback=header_callbackself.prepare_curl_callback=prepare_curl_callbackself.allow_nonstandard_methods=allow_nonstandard_methodsself.validate_cert=validate_certself.ca_certs=ca_certsself.allow_ipv6=allow_ipv6self.start_time=time.time()

[docs]classHTTPError(Exception):"""Exception thrown for an unsuccessful HTTP request. Attributes: code - HTTP error integer error code, e.g. 404. Error code 599 is used when no HTTP response was received, e.g. for a timeout. response - HTTPResponse object, if any. Note that if follow_redirects is False, redirects become HTTPErrors, and you can look at error.response.headers['Location'] to see the destination of the redirect. """def__init__(self,code,message=None,response=None):self.code=codemessage=messageorhttplib.responses.get(code,"Unknown")self.response=responseException.__init__(self,"HTTP %d: %s"%(self.code,message))