Source code for pex.http

# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).# Licensed under the Apache License, Version 2.0 (see LICENSE).importcontextlibimporthashlibimportosimportshutilimportuuidfromabcimportabstractmethodfromemailimportmessage_from_stringfrom.commonimportsafe_mkdtemp,safe_openfrom.compatibilityimportPY3,AbstractClassfrom.tracerimportTRACERfrom.variablesimportENVtry:importrequestsexceptImportError:requests=Nonetry:fromcachecontrolimportCacheControlfromcachecontrol.cachesimportFileCacheexceptImportError:CacheControl=FileCache=NoneifPY3:importurllib.requestasurllib_requestelse:importurllib2asurllib_request# This is available as hashlib.algorithms_guaranteed in >=3.2 and as# hashlib.algorithms in >=2.7, but not available in 2.6, so we enumerate# here.HASHLIB_ALGORITHMS=frozenset(['md5','sha1','sha224','sha256','sha384','sha512'])

[docs]classContext(AbstractClass):"""Encapsulate the networking necessary to do requirement resolution. At a minimum, the Context must implement ``open(link)`` by returning a file-like object. Reference implementations of ``read(link)`` and ``fetch(link)`` are provided based upon ``open(link)`` but may be further specialized by individual implementations. """DEFAULT_ENCODING='iso-8859-1'

[docs]defopen(self,link):"""Return an open file-like object to the link. :param link: The :class:`Link` to open. """

[docs]defread(self,link):"""Return the binary content associated with the link. :param link: The :class:`Link` to read. """withcontextlib.closing(self.open(link))asfp:returnfp.read()

[docs]defcontent(self,link):"""Return the encoded content associated with the link. :param link: The :class:`Link` to read. """

[docs]deffetch(self,link,into=None):"""Fetch the binary content associated with the link and write to a file. :param link: The :class:`Link` to fetch. :keyword into: If specified, write into the directory ``into``. If ``None``, creates a new temporary directory that persists for the duration of the interpreter. """target=os.path.join(intoorsafe_mkdtemp(),link.filename)ifos.path.exists(target):# Assume that if the local file already exists, it is safe to use.returntargetwithTRACER.timed('Fetching %s'%link.url,V=2):target_tmp='%s.%s'%(target,uuid.uuid4())withcontextlib.closing(self.open(link))asin_fp:withsafe_open(target_tmp,'wb')asout_fp:shutil.copyfileobj(in_fp,out_fp)os.rename(target_tmp,target)returntarget

[docs]classUrllibContext(Context):"""Default Python standard library Context."""defopen(self,link):returnurllib_request.urlopen(link.url)defcontent(self,link):iflink.local:raiseself.Error('Context.content only works with remote URLs.')withcontextlib.closing(self.open(link))asfp:encoding=message_from_string(str(fp.headers)).get_content_charset(self.DEFAULT_ENCODING)returnfp.read().decode(encoding,'replace')def__init__(self,*args,**kw):TRACER.log('Warning, using a UrllibContext which is known to be flaky.')TRACER.log('Please build pex with the requests module for more reliable downloads.')super(UrllibContext,self).__init__(*args,**kw)

[docs]defdetect_algorithm(cls,link):"""Detect the hashing algorithm from the fragment in the link, if any."""ifany(link.fragment.startswith('%s='%algorithm)foralgorithminHASHLIB_ALGORITHMS):algorithm,value=link.fragment.split('=',2)try:returnhashlib.new(algorithm),valueexceptValueError:# unsupported algorithmreturnNone,NonereturnNone,None

[docs]classRequestsContext(Context):"""A requests-based Context."""@staticmethoddef_create_session(max_retries):session=requests.session()retrying_adapter=requests.adapters.HTTPAdapter(max_retries=max_retries)session.mount('http://',retrying_adapter)session.mount('https://',retrying_adapter)returnsessiondef__init__(self,session=None,verify=True,env=ENV):ifrequestsisNone:raiseRuntimeError('requests is not available. Cannot use a RequestsContext.')self._verify=verifymax_retries=env.PEX_HTTP_RETRIESifmax_retries<0:raiseValueError('max_retries may not be negative.')self._max_retries=max_retriesself._session=sessionorself._create_session(self._max_retries)defopen(self,link):# requests does not support file:// -- so we must short-circuit manuallyiflink.local:returnopen(link.path,'rb')# noqa: T802forattemptinrange(self._max_retries+1):try:returnStreamFilelike(self._session.get(link.url,verify=self._verify,stream=True),link)exceptrequests.exceptions.ReadTimeout:# Connect timeouts are handled by the HTTPAdapter, unfortunately read timeouts are not# so we'll retry them ourselves.TRACER.log('Read timeout trying to fetch %s, retrying. %d retries remain.'%(link.url,self._max_retries-attempt))exceptrequests.exceptions.RequestExceptionase:raiseself.Error(e)raiseself.Error(requests.packages.urllib3.exceptions.MaxRetryError(None,link,'Exceeded max retries of %d'%self._max_retries))defcontent(self,link):iflink.local:raiseself.Error('Context.content only works with remote URLs.')withcontextlib.closing(self.open(link))asrequest:returnrequest.read().decode(request.encodingorself.DEFAULT_ENCODING,'replace')