"""perform ReST syntax, local and remote reference tests on .rst/.txt files."""importpyimportsys,os,redefpytest_addoption(parser):group=parser.getgroup("ReST","ReST documentation check options")group.addoption('-R','--urlcheck',action="store_true",dest="urlcheck",default=False,help="urlopen() remote links found in ReST text files.")group.addoption('--urltimeout',action="store",metavar="secs",type="int",dest="urlcheck_timeout",default=5,help="timeout in seconds for remote urlchecks")group.addoption('--forcegen',action="store_true",dest="forcegen",default=False,help="force generation of html files.")defpytest_collect_file(path,parent):ifpath.extin(".txt",".rst"):project=getproject(path)ifprojectisnotNone:returnReSTFile(path,parent=parent,project=project)defgetproject(path):forparentinpath.parts(reverse=True):confrest=parent.join("confrest.py")ifconfrest.check():print(confrest)Project=confrest.pyimport().ProjectreturnProject(parent)classReSTFile(py.test.collect.File):def__init__(self,fspath,parent,project):super(ReSTFile,self).__init__(fspath=fspath,parent=parent)self.project=projectdefcollect(self):return[ReSTSyntaxTest("ReSTSyntax",parent=self,project=self.project),LinkCheckerMaker("checklinks",parent=self),DoctestText("doctest",parent=self),]defdeindent(s,sep='\n'):leastspaces=-1lines=s.split(sep)forlineinlines:ifnotline.strip():continuespaces=len(line)-len(line.lstrip())ifleastspaces==-1orspaces<leastspaces:leastspaces=spacesifleastspaces==-1:returnsfori,lineinenumerate(lines):ifnotline.strip():lines[i]=''else:lines[i]=line[leastspaces:]returnsep.join(lines)classReSTSyntaxTest(py.test.collect.Item):def__init__(self,name,parent,project):super(ReSTSyntaxTest,self).__init__(name=name,parent=parent)self.project=projectdefreportinfo(self):returnself.fspath,None,"syntax check"defruntest(self):self.restcheck(py.path.svnwc(self.fspath))defrestcheck(self,path):py.test.importorskip("docutils")self.register_linkrole()fromdocutils.utilsimportSystemMessagetry:self._checkskip(path,self.project.get_htmloutputpath(path))self.project.process(path)exceptKeyboardInterrupt:raiseexceptSystemMessage:# we assume docutils printed info on stdoutpy.test.fail("docutils processing failed, see captured stderr")defregister_linkrole(self):#directive.register_linkrole('api', self.resolve_linkrole)#directive.register_linkrole('source', self.resolve_linkrole)## # XXX fake sphinx' "toctree" and refs# directive.register_linkrole('ref', self.resolve_linkrole)fromdocutils.parsers.rstimportdirectivesdeftoctree_directive(name,arguments,options,content,lineno,content_offset,block_text,state,state_machine):return[]toctree_directive.content=1toctree_directive.options={'maxdepth':int,'glob':directives.flag,'hidden':directives.flag}directives.register_directive('toctree',toctree_directive)self.register_pygments()defregister_pygments(self):# taken from pygments-main/external/rst-directive.pyfromdocutils.parsers.rstimportdirectivestry:frompygments.formattersimportHtmlFormatterexceptImportError:defpygments_directive(name,arguments,options,content,lineno,content_offset,block_text,state,state_machine):return[]pygments_directive.options={}else:# The default formatterDEFAULT=HtmlFormatter(noclasses=True)# Add name -> formatter pairs for every variant you want to useVARIANTS={# 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True),}fromdocutilsimportnodesfrompygmentsimporthighlightfrompygments.lexersimportget_lexer_by_name,TextLexerdefpygments_directive(name,arguments,options,content,lineno,content_offset,block_text,state,state_machine):try:lexer=get_lexer_by_name(arguments[0])exceptValueError:# no lexer found - use the text one instead of an exceptionlexer=TextLexer()# take an arbitrary option if more than one is givenformatter=optionsandVARIANTS[options.keys()[0]]orDEFAULTparsed=highlight('\n'.join(content),lexer,formatter)return[nodes.raw('',parsed,format='html')]pygments_directive.options=dict([(key,directives.flag)forkeyinVARIANTS])pygments_directive.arguments=(1,0,1)pygments_directive.content=1directives.register_directive('sourcecode',pygments_directive)defresolve_linkrole(self,name,text,check=True):apigen_relpath=self.project.apigen_relpathifname=='api':iftext=='py':return('py',apigen_relpath+'api/index.html')else:asserttext.startswith('py.'),('api link "%s" does not point to the py package')%(text,)dotted_name=textifdotted_name.find('(')>-1:dotted_name=dotted_name[:text.find('(')]# remove pkg rootpath=dotted_name.split('.')[1:]dotted_name='.'.join(path)obj=pyifcheck:forchunkinpath:try:obj=getattr(obj,chunk)exceptAttributeError:raiseAssertionError('problem with linkrole :api:`%s`: can not resolve ''dotted name %s'%(text,dotted_name,))return(text,apigen_relpath+'api/%s.html'%(dotted_name,))elifname=='source':asserttext.startswith('py/'),('source link "%s" does not point ''to the py package')%(text,)relpath='/'.join(text.split('/')[1:])ifcheck:pkgroot=py._pydirabspath=pkgroot.join(relpath)assertpkgroot.join(relpath).check(),('problem with linkrole :source:`%s`: ''path %s does not exist'%(text,relpath))ifrelpath.endswith('/')ornotrelpath:relpath+='index.html'else:relpath+='.html'return(text,apigen_relpath+'source/%s'%(relpath,))elifname=='ref':return("","")def_checkskip(self,lpath,htmlpath=None):ifnotself.config.getvalue("forcegen"):lpath=py.path.local(lpath)ifhtmlpathisnotNone:htmlpath=py.path.local(htmlpath)iflpath.ext=='.txt':htmlpath=htmlpathorlpath.new(ext='.html')ifhtmlpath.check(file=1)andhtmlpath.mtime()>=lpath.mtime():py.test.skip("html file is up to date, use --forcegen to regenerate")#return [] # no need to rebuildclassDoctestText(py.test.collect.Item):defreportinfo(self):returnself.fspath,None,"doctest"defruntest(self):content=self._normalize_linesep()newcontent=self.config.hook.pytest_doctest_prepare_content(content=content)ifnewcontentisnotNone:content=newcontents=contentl=[]prefix='.. >>> 'mod=py.std.types.ModuleType(self.fspath.purebasename)skipchunk=Falseforlineindeindent(s).split('\n'):stripped=line.strip()ifskipchunkandline.startswith(skipchunk):py.builtin.print_("skipping",line)continueskipchunk=Falseifstripped.startswith(prefix):try:py.builtin.exec_(py.code.Source(stripped[len(prefix):]).compile(),mod.__dict__)exceptValueError:e=sys.exc_info()[1]ife.argsande.args[0]=="skipchunk":skipchunk=" "*(len(line)-len(line.lstrip()))else:raiseelse:l.append(line)docstring="\n".join(l)mod.__doc__=docstringfailed,tot=py.std.doctest.testmod(mod,verbose=1)iffailed:py.test.fail("doctest %s: %s failed out of %s"%(self.fspath,failed,tot))def_normalize_linesep(self):# XXX quite nasty... but it works (fixes win32 issues)s=self.fspath.read()linesep='\n'if'\r'ins:if'\n'notins:linesep='\r'else:linesep='\r\n's=s.replace(linesep,'\n')returnsclassLinkCheckerMaker(py.test.collect.Collector):defcollect(self):returnlist(self.genlinkchecks())defgenlinkchecks(self):path=self.fspath# generating functions + args as single teststimeout=self.config.getvalue("urlcheck_timeout")forlineno,lineinenumerate(path.readlines()):line=line.strip()ifline.startswith('.. _'):ifline.startswith('.. _`'):delim='`:'else:delim=':'l=line.split(delim,1)iflen(l)!=2:continuetryfn=l[1].strip()name="%s:%d"%(tryfn,lineno)iftryfn.startswith('http:')ortryfn.startswith('https'):ifself.config.getvalue("urlcheck"):yieldCheckLink(name,parent=self,args=(tryfn,path,lineno,timeout),checkfunc=urlcheck)eliftryfn.startswith('webcal:'):continueelse:i=tryfn.find('#')ifi!=-1:checkfn=tryfn[:i]else:checkfn=tryfnifcheckfn.strip()and(1orcheckfn.endswith('.html')):yieldCheckLink(name,parent=self,args=(tryfn,path,lineno),checkfunc=localrefcheck)classCheckLink(py.test.collect.Item):def__init__(self,name,parent,args,checkfunc):super(CheckLink,self).__init__(name,parent)self.args=argsself.checkfunc=checkfuncdefruntest(self):returnself.checkfunc(*self.args)defreportinfo(self,basedir=None):return(self.fspath,self.args[2],"checklink: %s"%self.args[0])defurlcheck(tryfn,path,lineno,TIMEOUT_URLOPEN):old=py.std.socket.getdefaulttimeout()py.std.socket.setdefaulttimeout(TIMEOUT_URLOPEN)try:try:py.builtin.print_("trying remote",tryfn)py.std.urllib2.urlopen(tryfn)finally:py.std.socket.setdefaulttimeout(old)except(py.std.urllib2.URLError,py.std.urllib2.HTTPError):e=sys.exc_info()[1]ifgetattr(e,'code',None)in(401,403):# authorization required, forbiddenpy.test.skip("%s: %s"%(tryfn,str(e)))else:py.test.fail("remote reference error %r in %s:%d\n%s"%(tryfn,path.basename,lineno+1,e))deflocalrefcheck(tryfn,path,lineno):# assume it should be a filei=tryfn.find('#')iftryfn.startswith('javascript:'):return# don't check JS refsifi!=-1:anchor=tryfn[i+1:]tryfn=tryfn[:i]else:anchor=''fn=path.dirpath(tryfn)ishtml=fn.ext=='.html'fn=ishtmlandfn.new(ext='.txt')orfnpy.builtin.print_("filename is",fn)ifnotfn.check():# not ishtml or not fn.check():ifnotpy.path.local(tryfn).check():# the html could be therepy.test.fail("reference error %r in %s:%d"%(tryfn,path.basename,lineno+1))ifanchor:source=unicode(fn.read(),'latin1')source=source.lower().replace('-',' ')# aehemanchor=anchor.replace('-',' ')match2=".. _`%s`:"%anchormatch3=".. _%s:"%anchorcandidates=(anchor,match2,match3)py.builtin.print_("candidates",repr(candidates))forlineinsource.split('\n'):line=line.strip()iflineincandidates:breakelse:py.test.fail("anchor reference error %s#%s in %s:%d"%(tryfn,anchor,path.basename,lineno+1))ifhasattr(sys.stdout,'fileno')andos.isatty(sys.stdout.fileno()):deflog(msg):print(msg)else:deflog(msg):passdefconvert_rest_html(source,source_path,stylesheet=None,encoding='latin1'):""" return html latin1-encoded document for the given input. source a ReST-string sourcepath where to look for includes (basically) stylesheet path (to be used if any) """fromdocutils.coreimportpublish_stringkwargs={'stylesheet':stylesheet,'stylesheet_path':None,'traceback':1,'embed_stylesheet':0,'output_encoding':encoding,#'halt' : 0, # 'info','halt_level':2,}# docutils uses os.getcwd() :-(source_path=os.path.abspath(str(source_path))prevdir=os.getcwd()try:#os.chdir(os.path.dirname(source_path))returnpublish_string(source,source_path,writer_name='html',settings_overrides=kwargs)finally:os.chdir(prevdir)defprocess(txtpath,encoding='latin1'):""" process a textfile """log("processing %s"%txtpath)asserttxtpath.check(ext='.txt')ifisinstance(txtpath,py.path.svnwc):txtpath=txtpath.localpathhtmlpath=txtpath.new(ext='.html')#svninfopath = txtpath.localpath.new(ext='.svninfo')style=txtpath.dirpath('style.css')ifstyle.check():stylesheet=style.basenameelse:stylesheet=Nonecontent=unicode(txtpath.read(),encoding)doc=convert_rest_html(content,txtpath,stylesheet=stylesheet,encoding=encoding)htmlpath.open('wb').write(doc)#log("wrote %r" % htmlpath)#if txtpath.check(svnwc=1, versioned=1):# info = txtpath.info()# svninfopath.dump(info)ifsys.version_info>(3,0):def_uni(s):returnselse:def_uni(s):returnunicode(s)rex1=re.compile(r'.*<body>(.*)</body>.*',re.MULTILINE|re.DOTALL)rex2=re.compile(r'.*<div class="document">(.*)</div>.*',re.MULTILINE|re.DOTALL)defstrip_html_header(string,encoding='utf8'):""" return the content of the body-tag """uni=unicode(string,encoding)forrexinrex1,rex2:match=rex.search(uni)ifnotmatch:breakuni=match.group(1)returnuniclassProject:# used for confrest.py filesdef__init__(self,sourcepath):self.sourcepath=sourcepathdefprocess(self,path):returnprocess(path)defget_htmloutputpath(self,path):returnpath.new(ext='html')