# -*- coding: utf-8 -*-""" Pygments basic API tests ~~~~~~~~~~~~~~~~~~~~~~~~ :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details."""importosimportunittestimportrandomfrompygmentsimportlexers,formatters,filters,formatfrompygments.tokenimport_TokenType,Textfrompygments.lexerimportRegexLexerfrompygments.formatters.imgimportFontNotFoundfrompygments.utilimportBytesIO,StringIO,bytes,bimportsupportTESTFILE,TESTDIR=support.location(__file__)test_content=[chr(i)foriinxrange(33,128)]*5random.shuffle(test_content)test_content=''.join(test_content)+'\n'classLexersTest(unittest.TestCase):deftest_import_all(self):# instantiate every lexer, to see if the token type defs are correctforxinlexers.LEXERS.keys():c=getattr(lexers,x)()deftest_lexer_classes(self):a=self.assert_ae=self.assertEquals# test that every lexer class has the correct public APIforlexerinlexers._iter_lexerclasses():a(type(lexer.name)isstr)forattrin'aliases','filenames','alias_filenames','mimetypes':a(hasattr(lexer,attr))a(type(getattr(lexer,attr))islist,"%s: %s attribute wrong"%(lexer,attr))result=lexer.analyse_text("abc")a(isinstance(result,float)and0.0<=result<=1.0)inst=lexer(opt1="val1",opt2="val2")ifissubclass(lexer,RegexLexer):ifnothasattr(lexer,'_tokens'):# if there's no "_tokens", the lexer has to be one with# multiple tokendef variantsa(lexer.token_variants)forvariantinlexer.tokens:a('root'inlexer.tokens[variant])else:a('root'inlexer._tokens,'%s has no root state'%lexer)tokens=list(inst.get_tokens(test_content))txt=""fortokenintokens:a(isinstance(token,tuple))a(isinstance(token[0],_TokenType))ifisinstance(token[1],str):printrepr(token[1])a(isinstance(token[1],unicode))txt+=token[1]ae(txt,test_content,"%s lexer roundtrip failed: %r != %r"%(lexer.name,test_content,txt))deftest_get_lexers(self):a=self.assert_ae=self.assertEquals# test that the lexers functions workforfunc,argsin[(lexers.get_lexer_by_name,("python",)),(lexers.get_lexer_for_filename,("test.py",)),(lexers.get_lexer_for_mimetype,("text/x-python",)),(lexers.guess_lexer,("#!/usr/bin/python -O\nprint",)),(lexers.guess_lexer_for_filename,("a.py","<%= @foo %>"))]:x=func(opt="val",*args)a(isinstance(x,lexers.PythonLexer))ae(x.options["opt"],"val")classFiltersTest(unittest.TestCase):deftest_basic(self):filter_args={'whitespace':{'spaces':True,'tabs':True,'newlines':True},'highlight':{'names':['isinstance','lexers','x']},}forxinfilters.FILTERS.keys():lx=lexers.PythonLexer()lx.add_filter(x,**filter_args.get(x,{}))text=open(TESTFILE,'rb').read().decode('utf-8')tokens=list(lx.get_tokens(text))roundtext=''.join([t[1]fortintokens])ifxnotin('whitespace','keywordcase'):# these filters change the textself.assertEquals(roundtext,text,"lexer roundtrip with %s filter failed"%x)deftest_raiseonerror(self):lx=lexers.PythonLexer()lx.add_filter('raiseonerror',excclass=RuntimeError)self.assertRaises(RuntimeError,list,lx.get_tokens('$'))deftest_whitespace(self):lx=lexers.PythonLexer()lx.add_filter('whitespace',spaces='%')text=open(TESTFILE,'rb').read().decode('utf-8')lxtext=''.join([t[1]fortinlist(lx.get_tokens(text))])self.failIf(' 'inlxtext)deftest_keywordcase(self):lx=lexers.PythonLexer()lx.add_filter('keywordcase',case='capitalize')text=open(TESTFILE,'rb').read().decode('utf-8')lxtext=''.join([t[1]fortinlist(lx.get_tokens(text))])self.assert_('Def'inlxtextand'Class'inlxtext)deftest_codetag(self):lx=lexers.PythonLexer()lx.add_filter('codetagify')text=u'# BUG: text'tokens=list(lx.get_tokens(text))self.assertEquals('# ',tokens[0][1])self.assertEquals('BUG',tokens[1][1])deftest_codetag_boundary(self):# http://dev.pocoo.org/projects/pygments/ticket/368lx=lexers.PythonLexer()lx.add_filter('codetagify')text=u'# DEBUG: text'tokens=list(lx.get_tokens(text))self.assertEquals('# DEBUG: text',tokens[0][1])classFormattersTest(unittest.TestCase):deftest_public_api(self):a=self.assert_ae=self.assertEqualsts=list(lexers.PythonLexer().get_tokens("def f(): pass"))out=StringIO()# test that every formatter class has the correct public APIforformatter,infoinformatters.FORMATTERS.iteritems():a(len(info)==4)a(info[0],"missing formatter name")# namea(info[1],"missing formatter aliases")# aliasesa(info[3],"missing formatter docstring")# docifformatter.name=='Raw tokens':# will not work with Unicode output filecontinuetry:inst=formatter(opt1="val1")except(ImportError,FontNotFound):continuetry:inst.get_style_defs()exceptNotImplementedError:# may be raised by formatters for which it doesn't make sensepassinst.format(ts,out)deftest_encodings(self):frompygments.formattersimportHtmlFormatter# unicode outputfmt=HtmlFormatter()tokens=[(Text,u"ä")]out=format(tokens,fmt)self.assert_(type(out)isunicode)self.assert_(u"ä"inout)# encoding optionfmt=HtmlFormatter(encoding="latin1")tokens=[(Text,u"ä")]self.assert_(u"ä".encode("latin1")informat(tokens,fmt))# encoding and outencoding optionfmt=HtmlFormatter(encoding="latin1",outencoding="utf8")tokens=[(Text,u"ä")]self.assert_(u"ä".encode("utf8")informat(tokens,fmt))deftest_styles(self):frompygments.formattersimportHtmlFormatterfmt=HtmlFormatter(style="pastie")deftest_unicode_handling(self):# test that the formatter supports encoding and Unicodetokens=list(lexers.PythonLexer(encoding='utf-8').get_tokens("def f(): 'ä'"))forformatter,infoinformatters.FORMATTERS.iteritems():try:inst=formatter(encoding=None)except(ImportError,FontNotFound):# some dependency or font not installedcontinueifformatter.name!='Raw tokens':out=format(tokens,inst)ifformatter.unicodeoutput:self.assert_(type(out)isunicode)inst=formatter(encoding='utf-8')out=format(tokens,inst)self.assert_(type(out)isbytes,'%s: %r'%(formatter,out))# Cannot test for encoding, since formatters may have to escape# non-ASCII characters.else:inst=formatter()out=format(tokens,inst)self.assert_(type(out)isbytes,'%s: %r'%(formatter,out))deftest_get_formatters(self):a=self.assert_ae=self.assertEquals# test that the formatters functions workx=formatters.get_formatter_by_name("html",opt="val")a(isinstance(x,formatters.HtmlFormatter))ae(x.options["opt"],"val")x=formatters.get_formatter_for_filename("a.html",opt="val")a(isinstance(x,formatters.HtmlFormatter))ae(x.options["opt"],"val")