# -*- coding: utf-8 -*-# Natural Language Toolkit: Text Trees## Copyright (C) 2001-2013 NLTK Project# Author: Edward Loper <edloper@gradient.cis.upenn.edu># Steven Bird <stevenbird1@gmail.com># Peter Ljunglรถf <peter.ljunglof@gu.se># Nathan Bodenstab <bodenstab@cslu.ogi.edu> (tree transforms)# URL: <http://www.nltk.org/># For license information, see LICENSE.TXT"""Class for representing hierarchical language structures, such assyntax trees and morphological trees."""from__future__importprint_function,unicode_literals# TODO: add LabelledTree (can be used for dependency trees)importrefromnltk.grammarimportProduction,Nonterminalfromnltk.probabilityimportProbabilisticMixInfromnltk.utilimportslice_boundsfromnltk.compatimportstring_types,python_2_unicode_compatible,unicode_reprfromnltk.internalsimportraise_unorderable_types######################################################################## Trees######################################################################@python_2_unicode_compatibleclassTree(list):""" A Tree represents a hierarchical grouping of leaves and subtrees. For example, each constituent in a syntax tree is represented by a single Tree. A tree's children are encoded as a list of leaves and subtrees, where a leaf is a basic (non-tree) value; and a subtree is a nested Tree. >>> from nltk.tree import Tree >>> print(Tree(1, [2, Tree(3, [4]), 5])) (1 2 (3 4) 5) >>> vp = Tree('VP', [Tree('V', ['saw']), ... Tree('NP', ['him'])]) >>> s = Tree('S', [Tree('NP', ['I']), vp]) >>> print(s) (S (NP I) (VP (V saw) (NP him))) >>> print(s[1]) (VP (V saw) (NP him)) >>> print(s[1,1]) (NP him) >>> t = Tree("(S(NPI)(VP(Vsaw)(NPhim)))") >>> s == t True >>> t[1][1].node = "X" >>> print(t) (S (NP I) (VP (V saw) (X him))) >>> t[0], t[1,1] = t[1,1], t[0] >>> print(t) (S (X him) (VP (V saw) (NP I))) The length of a tree is the number of children it has. >>> len(t) 2 Any other properties that a Tree defines are known as node properties, and are used to add information about individual hierarchical groupings. For example, syntax trees use a NODE property to label syntactic constituents with phrase tags, such as "NP" and "VP". Several Tree methods use "treepositions"tospecifychildrenordescendantsofatree.Treepositionsaredefinedasfollows:-Thetreeposition*i*specifiesaTree's *i*\ th child. - The tree position ``()`` specifies the Tree itself. - If *p* is the tree position of descendant *d*, then *p+i* specifies the *i*\ th child of *d*. I.e., every tree position is either a single index *i*, specifying ``tree[i]``; or a sequence *i1, i2, ..., iN*, specifying ``tree[i1][i2]...[iN]``. Construct a new tree. This constructor can be called in one of two ways: - ``Tree(node, children)`` constructs a new tree with the specified node value and list of children. - ``Tree(s)`` constructs a new tree by parsing the string ``s``. It is equivalent to calling the class method ``Tree.parse(s)``. """ def __init__(self, node_or_str, children=None): if children is None: if not isinstance(node_or_str, string_types): raise TypeError("%s: Expected a node value and child list " "or a single string" % type(self).__name__) tree = type(self).parse(node_or_str) list.__init__(self, tree) self.node = tree.node elif isinstance(children, string_types): raise TypeError("%s() argument 2 should be a list, not a " "string" % type(self).__name__) else: list.__init__(self, children) self.node = node_or_str #//////////////////////////////////////////////////////////// # Comparison operators #//////////////////////////////////////////////////////////// def __eq__(self, other): return (self.__class__ is other.__class__ and (self.node, list(self)) == (other.node, list(other))) def __lt__(self, other): if not isinstance(other, Tree): # raise_unorderable_types("<", self, other) # Sometimes children can be pure strings, # so we need to be able to compare with non-trees: return self.__class__.__name__ < other.__class__.__name__ elif self.__class__ is other.__class__: return (self.node, list(self)) < (other.node, list(other)) else: return self.__class__.__name__ < other.__class__.__name__ # @total_ordering doesn'tworkhere,sincetheclassinheritsfromabuiltinclass__ne__=lambdaself,other:notself==other__gt__=lambdaself,other:not(self<otherorself==other)__le__=lambdaself,other:self<otherorself==other__ge__=lambdaself,other:notself<other#////////////////////////////////////////////////////////////#Disabledlistoperations#////////////////////////////////////////////////////////////def__mul__(self,v):raiseTypeError('Tree does not support multiplication')def__rmul__(self,v):raiseTypeError('Tree does not support multiplication')def__add__(self,v):raiseTypeError('Tree does not support addition')def__radd__(self,v):raiseTypeError('Tree does not support addition')#////////////////////////////////////////////////////////////#Indexing(withsupportfortreepositions)#////////////////////////////////////////////////////////////def__getitem__(self,index):ifisinstance(index,(int,slice)):returnlist.__getitem__(self,index)elifisinstance(index,(list,tuple)):iflen(index)==0:returnselfeliflen(index)==1:returnself[index[0]]else:returnself[index[0]][index[1:]]else:raiseTypeError("%s indices must be integers, not %s"%(type(self).__name__,type(index).__name__))def__setitem__(self,index,value):ifisinstance(index,(int,slice)):returnlist.__setitem__(self,index,value)elifisinstance(index,(list,tuple)):iflen(index)==0:raiseIndexError('The tree position () may not be ''assigned to.')eliflen(index)==1:self[index[0]]=valueelse:self[index[0]][index[1:]]=valueelse:raiseTypeError("%s indices must be integers, not %s"%(type(self).__name__,type(index).__name__))def__delitem__(self,index):ifisinstance(index,(int,slice)):returnlist.__delitem__(self,index)elifisinstance(index,(list,tuple)):iflen(index)==0:raiseIndexError('The tree position () may not be deleted.')eliflen(index)==1:delself[index[0]]else:delself[index[0]][index[1:]]else:raiseTypeError("%s indices must be integers, not %s"%(type(self).__name__,type(index).__name__))#////////////////////////////////////////////////////////////#Basictreeoperations#////////////////////////////////////////////////////////////defleaves(self):""" Return the leaves of the tree. >>> t = Tree("(S(NP(Dthe)(Ndog))(VP(Vchased)(NP(Dthe)(Ncat))))") >>> t.leaves() ['the', 'dog', 'chased', 'the', 'cat'] :return: a list containing this tree's leaves. The order reflects the order of the leaves in the tree's hierarchical structure. :rtype: list """leaves=[]forchildinself:ifisinstance(child,Tree):leaves.extend(child.leaves())else:leaves.append(child)returnleavesdefflatten(self):""" Return a flat version of the tree, with all non-root non-terminals removed. >>> t = Tree("(S(NP(Dthe)(Ndog))(VP(Vchased)(NP(Dthe)(Ncat))))") >>> print(t.flatten()) (S the dog chased the cat) :return: a tree consisting of this tree's root connected directly to its leaves, omitting all intervening non-terminal nodes. :rtype: Tree """returnTree(self.node,self.leaves())defheight(self):""" Return the height of the tree. >>> t = Tree("(S(NP(Dthe)(Ndog))(VP(Vchased)(NP(Dthe)(Ncat))))") >>> t.height() 5 >>> print(t[0,0]) (D the) >>> t[0,0].height() 2 :return: The height of this tree. The height of a tree containing no children is 1; the height of a tree containing only leaves is 2; and the height of any other tree is one plus the maximum of its children's heights. :rtype: int """max_child_height=0forchildinself:ifisinstance(child,Tree):max_child_height=max(max_child_height,child.height())else:max_child_height=max(max_child_height,1)return1+max_child_heightdeftreepositions(self,order='preorder'):""" >>> t = Tree("(S(NP(Dthe)(Ndog))(VP(Vchased)(NP(Dthe)(Ncat))))") >>> t.treepositions() # doctest: +ELLIPSIS [(), (0,), (0, 0), (0, 0, 0), (0, 1), (0, 1, 0), (1,), (1, 0), (1, 0, 0), ...] >>> for pos in t.treepositions('leaves'): ... t[pos] = t[pos][::-1].upper() >>> print(t) (S (NP (D EHT) (N GOD)) (VP (V DESAHC) (NP (D EHT) (N TAC)))) :param order: One of: ``preorder``, ``postorder``, ``bothorder``, ``leaves``. """positions=[]iforderin('preorder','bothorder'):positions.append(())fori,childinenumerate(self):ifisinstance(child,Tree):childpos=child.treepositions(order)positions.extend((i,)+pforpinchildpos)else:positions.append((i,))iforderin('postorder','bothorder'):positions.append(())returnpositionsdefsubtrees(self,filter=None):""" Generate all the subtrees of this tree, optionally restricted to trees matching the filter function. >>> t = Tree("(S(NP(Dthe)(Ndog))(VP(Vchased)(NP(Dthe)(Ncat))))") >>> for s in t.subtrees(lambda t: t.height() == 2): ... print(s) (D the) (N dog) (V chased) (D the) (N cat) :type filter: function :param filter: the function to filter all local trees """ifnotfilterorfilter(self):yieldselfforchildinself:ifisinstance(child,Tree):forsubtreeinchild.subtrees(filter):yieldsubtreedefproductions(self):""" Generate the productions that correspond to the non-terminal nodes of the tree. For each subtree of the form (P: C1 C2 ... Cn) this produces a production of the form P -> C1 C2 ... Cn. >>> t = Tree("(S(NP(Dthe)(Ndog))(VP(Vchased)(NP(Dthe)(Ncat))))") >>> t.productions() [S -> NP VP, NP -> D N, D -> 'the', N -> 'dog', VP -> V NP, V -> 'chased', NP -> D N, D -> 'the', N -> 'cat'] :rtype: list(Production) """ifnotisinstance(self.node,string_types):raiseTypeError('Productions can only be generated from trees having node labels that are strings')prods=[Production(Nonterminal(self.node),_child_names(self))]forchildinself:ifisinstance(child,Tree):prods+=child.productions()returnprodsdefpos(self):""" Return a sequence of pos-tagged words extracted from the tree. >>> t = Tree("(S(NP(Dthe)(Ndog))(VP(Vchased)(NP(Dthe)(Ncat))))") >>> t.pos() [('the', 'D'), ('dog', 'N'), ('chased', 'V'), ('the', 'D'), ('cat', 'N')] :return: a list of tuples containing leaves and pre-terminals (part-of-speech tags). The order reflects the order of the leaves in the tree's hierarchical structure. :rtype: list(tuple) """pos=[]forchildinself:ifisinstance(child,Tree):pos.extend(child.pos())else:pos.append((child,self.node))returnposdefleaf_treeposition(self,index):""" :return: The tree position of the ``index``-th leaf in this tree. I.e., if ``tp=self.leaf_treeposition(i)``, then ``self[tp]==self.leaves()[i]``. :raise IndexError: If this tree contains fewer than ``index+1`` leaves, or if ``index<0``. """ifindex<0:raiseIndexError('index must be non-negative')stack=[(self,())]whilestack:value,treepos=stack.pop()ifnotisinstance(value,Tree):ifindex==0:returntreeposelse:index-=1else:foriinrange(len(value)-1,-1,-1):stack.append((value[i],treepos+(i,)))raiseIndexError('index must be less than or equal to len(self)')deftreeposition_spanning_leaves(self,start,end):""" :return: The tree position of the lowest descendant of this tree that dominates ``self.leaves()[start:end]``. :raise ValueError: if ``end <= start`` """ifend<=start:raiseValueError('end must be greater than start')#Findthetreepositionsofthestart&endleaves,and#takethelongestcommonsubsequence.start_treepos=self.leaf_treeposition(start)end_treepos=self.leaf_treeposition(end-1)#Findthefirstindexwheretheymismatch:foriinrange(len(start_treepos)):ifi==len(end_treepos)orstart_treepos[i]!=end_treepos[i]:returnstart_treepos[:i]returnstart_treepos#////////////////////////////////////////////////////////////#Transforms#////////////////////////////////////////////////////////////defchomsky_normal_form(self,factor="right",horzMarkov=None,vertMarkov=0,childChar="|",parentChar="^"):""" This method can modify a tree in three ways: 1. Convert a tree into its Chomsky Normal Form (CNF) equivalent -- Every subtree has either two non-terminals or one terminal as its children. This process requires the creation of more"artificial" non-terminal nodes. 2. Markov (vertical) smoothing of children in new artificial nodes 3. Horizontal (parent) annotation of nodes :param factor: Right or left factoring method (default = "right") :type factor: str = [left|right] :param horzMarkov: Markov order for sibling smoothing in artificial nodes (None (default) = include all siblings) :type horzMarkov: int | None :param vertMarkov: Markov order for parent smoothing (0 (default) = no vertical annotation) :type vertMarkov: int | None :param childChar: A string used in construction of the artificial nodes, separating the head of the original subtree from the child nodes that have yet to be expanded (default = "|") :type childChar: str :param parentChar: A string used to separate the node representation from its vertical annotation :type parentChar: str """from.treetransformsimportchomsky_normal_formchomsky_normal_form(self,factor,horzMarkov,vertMarkov,childChar,parentChar)defun_chomsky_normal_form(self,expandUnary=True,childChar="|",parentChar="^",unaryChar="+"):""" This method modifies the tree in three ways: 1. Transforms a tree in Chomsky Normal Form back to its original structure (branching greater than two) 2. Removes any parent annotation (if it exists) 3. (optional) expands unary subtrees (if previously collapsed with collapseUnary(...) ) :param expandUnary: Flag to expand unary or not (default = True) :type expandUnary: bool :param childChar: A string separating the head node from its children in an artificial node (default = "|") :type childChar: str :param parentChar: A sting separating the node label from its parent annotation (default = "^") :type parentChar: str :param unaryChar: A string joining two non-terminals in a unary production (default = "+") :type unaryChar: str """from.treetransformsimportun_chomsky_normal_formun_chomsky_normal_form(self,expandUnary,childChar,parentChar,unaryChar)defcollapse_unary(self,collapsePOS=False,collapseRoot=False,joinChar="+"):""" Collapse subtrees with a single child (ie. unary productions) into a new non-terminal (Tree node) joined by 'joinChar'. This is useful when working with algorithms that do not allow unary productions, and completely removing the unary productions would require loss of useful information. The Tree is modified directly (since it is passed by reference) and no value is returned. :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie. Part-of-Speech tags) since they are always unary productions :type collapsePOS: bool :param collapseRoot: 'False' (default) will not modify the root production if it is unary. For the Penn WSJ treebank corpus, this corresponds to the TOP -> productions. :type collapseRoot: bool :param joinChar: A string used to connect collapsed node values (default = "+") :type joinChar: str """from.treetransformsimportcollapse_unarycollapse_unary(self,collapsePOS,collapseRoot,joinChar)#////////////////////////////////////////////////////////////#Convert,copy#////////////////////////////////////////////////////////////@classmethoddefconvert(cls,tree):""" Convert a tree between different subtypes of Tree. ``cls`` determines which class will be used to encode the new tree. :type tree: Tree :param tree: The tree that should be converted. :return: The new Tree. """ifisinstance(tree,Tree):children=[cls.convert(child)forchildintree]returncls(tree.node,children)else:returntreedefcopy(self,deep=False):ifnotdeep:returntype(self)(self.node,self)else:returntype(self).convert(self)def_frozen_class(self):returnImmutableTreedeffreeze(self,leaf_freezer=None):frozen_class=self._frozen_class()ifleaf_freezerisNone:newcopy=frozen_class.convert(self)else:newcopy=self.copy(deep=True)forposinnewcopy.treepositions('leaves'):newcopy[pos]=leaf_freezer(newcopy[pos])newcopy=frozen_class.convert(newcopy)hash(newcopy)#Makesuretheleavesarehashable.returnnewcopy#////////////////////////////////////////////////////////////#Parsing#////////////////////////////////////////////////////////////@classmethoddefparse(cls,s,brackets='()',parse_node=None,parse_leaf=None,node_pattern=None,leaf_pattern=None,remove_empty_top_bracketing=False):""" Parse a bracketed tree string and return the resulting tree. Trees are represented as nested brackettings, such as:: (S (NP (NNP John)) (VP (V runs))) :type s: str :param s: The string to parse :type brackets: str (length=2) :param brackets: The bracket characters used to mark the beginning and end of trees and subtrees. :type parse_node: function :type parse_leaf: function :param parse_node, parse_leaf: If specified, these functions are applied to the substrings of ``s`` corresponding to nodes and leaves (respectively) to obtain the values for those nodes and leaves. They should have the following signature: parse_node(str) -> value For example, these functions could be used to parse nodes and leaves whose values should be some type other than string (such as ``FeatStruct``). Note that by default, node strings and leaf strings are delimited by whitespace and brackets; to override this default, use the ``node_pattern`` and ``leaf_pattern`` arguments. :type node_pattern: str :type leaf_pattern: str :param node_pattern, leaf_pattern: Regular expression patterns used to find node and leaf substrings in ``s``. By default, both nodes patterns are defined to match any sequence of non-whitespace non-bracket characters. :type remove_empty_top_bracketing: bool :param remove_empty_top_bracketing: If the resulting tree has an empty node label, and is length one, then return its single child instead. This is useful for treebank trees, which sometimes contain an extra level of bracketing. :return: A tree corresponding to the string representation ``s``. If this class method is called using a subclass of Tree, then it will return a tree of that type. :rtype: Tree """ifnotisinstance(brackets,string_types)orlen(brackets)!=2:raiseTypeError('brackets must be a length-2 string')ifre.search('\s',brackets):raiseTypeError('whitespace brackets not allowed')#Constructaregexpthatwilltokenizethestring.open_b,close_b=bracketsopen_pattern,close_pattern=(re.escape(open_b),re.escape(close_b))ifnode_patternisNone:node_pattern='[^\s%s%s]+'% (open_pattern, close_pattern)ifleaf_patternisNone:leaf_pattern='[^\s%s%s]+'% (open_pattern, close_pattern)token_re=re.compile('%s\s*(%s)?|%s|(%s)'% (open_pattern,node_pattern,close_pattern,leaf_pattern))#Walkthrougheachtoken,updatingastackoftrees.stack=[(None,[])]#listof(node,children)tuplesformatchintoken_re.finditer(s):token=match.group()#Beginningofatree/subtreeiftoken[0]==open_b:iflen(stack)==1andlen(stack[0][1])>0:cls._parse_error(s,match,'end-of-string')node=token[1:].lstrip()ifparse_nodeisnotNone:node=parse_node(node)stack.append((node,[]))#Endofatree/subtreeeliftoken==close_b:iflen(stack)==1:iflen(stack[0][1])==0:cls._parse_error(s,match,open_b)else:cls._parse_error(s,match,'end-of-string')node,children=stack.pop()stack[-1][1].append(cls(node,children))#Leafnodeelse:iflen(stack)==1:cls._parse_error(s,match,open_b)ifparse_leafisnotNone:token=parse_leaf(token)stack[-1][1].append(token)#checkthatwegotexactlyonecompletetree.iflen(stack)>1:cls._parse_error(s,'end-of-string',close_b)eliflen(stack[0][1])==0:cls._parse_error(s,'end-of-string',open_b)else:assertstack[0][0]isNoneassertlen(stack[0][1])==1tree=stack[0][1][0]#Ifthetreehasanextralevelwithnode='',thengetridof#it.E.g.:"((S (NP ...) (VP ...)))"ifremove_empty_top_bracketingandtree.node==''andlen(tree)==1:tree=tree[0]#returnthetree.returntree@classmethoddef_parse_error(cls,s,match,expecting):""" Display a friendly error message when parsing a tree string fails. :param s: The string we're parsing. :param match: regexp match of the problem token. :param expecting: what we expected to see instead. """#Constructabasicerrormessageifmatch=='end-of-string':pos,token=len(s),'end-of-string'else:pos,token=match.start(),match.group()msg='%s.parse(): expected %r but got %r\n%sat index %d.'% (cls.__name__,expecting,token,' '*12,pos)#Addadisplayshowingtheerrortokenitsels:s=s.replace('\n',' ').replace('\t',' ')offset=posiflen(s)>pos+10:s=s[:pos+10]+'...'ifpos>10:s='...'+s[pos-10:]offset=13msg+='\n%s"%s"\n%s^'% (' '*16, s, ' '*(17+offset))raiseValueError(msg)#////////////////////////////////////////////////////////////#Visualization&StringRepresentation#////////////////////////////////////////////////////////////defdraw(self):""" Open a new window containing a graphical diagram of this tree. """fromnltk.draw.treeimportdraw_treesdraw_trees(self)def__repr__(self):childstr=", ".join(unicode_repr(c)forcinself)return'%s(%s, [%s])'% (type(self).__name__, unicode_repr(self.node), childstr)def__str__(self):returnself.pprint()defpprint(self,margin=70,indent=0,nodesep='',parens='()',quotes=False):""" :return: A pretty-printed string representation of this tree. :rtype: str :param margin: The right margin at which to do line-wrapping. :type margin: int :param indent: The indentation level at which printing begins. This number is used to decide how far to indent subsequent lines. :type indent: int :param nodesep: A string that is used to separate the node from the children. E.g., the default value ``':'`` gives trees like ``(S: (NP: I) (VP: (V: saw) (NP: it)))``. """#Trywritingitononeline.s=self._pprint_flat(nodesep,parens,quotes)iflen(s)+indent<margin:returns#Ifitdoesn't fit on one line, then write it on multi-lines. if isinstance(self.node, string_types): s = '%s%s%s' % (parens[0], self.node, nodesep)else:s='%s%s%s'% (parens[0], unicode_repr(self.node), nodesep)forchildinself:ifisinstance(child,Tree):s+='\n'+' '*(indent+2)+child.pprint(margin,indent+2,nodesep,parens,quotes)elifisinstance(child,tuple):s+='\n'+' '*(indent+2)+"/".join(child)elifisinstance(child,string_types)andnotquotes:s+='\n'+' '*(indent+2)+'%s'% childelse:s+='\n'+' '*(indent+2)+unicode_repr(child)returns+parens[1]defpprint_latex_qtree(self):r"""ReturnsarepresentationofthetreecompatiblewiththeLaTeXqtreepackage.Thisconsistsofthestring``\Tree``followedbytheparsetreerepresentedinbracketednotation.Forexample,thefollowingresultwasgeneratedfromaparsetreeofthesentence``Theannouncementastoundedus``::\Tree[.I''[.N''[.DThe][.N' [.N announcement ] ] ] [.I'[.V''[.V' [.V astounded ] [.N'' [.N'[.Nus]]]]]]]Seehttp://www.ling.upenn.edu/advice/latex.htmlfortheLaTeXstylefilefortheqtreepackage.:return:Alatexqtreerepresentationofthistree.:rtype:str"""reserved_chars=re.compile('([#\$%&~_\{\}])')pprint=self.pprint(indent=6,nodesep='',parens=('[.',' ]'))returnr'\Tree '+re.sub(reserved_chars,r'\\\1',pprint)def_pprint_flat(self,nodesep,parens,quotes):childstrs=[]forchildinself:ifisinstance(child,Tree):childstrs.append(child._pprint_flat(nodesep,parens,quotes))elifisinstance(child,tuple):childstrs.append("/".join(child))elifisinstance(child,string_types)andnotquotes:childstrs.append('%s'% child)else:childstrs.append(unicode_repr(child))ifisinstance(self.node,string_types):return'%s%s%s %s%s'% (parens[0], self.node, nodesep," ".join(childstrs),parens[1])else:return'%s%s%s %s%s'% (parens[0], unicode_repr(self.node), nodesep," ".join(childstrs),parens[1])classImmutableTree(Tree):def__init__(self,node_or_str,children=None):super(ImmutableTree,self).__init__(node_or_str,children)#Precomputeourhashvalue.Thisensuresthatwe're really # immutable. It also means we only have to calculate it once. try: self._hash = hash((self.node, tuple(self))) except (TypeError, ValueError): raise ValueError("%s: node value and children " "must be immutable" % type(self).__name__) def __setitem__(self, index, value): raise ValueError('%s may not be modified' % type(self).__name__)def__setslice__(self,i,j,value):raiseValueError('%s may not be modified'% type(self).__name__)def__delitem__(self,index):raiseValueError('%s may not be modified'% type(self).__name__)def__delslice__(self,i,j):raiseValueError('%s may not be modified'% type(self).__name__)def__iadd__(self,other):raiseValueError('%s may not be modified'% type(self).__name__)def__imul__(self,other):raiseValueError('%s may not be modified'% type(self).__name__)defappend(self,v):raiseValueError('%s may not be modified'% type(self).__name__)defextend(self,v):raiseValueError('%s may not be modified'% type(self).__name__)defpop(self,v=None):raiseValueError('%s may not be modified'% type(self).__name__)defremove(self,v):raiseValueError('%s may not be modified'% type(self).__name__)defreverse(self):raiseValueError('%s may not be modified'% type(self).__name__)defsort(self):raiseValueError('%s may not be modified'% type(self).__name__)def__hash__(self):returnself._hashdef_get_node(self):"""Get the node value"""returnself._nodedef_set_node(self,value):""" Set the node value. This will only succeed the first time the node value is set, which should occur in ImmutableTree.__init__(). """ifhasattr(self,'node'):raiseValueError('%s may not be modified'% type(self).__name__)self._node=valuenode=property(_get_node,_set_node)######################################################################## Parented trees######################################################################classAbstractParentedTree(Tree):""" An abstract base class for a ``Tree`` that automatically maintains pointers to parent nodes. These parent pointers are updated whenever any change is made to a tree's structure. Two subclasses are currently defined: - ``ParentedTree`` is used for tree structures where each subtree has at most one parent. This class should be used in cases where there is no"sharing" of subtrees. - ``MultiParentedTree`` is used for tree structures where a subtree may have zero or more parents. This class should be used in cases where subtrees may be shared. Subclassing =========== The ``AbstractParentedTree`` class redefines all operations that modify a tree's structure to call two methods, which are used by subclasses to update parent information: - ``_setparent()`` is called whenever a new child is added. - ``_delparent()`` is called whenever a child is removed. """def__init__(self,node_or_str,children=None):super(AbstractParentedTree,self).__init__(node_or_str,children)#IfchildrenisNone,thetreeisparsedfromnode_or_str,and#allparentswillbesetduringparsing.ifchildrenisnotNone:#Otherwisewehavetosettheparentofthechildren.#Iterateoverself,and*not*children,becausechildren#mightbeaniterator.fori,childinenumerate(self):ifisinstance(child,Tree):self._setparent(child,i,dry_run=True)fori,childinenumerate(self):ifisinstance(child,Tree):self._setparent(child,i)#////////////////////////////////////////////////////////////#Parentmanagement#////////////////////////////////////////////////////////////def_setparent(self,child,index,dry_run=False):""" Update the parent pointer of ``child`` to point to ``self``. This method is only called if the type of ``child`` is ``Tree``; i.e., it is not called when adding a leaf to a tree. This method is always called before the child is actually added to the child list of ``self``. :type child: Tree :type index: int :param index: The index of ``child`` in ``self``. :raise TypeError: If ``child`` is a tree with an impropriate type. Typically, if ``child`` is a tree, then its type needs to match the type of ``self``. This prevents mixing of different tree types (single-parented, multi-parented, and non-parented). :param dry_run: If true, the don't actually set the child's parent pointer; just check for any error conditions, and raise an exception if one is found. """raiseNotImplementedError()def_delparent(self,child,index):""" Update the parent pointer of ``child`` to not point to self. This method is only called if the type of ``child`` is ``Tree``; i.e., it is not called when removing a leaf from a tree. This method is always called before the child is actually removed from the child list of ``self``. :type child: Tree :type index: int :param index: The index of ``child`` in ``self``. """raiseNotImplementedError()#////////////////////////////////////////////////////////////#Methodsthatadd/removechildren#////////////////////////////////////////////////////////////#Everymethodthataddsorremovesachildmustmake#appropriatecallsto_setparent()and_delparent().def__delitem__(self,index):#delptree[start:stop]ifisinstance(index,slice):start,stop,step=slice_bounds(self,index,allow_step=True)#Clearallthechildrenpointers.foriinrange(start,stop,step):ifisinstance(self[i],Tree):self._delparent(self[i],i)#Deletethechildrenfromourchildlist.super(AbstractParentedTree,self).__delitem__(index)#delptree[i]elifisinstance(index,int):ifindex<0:index+=len(self)ifindex<0:raiseIndexError('index out of range')#Clearthechild's parent pointer. if isinstance(self[index], Tree): self._delparent(self[index], index) # Remove the child from our child list. super(AbstractParentedTree, self).__delitem__(index) elif isinstance(index, (list, tuple)): # del ptree[()] if len(index) == 0: raise IndexError('Thetreeposition()maynotbedeleted.') # del ptree[(i,)] elif len(index) == 1: del self[index[0]] # del ptree[i1, i2, i3] else: del self[index[0]][index[1:]] else: raise TypeError("%s indices must be integers, not %s" % (type(self).__name__, type(index).__name__)) def __setitem__(self, index, value): # ptree[start:stop] = value if isinstance(index, slice): start, stop, step = slice_bounds(self, index, allow_step=True) # make a copy of value, in case it'saniteratorifnotisinstance(value,(list,tuple)):value=list(value)#Checkforanyerrorconditions,sowecanavoidending#upinaninconsistentstateifanerrordoesoccur.fori,childinenumerate(value):ifisinstance(child,Tree):self._setparent(child,start+i*step,dry_run=True)#clearthechildpointersofallparentswe're removing for i in range(start, stop, step): if isinstance(self[i], Tree): self._delparent(self[i], i) # set the child pointers of the new children. We do this # after clearing *all* child pointers, in case we'ree.g.#reversingtheelementsinatree.fori,childinenumerate(value):ifisinstance(child,Tree):self._setparent(child,start+i*step)#finally,updatethecontentofthechildlistitself.super(AbstractParentedTree,self).__setitem__(index,value)#ptree[i]=valueelifisinstance(index,int):ifindex<0:index+=len(self)ifindex<0:raiseIndexError('index out of range')#ifthevalueisnotchanging,donothing.ifvalueisself[index]:return#Setthenewchild's parent pointer. if isinstance(value, Tree): self._setparent(value, index) # Remove the old child'sparentpointerifisinstance(self[index],Tree):self._delparent(self[index],index)#Updateourchildlist.super(AbstractParentedTree,self).__setitem__(index,value)elifisinstance(index,(list,tuple)):#ptree[()]=valueiflen(index)==0:raiseIndexError('The tree position () may not be assigned to.')#ptree[(i,)]=valueeliflen(index)==1:self[index[0]]=value#ptree[i1,i2,i3]=valueelse:self[index[0]][index[1:]]=valueelse:raiseTypeError("%s indices must be integers, not %s"%(type(self).__name__,type(index).__name__))defappend(self,child):ifisinstance(child,Tree):self._setparent(child,len(self))super(AbstractParentedTree,self).append(child)defextend(self,children):forchildinchildren:ifisinstance(child,Tree):self._setparent(child,len(self))super(AbstractParentedTree,self).append(child)definsert(self,index,child):#Handlenegativeindexes.Notethatifindex<-len(self),#wedo*not*raiseanIndexError,unlike__getitem__.This#isdoneforconsistencywithlist.__getitem__andlist.index.ifindex<0:index+=len(self)ifindex<0:index=0#Setthechild's parent, and update our child list. if isinstance(child, Tree): self._setparent(child, index) super(AbstractParentedTree, self).insert(index, child) def pop(self, index=-1): if index < 0: index += len(self) if index < 0: raise IndexError('indexoutofrange') if isinstance(self[index], Tree): self._delparent(self[index], index) return super(AbstractParentedTree, self).pop(index) # n.b.: like `list`, this is done by equality, not identity! # To remove a specific child, use del ptree[i]. def remove(self, child): index = self.index(child) if isinstance(self[index], Tree): self._delparent(self[index], index) super(AbstractParentedTree, self).remove(child) # We need to implement __getslice__ and friends, even though # they'redeprecated,becauseotherwiselist.__getslice__willget#called(sincewe're subclassing from list). Just delegate to # __getitem__ etc., but use max(0, start) and max(0, stop) because # because negative indices are already handled *before* # __getslice__ is called; and we don'twanttodouble-countthem.ifhasattr(list,'__getslice__'):def__getslice__(self,start,stop):returnself.__getitem__(slice(max(0,start),max(0,stop)))def__delslice__(self,start,stop):returnself.__delitem__(slice(max(0,start),max(0,stop)))def__setslice__(self,start,stop,value):returnself.__setitem__(slice(max(0,start),max(0,stop)),value)classParentedTree(AbstractParentedTree):""" A ``Tree`` that automatically maintains parent pointers for single-parented trees. The following are methods for querying the structure of a parented tree: ``parent``, ``parent_index``, ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``. Each ``ParentedTree`` may have at most one parent. In particular, subtrees may not be shared. Any attempt to reuse a single ``ParentedTree`` as a child of more than one parent (or as multiple children of the same parent) will cause a ``ValueError`` exception to be raised. ``ParentedTrees`` should never be used in the same tree as ``Trees`` or ``MultiParentedTrees``. Mixing tree implementations may result in incorrect parent pointers and in ``TypeError`` exceptions. """def__init__(self,node_or_str,children=None):self._parent=None"""The parent of this Tree, or None if it has no parent."""super(ParentedTree,self).__init__(node_or_str,children)ifchildrenisNone:#IfchildrenisNone,thetreeisparsedfromnode_or_str.#Afterparsing,theparentoftheimmediatechildren#willpointtoanintermediatetree,notself.#Wefixthisbybruteforce:fori,childinenumerate(self):ifisinstance(child,Tree):child._parent=Noneself._setparent(child,i)def_frozen_class(self):returnImmutableParentedTree#/////////////////////////////////////////////////////////////////#Methods#/////////////////////////////////////////////////////////////////defparent(self):"""The parent of this tree, or None if it has no parent."""returnself._parentdefparent_index(self):""" The index of this tree in its parent. I.e., ``ptree.parent()[ptree.parent_index()] is ptree``. Note that ``ptree.parent_index()`` is not necessarily equal to ``ptree.parent.index(ptree)``, since the ``index()`` method returns the first child that is equal to its argument. """ifself._parentisNone:returnNonefori,childinenumerate(self._parent):ifchildisself:returniassertFalse,'expected to find self in self._parent!'defleft_sibling(self):"""The left sibling of this tree, or None if it has none."""parent_index=self.parent_index()ifself._parentandparent_index>0:returnself._parent[parent_index-1]returnNone#noleftsiblingdefright_sibling(self):"""The right sibling of this tree, or None if it has none."""parent_index=self.parent_index()ifself._parentandparent_index<(len(self._parent)-1):returnself._parent[parent_index+1]returnNone#norightsiblingdefroot(self):""" The root of this tree. I.e., the unique ancestor of this tree whose parent is None. If ``ptree.parent()`` is None, then ``ptree`` is its own root. """root=selfwhileroot.parent()isnotNone:root=root.parent()returnrootdeftreeposition(self):""" The tree position of this tree, relative to the root of the tree. I.e., ``ptree.root[ptree.treeposition] is ptree``. """ifself.parent()isNone:return()else:returnself.parent().treeposition()+(self.parent_index(),)#/////////////////////////////////////////////////////////////////#ParentManagement#/////////////////////////////////////////////////////////////////def_delparent(self,child,index):#Sanitychecksassertisinstance(child,ParentedTree)assertself[index]ischildassertchild._parentisself#Deletechild's parent pointer. child._parent = None def _setparent(self, child, index, dry_run=False): # If the child'stypeisincorrect,thencomplain.ifnotisinstance(child,ParentedTree):raiseTypeError('Can not insert a non-ParentedTree '+'into a ParentedTree')#Ifchildalreadyhasaparent,thencomplain.ifchild._parentisnotNone:raiseValueError('Can not insert a subtree that already ''has a parent.')#Setchild's parent pointer & index. if not dry_run: child._parent = selfclass MultiParentedTree(AbstractParentedTree): """ A ``Tree`` that automatically maintains parent pointers for multi-parented trees. The following are methods for querying the structure of a multi-parented tree: ``parents()``, ``parent_indices()``, ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``. Each ``MultiParentedTree`` may have zero or more parents. In particular, subtrees may be shared. If a single ``MultiParentedTree`` is used as multiple children of the same parent, then that parent will appear multiple times in its ``parents()`` method. ``MultiParentedTrees`` should never be used in the same tree as ``Trees`` or ``ParentedTrees``. Mixing tree implementations may result in incorrect parent pointers and in ``TypeError`` exceptions. """ def __init__(self, node_or_str, children=None): self._parents = [] """A list of this tree'sparents.Thislistshouldnotcontainduplicates,evenifaparentcontainsthistreemultipletimes.""" super(MultiParentedTree, self).__init__(node_or_str, children) if children is None: # If children is None, the tree is parsed from node_or_str. # After parsing, the parent(s) of the immediate children # will point to an intermediate tree, not self. # We fix this by brute force: for i, child in enumerate(self): if isinstance(child, Tree): child._parents = [] self._setparent(child, i) def _frozen_class(self): return ImmutableMultiParentedTree #///////////////////////////////////////////////////////////////// # Methods #///////////////////////////////////////////////////////////////// def parents(self): """Thesetofparentsofthistree.Ifthistreehasnoparents,then``parents``istheemptyset.Tocheckifatreeisusedasmultiplechildrenofthesameparent,usethe``parent_indices()``method.:type:list(MultiParentedTree)""" return list(self._parents) def left_siblings(self): """Alistofallleftsiblingsofthistree,inanyofitsparenttrees.Atreemaybeitsownleftsiblingifitisusedasmultiplecontiguouschildrenofthesameparent.Atreemayappearmultipletimesinthislistifitistheleftsiblingofthistreewithrespecttomultipleparents.:type:list(MultiParentedTree)""" return [parent[index-1] for (parent, index) in self._get_parent_indices() if index > 0] def right_siblings(self): """Alistofallrightsiblingsofthistree,inanyofitsparenttrees.Atreemaybeitsownrightsiblingifitisusedasmultiplecontiguouschildrenofthesameparent.Atreemayappearmultipletimesinthislistifitistherightsiblingofthistreewithrespecttomultipleparents.:type:list(MultiParentedTree)""" return [parent[index+1] for (parent, index) in self._get_parent_indices() if index < (len(parent)-1)] def _get_parent_indices(self): return [(parent, index) for parent in self._parents for index, child in enumerate(parent) if child is self] def roots(self): """Thesetofallrootsofthistree.Thissetisformedbytracingallpossibleparentpathsuntiltreeswithnoparentsarefound.:type:list(MultiParentedTree)""" return list(self._get_roots_helper({}).values()) def _get_roots_helper(self, result): if self._parents: for parent in self._parents: parent._get_roots_helper(result) else: result[id(self)] = self return result def parent_indices(self, parent): """Returnalistoftheindiceswherethistreeoccursasachildof``parent``.Ifthischilddoesnotoccurasachildof``parent``,thentheemptylistisreturned.Thefollowingisalwaystrue::forparent_indexinptree.parent_indices(parent):parent[parent_index]isptree""" if parent not in self._parents: return [] else: return [index for (index, child) in enumerate(parent) if child is self] def treepositions(self, root): """Returnalistofalltreepositionsthatcanbeusedtoreachthismulti-parentedtreestartingfrom``root``.I.e.,thefollowingisalwaystrue::fortreeposinptree.treepositions(root):root[treepos]isptree""" if self is root: return [()] else: return [treepos+(index,) for parent in self._parents for treepos in parent.treepositions(root) for (index, child) in enumerate(parent) if child is self] #///////////////////////////////////////////////////////////////// # Parent Management #///////////////////////////////////////////////////////////////// def _delparent(self, child, index): # Sanity checks assert isinstance(child, MultiParentedTree) assert self[index] is child assert len([p for p in child._parents if p is self]) == 1 # If the only copy of child in self is at index, then delete # self from child's parent list. for i, c in enumerate(self): if c is child and i != index: break else: child._parents.remove(self) def _setparent(self, child, index, dry_run=False): # If the child's type is incorrect, then complain. if not isinstance(child, MultiParentedTree): raise TypeError('Can not insert a non-MultiParentedTree '+ 'into a MultiParentedTree') # Add self as a parent pointer if it's not already listed. if not dry_run: for parent in child._parents: if parent is self: break else: child._parents.append(self)class ImmutableParentedTree(ImmutableTree, ParentedTree): passclass ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree): pass######################################################################## Probabilistic trees######################################################################@python_2_unicode_compatibleclass ProbabilisticTree(Tree, ProbabilisticMixIn): def __init__(self, node_or_str, children=None, **prob_kwargs): Tree.__init__(self, node_or_str, children) ProbabilisticMixIn.__init__(self, **prob_kwargs) # We have to patch up these methods to make them work right: def _frozen_class(self): return ImmutableProbabilisticTree def __repr__(self): return '%s (p=%r)' % (Tree.unicode_repr(self), self.prob()) def __str__(self): return '%s (p=%.6g)' % (self.pprint(margin=60), self.prob()) def copy(self, deep=False): if not deep: return type(self)(self.node, self, prob=self.prob()) else: return type(self).convert(self) @classmethod def convert(cls, val): if isinstance(val, Tree): children = [cls.convert(child) for child in val] if isinstance(val, ProbabilisticMixIn): return cls(val.node, children, prob=val.prob()) else: return cls(val.node, children, prob=1.0) else: return val def __eq__(self, other): return (self.__class__ is other.__class__ and (self.node, list(self), self.prob()) == (other.node, list(other), other.prob())) def __lt__(self, other): if not isinstance(other, Tree): raise_unorderable_types("<", self, other) if self.__class__ is other.__class__: return ((self.node, list(self), self.prob()) < (other.node, list(other), other.prob())) else: return self.__class__.__name__ < other.__class__.__name__@python_2_unicode_compatibleclass ImmutableProbabilisticTree(ImmutableTree, ProbabilisticMixIn): def __init__(self, node_or_str, children=None, **prob_kwargs): ImmutableTree.__init__(self, node_or_str, children) ProbabilisticMixIn.__init__(self, **prob_kwargs) self._hash = hash((self.node, tuple(self), self.prob())) # We have to patch up these methods to make them work right: def _frozen_class(self): return ImmutableProbabilisticTree def __repr__(self): return '%s [%s]' % (Tree.unicode_repr(self), self.prob()) def __str__(self): return '%s [%s]' % (self.pprint(margin=60), self.prob()) def copy(self, deep=False): if not deep: return type(self)(self.node, self, prob=self.prob()) else: return type(self).convert(self) @classmethod def convert(cls, val): if isinstance(val, Tree): children = [cls.convert(child) for child in val] if isinstance(val, ProbabilisticMixIn): return cls(val.node, children, prob=val.prob()) else: return cls(val.node, children, prob=1.0) else: return valdef _child_names(tree): names = [] for child in tree: if isinstance(child, Tree): names.append(Nonterminal(child.node)) else: names.append(child) return names######################################################################## Parsing######################################################################def bracket_parse(s): """UseTree.parse(s,remove_empty_top_bracketing=True)instead.""" raise NameError("UseTree.parse(s,remove_empty_top_bracketing=True)instead.")def sinica_parse(s): """ParseaSinicaTreebankstringandreturnatree.Treesarerepresentedasnestedbrackettings,asshowninthefollowingexample(XrepresentsaChinesecharacter):S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY):return:Atreecorrespondingtothestringrepresentation.:rtype:Tree:params:Thestringtobeconverted:types:str""" tokens = re.split(r'([()| ])', s) for i in range(len(tokens)): if tokens[i] == '(': tokens[i-1], tokens[i] = tokens[i], tokens[i-1] # pull nonterminal inside parens elif ':' in tokens[i]: fields = tokens[i].split(':') if len(fields) == 2: # non-terminal tokens[i] = fields[1] else: tokens[i] = "(" + fields[-2] + "" + fields[-1] + ")" elif tokens[i] == '|': tokens[i] = '' treebank_string = "".join(tokens)returnTree.parse(treebank_string,remove_empty_top_bracketing=True)# s = re.sub(r'^#[^\s]*\s', '', s) # remove leading identifier# s = re.sub(r'\w+:', '', s) # remove role tags# return s######################################################################## Demonstration######################################################################defdemo():""" A demonstration showing how Trees and Trees can be used. This demonstration creates a Tree, and loads a Tree from the Treebank corpus, and shows the results of calling several of their methods. """fromnltkimporttree#Demonstratetreeparsing.s='(S (NP (DT the) (NN cat)) (VP (VBD ate) (NP (DT a) (NN cookie))))'t=Tree(s)print("Convert bracketed string into tree:")print(t)print(t.__repr__())print("Display tree properties:")print(t.node)#tree's constituent type print(t[0]) # tree'sfirstchildprint(t[1])#tree's second child print(t.height()) print(t.leaves()) print(t[1]) print(t[1,1]) print(t[1,1,0]) # Demonstrate tree modification. the_cat = t[0] the_cat.insert(1, tree.Tree.parse('(JJbig)')) print("Tree modification:") print(t) t[1,1,1] = tree.Tree.parse('(NNcake)') print(t) print() # Tree transforms print("Collapse unary:") t.collapse_unary() print(t) print("Chomsky normal form:") t.chomsky_normal_form() print(t) print() # Demonstrate probabilistic trees. pt = tree.ProbabilisticTree('x', ['y', 'z'], prob=0.5) print("Probabilistic Tree:") print(pt) print() # Demonstrate parsing of treebank output format. t = tree.Tree.parse(t.pprint()) print("Convert tree to bracketed string and back again:") print(t) print() # Demonstrate LaTeX output print("LaTeX output:") print(t.pprint_latex_qtree()) print() # Demonstrate Productions print("Production output:") print(t.productions()) print() # Demonstrate tree nodes containing objects other than strings t.node = ('test', 3) print(t)__all__ = ['ImmutableProbabilisticTree', 'ImmutableTree', 'ProbabilisticMixIn', 'ProbabilisticTree', 'Tree', 'bracket_parse', 'sinica_parse', 'ParentedTree', 'MultiParentedTree', 'ImmutableParentedTree', 'ImmutableMultiParentedTree']if__name__=="__main__":importdoctestdoctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)