108""" 109 Return C{True} if a module is to be introsepcted with the current 110 settings. 111 112 @param name: The name of the module to test 113 @type name: L{DottedName} or C{str} 114 """ 115returnself.introspect \ 116andnotself._matches_filter(name,self._introspect_regexp)

119""" 120 Return C{True} if a module is to be parsed with the current settings. 121 122 @param name: The name of the module to test 123 @type name: L{DottedName} or C{str} 124 """ 125returnself.parse \ 126andnotself._matches_filter(name,self._parse_regexp)

565""" 566 Construct and return the API documentation for the python 567 module with the given filename. 568 569 @param parent_docs: The C{ModuleDoc} of the containing package. 570 If C{parent_docs} is not provided, then this method will 571 check if the given filename is contained in a package; and 572 if so, it will construct a stub C{ModuleDoc} for the 573 containing package(s). C{parent_docs} is a tuple, where 574 the first element is the parent from introspection, and 575 the second element is the parent from parsing. 576 """ 577# Record our progress. 578modulename=os.path.splitext(os.path.split(filename)[1])[0] 579ifmodulename=='__init__': 580modulename=os.path.split(os.path.split(filename)[0])[1] 581ifparent_docs[0]: 582modulename=DottedName(parent_docs[0].canonical_name,modulename) 583elifparent_docs[1]: 584modulename=DottedName(parent_docs[1].canonical_name,modulename) 585ifoptions.must_introspect(modulename)oroptions.must_parse(modulename): 586log.progress(progress_estimator.progress(), 587'%s (%s)'%(modulename,filename)) 588progress_estimator.complete+=1 589 590# Normalize the filename. 591filename=os.path.normpath(os.path.abspath(filename)) 592 593# When possible, use the source version of the file. 594try: 595filename=py_src_filename(filename) 596src_file_available=True 597exceptValueError: 598src_file_available=False 599 600# Get the introspected & parsed docs (as appropriate) 601introspect_doc=parse_doc=None 602introspect_error=parse_error=None 603ifoptions.must_introspect(modulename): 604try: 605introspect_doc=introspect_docs( 606filename=filename,context=parent_docs[0]) 607ifintrospect_doc.canonical_nameisUNKNOWN: 608introspect_doc.canonical_name=modulename 609exceptImportError,e: 610introspect_error=str(e) 611ifsrc_file_availableandoptions.must_parse(modulename): 612try: 613parse_doc=parse_docs( 614filename=filename,context=parent_docs[1]) 615except(ParseError,ImportError,IOError,OSError),e: 616parse_error=str(e) 617 618# Report any errors we encountered. 619_report_errors(filename,introspect_doc,parse_doc, 620introspect_error,parse_error) 621 622# Return the docs we found. 623return(introspect_doc,parse_doc)

695""" 696 Used to keep track of progress when generating the initial docs 697 for the given items. (It is not known in advance how many items a 698 package directory will contain, since it might depend on those 699 packages' __path__ values.) 700 """

734 735###################################################################### 736## Doc Merger 737###################################################################### 738 739MERGE_PRECEDENCE={ 740'repr':'parse', 741 742# The names we get from introspection match the names that users 743# can actually use -- i.e., they take magic into account. 744'canonical_name':'introspect', 745 746# Only fall-back on the parser for is_imported if the introspecter 747# isn't sure. Otherwise, we can end up thinking that vars 748# containing modules are not imported, which can cause external 749# modules to show up in the docs (sf bug #1653486) 750'is_imported':'introspect', 751 752# The parser can tell if an assignment creates an alias or not. 753'is_alias':'parse', 754 755# The parser is better able to determine what text file something 756# came from; e.g., it can't be fooled by 'covert' imports. 757'docformat':'parse', 758 759# The parse should be able to tell definitively whether a module 760# is a package or not. 761'is_package':'parse', 762 763# Extract the sort spec from the order in which values are defined 764# in the source file. 765'sort_spec':'parse', 766 767'submodules':'introspect', 768 769# The filename used by 'parse' is the source file. 770'filename':'parse', 771 772# 'parse' is more likely to get the encoding right, but 773# 'introspect' will handle programatically generated docstrings. 774# Which is better? 775'docstring':'introspect', 776} 777"""Indicates whether information from introspection or parsing should be 778given precedence, for specific attributes. This dictionary maps from 779attribute names to either C{'introspect'} or C{'parse'}.""" 780 781DEFAULT_MERGE_PRECEDENCE='introspect' 782"""Indicates whether information from introspection or parsing should be 783given precedence. Should be either C{'introspect'} or C{'parse'}""" 784 785_attribute_mergefunc_registry={}

787""" 788 Register an attribute merge function. This function will be 789 called by L{merge_docs()} when it needs to merge the attribute 790 values of two C{APIDoc}s. 791 792 @param attrib: The name of the attribute whose values are merged 793 by C{mergefunc}. 794 795 @param mergefunc: The merge function, whose sinature is: 796 797 >>> def mergefunc(introspect_val, parse_val, precedence, cyclecheck, path): 798 ... return calculate_merged_value(introspect_val, parse_val) 799 800 Where C{introspect_val} and C{parse_val} are the two values to 801 combine; C{precedence} is a string indicating which value takes 802 precedence for this attribute (C{'introspect'} or C{'parse'}); 803 C{cyclecheck} is a value used by C{merge_docs()} to make sure that 804 it only visits each pair of docs once; and C{path} is a string 805 describing the path that was taken from the root to this 806 attribute (used to generate log messages). 807 808 If the merge function needs to call C{merge_docs}, then it should 809 pass C{cyclecheck} and C{path} back in. (When appropriate, a 810 suffix should be added to C{path} to describe the path taken to 811 the merged values.) 812 """ 813_attribute_mergefunc_registry[attrib]=mergefunc

816""" 817 Merge the API documentation information that was obtained from 818 introspection with information that was obtained from parsing. 819 C{introspect_doc} and C{parse_doc} should be two C{APIDoc} instances 820 that describe the same object. C{merge_docs} combines the 821 information from these two instances, and returns the merged 822 C{APIDoc}. 823 824 If C{introspect_doc} and C{parse_doc} are compatible, then they will 825 be I{merged} -- i.e., they will be coerced to a common class, and 826 their state will be stored in a shared dictionary. Once they have 827 been merged, any change made to the attributes of one will affect 828 the other. The value for the each of the merged C{APIDoc}'s 829 attributes is formed by combining the values of the source 830 C{APIDoc}s' attributes, as follows: 831 832 - If either of the source attributes' value is C{UNKNOWN}, then 833 use the other source attribute's value. 834 - Otherwise, if an attribute merge function has been registered 835 for the attribute, then use that function to calculate the 836 merged value from the two source attribute values. 837 - Otherwise, if L{MERGE_PRECEDENCE} is defined for the 838 attribute, then use the attribute value from the source that 839 it indicates. 840 - Otherwise, use the attribute value from the source indicated 841 by L{DEFAULT_MERGE_PRECEDENCE}. 842 843 If C{introspect_doc} and C{parse_doc} are I{not} compatible (e.g., if 844 their values have incompatible types), then C{merge_docs()} will 845 simply return either C{introspect_doc} or C{parse_doc}, depending on 846 the value of L{DEFAULT_MERGE_PRECEDENCE}. The two input 847 C{APIDoc}s will not be merged or modified in any way. 848 849 @param cyclecheck, path: These arguments should only be provided 850 when C{merge_docs()} is called by an attribute merge 851 function. See L{register_attribute_mergefunc()} for more 852 details. 853 """ 854assertisinstance(introspect_doc,APIDoc) 855assertisinstance(parse_doc,APIDoc) 856 857ifcyclecheckisNone: 858cyclecheck=set() 859ifintrospect_doc.canonical_namenotin(None,UNKNOWN): 860path='%s'%introspect_doc.canonical_name 861elifparse_doc.canonical_namenotin(None,UNKNOWN): 862path='%s'%parse_doc.canonical_name 863else: 864path='??' 865 866# If we've already examined this pair, then there's nothing 867# more to do. The reason that we check id's here is that we 868# want to avoid hashing the APIDoc objects for now, so we can 869# use APIDoc.merge_and_overwrite() later. 870if(id(introspect_doc),id(parse_doc))incyclecheck: 871returnintrospect_doc 872cyclecheck.add((id(introspect_doc),id(parse_doc))) 873 874# If these two are already merged, then we're done. (Two 875# APIDoc's compare equal iff they are identical or have been 876# merged.) 877ifintrospect_doc==parse_doc: 878returnintrospect_doc 879 880# If both values are GenericValueDoc, then we don't want to merge 881# them. E.g., we don't want to merge 2+2 with 4. So just copy 882# the parse_doc's parse_repr to introspect_doc, & return it. 883# (In particular, do *not* call merge_and_overwrite.) 884iftype(introspect_doc)==type(parse_doc)==GenericValueDoc: 885ifparse_doc.parse_reprisnotUNKNOWN: 886introspect_doc.parse_repr=parse_doc.parse_repr 887introspect_doc.docs_extracted_by='both' 888returnintrospect_doc 889 890# Perform several sanity checks here -- if we accidentally 891# merge values that shouldn't get merged, then bad things can 892# happen. 893mismatch=None 894if(introspect_doc.__class__!=parse_doc.__class__and 895not(issubclass(introspect_doc.__class__,parse_doc.__class__)or 896issubclass(parse_doc.__class__,introspect_doc.__class__))): 897mismatch=("value types don't match -- i=%r, p=%r."% 898(introspect_doc.__class__,parse_doc.__class__)) 899if(isinstance(introspect_doc,ValueDoc)and 900isinstance(parse_doc,ValueDoc)): 901if(introspect_doc.pyvalisnotUNKNOWNand 902parse_doc.pyvalisnotUNKNOWNand 903introspect_doc.pyvalisnotparse_doc.pyval): 904mismatch="values don't match." 905elif(introspect_doc.canonical_namenotin(None,UNKNOWN)and 906parse_doc.canonical_namenotin(None,UNKNOWN)and 907introspect_doc.canonical_name!=parse_doc.canonical_name): 908mismatch="canonical names don't match." 909ifmismatchisnotNone: 910log.info("Not merging the parsed & introspected values of %s, " 911"since their %s"%(path,mismatch)) 912ifDEFAULT_MERGE_PRECEDENCE=='introspect': 913returnintrospect_doc 914else: 915returnparse_doc 916 917# If one apidoc's class is a superclass of the other's, then 918# specialize it to the more specific class. 919ifintrospect_doc.__class__isnotparse_doc.__class__: 920ifissubclass(introspect_doc.__class__,parse_doc.__class__): 921parse_doc.specialize_to(introspect_doc.__class__) 922ifissubclass(parse_doc.__class__,introspect_doc.__class__): 923introspect_doc.specialize_to(parse_doc.__class__) 924assertintrospect_doc.__class__isparse_doc.__class__ 925 926# The posargs and defaults are tied together -- if we merge 927# the posargs one way, then we need to merge the defaults the 928# same way. So check them first. (This is a minor hack) 929if(isinstance(introspect_doc,RoutineDoc)and 930isinstance(parse_doc,RoutineDoc)): 931_merge_posargs_and_defaults(introspect_doc,parse_doc,path) 932 933# Merge the two api_doc's attributes. 934forattribinset(introspect_doc.__dict__.keys()+ 935parse_doc.__dict__.keys()): 936# Be sure not to merge any private attributes (especially 937# __mergeset or __has_been_hashed!) 938ifattrib.startswith('_'):continue 939merge_attribute(attrib,introspect_doc,parse_doc, 940cyclecheck,path) 941 942# Set the dictionaries to be shared. 943returnintrospect_doc.merge_and_overwrite(parse_doc)

999# Merge all variables that are in both sets.1000forvarname,var1invarlist1.items():1001var2=varlist2.get(varname)1002ifvar2isnotNone:1003var=merge_docs(var1,var2,cyclecheck,path+'.'+varname)1004varlist1[varname]=var1005varlist2[varname]=var10061007# Copy any variables that are not in varlist1 over.1008forvarname,varinvarlist2.items():1009varlist1.setdefault(varname,var)10101011returnvarlist1

1032# Be careful here -- if we get it wrong, then we could end up1033# merging two unrelated classes, which could lead to bad1034# things (e.g., a class that's its own subclass). So only1035# merge two bases if we're quite sure they're the same class.1036# (In particular, if they have the same canonical name.)10371038# If the lengths don't match up, then give up. This is most1039# often caused by __metaclass__.1040iflen(baselist1)!=len(baselist2):1041log.info("Not merging the introspected & parsed base lists "1042"for %s, since their lengths don't match (%s vs %s)"%1043(path,len(baselist1),len(baselist2)))1044ifprecedence=='introspect':returnbaselist11045else:returnbaselist210461047# If any names disagree, then give up.1048forbase1,base2inzip(baselist1,baselist2):1049if((base1.canonical_namenotin(None,UNKNOWN)and1050base2.canonical_namenotin(None,UNKNOWN))and1051base1.canonical_name!=base2.canonical_name):1052log.info("Not merging the parsed & introspected base "1053"lists for %s, since the bases' names don't match "1054"(%s vs %s)"%(path,base1.canonical_name,1055base2.canonical_name))1056ifprecedence=='introspect':returnbaselist11057else:returnbaselist210581059fori,(base1,base2)inenumerate(zip(baselist1,baselist2)):1060base=merge_docs(base1,base2,cyclecheck,1061'%s.__bases__[%d]'%(path,i))1062baselist1[i]=baselist2[i]=base10631064returnbaselist1

1120# Check if the ValueDoc has an unresolved proxy_for link.1121# If so, then resolve it.1122whileval_doc.proxy_fornotin(UNKNOWN,None):1123# Find the valuedoc that the proxy_for name points to.1124src_doc=docindex.get_valdoc(val_doc.proxy_for)11251126# If we don't have any valuedoc at that address, then1127# set that address as its canonical name.1128# [XXX] Do I really want to do this?1129ifsrc_docisNone:1130val_doc.canonical_name=val_doc.proxy_for1131return11321133# If we *do* have something at that address, then1134# merge the proxy `val_doc` with it.1135elifsrc_doc!=val_doc:1136# Copy any subclass information from val_doc->src_doc.1137if(isinstance(val_doc,ClassDoc)and1138isinstance(src_doc,ClassDoc)):1139forsubclassinval_doc.subclasses:1140ifsubclassnotinsrc_doc.subclasses:1141src_doc.subclasses.append(subclass)1142# Then overwrite val_doc with the contents of src_doc.1143src_doc.merge_and_overwrite(val_doc,ignore_hash_conflict=True)11441145# If the proxy_for link points back at src_doc1146# itself, then we most likely have a variable that's1147# shadowing a submodule that it should be equal to.1148# So just get rid of the variable.1149elifsrc_doc==val_doc:1150parent_name=val_doc.proxy_for[:-1]1151var_name=val_doc.proxy_for[-1]1152parent=docindex.get_valdoc(parent_name)1153ifparentisnotNoneandvar_nameinparent.variables:1154delparent.variables[var_name]1155src_doc.proxy_for=None

11561157######################################################################1158## Canonical Name Assignment1159######################################################################11601161_name_scores={}1162"""A dictionary mapping from each C{ValueDoc} to the score that has1163been assigned to its current cannonical name. If1164L{assign_canonical_names()} finds a canonical name with a better1165score, then it will replace the old name."""11661167_unreachable_names={DottedName(DottedName.UNREACHABLE):1}1168"""The set of names that have been used for unreachable objects. This1169is used to ensure there are no duplicate cannonical names assigned.1170C{_unreachable_names} is a dictionary mapping from dotted names to1171integer ids, where the next unused unreachable name derived from1172dotted name C{n} is1173C{DottedName('%s-%s' % (n, str(_unreachable_names[n]+1))}"""1174

1176"""1177 Assign a canonical name to C{val_doc} (if it doesn't have one1178 already), and (recursively) to each variable in C{val_doc}.1179 In particular, C{val_doc} will be assigned the canonical name1180 C{name} iff either:1181 - C{val_doc}'s canonical name is C{UNKNOWN}; or1182 - C{val_doc}'s current canonical name was assigned by this1183 method; but the score of the new name (C{score}) is higher1184 than the score of the current name (C{score_dict[val_doc]}).11851186 Note that canonical names will even be assigned to values1187 like integers and C{None}; but these should be harmless.1188 """1189# If we've already visited this node, and our new score1190# doesn't beat our old score, then there's nothing more to do.1191# Note that since score increases strictly monotonically, this1192# also prevents us from going in cycles.1193ifval_docin_name_scoresandscore<=_name_scores[val_doc]:1194return11951196# Update val_doc's canonical name, if appropriate.1197if(val_docnotin_name_scoresand1198val_doc.canonical_nameisnotUNKNOWN):1199# If this is the first time we've seen val_doc, and it1200# already has a name, then don't change that name.1201_name_scores[val_doc]=sys.maxint1202name=val_doc.canonical_name1203score=01204else:1205# Otherwise, update the name iff the new score is better1206# than the old one.1207if(val_docnotin_name_scoresor1208score>_name_scores[val_doc]):1209val_doc.canonical_name=name1210_name_scores[val_doc]=score12111212# Recurse to any contained values.1213ifisinstance(val_doc,NamespaceDoc):1214forvar_docinval_doc.variables.values():1215# Set the variable's canonical name.1216varname=DottedName(name,var_doc.name)1217var_doc.canonical_name=varname12181219# If the value is unknown, or is a generic value doc, then1220# the valuedoc doesn't get assigned a name; move on.1221if(var_doc.valueisUNKNOWN1222orisinstance(var_doc.value,GenericValueDoc)):1223continue12241225# [XX] After svn commit 1644-1647, I'm not sure if this1226# ever gets used: This check is for cases like1227# curses.wrapper, where an imported variable shadows its1228# value's "real" location.1229if_var_shadows_self(var_doc,varname):1230_fix_self_shadowing_var(var_doc,varname,docindex)12311232# Find the score for this new name. 1233vardoc_score=score-11234ifvar_doc.is_importedisUNKNOWN:vardoc_score-=101235elifvar_doc.is_imported:vardoc_score-=1001236ifvar_doc.is_aliasisUNKNOWN:vardoc_score-=101237elifvar_doc.is_alias:vardoc_score-=100012381239assign_canonical_names(var_doc.value,varname,1240docindex,vardoc_score)12411242# Recurse to any directly reachable values.1243forval_doc_2inval_doc.apidoc_links(variables=False):1244val_name,val_score=_unreachable_name_for(val_doc_2,docindex)1245assign_canonical_names(val_doc_2,val_name,docindex,val_score)

1254# If possible, find another name for the shadowed value.1255cname=var_doc.value.canonical_name1256foriinrange(1,len(cname)-1):1257new_name=cname[:i]+(cname[i]+"'")+cname[i+1:]1258val_doc=docindex.get_valdoc(new_name)1259ifval_docisnotNone:1260log.warning("%s shadows its own value -- using %s instead"%1261(varname,new_name))1262var_doc.value=val_doc1263return12641265# If we couldn't find the actual value, use an unreachable name.1266name,score=_unreachable_name_for(var_doc.value,docindex)1267log.warning('%s shadows itself -- using %s instead'%(varname,name))1268var_doc.value.canonical_name=name

1310"""1311 Set the C{overrides} attribute for all variables in C{class_doc}.1312 This needs to be done early (before docstring parsing), so we can1313 know which docstrings to suppress warnings for.1314 """1315forbase_classinlist(class_doc.mro(warn_about_bad_bases=True)):1316ifbase_class==class_doc:continue1317ifbase_class.variablesisUNKNOWN:continue1318forname,var_docinbase_class.variables.items():1319if(not(name.startswith('__')andnotname.endswith('__'))and1320base_class==var_doc.containerand1321nameinclass_doc.variablesand1322class_doc.variables[name].container==class_docand1323class_doc.variables[name].overridesisUNKNOWN):1324class_doc.variables[name].overrides=var_doc

1328forbase_classinlist(class_doc.mro(warn_about_bad_bases=True)):1329ifbase_class==class_doc:continue1330ifbase_class.pyvalisobjectandnotinherit_from_object:continue13311332# Inherit any groups. Place them *after* this class's groups,1333# so that any groups that are important to this class come1334# first.1335ifbase_class.group_specsnotin(None,UNKNOWN):1336class_doc.group_specs+=[gsforgsinbase_class.group_specs1337ifgsnotinclass_doc.group_specs]13381339# Inherit any variables.1340ifbase_class.variablesisUNKNOWN:continue1341forname,var_docinbase_class.variables.items():1342# If it's a __private variable, then don't inherit it.1343ifname.startswith('__')andnotname.endswith('__'):1344continue13451346# Inhetit only from the defining class. Or else, in case of1347# multiple inheritance, we may import from a grand-ancestor1348# variables overridden by a class that follows in mro.1349ifbase_class!=var_doc.container:1350continue13511352# If class_doc doesn't have a variable with this name,1353# then inherit it.1354ifnamenotinclass_doc.variables:1355class_doc.variables[name]=var_doc13561357# Otherwise, class_doc already contains a variable1358# that shadows var_doc. But if class_doc's var is1359# local, then record the fact that it overrides1360# var_doc.1361elifclass_doc.variables[name].container==class_doc:1362class_doc.variables[name].overrides=var_doc1363_inherit_info(class_doc.variables[name])

1373"""1374 Copy any relevant documentation information from the variable that1375 C{var_doc} overrides into C{var_doc} itself.1376 """1377src_var=var_doc.overrides1378src_val=var_doc.overrides.value1379val_doc=var_doc.value13801381# Special case: if the source value and target values are both c1382# extension methods, and the target value's signature is not1383# specified, then inherit the source value's signature.1384if(isinstance(val_doc,RoutineDoc)and1385isinstance(src_val,RoutineDoc)and1386(inspect.isbuiltin(val_doc.pyval)or1387isinstance(val_doc.pyval,_method_descriptor))and1388(inspect.isbuiltin(src_val.pyval)or1389isinstance(src_val.pyval,_method_descriptor))and1390val_doc.all_args()in(['...'],UNKNOWN)and1391src_val.all_args()notin(['...'],UNKNOWN)):1392forattribin['posargs','posarg_defaults','vararg',1393'kwarg','return_type']:1394setattr(val_doc,attrib,getattr(src_val,attrib))13951396# If the new variable has a docstring, then don't inherit1397# anything, even if the docstring is blank.1398ifvar_doc.docstringnotin(None,UNKNOWN):1399return1400# [xx] Do I want a check like this:?1401# # If it's a method and the signature doesn't match well enough,1402# # then give up.1403# if (isinstance(src_val, RoutineDoc) and1404# isinstance(val_doc, RoutineDoc)):1405# if (src_val.posargs != val_doc.posargs[:len(src_val.posargs)] or1406# src_val.vararg != None and src_val.vararg != val_doc.vararg):1407# log.docstring_warning(1408# "The signature of %s does not match the signature of the "1409# "method it overrides (%s); not inheriting documentation." %1410# (var_doc.canonical_name, src_var.canonical_name))1411# return14121413# Inherit attributes!1414forattribin_INHERITED_ATTRIBS:1415if(hasattr(var_doc,attrib)andhasattr(src_var,attrib)and1416getattr(src_var,attrib)notin(None,UNKNOWN)):1417setattr(var_doc,attrib,getattr(src_var,attrib))1418elif(src_valisnotNoneand1419hasattr(val_doc,attrib)andhasattr(src_val,attrib)and1420getattr(src_val,attrib)notin(None,UNKNOWN)and1421getattr(val_doc,attrib)in(None,UNKNOWN,[])):1422setattr(val_doc,attrib,getattr(src_val,attrib))