epydoc.docwriter.html

219 """ 220 A class that renders a python module's source code into HTML 221 pages. These HTML pages are intended to be provided along with 222 the API documentation for a module, in case a user wants to learn 223 more about a particular object by examining its source code. 224 Links are therefore generated from the API documentation to the 225 source code pages, and from the source code pages back into the 226 API documentation. 227 228 The HTML generated by C{PythonSourceColorizer} has several notable 229 features: 230 231 - CSS styles are used to color tokens according to their type. 232 (See L{CSS_CLASSES} for a list of the different token types 233 that are identified). 234 235 - Line numbers are included to the left of each line. 236 237 - The first line of each class and function definition includes 238 a link to the API source documentation for that object. 239 240 - The first line of each class and function definition includes 241 an anchor that can be used to link directly to that class or 242 function. 243 244 - If javascript is enabled, and the page is loaded using the 245 anchor for a class or function (i.e., if the url ends in 246 C{'#I{<name>}'}), then that class or function will automatically 247 be highlighted; and all other classes and function definition 248 blocks will be 'collapsed'. These collapsed blocks can be 249 expanded by clicking on them. 250 251 - Unicode input is supported (including automatic detection 252 of C{'coding:'} declarations). 253 254 """ 255 #: A look-up table that is used to determine which CSS class 256 #: should be used to colorize a given token. The following keys 257 #: may be used: 258 #: - Any token name (e.g., C{'STRING'}) 259 #: - Any operator token (e.g., C{'='} or C{'@'}). 260 #: - C{'KEYWORD'} -- Python keywords such as C{'for'} and C{'if'} 261 #: - C{'DEFNAME'} -- the name of a class or function at the top 262 #: of its definition statement. 263 #: - C{'BASECLASS'} -- names of base classes at the top of a class 264 #: definition statement. 265 #: - C{'PARAM'} -- function parameters 266 #: - C{'DOCSTRING'} -- docstrings 267 #: - C{'DECORATOR'} -- decorator names 268 #: If no CSS class can be found for a given token, then it won't 269 #: be marked with any CSS class. 270 CSS_CLASSES = { 271 'NUMBER': 'py-number', 272 'STRING': 'py-string', 273 'COMMENT': 'py-comment', 274 'NAME': 'py-name', 275 'KEYWORD': 'py-keyword', 276 'DEFNAME': 'py-def-name', 277 'BASECLASS': 'py-base-class', 278 'PARAM': 'py-param', 279 'DOCSTRING': 'py-docstring', 280 'DECORATOR': 'py-decorator', 281 'OP': 'py-op', 282 '@': 'py-decorator', 283 } 284 285 #: HTML code for the beginning of a collapsable function or class 286 #: definition block. The block contains two <div>...</div> 287 #: elements -- a collapsed version and an expanded version -- and 288 #: only one of these elements is visible at any given time. By 289 #: default, all definition blocks are expanded. 290 #: 291 #: This string should be interpolated with the following values:: 292 #: (name, indentation, name) 293 #: Where C{name} is the anchor name for the function or class; and 294 #: indentation is a string of whitespace used to indent the 295 #: ellipsis marker in the collapsed version. 296 START_DEF_BLOCK = ( 297 '<div id="%s-collapsed" style="display:none;" ' 298 'pad="%s" indent="%s"></div>' 299 '<div id="%s-expanded">') 300 301 #: HTML code for the end of a collapsable function or class 302 #: definition block. 303 END_DEF_BLOCK = '</div>' 304 305 #: A regular expression used to pick out the unicode encoding for 306 #: the source file. 307 UNICODE_CODING_RE = re.compile(r'.*?\n?.*?coding[:=]\s*([-\w.]+)') 308 309 #: A configuration constant, used to determine whether or not to add 310 #: collapsable <div> elements for definition blocks. 311 ADD_DEF_BLOCKS = True 312 313 #: A configuration constant, used to determine whether or not to 314 #: add line numbers. 315 ADD_LINE_NUMBERS = True 316 317 #: A configuration constant, used to determine whether or not to 318 #: add tooltips for linked names. 319 ADD_TOOLTIPS = True 320 321 #: If true, then try to guess which target is appropriate for 322 #: linked names; if false, then always open a div asking the 323 #: user which one they want. 324 GUESS_LINK_TARGETS = False 325

326 - def __init__(self, module_filename, module_name, 327 docindex=None, url_func=None, name_to_docs=None, 328 tab_width=8):

329 """ 330 Create a new HTML colorizer for the specified module. 331 332 @param module_filename: The name of the file containing the 333 module; its text will be loaded from this file. 334 @param module_name: The dotted name of the module; this will 335 be used to create links back into the API source 336 documentation. 337 """ 338 # Get the source version, if possible. 339 try: module_filename = py_src_filename(module_filename) 340 except: pass 341 342 #: The filename of the module we're colorizing. 343 self.module_filename = module_filename 344 345 #: The dotted name of the module we're colorizing. 346 self.module_name = module_name 347 348 #: A docindex, used to create href links from identifiers to 349 #: the API documentation for their values. 350 self.docindex = docindex 351 352 #: A mapping from short names to lists of ValueDoc, used to 353 #: decide which values an identifier might map to when creating 354 #: href links from identifiers to the API docs for their values. 355 self.name_to_docs = name_to_docs 356 357 #: A function that maps APIDoc -> URL, used to create href 358 #: links from identifiers to the API documentation for their 359 #: values. 360 self.url_func = url_func 361 362 #: The index in C{text} of the last character of the last 363 #: token we've processed. 364 self.pos = 0 365 366 #: A list that maps line numbers to character offsets in 367 #: C{text}. In particular, line C{M{i}} begins at character 368 #: C{line_offset[i]} in C{text}. Since line numbers begin at 369 #: 1, the first element of C{line_offsets} is C{None}. 370 self.line_offsets = [] 371 372 #: A list of C{(toktype, toktext)} for all tokens on the 373 #: logical line that we are currently processing. Once a 374 #: complete line of tokens has been collected in C{cur_line}, 375 #: it is sent to L{handle_line} for processing. 376 self.cur_line = [] 377 378 #: A list of the names of the class or functions that include 379 #: the current block. C{context} has one element for each 380 #: level of indentation; C{context[i]} is the name of the class 381 #: or function defined by the C{i}th level of indentation, or 382 #: C{None} if that level of indentation doesn't correspond to a 383 #: class or function definition. 384 self.context = [] 385 386 #: A list, corresponding one-to-one with L{self.context}, 387 #: indicating the type of each entry. Each element of 388 #: C{context_types} is one of: C{'func'}, C{'class'}, C{None}. 389 self.context_types = [] 390 391 #: A list of indentation strings for each of the current 392 #: block's indents. I.e., the current total indentation can 393 #: be found by taking C{''.join(self.indents)}. 394 self.indents = [] 395 396 #: The line number of the line we're currently processing. 397 self.lineno = 0 398 399 #: The name of the class or function whose definition started 400 #: on the previous logical line, or C{None} if the previous 401 #: logical line was not a class or function definition. 402 self.def_name = None 403 404 #: The type of the class or function whose definition started 405 #: on the previous logical line, or C{None} if the previous 406 #: logical line was not a class or function definition. 407 #: Can be C{'func'}, C{'class'}, C{None}. 408 self.def_type = None 409 410 #: The number of spaces to replace each tab in source code with 411 self.tab_width = tab_width

412 413

414 - def find_line_offsets(self):

415 """ 416 Construct the L{line_offsets} table from C{self.text}. 417 """ 418 # line 0 doesn't exist; line 1 starts at char offset 0. 419 self.line_offsets = [None, 0] 420 # Find all newlines in `text`, and add an entry to 421 # line_offsets for each one. 422 pos = self.text.find('\n') 423 while pos != -1: 424 self.line_offsets.append(pos+1) 425 pos = self.text.find('\n', pos+1) 426 # Add a final entry, marking the end of the string. 427 self.line_offsets.append(len(self.text))

428

429 - def lineno_to_html(self):

430 template = '%%%ds' % self.linenum_size 431 n = template % self.lineno 432 return '<a name="L%s"></a><tt class="py-lineno">%s</tt>' \ 433 % (self.lineno, n)

434

435 - def colorize(self):

436 """ 437 Return an HTML string that renders the source code for the 438 module that was specified in the constructor. 439 """ 440 # Initialize all our state variables 441 self.pos = 0 442 self.cur_line = [] 443 self.context = [] 444 self.context_types = [] 445 self.indents = [] 446 self.lineno = 1 447 self.def_name = None 448 self.def_type = None 449 self.has_decorators = False 450 451 # Cache, used so we only need to list the target elements once 452 # for each variable. 453 self.doclink_targets_cache = {} 454 455 # Load the module's text. 456 self.text = open(self.module_filename).read() 457 self.text = self.text.expandtabs(self.tab_width).rstrip()+'\n' 458 459 # Construct the line_offsets table. 460 self.find_line_offsets() 461 462 num_lines = self.text.count('\n')+1 463 self.linenum_size = len(`num_lines+1`) 464 465 # Call the tokenizer, and send tokens to our `tokeneater()` 466 # method. If anything goes wrong, then fall-back to using 467 # the input text as-is (with no colorization). 468 try: 469 output = StringIO() 470 self.out = output.write 471 tokenize.tokenize(StringIO(self.text).readline, self.tokeneater) 472 html = output.getvalue() 473 if self.has_decorators: 474 html = self._FIX_DECORATOR_RE.sub(r'\2\1', html) 475 except tokenize.TokenError, ex: 476 html = self.text 477 478 # Check for a unicode encoding declaration. 479 m = self.UNICODE_CODING_RE.match(self.text) 480 if m: coding = m.group(1) 481 else: coding = 'iso-8859-1' 482 483 # Decode the html string into unicode, and then encode it back 484 # into ascii, replacing any non-ascii characters with xml 485 # character references. 486 try: 487 html = html.decode(coding).encode('ascii', 'xmlcharrefreplace') 488 except LookupError: 489 coding = 'iso-8859-1' 490 html = html.decode(coding).encode('ascii', 'xmlcharrefreplace') 491 492 # Call expandto. 493 html += PYSRC_EXPANDTO_JAVASCRIPT 494 495 return html

496

497 - def tokeneater(self, toktype, toktext, (srow,scol), (erow,ecol), line):

498 """ 499 A callback function used by C{tokenize.tokenize} to handle 500 each token in the module. C{tokeneater} collects tokens into 501 the C{self.cur_line} list until a complete logical line has 502 been formed; and then calls L{handle_line} to process that line. 503 """ 504 # If we encounter any errors, then just give up. 505 if toktype == token.ERRORTOKEN: 506 raise tokenize.TokenError, toktype 507 508 # Did we skip anything whitespace? If so, add a pseudotoken 509 # for it, with toktype=None. (Note -- this skipped string 510 # might also contain continuation slashes; but I won't bother 511 # to colorize them.) 512 startpos = self.line_offsets[srow] + scol 513 if startpos > self.pos: 514 skipped = self.text[self.pos:startpos] 515 self.cur_line.append( (None, skipped) ) 516 517 # Update our position. 518 self.pos = startpos + len(toktext) 519 520 # Update our current line. 521 self.cur_line.append( (toktype, toktext) ) 522 523 # When we reach the end of a line, process it. 524 if toktype == token.NEWLINE or toktype == token.ENDMARKER: 525 self.handle_line(self.cur_line) 526 self.cur_line = []

527 528 _next_uid = 0 529 530 # [xx] note -- this works with byte strings, not unicode strings! 531 # I may change it to use unicode eventually, but when I do it 532 # needs to be changed all at once.

533 - def handle_line(self, line):

534 """ 535 Render a single logical line from the module, and write the 536 generated HTML to C{self.out}. 537 538 @param line: A single logical line, encoded as a list of 539 C{(toktype,tokttext)} pairs corresponding to the tokens in 540 the line. 541 """ 542 # def_name is the name of the function or class defined by 543 # this line; or None if no funciton or class is defined. 544 def_name = None 545 546 # def_type is the type of the function or class defined by 547 # this line; or None if no funciton or class is defined. 548 def_type = None 549 550 # does this line start a class/func def? 551 starting_def_block = False 552 553 in_base_list = False 554 in_param_list = False 555 in_param_default = 0 556 at_module_top = (self.lineno == 1) 557 558 ended_def_blocks = 0 559 560 # The html output. 561 if self.ADD_LINE_NUMBERS: 562 s = self.lineno_to_html() 563 self.lineno += 1 564 else: 565 s = '' 566 s += ' <tt class="py-line">' 567 568 # Loop through each token, and colorize it appropriately. 569 for i, (toktype, toktext) in enumerate(line): 570 if type(s) is not str: 571 if type(s) is unicode: 572 log.error('While colorizing %s -- got unexpected ' 573 'unicode string' % self.module_name) 574 s = s.encode('ascii', 'xmlcharrefreplace') 575 else: 576 raise ValueError('Unexpected value for s -- %s' % 577 type(s).__name__) 578 579 # For each token, determine its css class and whether it 580 # should link to a url. 581 css_class = None 582 url = None 583 tooltip = None 584 onclick = uid = targets = None # these 3 are used together. 585 586 # Is this token the class name in a class definition? If 587 # so, then make it a link back into the API docs. 588 if i>=2 and line[i-2][1] == 'class': 589 in_base_list = True 590 css_class = self.CSS_CLASSES['DEFNAME'] 591 def_name = toktext 592 def_type = 'class' 593 if 'func' not in self.context_types: 594 cls_name = self.context_name(def_name) 595 url = self.name2url(cls_name) 596 s = self.mark_def(s, cls_name) 597 starting_def_block = True 598 599 # Is this token the function name in a function def? If 600 # so, then make it a link back into the API docs. 601 elif i>=2 and line[i-2][1] == 'def': 602 in_param_list = True 603 css_class = self.CSS_CLASSES['DEFNAME'] 604 def_name = toktext 605 def_type = 'func' 606 if 'func' not in self.context_types: 607 cls_name = self.context_name() 608 func_name = self.context_name(def_name) 609 url = self.name2url(cls_name, def_name) 610 s = self.mark_def(s, func_name) 611 starting_def_block = True 612 613 # For each indent, update the indents list (which we use 614 # to keep track of indentation strings) and the context 615 # list. If this indent is the start of a class or 616 # function def block, then self.def_name will be its name; 617 # otherwise, it will be None. 618 elif toktype == token.INDENT: 619 self.indents.append(toktext) 620 self.context.append(self.def_name) 621 self.context_types.append(self.def_type) 622 623 # When we dedent, pop the last elements off the indents 624 # list and the context list. If the last context element 625 # is a name, then we're ending a class or function def 626 # block; so write an end-div tag. 627 elif toktype == token.DEDENT: 628 self.indents.pop() 629 self.context_types.pop() 630 if self.context.pop(): 631 ended_def_blocks += 1 632 633 # If this token contains whitespace, then don't bother to 634 # give it a css tag. 635 elif toktype in (None, tokenize.NL, token.NEWLINE, 636 token.ENDMARKER): 637 css_class = None 638 639 # Check if the token is a keyword. 640 elif toktype == token.NAME and keyword.iskeyword(toktext): 641 css_class = self.CSS_CLASSES['KEYWORD'] 642 643 elif in_base_list and toktype == token.NAME: 644 css_class = self.CSS_CLASSES['BASECLASS'] 645 646 elif (in_param_list and toktype == token.NAME and 647 not in_param_default): 648 css_class = self.CSS_CLASSES['PARAM'] 649 650 # Class/function docstring. 651 elif (self.def_name and line[i-1][0] == token.INDENT and 652 self.is_docstring(line, i)): 653 css_class = self.CSS_CLASSES['DOCSTRING'] 654 655 # Module docstring. 656 elif at_module_top and self.is_docstring(line, i): 657 css_class = self.CSS_CLASSES['DOCSTRING'] 658 659 # check for decorators?? 660 elif (toktype == token.NAME and 661 ((i>0 and line[i-1][1]=='@') or 662 (i>1 and line[i-1][0]==None and line[i-2][1] == '@'))): 663 css_class = self.CSS_CLASSES['DECORATOR'] 664 self.has_decorators = True 665 666 # If it's a name, try to link it. 667 elif toktype == token.NAME: 668 css_class = self.CSS_CLASSES['NAME'] 669 # If we have a variable named `toktext` in the current 670 # context, then link to that. Note that if we're inside 671 # a function, then that function is our context, not 672 # the namespace that contains it. [xx] this isn't always 673 # the right thing to do. 674 if (self.GUESS_LINK_TARGETS and self.docindex is not None 675 and self.url_func is not None): 676 context = [n for n in self.context if n is not None] 677 container = self.docindex.get_vardoc( 678 DottedName(self.module_name, *context)) 679 if isinstance(container, NamespaceDoc): 680 doc = container.variables.get(toktext) 681 if doc is not None: 682 url = self.url_func(doc) 683 tooltip = str(doc.canonical_name) 684 # Otherwise, check the name_to_docs index to see what 685 # else this name might refer to. 686 if (url is None and self.name_to_docs is not None 687 and self.url_func is not None): 688 docs = self.name_to_docs.get(toktext) 689 if docs: 690 tooltip='\n'.join([str(d.canonical_name) 691 for d in docs]) 692 if len(docs) == 1 and self.GUESS_LINK_TARGETS: 693 url = self.url_func(docs[0]) 694 else: 695 uid, onclick, targets = self.doclink(toktext, docs) 696 697 # For all other tokens, look up the CSS class to use 698 # based on the token's type. 699 else: 700 if toktype == token.OP and toktext in self.CSS_CLASSES: 701 css_class = self.CSS_CLASSES[toktext] 702 elif token.tok_name[toktype] in self.CSS_CLASSES: 703 css_class = self.CSS_CLASSES[token.tok_name[toktype]] 704 else: 705 css_class = None 706 707 # update our status.. 708 if toktext == ':': 709 in_base_list = False 710 in_param_list = False 711 if toktext == '=' and in_param_list: 712 in_param_default = True 713 if in_param_default: 714 if toktext in ('(','[','{'): in_param_default += 1 715 if toktext in (')',']','}'): in_param_default -= 1 716 if toktext == ',' and in_param_default == 1: 717 in_param_default = 0 718 719 # Write this token, with appropriate colorization. 720 if tooltip and self.ADD_TOOLTIPS: 721 tooltip_html = ' title="%s"' % tooltip 722 else: tooltip_html = '' 723 if css_class: css_class_html = ' class="%s"' % css_class 724 else: css_class_html = '' 725 if onclick: 726 if targets: targets_html = ' targets="%s"' % targets 727 else: targets_html = '' 728 s += ('<tt id="%s"%s%s><a%s%s href="#" onclick="%s">' % 729 (uid, css_class_html, targets_html, tooltip_html, 730 css_class_html, onclick)) 731 elif url: 732 if isinstance(url, unicode): 733 url = url.encode('ascii', 'xmlcharrefreplace') 734 s += ('<a%s%s href="%s">' % 735 (tooltip_html, css_class_html, url)) 736 elif css_class_html or tooltip_html: 737 s += '<tt%s%s>' % (tooltip_html, css_class_html) 738 if i == len(line)-1: 739 s += ' </tt>' # Closes <tt class="py-line"> 740 s += cgi.escape(toktext) 741 else: 742 try: 743 s += self.add_line_numbers(cgi.escape(toktext), css_class) 744 except Exception, e: 745 print (toktext, css_class, toktext.encode('ascii')) 746 raise 747 748 if onclick: s += "</a></tt>" 749 elif url: s += '</a>' 750 elif css_class_html or tooltip_html: s += '</tt>' 751 752 if self.ADD_DEF_BLOCKS: 753 for i in range(ended_def_blocks): 754 self.out(self.END_DEF_BLOCK) 755 756 # Strip any empty <tt>s. 757 s = re.sub(r'<tt class="[\w+]"></tt>', '', s) 758 759 # Write the line. 760 self.out(s) 761 762 if def_name and starting_def_block: 763 self.out('</div>') 764 765 # Add div's if we're starting a def block. 766 if (self.ADD_DEF_BLOCKS and def_name and starting_def_block and 767 (line[-2][1] == ':')): 768 indentation = (''.join(self.indents)+' ').replace(' ', '+') 769 linenum_padding = '+'*self.linenum_size 770 name=self.context_name(def_name) 771 self.out(self.START_DEF_BLOCK % (name, linenum_padding, 772 indentation, name)) 773 774 self.def_name = def_name 775 self.def_type = def_type

776

777 - def context_name(self, extra=None):

778 pieces = [n for n in self.context if n is not None] 779 if extra is not None: pieces.append(extra) 780 return '.'.join(pieces)

781

782 - def doclink(self, name, docs):

783 uid = 'link-%s' % self._next_uid 784 self._next_uid += 1 785 context = [n for n in self.context if n is not None] 786 container = DottedName(self.module_name, *context) 787 #else: 788 # container = None 789 targets = ','.join(['%s=%s' % (str(self.doc_descr(d,container)), 790 str(self.url_func(d))) 791 for d in docs]) 792 793 if targets in self.doclink_targets_cache: 794 onclick = ("return doclink('%s', '%s', '%s');" % 795 (uid, name, self.doclink_targets_cache[targets])) 796 return uid, onclick, None 797 else: 798 self.doclink_targets_cache[targets] = uid 799 onclick = ("return doclink('%s', '%s', '%s');" % 800 (uid, name, uid)) 801 return uid, onclick, targets

802

803 - def doc_descr(self, doc, context):

804 name = str(doc.canonical_name) 805 descr = '%s %s' % (self.doc_kind(doc), name) 806 if isinstance(doc, RoutineDoc): 807 descr += '()' 808 return descr

809 810 # [XX] copied streight from html.py; this should be consolidated, 811 # probably into apidoc.

812 - def doc_kind(self, doc):

813 if isinstance(doc, ModuleDoc) and doc.is_package == True: 814 return 'Package' 815 elif (isinstance(doc, ModuleDoc) and 816 doc.canonical_name[0].startswith('script')): 817 return 'Script' 818 elif isinstance(doc, ModuleDoc): 819 return 'Module' 820 elif isinstance(doc, ClassDoc): 821 return 'Class' 822 elif isinstance(doc, ClassMethodDoc): 823 return 'Class Method' 824 elif isinstance(doc, StaticMethodDoc): 825 return 'Static Method' 826 elif isinstance(doc, RoutineDoc): 827 if (self.docindex is not None and 828 isinstance(self.docindex.container(doc), ClassDoc)): 829 return 'Method' 830 else: 831 return 'Function' 832 else: 833 return 'Variable'

834

835 - def mark_def(self, s, name):

836 replacement = ('<a name="%s"></a><div id="%s-def">\\1' 837 '<a class="py-toggle" href="#" id="%s-toggle" ' 838 'onclick="return toggle(\'%s\');">-</a>\\2' % 839 (name, name, name, name)) 840 return re.sub('(.*) (<tt class="py-line">.*)\Z', replacement, s)

841

842 - def is_docstring(self, line, i):

843 if line[i][0] != token.STRING: return False 844 for toktype, toktext in line[i:]: 845 if toktype not in (token.NEWLINE, tokenize.COMMENT, 846 tokenize.NL, token.STRING, None): 847 return False 848 return True

849

850 - def add_line_numbers(self, s, css_class):

851 result = '' 852 start = 0 853 end = s.find('\n')+1 854 while end: 855 result += s[start:end-1] 856 if css_class: result += '</tt>' 857 result += ' </tt>' # py-line 858 result += '\n' 859 if self.ADD_LINE_NUMBERS: 860 result += self.lineno_to_html() 861 result += ' <tt class="py-line">' 862 if css_class: result += '<tt class="%s">' % css_class 863 start = end 864 end = s.find('\n', end)+1 865 self.lineno += 1 866 result += s[start:] 867 return result

868

869 - def name2url(self, class_name, func_name=None):

870 if class_name: 871 class_name = '%s.%s' % (self.module_name, class_name) 872 if func_name: 873 return '%s-class.html#%s' % (class_name, func_name) 874 else: 875 return '%s-class.html' % class_name 876 else: 877 return '%s-module.html#%s' % (self.module_name, func_name)

878 879 #: A regexp used to move the <div> that marks the beginning of a 880 #: function or method to just before the decorators. 881 _FIX_DECORATOR_RE = re.compile( 882 r'((?:^<a name="L\d+"></a><tt class="py-lineno">\s*\d+</tt>' 883 r'\s*<tt class="py-line">(?:<tt class="py-decorator">.*|\s*</tt>|' 884 r'\s*<tt class="py-comment">.*)\n)+)' 885 r'(<a name="\w+"></a><div id="\w+-def">)', re.MULTILINE)

Source Code for Module epydoc.docwriter.html_colorize