Replace Markdown toc extension and add improved version to macro package.

wiki/conf/settings.py

@@ -15,7 +15,7 @@ WIKI_LANGUAGE = 'markdown'
 EDITOR = getattr( django_settings, 'WIKI_EDITOR', 'wiki.editors.markitup.MarkItUp' )
 
 MARKDOWN_KWARGS = {
-    'extensions': ['footnotes', 'headerid', 'extra', 'toc'],
+    'extensions': ['footnotes', 'headerid', 'extra',],
     'safe_mode': 'replace',
     'extension_configs': {'toc': {'title': _('Table of Contents')}},
 }

wiki/plugins/macros/mdx/macro.py

+# -*- coding: utf-8 -*-
+import markdown
+import re
+
+from django.utils.translation import ugettext as _
+from django.template.loader import render_to_string
+from django.template import Context
+
+# See: http://stackoverflow.com/questions/430759/regex-for-managing-escaped-characters-for-items-like-string-literals
+re_sq_short = r"'([^'\\]*(?:\\.[^'\\]*)*)'"
+
+MACRO_RE = re.compile(r'.*(\[(?P<macro>\w+)(?P<kwargs>\s\w+\:.+)*\]).*', re.IGNORECASE)
+KWARG_RE = re.compile(r'\s*(?P<arg>\w+)(:(?P<value>([^\']+|%s)))?' % re_sq_short, re.IGNORECASE)
+
+from wiki.plugins.macros import settings
+
+class MacroExtension(markdown.Extension):
+    """ Macro plugin markdown extension for django-wiki. """
+
+    def extendMarkdown(self, md, md_globals):
+        """ Insert MacroPreprocessor before ReferencePreprocessor. """
+        md.preprocessors.add('dw-macros', MacroPreprocessor(md), '>html_block')
+
+
+class MacroPreprocessor(markdown.preprocessors.Preprocessor):
+    """django-wiki macro preprocessor - parse text for various [some_macro] and 
+    [some_macro (kw:arg)*] references. """
+    
+    def run(self, lines):
+        # Look at all those indentations.
+        # That's insane, let's get a helper library
+        # Please note that this pattern is also in plugins.images
+        new_text = []
+        for line in lines:
+            m = MACRO_RE.match(line)
+            if m:
+                macro = m.group('macro').strip()
+                if macro in settings.METHODS and hasattr(self, macro):
+                    kwargs = m.group('kwargs')
+                    if kwargs:
+                        kwargs_dict = {}
+                        for kwarg in KWARG_RE.finditer(kwargs):
+                            arg = kwarg.group('arg')
+                            value = kwarg.group('value')
+                            if value is None:
+                                value = True
+                            if isinstance(value, basestring):
+                                # If value is enclosed with ': Remove and remove escape sequences
+                                if value.startswith(u"'") and len(value) > 2:
+                                    value = value[1:-1]
+                                    value = value.replace(u"\\\\", u"¤KEEPME¤")
+                                    value = value.replace(u"\\", u"")
+                                    value = value.replace(u"¤KEEPME¤", u"\\")
+                            kwargs_dict[str(arg)] = value
+                        line = getattr(self, macro)(**kwargs_dict)
+                    else:
+                        line = getattr(self, macro)()
+            if not line is None:
+                new_text.append(line)
+        return new_text
+
+    def article_list(self, depth="2"):
+        html = render_to_string(
+            "wiki/plugins/macros/article_list.html",
+            Context({
+                'article_children': self.markdown.article.get_children(article__current_revision__deleted=False),
+                'depth': int(depth) + 1,
+            })
+        )
+        return self.markdown.htmlStash.store(html, safe=True)
+    article_list.meta = dict(
+        short_description = _(u'Article list'),
+        help_text = _(u'Insert a list of articles in this level.'),
+        example_code = _(u'[article_list depth:2]'),
+        args = {'depth': _('Maximum depth to show levels for.')}
+    )
+
+    def toc(self):
+        return "[TOC]"
+    toc.meta = dict(
+        short_description = _(u'Table of contents'),
+        help_text = _(u'Insert a table of contents matching the headings.'),
+        example_code = _(u'[TOC]'),
+        args = {}
+    )

wiki/plugins/macros/mdx/toc.py

+"""
+Table of Contents Extension for Python-Markdown
+* * *
+
+(c) 2008 [Jack Miller](http://codezen.org)
+
+Dependencies:
+* [Markdown 2.1+](http://packages.python.org/Markdown/)
+
+Pull request to include the below code in Python-Markdown:
+https://github.com/waylan/Python-Markdown/pull/191
+
+Until it's released, we have a copy here.
+
+/benjaoming
+
+"""
+import markdown
+from markdown.util import etree
+from markdown.extensions.headerid import slugify, unique, itertext
+
+import re
+
+from wiki.plugins.macros import settings
+
+def order_toc_list(toc_list):
+    """Given an unsorted list with errors and skips, return a nested one.
+    [{'level': 1}, {'level': 2}]
+    =>
+    [{'level': 1, 'children': [{'level': 2, 'children': []}]}]
+    
+    A wrong list is also converted:
+    [{'level': 2}, {'level': 1}]
+    =>
+    [{'level': 2, 'children': []}, {'level': 1, 'children': []}]
+    """
+    
+    def build_correct(remaining_list, prev_elements=[{'level': 1000}]):
+        
+        if not remaining_list:
+            return [], []
+        
+        current = remaining_list.pop(0)
+        if not 'children' in current.keys():
+            current['children'] = []
+        
+        if not prev_elements:
+            # This happens for instance with [8, 1, 1], ie. when some
+            # header level is outside a scope. We treat it as a
+            # top-level
+            next_elements, children = build_correct(remaining_list, [current])
+            current['children'].append(children)
+            return [current] + next_elements, []
+        
+        prev_element = prev_elements.pop()
+        children = []
+        next_elements = []
+        # Is current part of the child list or next list?
+        if current['level'] > prev_element['level']:
+            #print "%d is a child of %d" % (current['level'], prev_element['level'])
+            prev_elements.append(prev_element)
+            prev_elements.append(current)
+            prev_element['children'].append(current)
+            next_elements2, children2 = build_correct(remaining_list, prev_elements)
+            children += children2
+            next_elements += next_elements2
+        else:
+            #print "%d is ancestor of %d" % (current['level'], prev_element['level'])
+            if not prev_elements:
+                #print "No previous elements, so appending to the next set"
+                next_elements.append(current)
+                prev_elements = [current]
+                next_elements2, children2 = build_correct(remaining_list, prev_elements)
+                current['children'].extend(children2)
+            else:
+                #print "Previous elements, comparing to those first"
+                remaining_list.insert(0, current)
+                next_elements2, children2 = build_correct(remaining_list, prev_elements)
+                children.extend(children2)
+            next_elements += next_elements2
+        
+        return next_elements, children
+    
+    flattened_list, __ = build_correct(toc_list)
+    return flattened_list
+
+
+class TocTreeprocessor(markdown.treeprocessors.Treeprocessor):
+    
+    # Iterator wrapper to get parent and child all at once
+    def iterparent(self, root):
+        for parent in root.getiterator():
+            for child in parent:
+                yield parent, child
+    
+    def add_anchor(self, c, elem_id): #@ReservedAssignment
+        if self.use_anchors:
+            anchor = etree.Element("a")
+            anchor.text = c.text
+            anchor.attrib["href"] = "#" + elem_id
+            anchor.attrib["class"] = "toclink"
+            c.text = ""
+            for elem in c.getchildren():
+                anchor.append(elem)
+                c.remove(elem)
+            c.append(anchor)
+    
+    def build_toc_etree(self, div, toc_list):
+        # Add title to the div
+        if self.config["title"]:
+            header = etree.SubElement(div, "span")
+            header.attrib["class"] = "toctitle"
+            header.text = self.config["title"]
+
+        def build_etree_ul(toc_list, parent):
+            ul = etree.SubElement(parent, "ul")
+            for item in toc_list:
+                # List item link, to be inserted into the toc div
+                li = etree.SubElement(ul, "li")
+                link = etree.SubElement(li, "a")
+                link.text = item.get('name', '')
+                link.attrib["href"] = '#' + item.get('id', '')
+                if item['children']:
+                    build_etree_ul(item['children'], li)
+            return ul
+        
+        return build_etree_ul(toc_list, div)
+        
+    def run(self, doc):
+
+        div = etree.Element("div")
+        div.attrib["class"] = "toc"
+        header_rgx = re.compile("[Hh][123456]")
+        
+        self.use_anchors = self.config["anchorlink"] in [1, '1', True, 'True', 'true']
+        
+        # Get a list of id attributes
+        used_ids = []
+        for c in doc.getiterator():
+            if "id" in c.attrib:
+                used_ids.append(c.attrib["id"])
+
+        toc_list = []
+        marker_found = False
+        for (p, c) in self.iterparent(doc):
+            text = ''.join(itertext(c)).strip()
+            if not text:
+                continue
+
+            # To keep the output from screwing up the
+            # validation by putting a <div> inside of a <p>
+            # we actually replace the <p> in its entirety.
+            # We do not allow the marker inside a header as that
+            # would causes an enless loop of placing a new TOC 
+            # inside previously generated TOC.
+            if c.text and c.text.strip() == self.config["marker"] and \
+               not header_rgx.match(c.tag) and c.tag not in ['pre', 'code']:
+                for i in range(len(p)):
+                    if p[i] == c:
+                        p[i] = div
+                        break
+                marker_found = True
+                            
+            if header_rgx.match(c.tag):
+                
+                # Do not override pre-existing ids 
+                if not "id" in c.attrib:
+                    elem_id = unique(self.config["slugify"](text, '-'), used_ids)
+                    c.attrib["id"] = elem_id
+                else:
+                    elem_id = c.attrib["id"]
+
+                tag_level = int(c.tag[-1])
+                
+                toc_list.append({'level': tag_level,
+                    'id': elem_id,
+                    'name': c.text})
+                
+                self.add_anchor(c, elem_id)
+                
+        if marker_found:
+            toc_list_nested = order_toc_list(toc_list)
+            self.build_toc_etree(div, toc_list_nested)
+            # serialize and attach to markdown instance.
+            prettify = self.markdown.treeprocessors.get('prettify')
+            if prettify: prettify.run(div)
+            toc = self.markdown.serializer(div)
+            for pp in self.markdown.postprocessors.values():
+                toc = pp.run(toc)
+            self.markdown.toc = toc
+
+
+class TocExtension(markdown.Extension):
+    
+    TreeProcessorClass = TocTreeprocessor
+    
+    def __init__(self, configs=[]):
+        self.config = { "marker" : ["[TOC]", 
+                            "Text to find and replace with Table of Contents -"
+                            "Defaults to \"[TOC]\""],
+                        "slugify" : [slugify,
+                            "Function to generate anchors based on header text-"
+                            "Defaults to the headerid ext's slugify function."],
+                        "title" : [None,
+                            "Title to insert into TOC <div> - "
+                            "Defaults to None"],
+                        "anchorlink" : [0,
+                            "1 if header should be a self link"
+                            "Defaults to 0"]}
+
+        for key, value in configs:
+            self.setConfig(key, value)
+
+    def extendMarkdown(self, md, md_globals):
+        tocext = self.TreeProcessorClass(md)
+        tocext.config = self.getConfigs()
+        # Headerid ext is set to '>inline'. With this set to '<prettify',
+        # it should always come after headerid ext (and honor ids assinged 
+        # by the header id extension) if both are used. Same goes for 
+        # attr_list extension. This must come last because we don't want
+        # to redefine ids after toc is created. But we do want toc prettified.
+        md.treeprocessors.add("toc", tocext, "<prettify")
+
+
+def makeExtension(configs={}):
+    return TocExtension(configs=configs)
+
+
+class WikiTreeProcessorClass(TocTreeprocessor):
+    
+    def build_toc_etree(self, div, toc_list):
+        # Add title to the div
+        if self.config["title"]:
+            header = etree.SubElement(div, "span")
+            header.attrib["class"] = "toctitle"
+            header.text = self.config["title"]
+
+        def build_etree_ul(toc_list, parent):
+            ul = etree.SubElement(parent, "ul")
+            for item in toc_list:
+                # List item link, to be inserted into the toc div
+                li = etree.SubElement(ul, "li")
+                link = etree.SubElement(li, "a")
+                link.text = item.get('name', '')
+                link.attrib["href"] = '#' + item.get('id', '')
+                if item['children']:
+                    build_etree_ul(item['children'], li)
+            return ul
+        
+        return build_etree_ul(toc_list, div)
+    
+    
+class WikiTocExtension(TocExtension):
+    TreeProcessorClass = WikiTreeProcessorClass
+
+    def extendMarkdown(self, md, md_globals):
+        if 'toc' in settings.METHODS:
+            print md, md_globals
+            TocExtension.extendMarkdown(self, md, md_globals)

wiki/plugins/macros/settings.py

@@ -4,4 +4,4 @@ from django.conf import settings as django_settings
 SLUG = 'macros'
 APP_LABEL = 'wiki'
 
-METHODS = getattr(django_settings, 'WIKI_PLUGINS_METHODS', ('article_list',))
\ No newline at end of file
+METHODS = getattr(django_settings, 'WIKI_PLUGINS_METHODS', ('article_list', 'toc',))
\ No newline at end of file

wiki/plugins/macros/templatetags/wiki_macro_tags.py

 from django import template
-from wiki.plugins.macros import settings, markdown_extensions
+from wiki.plugins.macros import settings
+from wiki.plugins.macros.mdx.macro import MacroPreprocessor
 
 register = template.Library()
 
@@ -18,7 +19,7 @@ def article_list(context, urlpath, depth):
 def allowed_macros():
     for method in settings.METHODS:
         try:
-            yield getattr(markdown_extensions.MacroPreprocessor, method).meta
+            yield getattr(MacroPreprocessor, method).meta
         except AttributeError:
             continue
     

wiki/plugins/macros/wiki_plugin.py

@@ -4,8 +4,9 @@ from django.utils.translation import ugettext as _
 from wiki.core.plugins import registry
 from wiki.core.plugins.base import BasePlugin
 from wiki.plugins.macros import settings
-from wiki.plugins.macros.markdown_extensions import MacroExtension
 
+from wiki.plugins.macros.mdx.macro import MacroExtension
+from wiki.plugins.macros.mdx.toc import WikiTocExtension
 
 class MacroPlugin(BasePlugin):
     
@@ -17,7 +18,7 @@ class MacroPlugin(BasePlugin):
                'form_class': None,
                'get_form_kwargs': (lambda a: {})}
     
-    markdown_extensions = [MacroExtension()]
+    markdown_extensions = [MacroExtension(), WikiTocExtension()]
     
     def __init__(self):
         pass