Merge 'development-spec' into 'development': spec-compliance. Happy New Year! :)

This commit merges pvande's spec-compliant branch `e06bed51`
into the development branch with all tests (including v1.0.3 spec tests) passing.
Also, speed did not slow considerably.  Further refactoring and clean-up are
still needed.

Here are results of the benchmark script before and after (using
`python tests/benchmark.py 10000`):

development (before):

    Benchmarking: 10000x

    0.931457042694
    3.08717703819

development (after):

    Benchmarking: 10000x

    1.56453585625
    4.69798803329

And here were the test results before the commit (now all 280 are passing):

(using `nosetests --with-doctest --doctest-extension=rst -i spec`)

    Ran 280 tests in 0.535s

    FAILED (errors=5, failures=39)

examples/inner_partial.txt

-## Again, {{title}}! ##
+## Again, {{title}}! ##
\ No newline at end of file

spec @ 48c933b0

-Subproject commit 62871926ab5789ab6c55f5a1deda359ba5f7b2fa
+Subproject commit 48c933b0bb780875acbfd15816297e263c53d6f7

pystache/renderengine.py

@@ -5,8 +5,11 @@ Defines a class responsible for rendering logic.
 
 """
 
+import cgi
 import collections
+import inspect
 import re
+import types
 
 
 try:
@@ -113,11 +116,18 @@ class RenderEngine(object):
           context: a Context instance.
 
         """
-        self.context = context
+        # Be strict but not too strict.  In other words, accept str instead
+        # of unicode, but don't assume anything about the encoding (e.g.
+        # don't use self.literal).
+        template = unicode(template)
 
-        self._compile_regexps()
+        _template = Template(template=template)
 
-        return self._render(template)
+        _template.to_unicode = self.literal
+        _template.escape = self.escape
+        _template.get_partial = self.load_partial
+
+        return _template.render_template(template=template, context=context)
 
     def _compile_regexps(self):
         """
@@ -294,3 +304,315 @@ class RenderEngine(object):
         output = "".join(output)
         return output
 
+#
+
+
+END_OF_LINE_CHARACTERS = ['\r', '\n']
+
+
+# TODO: what are the possibilities for val?
+def call(val, view, template=None):
+    if callable(val):
+        (args, _, _, _) = inspect.getargspec(val)
+
+        args_count = len(args)
+
+        if not isinstance(val, types.FunctionType):
+            # Then val is an instance method.  Subtract one from the
+            # argument count because Python will automatically prepend
+            # self to the argument list when calling.
+            args_count -=1
+
+        if args_count is 0:
+            val = val()
+        elif args_count is 1 and args[0] in ['self', 'context']:
+            val = val(view)
+        elif args_count is 1:
+            val = val(template)
+        else:
+            val = val(view, template)
+
+    if callable(val):
+        val = val(template)
+
+    if val is None:
+        val = ''
+
+    return unicode(val)
+
+def render_parse_tree(parse_tree, context, template):
+    """
+    Convert a parse-tree into a string.
+
+    """
+    get_string = lambda val: call(val, context, template)
+    parts = map(get_string, parse_tree)
+    return ''.join(parts)
+
+def inverseTag(name, parsed, template, delims):
+    def func(self):
+        data = self.get(name)
+        if data:
+            return ''
+        return render_parse_tree(parsed, self, delims)
+    return func
+
+class EndOfSection(Exception):
+    def __init__(self, parse_tree, template, position):
+        self.parse_tree = parse_tree
+        self.template = template
+        self.position = position
+
+class Template(object):
+    tag_re = None
+    otag, ctag = '{{', '}}'
+
+    def __init__(self, template=None):
+        self.template = template
+
+    def _compile_regexps(self):
+
+        # The possible tag type characters following the opening tag,
+        # excluding "=" and "{".
+        tag_types = "!>&/#^"
+
+        # TODO: are we following this in the spec?
+        #
+        #   The tag's content MUST be a non-whitespace character sequence
+        #   NOT containing the current closing delimiter.
+        #
+        tag = r"""
+            (?P<content>[\s\S]*?)
+            (?P<whitespace>[\ \t]*)
+            %(otag)s \s*
+            (?:
+              (?P<change>=) \s* (?P<delims>.+?)   \s* = |
+              (?P<raw>{)    \s* (?P<raw_name>.+?) \s* } |
+              (?P<tag>[%(tag_types)s]?)  \s* (?P<name>[\s\S]+?)
+            )
+            \s* %(ctag)s
+        """ % {'tag_types': tag_types, 'otag': re.escape(self.otag), 'ctag': re.escape(self.ctag)}
+
+        self.tag_re = re.compile(tag, re.M | re.X)
+
+    def to_unicode(self, text):
+        return unicode(text)
+
+    def escape(self, text):
+        return cgi.escape(text, True)
+
+    def get_partial(self, name):
+        pass
+
+    def _get_string_value(self, context, tag_name):
+        """
+        Get a value from the given context as a basestring instance.
+
+        """
+        val = context.get(tag_name)
+
+        # We use "==" rather than "is" to compare integers, as using "is"
+        # relies on an implementation detail of CPython.  The test about
+        # rendering zeroes failed while using PyPy when using "is".
+        # See issue #34: https://github.com/defunkt/pystache/issues/34
+        if not val and val != 0:
+            if tag_name != '.':
+                return ''
+            val = context.top()
+
+        if callable(val):
+            # According to the spec:
+            #
+            #     When used as the data value for an Interpolation tag,
+            #     the lambda MUST be treatable as an arity 0 function,
+            #     and invoked as such.  The returned value MUST be
+            #     rendered against the default delimiters, then
+            #     interpolated in place of the lambda.
+            template = val()
+            if not isinstance(template, basestring):
+                # In case the template is an integer, for example.
+                template = str(template)
+            if type(template) is not unicode:
+                template = self.to_unicode(template)
+            val = self.render_template(template, context)
+
+        if not isinstance(val, basestring):
+            val = str(val)
+
+        return val
+
+    def escape_tag_function(self, name):
+        get_literal = self.literal_tag_function(name)
+        def func(context):
+            s = self._get_string_value(context, name)
+            s = self.escape(s)
+            return s
+        return func
+
+    def literal_tag_function(self, name):
+        def func(context):
+            s = self._get_string_value(context, name)
+            s = self.to_unicode(s)
+            return s
+
+        return func
+
+    def partial_tag_function(self, name, indentation=''):
+        def func(context):
+            nonblank = re.compile(r'^(.)', re.M)
+            template = self.get_partial(name)
+            # Indent before rendering.
+            template = re.sub(nonblank, indentation + r'\1', template)
+            return self.render_template(template, context)
+        return func
+
+    def section_tag_function(self, name, parse_tree_, template_, delims):
+        def func(context):
+            template = template_
+            parse_tree = parse_tree_
+            data = context.get(name)
+            if not data:
+                data = []
+            elif callable(data):
+                # TODO: should we check the arity?
+                template = data(template)
+                parse_tree = self.parse_string_to_tree(template, delims)
+                data = [ data ]
+            elif type(data) not in [list, tuple]:
+                data = [ data ]
+
+            parts = []
+            for element in data:
+                context.push(element)
+                parts.append(render_parse_tree(parse_tree, context, delims))
+                context.pop()
+
+            return ''.join(parts)
+        return func
+
+    def parse_string_to_tree(self, template, delims=('{{', '}}')):
+
+        template = Template(template)
+
+        template.otag = delims[0]
+        template.ctag = delims[1]
+
+        template.escape = self.escape
+        template.get_partial = self.get_partial
+        template.to_unicode = self.to_unicode
+
+        template._compile_regexps()
+
+        return template.parse_to_tree()
+
+    def parse_to_tree(self, index=0):
+        """
+        Parse a template into a syntax tree.
+
+        """
+        parse_tree = []
+        template = self.template
+        start_index = index
+
+        while True:
+            match = self.tag_re.search(template, index)
+
+            if match is None:
+                break
+
+            captures = match.groupdict()
+            match_index = match.end('content')
+            end_index = match.end()
+
+            index = self._handle_match(parse_tree, captures, start_index, match_index, end_index)
+
+        # Save the rest of the template.
+        parse_tree.append(template[index:])
+
+        return parse_tree
+
+    def _handle_match(self, parse_tree, captures, start_index, match_index, end_index):
+        template = self.template
+
+        # Normalize the captures dictionary.
+        if captures['change'] is not None:
+            captures.update(tag='=', name=captures['delims'])
+        elif captures['raw'] is not None:
+            captures.update(tag='{', name=captures['raw_name'])
+
+        parse_tree.append(captures['content'])
+
+        # Standalone (non-interpolation) tags consume the entire line,
+        # both leading whitespace and trailing newline.
+        did_tag_begin_line = match_index == 0 or template[match_index - 1] in END_OF_LINE_CHARACTERS
+        did_tag_end_line = end_index == len(template) or template[end_index] in END_OF_LINE_CHARACTERS
+        is_tag_interpolating = captures['tag'] in ['', '&', '{']
+
+        if did_tag_begin_line and did_tag_end_line and not is_tag_interpolating:
+            if end_index < len(template):
+                end_index += template[end_index] == '\r' and 1 or 0
+            if end_index < len(template):
+                end_index += template[end_index] == '\n' and 1 or 0
+        elif captures['whitespace']:
+            parse_tree.append(captures['whitespace'])
+            match_index += len(captures['whitespace'])
+            captures['whitespace'] = ''
+
+        name = captures['name']
+
+        if captures['tag'] == '!':
+            return end_index
+
+        if captures['tag'] == '=':
+            self.otag, self.ctag = name.split()
+            self._compile_regexps()
+            return end_index
+
+        if captures['tag'] == '>':
+            func = self.partial_tag_function(name, captures['whitespace'])
+        elif captures['tag'] in ['#', '^']:
+
+            try:
+                self.parse_to_tree(index=end_index)
+            except EndOfSection as e:
+                bufr = e.parse_tree
+                tmpl = e.template
+                end_index = e.position
+
+            tag = self.section_tag_function if captures['tag'] == '#' else inverseTag
+            func = tag(name, bufr, tmpl, (self.otag, self.ctag))
+
+        elif captures['tag'] in ['{', '&']:
+
+            func = self.literal_tag_function(name)
+
+        elif captures['tag'] == '':
+
+            func = self.escape_tag_function(name)
+
+        elif captures['tag'] == '/':
+
+            # TODO: don't use exceptions for flow control.
+            raise EndOfSection(parse_tree, template[start_index:match_index], end_index)
+
+        else:
+            raise Exception("'%s' is an unrecognized type!" % captures['tag'])
+
+        parse_tree.append(func)
+
+        return end_index
+
+    def render_template(self, template, context, delims=('{{', '}}')):
+        """
+        Arguments:
+
+          template: template string
+          context: a Context instance
+
+        """
+        if type(template) is not unicode:
+            raise Exception("Argument 'template' not unicode: %s: %s" % (type(template), repr(template)))
+
+        parse_tree = self.parse_string_to_tree(template=template, delims=delims)
+        return render_parse_tree(parse_tree, context, template)
+

tests/common.py

@@ -13,3 +13,17 @@ DATA_DIR = 'tests/data'
 def get_data_path(file_name):
     return os.path.join(DATA_DIR, file_name)
 
+
+def assert_strings(test_case, actual, expected):
+    # Show both friendly and literal versions.
+    message = """\
+
+
+    Expected: \"""%s\"""
+    Actual:   \"""%s\"""
+
+    Expected: %s
+    Actual:   %s""" % (expected, actual, repr(expected), repr(actual))
+    test_case.assertEquals(actual, expected, message)
+
+

tests/spec_cases.py

@@ -55,7 +55,11 @@ def buildTest(testData, spec_filename):
   Template: \"""%s\"""
 
   Expected: %s
-  Actual:   %s""" % (description, template, repr(expected), repr(actual))
+  Actual:   %s
+
+  Expected: \"""%s\"""
+  Actual:   \"""%s\"""
+  """ % (description, template, repr(expected), repr(actual), expected, actual)
 
         self.assertEquals(actual, expected, message)
 

tests/test_examples.py

@@ -13,6 +13,9 @@ from examples.unicode_output import UnicodeOutput
 from examples.unicode_input import UnicodeInput
 from examples.nested_context import NestedContext
 
+from tests.common import assert_strings
+
+
 class TestView(unittest.TestCase):
     def test_comments(self):
         self.assertEquals(Comments().render(), """<h1>A Comedy of Errors</h1>
@@ -47,20 +50,14 @@ Again, Welcome!""")
     def test_template_partial_extension(self):
         view = TemplatePartial()
         view.template_extension = 'txt'
-        self.assertEquals(view.render(), """Welcome
+        assert_strings(self, view.render(), u"""Welcome
 -------
 
-## Again, Welcome! ##
-
-""")
-
+## Again, Welcome! ##""")
 
     def test_delimiters(self):
-        self.assertEquals(Delimiters().render(), """
-* It worked the first time.
-
+        assert_strings(self, Delimiters().render(), """* It worked the first time.
 * And it worked the second time.
-
 * Then, surprisingly, it worked the third time.
 """)
 

tests/test_renderengine.py

@@ -68,7 +68,7 @@ class RenderEngineEnderTestCase(unittest.TestCase):
 
         """
         engine = self._engine()
-        partials = {'partial': "{{person}}"}
+        partials = {'partial': u"{{person}}"}
         engine.load_partial = lambda key: partials[key]
 
         self._assert_render('Hi Mom', 'Hi {{>partial}}', {'person': 'Mom'}, engine=engine)
@@ -243,12 +243,20 @@ class RenderEngineEnderTestCase(unittest.TestCase):
 
     def test_render__section__lambda__tag_in_output(self):
         """
-        Check that callable output isn't treated as a template string (issue #46).
+        Check that callable output is treated as a template string (issue #46).
+
+        The spec says--
+
+            When used as the data value for a Section tag, the lambda MUST
+            be treatable as an arity 1 function, and invoked as such (passing
+            a String containing the unprocessed section contents).  The
+            returned value MUST be rendered against the current delimiters,
+            then interpolated in place of the section.
 
         """
-        template = '{{#test}}Mom{{/test}}'
-        context = {'test': (lambda text: '{{hi}} %s' % text)}
-        self._assert_render('{{hi}} Mom', template, context)
+        template = '{{#test}}Hi {{person}}{{/test}}'
+        context = {'person': 'Mom', 'test': (lambda text: text + " :)")}
+        self._assert_render('Hi Mom :)', template, context)
 
     def test_render__section__comment__multiline(self):
         """

tests/test_simple.py

 import unittest
+
 import pystache
 from pystache import Renderer
 from examples.nested_context import NestedContext
@@ -7,6 +8,9 @@ from examples.lambdas import Lambdas
 from examples.template_partial import TemplatePartial
 from examples.simple import Simple
 
+from tests.common import assert_strings
+
+
 class TestSimple(unittest.TestCase):
 
     def test_nested_context(self):
@@ -44,11 +48,19 @@ class TestSimple(unittest.TestCase):
 
 
     def test_template_partial_extension(self):
+        """
+        Side note:
+
+        From the spec--
+
+            Partial tags SHOULD be treated as standalone when appropriate.
+
+        In particular, this means that trailing newlines should be removed.
+
+        """
         view = TemplatePartial()
         view.template_extension = 'txt'
-        self.assertEquals(view.render(), """Welcome
+        assert_strings(self, view.render(), u"""Welcome
 -------
 
-## Again, Welcome! ##
-
-""")
+## Again, Welcome! ##""")