Fix issue #66: "RenderEngine should operate only on unicode strings"

Merry Christmas! :)

pystache/renderengine.py

@@ -50,6 +50,16 @@ class RenderEngine(object):
 
     This class is meant only for internal use by the Template class.
 
+    As a rule, the code in this class operates on unicode strings where
+    possible rather than, say, strings of type str or markupsafe.Markup.
+    This means that strings obtained from "external" sources like partials
+    and variable tag values are immediately converted to unicode (or
+    escaped and converted to unicode) before being operated on further.
+    This makes maintaining, reasoning about, and testing the correctness
+    of the code much simpler.  In particular, it keeps the implementation
+    of this class independent of the API details of one (or possibly more)
+    unicode subclasses (e.g. markupsafe.Markup).
+
     """
     tag_re = None
     otag = '{{'
@@ -61,15 +71,30 @@ class RenderEngine(object):
         """
         Arguments:
 
-          load_partial: a function for loading templates by name when
-            loading partials.  The function should accept a template name
-            and return a unicode template string.
-
-          escape: a function that takes a unicode or str string,
-            converts it to unicode, and escapes and returns it.
-
-          literal: a function that converts a unicode or str string
-            to unicode without escaping, and returns it.
+          load_partial: the function to call when loading a partial.  The
+            function should accept a string template name and return a
+            template string of type unicode (not a subclass).
+
+          literal: the function used to convert unescaped variable tag
+            values to unicode, e.g. the value corresponding to a tag
+            "{{{name}}}".  The function should accept a string of type
+            str or unicode (or a subclass) and return a string of type
+            unicode (but not a proper subclass of unicode).
+                This class will only pass basestring instances to this
+            function.  For example, it will call str() on integer variable
+            values prior to passing them to this function.
+
+          escape: the function used to escape and convert variable tag
+            values to unicode, e.g. the value corresponding to a tag
+            "{{name}}".  The function should obey the same properties
+            described above for the "literal" function argument.
+                This function should take care to convert any str
+            arguments to unicode just as the literal function should, as
+            this class will not pass tag values to literal prior to passing
+            them to this function.  This allows for more flexibility,
+            for example using a custom escape function that handles
+            incoming strings of type markupssafe.Markup differently
+            from plain unicode strings.
 
         """
         self.escape = escape
@@ -78,9 +103,13 @@ class RenderEngine(object):
 
     def render(self, template, context):
         """
+        Return a template rendered as a string with type unicode.
+
         Arguments:
 
-          template: a unicode template string.
+          template: a template string of type unicode (but not a proper
+            subclass of unicode).
+
           context: a Context instance.
 
         """
@@ -122,7 +151,7 @@ class RenderEngine(object):
                 # Then there was no match.
                 break
 
-            start, tag_type, tag_name, template = parts
+            tag_type, tag_name, template = parts[1:]
 
             tag_name = tag_name.strip()
             func = self.modifiers[tag_type]
@@ -133,6 +162,7 @@ class RenderEngine(object):
             output.append(tag_value)
 
         output = "".join(output)
+
         return output
 
     def _render_dictionary(self, template, context):
@@ -149,35 +179,45 @@ class RenderEngine(object):
 
         return ''.join(insides)
 
-    @modifiers.set(None)
-    def _render_tag(self, tag_name):
+    def _get_string_context(self, tag_name):
         """
-        Return the value of a variable as an escaped unicode string.
+        Get a value from the current context as a basestring instance.
 
         """
-        raw = self.context.get(tag_name, '')
+        val = self.context.get(tag_name)
 
-        # For methods with no return value
-        #
-        # We use "==" rather than "is" to compare integers, as using "is" relies
-        # on an implementation detail of CPython.  The test about rendering
-        # zeroes failed while using PyPy when using "is".
+        # We use "==" rather than "is" to compare integers, as using "is"
+        # relies on an implementation detail of CPython.  The test about
+        # rendering zeroes failed while using PyPy when using "is".
         # See issue #34: https://github.com/defunkt/pystache/issues/34
-        if not raw and raw != 0:
-            if tag_name == '.':
-                raw = self.context.top()
-            else:
+        if not val and val != 0:
+            if tag_name != '.':
                 return ''
+            val = self.context.top()
 
-        # If we don't first convert to a string type, the call to self._unicode_and_escape()
-        # will yield an error like the following:
-        #
-        #   TypeError: coercing to Unicode: need string or buffer, ... found
-        #
-        if not isinstance(raw, basestring):
-            raw = str(raw)
+        if not isinstance(val, basestring):
+            val = str(val)
 
-        return self.escape(raw)
+        return val
+
+    @modifiers.set(None)
+    def _render_escaped(self, tag_name):
+        """
+        Return a variable value as an escaped unicode string.
+
+        """
+        s = self._get_string_context(tag_name)
+        return self.escape(s)
+
+    @modifiers.set('{')
+    @modifiers.set('&')
+    def _render_literal(self, tag_name):
+        """
+        Return a variable value as a unicode string (unescaped).
+
+        """
+        s = self._get_string_context(tag_name)
+        return self.literal(s)
 
     @modifiers.set('!')
     def _render_comment(self, tag_name):
@@ -185,8 +225,8 @@ class RenderEngine(object):
 
     @modifiers.set('>')
     def _render_partial(self, template_name):
-        markup = self.load_partial(template_name)
-        return self._render(markup)
+        template = self.load_partial(template_name)
+        return self._render(template)
 
     @modifiers.set('=')
     def _change_delimiter(self, tag_name):
@@ -199,20 +239,11 @@ class RenderEngine(object):
 
         return ''
 
-    @modifiers.set('{')
-    @modifiers.set('&')
-    def _render_unescaped(self, tag_name):
-        """
-        Render a tag without escaping it.
-
-        """
-        return self.literal(self.context.get(tag_name, ''))
-
     def _render(self, template):
         """
         Arguments:
 
-          template: a unicode template string.
+          template: a template string with type unicode.
 
         """
         output = []

pystache/renderer.py

@@ -63,9 +63,10 @@ class Renderer(object):
             example "utf-8".  See the render() method's documentation for
             more information.
 
-          escape: the function used to escape mustache variable values
-            when rendering a template.  The function should accept a
-            unicode string and return an escaped string of the same type.
+          escape: the function used to escape variable tag values when
+            rendering a template.  The function should accept a unicode
+            string (or subclass of unicode) and return an escaped string
+            that is again unicode (or a subclass of unicode).
                 This function need not handle strings of type `str` because
             this class will only pass it unicode strings.  The constructor
             assigns this function to the constructed instance's escape()
@@ -91,14 +92,13 @@ class Renderer(object):
             default_encoding = sys.getdefaultencoding()
 
         if escape is None:
+            # TODO: use 'quote=True' with cgi.escape and add tests.
             escape = markupsafe.escape if markupsafe else cgi.escape
 
         if loader is None:
             loader = Loader(encoding=default_encoding, decode_errors=decode_errors)
 
-        literal = markupsafe.Markup if markupsafe else unicode
-
-        self._literal = literal
+        self._literal = markupsafe.Markup if markupsafe else unicode
 
         self.decode_errors = decode_errors
         self.default_encoding = default_encoding
@@ -106,37 +106,46 @@ class Renderer(object):
         self.loader = loader
         self.output_encoding = output_encoding
 
-    def _unicode_and_escape(self, s):
-        if not isinstance(s, unicode):
-            s = self.unicode(s)
-        return self.escape(s)
+    def _to_unicode_soft(self, s):
+        """
+        Convert an str or unicode string to a unicode string (or subclass).
 
-    def unicode(self, s):
-        return unicode(s, self.default_encoding, self.decode_errors)
+        """
+        # Avoid the "double-decoding" TypeError.
+        return s if isinstance(s, unicode) else self.unicode(s)
 
-    def escape(self, u):
+    def _to_unicode_hard(self, s):
         """
-        Escape a unicode string, and return it.
+        Convert an str or unicode string to a unicode string (not subclass).
 
-        This function is initialized as the escape function that was passed
-        to the Template class's constructor when this instance was
-        constructed.  See the constructor docstring for more information.
+        """
+        return unicode(self._to_unicode_soft(s))
 
+    def _escape_to_unicode(self, s):
         """
-        pass
+        Convert an str or unicode string to unicode, and escape it.
+
+        Returns a unicode string (not subclass).
 
-    def literal(self, s):
         """
-        Convert the given string to a unicode string, without escaping it.
+        return unicode(self.escape(self._to_unicode_soft(s)))
 
-        This function internally calls the built-in function unicode() and
-        passes it the default_encoding and decode_errors attributes for this
-        Template instance.  If markupsafe was importable when loading this
-        module, this function returns an instance of the class
-        markupsafe.Markup (which subclasses unicode).
+    def unicode(self, s):
+        """
+        Convert a string to unicode, using default_encoding and decode_errors.
+
+        Raises:
+
+          TypeError: Because this method calls Python's built-in unicode()
+            function, this method raises the following exception if the
+            given string is already unicode:
+
+              TypeError: decoding Unicode is not supported
 
         """
-        return self._literal(self.unicode(s))
+        # TODO: Wrap UnicodeDecodeErrors with a message about setting
+        # the default_encoding and decode_errors attributes.
+        return unicode(s, self.default_encoding, self.decode_errors)
 
     def _make_context(self, context, **kwargs):
         """
@@ -157,21 +166,16 @@ class Renderer(object):
         return context
 
     def _make_load_partial(self):
-        """
-        Return the load_partial function for use by RenderEngine.
-
-        """
         def load_partial(name):
             template = self.loader.get(name)
-            # Make sure the return value is unicode since RenderEngine requires
-            # it.  Also, check that the string is not already unicode to
-            # avoid "double-decoding".  Otherwise, we would get the following
-            # error:
-            #   TypeError: decoding Unicode is not supported
-            if not isinstance(template, unicode):
-                template = self.unicode(template)
 
-            return template
+            if template is None:
+                # TODO: make a TemplateNotFoundException type that provides
+                # the original loader as an attribute.
+                raise Exception("Partial not found with name: %s" % repr(name))
+
+            # RenderEngine requires that the return value be unicode.
+            return self._to_unicode_hard(template)
 
         return load_partial
 
@@ -183,8 +187,8 @@ class Renderer(object):
         load_partial = self._make_load_partial()
 
         engine = RenderEngine(load_partial=load_partial,
-                              literal=self.literal,
-                              escape=self._unicode_and_escape)
+                              literal=self._to_unicode_hard,
+                              escape=self._escape_to_unicode)
         return engine
 
     def render(self, template, context=None, **kwargs):
@@ -194,30 +198,32 @@ class Renderer(object):
         Returns:
 
           If the output_encoding attribute is None, the return value is
-          a unicode string.  Otherwise, the return value is encoded to a
-          string of type str using the output encoding named by the
-          output_encoding attribute.
+          markupsafe.Markup if markup was importable and unicode if not.
+          Otherwise, the return value is encoded to a string of type str
+          using the output encoding named by the output_encoding attribute.
 
         Arguments:
 
           template: a template string that is either unicode or of type str.
             If the string has type str, it is first converted to unicode
-            using the default_encoding and decode_errors attributes of this
-            instance.  See the constructor docstring for more information.
+            using this instance's default_encoding and decode_errors
+            attributes.  See the constructor docstring for more information.
 
           context: a dictionary, Context, or object (e.g. a View instance).
 
-          **kwargs: additional key values to add to the context when rendering.
-            These values take precedence over the context on any key conflicts.
+          **kwargs: additional key values to add to the context when
+            rendering.  These values take precedence over the context on
+            any key conflicts.
 
         """
         engine = self._make_render_engine()
         context = self._make_context(context, **kwargs)
 
-        if not isinstance(template, unicode):
-            template = self.unicode(template)
+        # RenderEngine.render() requires that the template string be unicode.
+        template = self._to_unicode_hard(template)
 
         rendered = engine.render(template, context)
+        rendered = self._literal(rendered)
 
         if self.output_encoding is not None:
             rendered = rendered.encode(self.output_encoding)

tests/test_renderengine.py

@@ -16,25 +16,42 @@ class RenderEngineTestCase(unittest.TestCase):
 
     """Test the RenderEngine class."""
 
-    def _engine(self):
+    def test_init(self):
         """
-        Create and return a default RenderEngine for testing.
+        Test that __init__() stores all of the arguments correctly.
 
         """
-        to_unicode = unicode
+        # In real-life, these arguments would be functions
+        engine = RenderEngine(load_partial="foo", literal="literal", escape="escape")
+
+        self.assertEquals(engine.escape, "escape")
+        self.assertEquals(engine.literal, "literal")
+        self.assertEquals(engine.load_partial, "foo")
 
-        escape = lambda s: cgi.escape(to_unicode(s))
-        literal = to_unicode
 
-        engine = RenderEngine(literal=literal, escape=escape, load_partial=None)
+class RenderEngineEnderTestCase(unittest.TestCase):
+
+    """Test RenderEngine.render()."""
+
+    def _engine(self):
+        """
+        Create and return a default RenderEngine for testing.
+
+        """
+        escape = lambda s: unicode(cgi.escape(s))
+        engine = RenderEngine(literal=unicode, escape=escape, load_partial=None)
         return engine
 
     def _assert_render(self, expected, template, *context, **kwargs):
+        """
+        Test rendering the given template using the given context.
+
+        """
         partials = kwargs.get('partials')
         engine = kwargs.get('engine', self._engine())
 
         if partials is not None:
-            engine.load_partial = lambda key: partials[key]
+            engine.load_partial = lambda key: unicode(partials[key])
 
         context = Context(*context)
 
@@ -42,22 +59,10 @@ class RenderEngineTestCase(unittest.TestCase):
 
         self.assertEquals(actual, expected)
 
-    def test_init(self):
-        """
-        Test that __init__() stores all of the arguments correctly.
-
-        """
-        # In real-life, these arguments would be functions
-        engine = RenderEngine(load_partial="foo", literal="literal", escape="escape")
-
-        self.assertEquals(engine.load_partial, "foo")
-        self.assertEquals(engine.escape, "escape")
-        self.assertEquals(engine.literal, "literal")
-
     def test_render(self):
         self._assert_render('Hi Mom', 'Hi {{person}}', {'person': 'Mom'})
 
-    def test_render__load_partial(self):
+    def test__load_partial(self):
         """
         Test that render() uses the load_template attribute.
 
@@ -65,25 +70,106 @@ class RenderEngineTestCase(unittest.TestCase):
         engine = self._engine()
         partials = {'partial': "{{person}}"}
         engine.load_partial = lambda key: partials[key]
+
         self._assert_render('Hi Mom', 'Hi {{>partial}}', {'person': 'Mom'}, engine=engine)
 
-    def test_render__literal(self):
+    def test__literal(self):
         """
         Test that render() uses the literal attribute.
 
         """
         engine = self._engine()
         engine.literal = lambda s: s.upper()
-        self._assert_render('bar BAR', '{{foo}} {{{foo}}}', {'foo': 'bar'}, engine=engine)
 
-    def test_render__escape(self):
+        self._assert_render('BAR', '{{{foo}}}', {'foo': 'bar'}, engine=engine)
+
+    def test__escape(self):
         """
         Test that render() uses the escape attribute.
 
         """
         engine = self._engine()
         engine.escape = lambda s: "**" + s
-        self._assert_render('**bar bar', '{{foo}} {{{foo}}}', {'foo': 'bar'}, engine=engine)
+
+        self._assert_render('**bar', '{{foo}}', {'foo': 'bar'}, engine=engine)
+
+    def test__escape_does_not_call_literal(self):
+        """
+        Test that render() does not call literal before or after calling escape.
+
+        """
+        engine = self._engine()
+        engine.literal = lambda s: s.upper()  # a test version
+        engine.escape = lambda s: "**" + s
+
+        template = 'literal: {{{foo}}} escaped: {{foo}}'
+        context = {'foo': 'bar'}
+
+        self._assert_render('literal: BAR escaped: **bar', template, context, engine=engine)
+
+    def test__escape_preserves_unicode_subclasses(self):
+        """
+        Test that render() preserves unicode subclasses when passing to escape.
+
+        This is useful, for example, if one wants to respect whether a
+        variable value is markupsafe.Markup when escaping.
+
+        """
+        class MyUnicode(unicode):
+            pass
+
+        def escape(s):
+            if type(s) is MyUnicode:
+                return "**" + s
+            else:
+                return s + "**"
+
+        engine = self._engine()
+        engine.escape = escape
+
+        template = '{{foo1}} {{foo2}}'
+        context = {'foo1': MyUnicode('bar'), 'foo2': 'bar'}
+
+        self._assert_render('**bar bar**', template, context, engine=engine)
+
+    def test__non_basestring__literal_and_escaped(self):
+        """
+        Test a context value that is not a basestring instance.
+
+        """
+        # We use include upper() to make sure we are actually using
+        # our custom function in the tests
+        to_unicode = lambda s: unicode(s, encoding='ascii').upper()
+        engine = self._engine()
+        engine.escape = to_unicode
+        engine.literal = to_unicode
+
+        self.assertRaises(TypeError, engine.literal, 100)
+
+        template = '{{text}} {{int}} {{{int}}}'
+        context = {'int': 100, 'text': 'foo'}
+
+        self._assert_render('FOO 100 100', template, context, engine=engine)
+
+    def test__implicit_iterator__literal(self):
+        """
+        Test an implicit iterator in a literal tag.
+
+        """
+        template = """{{#test}}{{.}}{{/test}}"""
+        context = {'test': ['a', 'b']}
+
+        self._assert_render('ab', template, context)
+
+    def test__implicit_iterator__escaped(self):
+        """
+        Test an implicit iterator in a normal tag.
+
+        """
+        template = """{{#test}}{{{.}}}{{/test}}"""
+        context = {'test': ['a', 'b']}
+
+        self._assert_render('ab', template, context)
 
     def test_render_with_partial(self):
         partials = {'partial': "{{person}}"}
@@ -95,13 +181,11 @@ class RenderEngineTestCase(unittest.TestCase):
 
         """
         engine = self._engine()
-        engine.escape = lambda s: "**" + s
-        engine.literal = lambda s: s.upper()
 
-        template = '{{#test}}{{foo}} {{{foo}}}{{/test}}'
-        context = {'test': {'foo': 'bar'}}
+        template = '{{#test}}unescaped: {{{foo}}} escaped: {{foo}}{{/test}}'
+        context = {'test': {'foo': '<'}}
 
-        self._assert_render('**bar BAR', template, context, engine=engine)
+        self._assert_render('unescaped: < escaped: &lt;', template, context, engine=engine)
 
     def test_render__partial_context_values(self):
         """
@@ -109,12 +193,12 @@ class RenderEngineTestCase(unittest.TestCase):
 
         """
         engine = self._engine()
-        engine.escape = lambda s: "**" + s
-        engine.literal = lambda s: s.upper()
 
-        partials = {'partial': '{{foo}} {{{foo}}}'}
+        template = '{{>partial}}'
+        partials = {'partial': 'unescaped: {{{foo}}} escaped: {{foo}}'}
+        context = {'foo': '<'}
 
-        self._assert_render('**bar BAR', '{{>partial}}', {'foo': 'bar'}, engine=engine, partials=partials)
+        self._assert_render('unescaped: < escaped: &lt;', template, context, engine=engine, partials=partials)
 
     def test_render__list_referencing_outer_context(self):
         """

tests/test_renderer.py

@@ -13,6 +13,7 @@ from pystache import renderer
 from pystache.renderer import Renderer
 from pystache.loader import Loader
 
+
 class RendererInitTestCase(unittest.TestCase):
 
     """A class to test the Renderer.__init__() method."""
@@ -54,7 +55,6 @@ class RendererInitTestCase(unittest.TestCase):
         self.assertEquals(actual.__dict__, expected.__dict__)
 
 
-
 class RendererTestCase(unittest.TestCase):
 
     """Test the Renderer class."""
@@ -146,13 +146,13 @@ class RendererTestCase(unittest.TestCase):
         renderer = Renderer(decode_errors="foo")
         self.assertEquals(renderer.decode_errors, "foo")
 
-    def test_unicode(self):
-        renderer = Renderer()
-        actual = renderer.literal("abc")
-        self.assertEquals(actual, "abc")
-        self.assertEquals(type(actual), unicode)
+    ## Test Renderer.unicode().
 
     def test_unicode__default_encoding(self):
+        """
+        Test that the default_encoding attribute is respected.
+
+        """
         renderer = Renderer()
         s = "é"
 
@@ -163,40 +163,20 @@ class RendererTestCase(unittest.TestCase):
         self.assertEquals(renderer.unicode(s), u"é")
 
     def test_unicode__decode_errors(self):
-        renderer = Renderer()
-        s = "é"
+        """
+        Test that the decode_errors attribute is respected.
 
+        """
+        renderer = Renderer()
         renderer.default_encoding = "ascii"
-        renderer.decode_errors = "strict"
-        self.assertRaises(UnicodeDecodeError, renderer.unicode, s)
+        s = "déf"
+
+        renderer.decode_errors = "ignore"
+        self.assertEquals(renderer.unicode(s), "df")
 
         renderer.decode_errors = "replace"
         # U+FFFD is the official Unicode replacement character.
-        self.assertEquals(renderer.unicode(s), u'\ufffd\ufffd')
-
-    def test_literal__with_markupsafe(self):
-        if not self._was_markupsafe_imported():
-            # Then we cannot test this case.
-            return
-        self._restore_markupsafe()
-
-        _renderer = Renderer()
-        _renderer.default_encoding = "utf_8"
-
-        # Check the standard case.
-        actual = _renderer.literal("abc")
-        self.assertEquals(actual, "abc")
-        self.assertEquals(type(actual), renderer.markupsafe.Markup)
-
-        s = "é"
-        # Check that markupsafe respects default_encoding.
-        self.assertEquals(_renderer.literal(s), u"é")
-        _renderer.default_encoding = "ascii"
-        self.assertRaises(UnicodeDecodeError, _renderer.literal, s)
-
-        # Check that markupsafe respects decode_errors.
-        _renderer.decode_errors = "replace"
-        self.assertEquals(_renderer.literal(s), u'\ufffd\ufffd')
+        self.assertEquals(renderer.unicode(s), u'd\ufffd\ufffdf')
 
     def test_render__unicode(self):
         renderer = Renderer()
@@ -314,47 +294,176 @@ class RendererTestCase(unittest.TestCase):
         #   TypeError: decoding Unicode is not supported
         self.assertEquals(load_partial("partial"), "foo")
 
-    # By testing that Renderer.render() constructs the RenderEngine instance
-    # correctly, we no longer need to test the rendering code paths through
-    # the Renderer.  We can test rendering paths through only the RenderEngine
-    # for the same amount of code coverage.
-    def test_make_render_engine__load_partial(self):
+
+# By testing that Renderer.render() constructs the right RenderEngine,
+# we no longer need to exercise all rendering code paths through
+# the Renderer.  It suffices to test rendering paths through the
+# RenderEngine for the same amount of code coverage.
+class Renderer_MakeRenderEngineTests(unittest.TestCase):
+
+    """
+    Check the RenderEngine returned by Renderer._make_render_engine().
+
+    """
+
+    ## Test the engine's load_partial attribute.
+
+    def test__load_partial__returns_unicode(self):
         """
-        Test that _make_render_engine() constructs and passes load_partial correctly.
+        Check that load_partial returns unicode (and not a subclass).
 
         """
-        partials = {'partial': 'foo'}
-        renderer = Renderer(loader=partials)
-        renderer.unicode = lambda s: s.upper()  # a test version.
+        class MyUnicode(unicode):
+            pass
+
+        renderer = Renderer()
+        renderer.default_encoding = 'ascii'
+        renderer.loader = {'str': 'foo', 'subclass': MyUnicode('abc')}
 
         engine = renderer._make_render_engine()
-        # Make sure it calls unicode.
-        self.assertEquals(engine.load_partial('partial'), "FOO")
 
-    def test_make_render_engine__literal(self):
+        actual = engine.load_partial('str')
+        self.assertEquals(actual, "foo")
+        self.assertEquals(type(actual), unicode)
+
+        # Check that unicode subclasses are not preserved.
+        actual = engine.load_partial('subclass')
+        self.assertEquals(actual, "abc")
+        self.assertEquals(type(actual), unicode)
+
+    def test__load_partial__not_found(self):
         """
-        Test that _make_render_engine() passes the right literal.
+        Check that load_partial provides a nice message when a template is not found.
 
         """
         renderer = Renderer()
-        renderer.literal = "foo"  # in real life, this would be a function.
+        renderer.loader = {}
 
         engine = renderer._make_render_engine()
-        self.assertEquals(engine.literal, "foo")
+        load_partial = engine.load_partial
 
-    def test_make_render_engine__escape(self):
+        try:
+            load_partial("foo")
+            raise Exception("Shouldn't get here")
+        except Exception, err:
+            self.assertEquals(str(err), "Partial not found with name: 'foo'")
+
+    ## Test the engine's literal attribute.
+
+    def test__literal__uses_renderer_unicode(self):
         """
-        Test that _make_render_engine() passes the right escape.
+        Test that literal uses the renderer's unicode function.
 
         """
         renderer = Renderer()
-        renderer.unicode = lambda s: s.upper()  # a test version.
-        renderer.escape = lambda s: "**" + s  # a test version.
+        renderer.unicode = lambda s: s.upper()
+
+        engine = renderer._make_render_engine()
+        literal = engine.literal
+
+        self.assertEquals(literal("foo"), "FOO")
+
+    def test__literal__handles_unicode(self):
+        """
+        Test that literal doesn't try to "double decode" unicode.
+
+        """
+        renderer = Renderer()
+        renderer.default_encoding = 'ascii'
+
+        engine = renderer._make_render_engine()
+        literal = engine.literal
+
+        self.assertEquals(literal(u"foo"), "foo")
+
+    def test__literal__returns_unicode(self):
+        """
+        Test that literal returns unicode (and not a subclass).
+
+        """
+        renderer = Renderer()
+        renderer.default_encoding = 'ascii'
+
+        engine = renderer._make_render_engine()
+        literal = engine.literal
+
+        self.assertEquals(type(literal("foo")), unicode)
+
+        class MyUnicode(unicode):
+            pass
+
+        s = MyUnicode("abc")
+
+        self.assertEquals(type(s), MyUnicode)
+        self.assertTrue(isinstance(s, unicode))
+        self.assertEquals(type(literal(s)), unicode)
+
+    ## Test the engine's escape attribute.
+
+    def test__escape__uses_renderer_escape(self):
+        """
+        Test that escape uses the renderer's escape function.
+
+        """
+        renderer = Renderer()
+        renderer.escape = lambda s: "**" + s
+
+        engine = renderer._make_render_engine()
+        escape = engine.escape
+
+        self.assertEquals(escape("foo"), "**foo")
+
+    def test__escape__uses_renderer_unicode(self):
+        """
+        Test that escape uses the renderer's unicode function.
+
+        """
+        renderer = Renderer()
+        renderer.unicode = lambda s: s.upper()
+
+        engine = renderer._make_render_engine()
+        escape = engine.escape
+
+        self.assertEquals(escape("foo"), "FOO")
+
+    def test__escape__has_access_to_original_unicode_subclass(self):
+        """
+        Test that escape receives strings with the unicode subclass intact.
+
+        """
+        renderer = Renderer()
+        renderer.escape = lambda s: type(s).__name__
 
         engine = renderer._make_render_engine()
         escape = engine.escape
 
-        self.assertEquals(escape(u"foo"), "**foo")
+        class MyUnicode(unicode):
+            pass
+
+        self.assertEquals(escape("foo"), "unicode")
+        self.assertEquals(escape(u"foo"), "unicode")
+        self.assertEquals(escape(MyUnicode("foo")), "MyUnicode")
+
+    def test__escape__returns_unicode(self):
+        """
+        Test that literal returns unicode (and not a subclass).
+
+        """
+        renderer = Renderer()
+        renderer.default_encoding = 'ascii'
+
+        engine = renderer._make_render_engine()
+        escape = engine.escape
+
+        self.assertEquals(type(escape("foo")), unicode)
+
+        # Check that literal doesn't preserve unicode subclasses.
+        class MyUnicode(unicode):
+            pass
+
+        s = MyUnicode("abc")
+
+        self.assertEquals(type(s), MyUnicode)
+        self.assertTrue(isinstance(s, unicode))
+        self.assertEquals(type(escape(s)), unicode)
 
-        # Test that escape converts str strings to unicode first.
-        self.assertEquals(escape("foo"), "**FOO")