- added "bytestring passthru" mode, via `disable_unicode=True`

  argument passed to Template or TemplateLookup.  All
  unicode-awareness and filtering is turned off, and template
  modules are generated with the appropriate magic encoding
  comment.  In this mode, template expressions can only
  receive raw bytestrings or Unicode objects which represent
  straight ASCII, and render_unicode() may not be used.
  [ticket:77]  (courtesy anonymous guest)

6 files changed, 43 insertions, 25 deletions

diff --git a/lib/mako/codegen.py b/lib/mako/codegen.py
index 523c3b0..acf5bf1 100644
--- a/lib/mako/codegen.py
+++ b/lib/mako/codegen.py
@@ -14,21 +14,27 @@ from mako import util, ast, parsetree, filters
 MAGIC_NUMBER = 2
 
 
-def compile(node, uri, filename=None, default_filters=None, buffer_filters=None, imports=None, source_encoding=None):
+def compile(node, uri, filename=None, default_filters=None, buffer_filters=None, imports=None, source_encoding=None, generate_unicode=True):
     """generate module source code given a parsetree node, uri, and optional source filename"""
-    buf = util.FastEncodingBuffer()
+    
+    if generate_unicode:
+        buf = util.FastEncodingBuffer()  # creates Unicode
+    else:
+        buf = util.StringIO()  # returns whatever was passed in
+
     printer = PythonPrinter(buf)
-    _GenerateRenderMethod(printer, _CompileContext(uri, filename, default_filters, buffer_filters, imports, source_encoding), node)
+    _GenerateRenderMethod(printer, _CompileContext(uri, filename, default_filters, buffer_filters, imports, source_encoding, generate_unicode), node)
     return buf.getvalue()
 
 class _CompileContext(object):
-    def __init__(self, uri, filename, default_filters, buffer_filters, imports, source_encoding):
+    def __init__(self, uri, filename, default_filters, buffer_filters, imports, source_encoding, generate_unicode):
         self.uri = uri
         self.filename = filename
         self.default_filters = default_filters
         self.buffer_filters = buffer_filters
         self.imports = imports
         self.source_encoding = source_encoding
+        self.generate_unicode = generate_unicode
         
 class _GenerateRenderMethod(object):
     """a template visitor object which generates the full module source for a template."""
@@ -110,6 +116,9 @@ class _GenerateRenderMethod(object):
         module_identifiers.declared = module_ident
         
         # module-level names, python code
+        if not self.compiler.generate_unicode and self.compiler.source_encoding:
+            self.printer.writeline("# -*- encoding:%s -*-" % self.compiler.source_encoding)
+            
         self.printer.writeline("from mako import runtime, filters, cache")
         self.printer.writeline("UNDEFINED = runtime.UNDEFINED")
         self.printer.writeline("_magic_number = %s" % repr(MAGIC_NUMBER))
@@ -159,7 +168,7 @@ class _GenerateRenderMethod(object):
         )
         if buffered or filtered or cached:
             self.printer.writeline("context.push_buffer()")
-
+        
         self.identifier_stack.append(self.compiler.identifiers.branch(self.node))
         if not self.in_def and '**pageargs' in args:
             self.identifier_stack[-1].argument_declared.add('pageargs')
@@ -424,8 +433,6 @@ class _GenerateRenderMethod(object):
         def locate_encode(name):
             if re.match(r'decode\..+', name):
                 return "filters." + name
-            elif name == 'unicode':
-                return 'unicode'
             else:
                 return filters.DEFAULT_ESCAPES.get(name, name)
         
@@ -544,6 +551,7 @@ class _GenerateRenderMethod(object):
             )
         self.write_variable_declares(body_identifiers)
         self.identifier_stack.append(body_identifiers)
+        
         for n in node.nodes:
             n.accept_visitor(self)
         self.identifier_stack.pop()
@@ -563,7 +571,7 @@ class _GenerateRenderMethod(object):
             "try:")
         self.write_source_comment(node)
         self.printer.writelines(
-                "context.write(unicode(%s))" % node.attributes['expr'],
+                "context.write(%s)" % self.create_filter_callable([], node.attributes['expr'], True),
             "finally:",
                 "context.caller_stack.nextcaller = None",
             None
diff --git a/lib/mako/filters.py b/lib/mako/filters.py
index 94305d7..f252241 100644
--- a/lib/mako/filters.py
+++ b/lib/mako/filters.py
@@ -163,6 +163,7 @@ DEFAULT_ESCAPES = {
     'entity':'filters.html_entities_escape',
     'unicode':'unicode',
     'decode':'decode',
+    'str':'str',
     'n':'n'
 }
     
diff --git a/lib/mako/lexer.py b/lib/mako/lexer.py
index 621a92e..785461c 100644
--- a/lib/mako/lexer.py
+++ b/lib/mako/lexer.py
@@ -13,7 +13,7 @@ from mako.pygen import adjust_whitespace
 _regexp_cache = {}
 
 class Lexer(object):
-    def __init__(self, text, filename=None, input_encoding=None, preprocessor=None):
+    def __init__(self, text, filename=None, disable_unicode=False, input_encoding=None, preprocessor=None):
         self.text = text
         self.filename = filename
         self.template = parsetree.TemplateNode(self.filename)
@@ -23,6 +23,7 @@ class Lexer(object):
         self.match_position = 0
         self.tag = []
         self.control_line = []
+        self.disable_unicode = disable_unicode
         self.encoding = input_encoding
         if preprocessor is None:
             self.preprocessor = []
@@ -108,7 +109,7 @@ class Lexer(object):
                 raise exceptions.SyntaxException("Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), **self.exception_kwargs)
 
     def escape_code(self, text):
-        if self.encoding:
+        if not self.disable_unicode and self.encoding:
             return text.encode('ascii', 'backslashreplace')
         else:
             return text
@@ -126,8 +127,7 @@ class Lexer(object):
             parsed_encoding = self.match_encoding()
         if parsed_encoding:
             self.encoding = parsed_encoding
-        if not isinstance(self.text, unicode):
-
+        if not self.disable_unicode and not isinstance(self.text, unicode):
             if self.encoding:
                 try:
                     self.text = self.text.decode(self.encoding)
diff --git a/lib/mako/lookup.py b/lib/mako/lookup.py
index da4d1c3..290f71d 100644
--- a/lib/mako/lookup.py
+++ b/lib/mako/lookup.py
@@ -38,8 +38,8 @@ class TemplateCollection(object):
         
 class TemplateLookup(TemplateCollection):
     def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False, 
-    error_handler=None, output_encoding=None, encoding_errors='strict', cache_type=None, cache_dir=None, cache_url=None, 
-    modulename_callable=None, default_filters=['unicode'], buffer_filters=[], imports=None, input_encoding=None, preprocessor=None):
+    error_handler=None, disable_unicode=False, output_encoding=None, encoding_errors='strict', cache_type=None, cache_dir=None, cache_url=None, 
+    modulename_callable=None, default_filters=None, buffer_filters=[], imports=None, input_encoding=None, preprocessor=None):
         if isinstance(directories, basestring):
             directories = [directories]        
         self.directories = [posixpath.normpath(d) for d in directories or []]
@@ -47,7 +47,7 @@ class TemplateLookup(TemplateCollection):
         self.modulename_callable = modulename_callable
         self.filesystem_checks = filesystem_checks
         self.collection_size = collection_size
-        self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding, 'encoding_errors':encoding_errors, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'cache_url':cache_url, 'default_filters':default_filters, 'buffer_filters':buffer_filters,  'imports':imports, 'preprocessor':preprocessor}
+        self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'disable_unicode':disable_unicode, 'output_encoding':output_encoding, 'encoding_errors':encoding_errors, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'cache_url':cache_url, 'default_filters':default_filters, 'buffer_filters':buffer_filters,  'imports':imports, 'preprocessor':preprocessor}
         if collection_size == -1:
             self.__collection = {}
             self._uri_cache = {}
diff --git a/lib/mako/runtime.py b/lib/mako/runtime.py
index 56e3776..f0ae468 100644
--- a/lib/mako/runtime.py
+++ b/lib/mako/runtime.py
@@ -289,7 +289,7 @@ def _render(template, callable_, args, data, as_unicode=False):
     """create a Context and return the string output of the given template and template callable."""
     if as_unicode:
         buf = util.FastEncodingBuffer()
-    elif template.output_encoding:
+    elif not template.disable_unicode and template.output_encoding:
         buf = util.FastEncodingBuffer(template.output_encoding, template.encoding_errors)
     else:
         buf = util.StringIO()
diff --git a/lib/mako/template.py b/lib/mako/template.py
index e418b0f..36e4d43 100644
--- a/lib/mako/template.py
+++ b/lib/mako/template.py
@@ -18,7 +18,7 @@ class Template(object):
     """a compiled template"""
     def __init__(self, text=None, filename=None, uri=None, format_exceptions=False, error_handler=None, 
         lookup=None, output_encoding=None, encoding_errors='strict', module_directory=None, cache_type=None, 
-        cache_dir=None, cache_url=None, module_filename=None, input_encoding=None, default_filters=['unicode'], 
+        cache_dir=None, cache_url=None, module_filename=None, input_encoding=None, disable_unicode=False, default_filters=None, 
         buffer_filters=[], imports=None, preprocessor=None):
         """construct a new Template instance using either literal template text, or a previously loaded template module
         
@@ -42,9 +42,19 @@ class Template(object):
             self.module_id = "memory:" + hex(id(self))
             self.uri = self.module_id
         
-        self.default_filters = default_filters
-        self.buffer_filters = buffer_filters
         self.input_encoding = input_encoding
+        self.output_encoding = output_encoding
+        self.encoding_errors = encoding_errors
+        self.disable_unicode = disable_unicode
+        if default_filters is None:
+            if self.disable_unicode:
+                self.default_filters = ['str']
+            else:
+                self.default_filters = ['unicode']
+        else:
+            self.default_filters = default_filters
+        self.buffer_filters = buffer_filters
+            
         self.imports = imports
         self.preprocessor = preprocessor
         
@@ -94,8 +104,6 @@ class Template(object):
         self.format_exceptions = format_exceptions
         self.error_handler = error_handler
         self.lookup = lookup
-        self.output_encoding = output_encoding
-        self.encoding_errors = encoding_errors
         self.cache_type = cache_type
         self.cache_dir = cache_dir
         self.cache_url = cache_url
@@ -141,6 +149,7 @@ class DefTemplate(Template):
         self.buffer_filters = parent.buffer_filters
         self.input_encoding = parent.input_encoding
         self.imports = parent.imports
+        self.disable_unicode = parent.disable_unicode
         self.output_encoding = parent.output_encoding
         self.encoding_errors = parent.encoding_errors
         self.format_exceptions = parent.format_exceptions
@@ -191,9 +200,9 @@ class ModuleInfo(object):
         
 def _compile_text(template, text, filename):
     identifier = template.module_id
-    lexer = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
+    lexer = Lexer(text, filename, disable_unicode=template.disable_unicode, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
     node = lexer.parse()
-    source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports, source_encoding=lexer.encoding)
+    source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports, source_encoding=lexer.encoding, generate_unicode=not template.disable_unicode)
     #print source
     cid = identifier
     module = imp.new_module(cid)
@@ -203,9 +212,9 @@ def _compile_text(template, text, filename):
 
 def _compile_module_file(template, text, filename, outputpath):
     identifier = template.module_id
-    lexer = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
+    lexer = Lexer(text, filename, disable_unicode=template.disable_unicode, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
     node = lexer.parse()
-    source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports, source_encoding=lexer.encoding)
+    source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports, source_encoding=lexer.encoding, generate_unicode=not template.disable_unicode)
     (dest, name) = tempfile.mkstemp()
     os.write(dest, source)
     os.close(dest)