diff options
| author | Mike Bayer <mike_mp@zzzcomputing.com> | 2012-03-24 12:04:26 -0400 |
|---|---|---|
| committer | Mike Bayer <mike_mp@zzzcomputing.com> | 2012-03-24 12:04:26 -0400 |
| commit | 316c1ab2f3edc8807cf26e0d4718453cd0154872 (patch) | |
| tree | f63066b77304a67863e022686631be9086c0fdfb | |
| parent | 556151f5c2f5e491cb97ca7ffdbdb565b3145ccc (diff) | |
| download | external_python_mako-316c1ab2f3edc8807cf26e0d4718453cd0154872.tar.gz external_python_mako-316c1ab2f3edc8807cf26e0d4718453cd0154872.tar.bz2 external_python_mako-316c1ab2f3edc8807cf26e0d4718453cd0154872.zip | |
- apply the #125 patch
- changelog
| -rw-r--r-- | CHANGES | 14 | ||||
| -rw-r--r-- | mako/__init__.py | 2 | ||||
| -rw-r--r-- | mako/codegen.py | 110 | ||||
| -rw-r--r-- | mako/exceptions.py | 3 | ||||
| -rw-r--r-- | mako/lexer.py | 4 | ||||
| -rw-r--r-- | mako/parsetree.py | 12 | ||||
| -rw-r--r-- | mako/pygen.py | 5 | ||||
| -rw-r--r-- | mako/runtime.py | 117 | ||||
| -rw-r--r-- | mako/template.py | 12 | ||||
| -rw-r--r-- | mako/util.py | 27 | ||||
| -rw-r--r-- | test/test_loop.py | 201 |
11 files changed, 482 insertions, 25 deletions
@@ -1,4 +1,16 @@ -0.6.3 +0.7.0 +- [feature] Added new "loop" variable to templates, + is provided within a % for block to provide + info about the loop such as index, first/last, + odd/even, etc. Thanks to Ben Trofatter for all + the work on this [ticket:125] + +- [feature] Added a real check for "reserved" + names, that is names which are never pulled + from the context and cannot be passed to + the template.render() method. Current names + are "context", "loop". + - [feature] The html_error_template() will now apply Pygments highlighting to the source code displayed in the traceback, if Pygments diff --git a/mako/__init__.py b/mako/__init__.py index 4121e6b..f6f02eb 100644 --- a/mako/__init__.py +++ b/mako/__init__.py @@ -5,5 +5,5 @@ # the MIT License: http://www.opensource.org/licenses/mit-license.php -__version__ = '0.6.3' +__version__ = '0.7.0' diff --git a/mako/codegen.py b/mako/codegen.py index 704330c..c5e94c0 100644 --- a/mako/codegen.py +++ b/mako/codegen.py @@ -11,7 +11,12 @@ import re from mako.pygen import PythonPrinter from mako import util, ast, parsetree, filters, exceptions -MAGIC_NUMBER = 7 +MAGIC_NUMBER = 8 + +# names which are hardwired into the +# template and are not accessed via the +# context itself +RESERVED_NAMES = set(['context', 'loop']) def compile(node, uri, @@ -22,7 +27,8 @@ def compile(node, source_encoding=None, generate_magic_comment=True, disable_unicode=False, - strict_undefined=False): + strict_undefined=False, + reserved_names=()): """Generate module source code given a parsetree node, uri, and optional source filename""" @@ -47,7 +53,8 @@ def compile(node, source_encoding, generate_magic_comment, disable_unicode, - strict_undefined), + strict_undefined, + reserved_names), node) return buf.getvalue() @@ -61,7 +68,8 @@ class _CompileContext(object): source_encoding, generate_magic_comment, disable_unicode, - strict_undefined): + strict_undefined, + reserved_names): self.uri = uri self.filename = filename self.default_filters = default_filters @@ -71,6 +79,7 @@ class _CompileContext(object): self.generate_magic_comment = generate_magic_comment self.disable_unicode = disable_unicode self.strict_undefined = strict_undefined + self.reserved_names = reserved_names class _GenerateRenderMethod(object): """A template visitor object which generates the @@ -160,7 +169,7 @@ class _GenerateRenderMethod(object): for n in module_code: module_ident = module_ident.union(n.declared_identifiers()) - module_identifiers = _Identifiers() + module_identifiers = _Identifiers(self.compiler) module_identifiers.declared = module_ident # module-level names, python code @@ -388,6 +397,7 @@ class _GenerateRenderMethod(object): top-level, it is fully rendered as a local closure. """ + # collection of all defs available to us in this scope comp_idents = dict([(c.funcname, c) for c in identifiers.defs]) to_write = set() @@ -410,6 +420,9 @@ class _GenerateRenderMethod(object): # means that variable is now a "locally declared" var, # which cannot be referenced beforehand. to_write = to_write.difference(identifiers.locally_declared) + + has_loop = "loop" in to_write + to_write.discard("loop") # if a limiting set was sent, constraint to those items in that list # (this is used for the caching decorator) @@ -427,7 +440,12 @@ class _GenerateRenderMethod(object): ident, re.split(r'\s*,\s*', ns.attributes['import']) )) - + + if has_loop: + self.printer.writeline( + 'loop = __M_loop = runtime.LoopStack()' + ) + for ident in to_write: if ident in comp_idents: comp = comp_idents[ident] @@ -709,9 +727,17 @@ class _GenerateRenderMethod(object): if not node.get_children(): self.printer.writeline("pass") self.printer.writeline(None) + if node.has_loop_context: + self.printer.writeline('finally:') + self.printer.writeline("loop = __M_loop._exit()") + self.printer.writeline(None) else: self.write_source_comment(node) - self.printer.writeline(node.text) + if node.keyword == 'for': + text = mangle_mako_loop(node, self.printer) + else: + text = node.text + self.printer.writeline(text) def visitText(self, node): self.write_source_comment(node) @@ -739,6 +765,8 @@ class _GenerateRenderMethod(object): ) def visitCode(self, node): + # mangle loop variables within the scope of a loop context, + # if applicable if not node.ismodule: self.write_source_comment(node) self.printer.write_indented_block(node.text) @@ -865,7 +893,7 @@ class _GenerateRenderMethod(object): class _Identifiers(object): """tracks the status of identifier names as template code is rendered.""" - def __init__(self, node=None, parent=None, nested=False): + def __init__(self, compiler, node=None, parent=None, nested=False): if parent is not None: # if we are the branch created in write_namespaces(), # we don't share any context from the main body(). @@ -892,7 +920,9 @@ class _Identifiers(object): else: self.declared = set() self.topleveldefs = util.SetLikeDict() - + + self.compiler = compiler + # things within this level that are referenced before they # are declared (e.g. assigned to) self.undeclared = set() @@ -918,12 +948,19 @@ class _Identifiers(object): if node is not None: node.accept_visitor(self) - + + illegal_names = self.compiler.reserved_names.intersection(self.locally_declared) + if illegal_names: + raise exceptions.NameConflictError( + "Reserved words declared in template: %s" % + ", ".join(illegal_names)) + + def branch(self, node, **kwargs): """create a new Identifiers for a new Node, with this Identifiers as the parent.""" - return _Identifiers(node, self, **kwargs) + return _Identifiers(self.compiler, node, self, **kwargs) @property def defs(self): @@ -1055,3 +1092,54 @@ class _Identifiers(object): if ident != 'context' and ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident) + +_FOR_LOOP = re.compile( + r'^for\s+((?:\(?)\s*[A-Za-z_][A-Za-z_0-9]*' + r'(?:\s*,\s*(?:[A-Za-z_][A-Za-z0-9_]*),??)*\s*(?:\)?))\s+in\s+(.*):' + ) + +def mangle_mako_loop(node, printer): + """converts a for loop into a context manager wrapped around a for loop + when access to the `loop` variable has been detected in the for loop body + """ + loop_variable = LoopVariable() + node.accept_visitor(loop_variable) + if loop_variable.detected: + node.nodes[-1].has_loop_context = True + match = _FOR_LOOP.match(node.text) + if match: + printer.writelines( + 'loop = __M_loop._enter(%s)' % match.group(2), + 'try:' + #'with __M_loop(%s) as loop:' % match.group(2) + ) + text = 'for %s in loop:' % match.group(1) + else: + raise SyntaxError("Couldn't apply loop context: %s" % node.text) + else: + text = node.text + return text + + +class LoopVariable(object): + """A node visitor which looks for the name 'loop' within undeclared + identifiers.""" + + def __init__(self): + self.detected = False + + def _loop_reference_detected(self, node): + if 'loop' in node.undeclared_identifiers(): + self.detected = True + else: + for n in node.get_children(): + n.accept_visitor(self) + + def visitControlLine(self, node): + self._loop_reference_detected(node) + + def visitCode(self, node): + self._loop_reference_detected(node) + + def visitExpression(self, node): + self._loop_reference_detected(node) diff --git a/mako/exceptions.py b/mako/exceptions.py index c9e04ea..abb866e 100644 --- a/mako/exceptions.py +++ b/mako/exceptions.py @@ -41,6 +41,9 @@ class SyntaxException(MakoException): class UnsupportedError(MakoException): """raised when a retired feature is used.""" +class NameConflictError(MakoException): + """raised when a reserved word is used inappropriately""" + class TemplateLookupException(MakoException): pass diff --git a/mako/lexer.py b/mako/lexer.py index 9e0bc5e..8c8b849 100644 --- a/mako/lexer.py +++ b/mako/lexer.py @@ -128,6 +128,10 @@ class Lexer(object): self.tag[-1].nodes.append(node) else: self.template.nodes.append(node) + # build a set of child nodes for the control line + # (used for loop variable detection) + if self.control_line: + self.control_line[-1].nodes.append(node) if isinstance(node, parsetree.Tag): if len(self.tag): node.parent = self.tag[-1] diff --git a/mako/parsetree.py b/mako/parsetree.py index 52bd156..c3aa688 100644 --- a/mako/parsetree.py +++ b/mako/parsetree.py @@ -8,9 +8,10 @@ from mako import exceptions, ast, util, filters import re - + class Node(object): """base class for a Node in the parse tree.""" + def __init__(self, source, lineno, pos, filename): self.source = source self.lineno = lineno @@ -29,6 +30,7 @@ class Node(object): def traverse(node): for n in node.get_children(): n.accept_visitor(visitor) + method = getattr(visitor, "visit" + self.__class__.__name__, traverse) method(self) @@ -59,12 +61,15 @@ class ControlLine(Node): """ + has_loop_context = False + def __init__(self, keyword, isend, text, **kwargs): super(ControlLine, self).__init__(**kwargs) self.text = text self.keyword = keyword self.isend = isend - self.is_primary = keyword in ['for','if', 'while', 'try', 'with'] + self.is_primary = keyword in ['for', 'if', 'while', 'try', 'with'] + self.nodes = [] if self.isend: self._declared_identifiers = [] self._undeclared_identifiers = [] @@ -73,6 +78,9 @@ class ControlLine(Node): self._declared_identifiers = code.declared_identifiers self._undeclared_identifiers = code.undeclared_identifiers + def get_children(self): + return self.nodes + def declared_identifiers(self): return self._declared_identifiers diff --git a/mako/pygen.py b/mako/pygen.py index b50e60e..6b1767c 100644 --- a/mako/pygen.py +++ b/mako/pygen.py @@ -65,8 +65,6 @@ class PythonPrinter(object): self._flush_adjusted_lines() self.in_indent_lines = True - decreased_indent = False - if (line is None or re.match(r"^\s*#",line) or re.match(r"^\s*$", line) @@ -78,8 +76,7 @@ class PythonPrinter(object): is_comment = line and len(line) and line[0] == '#' # see if this line should decrease the indentation level - if (not decreased_indent and - not is_comment and + if (not is_comment and (not hastext or self._is_unindentor(line)) ): diff --git a/mako/runtime.py b/mako/runtime.py index b56fa67..5a83a81 100644 --- a/mako/runtime.py +++ b/mako/runtime.py @@ -10,6 +10,7 @@ Namespace, and various helper functions.""" from mako import exceptions, util import __builtin__, inspect, sys + class Context(object): """Provides runtime namespace, output buffer, and various callstacks for templates. @@ -23,6 +24,7 @@ class Context(object): self._buffer_stack = [buffer] self._data = data + self._kwargs = data.copy() self._with_template = None self._outputting_as_unicode = None @@ -34,7 +36,15 @@ class Context(object): # "caller" stack used by def calls with content self.caller_stack = self._data['caller'] = CallerStack() - + + def _set_with_template(self, t): + self._with_template = t + illegal_names = t.reserved_names.intersection(self._data) + if illegal_names: + raise exceptions.NameConflictError( + "Reserved words passed to render(): %s" % + ", ".join(illegal_names)) + @property def lookup(self): """Return the :class:`.TemplateLookup` associated @@ -187,6 +197,107 @@ class Undefined(object): UNDEFINED = Undefined() +class LoopStack(object): + """a stack for LoopContexts that implements the context manager protocol + to automatically pop off the top of the stack on context exit + """ + + def __init__(self): + self.stack = [] + + def _enter(self, iterable): + self._push(iterable) + return self._top + + def _exit(self): + self._pop() + return self._top + + @property + def _top(self): + if self.stack: + return self.stack[-1] + else: + return self + + def _pop(self): + return self.stack.pop() + + def _push(self, iterable): + new = LoopContext(iterable) + if self.stack: + new.parent = self.stack[-1] + return self.stack.append(new) + + def __getattr__(self, key): + raise exceptions.RuntimeException("No loop context is established") + + def __iter__(self): + return iter(self._top) + + +class LoopContext(object): + """A magic loop variable. + Automatically accessible in any %for block. + + :attr:`parent` -> LoopContext or None + The parent loop, if one exists + :attr:`index` -> int + The 0-based iteration count + :attr:`reverse_index` -> int + The number of iterations remaining + :attr:`first` -> bool + `True` on the first iteration, `False` otherwise + :attr:`last` -> bool + `True` on the last iteration, `False` otherwise + :attr:`even` -> bool + `True` when `index` is even + :attr:`odd` -> bool + `True` when `index` is odd + """ + + def __init__(self, iterable): + self._iterable = iterable + self.index = 0 + self.parent = None + + def __iter__(self): + for i in self._iterable: + yield i + self.index += 1 + + @util.memoized_instancemethod + def __len__(self): + return len(self._iterable) + + @property + def reverse_index(self): + return len(self) - self.index - 1 + + @property + def first(self): + return self.index == 0 + + @property + def last(self): + return self.index == len(self) - 1 + + @property + def even(self): + return not self.odd + + @property + def odd(self): + return bool(self.index % 2) + + def cycle(self, *values): + """cycle through values as the loop progresses + """ + if not values: + raise ValueError("You must provide values to cycle through") + return values[self.index % len(values)] + + class _NSAttr(object): def __init__(self, parent): self.__parent = parent @@ -644,7 +755,7 @@ def _render(template, callable_, args, data, as_unicode=False): errors=template.encoding_errors) context = Context(buf, **data) context._outputting_as_unicode = as_unicode - context._with_template = template + context._set_with_template(template) _render_context(template, callable_, context, *args, **_kwargs_for_callable(callable_, data)) @@ -721,5 +832,5 @@ def _render_error(template, context, error): error_template.output_encoding, error_template.encoding_errors)] - context._with_template = error_template + context._set_with_template(error_template) error_template.render_context(context, error=error) diff --git a/mako/template.py b/mako/template.py index f38a055..e3a6399 100644 --- a/mako/template.py +++ b/mako/template.py @@ -283,6 +283,10 @@ class Template(object): cache_type, cache_dir, cache_url ) + @util.memoized_property + def reserved_names(self): + return codegen.RESERVED_NAMES + def _setup_cache_args(self, cache_impl, cache_enabled, cache_args, cache_type, cache_dir, cache_url): @@ -396,7 +400,7 @@ class Template(object): """ if getattr(context, '_with_template', None) is None: - context._with_template = self + context._set_with_template(self) runtime._render_context(self, self.callable_, context, @@ -573,7 +577,8 @@ def _compile_text(template, text, filename): source_encoding=lexer.encoding, generate_magic_comment=template.disable_unicode, disable_unicode=template.disable_unicode, - strict_undefined=template.strict_undefined) + strict_undefined=template.strict_undefined, + reserved_names=template.reserved_names) cid = identifier if not util.py3k and isinstance(cid, unicode): @@ -601,7 +606,8 @@ def _compile_module_file(template, text, filename, outputpath, module_writer): source_encoding=lexer.encoding, generate_magic_comment=True, disable_unicode=template.disable_unicode, - strict_undefined=template.strict_undefined) + strict_undefined=template.strict_undefined, + reserved_names=template.reserved_names) if isinstance(source, unicode): source = source.encode(lexer.encoding or 'ascii') diff --git a/mako/util.py b/mako/util.py index 408dd3d..ce0bab8 100644 --- a/mako/util.py +++ b/mako/util.py @@ -129,6 +129,33 @@ class memoized_property(object): obj.__dict__[self.__name__] = result = self.fget(obj) return result +class memoized_instancemethod(object): + """Decorate a method memoize its return value. + + Best applied to no-arg methods: memoization is not sensitive to + argument values, and will always return the same value even when + called with different arguments. + + """ + def __init__(self, fget, doc=None): + self.fget = fget + self.__doc__ = doc or fget.__doc__ + self.__name__ = fget.__name__ + + def __get__(self, obj, cls): + if obj is None: + return self + def oneshot(*args, **kw): + result = self.fget(obj, *args, **kw) + memo = lambda *a, **kw: result + memo.__name__ = self.__name__ + memo.__doc__ = self.__doc__ + obj.__dict__[self.__name__] = memo + return result + oneshot.__name__ = self.__name__ + oneshot.__doc__ = self.__doc__ + return oneshot + class SetLikeDict(dict): """a dictionary that has some setlike methods on it""" def union(self, other): diff --git a/test/test_loop.py b/test/test_loop.py new file mode 100644 index 0000000..a86f8fd --- /dev/null +++ b/test/test_loop.py @@ -0,0 +1,201 @@ +import re +import unittest + +from mako.template import Template +from mako.codegen import ( + _FOR_LOOP, mangle_mako_loop, LoopVariable + ) +from mako.runtime import LoopStack, LoopContext +from mako import exceptions +from test import assert_raises_message + +class TestLoop(unittest.TestCase): + + def test__FOR_LOOP(self): + for statement, target_list, expression_list in ( + ('for x in y:', 'x', 'y'), + ('for x, y in z:', 'x, y', 'z'), + ('for (x,y) in z:', '(x,y)', 'z'), + ('for ( x, y, z) in a:', '( x, y, z)', 'a'), + ('for x in [1, 2, 3]:', 'x', '[1, 2, 3]'), + ('for x in "spam":', 'x', '"spam"'), + ('for k,v in dict(a=1,b=2).items():', 'k,v', + 'dict(a=1,b=2).items()'), + ('for x in [y+1 for y in [1, 2, 3]]:', 'x', + '[y+1 for y in [1, 2, 3]]') + ): + match = _FOR_LOOP.match(statement) + assert match and match.groups() == (target_list, expression_list) + + def test_no_loop(self): + template = Template("""% for x in 'spam': +${x} +% endfor""") + code = template.code + assert not re.match(r"loop = __M_loop._enter\(:", code), "No need to "\ + "generate a loop context if the loop variable wasn't accessed" + print template.render() + + def test_loop_demo(self): + template = Template("""x|index|reverse_index|first|last|cycle|even|odd +% for x in 'ham': +${x}|${loop.index}|${loop.reverse_index}|${loop.first}|${loop.last}|${loop.cycle('even', 'odd')}|${loop.even}|${loop.odd} +% endfor""") + expected = [ + "x|index|reverse_index|first|last|cycle|even|odd", + "h|0|2|True|False|even|True|False", + "a|1|1|False|False|odd|False|True", + "m|2|0|False|True|even|True|False" + ] + code = template.code + assert "loop = __M_loop._enter(" in code, "Generated a loop context since "\ + "the loop variable was accessed" + rendered = template.render() + print rendered + for line in expected: + assert line in rendered, "Loop variables give information about "\ + "the progress of the loop" + + def test_nested_loops(self): + template = Template("""% for x in 'ab': +${x} ${loop.index} <- start in outer loop +% for y in [0, 1]: +${y} ${loop.index} <- go to inner loop +% endfor +${x} ${loop.index} <- back to outer loop +% endfor""") + code = template.code + rendered = template.render() + expected = [ + "a 0 <- start in outer loop", + "0 0 <- go to inner loop", + "1 1 <- go to inner loop", + "a 0 <- back to outer loop", + "b 1 <- start in outer loop", + "0 0 <- go to inner loop", + "1 1 <- go to inner loop", + "b 1 <- back to outer loop", + ] + for line in expected: + assert line in rendered, "The LoopStack allows you to take "\ + "advantage of the loop variable even in embedded loops" + + def test_parent_loops(self): + template = Template("""% for x in 'ab': +${x} ${loop.index} <- outer loop +% for y in [0, 1]: +${y} ${loop.index} <- inner loop +${x} ${loop.parent.index} <- parent loop +% endfor +${x} ${loop.index} <- outer loop +% endfor""") + code = template.code + rendered = template.render() + expected = [ + "a 0 <- outer loop", + "a 0 <- parent loop", + "b 1 <- outer loop", + "b 1 <- parent loop" + ] + for line in expected: + print code + assert line in rendered, "The parent attribute of a loop gives "\ + "you the previous loop context in the stack" + + def test_out_of_context_access(self): + template = Template("""${loop.index}""") + assert_raises_message( + exceptions.RuntimeException, + "No loop context is established", + template.render + ) + +class TestLoopStack(unittest.TestCase): + + def setUp(self): + self.stack = LoopStack() + self.bottom = 'spam' + self.stack.stack = [self.bottom] + + def test_enter(self): + iterable = 'ham' + s = self.stack._enter(iterable) + assert s is self.stack.stack[-1], "Calling the stack with an iterable returns "\ + "the stack" + assert iterable == self.stack.stack[-1]._iterable, "and pushes the "\ + "iterable on the top of the stack" + + def test__top(self): + assert self.bottom == self.stack._top, "_top returns the last item "\ + "on the stack" + + def test__pop(self): + assert len(self.stack.stack) == 1 + top = self.stack._pop() + assert top == self.bottom + assert len(self.stack.stack) == 0 + + def test__push(self): + assert len(self.stack.stack) == 1 + iterable = 'ham' + self.stack._push(iterable) + assert len(self.stack.stack) == 2 + assert iterable is self.stack._top._iterable + + def test_exit(self): + iterable = 'ham' + self.stack._enter(iterable) + before = len(self.stack.stack) + self.stack._exit() + after = len(self.stack.stack) + assert before == (after + 1), "Exiting a context pops the stack" + + +class TestLoopContext(unittest.TestCase): + + def setUp(self): + self.iterable = [1, 2, 3] + self.ctx = LoopContext(self.iterable) + + def test___len__(self): + assert len(self.iterable) == len(self.ctx), "The LoopContext is the "\ + "same length as the iterable" + + def test_index(self): + expected = tuple(range(len(self.iterable))) + actual = tuple(self.ctx.index for i in self.ctx) + assert expected == actual, "The index is consistent with the current "\ + "iteration count" + + def test_reverse_index(self): + length = len(self.iterable) + expected = tuple([length-i-1 for i in range(length)]) + actual = tuple(self.ctx.reverse_index for i in self.ctx) + print expected, actual + assert expected == actual, "The reverse_index is the number of "\ + "iterations until the end" + + def test_first(self): + expected = (True, False, False) + actual = tuple(self.ctx.first for i in self.ctx) + assert expected == actual, "first is only true on the first iteration" + + def test_last(self): + expected = (False, False, True) + actual = tuple(self.ctx.last for i in self.ctx) + assert expected == actual, "last is only true on the last iteration" + + def test_even(self): + expected = (True, False, True) + actual = tuple(self.ctx.even for i in self.ctx) + assert expected == actual, "even is true on even iterations" + + def test_odd(self): + expected = (False, True, False) + actual = tuple(self.ctx.odd for i in self.ctx) + assert expected == actual, "odd is true on odd iterations" + + def test_cycle(self): + expected = ('a', 'b', 'a') + actual = tuple(self.ctx.cycle('a', 'b') for i in self.ctx) + assert expected == actual, "cycle endlessly cycles through the values" |
