From ca21a5353a4c88f3477a53f2f45cd9ca280b0479 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 10 Nov 2010 12:09:30 -0500 Subject: - The range of Python identifiers that are considered "undefined", meaning they are pulled from the context, has been trimmed back to not include variables declared inside of expressions (i.e. from list comprehensions), as well as in the argument list of lambdas. This to better support the strict_undefined feature. --- CHANGES | 14 ++++++++ mako/parsetree.py | 16 ++++++--- mako/pyparser.py | 34 ++++++++++++++---- test/test_ast.py | 32 +++++++++++++++++ test/test_template.py | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 10 deletions(-) diff --git a/CHANGES b/CHANGES index ecc9430..d25f1b9 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,20 @@ variables not found in the context to raise a NameError immediately, instead of defaulting to the UNDEFINED value. + +- The range of Python identifiers that + are considered "undefined", meaning they + are pulled from the context, has been + trimmed back to not include variables + declared inside of expressions (i.e. from + list comprehensions), as well as + in the argument list of lambdas. This + to better support the strict_undefined + feature. The change should be + fully backwards-compatible but involved + a little bit of tinkering in the AST code, + which hadn't really been touched for + a couple of years, just FYI. 0.3.5 - The <%namespace> tag allows expressions diff --git a/mako/parsetree.py b/mako/parsetree.py index 12b0498..554531e 100644 --- a/mako/parsetree.py +++ b/mako/parsetree.py @@ -176,7 +176,7 @@ class Expression(Node): self.escapes_code.undeclared_identifiers.difference( set(filters.DEFAULT_ESCAPES.keys()) ) - ) + ).difference(self.code.declared_identifiers) def __repr__(self): return "Expression(%r, %r, %r)" % ( @@ -274,6 +274,10 @@ class Tag(Node): if m: code = ast.PythonCode(m.group(1).rstrip(), **self.exception_kwargs) + # we aren't discarding "declared_identifiers" here, + # which we do so that list comprehension-declared variables + # aren't counted. As yet can't find a condition that + # requires it here. undeclared_identifiers = \ undeclared_identifiers.union( code.undeclared_identifiers) @@ -327,7 +331,9 @@ class IncludeTag(Tag): return [] def undeclared_identifiers(self): - identifiers = self.page_args.undeclared_identifiers.difference(set(["__DUMMY"])) + identifiers = self.page_args.undeclared_identifiers.\ + difference(set(["__DUMMY"])).\ + difference(self.page_args.declared_identifiers) return identifiers.union(super(IncludeTag, self).undeclared_identifiers()) class NamespaceTag(Tag): @@ -414,7 +420,8 @@ class CallTag(Tag): return self.code.declared_identifiers.union(self.body_decl.argnames) def undeclared_identifiers(self): - return self.code.undeclared_identifiers + return self.code.undeclared_identifiers.\ + difference(self.code.declared_identifiers) class CallNamespaceTag(Tag): @@ -443,7 +450,8 @@ class CallNamespaceTag(Tag): return self.code.declared_identifiers.union(self.body_decl.argnames) def undeclared_identifiers(self): - return self.code.undeclared_identifiers + return self.code.undeclared_identifiers.\ + difference(self.code.declared_identifiers) class InheritTag(Tag): __keyword__ = 'inherit' diff --git a/mako/pyparser.py b/mako/pyparser.py index b90278e..d011690 100644 --- a/mako/pyparser.py +++ b/mako/pyparser.py @@ -89,12 +89,19 @@ if _ast: for statement in node.body: self.visit(statement) + def visit_Lambda(self, node, *args): + self._visit_function(node, True) + def visit_FunctionDef(self, node): self._add_declared(node.name) + self._visit_function(node, False) + + def _visit_function(self, node, islambda): # push function state onto stack. dont log any # more identifiers as "declared" until outside of the function, # but keep logging identifiers as "undeclared". - # track argument names in each function header so they arent counted as "undeclared" + # track argument names in each function header + # so they arent counted as "undeclared" saved = {} inf = self.in_function self.in_function = True @@ -104,13 +111,16 @@ if _ast: saved[arg_id(arg)] = True else: self.local_ident_stack[arg_id(arg)] = True - for n in node.body: - self.visit(n) + if islambda: + self.visit(node.body) + else: + for n in node.body: + self.visit(n) self.in_function = inf for arg in node.args.args: if arg_id(arg) not in saved: del self.local_ident_stack[arg_id(arg)] - + def visit_For(self, node): # flip around visit self.visit(node.iter) @@ -138,7 +148,13 @@ if _ast: self._add_declared(name.asname) else: if name.name == '*': - raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared. Please use the form 'from import , , ...' instead.", **self.exception_kwargs) + raise exceptions.CompileException( + "'import *' is not supported, since all " + "identifier names must be explicitly " + "declared. Please use the form 'from " + " import , " + ", ...' instead.", + **self.exception_kwargs) self._add_declared(name.name) class FindTuple(_ast_util.NodeVisitor): @@ -197,12 +213,17 @@ else: self.visit(node.expr, *args) for n in node.nodes: self.visit(n, *args) + def visitLambda(self, node, *args): + self._visit_function(node, args) def visitFunction(self,node, *args): self._add_declared(node.name) + self._visit_function(node, args) + def _visit_function(self, node, args): # push function state onto stack. dont log any # more identifiers as "declared" until outside of the function, # but keep logging identifiers as "undeclared". - # track argument names in each function header so they arent counted as "undeclared" + # track argument names in each function header so + # they arent counted as "undeclared" saved = {} inf = self.in_function self.in_function = True @@ -217,6 +238,7 @@ else: for arg in node.argnames: if arg not in saved: del self.local_ident_stack[arg] + def visitFor(self, node, *args): # flip around visit self.visit(node.list, *args) diff --git a/test/test_ast.py b/test/test_ast.py index bfdfd90..b9fe948 100644 --- a/test/test_ast.py +++ b/test/test_ast.py @@ -156,7 +156,39 @@ class Hi(object): parsed = ast.PythonCode(code, **exception_kwargs) assert parsed.declared_identifiers == set(['Hi']) assert parsed.undeclared_identifiers == set() + + def test_locate_identifiers_9(self): + code = """ + ",".join([t for t in ("a", "b", "c")]) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + assert parsed.declared_identifiers == set(['t']) + assert parsed.undeclared_identifiers == set(['t']) + + code = """ + [(val, name) for val, name in x] +""" + parsed = ast.PythonCode(code, **exception_kwargs) + assert parsed.declared_identifiers == set(['val', 'name']) + assert parsed.undeclared_identifiers == set(['val', 'name', 'x']) + def test_locate_identifiers_10(self): + code = """ +lambda q: q + 5 +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, set()) + eq_(parsed.undeclared_identifiers, set()) + + def test_locate_identifiers_11(self): + code = """ +def x(q): + return q + 5 +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, set(['x'])) + eq_(parsed.undeclared_identifiers, set()) + def test_no_global_imports(self): code = """ from foo import * diff --git a/test/test_template.py b/test/test_template.py index ddf746a..dbfd068 100644 --- a/test/test_template.py +++ b/test/test_template.py @@ -591,6 +591,104 @@ class UndefinedVarsTest(TemplateTest): t.render, y=12 ) + def test_expression_declared(self): + t = Template(""" + ${",".join([t for t in ("a", "b", "c")])} + """, strict_undefined=True) + + eq_(result_lines(t.render()), ['a,b,c']) + + t = Template(""" + <%self:foo value="${[(val, n) for val, n in [(1, 2)]]}"/> + + <%def name="foo(value)"> + ${value} + + + """, strict_undefined=True) + + eq_(result_lines(t.render()), ['[(1, 2)]']) + + t = Template(""" + <%call expr="foo(value=[(val, n) for val, n in [(1, 2)]])" /> + + <%def name="foo(value)"> + ${value} + + + """, strict_undefined=True) + + eq_(result_lines(t.render()), ['[(1, 2)]']) + + l = TemplateLookup(strict_undefined=True) + l.put_string("i", "hi, ${pageargs['y']}") + l.put_string("t", """ + <%include file="i" args="y=[x for x in range(3)]" /> + """) + eq_( + result_lines(l.get_template("t").render()), ['hi, [0, 1, 2]'] + ) + + l.put_string('q', """ + <%namespace name="i" file="${(str([x for x in range(3)][2]) + 'i')[-1]}" /> + ${i.body(y='x')} + """) + eq_( + result_lines(l.get_template("q").render()), ['hi, x'] + ) + + t = Template(""" + <% + y = lambda q: str(q) + %> + ${y('hi')} + """, strict_undefined=True) + eq_( + result_lines(t.render()), ["hi"] + ) + + def test_list_comprehensions_plus_undeclared_nonstrict(self): + # traditional behavior. variable inside a list comprehension + # is treated as an "undefined", so is pulled from the context. + t = Template(""" + t is: ${t} + + ${",".join([t for t in ("a", "b", "c")])} + """) + + eq_( + result_lines(t.render(t="T")), + ['t is: T', 'a,b,c'] + ) + + def test_traditional_assignment_plus_undeclared(self): + t = Template(""" + t is: ${t} + + <% + t = 12 + %> + """) + assert_raises( + UnboundLocalError, + t.render, t="T" + ) + + def test_list_comprehensions_plus_undeclared_strict(self): + # with strict, a list comprehension now behaves + # like the undeclared case above. + t = Template(""" + t is: ${t} + + ${",".join([t for t in ("a", "b", "c")])} + """, strict_undefined=True) + + eq_( + result_lines(t.render(t="T")), + ['t is: T', 'a,b,c'] + ) + + class ControlTest(TemplateTest): def test_control(self): t = Template(""" -- cgit v1.2.3