path: root/mako/pyparser.py

# ast.py
# Copyright (C) Mako developers
#
# This module is part of Mako and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

"""Handles parsing of Python code.

Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
module is used.
"""

from StringIO import StringIO
from mako import exceptions, util
import operator

if util.py3k:
    # words that cannot be assigned to (notably 
    # smaller than the total keys in __builtins__)
    reserved = set(['True', 'False', 'None', 'print'])

    # the "id" attribute on a function node
    arg_id = operator.attrgetter('arg')
else:
    # words that cannot be assigned to (notably 
    # smaller than the total keys in __builtins__)
    reserved = set(['True', 'False', 'None'])
    
    # the "id" attribute on a function node
    arg_id = operator.attrgetter('id')


try:
    import _ast
    util.restore__ast(_ast)
    import _ast_util
except ImportError:
    _ast = None
    from compiler import parse as compiler_parse
    from compiler import visitor


def parse(code, mode='exec', **exception_kwargs):
    """Parse an expression into AST"""
    

    try:
        if _ast:
            return _ast_util.parse(code, '<unknown>', mode)
        else:
            if isinstance(code, unicode):
                code = code.encode('ascii', 'backslashreplace')
            return compiler_parse(code, mode)
    except Exception, e:
        raise exceptions.SyntaxException("(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), **exception_kwargs)


if _ast:
    class FindIdentifiers(_ast_util.NodeVisitor):
        def __init__(self, listener, **exception_kwargs):
            self.in_function = False
            self.in_assign_targets = False
            self.local_ident_stack = {}
            self.listener = listener
            self.exception_kwargs = exception_kwargs
        def _add_declared(self, name):
            if not self.in_function:
                self.listener.declared_identifiers.add(name)
        def visit_ClassDef(self, node):
            self._add_declared(node.name)
        def visit_Assign(self, node):
            # flip around the visiting of Assign so the expression gets evaluated first, 
            # in the case of a clause like "x=x+5" (x is undeclared)
            self.visit(node.value)
            in_a = self.in_assign_targets
            self.in_assign_targets = True
            for n in node.targets:
                self.visit(n)
            self.in_assign_targets = in_a
        
        if util.py3k:
            # ExceptHandler is in Python 2, but this
            # block only works in Python 3 (and is required there)
            def visit_ExceptHandler(self, node):
                if node.name is not None:
                    self._add_declared(node.name)
                if node.type is not None:
                    self.listener.undeclared_identifiers.add(node.type.id)
                for statement in node.body:
                    self.visit(statement)
                
        def visit_FunctionDef(self, node):
            self._add_declared(node.name)
            # push function state onto stack.  dont log any
            # more identifiers as "declared" until outside of the function,
            # but keep logging identifiers as "undeclared".
            # track argument names in each function header so they arent counted as "undeclared"
            saved = {}
            inf = self.in_function
            self.in_function = True
            
            for arg in node.args.args:
                if arg_id(arg) in self.local_ident_stack:
                    saved[arg_id(arg)] = True
                else:
                    self.local_ident_stack[arg_id(arg)] = True
            for n in node.body:
                self.visit(n)
            self.in_function = inf
            for arg in node.args.args:
                if arg_id(arg) not in saved:
                    del self.local_ident_stack[arg_id(arg)]
                    
        def visit_For(self, node):
            # flip around visit
            self.visit(node.iter)
            self.visit(node.target)
            for statement in node.body:
                self.visit(statement)
            for statement in node.orelse:
                self.visit(statement)
        def visit_Name(self, node):
            if isinstance(node.ctx, _ast.Store):
                self._add_declared(node.id)
            if node.id not in reserved and \
                        node.id not in self.listener.declared_identifiers and \
                        node.id not in self.local_ident_stack:
                self.listener.undeclared_identifiers.add(node.id)
        def visit_Import(self, node):
            for name in node.names:
                if name.asname is not None:
                    self._add_declared(name.asname)
                else:
                    self._add_declared(name.name.split('.')[0])
        def visit_ImportFrom(self, node):
            for name in node.names:
                if name.asname is not None:
                    self._add_declared(name.asname)
                else:
                    if name.name == '*':
                        raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared.  Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", **self.exception_kwargs)
                    self._add_declared(name.name)

    class FindTuple(_ast_util.NodeVisitor):
        def __init__(self, listener, code_factory, **exception_kwargs):
            self.listener = listener
            self.exception_kwargs = exception_kwargs
            self.code_factory = code_factory
        def visit_Tuple(self, node):
            for n in node.elts:
                p = self.code_factory(n, **self.exception_kwargs)
                self.listener.codeargs.append(p)
                self.listener.args.append(ExpressionGenerator(n).value())
                self.listener.declared_identifiers = self.listener.declared_identifiers.union(p.declared_identifiers)
                self.listener.undeclared_identifiers = self.listener.undeclared_identifiers.union(p.undeclared_identifiers)

    class ParseFunc(_ast_util.NodeVisitor):
        def __init__(self, listener, **exception_kwargs):
            self.listener = listener
            self.exception_kwargs = exception_kwargs
            
        def visit_FunctionDef(self, node):
            self.listener.funcname = node.name
            argnames = [arg_id(arg) for arg in node.args.args]
            if node.args.vararg:
                argnames.append(node.args.vararg)
            if node.args.kwarg:
                argnames.append(node.args.kwarg)
            self.listener.argnames = argnames
            self.listener.defaults = node.args.defaults # ast
            self.listener.varargs = node.args.vararg
            self.listener.kwargs = node.args.kwarg

    class ExpressionGenerator(object):
        def __init__(self, astnode):
            self.generator = _ast_util.SourceGenerator(' ' * 4)
            self.generator.visit(astnode)
        def value(self):
            return ''.join(self.generator.result)
else:
    class FindIdentifiers(object):
        def __init__(self, listener, **exception_kwargs):
            self.in_function = False
            self.local_ident_stack = {}
            self.listener = listener
            self.exception_kwargs = exception_kwargs
        def _add_declared(self, name):
            if not self.in_function:
                self.listener.declared_identifiers.add(name)
        def visitClass(self, node, *args):
            self._add_declared(node.name)
        def visitAssName(self, node, *args):
            self._add_declared(node.name)
        def visitAssign(self, node, *args):
            # flip around the visiting of Assign so the expression gets evaluated first, 
            # in the case of a clause like "x=x+5" (x is undeclared)
            self.visit(node.expr, *args)
            for n in node.nodes:
                self.visit(n, *args)
        def visitFunction(self,node, *args):
            self._add_declared(node.name)
            # push function state onto stack.  dont log any
            # more identifiers as "declared" until outside of the function,
            # but keep logging identifiers as "undeclared".
            # track argument names in each function header so they arent counted as "undeclared"
            saved = {}
            inf = self.in_function
            self.in_function = True
            for arg in node.argnames:
                if arg in self.local_ident_stack:
                    saved[arg] = True
                else:
                    self.local_ident_stack[arg] = True
            for n in node.getChildNodes():
                self.visit(n, *args)
            self.in_function = inf
            for arg in node.argnames:
                if arg not in saved:
                    del self.local_ident_stack[arg]
        def visitFor(self, node, *args):
            # flip around visit
            self.visit(node.list, *args)
            self.visit(node.assign, *args)
            self.visit(node.body, *args)
        def visitName(self, node, *args):
            if node.name not in reserved and node.name not in self.listener.declared_identifiers and node.name not in self.local_ident_stack:
                self.listener.undeclared_identifiers.add(node.name)
        def visitImport(self, node, *args):
            for (mod, alias) in node.names:
                if alias is not None:
                    self._add_declared(alias)
                else:
                    self._add_declared(mod.split('.')[0])
        def visitFrom(self, node, *args):
            for (mod, alias) in node.names:
                if alias is not None:
                    self._add_declared(alias)
                else:
                    if mod == '*':
                        raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared.  Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", **self.exception_kwargs)
                    self._add_declared(mod)
        def visit(self, expr):
            visitor.walk(expr, self) #, walker=walker())

    class FindTuple(object):
        def __init__(self, listener, code_factory, **exception_kwargs):
            self.listener = listener
            self.exception_kwargs = exception_kwargs
            self.code_factory = code_factory
        def visitTuple(self, node, *args):
            for n in node.nodes:
                p = self.code_factory(n, **self.exception_kwargs)
                self.listener.codeargs.append(p)
                self.listener.args.append(ExpressionGenerator(n).value())
                self.listener.declared_identifiers = self.listener.declared_identifiers.union(p.declared_identifiers)
                self.listener.undeclared_identifiers = self.listener.undeclared_identifiers.union(p.undeclared_identifiers)
        def visit(self, expr):
            visitor.walk(expr, self) #, walker=walker())

    class ParseFunc(object):
        def __init__(self, listener, **exception_kwargs):
            self.listener = listener
            self.exception_kwargs = exception_kwargs
        def visitFunction(self, node, *args):
            self.listener.funcname = node.name
            self.listener.argnames = node.argnames
            self.listener.defaults = node.defaults
            self.listener.varargs = node.varargs
            self.listener.kwargs = node.kwargs
        def visit(self, expr):
            visitor.walk(expr, self)

    class ExpressionGenerator(object):
        """given an AST node, generates an equivalent literal Python expression."""
        def __init__(self, astnode):
            self.buf = StringIO()
            visitor.walk(astnode, self) #, walker=walker())
        def value(self):
            return self.buf.getvalue()        
        def operator(self, op, node, *args):
            self.buf.write("(")
            self.visit(node.left, *args)
            self.buf.write(" %s " % op)
            self.visit(node.right, *args)
            self.buf.write(")")
        def booleanop(self, op, node, *args):
            self.visit(node.nodes[0])
            for n in node.nodes[1:]:
                self.buf.write(" " + op + " ")
                self.visit(n, *args)
        def visitConst(self, node, *args):
            self.buf.write(repr(node.value))
        def visitAssName(self, node, *args):
            # TODO: figure out OP_ASSIGN, other OP_s
            self.buf.write(node.name)
        def visitName(self, node, *args):
            self.buf.write(node.name)
        def visitMul(self, node, *args):
            self.operator("*", node, *args)
        def visitAnd(self, node, *args):
            self.booleanop("and", node, *args)
        def visitOr(self, node, *args):
            self.booleanop("or", node, *args)
        def visitBitand(self, node, *args):
            self.booleanop("&", node, *args)
        def visitBitor(self, node, *args):
            self.booleanop("|", node, *args)
        def visitBitxor(self, node, *args):
            self.booleanop("^", node, *args)
        def visitAdd(self, node, *args):
            self.operator("+", node, *args)
        def visitGetattr(self, node, *args):
            self.visit(node.expr, *args)
            self.buf.write(".%s" % node.attrname)
        def visitSub(self, node, *args):
            self.operator("-", node, *args)
        def visitNot(self, node, *args):
            self.buf.write("not ")
            self.visit(node.expr)
        def visitDiv(self, node, *args):
            self.operator("/", node, *args)
        def visitFloorDiv(self, node, *args):
            self.operator("//", node, *args)
        def visitSubscript(self, node, *args):
            self.visit(node.expr)
            self.buf.write("[")
            [self.visit(x) for x in node.subs]
            self.buf.write("]")
        def visitUnarySub(self, node, *args):
            self.buf.write("-")
            self.visit(node.expr)
        def visitUnaryAdd(self, node, *args):
            self.buf.write("-")
            self.visit(node.expr)
        def visitSlice(self, node, *args):
            self.visit(node.expr)
            self.buf.write("[")
            if node.lower is not None:
                self.visit(node.lower)
            self.buf.write(":")
            if node.upper is not None:
                self.visit(node.upper)
            self.buf.write("]")
        def visitDict(self, node):
            self.buf.write("{")
            c = node.getChildren()
            for i in range(0, len(c), 2):
                self.visit(c[i])
                self.buf.write(": ")
                self.visit(c[i+1])
                if i<len(c) -2:
                    self.buf.write(", ")
            self.buf.write("}")
        def visitTuple(self, node):
            self.buf.write("(")
            c = node.getChildren()
            for i in range(0, len(c)):
                self.visit(c[i])
                if i<len(c) - 1:
                    self.buf.write(", ")
            self.buf.write(")")
        def visitList(self, node):
            self.buf.write("[")
            c = node.getChildren()
            for i in range(0, len(c)):
                self.visit(c[i])
                if i<len(c) - 1:
                    self.buf.write(", ")
            self.buf.write("]")
        def visitListComp(self, node):
            self.buf.write("[")
            self.visit(node.expr)
            self.buf.write(" ")
            for n in node.quals:
                self.visit(n)
            self.buf.write("]")
        def visitListCompFor(self, node):
            self.buf.write(" for ")
            self.visit(node.assign)
            self.buf.write(" in ")
            self.visit(node.list)
            for n in node.ifs:
                self.visit(n)
        def visitListCompIf(self, node):
            self.buf.write(" if ")
            self.visit(node.test)
        def visitCompare(self, node):
            self.visit(node.expr)
            for tup in node.ops:
                self.buf.write(tup[0])
                self.visit(tup[1])
        def visitCallFunc(self, node, *args):
            self.visit(node.node)
            self.buf.write("(")
            if len(node.args):
                self.visit(node.args[0])
                for a in node.args[1:]:
                    self.buf.write(", ")
                    self.visit(a)
            self.buf.write(")")

    class walker(visitor.ASTVisitor):
        def dispatch(self, node, *args):
            print "Node:", str(node)
            #print "dir:", dir(node)
            return visitor.ASTVisitor.dispatch(self, node, *args)