diff options
Diffstat (limited to 'mako/lexer.py')
| -rw-r--r-- | mako/lexer.py | 148 |
1 files changed, 74 insertions, 74 deletions
diff --git a/mako/lexer.py b/mako/lexer.py index a445f47..267c0d1 100644 --- a/mako/lexer.py +++ b/mako/lexer.py @@ -13,8 +13,8 @@ from mako.pygen import adjust_whitespace _regexp_cache = {} class Lexer(object): - def __init__(self, text, filename=None, - disable_unicode=False, + def __init__(self, text, filename=None, + disable_unicode=False, input_encoding=None, preprocessor=None): self.text = text self.filename = filename @@ -28,29 +28,29 @@ class Lexer(object): self.ternary_stack = [] self.disable_unicode = disable_unicode self.encoding = input_encoding - + if util.py3k and disable_unicode: raise exceptions.UnsupportedError( "Mako for Python 3 does not " "support disabling Unicode") - + if preprocessor is None: self.preprocessor = [] elif not hasattr(preprocessor, '__iter__'): self.preprocessor = [preprocessor] else: self.preprocessor = preprocessor - + @property def exception_kwargs(self): - return {'source':self.text, - 'lineno':self.matched_lineno, - 'pos':self.matched_charpos, + return {'source':self.text, + 'lineno':self.matched_lineno, + 'pos':self.matched_charpos, 'filename':self.filename} - + def match(self, regexp, flags=None): """compile the given regexp, cache the reg, and call match_reg().""" - + try: reg = _regexp_cache[(regexp, flags)] except KeyError: @@ -59,15 +59,15 @@ class Lexer(object): else: reg = re.compile(regexp) _regexp_cache[(regexp, flags)] = reg - + return self.match_reg(reg) - + def match_reg(self, reg): """match the given regular expression object to the current text position. - + if a match occurs, update the current text and line position. - + """ mp = self.match_position @@ -86,12 +86,12 @@ class Lexer(object): cp -=1 self.matched_charpos = mp - cp self.lineno += len(lines) - #print "MATCHED:", match.group(0), "LINE START:", + #print "MATCHED:", match.group(0), "LINE START:", # self.matched_lineno, "LINE END:", self.lineno #print "MATCH:", regexp, "\n", self.text[mp : mp + 15], \ # (match and "TRUE" or "FALSE") return match - + def parse_until_text(self, *text): startpos = self.match_position text_re = r'|'.join(text) @@ -119,10 +119,10 @@ class Lexer(object): brace_level -= match.group(1).count('}') continue raise exceptions.SyntaxException( - "Expected: %s" % - ','.join(text), + "Expected: %s" % + ','.join(text), **self.exception_kwargs) - + def append_node(self, nodecls, *args, **kwargs): kwargs.setdefault('source', self.text) kwargs.setdefault('lineno', self.matched_lineno) @@ -185,8 +185,8 @@ class Lexer(object): if m is not None and m.group(1) != 'utf-8': raise exceptions.CompileException( "Found utf-8 BOM in file, with conflicting " - "magic encoding comment of '%s'" % m.group(1), - text.decode('utf-8', 'ignore'), + "magic encoding comment of '%s'" % m.group(1), + text.decode('utf-8', 'ignore'), 0, 0, filename) else: m = self._coding_re.match(text.decode('utf-8', 'ignore')) @@ -201,31 +201,31 @@ class Lexer(object): except UnicodeDecodeError, e: raise exceptions.CompileException( "Unicode decode operation of encoding '%s' failed" % - parsed_encoding, - text.decode('utf-8', 'ignore'), + parsed_encoding, + text.decode('utf-8', 'ignore'), 0, 0, filename) return parsed_encoding, text def parse(self): - self.encoding, self.text = self.decode_raw_stream(self.text, - not self.disable_unicode, + self.encoding, self.text = self.decode_raw_stream(self.text, + not self.disable_unicode, self.encoding, self.filename,) for preproc in self.preprocessor: self.text = preproc(self.text) - - # push the match marker past the + + # push the match marker past the # encoding comment. self.match_reg(self._coding_re) - + self.textlength = len(self.text) - + while (True): - if self.match_position > self.textlength: + if self.match_position > self.textlength: break - + if self.match_end(): break if self.match_expression(): @@ -234,28 +234,28 @@ class Lexer(object): continue if self.match_comment(): continue - if self.match_tag_start(): + if self.match_tag_start(): continue if self.match_tag_end(): continue if self.match_python_block(): continue - if self.match_text(): + if self.match_text(): continue - - if self.match_position > self.textlength: + + if self.match_position > self.textlength: break raise exceptions.CompileException("assertion failed") - + if len(self.tag): - raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % - self.tag[-1].keyword, + raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % + self.tag[-1].keyword, **self.exception_kwargs) if len(self.control_line): raise exceptions.SyntaxException( "Unterminated control keyword: '%s'" % - self.control_line[-1].keyword, - self.text, + self.control_line[-1].keyword, + self.text, self.control_line[-1].lineno, self.control_line[-1].pos, self.filename) return self.template @@ -263,20 +263,20 @@ class Lexer(object): def match_tag_start(self): match = self.match(r''' \<% # opening tag - + ([\w\.\:]+) # keyword - + ((?:\s+\w+|\s*=\s*|".*?"|'.*?')*) # attrname, = \ # sign, string expression - + \s* # more whitespace - + (/)?> # closing - - ''', - + + ''', + re.I | re.S | re.X) - + if match: keyword, attr, isend = match.groups() self.keyword = keyword @@ -296,22 +296,22 @@ class Lexer(object): match = self.match(r'(.*?)(?=\</%text>)', re.S) if not match: raise exceptions.SyntaxException( - "Unclosed tag: <%%%s>" % - self.tag[-1].keyword, + "Unclosed tag: <%%%s>" % + self.tag[-1].keyword, **self.exception_kwargs) self.append_node(parsetree.Text, match.group(1)) return self.match_tag_end() return True - else: + else: return False - + def match_tag_end(self): match = self.match(r'\</%[\t ]*(.+?)[\t ]*>') if match: if not len(self.tag): raise exceptions.SyntaxException( "Closing tag without opening tag: </%%%s>" % - match.group(1), + match.group(1), **self.exception_kwargs) elif self.tag[-1].keyword != match.group(1): raise exceptions.SyntaxException( @@ -322,7 +322,7 @@ class Lexer(object): return True else: return False - + def match_end(self): match = self.match(r'\Z', re.S) if match: @@ -333,13 +333,13 @@ class Lexer(object): return True else: return False - + def match_text(self): match = self.match(r""" (.*?) # anything, followed by: ( - (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based - # comment preceded by a + (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based + # comment preceded by a # consumed newline and whitespace | (?=\${) # an expression @@ -353,7 +353,7 @@ class Lexer(object): | \Z # end of string )""", re.X | re.S) - + if match: text = match.group(1) if text: @@ -361,23 +361,23 @@ class Lexer(object): return True else: return False - + def match_python_block(self): match = self.match(r"<%(!)?") if match: line, pos = self.matched_lineno, self.matched_charpos text, end = self.parse_until_text(r'%>') - # the trailing newline helps + # the trailing newline helps # compiler.parse() not complain about indentation - text = adjust_whitespace(text) + "\n" + text = adjust_whitespace(text) + "\n" self.append_node( - parsetree.Code, - text, + parsetree.Code, + text, match.group(1)=='!', lineno=line, pos=pos) return True else: return False - + def match_expression(self): match = self.match(r"\${") if match: @@ -389,8 +389,8 @@ class Lexer(object): escapes = "" text = text.replace('\r\n', '\n') self.append_node( - parsetree.Expression, - text, escapes.strip(), + parsetree.Expression, + text, escapes.strip(), lineno=line, pos=pos) return True else: @@ -407,22 +407,22 @@ class Lexer(object): m2 = re.match(r'(end)?(\w+)\s*(.*)', text) if not m2: raise exceptions.SyntaxException( - "Invalid control line: '%s'" % - text, + "Invalid control line: '%s'" % + text, **self.exception_kwargs) isend, keyword = m2.group(1, 2) isend = (isend is not None) - + if isend: if not len(self.control_line): raise exceptions.SyntaxException( - "No starting keyword '%s' for '%s'" % - (keyword, text), + "No starting keyword '%s' for '%s'" % + (keyword, text), **self.exception_kwargs) elif self.control_line[-1].keyword != keyword: raise exceptions.SyntaxException( - "Keyword '%s' doesn't match keyword '%s'" % - (text, self.control_line[-1].keyword), + "Keyword '%s' doesn't match keyword '%s'" % + (text, self.control_line[-1].keyword), **self.exception_kwargs) self.append_node(parsetree.ControlLine, keyword, isend, text) else: @@ -439,4 +439,4 @@ class Lexer(object): return True else: return False - + |