diff options
| author | Roman Imankulov <roman.imankulov@gmail.com> | 2015-10-13 11:44:18 +0000 |
|---|---|---|
| committer | Roman Imankulov <roman.imankulov@gmail.com> | 2015-10-13 16:26:07 +0000 |
| commit | d838e4496bb573c252253722401e4f05f55db0e6 (patch) | |
| tree | 2c2e6b5cd25542b3160a20a9ac9c926f1a30dcf6 /mako | |
| parent | 72e95faf46665753247796df7403c3b49bfe092d (diff) | |
| download | external_python_mako-d838e4496bb573c252253722401e4f05f55db0e6.tar.gz external_python_mako-d838e4496bb573c252253722401e4f05f55db0e6.tar.bz2 external_python_mako-d838e4496bb573c252253722401e4f05f55db0e6.zip | |
Ensure babel i18n extactor works properly with non-ascii input
If mako templates contain something like "_('Köln')", babel extractor converts
it to pure ASCII so that resulting .po file would contain "K\xf6ln". Not all
translation tools and translations are ready for such kind of escape sequences.
Babel allows message ids to be non-ascii, the plugin just has to return Unicode
objects instead of ASCII strings (and that's exactly how Babel built-in Python
and JavaScript extractors work).
This fix ensures mako extractor doesn't excape non-ascii symbols, works well
both for Unicode and non-unicode input (there is a test for cp1251 encoding),
and also provides a workaround for babel charset detector python-babel/babel#274.
Diffstat (limited to 'mako')
| -rw-r--r-- | mako/ext/extract.py | 12 |
1 files changed, 9 insertions, 3 deletions
diff --git a/mako/ext/extract.py b/mako/ext/extract.py index 313c088..8dd2e96 100644 --- a/mako/ext/extract.py +++ b/mako/ext/extract.py @@ -16,6 +16,7 @@ class MessageExtractor(object): def extract_nodes(self, nodes): translator_comments = [] in_translator_comments = False + input_encoding = self.config['encoding'] or 'ascii' comment_tags = list( filter(None, re.split(r'\s+', self.config['comment-tags']))) @@ -76,13 +77,18 @@ class MessageExtractor(object): comment[1] for comment in translator_comments] if isinstance(code, compat.text_type): - code = code.encode('ascii', 'backslashreplace') + code = code.encode(input_encoding, 'backslashreplace') used_translator_comments = False - code = compat.byte_buffer(code) + # We add extra newline to work around a pybabel bug + # (see python-babel/babel#274, parse_encoding dies if the first + # input string of the input is non-ascii) + # Also, because we added it, we have to subtract one from + # node.lineno + code = compat.byte_buffer(compat.b('\n') + code) for message in self.process_python( - code, node.lineno, translator_strings): + code, node.lineno - 1, translator_strings): yield message used_translator_comments = True |