add markerlib as _markerlib

--HG--
branch : distribute
extra : rebase_source : b9d8fa81db6c6fc3d89db54a70778eb3e8396e17

6 files changed, 342 insertions, 7 deletions

diff --git a/_markerlib/__init__.py b/_markerlib/__init__.py
new file mode 100644
index 00000000..ef8c994c
--- /dev/null
+++ b/_markerlib/__init__.py
@@ -0,0 +1 @@
+from _markerlib.markers import default_environment, compile, interpret, as_function
diff --git a/_markerlib/_markers_ast.py b/_markerlib/_markers_ast.py
new file mode 100644
index 00000000..b7a5e0b9
--- /dev/null
+++ b/_markerlib/_markers_ast.py
@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+"""
+Just enough of ast.py for markers.py
+"""
+
+from _ast import AST, PyCF_ONLY_AST
+
+def parse(source, filename='<unknown>', mode='exec'):
+    """
+    Parse the source into an AST node.
+    Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
+    """
+    return compile(source, filename, mode, PyCF_ONLY_AST)
+
+def copy_location(new_node, old_node):
+    """
+    Copy source location (`lineno` and `col_offset` attributes) from
+    *old_node* to *new_node* if possible, and return *new_node*.
+    """
+    for attr in 'lineno', 'col_offset':
+        if attr in old_node._attributes and attr in new_node._attributes \
+           and hasattr(old_node, attr):
+            setattr(new_node, attr, getattr(old_node, attr))
+    return new_node
+
+def iter_fields(node):
+    """
+    Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields``
+    that is present on *node*.
+    """
+    for field in node._fields:
+        try:
+            yield field, getattr(node, field)
+        except AttributeError:
+            pass
+
+class NodeVisitor(object):
+    """
+    A node visitor base class that walks the abstract syntax tree and calls a
+    visitor function for every node found.  This function may return a value
+    which is forwarded by the `visit` method.
+
+    This class is meant to be subclassed, with the subclass adding visitor
+    methods.
+
+    Per default the visitor functions for the nodes are ``'visit_'`` +
+    class name of the node.  So a `TryFinally` node visit function would
+    be `visit_TryFinally`.  This behavior can be changed by overriding
+    the `visit` method.  If no visitor function exists for a node
+    (return value `None`) the `generic_visit` visitor is used instead.
+
+    Don't use the `NodeVisitor` if you want to apply changes to nodes during
+    traversing.  For this a special visitor exists (`NodeTransformer`) that
+    allows modifications.
+    """
+
+    def visit(self, node):
+        """Visit a node."""
+        method = 'visit_' + node.__class__.__name__
+        visitor = getattr(self, method, self.generic_visit)
+        return visitor(node)
+
+#    def generic_visit(self, node):
+#        """Called if no explicit visitor function exists for a node."""
+#        for field, value in iter_fields(node):
+#            if isinstance(value, list):
+#                for item in value:
+#                    if isinstance(item, AST):
+#                        self.visit(item)
+#            elif isinstance(value, AST):
+#                self.visit(value)
+
+
+class NodeTransformer(NodeVisitor):
+    """
+    A :class:`NodeVisitor` subclass that walks the abstract syntax tree and
+    allows modification of nodes.
+
+    The `NodeTransformer` will walk the AST and use the return value of the
+    visitor methods to replace or remove the old node.  If the return value of
+    the visitor method is ``None``, the node will be removed from its location,
+    otherwise it is replaced with the return value.  The return value may be the
+    original node in which case no replacement takes place.
+
+    Here is an example transformer that rewrites all occurrences of name lookups
+    (``foo``) to ``data['foo']``::
+
+       class RewriteName(NodeTransformer):
+
+           def visit_Name(self, node):
+               return copy_location(Subscript(
+                   value=Name(id='data', ctx=Load()),
+                   slice=Index(value=Str(s=node.id)),
+                   ctx=node.ctx
+               ), node)
+
+    Keep in mind that if the node you're operating on has child nodes you must
+    either transform the child nodes yourself or call the :meth:`generic_visit`
+    method for the node first.
+
+    For nodes that were part of a collection of statements (that applies to all
+    statement nodes), the visitor may also return a list of nodes rather than
+    just a single node.
+
+    Usually you use the transformer like this::
+
+       node = YourTransformer().visit(node)
+    """
+
+    def generic_visit(self, node):
+        for field, old_value in iter_fields(node):
+            old_value = getattr(node, field, None)
+            if isinstance(old_value, list):
+                new_values = []
+                for value in old_value:
+                    if isinstance(value, AST):
+                        value = self.visit(value)
+                        if value is None:
+                            continue
+                        elif not isinstance(value, AST):
+                            new_values.extend(value)
+                            continue
+                    new_values.append(value)
+                old_value[:] = new_values
+            elif isinstance(old_value, AST):
+                new_node = self.visit(old_value)
+                if new_node is None:
+                    delattr(node, field)
+                else:
+                    setattr(node, field, new_node)
+        return node
+    
\ No newline at end of file
diff --git a/_markerlib/markers.py b/_markerlib/markers.py
new file mode 100644
index 00000000..293adf72
--- /dev/null
+++ b/_markerlib/markers.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+"""Interpret PEP 345 environment markers.
+
+EXPR [in|==|!=|not in] EXPR [or|and] ...
+
+where EXPR belongs to any of those:
+
+    python_version = '%s.%s' % (sys.version_info[0], sys.version_info[1])
+    python_full_version = sys.version.split()[0]
+    os.name = os.name
+    sys.platform = sys.platform
+    platform.version = platform.version()
+    platform.machine = platform.machine()
+    platform.python_implementation = platform.python_implementation()
+    a free string, like '2.4', or 'win32'
+"""
+
+__all__ = ['default_environment', 'compile', 'interpret']
+
+# Would import from ast but for Python 2.5
+from _ast import Compare, BoolOp, Attribute, Name, Load, Str, cmpop, boolop
+try:
+    from ast import parse, copy_location, NodeTransformer
+except ImportError: # pragma no coverage
+    from markerlib._markers_ast import parse, copy_location, NodeTransformer
+
+import os
+import platform
+import sys
+import weakref
+
+_builtin_compile = compile
+
+from platform import python_implementation
+
+# restricted set of variables
+_VARS = {'sys.platform': sys.platform,
+         'python_version': '%s.%s' % sys.version_info[:2],
+         # FIXME parsing sys.platform is not reliable, but there is no other
+         # way to get e.g. 2.7.2+, and the PEP is defined with sys.version
+         'python_full_version': sys.version.split(' ', 1)[0],
+         'os.name': os.name,
+         'platform.version': platform.version(),
+         'platform.machine': platform.machine(),
+         'platform.python_implementation': python_implementation(),
+         'extra': None # wheel extension
+        }
+
+def default_environment():
+    """Return copy of default PEP 385 globals dictionary."""
+    return dict(_VARS)
+
+class ASTWhitelist(NodeTransformer):
+    def __init__(self, statement):
+        self.statement = statement # for error messages
+    
+    ALLOWED = (Compare, BoolOp, Attribute, Name, Load, Str, cmpop, boolop)
+    
+    def visit(self, node):
+        """Ensure statement only contains allowed nodes."""
+        if not isinstance(node, self.ALLOWED):
+            raise SyntaxError('Not allowed in environment markers.\n%s\n%s' %
+                               (self.statement, 
+                               (' ' * node.col_offset) + '^'))
+        return NodeTransformer.visit(self, node)
+    
+    def visit_Attribute(self, node):
+        """Flatten one level of attribute access."""
+        new_node = Name("%s.%s" % (node.value.id, node.attr), node.ctx)
+        return copy_location(new_node, node)
+
+def parse_marker(marker):
+    tree = parse(marker, mode='eval')
+    new_tree = ASTWhitelist(marker).generic_visit(tree)
+    return new_tree
+
+def compile_marker(parsed_marker):
+    return _builtin_compile(parsed_marker, '<environment marker>', 'eval',
+                   dont_inherit=True)
+
+_cache = weakref.WeakValueDictionary()
+
+def compile(marker):
+    """Return compiled marker as a function accepting an environment dict."""
+    try:
+        return _cache[marker]
+    except KeyError:
+        pass
+    if not marker.strip():
+        def marker_fn(environment=None, override=None):
+            """"""
+            return True
+    else:
+        compiled_marker = compile_marker(parse_marker(marker))
+        def marker_fn(environment=None, override=None):
+            """override updates environment"""
+            if override is None:
+                override = {}
+            if environment is None:
+                environment = default_environment()
+            environment.update(override)
+            return eval(compiled_marker, environment)
+    marker_fn.__doc__ = marker
+    _cache[marker] = marker_fn
+    return _cache[marker]
+
+as_function = compile # bw compat
+
+def interpret(marker, environment=None):
+    return compile(marker)(environment)
diff --git a/_markerlib/test_markerlib.py b/_markerlib/test_markerlib.py
new file mode 100644
index 00000000..ff78d672
--- /dev/null
+++ b/_markerlib/test_markerlib.py
@@ -0,0 +1,90 @@
+import os
+import unittest
+import pkg_resources
+from setuptools.tests.py26compat import skipIf
+from unittest import expectedFailure
+
+try:
+    import _ast
+except ImportError:
+    pass
+
+class TestMarkerlib(unittest.TestCase):
+
+    def test_markers(self):
+        from _markerlib import interpret, default_environment, compile
+        
+        os_name = os.name
+        
+        self.assert_(interpret(""))
+        
+        self.assert_(interpret("os.name != 'buuuu'"))
+        self.assert_(interpret("python_version > '1.0'"))
+        self.assert_(interpret("python_version < '5.0'"))
+        self.assert_(interpret("python_version <= '5.0'"))
+        self.assert_(interpret("python_version >= '1.0'"))
+        self.assert_(interpret("'%s' in os.name" % os_name))
+        self.assert_(interpret("'buuuu' not in os.name"))
+        
+        self.assertFalse(interpret("os.name == 'buuuu'"))
+        self.assertFalse(interpret("python_version < '1.0'"))
+        self.assertFalse(interpret("python_version > '5.0'"))
+        self.assertFalse(interpret("python_version >= '5.0'"))
+        self.assertFalse(interpret("python_version <= '1.0'"))
+        self.assertFalse(interpret("'%s' not in os.name" % os_name))
+        self.assertFalse(interpret("'buuuu' in os.name and python_version >= '5.0'"))    
+        
+        environment = default_environment()
+        environment['extra'] = 'test'
+        self.assert_(interpret("extra == 'test'", environment))
+        self.assertFalse(interpret("extra == 'doc'", environment))
+        
+        @expectedFailure(NameError)
+        def raises_nameError():
+            interpret("python.version == '42'")
+        
+        raises_nameError()
+        
+        @expectedFailure(SyntaxError)
+        def raises_syntaxError():
+            interpret("(x for x in (4,))")
+            
+        raises_syntaxError()
+        
+        statement = "python_version == '5'"
+        self.assertEqual(compile(statement).__doc__, statement)
+        
+    @skipIf('_ast' not in globals(),
+        "ast not available (Python < 2.5?)")
+    def test_ast(self):
+        try:
+            import ast, nose
+            raise nose.SkipTest()
+        except ImportError:
+            pass
+        
+        # Nonsensical code coverage tests.
+        import _markerlib._markers_ast as _markers_ast
+        
+        class Node(_ast.AST):
+            _fields = ('bogus')
+        list(_markers_ast.iter_fields(Node()))
+        
+        class Node2(_ast.AST):
+            def __init__(self):
+                self._fields = ('bogus',)
+                self.bogus = [Node()]
+                
+        class NoneTransformer(_markers_ast.NodeTransformer):
+            def visit_Attribute(self, node):
+                return None
+            
+            def visit_Str(self, node):
+                return None
+            
+            def visit_Node(self, node):
+                return []
+                
+        NoneTransformer().visit(_markers_ast.parse('a.b = "c"'))
+        NoneTransformer().visit(Node2())
+        
diff --git a/pkg_resources.py b/pkg_resources.py
index ca0f21c4..63d18977 100644
--- a/pkg_resources.py
+++ b/pkg_resources.py
@@ -1746,7 +1746,7 @@ def find_on_path(importer, path_item, only=False):
             # scan for .egg and .egg-info in directory
             for entry in os.listdir(path_item):
                 lower = entry.lower()
-                if lower.endswith('.egg-info') or lower.endswith('.dist-info'):
+                if lower.endswith(('.egg-info', '.dist-info')):
                     fullpath = os.path.join(path_item, entry)
                     if os.path.isdir(fullpath):
                         # egg-info directory, allow getting metadata
@@ -2489,7 +2489,7 @@ class DistInfoDistribution(Distribution):
             marker_fn.__doc__ = marker
             return marker_fn
         try:
-            from markerlib import as_function
+            from _markerlib import as_function
         except ImportError:
             as_function = dummy_marker
         dm = self.__dep_map = {None: []}
diff --git a/setuptools/tests/test_dist_info.py b/setuptools/tests/test_dist_info.py
index 001c4433..70dce2d4 100644
--- a/setuptools/tests/test_dist_info.py
+++ b/setuptools/tests/test_dist_info.py
@@ -7,7 +7,7 @@ import unittest
 import textwrap
 
 try:
-    import markerlib
+    import _markerlib
 except:
     pass
 
@@ -34,8 +34,8 @@ class TestDistInfo(unittest.TestCase):
         assert versioned.version == '2.718' # from filename
         assert unversioned.version == '0.3' # from METADATA
 
-    @skipIf('markerlib' not in globals(),
-        "install markerlib to test conditional dependencies")
+    @skipIf('_markerlib' not in globals(),
+        "_markerlib is used to test conditional dependencies (Python >= 2.5)")
     def test_conditional_dependencies(self):
         requires = [pkg_resources.Requirement.parse('splort==4'),
                     pkg_resources.Requirement.parse('quux>=1.1')]
@@ -51,7 +51,8 @@ class TestDistInfo(unittest.TestCase):
                                  'VersionedDistribution-2.718.dist-info')
         os.mkdir(versioned)
         open(os.path.join(versioned, 'METADATA'), 'w+').write(DALS(
-            """Metadata-Version: 1.2
+            """
+            Metadata-Version: 1.2
             Name: VersionedDistribution
             Requires-Dist: splort (4)
             Provides-Extra: baz
@@ -62,7 +63,8 @@ class TestDistInfo(unittest.TestCase):
                                    'UnversionedDistribution.dist-info')
         os.mkdir(unversioned)
         open(os.path.join(unversioned, 'METADATA'), 'w+').write(DALS(
-            """Metadata-Version: 1.2
+            """
+            Metadata-Version: 1.2
             Name: UnversionedDistribution
             Version: 0.3
             Requires-Dist: splort (==4)