Merged latest changes from setuptools-0.6 branch

--HG--
rename : doc/formats.txt => docs/formats.txt

5 files changed, 262 insertions, 16 deletions

diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index 4cd058bb..146e1f47 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -283,7 +283,7 @@ class easy_install(Command):
                 else:
                     self.all_site_dirs.append(normalize_path(d))
         if not self.editable: self.check_site_dir()
-        self.index_url = self.index_url or "http://pypi.python.org/simple"
+        self.index_url = self.index_url or "https://pypi.python.org/simple"
         self.shadow_path = self.all_site_dirs[:]
         for path_item in self.install_dir, normalize_path(self.script_dir):
             if path_item not in self.shadow_path:
diff --git a/setuptools/dist.py b/setuptools/dist.py
index 89208da8..907ce550 100644
--- a/setuptools/dist.py
+++ b/setuptools/dist.py
@@ -48,7 +48,6 @@ def assert_string_list(dist, attr, value):
         raise DistutilsSetupError(
             "%r must be a list of strings (got %r)" % (attr,value)
         )
-
 def check_nsp(dist, attr, value):
     """Verify that namespace packages are valid"""
     assert_string_list(dist,attr,value)
@@ -70,6 +69,10 @@ def check_extras(dist, attr, value):
     """Verify that extras_require mapping is valid"""
     try:
         for k,v in value.items():
+            if ':' in k:
+                k,m = k.split(':',1)
+                if pkg_resources.invalid_marker(m):
+                    raise DistutilsSetupError("Invalid environment marker: "+m)
             list(pkg_resources.parse_requirements(v))
     except (TypeError,ValueError,AttributeError):
         raise DistutilsSetupError(
@@ -78,9 +81,6 @@ def check_extras(dist, attr, value):
             "requirement specifiers."
         )
 
-
-
-
 def assert_bool(dist, attr, value):
     """Verify that value is True, False, 0, or 1"""
     if bool(value) != value:
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index 3a6b6fac..04a30a45 100755
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -2,6 +2,7 @@
 import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO
 import base64
 import httplib, urllib
+from setuptools import ssl_support
 from pkg_resources import *
 from distutils import log
 from distutils.errors import DistutilsError
@@ -157,12 +158,11 @@ user_agent = "Python-urllib/%s setuptools/%s" % (
     sys.version[:3], require('setuptools')[0].version
 )
 
-
 class PackageIndex(Environment):
     """A distribution index that scans web pages for download URLs"""
 
-    def __init__(self, index_url="http://pypi.python.org/simple", hosts=('*',),
-        *args, **kw
+    def __init__(self, index_url="https://pypi.python.org/simple", hosts=('*',),
+        ca_bundle=None, verify_ssl=True, *args, **kw
     ):
         Environment.__init__(self,*args,**kw)
         self.index_url = index_url + "/"[:not index_url.endswith('/')]
@@ -171,8 +171,9 @@ class PackageIndex(Environment):
         self.package_pages = {}
         self.allows = re.compile('|'.join(map(translate,hosts))).match
         self.to_scan = []
-
-
+        if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()):
+            self.opener = ssl_support.opener_for(ca_bundle)
+        else: self.opener = urllib2.urlopen
 
     def process_url(self, url, retrieve=False):
         """Evaluate a URL as a possible download, and maybe retrieve it"""
@@ -601,7 +602,7 @@ class PackageIndex(Environment):
         if url.startswith('file:'):
             return local_open(url)
         try:
-            return open_with_auth(url)
+            return open_with_auth(url, self.opener)
         except (ValueError, httplib.InvalidURL), v:
             msg = ' '.join([str(arg) for arg in v.args])
             if warning:
@@ -659,7 +660,6 @@ class PackageIndex(Environment):
             self.url_ok(url, True)   # raises error if not allowed
             return self._attempt_download(url, filename)
 
-
     def scan_url(self, url):
         self.process_url(url, True)
 
@@ -859,7 +859,7 @@ def _encode_auth(auth):
     # strip the trailing carriage return
     return encoded.rstrip()
 
-def open_with_auth(url):
+def open_with_auth(url, opener=urllib2.urlopen):
     """Open a urllib2 request, handling HTTP authentication"""
 
     scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
@@ -883,7 +883,7 @@ def open_with_auth(url):
         request = urllib2.Request(url)
 
     request.add_header('User-Agent', user_agent)
-    fp = urllib2.urlopen(request)
+    fp = opener(request)
 
     if auth:
         # Put authentication info back into request URL if same host,
diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py
new file mode 100644
index 00000000..f1d8c920
--- /dev/null
+++ b/setuptools/ssl_support.py
@@ -0,0 +1,246 @@
+import sys, os, socket, urllib2, atexit, re
+from pkg_resources import ResolutionError, ExtractionError
+
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+__all__ = [
+    'VerifyingHTTPSHandler', 'find_ca_bundle', 'is_available', 'cert_paths',
+    'opener_for'
+]
+
+cert_paths = """
+/etc/pki/tls/certs/ca-bundle.crt
+/etc/ssl/certs/ca-certificates.crt
+/usr/share/ssl/certs/ca-bundle.crt
+/usr/local/share/certs/ca-root.crt
+/etc/ssl/cert.pem
+/System/Library/OpenSSL/certs/cert.pem
+""".strip().split()
+
+
+HTTPSHandler = HTTPSConnection = object
+
+for what, where in (
+    ('HTTPSHandler', ['urllib2','urllib.request']),
+    ('HTTPSConnection', ['httplib', 'http.client']),
+):
+    for module in where:
+        try:
+            exec("from %s import %s" % (module, what))
+        except ImportError:
+            pass
+
+is_available = ssl is not None and object not in (HTTPSHandler, HTTPSConnection)
+
+
+
+
+
+try:
+    from socket import create_connection
+except ImportError:
+    _GLOBAL_DEFAULT_TIMEOUT = getattr(socket, '_GLOBAL_DEFAULT_TIMEOUT', object())
+    def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
+                          source_address=None):
+        """Connect to *address* and return the socket object.
+    
+        Convenience function.  Connect to *address* (a 2-tuple ``(host,
+        port)``) and return the socket object.  Passing the optional
+        *timeout* parameter will set the timeout on the socket instance
+        before attempting to connect.  If no *timeout* is supplied, the
+        global default timeout setting returned by :func:`getdefaulttimeout`
+        is used.  If *source_address* is set it must be a tuple of (host, port)
+        for the socket to bind as a source address before making the connection.
+        An host of '' or port 0 tells the OS to use the default.
+        """   
+        host, port = address
+        err = None
+        for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
+            af, socktype, proto, canonname, sa = res
+            sock = None
+            try:
+                sock = socket.socket(af, socktype, proto)
+                if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
+                    sock.settimeout(timeout)
+                if source_address:
+                    sock.bind(source_address)
+                sock.connect(sa)
+                return sock
+
+            except error:
+                err = True
+                if sock is not None:
+                    sock.close()    
+        if err:
+            raise
+        else:
+            raise error("getaddrinfo returns an empty list")
+
+
+try:
+    from ssl import CertificateError, match_hostname
+except ImportError:
+    class CertificateError(ValueError):
+        pass
+       
+    def _dnsname_to_pat(dn):
+        pats = []
+        for frag in dn.split(r'.'):
+            if frag == '*':
+                # When '*' is a fragment by itself, it matches a non-empty dotless
+                # fragment.
+                pats.append('[^.]+')
+            else:
+                # Otherwise, '*' matches any dotless fragment.
+                frag = re.escape(frag)
+                pats.append(frag.replace(r'\*', '[^.]*'))
+        return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
+       
+    def match_hostname(cert, hostname):
+        """Verify that *cert* (in decoded format as returned by
+        SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 rules
+        are mostly followed, but IP addresses are not accepted for *hostname*.
+    
+        CertificateError is raised on failure. On success, the function
+        returns nothing.
+        """
+        if not cert:
+            raise ValueError("empty or no certificate")
+        dnsnames = []
+        san = cert.get('subjectAltName', ())
+        for key, value in san:
+            if key == 'DNS':
+                if _dnsname_to_pat(value).match(hostname):
+                    return
+                dnsnames.append(value)
+        if not dnsnames:
+            # The subject is only checked when there is no dNSName entry
+            # in subjectAltName
+            for sub in cert.get('subject', ()):
+                for key, value in sub:
+                    # XXX according to RFC 2818, the most specific Common Name
+                    # must be used.
+                    if key == 'commonName':
+                        if _dnsname_to_pat(value).match(hostname):
+                            return
+                        dnsnames.append(value)
+        if len(dnsnames) > 1:
+            raise CertificateError("hostname %r "
+                "doesn't match either of %s"
+                % (hostname, ', '.join(map(repr, dnsnames))))
+        elif len(dnsnames) == 1:
+            raise CertificateError("hostname %r "
+                "doesn't match %r"
+                % (hostname, dnsnames[0]))
+        else:
+            raise CertificateError("no appropriate commonName or "
+                "subjectAltName fields were found")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+class VerifyingHTTPSHandler(HTTPSHandler):
+    """Simple verifying handler: no auth, subclasses, timeouts, etc."""
+
+    def __init__(self, ca_bundle):
+        self.ca_bundle = ca_bundle
+        HTTPSHandler.__init__(self)
+
+    def https_open(self, req):
+        return self.do_open(
+            lambda host, **kw: VerifyingHTTPSConn(host, self.ca_bundle, **kw), req
+        )
+
+
+class VerifyingHTTPSConn(HTTPSConnection):
+    """Simple verifying connection: no auth, subclasses, timeouts, etc."""
+    def __init__(self, host, ca_bundle, **kw):         
+        HTTPSConnection.__init__(self, host, **kw)
+        self.ca_bundle = ca_bundle
+
+    def connect(self):
+        sock = create_connection(
+            (self.host, self.port), getattr(self,'source_address',None)
+        )
+        self.sock = ssl.wrap_socket(
+            sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_bundle
+        )               
+        try:
+            match_hostname(self.sock.getpeercert(), self.host)
+        except CertificateError:
+            self.sock.shutdown(socket.SHUT_RDWR)
+            self.sock.close()
+            raise
+
+def opener_for(ca_bundle=None):
+    """Get a urlopen() replacement that uses ca_bundle for verification"""
+    return urllib2.build_opener(
+        VerifyingHTTPSHandler(ca_bundle or find_ca_bundle())
+    ).open
+
+        
+
+_wincerts = None
+
+def get_win_certfile():
+    global _wincerts
+    if _wincerts is not None:
+        return _wincerts.name
+
+    try:        
+        from wincertstore import CertFile
+    except ImportError:
+        return None
+
+    class MyCertFile(CertFile):
+        def __init__(self, stores=(), certs=()):
+            CertFile.__init__(self)
+            for store in stores:
+                self.addstore(store)
+            self.addcerts(certs)
+            atexit.register(self.close)            
+
+    _wincerts = MyCertFile(stores=['CA', 'ROOT'])
+    return _wincerts.name
+
+
+def find_ca_bundle():
+    """Return an existing CA bundle path, or None"""
+    if os.name=='nt':
+        return get_win_certfile()
+    else:
+        for cert_path in cert_paths:
+            if os.path.isfile(cert_path):
+                return cert_path
+    try:
+        return pkg_resources.resource_filename('certifi', 'cacert.pem')
+    except (ImportError, ResolutionError, ExtractionError):
+        return None
+
+
+
+
+
diff --git a/setuptools/tests/test_resources.py b/setuptools/tests/test_resources.py
index 9c2ed9c9..34e341b5 100644
--- a/setuptools/tests/test_resources.py
+++ b/setuptools/tests/test_resources.py
@@ -527,7 +527,7 @@ class ScriptHeaderTests(TestCase):
 
         platform = sys.platform
         sys.platform = 'java1.5.0_13'
-        stdout = sys.stdout
+        stdout, stderr = sys.stdout, sys.stderr
         try:
             # A mock sys.executable that uses a shebang line (this file)
             exe = os.path.normpath(os.path.splitext(__file__)[0] + '.py')
@@ -550,7 +550,7 @@ class ScriptHeaderTests(TestCase):
         finally:
             del sys.modules["java"]
             sys.platform = platform
-            sys.stdout = stdout
+            sys.stdout, sys.stderr = stdout, stderr