Merge with default

--HG--
branch : distribute
extra : rebase_source : a19a5411e30665acdd86f2554921e385759b1ef3

19 files changed, 192 insertions, 79 deletions

diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py
index ab786f3d..5787753f 100755
--- a/setuptools/archive_util.py
+++ b/setuptools/archive_util.py
@@ -180,19 +180,22 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter):
     try:
         tarobj.chown = lambda *args: None   # don't do any chowning!
         for member in tarobj:
-            if member.isfile() or member.isdir():
-                name = member.name
-                # don't extract absolute paths or ones with .. in them
-                if not name.startswith('/') and '..' not in name:
-                    dst = os.path.join(extract_dir, *name.split('/'))                
-                    dst = progress_filter(name, dst)
-                    if dst:
-                        if dst.endswith(os.sep):
-                            dst = dst[:-1]
-                        try:
-                            tarobj._extract_member(member,dst)  # XXX Ugh
-                        except tarfile.ExtractError:
-                            pass    # chown/chmod/mkfifo/mknode/makedev failed
+            name = member.name
+            # don't extract absolute paths or ones with .. in them
+            if not name.startswith('/') and '..' not in name:
+                prelim_dst = os.path.join(extract_dir, *name.split('/'))
+                final_dst = progress_filter(name, prelim_dst)
+                # If progress_filter returns None, then we do not extract
+                # this file
+                # TODO: Do we really need to limit to just these file types?
+                # tarobj.extract() will handle all files on all platforms,
+                # turning file types that aren't allowed on that platform into
+                # regular files.
+                if final_dst and (member.isfile() or member.isdir() or
+                        member.islnk() or member.issym()):
+                    tarobj.extract(member, extract_dir)
+                    if final_dst != prelim_dst:
+                        shutil.move(prelim_dst, final_dst)
         return True
     finally:
         tarobj.close()
diff --git a/setuptools/cli-32.exe b/setuptools/cli-32.exe
new file mode 100644
index 00000000..9b7717b7
--- /dev/null
+++ b/setuptools/cli-32.exe
diff --git a/setuptools/cli-64.exe b/setuptools/cli-64.exe
new file mode 100644
index 00000000..265585af
--- /dev/null
+++ b/setuptools/cli-64.exe
diff --git a/setuptools/cli.exe b/setuptools/cli.exe
index 8906ff77..9b7717b7 100644
--- a/setuptools/cli.exe
+++ b/setuptools/cli.exe
diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py
index 152406b3..b063fa19 100644
--- a/setuptools/command/__init__.py
+++ b/setuptools/command/__init__.py
@@ -14,7 +14,6 @@ if sys.version>='2.5':
 
 from distutils.command.bdist import bdist
 
-
 if 'egg' not in bdist.format_commands:
     bdist.format_command['egg'] = ('bdist_egg', "Python .egg file")
     bdist.format_commands.append('egg')
diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py
index a01e2843..d53960fe 100644
--- a/setuptools/command/build_py.py
+++ b/setuptools/command/build_py.py
@@ -28,13 +28,8 @@ try:
             if not files:
                 return
             log.info("Fixing "+" ".join(files))
-            if not self.fixer_names:
-                self.fixer_names = []
-                for p in setuptools.lib2to3_fixer_packages:
-                    self.fixer_names.extend(get_fixers_from_package(p))
-                if self.distribution.use_2to3_fixers is not None:
-                    for p in self.distribution.use_2to3_fixers:
-                        self.fixer_names.extend(get_fixers_from_package(p))
+            self.__build_fixer_names()
+            self.__exclude_fixers()
             if doctests:
                 if setuptools.run_2to3_on_doctests:
                     r = DistutilsRefactoringTool(self.fixer_names)
@@ -42,6 +37,25 @@ try:
             else:
                 _Mixin2to3.run_2to3(self, files)
 
+        def __build_fixer_names(self):
+            if self.fixer_names: return
+            self.fixer_names = []
+            for p in setuptools.lib2to3_fixer_packages:
+                self.fixer_names.extend(get_fixers_from_package(p))
+            if self.distribution.use_2to3_fixers is not None:
+                for p in self.distribution.use_2to3_fixers:
+                    self.fixer_names.extend(get_fixers_from_package(p))
+
+        def __exclude_fixers(self):
+            excluded_fixers = getattr(self, 'exclude_fixers', [])
+            if self.distribution.use_2to3_exclude_fixers is not None:
+                excluded_fixers.extend(self.distribution.use_2to3_exclude_fixers)
+            for fixer_name in excluded_fixers:
+                if fixer_name not in self.fixer_names:
+                    log.warn("Excluded fixer %s not found", fixer_name)
+                    continue
+                self.fixer_names.remove(fixer_name)
+
 except ImportError:
     class Mixin2to3:
         def run_2to3(self, files, doctests=True):
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index 3f1b4228..d200dac1 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -42,6 +42,10 @@ __all__ = [
 import site
 HAS_USER_SITE = not sys.version < "2.6" and site.ENABLE_USER_SITE
 
+import struct
+def is_64bit():
+    return struct.calcsize("P") == 8
+
 def samefile(p1,p2):
     if hasattr(os.path,'samefile') and (
         os.path.exists(p1) and os.path.exists(p2)
@@ -733,22 +737,26 @@ Please make the appropriate changes for your system and try again.
         spec = str(dist.as_requirement())
         is_script = is_python_script(script_text, script_name)
 
-        if is_script and dev_path:
-            script_text = get_script_header(script_text) + (
-                "# EASY-INSTALL-DEV-SCRIPT: %(spec)r,%(script_name)r\n"
-                "__requires__ = %(spec)r\n"
-                "from pkg_resources import require; require(%(spec)r)\n"
-                "del require\n"
-                "__file__ = %(dev_path)r\n"
-                "execfile(__file__)\n"
-            ) % locals()
-        elif is_script:
-            script_text = get_script_header(script_text) + (
-                "# EASY-INSTALL-SCRIPT: %(spec)r,%(script_name)r\n"
-                "__requires__ = %(spec)r\n"
-                "import pkg_resources\n"
-                "pkg_resources.run_script(%(spec)r, %(script_name)r)\n"
-            ) % locals()
+        def get_template(filename):
+            """
+            There are a couple of template scripts in the package. This
+            function loads one of them and prepares it for use.
+
+            These templates use triple-quotes to escape variable
+            substitutions so the scripts get the 2to3 treatment when build
+            on Python 3. The templates cannot use triple-quotes naturally.
+            """
+            raw_bytes = resource_string('setuptools', template_name)
+            template_str = raw_bytes.decode('utf-8')
+            clean_template = template_str.replace('"""', '')
+            return clean_template
+
+        if is_script:
+            template_name = 'script template.py'
+            if dev_path:
+                template_name = template_name.replace('.py', ' (dev).py')
+            script_text = (get_script_header(script_text) +
+                get_template(template_name) % locals())
         self.write_script(script_name, _to_ascii(script_text), 'b')
 
     def write_script(self, script_name, contents, mode="t", blockers=()):
@@ -1801,7 +1809,10 @@ def get_script_args(dist, executable=sys_executable, wininst=False):
                     ext, launcher = '-script.py', 'cli.exe'
                     old = ['.py','.pyc','.pyo']
                     new_header = re.sub('(?i)pythonw.exe','python.exe',header)
-
+                if is_64bit():
+                    launcher = launcher.replace(".", "-64.")
+                else:
+                    launcher = launcher.replace(".", "-32.")
                 if os.path.exists(new_header[2:-1]) or sys.platform!='win32':
                     hdr = new_header
                 else:
diff --git a/setuptools/command/install_egg_info.py b/setuptools/command/install_egg_info.py
index dd95552e..f44b34b5 100755
--- a/setuptools/command/install_egg_info.py
+++ b/setuptools/command/install_egg_info.py
@@ -89,6 +89,8 @@ class install_egg_info(Command):
         if not self.dry_run:
             f = open(filename,'wt')
             for pkg in nsp:
+                # ensure pkg is not a unicode string under Python 2.7
+                pkg = str(pkg)
                 pth = tuple(pkg.split('.'))
                 trailer = '\n'
                 if '.' in pkg:
diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py
index 3442fe4b..c49839cd 100755
--- a/setuptools/command/sdist.py
+++ b/setuptools/command/sdist.py
@@ -199,15 +199,25 @@ class sdist(_sdist):
             build_scripts = self.get_finalized_command('build_scripts')
             self.filelist.extend(build_scripts.get_source_files())
 
-    def read_template(self):
+    def __read_template_hack(self):
+        # This grody hack closes the template file (MANIFEST.in) if an
+        #  exception occurs during read_template.
+        # Doing so prevents an error when easy_install attempts to delete the
+        #  file.
         try:
             _sdist.read_template(self)
         except:
-            # grody hack to close the template file (MANIFEST.in)
-            # this prevents easy_install's attempt at deleting the file from
-            # dying and thus masking the real error
             sys.exc_info()[2].tb_next.tb_frame.f_locals['template'].close()
             raise
+    # Beginning with Python 2.7.2, 3.1.4, and 3.2.1, this leaky file handle
+    #  has been fixed, so only override the method if we're using an earlier
+    #  Python.
+    if (
+            sys.version_info < (2,7,2)
+            or (3,0) <= sys.version_info < (3,1,4)
+            or (3,2) <= sys.version_info < (3,2,1)
+        ):
+        read_template = __read_template_hack
 
     def check_readme(self):
         alts = ("README", "README.txt")
diff --git a/setuptools/command/upload.py b/setuptools/command/upload.py
index 1f49745e..4bd6021d 100755
--- a/setuptools/command/upload.py
+++ b/setuptools/command/upload.py
@@ -181,3 +181,4 @@ class upload(Command):
                           log.ERROR)
         if self.show_response:
             print '-'*75, r.read(), '-'*75
+
diff --git a/setuptools/extension.py b/setuptools/extension.py
index 980ee0a7..eb8b836c 100644
--- a/setuptools/extension.py
+++ b/setuptools/extension.py
@@ -1,40 +1,46 @@
-from distutils.core import Extension as _Extension
+import sys
+import distutils.core
+import distutils.extension
+
 from setuptools.dist import _get_unpatched
-_Extension = _get_unpatched(_Extension)
 
-# Prefer Cython to Pyrex
-pyrex_impls = 'Cython.Distutils.build_ext', 'Pyrex.Distutils.build_ext'
-for pyrex_impl in pyrex_impls:
-	try:
-		# from (pyrex_impl) import build_ext
-		build_ext = __import__(pyrex_impl, fromlist=['build_ext']).build_ext
-		break
-	except:
-		pass
-have_pyrex = 'build_ext' in globals()
+_Extension = _get_unpatched(distutils.core.Extension)
+
+def have_pyrex():
+    """
+    Return True if Cython or Pyrex can be imported.
+    """
+    pyrex_impls = 'Cython.Distutils.build_ext', 'Pyrex.Distutils.build_ext'
+    for pyrex_impl in pyrex_impls:
+        try:
+            # from (pyrex_impl) import build_ext
+            __import__(pyrex_impl, fromlist=['build_ext']).build_ext
+            return True
+        except Exception:
+            pass
+    return False
 
 
 class Extension(_Extension):
     """Extension that uses '.c' files in place of '.pyx' files"""
 
-    if not have_pyrex:
-        # convert .pyx extensions to .c 
-        def __init__(self,*args,**kw):
-            _Extension.__init__(self,*args,**kw)
-            sources = []
-            for s in self.sources:
-                if s.endswith('.pyx'):
-                    sources.append(s[:-3]+'c')
-                else:
-                    sources.append(s)
-            self.sources = sources
+    def __init__(self, *args, **kw):
+        _Extension.__init__(self, *args, **kw)
+        if not have_pyrex():
+            self._convert_pyx_sources_to_c()
+
+    def _convert_pyx_sources_to_c(self):
+        "convert .pyx extensions to .c"
+        def pyx_to_c(source):
+            if source.endswith('.pyx'):
+                source = source[:-4] + '.c'
+            return source
+        self.sources = map(pyx_to_c, self.sources)
 
 class Library(Extension):
     """Just like a regular Extension, but built as a library instead"""
 
-import sys, distutils.core, distutils.extension
 distutils.core.Extension = Extension
 distutils.extension.Extension = Extension
 if 'distutils.command.build_ext' in sys.modules:
     sys.modules['distutils.command.build_ext'].Extension = Extension
-
diff --git a/setuptools/gui-32.exe b/setuptools/gui-32.exe
new file mode 100644
index 00000000..3f64af7d
--- /dev/null
+++ b/setuptools/gui-32.exe
diff --git a/setuptools/gui-64.exe b/setuptools/gui-64.exe
new file mode 100644
index 00000000..3ab4378e
--- /dev/null
+++ b/setuptools/gui-64.exe
diff --git a/setuptools/gui.exe b/setuptools/gui.exe
index 474838d5..3f64af7d 100644
--- a/setuptools/gui.exe
+++ b/setuptools/gui.exe
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index bb0ae129..d0896feb 100755
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -1,5 +1,6 @@
 """PyPI and direct package downloading"""
 import sys, os.path, re, urlparse, urllib, urllib2, shutil, random, socket, cStringIO
+import base64
 import httplib
 from pkg_resources import *
 from distutils import log
@@ -756,6 +757,22 @@ def socket_timeout(timeout=15):
         return _socket_timeout
     return _socket_timeout
 
+def _encode_auth(auth):
+    """
+    A function compatible with Python 2.3-3.3 that will encode
+    auth from a URL suitable for an HTTP header.
+    >>> _encode_auth('username%3Apassword')
+    u'dXNlcm5hbWU6cGFzc3dvcmQ='
+    """
+    auth_s = urllib2.unquote(auth)
+    # convert to bytes
+    auth_bytes = auth_s.encode()
+    # use the legacy interface for Python 2.3 support
+    encoded_bytes = base64.encodestring(auth_bytes)
+    # convert back to a string
+    encoded = encoded_bytes.decode()
+    # strip the trailing carriage return
+    return encoded.rstrip()
 
 def open_with_auth(url):
     """Open a urllib2 request, handling HTTP authentication"""
@@ -768,7 +785,7 @@ def open_with_auth(url):
         auth = None
 
     if auth:
-        auth = "Basic " + urllib2.unquote(auth).encode('base64').strip()
+        auth = "Basic " + _encode_auth(auth)
         new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
         request = urllib2.Request(new_url)
         request.add_header("Authorization", auth)
diff --git a/setuptools/script template (dev).py b/setuptools/script template (dev).py
new file mode 100644
index 00000000..6dd9dd45
--- /dev/null
+++ b/setuptools/script template (dev).py
@@ -0,0 +1,6 @@
+# EASY-INSTALL-DEV-SCRIPT: %(spec)r,%(script_name)r
+__requires__ = """%(spec)r"""
+from pkg_resources import require; require("""%(spec)r""")
+del require
+__file__ = """%(dev_path)r"""
+execfile(__file__)
diff --git a/setuptools/script template.py b/setuptools/script template.py
new file mode 100644
index 00000000..8dd5d510
--- /dev/null
+++ b/setuptools/script template.py
@@ -0,0 +1,4 @@
+# EASY-INSTALL-SCRIPT: %(spec)r,%(script_name)r
+__requires__ = """%(spec)r"""
+import pkg_resources
+pkg_resources.run_script("""%(spec)r""", """%(script_name)r""")
diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py
index 85616605..4150ad10 100644
--- a/setuptools/tests/test_easy_install.py
+++ b/setuptools/tests/test_easy_install.py
@@ -67,7 +67,7 @@ class TestEasyInstallTest(unittest.TestCase):
 
         old_platform = sys.platform
         try:
-            name, script = get_script_args(dist).next()
+            name, script = [i for i in get_script_args(dist).next()][0:2]
         finally:
             sys.platform = old_platform
 
@@ -141,9 +141,13 @@ class TestPTHFileWriter(unittest.TestCase):
         self.assert_(pth.dirty)
 
     def test_add_from_site_is_ignored(self):
-        pth = PthDistributions('does-not_exist', ['/test/location/does-not-have-to-exist'])
+        if os.name != 'nt':
+            location = '/test/location/does-not-have-to-exist'
+        else:
+            location = 'c:\\does_not_exist'
+        pth = PthDistributions('does-not_exist', [location, ])
         self.assert_(not pth.dirty)
-        pth.add(PRDistribution('/test/location/does-not-have-to-exist'))
+        pth.add(PRDistribution(location))
         self.assert_(not pth.dirty)
 
 
@@ -221,7 +225,7 @@ class TestUserInstallTest(unittest.TestCase):
 
         sys.path.append(target)
         old_ppath = os.environ.get('PYTHONPATH')
-        os.environ['PYTHONPATH'] = ':'.join(sys.path)
+        os.environ['PYTHONPATH'] = os.path.pathsep.join(sys.path)
         try:
             dist = Distribution()
             dist.script_name = 'setup.py'
@@ -234,8 +238,13 @@ class TestUserInstallTest(unittest.TestCase):
             self.assertEquals(res.location, new_location)
         finally:
             sys.path.remove(target)
-            shutil.rmtree(new_location)
-            shutil.rmtree(target)
+            for basedir in [new_location, target, ]:
+                if not os.path.exists(basedir) or not os.path.isdir(basedir):
+                    continue
+                try:
+                    shutil.rmtree(basedir)
+                except:
+                    pass
             if old_ppath is not None:
                 os.environ['PYTHONPATH'] = old_ppath
             else:
diff --git a/setuptools/tests/test_resources.py b/setuptools/tests/test_resources.py
index c10ca210..3e0309f1 100644
--- a/setuptools/tests/test_resources.py
+++ b/setuptools/tests/test_resources.py
@@ -8,6 +8,16 @@ try: frozenset
 except NameError:
     from sets import ImmutableSet as frozenset
 
+def safe_repr(obj, short=False):
+    """ copied from Python2.7"""
+    try:
+        result = repr(obj)
+    except Exception:
+        result = object.__repr__(obj)
+    if not short or len(result) < _MAX_LENGTH:
+        return result
+    return result[:_MAX_LENGTH] + ' [truncated]...'
+
 class Metadata(EmptyProvider):
     """Mock object to return metadata as if from an on-disk distribution"""
 
@@ -467,14 +477,13 @@ class ParseTests(TestCase):
             p1, p2 = parse_version(s1),parse_version(s2)
             self.assertEqual(p1,p2, (s1,s2,p1,p2))
 
-        c('1.2-rc1', '1.2rc1')
         c('0.4', '0.4.0')
         c('0.4.0.0', '0.4.0')
         c('0.4.0-0', '0.4-0')
         c('0pl1', '0.0pl1')
         c('0pre1', '0.0c1')
         c('0.0.0preview1', '0c1')
-        c('0.0c1', '0-rc1')
+        c('0.0c1', '0rc1')
         c('1.2a1', '1.2.a.1'); c('1.2...a', '1.2a')
 
     def testVersionOrdering(self):
@@ -483,11 +492,14 @@ class ParseTests(TestCase):
             self.assert_(p1<p2, (s1,s2,p1,p2))
 
         c('2.1','2.1.1')
+        c('2.1.0','2.10')
         c('2a1','2b0')
+        c('2b1','2c0')
         c('2a1','2.1')
         c('2.3a1', '2.3')
         c('2.1-1', '2.1-2')
         c('2.1-1', '2.1.1')
+        c('2.1', '2.1.1-1')
         c('2.1', '2.1pl4')
         c('2.1a0-20040501', '2.1')
         c('1.1', '02.1')
@@ -498,8 +510,20 @@ class ParseTests(TestCase):
         c('0.4', '4.0')
         c('0.0.4', '0.4.0')
         c('0pl1', '0.4pl1')
-        c('2.1.0-rc1','2.1.0')
         c('2.1dev','2.1a0')
+        c('2.1.0rc1','2.1.0')
+        c('2.1.0','2.1.0-rc0')
+        c('2.1.0','2.1.0-a')
+        c('2.1.0','2.1.0-alpha')
+        c('2.1.0','2.1.0-foo')
+        c('1.0','1.0-1')
+        c('1.0-1','1.0.1')
+        c('1.0a','1.0b')
+        c('1.0dev','1.0rc1')
+        c('1.0pre','1.0')
+        c('1.0pre','1.0')
+        c('1.0a','1.0-a')
+        c('1.0rc1','1.0-rc1')
 
         torture ="""
         0.80.1-3 0.80.1-2 0.80.1-1 0.79.9999+0.80.0pre4-1
@@ -580,6 +604,13 @@ class NamespaceTests(TestCase):
         pkg_resources._namespace_packages = self._ns_pkgs.copy()
         sys.path = self._prev_sys_path[:]
 
+    def _assertIn(self, member, container):
+        """ assertIn and assertTrue does not exist in Python2.3"""
+        if member not in container:
+            standardMsg = '%s not found in %s' % (safe_repr(member),
+                                                  safe_repr(container))
+            self.fail(self._formatMessage(msg, standardMsg))
+
     def test_two_levels_deep(self):
         """
         Test nested namespace packages
@@ -603,13 +634,13 @@ class NamespaceTests(TestCase):
             pkg2_init.write(ns_str)
             pkg2_init.close()
         import pkg1
-        self.assertTrue("pkg1" in pkg_resources._namespace_packages.keys())
+        self._assertIn("pkg1", pkg_resources._namespace_packages.keys())
         try:
             import pkg1.pkg2
         except ImportError, e:
             self.fail("Distribute tried to import the parent namespace package")
         # check the _namespace_packages dict
-        self.assertTrue("pkg1.pkg2" in pkg_resources._namespace_packages.keys())
+        self._assertIn("pkg1.pkg2", pkg_resources._namespace_packages.keys())
         self.assertEqual(pkg_resources._namespace_packages["pkg1"], ["pkg1.pkg2"])
         # check the __path__ attribute contains both paths
         self.assertEqual(pkg1.pkg2.__path__, [