Merge with upstream

--HG--
branch : distribute

11 files changed, 254 insertions, 54 deletions

diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py
index 152406b3..b063fa19 100644
--- a/setuptools/command/__init__.py
+++ b/setuptools/command/__init__.py
@@ -14,7 +14,6 @@ if sys.version>='2.5':
 
 from distutils.command.bdist import bdist
 
-
 if 'egg' not in bdist.format_commands:
     bdist.format_command['egg'] = ('bdist_egg', "Python .egg file")
     bdist.format_commands.append('egg')
diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py
index 007f3ba9..875971f0 100644
--- a/setuptools/command/bdist_egg.py
+++ b/setuptools/command/bdist_egg.py
@@ -426,8 +426,12 @@ def scan_module(egg_dir, base, name, stubs):
         return True     # Extension module
     pkg = base[len(egg_dir)+1:].replace(os.sep,'.')
     module = pkg+(pkg and '.' or '')+os.path.splitext(name)[0]
-    f = open(filename,'rb'); f.read(8)   # skip magic & date
-    code = marshal.load(f);  f.close()
+    if sys.version_info < (3, 3):
+        skip = 8   # skip magic & date
+    else:
+        skip = 12  # skip magic & date & file size
+    f = open(filename,'rb'); f.read(skip)
+    code = marshal.load(f); f.close()
     safe = True
     symbols = dict.fromkeys(iter_symbols(code))
     for bad in ['__file__', '__path__']:
diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py
index a01e2843..8751acd4 100644
--- a/setuptools/command/build_py.py
+++ b/setuptools/command/build_py.py
@@ -28,13 +28,8 @@ try:
             if not files:
                 return
             log.info("Fixing "+" ".join(files))
-            if not self.fixer_names:
-                self.fixer_names = []
-                for p in setuptools.lib2to3_fixer_packages:
-                    self.fixer_names.extend(get_fixers_from_package(p))
-                if self.distribution.use_2to3_fixers is not None:
-                    for p in self.distribution.use_2to3_fixers:
-                        self.fixer_names.extend(get_fixers_from_package(p))
+            self.__build_fixer_names()
+            self.__exclude_fixers()
             if doctests:
                 if setuptools.run_2to3_on_doctests:
                     r = DistutilsRefactoringTool(self.fixer_names)
@@ -42,6 +37,23 @@ try:
             else:
                 _Mixin2to3.run_2to3(self, files)
 
+        def __build_fixer_names(self):
+            if self.fixer_names: return
+            self.fixer_names = []
+            for p in setuptools.lib2to3_fixer_packages:
+                self.fixer_names.extend(get_fixers_from_package(p))
+            if self.distribution.use_2to3_fixers is not None:
+                for p in self.distribution.use_2to3_fixers:
+                    self.fixer_names.extend(get_fixers_from_package(p))
+
+        def __exclude_fixers(self):
+            excluded_fixers = getattr(self, 'exclude_fixers', [])
+            if self.distribution.use_2to3_exclude_fixers is not None:
+                excluded_fixers.extend(self.distribution.use_2to3_exclude_fixers)
+            for fixer_name in excluded_fixers:
+                if fixer_name in self.fixer_names:
+                    self.fixer_names.remove(fixer_name)
+
 except ImportError:
     class Mixin2to3:
         def run_2to3(self, files, doctests=True):
@@ -201,8 +213,8 @@ class build_py(_build_py, Mixin2to3):
         else:
             return init_py
 
-        f = open(init_py,'rU')
-        if 'declare_namespace' not in f.read():
+        f = open(init_py,'rbU')
+        if 'declare_namespace'.encode() not in f.read():
             from distutils import log
             log.warn(
                "WARNING: %s is a namespace package, but its __init__.py does\n"
diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py
index 93b7773c..709e349c 100755
--- a/setuptools/command/develop.py
+++ b/setuptools/command/develop.py
@@ -3,7 +3,7 @@ from distutils.util import convert_path, subst_vars
 from pkg_resources import Distribution, PathMetadata, normalize_path
 from distutils import log
 from distutils.errors import DistutilsError, DistutilsOptionError
-import os, setuptools, glob
+import os, sys, setuptools, glob
 
 class develop(easy_install):
     """Set up package for development"""
@@ -84,11 +84,35 @@ class develop(easy_install):
                 " installation directory", p, normalize_path(os.curdir))
 
     def install_for_development(self):
-        # Ensure metadata is up-to-date
-        self.run_command('egg_info')
-        # Build extensions in-place
-        self.reinitialize_command('build_ext', inplace=1)
-        self.run_command('build_ext')
+        if sys.version_info >= (3,) and getattr(self.distribution, 'use_2to3', False):
+            # If we run 2to3 we can not do this inplace:
+
+            # Ensure metadata is up-to-date
+            self.reinitialize_command('build_py', inplace=0)
+            self.run_command('build_py')
+            bpy_cmd = self.get_finalized_command("build_py")
+            build_path = normalize_path(bpy_cmd.build_lib)
+
+            # Build extensions
+            self.reinitialize_command('egg_info', egg_base=build_path)
+            self.run_command('egg_info')
+
+            self.reinitialize_command('build_ext', inplace=0)
+            self.run_command('build_ext')
+            
+            # Fixup egg-link and easy-install.pth
+            ei_cmd = self.get_finalized_command("egg_info")
+            self.egg_path = build_path
+            self.dist.location = build_path
+            self.dist._provider = PathMetadata(build_path, ei_cmd.egg_info)    # XXX
+        else:
+            # Without 2to3 inplace works fine:
+            self.run_command('egg_info')
+
+            # Build extensions in-place
+            self.reinitialize_command('build_ext', inplace=1)
+            self.run_command('build_ext')
+        
         self.install_site_py()  # ensure that target dir is site-safe
         if setuptools.bootstrap_install_from:
             self.easy_install(setuptools.bootstrap_install_from)
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index b8a10346..75d7b24b 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -10,7 +10,15 @@ file, or visit the `EasyInstall home page`__.
 __ http://packages.python.org/distribute/easy_install.html
 
 """
-import sys, os.path, zipimport, shutil, tempfile, zipfile, re, stat, random
+import sys
+import os
+import zipimport
+import shutil
+import tempfile
+import zipfile
+import re
+import stat
+import random
 from glob import glob
 from setuptools import Command, _dont_write_bytecode
 from setuptools.sandbox import run_setup
@@ -21,6 +29,7 @@ from distutils.sysconfig import get_python_lib, get_config_vars
 from distutils.errors import DistutilsArgError, DistutilsOptionError, \
     DistutilsError, DistutilsPlatformError
 from distutils.command.install import INSTALL_SCHEMES, SCHEME_KEYS
+from setuptools.command import setopt
 from setuptools.archive_util import unpack_archive
 from setuptools.package_index import PackageIndex
 from setuptools.package_index import URL_SCHEME
@@ -43,6 +52,10 @@ __all__ = [
 import site
 HAS_USER_SITE = not sys.version < "2.6" and site.ENABLE_USER_SITE
 
+import struct
+def is_64bit():
+    return struct.calcsize("P") == 8
+
 def samefile(p1,p2):
     if hasattr(os.path,'samefile') and (
         os.path.exists(p1) and os.path.exists(p2)
@@ -730,22 +743,26 @@ Please make the appropriate changes for your system and try again.
         spec = str(dist.as_requirement())
         is_script = is_python_script(script_text, script_name)
 
-        if is_script and dev_path:
-            script_text = get_script_header(script_text) + (
-                "# EASY-INSTALL-DEV-SCRIPT: %(spec)r,%(script_name)r\n"
-                "__requires__ = %(spec)r\n"
-                "from pkg_resources import require; require(%(spec)r)\n"
-                "del require\n"
-                "__file__ = %(dev_path)r\n"
-                "execfile(__file__)\n"
-            ) % locals()
-        elif is_script:
-            script_text = get_script_header(script_text) + (
-                "# EASY-INSTALL-SCRIPT: %(spec)r,%(script_name)r\n"
-                "__requires__ = %(spec)r\n"
-                "import pkg_resources\n"
-                "pkg_resources.run_script(%(spec)r, %(script_name)r)\n"
-            ) % locals()
+        def get_template(filename):
+            """
+            There are a couple of template scripts in the package. This
+            function loads one of them and prepares it for use.
+
+            These templates use triple-quotes to escape variable
+            substitutions so the scripts get the 2to3 treatment when build
+            on Python 3. The templates cannot use triple-quotes naturally.
+            """
+            raw_bytes = resource_string('setuptools', template_name)
+            template_str = raw_bytes.decode('utf-8')
+            clean_template = template_str.replace('"""', '')
+            return clean_template
+
+        if is_script:
+            template_name = 'script template.py'
+            if dev_path:
+                template_name = template_name.replace('.py', ' (dev).py')
+            script_text = (get_script_header(script_text) +
+                get_template(template_name) % locals())
         self.write_script(script_name, _to_ascii(script_text), 'b')
 
     def write_script(self, script_name, contents, mode="t", blockers=()):
@@ -756,12 +773,13 @@ Please make the appropriate changes for your system and try again.
         target = os.path.join(self.script_dir, script_name)
         self.add_output(target)
 
+        mask = current_umask()
         if not self.dry_run:
             ensure_directory(target)
             f = open(target,"w"+mode)
             f.write(contents)
             f.close()
-            chmod(target,0x1ED) # 0755
+            chmod(target, 0x1FF-mask) # 0777
 
 
 
@@ -1078,11 +1096,14 @@ See the setuptools documentation for the "develop" command for more info.
 
     def build_and_install(self, setup_script, setup_base):
         args = ['bdist_egg', '--dist-dir']
+
         dist_dir = tempfile.mkdtemp(
             prefix='egg-dist-tmp-', dir=os.path.dirname(setup_script)
         )
         try:
+            self._set_fetcher_options(os.path.dirname(setup_script))
             args.append(dist_dir)
+
             self.run_setup(setup_script, setup_base, args)
             all_eggs = Environment([dist_dir])
             eggs = []
@@ -1097,6 +1118,30 @@ See the setuptools documentation for the "develop" command for more info.
             rmtree(dist_dir)
             log.set_verbosity(self.verbose) # restore our log verbosity
 
+    def _set_fetcher_options(self, base):
+        """
+        When easy_install is about to run bdist_egg on a source dist, that
+        source dist might have 'setup_requires' directives, requiring
+        additional fetching. Ensure the fetcher options given to easy_install
+        are available to that command as well.
+        """
+        # find the fetch options from easy_install and write them out
+        #  to the setup.cfg file.
+        ei_opts = self.distribution.get_option_dict('easy_install').copy()
+        fetch_directives = (
+            'find_links', 'site_dirs', 'index_url', 'optimize',
+            'site_dirs', 'allow_hosts',
+        )
+        fetch_options = {}
+        for key, val in ei_opts.iteritems():
+            if key not in fetch_directives: continue
+            fetch_options[key.replace('_', '-')] = val[1]
+        # create a settings dictionary suitable for `edit_config`
+        settings = dict(easy_install=fetch_options)
+        cfg_filename = os.path.join(base, 'setup.cfg')
+        setopt.edit_config(cfg_filename, settings)
+
+
     def update_pth(self,dist):
         if self.pth_file is None:
             return
@@ -1431,7 +1476,19 @@ def extract_wininst_cfg(dist_filename):
         f.seek(prepended-(12+cfglen))
         cfg = ConfigParser.RawConfigParser({'version':'','target_version':''})
         try:
-            cfg.readfp(StringIO(f.read(cfglen).split(chr(0),1)[0]))
+            part = f.read(cfglen)
+            # part is in bytes, but we need to read up to the first null
+            #  byte.
+            if sys.version_info >= (2,6):
+                null_byte = bytes([0])
+            else:
+                null_byte = chr(0)
+            config = part.split(null_byte, 1)[0]
+            # Now the config is in bytes, but on Python 3, it must be
+            #  unicode for the RawConfigParser, so decode it. Is this the
+            #  right encoding?
+            config = config.decode('ascii')
+            cfg.readfp(StringIO(config))
         except ConfigParser.Error:
             return None
         if not cfg.has_section('metadata') or not cfg.has_section('Setup'):
@@ -1777,7 +1834,10 @@ def get_script_args(dist, executable=sys_executable, wininst=False):
                     ext, launcher = '-script.py', 'cli.exe'
                     old = ['.py','.pyc','.pyo']
                     new_header = re.sub('(?i)pythonw.exe','python.exe',header)
-
+                if is_64bit():
+                    launcher = launcher.replace(".", "-64.")
+                else:
+                    launcher = launcher.replace(".", "-32.")
                 if os.path.exists(new_header[2:-1]) or sys.platform!='win32':
                     hdr = new_header
                 else:
@@ -1827,6 +1887,11 @@ def rmtree(path, ignore_errors=False, onerror=auto_chmod):
     except os.error:
         onerror(os.rmdir, path, sys.exc_info())
 
+def current_umask():
+    tmp = os.umask(022)
+    os.umask(tmp)
+    return tmp
+
 def bootstrap():
     # This function is called when setuptools*.egg is run using /bin/sh
     import setuptools; argv0 = os.path.dirname(setuptools.__path__[0])
diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py
index 9ccbe68f..124c410e 100755
--- a/setuptools/command/egg_info.py
+++ b/setuptools/command/egg_info.py
@@ -163,7 +163,12 @@ class egg_info(Command):
             os.unlink(filename)
 
     def tagged_version(self):
-        return safe_version(self.distribution.get_version() + self.vtags)
+        version = self.distribution.get_version()
+        # egg_info may be called more than once for a distribution,
+        # in which case the version string already contains all tags.
+        if self.vtags and version.endswith(self.vtags):
+            return safe_version(version)
+        return safe_version(version + self.vtags)
 
     def run(self):
         self.mkpath(self.egg_info)
@@ -288,6 +293,19 @@ class FileList(FileList):
 
 
 
+def compose(path):
+    # Apple's HFS Plus returns decomposed UTF-8. Since just about
+    # everyone else chokes on it, we must make sure to return fully
+    # composed UTF-8 only.
+    if sys.getfilesystemencoding().lower() == 'utf-8':
+        from unicodedata import normalize
+        if sys.version_info >= (3,):
+            path = normalize('NFC', path)
+        else:
+            path = normalize('NFC', path.decode('utf-8')).encode('utf-8')
+    return path
+
+
 class manifest_maker(sdist):
 
     template = "MANIFEST.in"
@@ -312,6 +330,7 @@ class manifest_maker(sdist):
         self.prune_file_list()
         self.filelist.sort()
         self.filelist.remove_duplicates()
+        self.filelist.files = [compose(path) for path in self.filelist.files]
         self.write_manifest()
 
     def write_manifest (self):
diff --git a/setuptools/command/install_egg_info.py b/setuptools/command/install_egg_info.py
index dd95552e..f44b34b5 100755
--- a/setuptools/command/install_egg_info.py
+++ b/setuptools/command/install_egg_info.py
@@ -89,6 +89,8 @@ class install_egg_info(Command):
         if not self.dry_run:
             f = open(filename,'wt')
             for pkg in nsp:
+                # ensure pkg is not a unicode string under Python 2.7
+                pkg = str(pkg)
                 pth = tuple(pkg.split('.'))
                 trailer = '\n'
                 if '.' in pkg:
diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py
index 251190ba..105dabca 100755
--- a/setuptools/command/install_scripts.py
+++ b/setuptools/command/install_scripts.py
@@ -39,15 +39,16 @@ class install_scripts(_install_scripts):
 
     def write_script(self, script_name, contents, mode="t", *ignored):
         """Write an executable file to the scripts directory"""
-        from setuptools.command.easy_install import chmod
+        from setuptools.command.easy_install import chmod, current_umask
         log.info("Installing %s script to %s", script_name, self.install_dir)
         target = os.path.join(self.install_dir, script_name)
         self.outfiles.append(target)
 
+        mask = current_umask()
         if not self.dry_run:
             ensure_directory(target)
             f = open(target,"w"+mode)
             f.write(contents)
             f.close()
-            chmod(target,0x1ED)  # 0755
+            chmod(target, 0x1FF-mask)  # 0777
 
diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py
index 499a3fb9..56ef8a66 100755
--- a/setuptools/command/sdist.py
+++ b/setuptools/command/sdist.py
@@ -4,6 +4,8 @@ from distutils import log
 import os, re, sys, pkg_resources
 from glob import glob
 
+READMES = ('README', 'README.rst', 'README.txt')
+
 entities = [
     ("&lt;","<"), ("&gt;", ">"), ("&quot;", '"'), ("&apos;", "'"),
     ("&amp;", "&")
@@ -97,7 +99,7 @@ def entries_finder(dirname, filename):
         for match in entries_pattern.finditer(data):
             yield joinpath(dirname,unescape(match.group(1)))
     else:
-        log.warn("unrecognized .svn/entries format in %s", dirname)
+        log.warn("unrecognized .svn/entries format in %s", os.path.abspath(dirname))
 
 
 finders = [
@@ -145,7 +147,17 @@ class sdist(_sdist):
         self.filelist = ei_cmd.filelist
         self.filelist.append(os.path.join(ei_cmd.egg_info,'SOURCES.txt'))
         self.check_readme()
-        self.check_metadata()
+
+        # Run sub commands
+        for cmd_name in self.get_sub_commands():
+            self.run_command(cmd_name)
+
+        # Call check_metadata only if no 'check' command
+        # (distutils <= 2.6)
+        import distutils.command
+        if 'check' not in distutils.command.__all__:
+            self.check_metadata()
+            
         self.make_distribution()
 
         dist_files = getattr(self.distribution,'dist_files',[])
@@ -155,7 +167,7 @@ class sdist(_sdist):
                 dist_files.append(data)
 
     def add_defaults(self):
-        standards = [('README', 'README.txt'),
+        standards = [READMES,
                      self.distribution.script_name]
         for fn in standards:
             if isinstance(fn, tuple):
@@ -186,6 +198,14 @@ class sdist(_sdist):
         if self.distribution.has_pure_modules():
             build_py = self.get_finalized_command('build_py')
             self.filelist.extend(build_py.get_source_files())
+            # This functionality is incompatible with include_package_data, and
+            # will in fact create an infinite recursion if include_package_data
+            # is True.  Use of include_package_data will imply that
+            # distutils-style automatic handling of package_data is disabled
+            if not self.distribution.include_package_data:
+                for _, src_dir, _, filenames in build_py.data_files:
+                    self.filelist.extend([os.path.join(src_dir, filename)
+                                          for filename in filenames])
 
         if self.distribution.has_ext_modules():
             build_ext = self.get_finalized_command('build_ext')
@@ -199,24 +219,33 @@ class sdist(_sdist):
             build_scripts = self.get_finalized_command('build_scripts')
             self.filelist.extend(build_scripts.get_source_files())
 
-    def read_template(self):
+    def __read_template_hack(self):
+        # This grody hack closes the template file (MANIFEST.in) if an
+        #  exception occurs during read_template.
+        # Doing so prevents an error when easy_install attempts to delete the
+        #  file.
         try:
             _sdist.read_template(self)
         except:
-            # grody hack to close the template file (MANIFEST.in)
-            # this prevents easy_install's attempt at deleting the file from
-            # dying and thus masking the real error
             sys.exc_info()[2].tb_next.tb_frame.f_locals['template'].close()
             raise
+    # Beginning with Python 2.7.2, 3.1.4, and 3.2.1, this leaky file handle
+    #  has been fixed, so only override the method if we're using an earlier
+    #  Python.
+    if (
+            sys.version_info < (2,7,2)
+            or (3,0) <= sys.version_info < (3,1,4)
+            or (3,2) <= sys.version_info < (3,2,1)
+        ):
+        read_template = __read_template_hack
 
     def check_readme(self):
-        alts = ("README", "README.txt")
-        for f in alts:
+        for f in READMES:
             if os.path.exists(f):
                 return
         else:
             self.warn(
-                "standard file not found: should have one of " +', '.join(alts)
+                "standard file not found: should have one of " +', '.join(READMES)
             )
 
 
@@ -233,7 +262,34 @@ class sdist(_sdist):
 
         self.get_finalized_command('egg_info').save_version_info(dest)
 
+    def _manifest_is_not_generated(self):
+        # check for special comment used in 2.7.1 and higher
+        if not os.path.isfile(self.manifest):
+            return False
 
+        fp = open(self.manifest, 'rbU')
+        try:
+            first_line = fp.readline()
+        finally:
+            fp.close()
+        return first_line != '# file GENERATED by distutils, do NOT edit\n'.encode()
+
+    def read_manifest(self):
+        """Read the manifest file (named by 'self.manifest') and use it to
+        fill in 'self.filelist', the list of files to include in the source
+        distribution.
+        """
+        log.info("reading manifest file '%s'", self.manifest)
+        manifest = open(self.manifest, 'rbU')
+        for line in manifest:
+            if sys.version_info >= (3,):
+                line = line.decode('UTF-8')
+            # ignore comments and blank lines
+            line = line.strip()
+            if line.startswith('#') or not line:
+                continue
+            self.filelist.append(line)
+        manifest.close()
 
 
 
diff --git a/setuptools/command/test.py b/setuptools/command/test.py
index b7aef969..a02ac142 100644
--- a/setuptools/command/test.py
+++ b/setuptools/command/test.py
@@ -2,6 +2,7 @@ from setuptools import Command
 from distutils.errors import DistutilsOptionError
 import sys
 from pkg_resources import *
+from pkg_resources import _namespace_packages
 from unittest import TestLoader, main
 
 class ScanningLoader(TestLoader):
@@ -81,7 +82,7 @@ class test(Command):
 
 
     def with_project_on_sys_path(self, func):
-        if getattr(self.distribution, 'use_2to3', False):
+        if sys.version_info >= (3,) and getattr(self.distribution, 'use_2to3', False):
             # If we run 2to3 we can not do this inplace:
 
             # Ensure metadata is up-to-date
@@ -139,11 +140,28 @@ class test(Command):
 
     def run_tests(self):
         import unittest
+
+        # Purge modules under test from sys.modules. The test loader will
+        # re-import them from the build location. Required when 2to3 is used
+        # with namespace packages.
+        if sys.version_info >= (3,) and getattr(self.distribution, 'use_2to3', False):
+            module = self.test_args[-1].split('.')[0]
+            if module in _namespace_packages:
+                del_modules = []
+                if module in sys.modules:
+                    del_modules.append(module)
+                module += '.'
+                for name in sys.modules:
+                    if name.startswith(module):
+                        del_modules.append(name)
+                map(sys.modules.__delitem__, del_modules)
+
         loader_ep = EntryPoint.parse("x="+self.test_loader)
         loader_class = loader_ep.load(require=False)
+        cks = loader_class()
         unittest.main(
             None, None, [unittest.__file__]+self.test_args,
-            testLoader = loader_class()
+            testLoader = cks
         )
 
 
diff --git a/setuptools/command/upload.py b/setuptools/command/upload.py
index 6b18d761..bf9c0668 100755
--- a/setuptools/command/upload.py
+++ b/setuptools/command/upload.py
@@ -91,7 +91,7 @@ class upload(Command):
             comment = "built on %s" % platform.platform(terse=1)
         data = {
             ':action':'file_upload',
-            'protcol_version':'1',
+            'protocol_version':'1',
             'name':self.distribution.get_name(),
             'version':self.distribution.get_version(),
             'content':(basename,content),