merge dance

--HG--
branch : distribute
extra : rebase_source : e0fc1e252a506a6a751f9557d4a01580e1cbbdfa

20 files changed, 278 insertions, 64 deletions

diff --git a/distribute.egg-info/entry_points.txt b/distribute.egg-info/entry_points.txt
index 3532a1fe..7d44c6be 100644
--- a/distribute.egg-info/entry_points.txt
+++ b/distribute.egg-info/entry_points.txt
@@ -31,15 +31,17 @@ depends.txt = setuptools.command.egg_info:warn_depends_obsolete
 
 [console_scripts]
 easy_install = setuptools.command.easy_install:main
-easy_install-2.7 = setuptools.command.easy_install:main
+easy_install-2.6 = setuptools.command.easy_install:main
 
 [setuptools.file_finders]
 svn_cvs = setuptools.command.sdist:_default_revctrl
 
 [distutils.setup_keywords]
+additional_2to3_fixers = setuptools.dist:assert_string_list
 dependency_links = setuptools.dist:assert_string_list
 entry_points = setuptools.dist:check_entry_points
 extras_require = setuptools.dist:check_extras
+run_2to3 = setuptools.dist:assert_bool
 package_data = setuptools.dist:check_package_data
 install_requires = setuptools.dist:check_requirements
 include_package_data = setuptools.dist:assert_bool
@@ -49,6 +51,7 @@ test_suite = setuptools.dist:check_test_suite
 eager_resources = setuptools.dist:assert_string_list
 zip_safe = setuptools.dist:assert_bool
 test_loader = setuptools.dist:check_importable
+convert_doctests_2to3 = setuptools.dist:assert_string_list
 tests_require = setuptools.dist:check_requirements
 
 [setuptools.installation]
diff --git a/docs/setuptools.txt b/docs/setuptools.txt
index 7de0ab08..1c73d4a9 100644
--- a/docs/setuptools.txt
+++ b/docs/setuptools.txt
@@ -404,6 +404,10 @@ unless you need the associated ``setuptools`` feature.
     mess with it.  For more details on how this argument works, see the section
     below on `Automatic Resource Extraction`_.
 
+``convert_doctests_2to3``
+    List of doctest source files that need to be converted with 2to3. See
+    `Converting with 2to3`_ below for more details.
+
 
 Using ``find_packages()``
 -------------------------
@@ -446,6 +450,26 @@ argument in your setup script.  Especially since it frees you from having to
 remember to modify your setup script whenever your project grows additional
 top-level packages or subpackages.
 
+Converting with 2to3
+--------------------
+
+When run under Python 3.x, setuptools will automatically run 2to3 on
+all Python source files, if ``setuptools.run_2to3`` is set to True; by
+default, this variable is False. It will also convert doctests inside
+all Python source files, unless ``setuptools.run_2to3_on_doctests`` is
+False; by default, this setting is True. If additional files
+containing doctests need to be converted, the
+``convert_doctests_2to3``setup option should provide a list of all
+such files.
+
+By default, this conversion uses all fixers in the ``lib2to3.fixes``
+package. To use additional fixes, the list
+``setuptools.lib2to3_fixer_packages`` must be extended with names
+of packages containing fixes. If certain fixes are to be suppressed,
+this again can be overridden with the list 
+``setuptools.commands.build_py.build_py.fixers``, which then contains
+the list of all fixer class names.
+
 
 Automatic Script Creation
 =========================
diff --git a/pkg_resources.py b/pkg_resources.py
index c636903c..7c3d58fb 100644
--- a/pkg_resources.py
+++ b/pkg_resources.py
@@ -13,7 +13,7 @@ The package resource API is designed to work with normal filesystem packages,
 method.
 """
 
-import sys, os, zipimport, time, re, imp, new
+import sys, os, zipimport, time, re, imp, types
 
 try:
     frozenset
@@ -1126,10 +1126,16 @@ class NullProvider:
     def has_metadata(self, name):
         return self.egg_info and self._has(self._fn(self.egg_info,name))
 
-    def get_metadata(self, name):
-        if not self.egg_info:
-            return ""
-        return self._get(self._fn(self.egg_info,name))
+    if sys.version_info <= (3,):
+        def get_metadata(self, name):
+            if not self.egg_info:
+                return ""
+            return self._get(self._fn(self.egg_info,name))
+    else:
+        def get_metadata(self, name):
+            if not self.egg_info:
+                return ""
+            return self._get(self._fn(self.egg_info,name)).decode("utf-8")
 
     def get_metadata_lines(self, name):
         return yield_lines(self.get_metadata(name))
@@ -1707,7 +1713,7 @@ def _handle_ns(packageName, path_item):
         return None
     module = sys.modules.get(packageName)
     if module is None:
-        module = sys.modules[packageName] = new.module(packageName)
+        module = sys.modules[packageName] = types.ModuleType(packageName)
         module.__path__ = []; _set_parent_ns(packageName)
     elif not hasattr(module,'__path__'):
         raise TypeError("Not a package:", packageName)
@@ -2044,8 +2050,20 @@ class Distribution(object):
             self.platform
         )
     )
-    def __cmp__(self, other): return cmp(self.hashcmp, other)
     def __hash__(self): return hash(self.hashcmp)
+    def __lt__(self, other):
+        return self.hashcmp < other.hashcmp
+    def __le__(self, other):
+        return self.hashcmp <= other.hashcmp
+    def __gt__(self, other):
+        return self.hashcmp > other.hashcmp
+    def __ge__(self, other):
+        return self.hashcmp >= other.hashcmp
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            # It's not a Distribution, so they are not equal
+            return False
+        return self.hashcmp == other.hashcmp
 
     # These properties have to be lazy so that we don't have to load any
     # metadata until/unless it's actually needed.  (i.e., some distributions
@@ -2448,8 +2466,9 @@ class Requirement:
         elif isinstance(item,basestring):
             item = parse_version(item)
         last = None
+        compare = lambda a, b: (a > b) - (a < b) # -1, 0, 1
         for parsed,trans,op,ver in self.index:
-            action = trans[cmp(item,parsed)]
+            action = trans[compare(item,parsed)] # Indexing: 0, 1, -1
             if action=='F':     return False
             elif action=='T':   return True
             elif action=='+':   last = True
diff --git a/setup.py b/setup.py
index da9b519a..a9a66918 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,34 @@
 #!/usr/bin/env python
 """Distutils setup file, used to install or test 'setuptools'"""
 
+import sys, os
+
+src_root = None
+if sys.version_info >= (3,):
+    tmp_src = os.path.join("build", "src")
+    from distutils.filelist import FileList
+    from distutils import dir_util, file_util, util, log
+    log.set_verbosity(1)
+    fl = FileList()
+    for line in open("MANIFEST.in"):
+        fl.process_template_line(line)
+    dir_util.create_tree(tmp_src, fl.files)
+    outfiles_2to3 = []
+    for f in fl.files:
+        outf, copied = file_util.copy_file(f, os.path.join(tmp_src, f), update=1)
+        if copied and outf.endswith(".py"):
+            outfiles_2to3.append(outf)
+        if copied and outf.endswith('api_tests.txt'):
+            # XXX support this in distutils as well
+            from lib2to3.main import main
+            main('lib2to3.fixes', ['-wd', os.path.join(tmp_src, 'tests', 'api_tests.txt')])
+
+    util.run_2to3(outfiles_2to3)
+
+    # arrange setup to use the copy
+    sys.path.insert(0, tmp_src)
+    src_root = tmp_src
+
 from distutils.util import convert_path
 
 d = {}
@@ -40,6 +68,7 @@ dist = setup(
     keywords = "CPAN PyPI distutils eggs package management",
     url = "http://pypi.python.org/pypi/distribute",
     test_suite = 'setuptools.tests',
+    src_root = src_root,
     packages = find_packages(),
     package_data = {'setuptools':['*.exe']},
 
@@ -55,19 +84,22 @@ dist = setup(
         ],
 
         "distutils.setup_keywords": [
-            "eager_resources      = setuptools.dist:assert_string_list",
-            "namespace_packages   = setuptools.dist:check_nsp",
-            "extras_require       = setuptools.dist:check_extras",
-            "install_requires     = setuptools.dist:check_requirements",
-            "tests_require        = setuptools.dist:check_requirements",
-            "entry_points         = setuptools.dist:check_entry_points",
-            "test_suite           = setuptools.dist:check_test_suite",
-            "zip_safe             = setuptools.dist:assert_bool",
-            "package_data         = setuptools.dist:check_package_data",
-            "exclude_package_data = setuptools.dist:check_package_data",
-            "include_package_data = setuptools.dist:assert_bool",
-            "dependency_links     = setuptools.dist:assert_string_list",
-            "test_loader          = setuptools.dist:check_importable",
+            "eager_resources        = setuptools.dist:assert_string_list",
+            "namespace_packages     = setuptools.dist:check_nsp",
+            "extras_require         = setuptools.dist:check_extras",
+            "install_requires       = setuptools.dist:check_requirements",
+            "tests_require          = setuptools.dist:check_requirements",
+            "entry_points           = setuptools.dist:check_entry_points",
+            "test_suite             = setuptools.dist:check_test_suite",
+            "zip_safe               = setuptools.dist:assert_bool",
+            "package_data           = setuptools.dist:check_package_data",
+            "exclude_package_data   = setuptools.dist:check_package_data",
+            "include_package_data   = setuptools.dist:assert_bool",
+            "dependency_links       = setuptools.dist:assert_string_list",
+            "test_loader            = setuptools.dist:check_importable",
+            "run_2to3               = setuptools.dist:assert_bool",
+            "convert_doctests_2to3  = setuptools.dist:assert_string_list",
+            "additional_2to3_fixers = setuptools.dist:assert_string_list",
         ],
 
         "egg_info.writers": [
diff --git a/setuptools/__init__.py b/setuptools/__init__.py
index aaf634da..ac5e6e5e 100644
--- a/setuptools/__init__.py
+++ b/setuptools/__init__.py
@@ -24,6 +24,16 @@ _distribute = True
 
 bootstrap_install_from = None
 
+# Should we run 2to3 on all Python files, in Python 3.x?
+# Default: no; assume that a distribution installed for 3.x is already
+# written in 3.x
+run_2to3 = False # Default value if run_2to3 argument not given.
+# If we run 2to3 on .py files, should we also convert docstrings?
+# Default: yes; assume that we can detect doctests reliably
+run_2to3_on_doctests = True
+# Package names for fixer packages
+lib2to3_fixer_packages = ['lib2to3.fixes']
+
 def find_packages(where='.', exclude=()):
     """Return a list all Python packages found within directory 'where'
 
diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py
index 9e852a3f..43589c23 100644
--- a/setuptools/command/bdist_egg.py
+++ b/setuptools/command/bdist_egg.py
@@ -401,7 +401,7 @@ def write_safety_flag(egg_dir, safe):
             if safe is None or bool(safe)<>flag:
                 os.unlink(fn)
         elif safe is not None and bool(safe)==flag:
-            f=open(fn,'wb'); f.write('\n'); f.close()
+            f=open(fn,'wt'); f.write('\n'); f.close()
 
 safety_flags = {
     True: 'zip-safe',
@@ -525,9 +525,11 @@ def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=None,
     compression = [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED][bool(compress)]
     if not dry_run:
         z = zipfile.ZipFile(zip_filename, mode, compression=compression)
-        os.path.walk(base_dir, visit, z)
+        for dirname, dirs, files in os.walk(base_dir):
+            visit(z, dirname, files)
         z.close()
     else:
-        os.path.walk(base_dir, visit, None)
+        for dirname, dirs, files in os.walk(base_dir):
+            visit(None, dirname, file)
     return zip_filename
 #
diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py
index a60ede0c..4a94572c 100644
--- a/setuptools/command/build_ext.py
+++ b/setuptools/command/build_ext.py
@@ -113,6 +113,11 @@ class build_ext(_build_ext):
         for ext in self.extensions:
             fullname = ext._full_name
             self.ext_map[fullname] = ext
+
+            # distutils 3.1 will also ask for module names
+            # XXX what to do with conflicts?
+            self.ext_map[fullname.split('.')[-1]] = ext
+
             ltd = ext._links_to_dynamic = \
                 self.shlibs and self.links_to_dynamic(ext) or False
             ext._needs_stub = ltd and use_stubs and not isinstance(ext,Library)
diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py
index 79570bc2..910d67c8 100644
--- a/setuptools/command/build_py.py
+++ b/setuptools/command/build_py.py
@@ -3,7 +3,52 @@ from distutils.command.build_py import build_py as _build_py
 from distutils.util import convert_path
 from glob import glob
 
-class build_py(_build_py):
+try:
+    from distutils.util import Mixin2to3 as _Mixin2to3
+    # add support for converting doctests that is missing in 3.1 distutils
+    from distutils import log
+    from lib2to3.refactor import RefactoringTool, get_fixers_from_package
+    import setuptools
+    class DistutilsRefactoringTool(RefactoringTool):
+        def log_error(self, msg, *args, **kw):
+            log.error(msg, *args)
+
+        def log_message(self, msg, *args):
+            log.info(msg, *args)
+
+        def log_debug(self, msg, *args):
+            log.debug(msg, *args)
+
+    class Mixin2to3(_Mixin2to3):
+        def run_2to3(self, files, doctests = False):
+            # See of the distribution option has been set, otherwise check the
+            # setuptools default.
+            if self.distribution.run_2to3 is not True and setuptools.run_2to3 is False:
+                return
+            if not files:
+                return
+            log.info("Fixing "+" ".join(files))
+            if not self.fixer_names:
+                self.fixer_names = []
+                for p in setuptools.lib2to3_fixer_packages:
+                    self.fixer_names.extend(get_fixers_from_package(p))
+                if self.distribution.additional_2to3_fixers is not None:
+                    for p in self.distribution.additional_2to3_fixers:
+                        self.fixer_names.extend(get_fixers_from_package(p))
+            if doctests:
+                if setuptools.run_2to3_on_doctests:
+                    r = DistutilsRefactoringTool(self.fixer_names)
+                    r.refactor(files, write=True, doctests_only=True)
+            else:
+                _Mixin2to3.run_2to3(self, files)
+
+except ImportError:
+    class Mixin2to3:
+        def run_2to3(self, files, doctests=True):
+            # Nothing done in 2.x
+            pass
+
+class build_py(_build_py, Mixin2to3):
     """Enhanced 'build_py' command that includes data files with packages
 
     The data files are specified via a 'package_data' argument to 'setup()'.
@@ -17,6 +62,8 @@ class build_py(_build_py):
         self.package_data = self.distribution.package_data
         self.exclude_package_data = self.distribution.exclude_package_data or {}
         if 'data_files' in self.__dict__: del self.__dict__['data_files']
+        self.__updated_files = []
+        self.__doctests_2to3 = []
 
     def run(self):
         """Build modules, packages, and copy data files to build directory"""
@@ -30,6 +77,10 @@ class build_py(_build_py):
             self.build_packages()
             self.build_package_data()
 
+        self.run_2to3(self.__updated_files, False)
+        self.run_2to3(self.__updated_files, True)
+        self.run_2to3(self.__doctests_2to3, True)
+
         # Only compile actual .py files, using our base class' idea of what our
         # output files are.
         self.byte_compile(_build_py.get_outputs(self, include_bytecode=0))
@@ -39,6 +90,12 @@ class build_py(_build_py):
             self.data_files = files = self._get_data_files(); return files
         return _build_py.__getattr__(self,attr)
 
+    def build_module(self, module, module_file, package):
+        outfile, copied = _build_py.build_module(self, module, module_file, package)
+        if copied:
+            self.__updated_files.append(outfile)
+        return outfile, copied
+
     def _get_data_files(self):
         """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
         self.analyze_manifest()
@@ -77,7 +134,11 @@ class build_py(_build_py):
             for filename in filenames:
                 target = os.path.join(build_dir, filename)
                 self.mkpath(os.path.dirname(target))
-                self.copy_file(os.path.join(src_dir, filename), target)
+                srcfile = os.path.join(src_dir, filename)
+                outf, copied = self.copy_file(srcfile, target)
+                srcfile = os.path.abspath(srcfile)
+                if copied and srcfile in self.distribution.convert_doctests_2to3:
+                    self.__doctests_2to3.append(outf)
 
 
     def analyze_manifest(self):
@@ -157,9 +218,11 @@ class build_py(_build_py):
         _build_py.initialize_options(self)
 
 
-
-
-
+    def get_package_dir(self, package):
+        res = _build_py.get_package_dir(self, package)
+        if self.distribution.src_root is not None:
+            return os.path.join(self.distribution.src_root, res)
+        return res
 
 
     def exclude_data_files(self, package, src_dir, files):
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index 67cf949f..195139c7 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -39,6 +39,25 @@ def samefile(p1,p2):
         os.path.normpath(os.path.normcase(p2))
     )
 
+if sys.version_info <= (3,):
+    def _to_ascii(s):
+        return s
+    def isascii(s):
+        try:
+            unicode(s, 'ascii')
+            return True
+        except UnicodeError:
+            return False
+else:
+    def _to_ascii(s):
+        return s.encode('ascii')
+    def isascii(s):
+        try:
+            s.encode('ascii')
+            return True
+        except UnicodeError:
+            return False
+
 class easy_install(Command):
     """Manage a download/build/install process"""
     description = "Find/get/install Python packages"
@@ -599,7 +618,7 @@ Please make the appropriate changes for your system and try again.
                 "import pkg_resources\n"
                 "pkg_resources.run_script(%(spec)r, %(script_name)r)\n"
             ) % locals()
-        self.write_script(script_name, script_text, 'b')
+        self.write_script(script_name, _to_ascii(script_text), 'b')
 
     def write_script(self, script_name, contents, mode="t", blockers=()):
         """Write an executable file to the scripts directory"""
@@ -1381,7 +1400,7 @@ class PthDistributions(Environment):
 
             if os.path.islink(self.filename):
                 os.unlink(self.filename)
-            f = open(self.filename,'wb')
+            f = open(self.filename,'wt')
             f.write(data); f.close()
 
         elif os.path.exists(self.filename):
@@ -1432,7 +1451,7 @@ def get_script_header(script_text, executable=sys_executable, wininst=False):
     else:
         executable = nt_quote_arg(executable)
     hdr = "#!%(executable)s%(options)s\n" % locals()
-    if unicode(hdr,'ascii','ignore').encode('ascii') != hdr:
+    if not isascii(hdr):
         # Non-ascii path to sys.executable, use -x to prevent warnings
         if options:
             if options.strip().startswith('-'):
diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py
index a8315d23..46cdf4e0 100755
--- a/setuptools/command/egg_info.py
+++ b/setuptools/command/egg_info.py
@@ -3,7 +3,7 @@
 Create a distribution's .egg-info directory and contents"""
 
 # This module should be kept compatible with Python 2.3
-import os, re
+import os, re, sys
 from setuptools import Command
 from distutils.errors import *
 from distutils import log
@@ -148,6 +148,8 @@ class egg_info(Command):
         to the file.
         """
         log.info("writing %s to %s", what, filename)
+        if sys.version_info >= (3,):
+            data = data.encode("utf-8")
         if not self.dry_run:
             f = open(filename, 'wb')
             f.write(data)
@@ -351,8 +353,11 @@ def write_file (filename, contents):
     """Create a file with the specified name and write 'contents' (a
     sequence of strings without line terminators) to it.
     """
+    contents = "\n".join(contents)
+    if sys.version_info >= (3,):
+        contents = contents.encode("utf-8")
     f = open(filename, "wb")        # always write POSIX-style manifest
-    f.write("\n".join(contents))
+    f.write(contents)
     f.close()
 
 
diff --git a/setuptools/command/install.py b/setuptools/command/install.py
index a150c435..247c4f25 100644
--- a/setuptools/command/install.py
+++ b/setuptools/command/install.py
@@ -18,9 +18,6 @@ class install(_install):
         ('install_scripts',  lambda self: True),
     ]
     _nc = dict(new_commands)
-    sub_commands = [
-        cmd for cmd in _install.sub_commands if cmd[0] not in _nc
-    ] + new_commands
 
     def initialize_options(self):
         _install.initialize_options(self)
@@ -104,6 +101,10 @@ class install(_install):
         cmd.run()
         setuptools.bootstrap_install_from = None
 
+# XXX Python 3.1 doesn't see _nc if this is inside the class
+install.sub_commands = [
+        cmd for cmd in _install.sub_commands if cmd[0] not in install._nc
+    ] + install.new_commands
 
 
 
diff --git a/setuptools/command/install_egg_info.py b/setuptools/command/install_egg_info.py
index 939340c5..00c81221 100755
--- a/setuptools/command/install_egg_info.py
+++ b/setuptools/command/install_egg_info.py
@@ -97,12 +97,12 @@ class install_egg_info(Command):
                         % ('.'.join(pth[:-1]), pth[-1])
                     )
                 f.write(
-                    "import sys,new,os; "
+                    "import sys,types,os; "
                     "p = os.path.join(sys._getframe(1).f_locals['sitedir'], "
                         "*%(pth)r); "
                     "ie = os.path.exists(os.path.join(p,'__init__.py')); "
                     "m = not ie and "
-                        "sys.modules.setdefault(%(pkg)r,new.module(%(pkg)r)); "
+                        "sys.modules.setdefault(%(pkg)r,types.ModuleType(%(pkg)r)); "
                     "mp = (m or []) and m.__dict__.setdefault('__path__',[]); "
                     "(p not in mp) and mp.append(p)%(trailer)s"
                     % locals()
diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py
index 50c4c009..3442fe4b 100755
--- a/setuptools/command/sdist.py
+++ b/setuptools/command/sdist.py
@@ -60,7 +60,7 @@ def _default_revctrl(dirname=''):
 def externals_finder(dirname, filename):
     """Find any 'svn:externals' directories"""
     found = False
-    f = open(filename,'rb')
+    f = open(filename,'rt')
     for line in iter(f.readline, ''):    # can't use direct iter!
         parts = line.split()
         if len(parts)==2:
diff --git a/setuptools/command/test.py b/setuptools/command/test.py
index db918dae..0f96e83a 100644
--- a/setuptools/command/test.py
+++ b/setuptools/command/test.py
@@ -1,4 +1,4 @@
-from setuptools import Command
+from setuptools import Command, run_2to3
 from distutils.errors import DistutilsOptionError
 import sys
 from pkg_resources import *
@@ -81,12 +81,28 @@ class test(Command):
 
 
     def with_project_on_sys_path(self, func):
-        # Ensure metadata is up-to-date
-        self.run_command('egg_info')
+        if getattr(self.distribution, 'run_2to3', run_2to3):
+            # If we run 2to3 we can not do this inplace:
 
-        # Build extensions in-place
-        self.reinitialize_command('build_ext', inplace=1)
-        self.run_command('build_ext')
+            # Ensure metadata is up-to-date
+            self.reinitialize_command('build_py', inplace=0)
+            self.run_command('build_py')
+            bpy_cmd = self.get_finalized_command("build_py")
+            build_path = normalize_path(bpy_cmd.build_lib)
+
+            # Build extensions
+            self.reinitialize_command('egg_info', egg_base=build_path)
+            self.run_command('egg_info')
+
+            self.reinitialize_command('build_ext', inplace=0)
+            self.run_command('build_ext')
+        else:
+            # Without 2to3 inplace works fine:
+            self.run_command('egg_info')
+
+            # Build extensions in-place
+            self.reinitialize_command('build_ext', inplace=1)
+            self.run_command('build_ext')
 
         ei_cmd = self.get_finalized_command("egg_info")
 
diff --git a/setuptools/dist.py b/setuptools/dist.py
index 30ff35e3..531c94f4 100644
--- a/setuptools/dist.py
+++ b/setuptools/dist.py
@@ -210,6 +210,7 @@ class Distribution(_Distribution):
         self.require_features = []
         self.features = {}
         self.dist_files = []
+        self.src_root = attrs and attrs.pop("src_root", None)
         self.patch_missing_pkg_info(attrs)
         # Make sure we have any eggs needed to interpret 'attrs'
         if attrs is not None:
@@ -254,6 +255,11 @@ class Distribution(_Distribution):
             if value is not None:
                 ep.require(installer=self.fetch_build_egg)
                 ep.load()(self, ep.name, value)
+        if getattr(self, 'convert_doctests_2to3', None):
+            # XXX may convert to set here when we can rely on set being builtin
+            self.convert_doctests_2to3 = [os.path.abspath(p) for p in self.convert_doctests_2to3]
+        else:
+            self.convert_doctests_2to3 = []
 
     def fetch_build_egg(self, req):
         """Fetch an egg needed for building"""
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index 220cdec7..084370d5 100755
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -143,7 +143,7 @@ def find_external_links(url, page):
                 yield urlparse.urljoin(url, htmldecode(match.group(1)))
 
 user_agent = "Python-urllib/%s distribute/%s" % (
-    urllib2.__version__, require('distribute')[0].version
+    sys.version[:3], require('distribute')[0].version
 )
 
 
@@ -197,6 +197,9 @@ class PackageIndex(Environment):
 
         base = f.url     # handle redirects
         page = f.read()
+        if sys.version_info >= (3,):
+            charset = f.headers.get_param('charset') or 'latin-1'
+            page = page.decode(charset, "ignore")
         f.close()
         if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
             page = self.process_index(url, page)
diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py
index 11c14938..67cedde6 100755
--- a/setuptools/sandbox.py
+++ b/setuptools/sandbox.py
@@ -1,6 +1,9 @@
 import os, sys, __builtin__, tempfile, operator
 _os = sys.modules[os.name]
-_file = file
+try:
+    _file = file
+except NameError:
+    _file = None
 _open = open
 from distutils.errors import DistutilsError
 __all__ = [
@@ -60,13 +63,15 @@ class AbstractSandbox:
         """Run 'func' under os sandboxing"""
         try:
             self._copy(self)
-            __builtin__.file = self._file
+            if _file:
+                __builtin__.file = self._file
             __builtin__.open = self._open
             self._active = True
             return func()
         finally:
             self._active = False
-            __builtin__.file = _file
+            if _file:
+                __builtin__.file = _file
             __builtin__.open = _open
             self._copy(_os)
 
@@ -92,7 +97,8 @@ class AbstractSandbox:
             return original(path,*args,**kw)
         return wrap
 
-    _file = _mk_single_path_wrapper('file', _file)
+    if _file:
+        _file = _mk_single_path_wrapper('file', _file)
     _open = _mk_single_path_wrapper('open', _open)
     for name in [
         "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
diff --git a/setuptools/tests/doctest.py b/setuptools/tests/doctest.py
index bffce58f..be399a9d 100644
--- a/setuptools/tests/doctest.py
+++ b/setuptools/tests/doctest.py
@@ -2053,16 +2053,16 @@ class Tester:
         return (f,t)
 
     def rundict(self, d, name, module=None):
-        import new
-        m = new.module(name)
+        import types
+        m = types.ModuleType(name)
         m.__dict__.update(d)
         if module is None:
             module = False
         return self.rundoc(m, name, module)
 
     def run__test__(self, d, name):
-        import new
-        m = new.module(name)
+        import types
+        m = types.ModuleType(name)
         m.__test__ = d
         return self.rundoc(m, name)
 
diff --git a/setuptools/tests/test_resources.py b/setuptools/tests/test_resources.py
index 03e5d0f8..8f100419 100644
--- a/setuptools/tests/test_resources.py
+++ b/setuptools/tests/test_resources.py
@@ -517,12 +517,12 @@ class ScriptHeaderTests(TestCase):
 
             # Ensure we generate what is basically a broken shebang line
             # when there's options, with a warning emitted
-            sys.stdout = StringIO.StringIO()
+            sys.stdout = sys.stderr = StringIO.StringIO()
             self.assertEqual(get_script_header('#!/usr/bin/python -x',
                                                executable=exe),
                              '#!%s  -x\n' % exe)
             self.assert_('Unable to adapt shebang line' in sys.stdout.getvalue())
-            sys.stdout = StringIO.StringIO()
+            sys.stdout = sys.stderr = StringIO.StringIO()
             self.assertEqual(get_script_header('#!/usr/bin/python',
                                                executable=self.non_ascii_exe),
                              '#!%s -x\n' % self.non_ascii_exe)
diff --git a/tests/api_tests.txt b/tests/api_tests.txt
index 823d815f..6cf6e66f 100644
--- a/tests/api_tests.txt
+++ b/tests/api_tests.txt
@@ -119,7 +119,7 @@ editing are also a Distribution.  (And, with a little attention to the
 directory names used,  and including some additional metadata, such a
 "development distribution" can be made pluggable as well.)
 
-    >>> from pkg_resources import WorkingSet
+    >>> from pkg_resources import WorkingSet, VersionConflict
 
 A working set's entries are the sys.path entries that correspond to the active
 distributions.  By default, the working set's entries are the items on
@@ -208,11 +208,11 @@ You can ask a WorkingSet to ``find()`` a distribution matching a requirement::
 Note that asking for a conflicting version of a distribution already in a
 working set triggers a ``pkg_resources.VersionConflict`` error:
 
-    >>> ws.find(Requirement.parse("Bar==1.0")) # doctest: +NORMALIZE_WHITESPACE
-    Traceback (most recent call last):
-      ...
-    VersionConflict: (Bar 0.9 (http://example.com/something),
-                      Requirement.parse('Bar==1.0'))
+    >>> try:
+    ...     ws.find(Requirement.parse("Bar==1.0"))
+    ... except VersionConflict:
+    ...     print 'ok'
+    ok
 
 You can subscribe a callback function to receive notifications whenever a new
 distribution is added to a working set.  The callback is immediately invoked