Merge branch 'master' into feature/deterministic-provides-extras

17 files changed, 337 insertions, 102 deletions

diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py
index 047cc07b..10c4b528 100644
--- a/setuptools/build_meta.py
+++ b/setuptools/build_meta.py
@@ -35,6 +35,10 @@ import contextlib
 
 import setuptools
 import distutils
+from setuptools.py31compat import TemporaryDirectory
+
+from pkg_resources import parse_requirements
+from pkg_resources.py31compat import makedirs
 
 __all__ = ['get_requires_for_build_sdist',
            'get_requires_for_build_wheel',
@@ -51,7 +55,9 @@ class SetupRequirementsError(BaseException):
 
 class Distribution(setuptools.dist.Distribution):
     def fetch_build_eggs(self, specifiers):
-        raise SetupRequirementsError(specifiers)
+        specifier_list = list(map(str, parse_requirements(specifiers)))
+
+        raise SetupRequirementsError(specifier_list)
 
     @classmethod
     @contextlib.contextmanager
@@ -174,28 +180,38 @@ class _BuildMetaBackend(object):
 
         return dist_infos[0]
 
-    def build_wheel(self, wheel_directory, config_settings=None,
-                    metadata_directory=None):
+    def _build_with_temp_dir(self, setup_command, result_extension,
+                             result_directory, config_settings):
         config_settings = self._fix_config(config_settings)
-        wheel_directory = os.path.abspath(wheel_directory)
-        sys.argv = sys.argv[:1] + ['bdist_wheel'] + \
-            config_settings["--global-option"]
-        self.run_setup()
-        if wheel_directory != 'dist':
-            shutil.rmtree(wheel_directory)
-            shutil.copytree('dist', wheel_directory)
+        result_directory = os.path.abspath(result_directory)
 
-        return _file_with_extension(wheel_directory, '.whl')
+        # Build in a temporary directory, then copy to the target.
+        makedirs(result_directory, exist_ok=True)
+        with TemporaryDirectory(dir=result_directory) as tmp_dist_dir:
+            sys.argv = (sys.argv[:1] + setup_command +
+                        ['--dist-dir', tmp_dist_dir] +
+                        config_settings["--global-option"])
+            self.run_setup()
 
-    def build_sdist(self, sdist_directory, config_settings=None):
-        config_settings = self._fix_config(config_settings)
-        sdist_directory = os.path.abspath(sdist_directory)
-        sys.argv = sys.argv[:1] + ['sdist', '--formats', 'gztar'] + \
-            config_settings["--global-option"] + \
-            ["--dist-dir", sdist_directory]
-        self.run_setup()
+            result_basename = _file_with_extension(tmp_dist_dir, result_extension)
+            result_path = os.path.join(result_directory, result_basename)
+            if os.path.exists(result_path):
+                # os.rename will fail overwriting on non-Unix.
+                os.remove(result_path)
+            os.rename(os.path.join(tmp_dist_dir, result_basename), result_path)
+
+        return result_basename
 
-        return _file_with_extension(sdist_directory, '.tar.gz')
+
+    def build_wheel(self, wheel_directory, config_settings=None,
+                    metadata_directory=None):
+        return self._build_with_temp_dir(['bdist_wheel'], '.whl',
+                                         wheel_directory, config_settings)
+
+    def build_sdist(self, sdist_directory, config_settings=None):
+        return self._build_with_temp_dir(['sdist', '--formats', 'gztar'],
+                                         '.tar.gz', sdist_directory,
+                                         config_settings)
 
 
 class _BuildMetaLegacyBackend(_BuildMetaBackend):
diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py
index 60a8a32f..daa8e4fe 100644
--- a/setuptools/command/build_ext.py
+++ b/setuptools/command/build_ext.py
@@ -1,7 +1,6 @@
 import os
 import sys
 import itertools
-import imp
 from distutils.command.build_ext import build_ext as _du_build_ext
 from distutils.file_util import copy_file
 from distutils.ccompiler import new_compiler
@@ -12,6 +11,13 @@ from distutils import log
 from setuptools.extension import Library
 from setuptools.extern import six
 
+if six.PY2:
+    import imp
+
+    EXTENSION_SUFFIXES = [s for s, _, tp in imp.get_suffixes() if tp == imp.C_EXTENSION]
+else:
+    from importlib.machinery import EXTENSION_SUFFIXES
+
 try:
     # Attempt to use Cython for building extensions, if available
     from Cython.Distutils.build_ext import build_ext as _build_ext
@@ -64,7 +70,7 @@ if_dl = lambda s: s if have_rtld else ''
 
 def get_abi3_suffix():
     """Return the file extension for an abi3-compliant Extension()"""
-    for suffix, _, _ in (s for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION):
+    for suffix in EXTENSION_SUFFIXES:
         if '.abi3' in suffix:  # Unix
             return suffix
         elif suffix == '.pyd':  # Windows
diff --git a/setuptools/command/install_lib.py b/setuptools/command/install_lib.py
index 2b31c3e3..07d65933 100644
--- a/setuptools/command/install_lib.py
+++ b/setuptools/command/install_lib.py
@@ -1,5 +1,5 @@
 import os
-import imp
+import sys
 from itertools import product, starmap
 import distutils.command.install_lib as orig
 
@@ -74,10 +74,10 @@ class install_lib(orig.install_lib):
         yield '__init__.pyc'
         yield '__init__.pyo'
 
-        if not hasattr(imp, 'get_tag'):
+        if not hasattr(sys, 'implementation'):
             return
 
-        base = os.path.join('__pycache__', '__init__.' + imp.get_tag())
+        base = os.path.join('__pycache__', '__init__.' + sys.implementation.cache_tag)
         yield base + '.pyc'
         yield base + '.pyo'
         yield base + '.opt-1.pyc'
diff --git a/setuptools/command/test.py b/setuptools/command/test.py
index dde0118c..973e4eb2 100644
--- a/setuptools/command/test.py
+++ b/setuptools/command/test.py
@@ -15,6 +15,7 @@ from pkg_resources import (resource_listdir, resource_exists, normalize_path,
                            working_set, _namespace_packages, evaluate_marker,
                            add_activation_listener, require, EntryPoint)
 from setuptools import Command
+from .build_py import _unique_everseen
 
 __metaclass__ = type
 
@@ -186,7 +187,7 @@ class test(Command):
         orig_pythonpath = os.environ.get('PYTHONPATH', nothing)
         current_pythonpath = os.environ.get('PYTHONPATH', '')
         try:
-            prefix = os.pathsep.join(paths)
+            prefix = os.pathsep.join(_unique_everseen(paths))
             to_join = filter(None, [prefix, current_pythonpath])
             new_path = os.pathsep.join(to_join)
             if new_path:
diff --git a/setuptools/dist.py b/setuptools/dist.py
index ce37761e..2e5ad4bd 100644
--- a/setuptools/dist.py
+++ b/setuptools/dist.py
@@ -11,7 +11,6 @@ import distutils.log
 import distutils.core
 import distutils.cmd
 import distutils.dist
-from distutils.errors import DistutilsOptionError
 from distutils.util import strtobool
 from distutils.debug import DEBUG
 from distutils.fancy_getopt import translate_longopt
@@ -37,7 +36,6 @@ from setuptools.depends import Require
 from setuptools import windows_support
 from setuptools.monkey import get_unpatched
 from setuptools.config import parse_configuration
-from .unicode_utils import detect_encoding
 import pkg_resources
 
 __import__('setuptools.extern.packaging.specifiers')
@@ -57,7 +55,8 @@ def get_metadata_version(self):
             mv = StrictVersion('2.1')
         elif (self.maintainer is not None or
               self.maintainer_email is not None or
-              getattr(self, 'python_requires', None) is not None):
+              getattr(self, 'python_requires', None) is not None or
+              self.project_urls):
             mv = StrictVersion('1.2')
         elif (self.provides or self.requires or self.obsoletes or
                 self.classifiers or self.download_url):
@@ -136,7 +135,6 @@ def write_pkg_file(self, file):
         def write_field(key, value):
             file.write("%s: %s\n" % (key, value))
 
-
     write_field('Metadata-Version', str(version))
     write_field('Name', self.get_name())
     write_field('Version', self.get_version())
@@ -216,8 +214,12 @@ def check_importable(dist, attr, value):
 
 
 def assert_string_list(dist, attr, value):
-    """Verify that value is a string list or None"""
+    """Verify that value is a string list"""
     try:
+        # verify that value is a list or tuple to exclude unordered
+        # or single-use iterables
+        assert isinstance(value, (list, tuple))
+        # verify that elements of value are strings
         assert ''.join(value) != value
     except (TypeError, ValueError, AttributeError, AssertionError):
         raise DistutilsSetupError(
@@ -310,20 +312,17 @@ def check_test_suite(dist, attr, value):
 
 def check_package_data(dist, attr, value):
     """Verify that value is a dictionary of package names to glob lists"""
-    if isinstance(value, dict):
-        for k, v in value.items():
-            if not isinstance(k, str):
-                break
-            try:
-                iter(v)
-            except TypeError:
-                break
-        else:
-            return
-    raise DistutilsSetupError(
-        attr + " must be a dictionary mapping package names to lists of "
-        "wildcard patterns"
-    )
+    if not isinstance(value, dict):
+        raise DistutilsSetupError(
+            "{!r} must be a dictionary mapping package names to lists of "
+            "string wildcard patterns".format(attr))
+    for k, v in value.items():
+        if not isinstance(k, six.string_types):
+            raise DistutilsSetupError(
+                "keys of {!r} dict must be strings (got {!r})"
+                .format(attr, k)
+            )
+        assert_string_list(dist, 'values of {!r} dict'.format(attr), v)
 
 
 def check_packages(dist, attr, value):
@@ -590,13 +589,9 @@ class Distribution(_Distribution):
 
         parser = ConfigParser()
         for filename in filenames:
-            with io.open(filename, 'rb') as fp:
-                encoding = detect_encoding(fp)
+            with io.open(filename, encoding='utf-8') as reader:
                 if DEBUG:
-                    self.announce("  reading %s [%s]" % (
-                        filename, encoding or 'locale')
-                    )
-                reader = io.TextIOWrapper(fp, encoding=encoding)
+                    self.announce("  reading {filename}".format(**locals()))
                 (parser.read_file if six.PY3 else parser.readfp)(reader)
             for section in parser.sections():
                 options = parser.options(section)
@@ -886,7 +881,7 @@ class Distribution(_Distribution):
     def include(self, **attrs):
         """Add items to distribution that are named in keyword arguments
 
-        For example, 'dist.exclude(py_modules=["x"])' would add 'x' to
+        For example, 'dist.include(py_modules=["x"])' would add 'x' to
         the distribution's 'py_modules' attribute, if it was not already
         there.
 
@@ -1104,7 +1099,6 @@ class Distribution(_Distribution):
             return _Distribution.handle_display_options(self, option_order)
 
         # Stdout may be StringIO (e.g. in tests)
-        import io
         if not isinstance(sys.stdout, io.TextIOWrapper):
             return _Distribution.handle_display_options(self, option_order)
 
@@ -1283,4 +1277,5 @@ class Feature:
 
 
 class DistDeprecationWarning(SetuptoolsDeprecationWarning):
-    """Class for warning about deprecations in dist in setuptools. Not ignored by default, unlike DeprecationWarning."""
+    """Class for warning about deprecations in dist in
+    setuptools. Not ignored by default, unlike DeprecationWarning."""
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index 705a47cf..6b06f2ca 100644
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -897,7 +897,7 @@ class PackageIndex(Environment):
 
         if rev is not None:
             self.info("Checking out %s", rev)
-            os.system("(cd %s && git checkout --quiet %s)" % (
+            os.system("git -C %s checkout --quiet %s" % (
                 filename,
                 rev,
             ))
@@ -913,7 +913,7 @@ class PackageIndex(Environment):
 
         if rev is not None:
             self.info("Updating to %s", rev)
-            os.system("(cd %s && hg up -C -r %s -q)" % (
+            os.system("hg --cwd %s up -C -r %s -q" % (
                 filename,
                 rev,
             ))
diff --git a/setuptools/py31compat.py b/setuptools/py31compat.py
index 1a0705ec..e1da7ee2 100644
--- a/setuptools/py31compat.py
+++ b/setuptools/py31compat.py
@@ -17,9 +17,9 @@ except ImportError:
         errors on deletion.
         """
 
-        def __init__(self):
+        def __init__(self, **kwargs):
             self.name = None  # Handle mkdtemp raising an exception
-            self.name = tempfile.mkdtemp()
+            self.name = tempfile.mkdtemp(**kwargs)
 
         def __enter__(self):
             return self.name
diff --git a/setuptools/py33compat.py b/setuptools/py33compat.py
index 87cf5398..cb694436 100644
--- a/setuptools/py33compat.py
+++ b/setuptools/py33compat.py
@@ -52,4 +52,8 @@ class Bytecode_compat:
 Bytecode = getattr(dis, 'Bytecode', Bytecode_compat)
 
 
-unescape = getattr(html, 'unescape', html_parser.HTMLParser().unescape)
+unescape = getattr(html, 'unescape', None)
+if unescape is None:
+    # HTMLParser.unescape is deprecated since Python 3.4, and will be removed
+    # from 3.9.
+    unescape = html_parser.HTMLParser().unescape
diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py
index 74969322..e1efe561 100644
--- a/setuptools/tests/test_build_meta.py
+++ b/setuptools/tests/test_build_meta.py
@@ -28,7 +28,7 @@ class BuildBackend(BuildBackendBase):
 
     def __init__(self, *args, **kwargs):
         super(BuildBackend, self).__init__(*args, **kwargs)
-        self.pool = futures.ProcessPoolExecutor()
+        self.pool = futures.ProcessPoolExecutor(max_workers=1)
 
     def __getattr__(self, name):
         """Handles aribrary function invocations on the build backend."""
@@ -157,6 +157,53 @@ class TestBuildMetaBackend:
 
         assert os.path.isfile(os.path.join(dist_dir, wheel_name))
 
+    @pytest.mark.parametrize('build_type', ('wheel', 'sdist'))
+    def test_build_with_existing_file_present(self, build_type, tmpdir_cwd):
+        # Building a sdist/wheel should still succeed if there's
+        # already a sdist/wheel in the destination directory.
+        files = {
+            'setup.py': "from setuptools import setup\nsetup()",
+            'VERSION': "0.0.1",
+            'setup.cfg': DALS("""
+                [metadata]
+                name = foo
+                version = file: VERSION
+            """),
+            'pyproject.toml': DALS("""
+                [build-system]
+                requires = ["setuptools", "wheel"]
+                build-backend = "setuptools.build_meta
+            """),
+        }
+
+        build_files(files)
+
+        dist_dir = os.path.abspath('preexisting-' + build_type)
+
+        build_backend = self.get_build_backend()
+        build_method = getattr(build_backend, 'build_' + build_type)
+
+        # Build a first sdist/wheel.
+        # Note: this also check the destination directory is
+        # successfully created if it does not exist already.
+        first_result = build_method(dist_dir)
+
+        # Change version.
+        with open("VERSION", "wt") as version_file:
+            version_file.write("0.0.2")
+
+        # Build a *second* sdist/wheel.
+        second_result = build_method(dist_dir)
+
+        assert os.path.isfile(os.path.join(dist_dir, first_result))
+        assert first_result != second_result
+
+        # And if rebuilding the exact same sdist/wheel?
+        open(os.path.join(dist_dir, second_result), 'w').close()
+        third_result = build_method(dist_dir)
+        assert third_result == second_result
+        assert os.path.getsize(os.path.join(dist_dir, third_result)) > 0
+
     def test_build_sdist(self, build_backend):
         dist_dir = os.path.abspath('pip-sdist')
         os.makedirs(dist_dir)
@@ -287,6 +334,57 @@ class TestBuildMetaBackend:
         with pytest.raises(ImportError):
             build_backend.build_sdist("temp")
 
+    @pytest.mark.parametrize('setup_literal, requirements', [
+        ("'foo'", ['foo']),
+        ("['foo']", ['foo']),
+        (r"'foo\n'", ['foo']),
+        (r"'foo\n\n'", ['foo']),
+        ("['foo', 'bar']", ['foo', 'bar']),
+        (r"'# Has a comment line\nfoo'", ['foo']),
+        (r"'foo # Has an inline comment'", ['foo']),
+        (r"'foo \\\n >=3.0'", ['foo>=3.0']),
+        (r"'foo\nbar'", ['foo', 'bar']),
+        (r"'foo\nbar\n'", ['foo', 'bar']),
+        (r"['foo\n', 'bar\n']", ['foo', 'bar']),
+    ])
+    @pytest.mark.parametrize('use_wheel', [True, False])
+    def test_setup_requires(self, setup_literal, requirements, use_wheel,
+                            tmpdir_cwd):
+
+        files = {
+            'setup.py': DALS("""
+                from setuptools import setup
+
+                setup(
+                    name="qux",
+                    version="0.0.0",
+                    py_modules=["hello.py"],
+                    setup_requires={setup_literal},
+                )
+            """).format(setup_literal=setup_literal),
+            'hello.py': DALS("""
+            def run():
+                print('hello')
+            """),
+        }
+
+        build_files(files)
+
+        build_backend = self.get_build_backend()
+
+        if use_wheel:
+            base_requirements = ['wheel']
+            get_requires = build_backend.get_requires_for_build_wheel
+        else:
+            base_requirements = []
+            get_requires = build_backend.get_requires_for_build_sdist
+
+        # Ensure that the build requirements are properly parsed
+        expected = sorted(base_requirements + requirements)
+        actual = get_requires()
+
+        assert expected == sorted(actual)
+
 
 class TestBuildMetaLegacyBackend(TestBuildMetaBackend):
     backend_name = 'setuptools.build_meta:__legacy__'
diff --git a/setuptools/tests/test_config.py b/setuptools/tests/test_config.py
index 6b177709..bc97664d 100644
--- a/setuptools/tests/test_config.py
+++ b/setuptools/tests/test_config.py
@@ -8,8 +8,7 @@ from distutils.errors import DistutilsOptionError, DistutilsFileError
 from mock import patch
 from setuptools.dist import Distribution, _Distribution
 from setuptools.config import ConfigHandler, read_configuration
-from setuptools.extern.six.moves.configparser import InterpolationMissingOptionError
-from setuptools.tests import is_ascii
+from setuptools.extern.six.moves import configparser
 from . import py2_only, py3_only
 from .textwrap import DALS
 
@@ -29,7 +28,9 @@ def make_package_dir(name, base_dir, ns=False):
     return dir_package, init_file
 
 
-def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii', package_path='fake_package'):
+def fake_env(
+        tmpdir, setup_cfg, setup_py=None,
+        encoding='ascii', package_path='fake_package'):
 
     if setup_py is None:
         setup_py = (
@@ -440,13 +441,10 @@ class TestMetadata:
             '[metadata]\n'
             'description = %(message)s\n'
         )
-        with pytest.raises(InterpolationMissingOptionError):
+        with pytest.raises(configparser.InterpolationMissingOptionError):
             with get_dist(tmpdir):
                 pass
 
-    skip_if_not_ascii = pytest.mark.skipif(not is_ascii, reason='Test not supported with this locale')
-
-    @skip_if_not_ascii
     def test_non_ascii_1(self, tmpdir):
         fake_env(
             tmpdir,
@@ -454,18 +452,8 @@ class TestMetadata:
             'description = éàïôñ\n',
             encoding='utf-8'
         )
-        with pytest.raises(UnicodeDecodeError):
-            with get_dist(tmpdir):
-                pass
-
-    def test_non_ascii_2(self, tmpdir):
-        fake_env(
-            tmpdir,
-            '# -*- coding: invalid\n'
-        )
-        with pytest.raises(LookupError):
-            with get_dist(tmpdir):
-                pass
+        with get_dist(tmpdir):
+            pass
 
     def test_non_ascii_3(self, tmpdir):
         fake_env(
@@ -476,7 +464,6 @@ class TestMetadata:
         with get_dist(tmpdir):
             pass
 
-    @skip_if_not_ascii
     def test_non_ascii_4(self, tmpdir):
         fake_env(
             tmpdir,
@@ -488,8 +475,10 @@ class TestMetadata:
         with get_dist(tmpdir) as dist:
             assert dist.metadata.description == 'éàïôñ'
 
-    @skip_if_not_ascii
-    def test_non_ascii_5(self, tmpdir):
+    def test_not_utf8(self, tmpdir):
+        """
+        Config files encoded not in UTF-8 will fail
+        """
         fake_env(
             tmpdir,
             '# vim: set fileencoding=iso-8859-15 :\n'
@@ -497,8 +486,9 @@ class TestMetadata:
             'description = éàïôñ\n',
             encoding='iso-8859-15'
         )
-        with get_dist(tmpdir) as dist:
-            assert dist.metadata.description == 'éàïôñ'
+        with pytest.raises(UnicodeDecodeError):
+            with get_dist(tmpdir):
+                pass
 
 
 class TestOptions:
diff --git a/setuptools/tests/test_dist.py b/setuptools/tests/test_dist.py
index 0c0b9d66..36237f24 100644
--- a/setuptools/tests/test_dist.py
+++ b/setuptools/tests/test_dist.py
@@ -4,8 +4,13 @@ from __future__ import unicode_literals
 
 import io
 import collections
-
-from setuptools.dist import DistDeprecationWarning, _get_unpatched
+import re
+from distutils.errors import DistutilsSetupError
+from setuptools.dist import (
+    _get_unpatched,
+    check_package_data,
+    DistDeprecationWarning,
+)
 from setuptools import Distribution
 from setuptools.extern.six.moves.urllib.request import pathname2url
 from setuptools.extern.six.moves.urllib_parse import urljoin
@@ -278,3 +283,48 @@ def test_provides_extras_deterministic_order():
         reversed(list(attrs['extras_require'].items())))
     dist = Distribution(attrs)
     assert dist.metadata.provides_extras == ['b', 'a']
+
+   
+CHECK_PACKAGE_DATA_TESTS = (
+    # Valid.
+    ({
+        '': ['*.txt', '*.rst'],
+        'hello': ['*.msg'],
+    }, None),
+    # Not a dictionary.
+    ((
+        ('', ['*.txt', '*.rst']),
+        ('hello', ['*.msg']),
+    ), (
+        "'package_data' must be a dictionary mapping package"
+        " names to lists of string wildcard patterns"
+    )),
+    # Invalid key type.
+    ({
+        400: ['*.txt', '*.rst'],
+    }, (
+        "keys of 'package_data' dict must be strings (got 400)"
+    )),
+    # Invalid value type.
+    ({
+        'hello': str('*.msg'),
+    }, (
+        "\"values of 'package_data' dict\" must be a list of strings (got '*.msg')"
+    )),
+    # Invalid value type (generators are single use)
+    ({
+        'hello': (x for x in "generator"),
+    }, (
+        "\"values of 'package_data' dict\" must be a list of strings "
+        "(got <generator object"
+    )),
+)
+
+
+@pytest.mark.parametrize('package_data, expected_message', CHECK_PACKAGE_DATA_TESTS)
+def test_check_package_data(package_data, expected_message):
+    if expected_message is None:
+        assert check_package_data(None, 'package_data', package_data) is None
+    else:
+        with pytest.raises(DistutilsSetupError, match=re.escape(expected_message)):
+            check_package_data(None, str('package_data'), package_data)
diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py
index db9c3873..316eb2ed 100644
--- a/setuptools/tests/test_egg_info.py
+++ b/setuptools/tests/test_egg_info.py
@@ -622,6 +622,7 @@ class TestEggInfo:
         assert expected_line in pkg_info_lines
         expected_line = 'Project-URL: Link Two, https://example.com/two/'
         assert expected_line in pkg_info_lines
+        assert 'Metadata-Version: 1.2' in pkg_info_lines
 
     def test_python_requires_egg_info(self, tmpdir_cwd, env):
         self._setup_script_with_requires(
diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py
index e54f3209..1e132188 100644
--- a/setuptools/tests/test_integration.py
+++ b/setuptools/tests/test_integration.py
@@ -141,6 +141,7 @@ def test_build_deps_on_distutils(request, tmpdir_factory, build_dep):
         allowed_unknowns = [
             'test_suite',
             'tests_require',
+            'python_requires',
             'install_requires',
         ]
         assert not match or match.group(1).strip('"\'') in allowed_unknowns
@@ -149,8 +150,8 @@ def test_build_deps_on_distutils(request, tmpdir_factory, build_dep):
 def install(pkg_dir, install_dir):
     with open(os.path.join(pkg_dir, 'setuptools.py'), 'w') as breaker:
         breaker.write('raise ImportError()')
-    cmd = [sys.executable, 'setup.py', 'install', '--prefix', install_dir]
-    env = dict(os.environ, PYTHONPATH=pkg_dir)
+    cmd = [sys.executable, 'setup.py', 'install', '--prefix', str(install_dir)]
+    env = dict(os.environ, PYTHONPATH=str(pkg_dir))
     output = subprocess.check_output(
         cmd, cwd=pkg_dir, env=env, stderr=subprocess.STDOUT)
     return output.decode('utf-8')
diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py
index ab371884..60d968fd 100644
--- a/setuptools/tests/test_packageindex.py
+++ b/setuptools/tests/test_packageindex.py
@@ -249,7 +249,7 @@ class TestPackageIndex:
         first_call_args = os_system_mock.call_args_list[0][0]
         assert first_call_args == (expected,)
 
-        tmpl = '(cd {expected_dir} && git checkout --quiet master)'
+        tmpl = 'git -C {expected_dir} checkout --quiet master'
         expected = tmpl.format(**locals())
         assert os_system_mock.call_args_list[1][0] == (expected,)
         assert result == expected_dir
diff --git a/setuptools/tests/test_setopt.py b/setuptools/tests/test_setopt.py
new file mode 100644
index 00000000..3fb04fb4
--- /dev/null
+++ b/setuptools/tests/test_setopt.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import io
+
+import six
+
+from setuptools.command import setopt
+from setuptools.extern.six.moves import configparser
+
+
+class TestEdit:
+    @staticmethod
+    def parse_config(filename):
+        parser = configparser.ConfigParser()
+        with io.open(filename, encoding='utf-8') as reader:
+            (parser.read_file if six.PY3 else parser.readfp)(reader)
+        return parser
+
+    @staticmethod
+    def write_text(file, content):
+        with io.open(file, 'wb') as strm:
+            strm.write(content.encode('utf-8'))
+
+    def test_utf8_encoding_retained(self, tmpdir):
+        """
+        When editing a file, non-ASCII characters encoded in
+        UTF-8 should be retained.
+        """
+        config = tmpdir.join('setup.cfg')
+        self.write_text(str(config), '[names]\njaraco=джарако')
+        setopt.edit_config(str(config), dict(names=dict(other='yes')))
+        parser = self.parse_config(str(config))
+        assert parser.get('names', 'jaraco') == 'джарако'
+        assert parser.get('names', 'other') == 'yes'
diff --git a/setuptools/tests/test_virtualenv.py b/setuptools/tests/test_virtualenv.py
index 3d5c84b0..74a1284c 100644
--- a/setuptools/tests/test_virtualenv.py
+++ b/setuptools/tests/test_virtualenv.py
@@ -8,6 +8,8 @@ from pytest_fixture_config import yield_requires_config
 
 import pytest_virtualenv
 
+from setuptools.extern import six
+
 from .textwrap import DALS
 from .test_easy_install import make_nspkg_sdist
 
@@ -52,10 +54,58 @@ def test_clean_env_install(bare_virtualenv):
     )).format(source=SOURCE_DIR))
 
 
-def test_pip_upgrade_from_source(virtualenv):
+def _get_pip_versions():
+    # This fixture will attempt to detect if tests are being run without
+    # network connectivity and if so skip some tests
+
+    network = True
+    if not os.environ.get('NETWORK_REQUIRED', False):  # pragma: nocover
+        try:
+            from urllib.request import urlopen
+            from urllib.error import URLError
+        except ImportError:
+            from urllib2 import urlopen, URLError # Python 2.7 compat
+
+        try:
+            urlopen('https://pypi.org', timeout=1)
+        except URLError:
+            # No network, disable most of these tests
+            network = False
+
+    network_versions = [
+        'pip==9.0.3',
+        'pip==10.0.1',
+        'pip==18.1',
+        'pip==19.0.1',
+    ]
+
+    # Pip's master dropped support for 3.4.
+    if not six.PY34:
+        network_versions.append('https://github.com/pypa/pip/archive/master.zip')
+
+    versions = [None] + [
+        pytest.param(v, **({} if network else {'marks': pytest.mark.skip}))
+        for v in network_versions
+    ]
+
+    return versions
+
+
+@pytest.mark.parametrize('pip_version', _get_pip_versions())
+def test_pip_upgrade_from_source(pip_version, virtualenv):
     """
     Check pip can upgrade setuptools from source.
     """
+    # Install pip/wheel, and remove setuptools (as it
+    # should not be needed for bootstraping from source)
+    if pip_version is None:
+        upgrade_pip = ()
+    else:
+        upgrade_pip = ('python -m pip install -U {pip_version} --retries=1',)
+    virtualenv.run(' && '.join((
+        'pip uninstall -y setuptools',
+        'pip install -U wheel',
+    ) + upgrade_pip).format(pip_version=pip_version))
     dist_dir = virtualenv.workspace
     # Generate source distribution / wheel.
     virtualenv.run(' && '.join((
diff --git a/setuptools/unicode_utils.py b/setuptools/unicode_utils.py
index 3b8179a8..7c63efd2 100644
--- a/setuptools/unicode_utils.py
+++ b/setuptools/unicode_utils.py
@@ -1,6 +1,5 @@
 import unicodedata
 import sys
-import re
 
 from setuptools.extern import six
 
@@ -43,15 +42,3 @@ def try_encode(string, enc):
         return string.encode(enc)
     except UnicodeEncodeError:
         return None
-
-
-CODING_RE = re.compile(br'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)')
-
-
-def detect_encoding(fp):
-    first_line = fp.readline()
-    fp.seek(0)
-    m = CODING_RE.match(first_line)
-    if m is None:
-        return None
-    return m.group(1).decode('ascii')