Merge branch 'master' into bugfix/2232-adopt-distutils-defaultbugfix/2232-adopt-distutils-default

9 files changed, 135 insertions, 58 deletions

diff --git a/setuptools/__init__.py b/setuptools/__init__.py
index 83882511..99094230 100644
--- a/setuptools/__init__.py
+++ b/setuptools/__init__.py
@@ -1,17 +1,15 @@
 """Extensions to the 'distutils' for large or complex distributions"""
 
-import os
+from fnmatch import fnmatchcase
 import functools
+import os
+import re
 
-# Disabled for now due to: #2228, #2230
-import setuptools.distutils_patch  # noqa: F401
+import _distutils_hack.override  # noqa: F401
 
 import distutils.core
-import distutils.filelist
-import re
 from distutils.errors import DistutilsOptionError
 from distutils.util import convert_path
-from fnmatch import fnmatchcase
 
 from ._deprecation_warning import SetuptoolsDeprecationWarning
 
diff --git a/setuptools/_distutils/_msvccompiler.py b/setuptools/_distutils/_msvccompiler.py
index 0e98692e..2d56ee0a 100644
--- a/setuptools/_distutils/_msvccompiler.py
+++ b/setuptools/_distutils/_msvccompiler.py
@@ -15,7 +15,9 @@ for older versions in distutils.msvc9compiler and distutils.msvccompiler.
 
 import os
 import subprocess
-import winreg
+import contextlib
+with contextlib.suppress(ImportError):
+    import winreg
 
 from distutils.errors import DistutilsExecError, DistutilsPlatformError, \
                              CompileError, LibError, LinkError
@@ -501,7 +503,7 @@ class MSVCCompiler(CCompiler) :
             log.debug("skipping %s (up-to-date)", output_filename)
 
     def spawn(self, cmd):
-        env = dict(os.environ, path=self._paths)
+        env = dict(os.environ, PATH=self._paths)
         return super().spawn(cmd, env=env)
 
     # -- Miscellaneous methods -----------------------------------------
diff --git a/setuptools/_distutils/ccompiler.py b/setuptools/_distutils/ccompiler.py
index b5ef143e..57bb94e8 100644
--- a/setuptools/_distutils/ccompiler.py
+++ b/setuptools/_distutils/ccompiler.py
@@ -906,8 +906,8 @@ int main (int argc, char **argv) {
     def execute(self, func, args, msg=None, level=1):
         execute(func, args, msg, self.dry_run)
 
-    def spawn(self, cmd):
-        spawn(cmd, dry_run=self.dry_run)
+    def spawn(self, cmd, **kwargs):
+        spawn(cmd, dry_run=self.dry_run, **kwargs)
 
     def move_file(self, src, dst):
         return move_file(src, dst, dry_run=self.dry_run)
diff --git a/setuptools/_distutils/tests/test_msvccompiler.py b/setuptools/_distutils/tests/test_msvccompiler.py
index b518d6a7..88d912b1 100644
--- a/setuptools/_distutils/tests/test_msvccompiler.py
+++ b/setuptools/_distutils/tests/test_msvccompiler.py
@@ -2,6 +2,7 @@
 import sys
 import unittest
 import os
+import threading
 
 from distutils.errors import DistutilsPlatformError
 from distutils.tests import support
@@ -74,6 +75,42 @@ class msvccompilerTestCase(support.TempdirManager,
         else:
             raise unittest.SkipTest("VS 2015 is not installed")
 
+
+class CheckThread(threading.Thread):
+    exc_info = None
+
+    def run(self):
+        try:
+            super().run()
+        except Exception:
+            self.exc_info = sys.exc_info()
+
+    def __bool__(self):
+        return not self.exc_info
+
+
+class TestSpawn(unittest.TestCase):
+    def test_concurrent_safe(self):
+        """
+        Concurrent calls to spawn should have consistent results.
+        """
+        import distutils._msvccompiler as _msvccompiler
+        compiler = _msvccompiler.MSVCCompiler()
+        compiler._paths = "expected"
+        inner_cmd = 'import os; assert os.environ["PATH"] == "expected"'
+        command = ['python', '-c', inner_cmd]
+
+        threads = [
+            CheckThread(target=compiler.spawn, args=[command])
+            for n in range(100)
+        ]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+        assert all(threads)
+
+
 def test_suite():
     return unittest.makeSuite(msvccompilerTestCase)
 
diff --git a/setuptools/distutils_patch.py b/setuptools/distutils_patch.py
deleted file mode 100644
index e6b2aad5..00000000
--- a/setuptools/distutils_patch.py
+++ /dev/null
@@ -1,44 +0,0 @@
-"""
-Ensure that the local copy of distutils is preferred over stdlib.
-
-See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
-for more motivation.
-"""
-
-import sys
-import re
-import os
-import importlib
-import warnings
-
-
-def clear_distutils():
-    if 'distutils' not in sys.modules:
-        return
-    warnings.warn("Setuptools is replacing distutils")
-    mods = [name for name in sys.modules if re.match(r'distutils\b', name)]
-    for name in mods:
-        del sys.modules[name]
-
-
-def enabled():
-    """
-    Allow selection of distutils by environment variable.
-    """
-    which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local')
-    return which == 'local'
-
-
-def ensure_local_distutils():
-    clear_distutils()
-    distutils = importlib.import_module('setuptools._distutils')
-    distutils.__name__ = 'distutils'
-    sys.modules['distutils'] = distutils
-
-    # sanity check that submodules load as expected
-    core = importlib.import_module('distutils.core')
-    assert '_distutils' in core.__file__, core.__file__
-
-
-if enabled():
-    ensure_local_distutils()
diff --git a/setuptools/monkey.py b/setuptools/monkey.py
index 3c77f8cf..e5f1377b 100644
--- a/setuptools/monkey.py
+++ b/setuptools/monkey.py
@@ -138,7 +138,7 @@ def patch_for_msvc_specialized_compiler():
     msvc = import_module('setuptools.msvc')
 
     if platform.system() != 'Windows':
-        # Compilers only availables on Microsoft Windows
+        # Compilers only available on Microsoft Windows
         return
 
     def patch_params(mod_name, func_name):
diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py
index 93ae8eb4..24a36080 100644
--- a/setuptools/sandbox.py
+++ b/setuptools/sandbox.py
@@ -185,8 +185,8 @@ def setup_context(setup_dir):
     temp_dir = os.path.join(setup_dir, 'temp')
     with save_pkg_resources_state():
         with save_modules():
-            hide_setuptools()
             with save_path():
+                hide_setuptools()
                 with save_argv():
                     with override_temp(temp_dir):
                         with pushd(setup_dir):
@@ -195,6 +195,15 @@ def setup_context(setup_dir):
                             yield
 
 
+_MODULES_TO_HIDE = {
+    'setuptools',
+    'distutils',
+    'pkg_resources',
+    'Cython',
+    '_distutils_hack',
+}
+
+
 def _needs_hiding(mod_name):
     """
     >>> _needs_hiding('setuptools')
@@ -212,8 +221,8 @@ def _needs_hiding(mod_name):
     >>> _needs_hiding('Cython')
     True
     """
-    pattern = re.compile(r'(setuptools|pkg_resources|distutils|Cython)(\.|$)')
-    return bool(pattern.match(mod_name))
+    base_module = mod_name.split('.', 1)[0]
+    return base_module in _MODULES_TO_HIDE
 
 
 def hide_setuptools():
@@ -223,6 +232,10 @@ def hide_setuptools():
     necessary to avoid issues such as #315 where setuptools upgrading itself
     would fail to find a function declared in the metadata.
     """
+    _distutils_hack = sys.modules.get('_distutils_hack', None)
+    if _distutils_hack is not None:
+        _distutils_hack.remove_shim()
+
     modules = filter(_needs_hiding, sys.modules)
     _clear_modules(modules)
 
diff --git a/setuptools/tests/requirements.txt b/setuptools/tests/requirements.txt
index 19bf5aef..d0d07f70 100644
--- a/setuptools/tests/requirements.txt
+++ b/setuptools/tests/requirements.txt
@@ -10,3 +10,4 @@ pytest-cov>=2.5.1
 paver; python_version>="3.6"
 futures; python_version=="2.7"
 pip>=19.1 # For proper file:// URLs support.
+jaraco.envs
diff --git a/setuptools/tests/test_distutils_adoption.py b/setuptools/tests/test_distutils_adoption.py
new file mode 100644
index 00000000..daccc473
--- /dev/null
+++ b/setuptools/tests/test_distutils_adoption.py
@@ -0,0 +1,70 @@
+import os
+import sys
+import functools
+import subprocess
+import platform
+
+import pytest
+import jaraco.envs
+import path
+
+
+IS_PYPY = '__pypy__' in sys.builtin_module_names
+
+
+class VirtualEnv(jaraco.envs.VirtualEnv):
+    name = '.env'
+
+    def run(self, cmd, *args, **kwargs):
+        cmd = [self.exe(cmd[0])] + cmd[1:]
+        return subprocess.check_output(cmd, *args, cwd=self.root, **kwargs)
+
+
+@pytest.fixture
+def venv(tmpdir):
+    env = VirtualEnv()
+    env.root = path.Path(tmpdir)
+    env.req = os.getcwd()
+    return env.create()
+
+
+def popen_text(call):
+    """
+    Augment the Popen call with the parameters to ensure unicode text.
+    """
+    return functools.partial(call, universal_newlines=True) \
+        if sys.version_info < (3, 7) else functools.partial(call, text=True)
+
+
+def find_distutils(venv, imports='distutils', env=None, **kwargs):
+    py_cmd = 'import {imports}; print(distutils.__file__)'.format(**locals())
+    cmd = ['python', '-c', py_cmd]
+    if platform.system() == 'Windows':
+        env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
+    return popen_text(venv.run)(cmd, env=env, **kwargs)
+
+
+def test_distutils_stdlib(venv):
+    """
+    Ensure stdlib distutils is used when appropriate.
+    """
+    assert venv.name not in find_distutils(venv, env=dict()).split(os.sep)
+
+
+def test_distutils_local_with_setuptools(venv):
+    """
+    Ensure local distutils is used when appropriate.
+    """
+    env = dict(SETUPTOOLS_USE_DISTUTILS='local')
+    loc = find_distutils(venv, imports='setuptools, distutils', env=env)
+    assert venv.name in loc.split(os.sep)
+
+
+@pytest.mark.xfail('IS_PYPY', reason='pypy imports distutils on startup')
+def test_distutils_local(venv):
+    """
+    Even without importing, the setuptools-local copy of distutils is
+    preferred.
+    """
+    env = dict(SETUPTOOLS_USE_DISTUTILS='local')
+    assert venv.name in find_distutils(venv, env=env).split(os.sep)