Merge pull request #2305 from pypa/distutils-import-hack

Prefer included distutils even without importing setuptools. Closes #2259.

8 files changed, 118 insertions, 22 deletions

diff --git a/setuptools/distutils_patch.py b/_distutils_hack/__init__.py
index 33f1e7f9..71fa7ce1 100644
--- a/setuptools/distutils_patch.py
+++ b/_distutils_hack/__init__.py
@@ -1,13 +1,6 @@
-"""
-Ensure that the local copy of distutils is preferred over stdlib.
-
-See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
-for more motivation.
-"""
-
 import sys
-import re
 import os
+import re
 import importlib
 import warnings
 
@@ -56,6 +49,48 @@ def ensure_local_distutils():
     assert '_distutils' in core.__file__, core.__file__
 
 
-warn_distutils_present()
-if enabled():
-    ensure_local_distutils()
+def do_override():
+    """
+    Ensure that the local copy of distutils is preferred over stdlib.
+
+    See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
+    for more motivation.
+    """
+    warn_distutils_present()
+    if enabled():
+        ensure_local_distutils()
+
+
+class DistutilsMetaFinder:
+    def find_spec(self, fullname, path, target=None):
+        if path is not None or fullname != "distutils":
+            return None
+
+        return self.get_distutils_spec()
+
+    def get_distutils_spec(self):
+        import importlib.util
+
+        class DistutilsLoader(importlib.util.abc.Loader):
+
+            def create_module(self, spec):
+                return importlib.import_module('._distutils', 'setuptools')
+
+            def exec_module(self, module):
+                pass
+
+        return importlib.util.spec_from_loader('distutils', DistutilsLoader())
+
+
+DISTUTILS_FINDER = DistutilsMetaFinder()
+
+
+def add_shim():
+    sys.meta_path.insert(0, DISTUTILS_FINDER)
+
+
+def remove_shim():
+    try:
+        sys.meta_path.remove(DISTUTILS_FINDER)
+    except ValueError:
+        pass
diff --git a/_distutils_hack/override.py b/_distutils_hack/override.py
new file mode 100644
index 00000000..2cc433a4
--- /dev/null
+++ b/_distutils_hack/override.py
@@ -0,0 +1 @@
+__import__('_distutils_hack').do_override()
diff --git a/changelog.d/2259.change.rst b/changelog.d/2259.change.rst
new file mode 100644
index 00000000..43701ec2
--- /dev/null
+++ b/changelog.d/2259.change.rst
@@ -0,0 +1 @@
+Setuptools now provides a .pth file (except for editable installs of setuptools) to the target environment to ensure that when enabled, the setuptools-provided distutils is preferred before setuptools has been imported (and even if setuptools is never imported). Honors the SETUPTOOLS_USE_DISTUTILS environment variable.
diff --git a/conftest.py b/conftest.py
index 6013e187..25537f56 100644
--- a/conftest.py
+++ b/conftest.py
@@ -15,7 +15,7 @@ collect_ignore = [
     'tests/manual_test.py',
     'setuptools/tests/mod_with_constant.py',
     'setuptools/_distutils',
-    'setuptools/distutils_patch.py',
+    '_distutils_hack',
 ]
 
 
diff --git a/setup.py b/setup.py
index 1fe18bd1..5d98c029 100755
--- a/setup.py
+++ b/setup.py
@@ -5,8 +5,10 @@ Distutils setup file, used to install or test 'setuptools'
 
 import os
 import sys
+import textwrap
 
 import setuptools
+from setuptools.command.install import install
 
 here = os.path.dirname(__file__)
 
@@ -81,8 +83,47 @@ def pypi_link(pkg_filename):
     return '/'.join(parts)
 
 
+class install_with_pth(install):
+    """
+    Custom install command to install a .pth file for distutils patching.
+
+    This hack is necessary because there's no standard way to install behavior
+    on startup (and it's debatable if there should be one). This hack (ab)uses
+    the `extra_path` behavior in Setuptools to install a `.pth` file with
+    implicit behavior on startup to give higher precedence to the local version
+    of `distutils` over the version from the standard library.
+
+    Please do not replicate this behavior.
+    """
+
+    _pth_name = 'distutils-precedence'
+    _pth_contents = textwrap.dedent("""
+        import os
+        enabled = os.environ.get('SETUPTOOLS_USE_DISTUTILS') == 'local'
+        enabled and __import__('_distutils_hack').add_shim()
+        """).lstrip().replace('\n', '; ')
+
+    def initialize_options(self):
+        install.initialize_options(self)
+        self.extra_path = self._pth_name, self._pth_contents
+
+    def finalize_options(self):
+        install.finalize_options(self)
+        self._restore_install_lib()
+
+    def _restore_install_lib(self):
+        """
+        Undo secondary effect of `extra_path` adding to `install_lib`
+        """
+        suffix = os.path.relpath(self.install_lib, self.install_libbase)
+
+        if suffix.strip() == self._pth_contents.strip():
+            self.install_lib = self.install_libbase
+
+
 setup_params = dict(
     src_root=None,
+    cmdclass={'install': install_with_pth},
     package_data=package_data,
     entry_points={
         "distutils.commands": [
diff --git a/setuptools/__init__.py b/setuptools/__init__.py
index 83882511..99094230 100644
--- a/setuptools/__init__.py
+++ b/setuptools/__init__.py
@@ -1,17 +1,15 @@
 """Extensions to the 'distutils' for large or complex distributions"""
 
-import os
+from fnmatch import fnmatchcase
 import functools
+import os
+import re
 
-# Disabled for now due to: #2228, #2230
-import setuptools.distutils_patch  # noqa: F401
+import _distutils_hack.override  # noqa: F401
 
 import distutils.core
-import distutils.filelist
-import re
 from distutils.errors import DistutilsOptionError
 from distutils.util import convert_path
-from fnmatch import fnmatchcase
 
 from ._deprecation_warning import SetuptoolsDeprecationWarning
 
diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py
index 93ae8eb4..24a36080 100644
--- a/setuptools/sandbox.py
+++ b/setuptools/sandbox.py
@@ -185,8 +185,8 @@ def setup_context(setup_dir):
     temp_dir = os.path.join(setup_dir, 'temp')
     with save_pkg_resources_state():
         with save_modules():
-            hide_setuptools()
             with save_path():
+                hide_setuptools()
                 with save_argv():
                     with override_temp(temp_dir):
                         with pushd(setup_dir):
@@ -195,6 +195,15 @@ def setup_context(setup_dir):
                             yield
 
 
+_MODULES_TO_HIDE = {
+    'setuptools',
+    'distutils',
+    'pkg_resources',
+    'Cython',
+    '_distutils_hack',
+}
+
+
 def _needs_hiding(mod_name):
     """
     >>> _needs_hiding('setuptools')
@@ -212,8 +221,8 @@ def _needs_hiding(mod_name):
     >>> _needs_hiding('Cython')
     True
     """
-    pattern = re.compile(r'(setuptools|pkg_resources|distutils|Cython)(\.|$)')
-    return bool(pattern.match(mod_name))
+    base_module = mod_name.split('.', 1)[0]
+    return base_module in _MODULES_TO_HIDE
 
 
 def hide_setuptools():
@@ -223,6 +232,10 @@ def hide_setuptools():
     necessary to avoid issues such as #315 where setuptools upgrading itself
     would fail to find a function declared in the metadata.
     """
+    _distutils_hack = sys.modules.get('_distutils_hack', None)
+    if _distutils_hack is not None:
+        _distutils_hack.remove_shim()
+
     modules = filter(_needs_hiding, sys.modules)
     _clear_modules(modules)
 
diff --git a/setuptools/tests/test_distutils_adoption.py b/setuptools/tests/test_distutils_adoption.py
index 7f28a217..daccc473 100644
--- a/setuptools/tests/test_distutils_adoption.py
+++ b/setuptools/tests/test_distutils_adoption.py
@@ -9,6 +9,9 @@ import jaraco.envs
 import path
 
 
+IS_PYPY = '__pypy__' in sys.builtin_module_names
+
+
 class VirtualEnv(jaraco.envs.VirtualEnv):
     name = '.env'
 
@@ -57,7 +60,11 @@ def test_distutils_local_with_setuptools(venv):
     assert venv.name in loc.split(os.sep)
 
 
-@pytest.mark.xfail(reason="#2259")
+@pytest.mark.xfail('IS_PYPY', reason='pypy imports distutils on startup')
 def test_distutils_local(venv):
+    """
+    Even without importing, the setuptools-local copy of distutils is
+    preferred.
+    """
     env = dict(SETUPTOOLS_USE_DISTUTILS='local')
     assert venv.name in find_distutils(venv, env=env).split(os.sep)