Detect and handle conflicts with "unmanaged" packages when installing

packages managed by EasyInstall.  Also, add an option to exclude source
files from .egg distributions.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041109

7 files changed, 270 insertions, 65 deletions

diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py
index 4def0a70..d24c6c13 100755
--- a/setuptools/archive_util.py
+++ b/setuptools/archive_util.py
@@ -8,8 +8,9 @@ __all__ = [
 
 import zipfile, tarfile, os
 from pkg_resources import ensure_directory
+from distutils.errors import DistutilsError
 
-class UnrecognizedFormat(RuntimeError):
+class UnrecognizedFormat(DistutilsError):
     """Couldn't recognize the archive type"""
 
 def default_filter(src,dst):
@@ -38,7 +39,6 @@ def default_filter(src,dst):
 
 
 
-
 def unpack_archive(filename, extract_dir, progress_filter=default_filter,
     drivers=None
 ):
diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py
index 74a2cd26..552bd7e8 100644
--- a/setuptools/command/bdist_egg.py
+++ b/setuptools/command/bdist_egg.py
@@ -49,6 +49,8 @@ class bdist_egg(Command):
         ('plat-name=', 'p',
                      "platform name to embed in generated filenames "
                      "(default: %s)" % get_platform()),
+        ('exclude-source-files', None,
+                     "remove all .py files from the generated egg"),
         ('keep-temp', 'k',
                      "keep the pseudo-installation tree around after " +
                      "creating the distribution archive"),
@@ -59,7 +61,7 @@ class bdist_egg(Command):
     ]
 
     boolean_options = [
-        'keep-temp', 'skip-build',
+        'keep-temp', 'skip-build', 'exclude-source-files'
     ]
 
 
@@ -78,8 +80,6 @@ class bdist_egg(Command):
 
 
 
-
-
     def initialize_options (self):
         self.bdist_dir = None
         self.plat_name = None
@@ -87,7 +87,7 @@ class bdist_egg(Command):
         self.dist_dir = None
         self.skip_build = 0
         self.egg_output = None
-
+        self.exclude_source_files = None
 
 
     def finalize_options(self):
@@ -227,6 +227,8 @@ class bdist_egg(Command):
                 "Use the install_requires/extras_require setup() args instead."
             )
 
+        if self.exclude_source_files: self.zap_pyfiles()
+
         # Make the archive
         make_zipfile(self.egg_output, archive_root, verbose=self.verbose,
                           dry_run=self.dry_run)
@@ -242,6 +244,45 @@ class bdist_egg(Command):
 
 
 
+    def zap_pyfiles(self):
+        log.info("Removing .py files from temporary directory")
+        for base,dirs,files in os.walk(self.bdist_dir):
+            if 'EGG-INFO' in dirs:
+                dirs.remove('EGG-INFO')
+            for name in files:
+                if name.endswith('.py'):
+                    path = os.path.join(base,name)
+                    log.debug("Deleting %s", path)
+                    os.unlink(path)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 # Attribute names of options for commands that might need to be convinced to
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index 9d61bb29..4ac6bac7 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -17,7 +17,7 @@ from setuptools import Command
 from setuptools.sandbox import run_setup
 from distutils import log, dir_util
 from distutils.sysconfig import get_python_lib
-from distutils.errors import DistutilsArgError, DistutilsOptionError
+from distutils.errors import DistutilsArgError, DistutilsOptionError, DistutilsError
 from setuptools.archive_util import unpack_archive
 from setuptools.package_index import PackageIndex, parse_bdist_wininst
 from setuptools.package_index import URL_SCHEME
@@ -43,7 +43,9 @@ class easy_install(Command):
     """Manage a download/build/install process"""
 
     description = "Find/get/install Python packages"
+
     command_consumes_arguments = True
+
     user_options = [
         ("zip-ok", "z", "install package as a zipfile"),
         ("multi-version", "m", "make apps have to require() a version"),
@@ -54,6 +56,10 @@ class easy_install(Command):
         ("always-copy", "a", "Copy all needed packages to install dir"),
         ("index-url=", "i", "base URL of Python Package Index"),
         ("find-links=", "f", "additional URL(s) to search for packages"),
+        ("delete-conflicting", "D", "delete old packages that get in the way"),
+        ("ignore-conflicts-at-my-risk", None,
+            "install even if old packages are in the way, even though it "
+            "most likely means the new package won't work."),
         ("build-directory=", "b",
             "download/extract/build in DIR; keep the results"),
         ('optimize=', 'O',
@@ -62,11 +68,18 @@ class easy_install(Command):
         ('record=', None,
          "filename in which to record list of installed files"),
     ]
+
     boolean_options = [
-        'zip-ok', 'multi-version', 'exclude-scripts', 'upgrade', 'always-copy'
+        'zip-ok', 'multi-version', 'exclude-scripts', 'upgrade', 'always-copy',
+        'delete-conflicting', 'ignore-conflicts-at-my-risk',
     ]
+
     create_index = PackageIndex
 
+
+
+
+
     def initialize_options(self):
         self.zip_ok = None
         self.install_dir = self.script_dir = self.exclude_scripts = None
@@ -79,6 +92,34 @@ class easy_install(Command):
         # Options not specifiable via command line
         self.package_index = None
         self.pth_file = None
+        self.delete_conflicting = None
+        self.ignore_conflicts_at_my_risk = None
+
+
+    def delete_blockers(self, blockers):
+        for filename in blockers:
+            log.info("Deleting %s", filename)
+            if not self.dry_run:
+                if os.path.isdir(filename):
+                    shutil.rmtree(filename)
+                else:
+                    os.unlink(filename)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
     def finalize_options(self):
         # If a non-default installation directory was specified, default the
@@ -122,16 +163,13 @@ class easy_install(Command):
         self.index_url = self.index_url or "http://www.python.org/pypi"
 
         self.shadow_path = sys.path[:]
-
         for path_item in self.install_dir, self.script_dir:
             if path_item not in self.shadow_path:
                 self.shadow_path.insert(0, self.install_dir)
-
         if self.package_index is None:
             self.package_index = self.create_index(
                 self.index_url, search_path = self.shadow_path
             )
-
         self.local_index = AvailableDistributions(self.shadow_path)
 
         if self.find_links is not None:
@@ -149,6 +187,11 @@ class easy_install(Command):
             except ValueError:
                 raise DistutilsOptionError("--optimize must be 0, 1, or 2")
 
+        if self.delete_conflicting and self.ignore_conflicts_at_my_risk:
+            raise DistutilsOptionError(
+                "Can't use both --delete-conflicting and "
+                "--ignore-conflicts-at-my-risk at the same time"
+            )
         if not self.args:
             raise DistutilsArgError(
                 "No urls, filenames, or requirements specified (see --help)")
@@ -160,8 +203,6 @@ class easy_install(Command):
 
         self.outputs = []
 
-
-
     def alloc_tmp(self):
         if self.build_directory is None:
             return tempfile.mkdtemp(prefix="easy_install-")
@@ -221,7 +262,7 @@ class easy_install(Command):
                     try:
                         spec = Requirement.parse(spec)
                     except ValueError:
-                        raise RuntimeError(
+                        raise DistutilsError(
                             "Not a URL, existing file, or requirement spec: %r"
                             % (spec,)
                         )
@@ -232,7 +273,7 @@ class easy_install(Command):
                 spec = None
 
             if download is None:
-                raise RuntimeError(
+                raise DistutilsError(
                     "Could not find distribution for %r" % spec
                 )
 
@@ -254,7 +295,7 @@ class easy_install(Command):
             for dist in dists:
                 self.process_distribution(spec, dist)
         else:
-            dists = [self.egg_distribution(download)]
+            dists = [self.check_conflicts(self.egg_distribution(download))]
             self.process_distribution(spec, dists[0], "Using")
         if spec is not None:
             for dist in dists:
@@ -276,11 +317,11 @@ class easy_install(Command):
                     [requirement], self.shadow_path, self.easy_install
                 )
             except DistributionNotFound, e:
-                raise RuntimeError(
+                raise DistutilsError(
                     "Could not find required distribution %s" % e.args
                 )
             except VersionConflict, e:
-                raise RuntimeError(
+                raise DistutilsError(
                     "Installed distribution %s conflicts with requirement %s"
                     % e.args
                 )
@@ -385,11 +426,11 @@ class easy_install(Command):
         if not os.path.exists(setup_script):
             setups = glob(os.path.join(tmpdir, '*', 'setup.py'))
             if not setups:
-                raise RuntimeError(
+                raise DistutilsError(
                     "Couldn't find a setup script in %s" % dist_filename
                 )
             if len(setups)>1:
-                raise RuntimeError(
+                raise DistutilsError(
                     "Multiple setup scripts in %s" % dist_filename
                 )
             setup_script = setups[0]
@@ -418,10 +459,10 @@ class easy_install(Command):
     def install_egg(self, egg_path, zip_ok, tmpdir):
         destination = os.path.join(self.install_dir,os.path.basename(egg_path))
         destination = os.path.abspath(destination)
-
         if not self.dry_run:
             ensure_directory(destination)
 
+        self.check_conflicts(self.egg_distribution(egg_path))
         if not samefile(egg_path, destination):
             if os.path.isdir(destination):
                 dir_util.remove_tree(destination, dry_run=self.dry_run)
@@ -453,7 +494,7 @@ class easy_install(Command):
         # See if it's valid, get data
         cfg = extract_wininst_cfg(dist_filename)
         if cfg is None:
-            raise RuntimeError(
+            raise DistutilsError(
                 "%s is not a valid distutils Windows .exe" % dist_filename
             )
 
@@ -495,18 +536,23 @@ class easy_install(Command):
 
         # Check for .pth file and set up prefix translations
         prefixes = get_exe_prefixes(dist_filename)
+
         to_compile = []
         native_libs = []
+        top_level = {}
 
         def process(src,dst):
             for old,new in prefixes:
                 if src.startswith(old):
                     src = new+src[len(old):]
-                    dst = os.path.join(egg_tmp, *src.split('/'))
+                    parts = src.split('/')
+                    dst = os.path.join(egg_tmp, *parts)
                     dl = dst.lower()
                     if dl.endswith('.pyd') or dl.endswith('.dll'):
+                        top_level[os.path.splitext(parts[0])[0]] = 1
                         native_libs.append(src)
                     elif dl.endswith('.py') and old!='SCRIPTS/':
+                        top_level[os.path.splitext(parts[0])[0]] = 1
                         to_compile.append(dst)
                     return dst
             if not src.endswith('.pth'):
@@ -526,10 +572,87 @@ class easy_install(Command):
 
         self.byte_compile(to_compile)   # compile .py's
 
-        if native_libs:     # write EGG-INFO/native_libs.txt
-            nl_txt = os.path.join(egg_tmp, 'EGG-INFO', 'native_libs.txt')
-            ensure_directory(nl_txt)
-            open(nl_txt,'w').write('\n'.join(native_libs)+'\n')
+        for name in 'top_level','native_libs':
+            if locals()[name]:
+                txt = os.path.join(egg_tmp, 'EGG-INFO', name+'.txt')
+                ensure_directory(txt)
+                open(txt,'w').write('\n'.join(locals()[name])+'\n')
+
+    def check_conflicts(self, dist):
+        """Verify that there are no conflicting "old-style" packages"""
+
+        from imp import find_module, get_suffixes
+        from glob import glob
+
+        blockers = []
+        names = dict.fromkeys(dist._get_metadata('top_level.txt')) # XXX private attr
+
+        exts = {'.pyc':1, '.pyo':1}     # get_suffixes() might leave one out
+        for ext,mode,typ in get_suffixes():
+            exts[ext] = 1
+
+        for path,files in expand_paths([self.install_dir, get_python_lib()]):
+            for filename in files:
+                base,ext = os.path.splitext(filename)
+                if base in names:
+                    if not ext:
+                        # no extension, check for package
+                        try:
+                            f, filename, descr = find_module(base, [path])
+                        except ImportError:
+                            continue
+                        else:
+                            if f: f.close()
+                            if filename not in blockers:
+                                blockers.append(filename)
+                    elif ext in exts:
+                        blockers.append(os.path.join(path,filename))
+
+        if blockers:
+            self.found_conflicts(dist, blockers)
+
+        return dist
+
+    def found_conflicts(self, dist, blockers):
+        if self.delete_conflicting:
+            log.warn("Attempting to delete conflicting packages:")
+            return self.delete_blockers(blockers)
+
+        msg = """\
+-------------------------------------------------------------------------
+CONFLICT WARNING:
+
+The following modules or packages have the same names as modules or
+packages being installed, and will be *before* the installed packages in
+Python's search path.  You MUST remove all of the relevant files and
+directories before you will be able to use the package(s) you are
+installing:
+
+   %s
+
+""" % '\n   '.join(blockers)
+
+        if self.ignore_conflicts_at_my_risk:
+            msg += """\
+(Note: you can run EasyInstall on '%s' with the
+--delete-conflicting option to attempt deletion of the above files
+and/or directories.)
+""" % dist.name
+        else:
+            msg += """\
+Note: you can attempt this installation again with EasyInstall, and use
+either the --delete-conflicting (-D) option or the
+--ignore-conflicts-at-my-risk option, to either delete the above files
+and directories, or to ignore the conflicts, respectively.  Note that if
+you ignore the conflicts, the installed package(s) may not work.
+"""
+        msg += """\
+-------------------------------------------------------------------------
+"""
+        sys.stderr.write(msg)
+        sys.stderr.flush()
+        if not self.ignore_conflicts_at_my_risk:
+            raise DistutilsError("Installation aborted due to conflicts")
 
     def installation_report(self, dist, what="Installed"):
         """Helpful installation message for display to package users"""
@@ -589,7 +712,7 @@ PYTHONPATH, or by being added to sys.path by your code.)
             try:
                 run_setup(setup_script, args)
             except SystemExit, v:
-                raise RuntimeError(
+                raise DistutilsError(
                     "Setup script exited with %s" % (v.args[0],)
                 )
         finally:
@@ -695,6 +818,47 @@ PYTHONPATH, or by being added to sys.path by your code.)
 
 
 
+def expand_paths(inputs):
+    """Yield sys.path directories that might contain "old-style" packages"""
+
+    seen = {}
+
+    for dirname in inputs:
+        if dirname in seen:
+            continue
+
+        seen[dirname] = 1
+        if not os.path.isdir(dirname):
+            continue
+
+        files = os.listdir(dirname)
+        yield dirname, files
+
+        for name in files:
+            if not name.endswith('.pth'):
+                # We only care about the .pth files
+                continue
+            if name in ('easy-install.pth','setuptools.pth'):
+                # Ignore .pth files that we control
+                continue
+
+            # Read the .pth file
+            f = open(os.path.join(dirname,name))
+            lines = list(yield_lines(f))
+            f.close()
+
+            # Yield existing non-dupe, non-import directory lines from it
+            for line in lines:
+                if not line.startswith("import"):
+                    line = line.rstrip()
+                    if line not in seen:
+                        seen[line] = 1
+                        if not os.path.isdir(line):
+                            continue
+                        yield line, os.listdir(line)
+
+
+
 def extract_wininst_cfg(dist_filename):
     """Extract configuration data from a bdist_wininst .exe
 
@@ -820,11 +984,11 @@ class PthDistributions(AvailableDistributions):
 
 def main(argv, **kw):
     from setuptools import setup
-    try:
-        setup(script_args = ['-q','easy_install', '-v']+argv, **kw)
-    except RuntimeError, v:
-        print >>sys.stderr,"error:",v
-        sys.exit(1)
+    setup(script_args = ['-q','easy_install', '-v']+argv, **kw)
+
+
+
+
 
 
 
diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py
index 907e47fb..dca4e2a3 100755
--- a/setuptools/command/egg_info.py
+++ b/setuptools/command/egg_info.py
@@ -97,6 +97,7 @@ class egg_info(Command):
                 metadata.name, metadata.version = oldname, oldver
 
         self.write_requirements()
+        self.write_toplevel_names()
         if os.path.exists(os.path.join(self.egg_info,'depends.txt')):
             log.warn(
                 "WARNING: 'depends.txt' will not be used by setuptools 0.6!\n"
@@ -106,7 +107,6 @@ class egg_info(Command):
 
     def write_requirements(self):
         dist = self.distribution
-
         if not getattr(dist,'install_requires',None) and \
            not getattr(dist,'extras_require',None): return
 
@@ -125,14 +125,11 @@ class egg_info(Command):
         version = self.distribution.get_version()
         if self.tag_build:
             version+=self.tag_build
-
         if self.tag_svn_revision and os.path.exists('.svn'):
             version += '-r%s' % self.get_svn_revision()
-
         if self.tag_date:
             import time
             version += time.strftime("-%Y%m%d")
-
         return safe_version(version)
 
 
@@ -142,23 +139,26 @@ class egg_info(Command):
         import re
         match = re.search(r'Last Changed Rev: (\d+)', result)
         if not match:
-            raise RuntimeError("svn info error: %s" % result.strip())
+            raise DistutilsError("svn info error: %s" % result.strip())
         return match.group(1)
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+    def write_toplevel_names(self):
+        pkgs = dict.fromkeys(self.distribution.packages or ())
+        pkgs.update(dict.fromkeys(self.distribution.py_modules or ()))
+        for ext in self.distribution.ext_modules or ():
+            if isinstance(ext,tuple):
+                name,buildinfo = ext
+            else:
+                name = ext.name
+            pkgs[name]=1
+        pkgs = dict.fromkeys([k.split('.',1)[0] for k in pkgs])
+        toplevel = os.path.join(self.egg_info,"top_level.txt")
+        log.info("writing list of top-level names to %s" % toplevel)
+        if not self.dry_run:
+            f = open(toplevel, 'wt')
+            f.write('\n'.join(pkgs))
+            f.write('\n')
+            f.close()
 
 
diff --git a/setuptools/dist.py b/setuptools/dist.py
index 8fcd19ea..4fef454f 100644
--- a/setuptools/dist.py
+++ b/setuptools/dist.py
@@ -420,13 +420,16 @@ class Distribution(_Distribution):
 
     def install_eggs(self):
         from setuptools.command.easy_install import easy_install
-        cmd = easy_install(self, args="x")
+        cmd = easy_install(self, args="x", ignore_conflicts_at_my_risk=1)
         cmd.ensure_finalized()  # finalize before bdist_egg munges install cmd
+
         self.run_command('bdist_egg')
         args = [self.get_command_obj('bdist_egg').egg_output]
+
         if setuptools.bootstrap_install_from:
             # Bootstrap self-installation of setuptools
             args.insert(0, setuptools.bootstrap_install_from)
+
         cmd.args = args
         cmd.run()
         self.have_run['install'] = 1
@@ -446,9 +449,6 @@ class Distribution(_Distribution):
 
 
 
-
-
-
     def get_cmdline_options(self):
         """Return a '{cmd: {opt:val}}' map of all command-line options
 
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index 0a8ccddc..21d37cf9 100755
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -3,6 +3,7 @@
 import sys, os.path, re, urlparse, urllib2
 from pkg_resources import *
 from distutils import log
+from distutils.errors import DistutilsError
 
 HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I)
 URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
@@ -38,7 +39,6 @@ def parse_bdist_wininst(name):
 
 
 
-
 def distros_for_url(url, metadata=None):
     """Yield egg or source distribution objects that might be found at a URL"""
 
@@ -272,7 +272,7 @@ class PackageIndex(AvailableDistributions):
                 try:
                     spec = Requirement.parse(spec)
                 except ValueError:
-                    raise RuntimeError(
+                    raise DistutilsError(
                         "Not a URL, existing file, or requirement spec: %r" %
                         (spec,)
                     )
@@ -336,7 +336,7 @@ class PackageIndex(AvailableDistributions):
         try:
             fp = self.open_url(url)
             if isinstance(fp, urllib2.HTTPError):
-                raise RuntimeError(
+                raise DistutilsError(
                     "Can't download %s: %s %s" % (url, fp.code,fp.msg)
                 )
 
@@ -373,7 +373,7 @@ class PackageIndex(AvailableDistributions):
         except urllib2.HTTPError, v:
             return v
         except urllib2.URLError, v:
-            raise RuntimeError("Download error: %s" % v.reason)
+            raise DistutilsError("Download error: %s" % v.reason)
 
 
     def _download_url(self, scheme, url, tmpdir):
@@ -432,7 +432,7 @@ class PackageIndex(AvailableDistributions):
                         return self._download_sourceforge(url, page, tmpdir)
                 break   # not an index page
         file.close()
-        raise RuntimeError("Unexpected HTML page found at "+url)
+        raise DistutilsError("Unexpected HTML page found at "+url)
 
 
     def _download_svn(self, url, filename):
@@ -457,7 +457,7 @@ class PackageIndex(AvailableDistributions):
         mirror_regex = re.compile(r'HREF=(/.*?\?use_mirror=[^>]*)')
         urls = [m.group(1) for m in mirror_regex.finditer(sf_page)]
         if not urls:
-            raise RuntimeError(
+            raise DistutilsError(
                 "URL looks like a Sourceforge mirror page, but no URLs found"
             )
 
@@ -481,7 +481,7 @@ class PackageIndex(AvailableDistributions):
             scheme = URL_SCHEME(download_url)
             return self._download_url(scheme.group(1), download_url, tmpdir)
         else:
-            raise RuntimeError(
+            raise DistutilsError(
                 'No META HTTP-EQUIV="refresh" found in Sourceforge page at %s'
                 % url
             )
diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py
index 8aa58f8d..f124289a 100755
--- a/setuptools/sandbox.py
+++ b/setuptools/sandbox.py
@@ -1,7 +1,7 @@
 import os, sys, __builtin__, tempfile
 _os = sys.modules[os.name]
 _open = open
-
+from distutils.errors import DistutilsError
 __all__ = [
     "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
 ]
@@ -188,7 +188,7 @@ class DirectorySandbox(AbstractSandbox):
         return (src,dst)
 
 
-class SandboxViolation(RuntimeError):
+class SandboxViolation(DistutilsError):
     """A setup script attempted to modify the filesystem outside the sandbox"""
 
     def __str__(self):