Add support for quiet/verbose/dry-run/optimize flags.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041064

4 files changed, 268 insertions, 78 deletions

diff --git a/EasyInstall.txt b/EasyInstall.txt
index ad3aa32e..cbc352de 100755
--- a/EasyInstall.txt
+++ b/EasyInstall.txt
@@ -23,7 +23,7 @@ Installing "Easy Install"
 -------------------------
 
 Windows users can just download and run the `setuptools binary installer for
-Windows <http://peak.telecommunity.com/dist/setuptools-0.4a2.win32.exe>`_.
+Windows <http://peak.telecommunity.com/dist/setuptools-0.4a4.win32.exe>`_.
 All others should just download `ez_setup.py
 <http://peak.telecommunity.com/dist/ez_setup.py>`_, and run it; this will
 download and install the correct version of ``setuptools`` for your Python
@@ -62,7 +62,7 @@ version, and automatically downloading, building, and installing it::
 **Example 2**. Install or upgrade a package by name and version by finding
 links on a given "download page"::
 
-    easy_install -f http://peak.telecommunity.com/dist "setuptools>=0.4a3"
+    easy_install -f http://peak.telecommunity.com/dist "setuptools>=0.4a4"
 
 **Example 3**. Download a source distribution from a specified URL,
 automatically building and installing it::
@@ -229,6 +229,26 @@ installations, so that Python won't lock us out of using anything but the most
 recently-installed version of the package.)
 
 
+Controlling Build Options
+-------------------------
+
+EasyInstall respects standard distutils `Configuration Files`_, so you can use
+them to configure build options for packages that it installs from source.  For
+example, if you are on Windows using the MinGW compiler, you can configure the
+default compiler by putting something like this::
+
+    [build]
+    compiler = mingw32
+
+into the appropriate distutils configuration file.  In fact, since this is just
+normal distutils configuration, it will affect any builds using that config
+file, not just ones done by EasyInstall.  For example, if you add those lines
+to ``distutils.cfg`` in the ``distutils`` package directory, it will be the
+default compiler for *all* packages you build.  See `Configuration Files`_
+below for a list of the standard configuration file locations, and links to
+more documentation on using distutils configuration files.
+
+
 Reference Manual
 ================
 
@@ -257,8 +277,17 @@ and Windows, respectively), and finally a ``distutils.cfg`` file in the
     find_links = http://sqlobject.org/
                  http://peak.telecommunity.com/dist/
 
-See also the current Python documentation on the `use and location of distutils
-configuration files <http://docs.python.org/inst/config-syntax.html>`_.
+In addition to accepting configuration for its own options under
+``[easy_install]``, EasyInstall also respects defaults specified for other
+distutils commands.  For example, if you don't set an ``install_dir`` for
+``[easy_install]``, but *have* set an ``install_lib`` for the ``[install]``
+command, this will become EasyInstall's default installation directory.  Thus,
+if you are already using distutils configuration files to set default install
+locations, build options, etc., EasyInstall will respect your existing settings
+until and unless you override them explicitly in an ``[easy_install]`` section.
+
+For more information, see also the current Python documentation on the `use and
+location of distutils configuration files <http://docs.python.org/inst/config-syntax.html>`_.
 
 
 Command-Line Options
@@ -377,6 +406,34 @@ Command-Line Options
     URL or filename, so that the installer will not be confused by the presence
     of multiple ``setup.py`` files in the build directory.
 
+``--verbose, -v, --quiet, -q`` (New in 0.4a4)
+    Control the level of detail of EasyInstall's progress messages.  The
+    default detail level is "info", which prints information only about
+    relatively time-consuming operations like running a setup script, unpacking
+    an archive, or retrieving a URL.  Using ``-q`` or ``--quiet`` drops the
+    detail level to "warn", which will only display installation reports,
+    warnings, and errors.  Using ``-v`` or ``--verbose`` increases the detail
+    level to include individual file-level operations, link analysis messages,
+    and distutils messages from any setup scripts that get run.  If you include
+    the ``-v`` option more than once, the second and subsequent uses are passed
+    down to any setup scripts, increasing the verbosity of their reporting as
+    well.
+
+``--dry-run, -n`` (New in 0.4a4)
+    Don't actually install the package or scripts.  This option is passed down
+    to any setup scripts run, so packages should not actually build either.
+    This does *not* skip downloading, nor does it skip extracting source
+    distributions to a temporary/build directory.
+
+``--optimize=LEVEL``, ``-O LEVEL`` (New in 0.4a4)
+    If you are installing from a source distribution, and are *not* using the
+    ``--zip-ok`` option, this option controls the optimization level for
+    compiling installed ``.py`` files to ``.pyo`` files.  It does not affect
+    the compilation of modules contained in ``.egg`` files, only those in
+    ``.egg`` directories.  The optimization level can be set to 0, 1, or 2;
+    the default is 0 (unless it's set under ``install`` or ``install_lib`` in
+    one of your distutils configuration files).
+
 
 Release Notes/Change History
 ============================
@@ -385,8 +442,15 @@ Known Issues
  * There's no automatic retry for borked Sourceforge mirrors, which can easily
    time out or be missing a file.
 
- * EasyInstall does not yet respect the distutils "verbose/quiet" and "dry-run"
-   options, even though it accepts them.
+0.4a4
+ * Added support for the distutils "verbose/quiet" and "dry-run" options, as
+   well as the "optimize" flag.
+
+ * Support downloading packages that were uploaded to PyPI (by scanning all
+   links on package pages, not just the homepage/download links).
+
+ * Fix problems with ``resource_listdir()``, ``resource_isdir()`` and resource
+   directory extraction for zipped eggs.
 
 0.4a3
  * Fixed scripts not being able to see a ``__file__`` variable in ``__main__``
@@ -493,10 +557,13 @@ Known Issues
 Future Plans
 ============
 
-* Log progress to a logger, with -v and -q options to control verbosity
-* Display more information about downloads and progress when being verbose
 * Process the installed package's dependencies as well as the base package
+* Support "self-installation" - bootstrapping setuptools install into another
+  package's installation process (copy egg, write setuptools.pth)
 * Support installation from bdist_wininst packages?
 * Additional utilities to list/remove/verify packages
 * Signature checking?  SSL?  Ability to suppress PyPI search?
+* Display byte progress meter when downloading distributions and long pages?
+* Redirect stdout/stderr to log during run_setup?
+
 
diff --git a/easy_install.py b/easy_install.py
index 24f33a59..c35955fc 100755
--- a/easy_install.py
+++ b/easy_install.py
@@ -16,8 +16,9 @@ import sys, os.path, zipimport, shutil, tempfile
 
 from setuptools import Command
 from setuptools.sandbox import run_setup
+from distutils import log, dir_util
 from distutils.sysconfig import get_python_lib
-from distutils.errors import DistutilsArgError
+from distutils.errors import DistutilsArgError, DistutilsOptionError
 from setuptools.archive_util import unpack_archive
 from setuptools.package_index import PackageIndex
 from pkg_resources import *
@@ -38,7 +39,6 @@ def samefile(p1,p2):
 
 
 
-
 class easy_install(Command):
     """Manage a download/build/install process"""
 
@@ -54,11 +54,14 @@ class easy_install(Command):
         ("find-links=", "f", "additional URL(s) to search for packages"),
         ("build-directory=", "b",
             "download/extract/build in DIR; keep the results"),
+        ('optimize=', 'O',
+         "also compile with optimization: -O1 for \"python -O\", "
+         "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"),
     ]
 
     boolean_options = [ 'zip-ok', 'multi-version', 'exclude-scripts' ]
     create_index = PackageIndex
-    
+
     def initialize_options(self):
         self.zip_ok = None
         self.multi_version = None
@@ -67,19 +70,16 @@ class easy_install(Command):
         self.find_links = None
         self.build_directory = None
         self.args = None
-        
+        self.optimize = None
+
         # Options not specifiable via command line
         self.package_index = None
         self.pth_file = None
 
-    def alloc_tmp(self):
-        if self.build_directory is None:
-            return tempfile.mkdtemp(prefix="easy_install-")
-        tmpdir = os.path.realpath(self.build_directory)
-        if not os.path.isdir(tmpdir):
-            os.makedirs(tmpdir)
-        return tmpdir
-        
+
+
+
+
     def finalize_options(self):
         # If a non-default installation directory was specified, default the
         # script directory to match it.
@@ -91,13 +91,13 @@ class easy_install(Command):
         # --prefix and --home and all that other crud.
         self.set_undefined_options('install_lib',
             ('install_dir','install_dir')
-        )         
+        )
         # Likewise, set default script_dir from 'install_scripts.install_dir'
         self.set_undefined_options('install_scripts',
             ('install_dir', 'script_dir')
         )
 
-        site_packages = get_python_lib()       
+        site_packages = get_python_lib()
         instdir = self.install_dir
 
         if instdir is None or samefile(site_packages,instdir):
@@ -106,7 +106,7 @@ class easy_install(Command):
                 self.pth_file = PthDistributions(
                     os.path.join(instdir,'easy-install.pth')
                 )
-            self.install_dir = instdir    
+            self.install_dir = instdir
 
         elif self.multi_version is None:
             self.multi_version = True
@@ -124,23 +124,66 @@ class easy_install(Command):
         if self.find_links is not None:
             if isinstance(self.find_links, basestring):
                 self.find_links = self.find_links.split()
-            for link in self.find_links:
-                self.package_index.scan_url(link)
+        else:
+            self.find_links = []
+
+        self.set_undefined_options('install_lib', ('optimize','optimize'))
+
+        if not isinstance(self.optimize,int):
+            try:
+                self.optimize = int(self.optimize)
+                if not (0 <= self.optimize <= 2): raise ValueError
+            except ValueError:
+                raise DistutilsOptionError("--optimize must be 0, 1, or 2")
 
         if not self.args:
             raise DistutilsArgError(
                 "No urls, filenames, or requirements specified (see --help)")
+
         elif len(self.args)>1 and self.build_directory is not None:
             raise DistutilsArgError(
-                "Build directory can only be set when using one URL"   
+                "Build directory can only be set when using one URL"
             )
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    def alloc_tmp(self):
+        if self.build_directory is None:
+            return tempfile.mkdtemp(prefix="easy_install-")
+        tmpdir = os.path.realpath(self.build_directory)
+        if not os.path.isdir(tmpdir):
+            os.makedirs(tmpdir)
+        return tmpdir
+
+
     def run(self):
-        for spec in self.args:
-            self.easy_install(spec)
+        if self.verbose<>self.distribution.verbose:
+            log.set_verbosity(self.verbose)
+        try:
+            for link in self.find_links:
+                self.package_index.scan_url(link)
+            for spec in self.args:
+                self.easy_install(spec)
+        finally:
+            log.set_verbosity(self.distribution.verbose)
 
 
-    def easy_install(self, spec):       
+    def easy_install(self, spec):
         tmpdir = self.alloc_tmp()
         try:
             download = self.package_index.download(spec, tmpdir)
@@ -149,19 +192,17 @@ class easy_install(Command):
                     "Could not find distribution for %r" % spec
                 )
 
-            print "Processing", os.path.basename(download)
+            log.info("Processing %s", os.path.basename(download))
             for dist in self.install_eggs(download, self.zip_ok, tmpdir):
                 self.package_index.add(dist)
                 self.install_egg_scripts(dist)
-                print self.installation_report(dist)
+                log.warn(self.installation_report(dist))
 
         finally:
             if self.build_directory is None:
                 shutil.rmtree(tmpdir)
 
 
-
-
     def install_egg_scripts(self, dist):
         metadata = dist.metadata
         if self.exclude_scripts or not metadata.metadata_isdir('scripts'):
@@ -172,7 +213,7 @@ class easy_install(Command):
         for script_name in metadata.metadata_listdir('scripts'):
             target = os.path.join(self.script_dir, script_name)
 
-            print "Installing", script_name, "script to", self.script_dir
+            log.info("Installing %s script to %s", script_name,self.script_dir)
 
             script_text = metadata.get_metadata('scripts/'+script_name)
             script_text = script_text.replace('\r','\n')
@@ -193,11 +234,11 @@ class easy_install(Command):
                 "import pkg_resources",
                 "pkg_resources.run_main(%r, %r)" % (spec, script_name)
             ])
+            if not self.dry_run:
+                f = open(target,"w")
+                f.write(script_text)
+                f.close()
 
-            f = open(target,"w")
-            f.write(script_text)
-            f.close()
-            
 
 
 
@@ -210,7 +251,7 @@ class easy_install(Command):
 
         # Anything else, try to extract and build
         if os.path.isfile(dist_filename):
-            unpack_archive(dist_filename, tmpdir)  # XXX add progress log
+            unpack_archive(dist_filename, tmpdir, self.unpack_progress)
 
         # Find the setup.py file
         from glob import glob
@@ -226,48 +267,51 @@ class easy_install(Command):
                     "Multiple setup scripts in %s" % dist_filename
                 )
             setup_script = setups[0]
-        from setuptools.command import bdist_egg
-        sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
-        try:
-            print "Running", setup_script[len(tmpdir)+1:]
-            run_setup(setup_script, ['-q', 'bdist_egg'])
-        except SystemExit, v:
-            raise RuntimeError(
-                "Setup script exited with %s" % (v.args[0],)
-            )
+
+        self.build_egg(tmpdir, setup_script)
+        dist_dir = os.path.join(os.path.dirname(setup_script),'dist')
 
         eggs = []
-        for egg in glob(
-            os.path.join(os.path.dirname(setup_script),'dist','*.egg')
-        ):
+        for egg in glob(os.path.join(dist_dir,'*.egg')):
             eggs.append(self.install_egg(egg, zip_ok, tmpdir))
 
+        if not eggs and not self.dry_run:
+            log.warn("No eggs found in %s (setup script problem?)", dist_dir)
+
         return eggs
 
-    def install_egg(self, egg_path, zip_ok, tmpdir):
 
+
+
+
+
+    def install_egg(self, egg_path, zip_ok, tmpdir):
         destination = os.path.join(self.install_dir,os.path.basename(egg_path))
         destination = os.path.abspath(destination)
-        ensure_directory(destination)
+        if not self.dry_run:
+            ensure_directory(destination)
 
         if not samefile(egg_path, destination):
             if os.path.isdir(destination):
-                shutil.rmtree(destination)
+                dir_util.remove_tree(destination, dry_run=self.dry_run)
+
             elif os.path.isfile(destination):
-                os.unlink(destination)
+                self.execute(os.unlink,(destination,),"Removing "+destination)
 
             if zip_ok:
                 if egg_path.startswith(tmpdir):
-                    shutil.move(egg_path, destination)
+                    f,m = shutil.move, "Moving"
                 else:
-                    shutil.copy2(egg_path, destination)
-
+                    f,m = shutil.copy2, "Copying"
             elif os.path.isdir(egg_path):
-                shutil.move(egg_path, destination)
-
+                f,m = shutil.move, "Moving"
             else:
-                os.mkdir(destination)
-                unpack_archive(egg_path, destination)   # XXX add progress??
+                self.mkpath(destination)
+                f,m = self.unpack_and_compile, "Extracting"
+
+            self.execute(f, (egg_path, destination),
+                (m+" %s to %s") %
+                (os.path.basename(egg_path),os.path.dirname(destination)))
 
         if os.path.isdir(destination):
             dist = Distribution.from_filename(
@@ -282,13 +326,10 @@ class easy_install(Command):
         self.update_pth(dist)
         return dist
 
-
-
-
     def installation_report(self, dist):
         """Helpful installation message for display to package users"""
 
-        msg = "Installed %(eggloc)s to %(instdir)s"
+        msg = "\nInstalled %(eggloc)s to %(instdir)s"
         if self.multi_version:
             msg += """
 
@@ -318,12 +359,94 @@ PYTHONPATH, or by being added to sys.path by your code.)
         if self.pth_file is not None:
             remove = self.pth_file.remove
             for d in self.pth_file.get(dist.key,()):    # drop old entries
+                log.info("Removing %s from .pth file", d)
                 remove(d)
             if not self.multi_version:
+                log.info("Adding %s to .pth file", dist)
                 self.pth_file.add(dist) # add new entry
             self.pth_file.save()
 
 
+    def build_egg(self, tmpdir, setup_script):
+        from setuptools.command import bdist_egg
+        sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
+
+        args = ['bdist_egg']
+        if self.verbose>2:
+            v = 'v' * self.verbose - 1
+            args.insert(0,'-'+v)
+        elif self.verbose<2:
+            args.insert(0,'-q')
+        if self.dry_run:
+            args.insert(0,'-n')
+
+        log.info("Running %s %s", setup_script[len(tmpdir)+1:], ' '.join(args))
+        try:
+            try:
+                run_setup(setup_script, args)
+            except SystemExit, v:
+                raise RuntimeError(
+                    "Setup script exited with %s" % (v.args[0],)
+                )
+        finally:
+            log.set_verbosity(self.verbose) # restore our log verbosity
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    def unpack_progress(self, src, dst):
+        # Progress filter for unpacking
+        log.debug("Unpacking %s to %s", src, dst)
+        return True     # only unpack-and-compile skips files for dry run
+
+
+    def unpack_and_compile(self, egg_path, destination):
+        to_compile = []
+
+        def pf(src,dst):
+            if dst.endswith('.py'):
+                to_compile.append(dst)
+            self.unpack_progress(src,dst)
+            return not self.dry_run
+
+        unpack_archive(egg_path, destination, pf)
+
+        from distutils.util import byte_compile
+        try:
+            # try to make the byte compile messages quieter
+            log.set_verbosity(self.verbose - 1)
+
+            byte_compile(to_compile, optimize=0, force=1, dry_run=self.dry_run) 
+            if self.optimize:
+                byte_compile(
+                    to_compile, optimize=self.optimize, force=1,
+                    dry_run=self.dry_run
+                )
+        finally:
+            log.set_verbosity(self.verbose)     # restore original verbosity
+
+
+
+
+
+
+
+
+
 
 
 class PthDistributions(AvailableDistributions):
diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py
index cabc3350..98ebda79 100644
--- a/setuptools/command/bdist_egg.py
+++ b/setuptools/command/bdist_egg.py
@@ -190,7 +190,7 @@ class bdist_egg(Command):
             if not self.dry_run:
                 os.unlink(native_libs)
 
-        if self.egg_info:
+        if self.egg_info and os.path.exists(self.egg_info):
             for filename in os.listdir(self.egg_info):
                 path = os.path.join(self.egg_info,filename)
                 if os.path.isfile(path):
@@ -231,7 +231,7 @@ class bdist_egg(Command):
         for dirname in INSTALL_DIRECTORY_ATTRS:
             kw.setdefault(dirname,self.bdist_dir)
         kw.setdefault('skip_build',self.skip_build)
-
+        kw.setdefault('dry_run', self.dry_run)
         cmd = self.reinitialize_command(cmdname, **kw)
         self.run_command(cmdname)
         return cmd
@@ -262,24 +262,24 @@ def make_zipfile (zip_filename, base_dir, verbose=0, dry_run=0):
     import zipfile
     mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
 
-    # If zipfile module is not available, try spawning an external
-    # 'zip' command.
-    log.info("creating '%s' and adding '%s' to it",
-             zip_filename, base_dir)
+    log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)
 
     def visit (z, dirname, names):
         for name in names:
             path = os.path.normpath(os.path.join(dirname, name))
             if os.path.isfile(path):
                 p = path[len(base_dir)+1:]
-                z.write(path, p)
-                log.info("adding '%s'" % p)
+                if not dry_run:
+                    z.write(path, p)
+                log.debug("adding '%s'" % p)
 
     if not dry_run:
         z = zipfile.ZipFile(zip_filename, "w",
                             compression=zipfile.ZIP_DEFLATED)
         os.path.walk(base_dir, visit, z)
         z.close()
+    else:
+        os.path.walk(base_dir, visit, None)
 
     return zip_filename
 
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index 0645dc3e..be40df9c 100755
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -2,6 +2,7 @@
 
 import sys, os.path, re, urlparse, urllib2
 from pkg_resources import *
+from distutils import log
 
 HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I)
 URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
@@ -38,7 +39,6 @@ __all__ = [
 
 
 
-
 def distros_for_url(url, metadata=None):
     """Yield egg or source distribution objects that might be found at a URL"""
 
@@ -359,13 +359,13 @@ class PackageIndex(AvailableDistributions):
         return filename
 
     def debug(self, msg, *args):
-        pass #print msg % args    # XXX
+        log.debug(msg, *args)
 
     def info(self, msg, *args):
-        print msg % args    # XXX
+        log.info(msg, *args)
         
     def warn(self, msg, *args):
-        print msg % args    # XXX
+        log.warn(msg, *args)
 
     def _download_sourceforge(self, source_url, sf_page, tmpdir):
         """Download package from randomly-selected SourceForge mirror"""