Split setup-running and archive-extraction utilities into separate modules,

for easy use by tools other than EasyInstall.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041049

4 files changed, 402 insertions, 275 deletions

diff --git a/EasyInstall.txt b/EasyInstall.txt
index 87468cfd..c4083be4 100755
--- a/EasyInstall.txt
+++ b/EasyInstall.txt
@@ -270,6 +270,10 @@ Command-Line Options
 Release Notes/Change History
 ============================
 
+Known Issues
+ * There's no automatic retry for borked Sourceforge mirrors, which can easily
+   time out or be missing a file.
+
 0.4a1
  * Added ``--scan-url`` and ``--index-url`` options, to scan download pages
    and search PyPI for needed packages.
diff --git a/easy_install.py b/easy_install.py
index 81415677..00205ddc 100755
--- a/easy_install.py
+++ b/easy_install.py
@@ -17,19 +17,17 @@ import os.path
 import pkg_resources
 import re
 import zipimport
-import zipfile
-import tarfile
 import shutil
 import urlparse
 import urllib
 import tempfile
-import __builtin__
 
+from setuptools.sandbox import run_setup
+from setuptools.archive_util import unpack_archive
 from distutils.sysconfig import get_python_lib
 from shutil import rmtree   # must have, because it can be called from __del__
 from pkg_resources import *
-_os = sys.modules[os.name]
-_open = open
+
 
 class Opener(urllib.FancyURLopener):
     def http_error_default(self, url, fp, errcode, errmsg, headers):
@@ -39,6 +37,8 @@ class Opener(urllib.FancyURLopener):
         return info
 opener = Opener()
 
+
+
 HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I)
 EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
 
@@ -251,7 +251,7 @@ class Installer:
 
         # Anything else, try to extract and build
         if os.path.isfile(dist_filename):
-            self._extract_file(dist_filename)
+            unpack_archive(dist_filename, self.tmpdir)  # XXX add progress log
 
         # Find the setup.py file
         from glob import glob
@@ -268,7 +268,14 @@ class Installer:
                 )
             setup_script = setups[0]
 
-        self._run_setup(setup_script)
+        from setuptools.command import bdist_egg
+        sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
+        try:
+            run_setup(setup_script, '-q', 'bdist_egg')
+        except SystemExit, v:
+            raise RuntimeError(
+                "Setup script exited with %s" % (v.args[0],)
+            )
 
         eggs = []
         for egg in glob(
@@ -278,95 +285,6 @@ class Installer:
 
         return eggs
 
-
-
-
-
-
-
-
-    def _extract_zip(self,zipname,extract_dir):
-        z = zipfile.ZipFile(zipname)
-        try:
-            for info in z.infolist():
-                name = info.filename
-
-                # don't extract absolute paths or ones with .. in them
-                if name.startswith('/') or '..' in name:
-                    continue
-
-                target = os.path.join(extract_dir,name)
-                if name.endswith('/'):
-                    # directory
-                    ensure_directory(target)
-                else:
-                    # file
-                    ensure_directory(target)
-                    data = z.read(info.filename)
-                    f = open(target,'wb')
-                    try:
-                        f.write(data)
-                    finally:
-                        f.close()
-                        del data
-        finally:
-            z.close()
-
-    def _extract_tar(self,tarobj):
-        try:
-            tarobj.chown = lambda *args: None   # don't do any chowning!
-            for member in tarobj:
-                if member.isfile() or member.isdir():
-                    name = member.name
-                    # don't extract absolute paths or ones with .. in them
-                    if not name.startswith('/') and '..' not in name:
-                        tarobj.extract(member,self.tmpdir)
-        finally:
-            tarobj.close()
-
-
-
-    def _run_setup(self, setup_script):
-        from setuptools.command import bdist_egg
-        sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
-        old_dir = os.getcwd()
-        save_argv = sys.argv[:]
-        save_path = sys.path[:]
-        try:
-            os.chdir(os.path.dirname(setup_script))
-            try:
-                sys.argv[:] = [setup_script, '-q', 'bdist_egg']
-                sys.path.insert(0,os.getcwd())
-                DirectorySandbox(self.tmpdir).run(
-                    lambda: execfile(
-                        "setup.py",
-                        {'__file__':setup_script, '__name__':'__main__'}
-                    )
-                )
-            except SystemExit, v:
-                if v.args and v.args[0]:
-                    raise RuntimeError(
-                        "Setup script exited with %s" % (v.args[0],)
-                    )
-        finally:
-            os.chdir(old_dir)
-            sys.path[:] = save_path
-            sys.argv[:] = save_argv
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
     def install_egg(self, egg_path, zip_ok):
 
         destination = os.path.join(self.instdir, os.path.basename(egg_path))
@@ -389,7 +307,7 @@ class Installer:
 
             else:
                 os.mkdir(destination)
-                self._extract_zip(egg_path, destination)
+                unpack_archive(egg_path, destination)   # XXX add progress??
 
         if os.path.isdir(destination):
             dist = Distribution.from_filename(
@@ -449,20 +367,6 @@ class Installer:
 
 
 
-    def _extract_file(self, dist_filename):
-        if zipfile.is_zipfile(dist_filename):
-            self._extract_zip(dist_filename, self.tmpdir)
-        else:
-            try:
-                tar = tarfile.open(dist_filename)
-            except tarfile.TarError:
-                raise RuntimeError(
-                    "Not a valid tar or zip archive: %s" % dist_filename
-                )
-            else:
-                self._extract_tar(tar)
-
-
     def _download_html(self, url, headers, filename):
         # Check for a sourceforge URL
         sf_url = url.startswith('http://prdownloads.')
@@ -494,6 +398,16 @@ class Installer:
         os.system("svn checkout -q %s %s" % (url, filename))
         return filename
 
+
+
+
+
+
+
+
+
+
+
     def _download_sourceforge(self, source_url, sf_page):
         """Download package from randomly-selected SourceForge mirror"""
 
@@ -531,6 +445,10 @@ class Installer:
 
 
 
+
+
+
+
     def installation_report(self, dist):
         """Helpful installation message for display to package users"""
 
@@ -572,170 +490,6 @@ PYTHONPATH, or by being added to sys.path by your code.)
 
 
 
-class AbstractSandbox:
-    """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""
-
-    _active = False
-
-    def __init__(self):
-        self._attrs = [
-            name for name in dir(_os)
-                if not name.startswith('_') and hasattr(self,name)
-        ]
-
-    def _copy(self, source):
-        for name in self._attrs:
-            setattr(os, name, getattr(source,name))
-
-    def run(self, func):
-        """Run 'func' under os sandboxing"""
-        try:
-            self._copy(self)
-            __builtin__.open = __builtin__.file = self._open
-            self._active = True
-            return func()
-        finally:
-            self._active = False
-            __builtin__.open = __builtin__.file = _open
-            self._copy(_os)
-
-
-    def _mk_dual_path_wrapper(name):
-        original = getattr(_os,name)
-        def wrap(self,src,dst,*args,**kw):
-            if self._active:
-                src,dst = self._remap_pair(name,src,dst,*args,**kw)
-            return original(src,dst,*args,**kw)
-        return wrap
-
-
-    for name in ["rename", "link", "symlink"]:
-        if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)
-
-
-    def _mk_single_path_wrapper(name, original=None):
-        original = original or getattr(_os,name)
-        def wrap(self,path,*args,**kw):
-            if self._active:
-                path = self._remap_input(name,path,*args,**kw)
-            return original(path,*args,**kw)
-        return wrap
-
-    _open = _mk_single_path_wrapper('file', _open)
-    for name in [
-        "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
-        "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
-        "startfile", "mkfifo", "mknod", "pathconf", "access"
-    ]:
-        if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)
-
-
-    def _mk_single_with_return(name):
-        original = getattr(_os,name)
-        def wrap(self,path,*args,**kw):
-            if self._active:
-                path = self._remap_input(name,path,*args,**kw)
-                return self._remap_output(name, original(path,*args,**kw))
-            return original(path,*args,**kw)
-        return wrap
-
-    for name in ['readlink', 'tempnam']:
-        if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)
-
-    def _mk_query(name):
-        original = getattr(_os,name)
-        def wrap(self,*args,**kw):
-            retval = original(*args,**kw)
-            if self._active:
-                return self._remap_output(name, retval)
-            return retval
-        return wrap
-
-    for name in ['getcwd', 'tmpnam']:
-        if hasattr(_os,name): locals()[name] = _mk_query(name)
-
-    def _validate_path(self,path):
-        """Called to remap or validate any path, whether input or output"""
-        return path
-
-    def _remap_input(self,operation,path,*args,**kw):
-        """Called for path inputs"""
-        return self._validate_path(path)
-
-    def _remap_output(self,operation,path):
-        """Called for path outputs"""
-        return self._validate_path(path)
-
-    def _remap_pair(self,operation,src,dst,*args,**kw):
-        """Called for path pairs like rename, link, and symlink operations"""
-        return (
-            self._remap_input(operation+'-from',src,*args,**kw),
-            self._remap_input(operation+'-to',dst,*args,**kw)
-        )
-
-
-class DirectorySandbox(AbstractSandbox):
-    """Restrict operations to a single subdirectory - pseudo-chroot"""
-
-    write_ops = dict.fromkeys([
-        "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
-        "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
-    ])
-
-    def __init__(self,sandbox):
-        self._sandbox = os.path.realpath(sandbox)
-        self._prefix = os.path.join(self._sandbox,'')
-        AbstractSandbox.__init__(self)
-
-    def _violation(self, operation, *args, **kw):
-        raise SandboxViolation(operation, args, kw)
-
-    def _open(self, path, mode='r', *args, **kw):
-        if mode not in ('r', 'rt', 'rb', 'rU') and not self._ok(path):
-            self._violation("open", path, mode, *args, **kw)
-        return _open(path,mode,*args,**kw)
-
-    def tmpnam(self):
-        self._violation("tmpnam")
-
-    def _ok(self,path):
-        active = self._active
-        try:
-            self._active = False
-            realpath = os.path.realpath(path)
-            if realpath==self._sandbox or realpath.startswith(self._prefix):
-                return True
-        finally:
-            self._active = active
-
-    def _remap_input(self,operation,path,*args,**kw):
-        """Called for path inputs"""
-        if operation in self.write_ops and not self._ok(path):
-            self._violation(operation, os.path.realpath(path), *args, **kw)
-        return path
-
-    def _remap_pair(self,operation,src,dst,*args,**kw):
-        """Called for path pairs like rename, link, and symlink operations"""
-        if not self._ok(src) or not self._ok(dst):
-            self._violation(operation, src, dst, *args, **kw)
-        return (src,dst)
-
-
-class SandboxViolation(RuntimeError):
-    """A setup script attempted to modify the filesystem outside the sandbox"""
-
-    def __str__(self):
-        return """SandboxViolation: %s%r %s
-
-The package setup script has attempted to modify files on your system
-that are not within the EasyInstall build area, and has been aborted.
-
-This package cannot be safely installed by EasyInstall, and may not
-support alternate installation locations even if you run its setup
-script by hand.  Please inform the package's author and the EasyInstall
-maintainers to find out if a fix or workaround is available.""" % self.args
-
-
 class PthDistributions(AvailableDistributions):
     """A .pth file with Distribution paths in it"""
 
diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py
new file mode 100755
index 00000000..f39d073e
--- /dev/null
+++ b/setuptools/archive_util.py
@@ -0,0 +1,164 @@
+"""Utilities for extracting common archive formats"""
+
+
+__all__ = [
+    "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter",
+    "UnrecognizedFormat", "extraction_drivers"
+]
+
+import zipfile, tarfile, os
+from pkg_resources import ensure_directory
+
+class UnrecognizedFormat(RuntimeError):
+    """Couldn't recognize the archive type"""
+
+def default_filter(src,dst):
+    """The default progress/filter callback; returns True for all files"""
+    
+    return True
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+def unpack_archive(filename, extract_dir, progress_filter=default_filter,
+    drivers=None
+):
+    """Unpack `filename` to `extract_dir`, or raise ``UnrecognizedFormat``
+
+    `progress_filter` is a function taking two arguments: a source path
+    internal to the archive ('/'-separated), and a filesystem path where it
+    will be extracted.  The callback must return a true value, or else that
+    file or directory will be skipped.  The callback can thus be used to
+    report on the progress of the extraction, as well as to filter the items
+    extracted.
+
+    `drivers`, if supplied, must be a non-empty sequence of functions with the
+    same signature as this function (minus the `drivers` argument), that raise
+    ``UnrecognizedFormat`` if they do not support extracting the designated
+    archive type.  The `drivers` are tried in sequence until one is found that
+    does not raise an error, or until all are exhausted (in which case
+    ``UnrecognizedFormat`` is raised).  If you do not supply a sequence of
+    drivers, the module's ``extraction_drivers`` constant will be used, which
+    means that ``unpack_zipfile`` and ``unpack_tarfile`` will be tried, in that
+    order.
+    """
+    for driver in drivers or extraction_drivers:
+        try:
+            driver(filename, extract_dir, progress_filter)
+        except UnrecognizedFormat:
+            continue
+        else:
+            return
+    else:
+        raise UnrecognizedFormat(
+            "Not a recognized archive type: %s" % filename
+        )
+
+
+
+
+
+
+
+
+def unpack_zipfile(filename, extract_dir, progress_filter=default_filter):
+    """Unpack zip `filename` to `extract_dir`
+
+    Raises ``UnrecognizedFormat`` if `filename` is not a zipfile (as determined
+    by ``zipfile.is_zipfile()``).  See ``unpack_archive()`` for an explanation
+    of the `progress_filter` argument.
+    """
+
+    if not zipfile.is_zipfile(filename):
+        raise UnrecognizedFormat("%s is not a zip file" % (filename,))
+
+    z = zipfile.ZipFile(filename)
+    try:
+        for info in z.infolist():
+            name = info.filename
+
+            # don't extract absolute paths or ones with .. in them
+            if name.startswith('/') or '..' in name:
+                continue
+
+            target = os.path.join(extract_dir,name)
+            if not progress_filter(name,target):
+                continue
+            if name.endswith('/'):
+                # directory
+                ensure_directory(target)
+            else:
+                # file
+                ensure_directory(target)
+                data = z.read(info.filename)
+                f = open(target,'wb')
+                try:
+                    f.write(data)
+                finally:
+                    f.close()
+                    del data
+    finally:
+        z.close()
+
+
+
+def unpack_tarfile(filename, extract_dir, progress_filter=default_filter):
+    """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir`
+
+    Raises ``UnrecognizedFormat`` if `filename` is not a tarfile (as determined
+    by ``tarfile.open()``).  See ``unpack_archive()`` for an explanation
+    of the `progress_filter` argument.
+    """
+
+    try:
+        tarobj = tarfile.open(filename)
+    except tarfile.TarError:
+        raise UnrecognizedFormat(
+            "%s is not a compressed or uncompressed tar file" % (filename,)
+        )
+
+    try:
+        tarobj.chown = lambda *args: None   # don't do any chowning!
+        for member in tarobj:
+            if member.isfile() or member.isdir():
+                name = member.name
+                # don't extract absolute paths or ones with .. in them
+                if not name.startswith('/') and '..' not in name:
+                    dst = os.path.join(extract_dir, *name.split('/'))                
+                    if progress_filter(name, dst):
+                        tarobj.extract(member,extract_dir)
+        return True
+    finally:
+        tarobj.close()
+
+
+
+
+extraction_drivers = unpack_zipfile, unpack_tarfile
+
+
+
+
+
+
+
+
diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py
new file mode 100755
index 00000000..6407947c
--- /dev/null
+++ b/setuptools/sandbox.py
@@ -0,0 +1,205 @@
+import os, sys, __builtin__
+_os = sys.modules[os.name]
+_open = open
+
+
+__all__ = [
+    "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
+]
+
+
+def run_setup(setup_script, *args):
+    """Run a distutils setup script, sandboxed in its directory"""
+
+    old_dir = os.getcwd()
+    save_argv = sys.argv[:]
+    save_path = sys.path[:]
+    setup_dir = os.path.abspath(os.path.dirname(setup_script))
+
+    try:
+        os.chdir(setup_dir)
+        try:
+            sys.argv[:] = [setup_script]+list(args)
+            sys.path.insert(0, setup_dir)
+            DirectorySandbox(setup_dir).run(
+                lambda: execfile(
+                    "setup.py",
+                    {'__file__':setup_script, '__name__':'__main__'}
+                )
+            )
+        except SystemExit, v:
+            if v.args and v.args[0]:
+                raise
+            # Normal exit, just return
+    finally:
+        os.chdir(old_dir)
+        sys.path[:] = save_path
+        sys.argv[:] = save_argv
+
+
+
+
+class AbstractSandbox:
+    """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""
+
+    _active = False
+
+    def __init__(self):
+        self._attrs = [
+            name for name in dir(_os)
+                if not name.startswith('_') and hasattr(self,name)
+        ]
+
+    def _copy(self, source):
+        for name in self._attrs:
+            setattr(os, name, getattr(source,name))
+
+    def run(self, func):
+        """Run 'func' under os sandboxing"""
+        try:
+            self._copy(self)
+            __builtin__.open = __builtin__.file = self._open
+            self._active = True
+            return func()
+        finally:
+            self._active = False
+            __builtin__.open = __builtin__.file = _open
+            self._copy(_os)
+
+
+    def _mk_dual_path_wrapper(name):
+        original = getattr(_os,name)
+        def wrap(self,src,dst,*args,**kw):
+            if self._active:
+                src,dst = self._remap_pair(name,src,dst,*args,**kw)
+            return original(src,dst,*args,**kw)
+        return wrap
+
+
+    for name in ["rename", "link", "symlink"]:
+        if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)
+
+
+    def _mk_single_path_wrapper(name, original=None):
+        original = original or getattr(_os,name)
+        def wrap(self,path,*args,**kw):
+            if self._active:
+                path = self._remap_input(name,path,*args,**kw)
+            return original(path,*args,**kw)
+        return wrap
+
+    _open = _mk_single_path_wrapper('file', _open)
+    for name in [
+        "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
+        "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
+        "startfile", "mkfifo", "mknod", "pathconf", "access"
+    ]:
+        if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)
+
+
+    def _mk_single_with_return(name):
+        original = getattr(_os,name)
+        def wrap(self,path,*args,**kw):
+            if self._active:
+                path = self._remap_input(name,path,*args,**kw)
+                return self._remap_output(name, original(path,*args,**kw))
+            return original(path,*args,**kw)
+        return wrap
+
+    for name in ['readlink', 'tempnam']:
+        if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)
+
+    def _mk_query(name):
+        original = getattr(_os,name)
+        def wrap(self,*args,**kw):
+            retval = original(*args,**kw)
+            if self._active:
+                return self._remap_output(name, retval)
+            return retval
+        return wrap
+
+    for name in ['getcwd', 'tmpnam']:
+        if hasattr(_os,name): locals()[name] = _mk_query(name)
+
+    def _validate_path(self,path):
+        """Called to remap or validate any path, whether input or output"""
+        return path
+
+    def _remap_input(self,operation,path,*args,**kw):
+        """Called for path inputs"""
+        return self._validate_path(path)
+
+    def _remap_output(self,operation,path):
+        """Called for path outputs"""
+        return self._validate_path(path)
+
+    def _remap_pair(self,operation,src,dst,*args,**kw):
+        """Called for path pairs like rename, link, and symlink operations"""
+        return (
+            self._remap_input(operation+'-from',src,*args,**kw),
+            self._remap_input(operation+'-to',dst,*args,**kw)
+        )
+
+
+class DirectorySandbox(AbstractSandbox):
+    """Restrict operations to a single subdirectory - pseudo-chroot"""
+
+    write_ops = dict.fromkeys([
+        "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
+        "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
+    ])
+
+    def __init__(self,sandbox):
+        self._sandbox = os.path.realpath(sandbox)
+        self._prefix = os.path.join(self._sandbox,'')
+        AbstractSandbox.__init__(self)
+
+    def _violation(self, operation, *args, **kw):
+        raise SandboxViolation(operation, args, kw)
+
+    def _open(self, path, mode='r', *args, **kw):
+        if mode not in ('r', 'rt', 'rb', 'rU') and not self._ok(path):
+            self._violation("open", path, mode, *args, **kw)
+        return _open(path,mode,*args,**kw)
+
+    def tmpnam(self):
+        self._violation("tmpnam")
+
+    def _ok(self,path):
+        active = self._active
+        try:
+            self._active = False
+            realpath = os.path.realpath(path)
+            if realpath==self._sandbox or realpath.startswith(self._prefix):
+                return True
+        finally:
+            self._active = active
+
+    def _remap_input(self,operation,path,*args,**kw):
+        """Called for path inputs"""
+        if operation in self.write_ops and not self._ok(path):
+            self._violation(operation, os.path.realpath(path), *args, **kw)
+        return path
+
+    def _remap_pair(self,operation,src,dst,*args,**kw):
+        """Called for path pairs like rename, link, and symlink operations"""
+        if not self._ok(src) or not self._ok(dst):
+            self._violation(operation, src, dst, *args, **kw)
+        return (src,dst)
+
+
+class SandboxViolation(RuntimeError):
+    """A setup script attempted to modify the filesystem outside the sandbox"""
+
+    def __str__(self):
+        return """SandboxViolation: %s%r %s
+
+The package setup script has attempted to modify files on your system
+that are not within the EasyInstall build area, and has been aborted.
+
+This package cannot be safely installed by EasyInstall, and may not
+support alternate installation locations even if you run its setup
+script by hand.  Please inform the package's author and the EasyInstall
+maintainers to find out if a fix or workaround is available.""" % self.args
+
+