Added "--allow-hosts" option to restrict downloading and spidering to

a specified list of server glob patterns.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041266

2 files changed, 31 insertions, 31 deletions

diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index c076a5fd..ad19a732 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -43,7 +43,6 @@ class easy_install(Command):
     """Manage a download/build/install process"""
 
     description = "Find/get/install Python packages"
-
     command_consumes_arguments = True
 
     user_options = [
@@ -71,6 +70,7 @@ class easy_install(Command):
         ('site-dirs=','S',"list of directories where .pth files work"),
         ('editable', 'e', "Install specified packages in editable form"),
         ('no-deps', 'N', "don't install dependencies"),
+        ('allow-hosts=', 'H', "pattern(s) that hostnames must match"),
     ]
     boolean_options = [
         'zip-ok', 'multi-version', 'exclude-scripts', 'upgrade', 'always-copy',
@@ -89,7 +89,7 @@ class easy_install(Command):
         self.args = None
         self.optimize = self.record = None
         self.upgrade = self.always_copy = self.multi_version = None
-        self.editable = self.no_deps = None
+        self.editable = self.no_deps = self.allow_hosts = None
         self.root = None
 
         # Options not specifiable via command line
@@ -177,9 +177,15 @@ class easy_install(Command):
         for path_item in self.install_dir, normalize_path(self.script_dir):
             if path_item not in self.shadow_path:
                 self.shadow_path.insert(0, path_item)
+
+        if self.allow_hosts is not None:
+            hosts = [s.strip() for s in self.allow_hosts.split(',')]
+        else:
+            hosts = ['*']
+
         if self.package_index is None:
             self.package_index = self.create_index(
-                self.index_url, search_path = self.shadow_path
+                self.index_url, search_path = self.shadow_path, hosts=hosts
             )
         self.local_index = Environment(self.shadow_path)
 
@@ -202,7 +208,6 @@ class easy_install(Command):
                 "Can't use both --delete-conflicting and "
                 "--ignore-conflicts-at-my-risk at the same time"
             )
-
         if self.editable and not self.build_directory:
             raise DistutilsArgError(
                 "Must specify a build directory (-b) when using --editable"
@@ -239,11 +244,6 @@ class easy_install(Command):
             log.set_verbosity(self.distribution.verbose)
 
 
-
-
-
-
-
     def install_egg_scripts(self, dist):
         """Write all the scripts for `dist`, unless scripts are excluded"""
 
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index f5b322b0..76d53d11 100755
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -5,11 +5,11 @@ from pkg_resources import *
 from distutils import log
 from distutils.errors import DistutilsError
 from md5 import md5
+from fnmatch import translate
 
 EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$')
 HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I)
 # this is here to fix emacs' cruddy broken syntax highlighting
-
 PYPI_MD5 = re.compile(
     '<a href="([^"#]+)">([^<]+)</a>\n\s+\\(<a href="[^?]+\?:action=show_md5'
     '&amp;digest=([0-9a-f]{32})">md5</a>\\)'
@@ -124,25 +124,25 @@ def interpret_distro_name(location, basename, metadata,
 class PackageIndex(Environment):
     """A distribution index that scans web pages for download URLs"""
 
-    def __init__(self,index_url="http://www.python.org/pypi",*args,**kw):
+    def __init__(self,index_url="http://www.python.org/pypi",hosts=('*',),*args,**kw):
         Environment.__init__(self,*args,**kw)
         self.index_url = index_url + "/"[:not index_url.endswith('/')]
         self.scanned_urls = {}
         self.fetched_urls = {}
         self.package_pages = {}
+        self.allows = re.compile('|'.join(map(translate,hosts))).match
 
     def process_url(self, url, retrieve=False):
         """Evaluate a URL as a possible download, and maybe retrieve it"""
-
         if url in self.scanned_urls and not retrieve:
             return
 
         self.scanned_urls[url] = True
-
         if not URL_SCHEME(url):
             # process filenames or directories
             if os.path.isfile(url):
-                dists = list(distros_for_filename(url))
+                map(self.add, distros_for_filename(url))
+                return    # no need to retrieve anything
             elif os.path.isdir(url):
                 url = os.path.realpath(url)
                 for item in os.listdir(url):
@@ -153,13 +153,16 @@ class PackageIndex(Environment):
                 return
         else:
             dists = list(distros_for_url(url))
+            if dists:
+                if not self.url_ok(url):
+                    return
+                self.debug("Found link: %s", url)
 
-        if dists:
-            self.debug("Found link: %s", url)
         if dists or not retrieve or url in self.fetched_urls:
-            for dist in dists:
-                self.add(dist)
-            # don't need the actual page
+            map(self.add, dists)
+            return  # don't need the actual page
+
+        if not self.url_ok(url):
             return
 
         self.info("Reading %s", url)
@@ -181,17 +184,14 @@ class PackageIndex(Environment):
             self.process_url(link)
 
 
-
-
-
-
-
-
-
-
-
-
-
+    def url_ok(self, url, fatal=False):
+        if self.allows(urlparse.urlparse(url)[1]):
+            return True
+        msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n"
+        if fatal:
+            raise DistutilsError(msg % url)
+        else:
+            self.warn(msg, url)
 
 
 
@@ -368,8 +368,8 @@ class PackageIndex(Environment):
 
 
     dl_blocksize = 8192
-
     def _download_to(self, url, filename):
+        self.url_ok(url,True)   # raises error if not allowed
         self.info("Downloading %s", url)
         # Download the file
         fp, tfp, info = None, None, None