diff options
Diffstat (limited to 'setuptools/package_index.py')
| -rwxr-xr-x | setuptools/package_index.py | 82 |
1 files changed, 45 insertions, 37 deletions
diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b0388628..85a1deeb 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,11 +1,14 @@ """PyPI and direct package downloading""" -import sys, os.path, re, urlparse, urllib, urllib2, shutil, random, socket, cStringIO +import sys, os.path, re, shutil, random, socket import itertools import base64 -import httplib from pkg_resources import * from distutils import log from distutils.errors import DistutilsError +from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, + urlparse, urlunparse, unquote, splituser, + url2pathname, name2codepoint, + unichr, urljoin) try: from hashlib import md5 except ImportError: @@ -56,8 +59,8 @@ def parse_bdist_wininst(name): def egg_info_for_url(url): - scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) - base = urllib2.unquote(path.split('/')[-1]) + scheme, server, path, parameters, query, fragment = urlparse(url) + base = unquote(path.split('/')[-1]) if '#' in base: base, fragment = base.split('#',1) return base,fragment @@ -177,14 +180,14 @@ def find_external_links(url, page): rels = map(str.strip, rel.lower().split(',')) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - yield urlparse.urljoin(url, htmldecode(match.group(1))) + yield urljoin(url, htmldecode(match.group(1))) for tag in ("<th>Home Page", "<th>Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - yield urlparse.urljoin(url, htmldecode(match.group(1))) + yield urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s distribute/%s" % ( @@ -224,7 +227,7 @@ class PackageIndex(Environment): self.debug("Found link: %s", url) if dists or not retrieve or url in self.fetched_urls: - map(self.add, dists) + list(map(self.add, dists)) return # don't need the actual page if not self.url_ok(url): @@ -243,7 +246,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. - if isinstance(f, urllib2.HTTPError): + if isinstance(f, HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' else: @@ -251,7 +254,7 @@ class PackageIndex(Environment): page = page.decode(charset, "ignore") f.close() for match in HREF.finditer(page): - link = urlparse.urljoin(base, htmldecode(match.group(1))) + link = urljoin(base, htmldecode(match.group(1))) self.process_url(link) if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) @@ -270,11 +273,11 @@ class PackageIndex(Environment): dists = distros_for_filename(fn) if dists: self.debug("Found: %s", fn) - map(self.add, dists) + list(map(self.add, dists)) def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower()=='file') or self.allows(urlparse.urlparse(url)[1]): + if (s and s.group(1).lower()=='file') or self.allows(urlparse(url)[1]): return True msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n" if fatal: @@ -290,7 +293,8 @@ class PackageIndex(Environment): self.scan_egg_link(item, entry) def scan_egg_link(self, path, entry): - lines = filter(None, map(str.strip, open(os.path.join(path, entry)))) + lines = [_f for _f in map(str.strip, + open(os.path.join(path, entry))) if _f] if len(lines)==2: for dist in find_distributions(os.path.join(path, lines[0])): dist.location = os.path.join(path, *lines) @@ -302,9 +306,9 @@ class PackageIndex(Environment): def scan(link): # Process a URL to see if it's for a package page if link.startswith(self.index_url): - parts = map( - urllib2.unquote, link[len(self.index_url):].split('/') - ) + parts = list(map( + unquote, link[len(self.index_url):].split('/') + )) if len(parts)==2 and '#' not in parts[1]: # it's a package page, sanitize and index it pkg = safe_name(parts[0]) @@ -316,7 +320,7 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): try: - scan( urlparse.urljoin(url, htmldecode(match.group(1))) ) + scan( urljoin(url, htmldecode(match.group(1))) ) except ValueError: pass @@ -385,7 +389,7 @@ class PackageIndex(Environment): def check_md5(self, cs, info, filename, tfp): if re.match('md5=[0-9a-f]{32}$', info): self.debug("Validating md5 checksum for %s", filename) - if cs.hexdigest()<>info[4:]: + if cs.hexdigest() != info[4:]: tfp.close() os.unlink(filename) raise DistutilsError( @@ -411,7 +415,7 @@ class PackageIndex(Environment): def prescan(self): """Scan urls scheduled for prescanning (e.g. --find-links)""" if self.to_scan: - map(self.scan_url, self.to_scan) + list(map(self.scan_url, self.to_scan)) self.to_scan = None # from now on, go ahead and process immediately def not_found_in_index(self, requirement): @@ -603,7 +607,7 @@ class PackageIndex(Environment): if '#' in url: url, info = url.split('#', 1) fp = self.open_url(url) - if isinstance(fp, urllib2.HTTPError): + if isinstance(fp, HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) @@ -642,28 +646,33 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url) - except (ValueError, httplib.InvalidURL), v: + except (ValueError, httplib.InvalidURL): + v = sys.exc_info()[1] msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib2.HTTPError, v: + except urllib2.HTTPError: + v = sys.exc_info()[1] return v - except urllib2.URLError, v: + except urllib2.URLError: + v = sys.exc_info()[1] if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine, v: + except httplib.BadStatusLine: + v = sys.exc_info()[1] if warning: self.warn(warning, v.line) else: raise DistutilsError('%s returned a bad status line. ' 'The server might be down, %s' % \ (url, v.line)) - except httplib.HTTPException, v: + except httplib.HTTPException: + v = sys.exc_info()[1] if warning: self.warn(warning, v) else: @@ -673,7 +682,7 @@ class PackageIndex(Environment): def _download_url(self, scheme, url, tmpdir): # Determine download filename # - name = filter(None,urlparse.urlparse(url)[2].split('/')) + name = [_f for _f in urlparse(url)[2].split('/') if _f] if name: name = name[-1] while '..' in name: @@ -695,7 +704,7 @@ class PackageIndex(Environment): elif scheme.startswith('hg+'): return self._download_hg(url, filename) elif scheme=='file': - return urllib.url2pathname(urlparse.urlparse(url)[2]) + return url2pathname(urlparse.urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) @@ -809,7 +818,6 @@ def decode_entity(match): elif what.startswith('#'): what = int(what[1:]) else: - from htmlentitydefs import name2codepoint what = name2codepoint.get(what, match.group(0)) return uchr(what) @@ -850,7 +858,7 @@ def _encode_auth(auth): >>> _encode_auth('username%3Apassword') u'dXNlcm5hbWU6cGFzc3dvcmQ=' """ - auth_s = urllib2.unquote(auth) + auth_s = unquote(auth) # convert to bytes auth_bytes = auth_s.encode() # use the legacy interface for Python 2.3 support @@ -863,7 +871,7 @@ def _encode_auth(auth): def open_with_auth(url): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urlparse.urlparse(url) + scheme, netloc, path, params, query, frag = urlparse(url) # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. @@ -871,13 +879,13 @@ def open_with_auth(url): raise httplib.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): - auth, host = urllib2.splituser(netloc) + auth, host = splituser(netloc) else: auth = None if auth: auth = "Basic " + _encode_auth(auth) - new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) + new_url = urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) else: @@ -889,9 +897,9 @@ def open_with_auth(url): if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work - s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url) + s2, h2, path2, param2, query2, frag2 = urlparse(fp.url) if s2==scheme and h2==host: - fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2)) + fp.url = urlunparse((s2,netloc,path2,param2,query2,frag2)) return fp @@ -913,8 +921,8 @@ def fix_sf_url(url): def local_open(url): """Read a local path, with special support for directories""" - scheme, server, path, param, query, frag = urlparse.urlparse(url) - filename = urllib.url2pathname(path) + scheme, server, path, param, query, frag = urlparse(url) + filename = url2pathname(path) if os.path.isfile(filename): return urllib2.urlopen(url) elif path.endswith('/') and os.path.isdir(filename): @@ -935,8 +943,8 @@ def local_open(url): else: status, message, body = 404, "Path not found", "Not found" - return urllib2.HTTPError(url, status, message, - {'content-type':'text/html'}, cStringIO.StringIO(body)) + return HTTPError(url, status, message, + {'content-type':'text/html'}, StringIO(body)) |