From ec81bfe54962ae4f902e8536b5841e11402a6505 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Fri, 22 Jun 2012 23:33:19 -0400 Subject: find .dist-info distributions as well --HG-- branch : distribute extra : rebase_source : 9ec6872c031166c66743ee87d7018161aa415fb1 --- pkg_resources.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index a61c0efe..350a50ef 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -15,6 +15,7 @@ method. import sys, os, zipimport, time, re, imp, types from urlparse import urlparse, urlunparse +from email.parser import Parser try: frozenset @@ -1746,7 +1747,7 @@ def find_on_path(importer, path_item, only=False): # scan for .egg and .egg-info in directory for entry in os.listdir(path_item): lower = entry.lower() - if lower.endswith('.egg-info'): + if lower.endswith('.egg-info') or lower.endswith('.dist-info'): fullpath = os.path.join(path_item, entry) if os.path.isdir(fullpath): # egg-info directory, allow getting metadata @@ -2119,6 +2120,8 @@ def _remove_md5_fragment(location): class Distribution(object): """Wrap an actual or potential sys.path entry w/metadata""" + PKG_INFO = 'PKG-INFO' + def __init__(self, location=None, metadata=None, project_name=None, version=None, py_version=PY_MAJOR, platform=None, precedence = EGG_DIST @@ -2136,12 +2139,14 @@ class Distribution(object): def from_location(cls,location,basename,metadata=None,**kw): project_name, version, py_version, platform = [None]*4 basename, ext = os.path.splitext(basename) - if ext.lower() in (".egg",".egg-info"): + if ext.lower() in _distributionImpl: + # .dist-info gets much metadata differently match = EGG_NAME(basename) if match: project_name, version, py_version, platform = match.group( 'name','ver','pyver','plat' ) + cls = _distributionImpl[ext.lower()] return cls( location, metadata, project_name=project_name, version=version, py_version=py_version, platform=platform, **kw @@ -2204,13 +2209,13 @@ class Distribution(object): try: return self._version except AttributeError: - for line in self._get_metadata('PKG-INFO'): + for line in self._get_metadata(self.PKG_INFO): if line.lower().startswith('version:'): self._version = safe_version(line.split(':',1)[1].strip()) return self._version else: raise ValueError( - "Missing 'Version:' header and/or PKG-INFO file", self + "Missing 'Version:' header and/or %s file" % self.PKG_INFO, self ) version = property(version) @@ -2441,6 +2446,54 @@ class Distribution(object): extras = property(extras) +class DistInfoDistribution(Distribution): + """Wrap an actual or potential sys.path entry w/metadata, .dist-info style""" + PKG_INFO = 'METADATA' + + @property + def _parsed_pkg_info(self): + """Parse and cache metadata""" + try: + return self._pkg_info + except AttributeError: + self._pkg_info = Parser().parsestr(self.get_metadata(self.PKG_INFO)) + return self._pkg_info + + @property + def _dep_map(self): + try: + return self.__dep_map + except AttributeError: + dm = self.__dep_map = {None: []} + # Including condition expressions + # XXX parse condition expressions, extras + reqs = self._parsed_pkg_info.get_all('Requires-Dist') +# extras = self._parsed_pkg_info.get_all('Provides-Extra') or [] +# for extra,reqs in split_sections(self._get_metadata(name)): +# if extra: extra = safe_extra(extra) + dm.setdefault(None,[]).extend(parse_requirements(reqs)) + return dm + + def requires(self,extras=()): + """List of Requirements needed for this distro if `extras` are used""" + dm = self._dep_map + deps = [] + deps.extend(dm.get(None,())) + for ext in extras: + try: + deps.extend(dm[safe_extra(ext)]) + except KeyError: + raise UnknownExtra( + "%s has no such extra feature %r" % (self, ext) + ) + return deps + + +_distributionImpl = {'.egg': Distribution, + '.egg-info': Distribution, + '.dist-info': DistInfoDistribution } + + def issue_warning(*args,**kw): level = 1 g = globals() -- cgit v1.2.3 From ac2bc6ee75884f521d9097b83315adf413ff2a7c Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Fri, 29 Jun 2012 03:50:45 -0400 Subject: implement environment markers --HG-- branch : distribute extra : rebase_source : 16af7be95f787ab5de573388270b97cda6c9cea7 --- pkg_resources.py | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index 350a50ef..5e3967f3 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -15,7 +15,6 @@ method. import sys, os, zipimport, time, re, imp, types from urlparse import urlparse, urlunparse -from email.parser import Parser try: frozenset @@ -2456,6 +2455,7 @@ class DistInfoDistribution(Distribution): try: return self._pkg_info except AttributeError: + from email.parser import Parser self._pkg_info = Parser().parsestr(self.get_metadata(self.PKG_INFO)) return self._pkg_info @@ -2464,15 +2464,41 @@ class DistInfoDistribution(Distribution): try: return self.__dep_map except AttributeError: - dm = self.__dep_map = {None: []} - # Including condition expressions - # XXX parse condition expressions, extras - reqs = self._parsed_pkg_info.get_all('Requires-Dist') -# extras = self._parsed_pkg_info.get_all('Provides-Extra') or [] -# for extra,reqs in split_sections(self._get_metadata(name)): -# if extra: extra = safe_extra(extra) - dm.setdefault(None,[]).extend(parse_requirements(reqs)) - return dm + self.__dep_map = self._compute_dependencies() + return self.__dep_map + + def _compute_dependencies(self): + """Recompute this distribution's dependencies.""" + def dummy_marker(marker): + def marker_fn(environment=None, override=None): + return True + return marker_fn + try: + from wheel.markers import as_function + except ImportError: + as_function = dummy_marker + dm = self.__dep_map = {None: []} + + reqs = [] + # Including any condition expressions + for req in self._parsed_pkg_info.get_all('Requires-Dist'): + rs = req.split(';', 1) + [''] + parsed = parse_requirements(rs[0]).next() + parsed.marker_fn = as_function(rs[1].strip()) + reqs.append(parsed) + + def reqs_for_extra(extra): + for req in reqs: + if req.marker_fn(override={'extra':extra}): + yield req + + common = set(reqs_for_extra(None)) + dm[None].extend(common) + + for extra in self._parsed_pkg_info.get_all('Provides-Extra'): + dm[extra] = list(set(reqs_for_extra(extra)) - common) + + return dm def requires(self,extras=()): """List of Requirements needed for this distro if `extras` are used""" -- cgit v1.2.3 From 07f0901db42f1bc5d411868e92e7e77d27f90323 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Fri, 29 Jun 2012 04:11:44 -0400 Subject: use markerlib --HG-- branch : distribute extra : rebase_source : a684fafb1aeb44954bdb8604a77fcc8d1549648c --- pkg_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg_resources.py b/pkg_resources.py index 5e3967f3..dea54490 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -2474,7 +2474,7 @@ class DistInfoDistribution(Distribution): return True return marker_fn try: - from wheel.markers import as_function + from markerlib import as_function except ImportError: as_function = dummy_marker dm = self.__dep_map = {None: []} -- cgit v1.2.3 From d87cabbf4cf11c0049c95d3359fc249b002b542e Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Fri, 29 Jun 2012 19:48:39 -0400 Subject: add missing == to prefix-less Requires-Dist version specifiers --HG-- branch : distribute extra : rebase_source : 131e2f27792688da5dd1f8bf984939ab51403a64 --- pkg_resources.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index dea54490..10c8aff5 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -2466,12 +2466,36 @@ class DistInfoDistribution(Distribution): except AttributeError: self.__dep_map = self._compute_dependencies() return self.__dep_map + + def _preparse_requirement(self, requires_dist): + """Return (dist, versions, marker). + Add == prefix to version specifiers as necessary. + """ + m = re.compile(r"^(?P.*?)\s*(\((?P.*?)\))?\s*(;\s*(?P.*))?$") + match = m.match(requires_dist) + if not match: + raise ValueError("Unexpected Requires-Dist: %s" % requires_dist) + dist = match.group('d') + if match.group('v'): + vers = match.group('v').split(',') + else: + vers = [] + mark = match.group('m') or '' + for i, v in enumerate(vers): + if not VERSION(v): + v = "==%s" % v + if not VERSION(v): + raise ValueError("Unexpected version: (%s)" % + match.group('v')) + vers[i] = v + return (dist, ', '.join(vers), mark) def _compute_dependencies(self): """Recompute this distribution's dependencies.""" def dummy_marker(marker): def marker_fn(environment=None, override=None): return True + marker_fn.__doc__ = marker return marker_fn try: from markerlib import as_function @@ -2481,10 +2505,10 @@ class DistInfoDistribution(Distribution): reqs = [] # Including any condition expressions - for req in self._parsed_pkg_info.get_all('Requires-Dist'): - rs = req.split(';', 1) + [''] - parsed = parse_requirements(rs[0]).next() - parsed.marker_fn = as_function(rs[1].strip()) + for req in self._parsed_pkg_info.get_all('Requires-Dist'): + dist, vers, mark = self._preparse_requirement(req) + parsed = parse_requirements("%s %s" % (dist, vers)).next() + parsed.marker_fn = as_function(mark) reqs.append(parsed) def reqs_for_extra(extra): @@ -2492,11 +2516,12 @@ class DistInfoDistribution(Distribution): if req.marker_fn(override={'extra':extra}): yield req - common = set(reqs_for_extra(None)) + common = set(reqs_for_extra(None)) dm[None].extend(common) for extra in self._parsed_pkg_info.get_all('Provides-Extra'): - dm[extra] = list(set(reqs_for_extra(extra)) - common) + extra = safe_extra(extra.strip()) + dm[extra] = list(set(reqs_for_extra(extra)) - common) return dm -- cgit v1.2.3 From 9b885a3c4843c3985413181a6d18b18ce015cdce Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sat, 30 Jun 2012 08:52:43 -0400 Subject: simplify Requires-Dist pre-parser --HG-- branch : distribute extra : rebase_source : 10900b24f80bc412881812fcea4e54e473a86068 --- pkg_resources.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index 10c8aff5..20513dc7 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -2448,6 +2448,7 @@ class Distribution(object): class DistInfoDistribution(Distribution): """Wrap an actual or potential sys.path entry w/metadata, .dist-info style""" PKG_INFO = 'METADATA' + EQEQ = re.compile(r"([\(,])\s*(\d.*?)\s*([,\)])") @property def _parsed_pkg_info(self): @@ -2471,24 +2472,12 @@ class DistInfoDistribution(Distribution): """Return (dist, versions, marker). Add == prefix to version specifiers as necessary. """ - m = re.compile(r"^(?P.*?)\s*(\((?P.*?)\))?\s*(;\s*(?P.*))?$") - match = m.match(requires_dist) - if not match: - raise ValueError("Unexpected Requires-Dist: %s" % requires_dist) - dist = match.group('d') - if match.group('v'): - vers = match.group('v').split(',') - else: - vers = [] - mark = match.group('m') or '' - for i, v in enumerate(vers): - if not VERSION(v): - v = "==%s" % v - if not VERSION(v): - raise ValueError("Unexpected version: (%s)" % - match.group('v')) - vers[i] = v - return (dist, ', '.join(vers), mark) + parts = requires_dist.split(';', 1) + [''] + distvers = parts[0].strip() + mark = parts[1].strip() + distvers = re.sub(self.EQEQ, r"\1==\2\3", distvers) + distvers = distvers.replace('(', '').replace(')', '') + return (distvers, mark) def _compute_dependencies(self): """Recompute this distribution's dependencies.""" @@ -2506,8 +2495,8 @@ class DistInfoDistribution(Distribution): reqs = [] # Including any condition expressions for req in self._parsed_pkg_info.get_all('Requires-Dist'): - dist, vers, mark = self._preparse_requirement(req) - parsed = parse_requirements("%s %s" % (dist, vers)).next() + distvers, mark = self._preparse_requirement(req) + parsed = parse_requirements(distvers).next() parsed.marker_fn = as_function(mark) reqs.append(parsed) -- cgit v1.2.3 From 7afb6a596a2d16b4d74b1b890eb9ca44d11197bd Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sat, 30 Jun 2012 09:02:41 -0400 Subject: update docstring --HG-- branch : distribute extra : rebase_source : ee1bf5fd6c851fa3d9c87c1132f0864a811f09e5 --- pkg_resources.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index 20513dc7..652079ec 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -2469,8 +2469,9 @@ class DistInfoDistribution(Distribution): return self.__dep_map def _preparse_requirement(self, requires_dist): - """Return (dist, versions, marker). - Add == prefix to version specifiers as necessary. + """Convert 'Foobar (1); baz' to ('Foobar ==1', 'baz') + Split environment marker, add == prefix to version specifiers as + necessary, and remove parenthesis. """ parts = requires_dist.split(';', 1) + [''] distvers = parts[0].strip() -- cgit v1.2.3 From 0d5901c018dbccaa9bffc8902796b40a546f7f3d Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sat, 30 Jun 2012 12:34:31 -0400 Subject: handle missing provides-extra or requires-dist --HG-- branch : distribute extra : rebase_source : 99392449586f9f299abfd1627ba60b9752caf26d --- pkg_resources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index 652079ec..285aa1bb 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -2495,7 +2495,7 @@ class DistInfoDistribution(Distribution): reqs = [] # Including any condition expressions - for req in self._parsed_pkg_info.get_all('Requires-Dist'): + for req in self._parsed_pkg_info.get_all('Requires-Dist') or []: distvers, mark = self._preparse_requirement(req) parsed = parse_requirements(distvers).next() parsed.marker_fn = as_function(mark) @@ -2509,7 +2509,7 @@ class DistInfoDistribution(Distribution): common = set(reqs_for_extra(None)) dm[None].extend(common) - for extra in self._parsed_pkg_info.get_all('Provides-Extra'): + for extra in self._parsed_pkg_info.get_all('Provides-Extra') or []: extra = safe_extra(extra.strip()) dm[extra] = list(set(reqs_for_extra(extra)) - common) -- cgit v1.2.3 From b73bf20cea512d0d95a29db4fd9ed837130c54fc Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sat, 7 Jul 2012 23:24:20 -0400 Subject: test .dist-info distributions --HG-- branch : distribute extra : rebase_source : ea6870d73aa69f2deacc50beb2e257d3c21073e4 --- pkg_resources.py | 16 +---------- setuptools/tests/test_dist_info.py | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 setuptools/tests/test_dist_info.py diff --git a/pkg_resources.py b/pkg_resources.py index 285aa1bb..27b9f834 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -2514,21 +2514,7 @@ class DistInfoDistribution(Distribution): dm[extra] = list(set(reqs_for_extra(extra)) - common) return dm - - def requires(self,extras=()): - """List of Requirements needed for this distro if `extras` are used""" - dm = self._dep_map - deps = [] - deps.extend(dm.get(None,())) - for ext in extras: - try: - deps.extend(dm[safe_extra(ext)]) - except KeyError: - raise UnknownExtra( - "%s has no such extra feature %r" % (self, ext) - ) - return deps - + _distributionImpl = {'.egg': Distribution, '.egg-info': Distribution, diff --git a/setuptools/tests/test_dist_info.py b/setuptools/tests/test_dist_info.py new file mode 100644 index 00000000..c40886d3 --- /dev/null +++ b/setuptools/tests/test_dist_info.py @@ -0,0 +1,55 @@ +"""Test .dist-info style distributions. +""" +import os, shutil, tempfile, unittest +import pkg_resources +from pkg_resources import Requirement + +class TestDistInfo(unittest.TestCase): + + def test_distinfo(self): + dists = {} + for d in pkg_resources.find_distributions(self.tmpdir): + dists[d.project_name] = d + + unversioned = dists['UnversionedDistribution'] + versioned = dists['VersionedDistribution'] + + assert versioned.version == '2.718' # from filename + assert unversioned.version == '0.3' # from METADATA + + requires = [Requirement.parse('splort==4'), + Requirement.parse('quux>=1.1')] + + for d in (unversioned, versioned): + self.assertEquals(d.requires(), requires[:1]) + self.assertEquals(d.requires(extras=('baz',)), requires) + self.assertEquals(d.extras, ['baz']) + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + versioned = os.path.join(self.tmpdir, + 'VersionedDistribution-2.718.dist-info') + os.mkdir(versioned) + open(os.path.join(versioned, 'METADATA'), 'w+').write( +"""Metadata-Version: 1.2 +Name: VersionedDistribution +Requires-Dist: splort (4) +Provides-Extra: baz +Requires-Dist: quux (>=1.1); extra == 'baz' +""") + + unversioned = os.path.join(self.tmpdir, + 'UnversionedDistribution.dist-info') + os.mkdir(unversioned) + open(os.path.join(unversioned, 'METADATA'), 'w+').write( +"""Metadata-Version: 1.2 +Name: UnversionedDistribution +Version: 0.3 +Requires-Dist: splort (==4) +Provides-Extra: baz +Requires-Dist: quux (>=1.1); extra == 'baz' +""") + + def tearDown(self): + shutil.rmtree(self.tmpdir) + -- cgit v1.2.3 From c62d8121c4660daa4fd6132643b8e98918faf58f Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sat, 7 Jul 2012 23:34:12 -0400 Subject: improve .dist-info test --HG-- branch : distribute extra : rebase_source : 23b6b4023d413bf1b27b114687da8736a4b38d8b --- setuptools/tests/test_dist_info.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/setuptools/tests/test_dist_info.py b/setuptools/tests/test_dist_info.py index c40886d3..119cc68b 100644 --- a/setuptools/tests/test_dist_info.py +++ b/setuptools/tests/test_dist_info.py @@ -3,6 +3,11 @@ import os, shutil, tempfile, unittest import pkg_resources from pkg_resources import Requirement +try: + import markerlib + has_markerlib = True +except: + has_markerlib = False class TestDistInfo(unittest.TestCase): @@ -11,19 +16,24 @@ class TestDistInfo(unittest.TestCase): for d in pkg_resources.find_distributions(self.tmpdir): dists[d.project_name] = d + assert len(dists) == 2, dists + unversioned = dists['UnversionedDistribution'] versioned = dists['VersionedDistribution'] assert versioned.version == '2.718' # from filename assert unversioned.version == '0.3' # from METADATA - + + @unittest.skipIf(not has_markerlib, + "install markerlib to test conditional dependencies") + def test_conditional_dependencies(self): requires = [Requirement.parse('splort==4'), Requirement.parse('quux>=1.1')] - for d in (unversioned, versioned): + for d in pkg_resources.find_distributions(self.tmpdir): self.assertEquals(d.requires(), requires[:1]) self.assertEquals(d.requires(extras=('baz',)), requires) - self.assertEquals(d.extras, ['baz']) + self.assertEquals(d.extras, ['baz']) def setUp(self): self.tmpdir = tempfile.mkdtemp() -- cgit v1.2.3