diff options
Diffstat (limited to 'setuptools')
80 files changed, 12361 insertions, 585 deletions
diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 77b4a374..a71b2bbd 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -1,6 +1,7 @@ """Extensions to the 'distutils' for large or complex distributions""" import os +import sys import functools import distutils.core import distutils.filelist @@ -9,7 +10,9 @@ from distutils.errors import DistutilsOptionError from distutils.util import convert_path from fnmatch import fnmatchcase -from setuptools.extern.six import string_types +from ._deprecation_warning import SetuptoolsDeprecationWarning + +from setuptools.extern.six import PY3, string_types from setuptools.extern.six.moves import filter, map import setuptools.version @@ -18,11 +21,18 @@ from setuptools.dist import Distribution, Feature from setuptools.depends import Require from . import monkey +__metaclass__ = type + + __all__ = [ 'setup', 'Distribution', 'Feature', 'Command', 'Extension', 'Require', - 'find_packages', + 'SetuptoolsDeprecationWarning', + 'find_packages' ] +if PY3: + __all__.append('find_namespace_packages') + __version__ = setuptools.version.__version__ bootstrap_install_from = None @@ -34,7 +44,7 @@ run_2to3_on_doctests = True lib2to3_fixer_packages = ['lib2to3.fixes'] -class PackageFinder(object): +class PackageFinder: """ Generate a list of all Python packages found within a directory """ @@ -112,7 +122,30 @@ class PEP420PackageFinder(PackageFinder): find_packages = PackageFinder.find -setup = distutils.core.setup +if PY3: + find_namespace_packages = PEP420PackageFinder.find + + +def _install_setup_requires(attrs): + # Note: do not use `setuptools.Distribution` directly, as + # our PEP 517 backend patch `distutils.core.Distribution`. + dist = distutils.core.Distribution(dict( + (k, v) for k, v in attrs.items() + if k in ('dependency_links', 'setup_requires') + )) + # Honor setup.cfg's options. + dist.parse_config_files(ignore_option_errors=True) + if dist.setup_requires: + dist.fetch_build_eggs(dist.setup_requires) + + +def setup(**attrs): + # Make sure we have any requirements needed to interpret 'attrs'. + _install_setup_requires(attrs) + return distutils.core.setup(**attrs) + +setup.__doc__ = distutils.core.setup.__doc__ + _Command = monkey.get_unpatched(distutils.core.Command) @@ -191,4 +224,5 @@ def findall(dir=os.curdir): return list(files) +# Apply monkey patches monkey.patch_all() diff --git a/setuptools/_deprecation_warning.py b/setuptools/_deprecation_warning.py new file mode 100644 index 00000000..086b64dd --- /dev/null +++ b/setuptools/_deprecation_warning.py @@ -0,0 +1,7 @@ +class SetuptoolsDeprecationWarning(Warning): + """ + Base class for warning deprecations in ``setuptools`` + + This class is not derived from ``DeprecationWarning``, and as such is + visible by default. + """ diff --git a/setuptools/_vendor/__init__.py b/setuptools/_vendor/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/setuptools/_vendor/__init__.py diff --git a/setuptools/_vendor/packaging/__about__.py b/setuptools/_vendor/packaging/__about__.py new file mode 100644 index 00000000..95d330ef --- /dev/null +++ b/setuptools/_vendor/packaging/__about__.py @@ -0,0 +1,21 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +__all__ = [ + "__title__", "__summary__", "__uri__", "__version__", "__author__", + "__email__", "__license__", "__copyright__", +] + +__title__ = "packaging" +__summary__ = "Core utilities for Python packages" +__uri__ = "https://github.com/pypa/packaging" + +__version__ = "16.8" + +__author__ = "Donald Stufft and individual contributors" +__email__ = "donald@stufft.io" + +__license__ = "BSD or Apache License, Version 2.0" +__copyright__ = "Copyright 2014-2016 %s" % __author__ diff --git a/setuptools/_vendor/packaging/__init__.py b/setuptools/_vendor/packaging/__init__.py new file mode 100644 index 00000000..5ee62202 --- /dev/null +++ b/setuptools/_vendor/packaging/__init__.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +from .__about__ import ( + __author__, __copyright__, __email__, __license__, __summary__, __title__, + __uri__, __version__ +) + +__all__ = [ + "__title__", "__summary__", "__uri__", "__version__", "__author__", + "__email__", "__license__", "__copyright__", +] diff --git a/setuptools/_vendor/packaging/_compat.py b/setuptools/_vendor/packaging/_compat.py new file mode 100644 index 00000000..210bb80b --- /dev/null +++ b/setuptools/_vendor/packaging/_compat.py @@ -0,0 +1,30 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import sys + + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +# flake8: noqa + +if PY3: + string_types = str, +else: + string_types = basestring, + + +def with_metaclass(meta, *bases): + """ + Create a base class with a metaclass. + """ + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) diff --git a/setuptools/_vendor/packaging/_structures.py b/setuptools/_vendor/packaging/_structures.py new file mode 100644 index 00000000..ccc27861 --- /dev/null +++ b/setuptools/_vendor/packaging/_structures.py @@ -0,0 +1,68 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + + +class Infinity(object): + + def __repr__(self): + return "Infinity" + + def __hash__(self): + return hash(repr(self)) + + def __lt__(self, other): + return False + + def __le__(self, other): + return False + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __ne__(self, other): + return not isinstance(other, self.__class__) + + def __gt__(self, other): + return True + + def __ge__(self, other): + return True + + def __neg__(self): + return NegativeInfinity + +Infinity = Infinity() + + +class NegativeInfinity(object): + + def __repr__(self): + return "-Infinity" + + def __hash__(self): + return hash(repr(self)) + + def __lt__(self, other): + return True + + def __le__(self, other): + return True + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __ne__(self, other): + return not isinstance(other, self.__class__) + + def __gt__(self, other): + return False + + def __ge__(self, other): + return False + + def __neg__(self): + return Infinity + +NegativeInfinity = NegativeInfinity() diff --git a/setuptools/_vendor/packaging/markers.py b/setuptools/_vendor/packaging/markers.py new file mode 100644 index 00000000..031332a3 --- /dev/null +++ b/setuptools/_vendor/packaging/markers.py @@ -0,0 +1,301 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import operator +import os +import platform +import sys + +from setuptools.extern.pyparsing import ParseException, ParseResults, stringStart, stringEnd +from setuptools.extern.pyparsing import ZeroOrMore, Group, Forward, QuotedString +from setuptools.extern.pyparsing import Literal as L # noqa + +from ._compat import string_types +from .specifiers import Specifier, InvalidSpecifier + + +__all__ = [ + "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", + "Marker", "default_environment", +] + + +class InvalidMarker(ValueError): + """ + An invalid marker was found, users should refer to PEP 508. + """ + + +class UndefinedComparison(ValueError): + """ + An invalid operation was attempted on a value that doesn't support it. + """ + + +class UndefinedEnvironmentName(ValueError): + """ + A name was attempted to be used that does not exist inside of the + environment. + """ + + +class Node(object): + + def __init__(self, value): + self.value = value + + def __str__(self): + return str(self.value) + + def __repr__(self): + return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) + + def serialize(self): + raise NotImplementedError + + +class Variable(Node): + + def serialize(self): + return str(self) + + +class Value(Node): + + def serialize(self): + return '"{0}"'.format(self) + + +class Op(Node): + + def serialize(self): + return str(self) + + +VARIABLE = ( + L("implementation_version") | + L("platform_python_implementation") | + L("implementation_name") | + L("python_full_version") | + L("platform_release") | + L("platform_version") | + L("platform_machine") | + L("platform_system") | + L("python_version") | + L("sys_platform") | + L("os_name") | + L("os.name") | # PEP-345 + L("sys.platform") | # PEP-345 + L("platform.version") | # PEP-345 + L("platform.machine") | # PEP-345 + L("platform.python_implementation") | # PEP-345 + L("python_implementation") | # undocumented setuptools legacy + L("extra") +) +ALIASES = { + 'os.name': 'os_name', + 'sys.platform': 'sys_platform', + 'platform.version': 'platform_version', + 'platform.machine': 'platform_machine', + 'platform.python_implementation': 'platform_python_implementation', + 'python_implementation': 'platform_python_implementation' +} +VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) + +VERSION_CMP = ( + L("===") | + L("==") | + L(">=") | + L("<=") | + L("!=") | + L("~=") | + L(">") | + L("<") +) + +MARKER_OP = VERSION_CMP | L("not in") | L("in") +MARKER_OP.setParseAction(lambda s, l, t: Op(t[0])) + +MARKER_VALUE = QuotedString("'") | QuotedString('"') +MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) + +BOOLOP = L("and") | L("or") + +MARKER_VAR = VARIABLE | MARKER_VALUE + +MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) + +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() + +MARKER_EXPR = Forward() +MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) +MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) + +MARKER = stringStart + MARKER_EXPR + stringEnd + + +def _coerce_parse_result(results): + if isinstance(results, ParseResults): + return [_coerce_parse_result(i) for i in results] + else: + return results + + +def _format_marker(marker, first=True): + assert isinstance(marker, (list, tuple, string_types)) + + # Sometimes we have a structure like [[...]] which is a single item list + # where the single item is itself it's own list. In that case we want skip + # the rest of this function so that we don't get extraneous () on the + # outside. + if (isinstance(marker, list) and len(marker) == 1 and + isinstance(marker[0], (list, tuple))): + return _format_marker(marker[0]) + + if isinstance(marker, list): + inner = (_format_marker(m, first=False) for m in marker) + if first: + return " ".join(inner) + else: + return "(" + " ".join(inner) + ")" + elif isinstance(marker, tuple): + return " ".join([m.serialize() for m in marker]) + else: + return marker + + +_operators = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _eval_op(lhs, op, rhs): + try: + spec = Specifier("".join([op.serialize(), rhs])) + except InvalidSpecifier: + pass + else: + return spec.contains(lhs) + + oper = _operators.get(op.serialize()) + if oper is None: + raise UndefinedComparison( + "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs) + ) + + return oper(lhs, rhs) + + +_undefined = object() + + +def _get_env(environment, name): + value = environment.get(name, _undefined) + + if value is _undefined: + raise UndefinedEnvironmentName( + "{0!r} does not exist in evaluation environment.".format(name) + ) + + return value + + +def _evaluate_markers(markers, environment): + groups = [[]] + + for marker in markers: + assert isinstance(marker, (list, tuple, string_types)) + + if isinstance(marker, list): + groups[-1].append(_evaluate_markers(marker, environment)) + elif isinstance(marker, tuple): + lhs, op, rhs = marker + + if isinstance(lhs, Variable): + lhs_value = _get_env(environment, lhs.value) + rhs_value = rhs.value + else: + lhs_value = lhs.value + rhs_value = _get_env(environment, rhs.value) + + groups[-1].append(_eval_op(lhs_value, op, rhs_value)) + else: + assert marker in ["and", "or"] + if marker == "or": + groups.append([]) + + return any(all(item) for item in groups) + + +def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + +def default_environment(): + if hasattr(sys, 'implementation'): + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + else: + iver = '0' + implementation_name = '' + + return { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": platform.python_version()[:3], + "sys_platform": sys.platform, + } + + +class Marker(object): + + def __init__(self, marker): + try: + self._markers = _coerce_parse_result(MARKER.parseString(marker)) + except ParseException as e: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e.loc:e.loc + 8]) + raise InvalidMarker(err_str) + + def __str__(self): + return _format_marker(self._markers) + + def __repr__(self): + return "<Marker({0!r})>".format(str(self)) + + def evaluate(self, environment=None): + """Evaluate a marker. + + Return the boolean from evaluating the given marker against the + environment. environment is an optional argument to override all or + part of the determined environment. + + The environment is determined from the current Python process. + """ + current_environment = default_environment() + if environment is not None: + current_environment.update(environment) + + return _evaluate_markers(self._markers, current_environment) diff --git a/setuptools/_vendor/packaging/requirements.py b/setuptools/_vendor/packaging/requirements.py new file mode 100644 index 00000000..5b493416 --- /dev/null +++ b/setuptools/_vendor/packaging/requirements.py @@ -0,0 +1,127 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import string +import re + +from setuptools.extern.pyparsing import stringStart, stringEnd, originalTextFor, ParseException +from setuptools.extern.pyparsing import ZeroOrMore, Word, Optional, Regex, Combine +from setuptools.extern.pyparsing import Literal as L # noqa +from setuptools.extern.six.moves.urllib import parse as urlparse + +from .markers import MARKER_EXPR, Marker +from .specifiers import LegacySpecifier, Specifier, SpecifierSet + + +class InvalidRequirement(ValueError): + """ + An invalid requirement was found, users should refer to PEP 508. + """ + + +ALPHANUM = Word(string.ascii_letters + string.digits) + +LBRACKET = L("[").suppress() +RBRACKET = L("]").suppress() +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() +COMMA = L(",").suppress() +SEMICOLON = L(";").suppress() +AT = L("@").suppress() + +PUNCTUATION = Word("-_.") +IDENTIFIER_END = ALPHANUM | (ZeroOrMore(PUNCTUATION) + ALPHANUM) +IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) + +NAME = IDENTIFIER("name") +EXTRA = IDENTIFIER + +URI = Regex(r'[^ ]+')("url") +URL = (AT + URI) + +EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) +EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") + +VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) +VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) + +VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY +VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), + joinString=",", adjacent=False)("_raw_spec") +_VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') + +VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") +VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) + +MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") +MARKER_EXPR.setParseAction( + lambda s, l, t: Marker(s[t._original_start:t._original_end]) +) +MARKER_SEPERATOR = SEMICOLON +MARKER = MARKER_SEPERATOR + MARKER_EXPR + +VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) +URL_AND_MARKER = URL + Optional(MARKER) + +NAMED_REQUIREMENT = \ + NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) + +REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd + + +class Requirement(object): + """Parse a requirement. + + Parse a given requirement string into its parts, such as name, specifier, + URL, and extras. Raises InvalidRequirement on a badly-formed requirement + string. + """ + + # TODO: Can we test whether something is contained within a requirement? + # If so how do we do that? Do we need to test against the _name_ of + # the thing as well as the version? What about the markers? + # TODO: Can we normalize the name and extra name? + + def __init__(self, requirement_string): + try: + req = REQUIREMENT.parseString(requirement_string) + except ParseException as e: + raise InvalidRequirement( + "Invalid requirement, parse error at \"{0!r}\"".format( + requirement_string[e.loc:e.loc + 8])) + + self.name = req.name + if req.url: + parsed_url = urlparse.urlparse(req.url) + if not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc): + raise InvalidRequirement("Invalid URL given") + self.url = req.url + else: + self.url = None + self.extras = set(req.extras.asList() if req.extras else []) + self.specifier = SpecifierSet(req.specifier) + self.marker = req.marker if req.marker else None + + def __str__(self): + parts = [self.name] + + if self.extras: + parts.append("[{0}]".format(",".join(sorted(self.extras)))) + + if self.specifier: + parts.append(str(self.specifier)) + + if self.url: + parts.append("@ {0}".format(self.url)) + + if self.marker: + parts.append("; {0}".format(self.marker)) + + return "".join(parts) + + def __repr__(self): + return "<Requirement({0!r})>".format(str(self)) diff --git a/setuptools/_vendor/packaging/specifiers.py b/setuptools/_vendor/packaging/specifiers.py new file mode 100644 index 00000000..7f5a76cf --- /dev/null +++ b/setuptools/_vendor/packaging/specifiers.py @@ -0,0 +1,774 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import abc +import functools +import itertools +import re + +from ._compat import string_types, with_metaclass +from .version import Version, LegacyVersion, parse + + +class InvalidSpecifier(ValueError): + """ + An invalid specifier was found, users should refer to PEP 440. + """ + + +class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): + + @abc.abstractmethod + def __str__(self): + """ + Returns the str representation of this Specifier like object. This + should be representative of the Specifier itself. + """ + + @abc.abstractmethod + def __hash__(self): + """ + Returns a hash value for this Specifier like object. + """ + + @abc.abstractmethod + def __eq__(self, other): + """ + Returns a boolean representing whether or not the two Specifier like + objects are equal. + """ + + @abc.abstractmethod + def __ne__(self, other): + """ + Returns a boolean representing whether or not the two Specifier like + objects are not equal. + """ + + @abc.abstractproperty + def prereleases(self): + """ + Returns whether or not pre-releases as a whole are allowed by this + specifier. + """ + + @prereleases.setter + def prereleases(self, value): + """ + Sets whether or not pre-releases as a whole are allowed by this + specifier. + """ + + @abc.abstractmethod + def contains(self, item, prereleases=None): + """ + Determines if the given item is contained within this specifier. + """ + + @abc.abstractmethod + def filter(self, iterable, prereleases=None): + """ + Takes an iterable of items and filters them so that only items which + are contained within this specifier are allowed in it. + """ + + +class _IndividualSpecifier(BaseSpecifier): + + _operators = {} + + def __init__(self, spec="", prereleases=None): + match = self._regex.search(spec) + if not match: + raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) + + self._spec = ( + match.group("operator").strip(), + match.group("version").strip(), + ) + + # Store whether or not this Specifier should accept prereleases + self._prereleases = prereleases + + def __repr__(self): + pre = ( + ", prereleases={0!r}".format(self.prereleases) + if self._prereleases is not None + else "" + ) + + return "<{0}({1!r}{2})>".format( + self.__class__.__name__, + str(self), + pre, + ) + + def __str__(self): + return "{0}{1}".format(*self._spec) + + def __hash__(self): + return hash(self._spec) + + def __eq__(self, other): + if isinstance(other, string_types): + try: + other = self.__class__(other) + except InvalidSpecifier: + return NotImplemented + elif not isinstance(other, self.__class__): + return NotImplemented + + return self._spec == other._spec + + def __ne__(self, other): + if isinstance(other, string_types): + try: + other = self.__class__(other) + except InvalidSpecifier: + return NotImplemented + elif not isinstance(other, self.__class__): + return NotImplemented + + return self._spec != other._spec + + def _get_operator(self, op): + return getattr(self, "_compare_{0}".format(self._operators[op])) + + def _coerce_version(self, version): + if not isinstance(version, (LegacyVersion, Version)): + version = parse(version) + return version + + @property + def operator(self): + return self._spec[0] + + @property + def version(self): + return self._spec[1] + + @property + def prereleases(self): + return self._prereleases + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + def __contains__(self, item): + return self.contains(item) + + def contains(self, item, prereleases=None): + # Determine if prereleases are to be allowed or not. + if prereleases is None: + prereleases = self.prereleases + + # Normalize item to a Version or LegacyVersion, this allows us to have + # a shortcut for ``"2.0" in Specifier(">=2") + item = self._coerce_version(item) + + # Determine if we should be supporting prereleases in this specifier + # or not, if we do not support prereleases than we can short circuit + # logic if this version is a prereleases. + if item.is_prerelease and not prereleases: + return False + + # Actually do the comparison to determine if this item is contained + # within this Specifier or not. + return self._get_operator(self.operator)(item, self.version) + + def filter(self, iterable, prereleases=None): + yielded = False + found_prereleases = [] + + kw = {"prereleases": prereleases if prereleases is not None else True} + + # Attempt to iterate over all the values in the iterable and if any of + # them match, yield them. + for version in iterable: + parsed_version = self._coerce_version(version) + + if self.contains(parsed_version, **kw): + # If our version is a prerelease, and we were not set to allow + # prereleases, then we'll store it for later incase nothing + # else matches this specifier. + if (parsed_version.is_prerelease and not + (prereleases or self.prereleases)): + found_prereleases.append(version) + # Either this is not a prerelease, or we should have been + # accepting prereleases from the begining. + else: + yielded = True + yield version + + # Now that we've iterated over everything, determine if we've yielded + # any values, and if we have not and we have any prereleases stored up + # then we will go ahead and yield the prereleases. + if not yielded and found_prereleases: + for version in found_prereleases: + yield version + + +class LegacySpecifier(_IndividualSpecifier): + + _regex_str = ( + r""" + (?P<operator>(==|!=|<=|>=|<|>)) + \s* + (?P<version> + [^,;\s)]* # Since this is a "legacy" specifier, and the version + # string can be just about anything, we match everything + # except for whitespace, a semi-colon for marker support, + # a closing paren since versions can be enclosed in + # them, and a comma since it's a version separator. + ) + """ + ) + + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + + _operators = { + "==": "equal", + "!=": "not_equal", + "<=": "less_than_equal", + ">=": "greater_than_equal", + "<": "less_than", + ">": "greater_than", + } + + def _coerce_version(self, version): + if not isinstance(version, LegacyVersion): + version = LegacyVersion(str(version)) + return version + + def _compare_equal(self, prospective, spec): + return prospective == self._coerce_version(spec) + + def _compare_not_equal(self, prospective, spec): + return prospective != self._coerce_version(spec) + + def _compare_less_than_equal(self, prospective, spec): + return prospective <= self._coerce_version(spec) + + def _compare_greater_than_equal(self, prospective, spec): + return prospective >= self._coerce_version(spec) + + def _compare_less_than(self, prospective, spec): + return prospective < self._coerce_version(spec) + + def _compare_greater_than(self, prospective, spec): + return prospective > self._coerce_version(spec) + + +def _require_version_compare(fn): + @functools.wraps(fn) + def wrapped(self, prospective, spec): + if not isinstance(prospective, Version): + return False + return fn(self, prospective, spec) + return wrapped + + +class Specifier(_IndividualSpecifier): + + _regex_str = ( + r""" + (?P<operator>(~=|==|!=|<=|>=|<|>|===)) + (?P<version> + (?: + # The identity operators allow for an escape hatch that will + # do an exact string match of the version you wish to install. + # This will not be parsed by PEP 440 and we cannot determine + # any semantic meaning from it. This operator is discouraged + # but included entirely as an escape hatch. + (?<====) # Only match for the identity operator + \s* + [^\s]* # We just match everything, except for whitespace + # since we are only testing for strict identity. + ) + | + (?: + # The (non)equality operators allow for wild card and local + # versions to be specified so we have to define these two + # operators separately to enable that. + (?<===|!=) # Only match for equals and not equals + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)* # release + (?: # pre release + [-_\.]? + (a|b|c|rc|alpha|beta|pre|preview) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + + # You cannot use a wild card and a dev or local version + # together so group them with a | and make them optional. + (?: + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local + | + \.\* # Wild card syntax of .* + )? + ) + | + (?: + # The compatible operator requires at least two digits in the + # release segment. + (?<=~=) # Only match for the compatible operator + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) + (?: # pre release + [-_\.]? + (a|b|c|rc|alpha|beta|pre|preview) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + ) + | + (?: + # All other operators only allow a sub set of what the + # (non)equality operators do. Specifically they do not allow + # local versions to be specified nor do they allow the prefix + # matching wild cards. + (?<!==|!=|~=) # We have special cases for these + # operators so we want to make sure they + # don't match here. + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)* # release + (?: # pre release + [-_\.]? + (a|b|c|rc|alpha|beta|pre|preview) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + ) + ) + """ + ) + + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + + _operators = { + "~=": "compatible", + "==": "equal", + "!=": "not_equal", + "<=": "less_than_equal", + ">=": "greater_than_equal", + "<": "less_than", + ">": "greater_than", + "===": "arbitrary", + } + + @_require_version_compare + def _compare_compatible(self, prospective, spec): + # Compatible releases have an equivalent combination of >= and ==. That + # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to + # implement this in terms of the other specifiers instead of + # implementing it ourselves. The only thing we need to do is construct + # the other specifiers. + + # We want everything but the last item in the version, but we want to + # ignore post and dev releases and we want to treat the pre-release as + # it's own separate segment. + prefix = ".".join( + list( + itertools.takewhile( + lambda x: (not x.startswith("post") and not + x.startswith("dev")), + _version_split(spec), + ) + )[:-1] + ) + + # Add the prefix notation to the end of our string + prefix += ".*" + + return (self._get_operator(">=")(prospective, spec) and + self._get_operator("==")(prospective, prefix)) + + @_require_version_compare + def _compare_equal(self, prospective, spec): + # We need special logic to handle prefix matching + if spec.endswith(".*"): + # In the case of prefix matching we want to ignore local segment. + prospective = Version(prospective.public) + # Split the spec out by dots, and pretend that there is an implicit + # dot in between a release segment and a pre-release segment. + spec = _version_split(spec[:-2]) # Remove the trailing .* + + # Split the prospective version out by dots, and pretend that there + # is an implicit dot in between a release segment and a pre-release + # segment. + prospective = _version_split(str(prospective)) + + # Shorten the prospective version to be the same length as the spec + # so that we can determine if the specifier is a prefix of the + # prospective version or not. + prospective = prospective[:len(spec)] + + # Pad out our two sides with zeros so that they both equal the same + # length. + spec, prospective = _pad_version(spec, prospective) + else: + # Convert our spec string into a Version + spec = Version(spec) + + # If the specifier does not have a local segment, then we want to + # act as if the prospective version also does not have a local + # segment. + if not spec.local: + prospective = Version(prospective.public) + + return prospective == spec + + @_require_version_compare + def _compare_not_equal(self, prospective, spec): + return not self._compare_equal(prospective, spec) + + @_require_version_compare + def _compare_less_than_equal(self, prospective, spec): + return prospective <= Version(spec) + + @_require_version_compare + def _compare_greater_than_equal(self, prospective, spec): + return prospective >= Version(spec) + + @_require_version_compare + def _compare_less_than(self, prospective, spec): + # Convert our spec to a Version instance, since we'll want to work with + # it as a version. + spec = Version(spec) + + # Check to see if the prospective version is less than the spec + # version. If it's not we can short circuit and just return False now + # instead of doing extra unneeded work. + if not prospective < spec: + return False + + # This special case is here so that, unless the specifier itself + # includes is a pre-release version, that we do not accept pre-release + # versions for the version mentioned in the specifier (e.g. <3.1 should + # not match 3.1.dev0, but should match 3.0.dev0). + if not spec.is_prerelease and prospective.is_prerelease: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # If we've gotten to here, it means that prospective version is both + # less than the spec version *and* it's not a pre-release of the same + # version in the spec. + return True + + @_require_version_compare + def _compare_greater_than(self, prospective, spec): + # Convert our spec to a Version instance, since we'll want to work with + # it as a version. + spec = Version(spec) + + # Check to see if the prospective version is greater than the spec + # version. If it's not we can short circuit and just return False now + # instead of doing extra unneeded work. + if not prospective > spec: + return False + + # This special case is here so that, unless the specifier itself + # includes is a post-release version, that we do not accept + # post-release versions for the version mentioned in the specifier + # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). + if not spec.is_postrelease and prospective.is_postrelease: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # Ensure that we do not allow a local version of the version mentioned + # in the specifier, which is techincally greater than, to match. + if prospective.local is not None: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # If we've gotten to here, it means that prospective version is both + # greater than the spec version *and* it's not a pre-release of the + # same version in the spec. + return True + + def _compare_arbitrary(self, prospective, spec): + return str(prospective).lower() == str(spec).lower() + + @property + def prereleases(self): + # If there is an explicit prereleases set for this, then we'll just + # blindly use that. + if self._prereleases is not None: + return self._prereleases + + # Look at all of our specifiers and determine if they are inclusive + # operators, and if they are if they are including an explicit + # prerelease. + operator, version = self._spec + if operator in ["==", ">=", "<=", "~=", "==="]: + # The == specifier can include a trailing .*, if it does we + # want to remove before parsing. + if operator == "==" and version.endswith(".*"): + version = version[:-2] + + # Parse the version, and if it is a pre-release than this + # specifier allows pre-releases. + if parse(version).is_prerelease: + return True + + return False + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + +_prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") + + +def _version_split(version): + result = [] + for item in version.split("."): + match = _prefix_regex.search(item) + if match: + result.extend(match.groups()) + else: + result.append(item) + return result + + +def _pad_version(left, right): + left_split, right_split = [], [] + + # Get the release segment of our versions + left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) + right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) + + # Get the rest of our versions + left_split.append(left[len(left_split[0]):]) + right_split.append(right[len(right_split[0]):]) + + # Insert our padding + left_split.insert( + 1, + ["0"] * max(0, len(right_split[0]) - len(left_split[0])), + ) + right_split.insert( + 1, + ["0"] * max(0, len(left_split[0]) - len(right_split[0])), + ) + + return ( + list(itertools.chain(*left_split)), + list(itertools.chain(*right_split)), + ) + + +class SpecifierSet(BaseSpecifier): + + def __init__(self, specifiers="", prereleases=None): + # Split on , to break each indidivual specifier into it's own item, and + # strip each item to remove leading/trailing whitespace. + specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] + + # Parsed each individual specifier, attempting first to make it a + # Specifier and falling back to a LegacySpecifier. + parsed = set() + for specifier in specifiers: + try: + parsed.add(Specifier(specifier)) + except InvalidSpecifier: + parsed.add(LegacySpecifier(specifier)) + + # Turn our parsed specifiers into a frozen set and save them for later. + self._specs = frozenset(parsed) + + # Store our prereleases value so we can use it later to determine if + # we accept prereleases or not. + self._prereleases = prereleases + + def __repr__(self): + pre = ( + ", prereleases={0!r}".format(self.prereleases) + if self._prereleases is not None + else "" + ) + + return "<SpecifierSet({0!r}{1})>".format(str(self), pre) + + def __str__(self): + return ",".join(sorted(str(s) for s in self._specs)) + + def __hash__(self): + return hash(self._specs) + + def __and__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + specifier = SpecifierSet() + specifier._specs = frozenset(self._specs | other._specs) + + if self._prereleases is None and other._prereleases is not None: + specifier._prereleases = other._prereleases + elif self._prereleases is not None and other._prereleases is None: + specifier._prereleases = self._prereleases + elif self._prereleases == other._prereleases: + specifier._prereleases = self._prereleases + else: + raise ValueError( + "Cannot combine SpecifierSets with True and False prerelease " + "overrides." + ) + + return specifier + + def __eq__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif isinstance(other, _IndividualSpecifier): + other = SpecifierSet(str(other)) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + return self._specs == other._specs + + def __ne__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif isinstance(other, _IndividualSpecifier): + other = SpecifierSet(str(other)) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + return self._specs != other._specs + + def __len__(self): + return len(self._specs) + + def __iter__(self): + return iter(self._specs) + + @property + def prereleases(self): + # If we have been given an explicit prerelease modifier, then we'll + # pass that through here. + if self._prereleases is not None: + return self._prereleases + + # If we don't have any specifiers, and we don't have a forced value, + # then we'll just return None since we don't know if this should have + # pre-releases or not. + if not self._specs: + return None + + # Otherwise we'll see if any of the given specifiers accept + # prereleases, if any of them do we'll return True, otherwise False. + return any(s.prereleases for s in self._specs) + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + def __contains__(self, item): + return self.contains(item) + + def contains(self, item, prereleases=None): + # Ensure that our item is a Version or LegacyVersion instance. + if not isinstance(item, (LegacyVersion, Version)): + item = parse(item) + + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + + # We can determine if we're going to allow pre-releases by looking to + # see if any of the underlying items supports them. If none of them do + # and this item is a pre-release then we do not allow it and we can + # short circuit that here. + # Note: This means that 1.0.dev1 would not be contained in something + # like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0 + if not prereleases and item.is_prerelease: + return False + + # We simply dispatch to the underlying specs here to make sure that the + # given version is contained within all of them. + # Note: This use of all() here means that an empty set of specifiers + # will always return True, this is an explicit design decision. + return all( + s.contains(item, prereleases=prereleases) + for s in self._specs + ) + + def filter(self, iterable, prereleases=None): + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + + # If we have any specifiers, then we want to wrap our iterable in the + # filter method for each one, this will act as a logical AND amongst + # each specifier. + if self._specs: + for spec in self._specs: + iterable = spec.filter(iterable, prereleases=bool(prereleases)) + return iterable + # If we do not have any specifiers, then we need to have a rough filter + # which will filter out any pre-releases, unless there are no final + # releases, and which will filter out LegacyVersion in general. + else: + filtered = [] + found_prereleases = [] + + for item in iterable: + # Ensure that we some kind of Version class for this item. + if not isinstance(item, (LegacyVersion, Version)): + parsed_version = parse(item) + else: + parsed_version = item + + # Filter out any item which is parsed as a LegacyVersion + if isinstance(parsed_version, LegacyVersion): + continue + + # Store any item which is a pre-release for later unless we've + # already found a final version or we are accepting prereleases + if parsed_version.is_prerelease and not prereleases: + if not filtered: + found_prereleases.append(item) + else: + filtered.append(item) + + # If we've found no items except for pre-releases, then we'll go + # ahead and use the pre-releases + if not filtered and found_prereleases and prereleases is None: + return found_prereleases + + return filtered diff --git a/setuptools/_vendor/packaging/utils.py b/setuptools/_vendor/packaging/utils.py new file mode 100644 index 00000000..942387ce --- /dev/null +++ b/setuptools/_vendor/packaging/utils.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import re + + +_canonicalize_regex = re.compile(r"[-_.]+") + + +def canonicalize_name(name): + # This is taken from PEP 503. + return _canonicalize_regex.sub("-", name).lower() diff --git a/setuptools/_vendor/packaging/version.py b/setuptools/_vendor/packaging/version.py new file mode 100644 index 00000000..83b5ee8c --- /dev/null +++ b/setuptools/_vendor/packaging/version.py @@ -0,0 +1,393 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import collections +import itertools +import re + +from ._structures import Infinity + + +__all__ = [ + "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" +] + + +_Version = collections.namedtuple( + "_Version", + ["epoch", "release", "dev", "pre", "post", "local"], +) + + +def parse(version): + """ + Parse the given version string and return either a :class:`Version` object + or a :class:`LegacyVersion` object depending on if the given version is + a valid PEP 440 version or a legacy version. + """ + try: + return Version(version) + except InvalidVersion: + return LegacyVersion(version) + + +class InvalidVersion(ValueError): + """ + An invalid version was found, users should refer to PEP 440. + """ + + +class _BaseVersion(object): + + def __hash__(self): + return hash(self._key) + + def __lt__(self, other): + return self._compare(other, lambda s, o: s < o) + + def __le__(self, other): + return self._compare(other, lambda s, o: s <= o) + + def __eq__(self, other): + return self._compare(other, lambda s, o: s == o) + + def __ge__(self, other): + return self._compare(other, lambda s, o: s >= o) + + def __gt__(self, other): + return self._compare(other, lambda s, o: s > o) + + def __ne__(self, other): + return self._compare(other, lambda s, o: s != o) + + def _compare(self, other, method): + if not isinstance(other, _BaseVersion): + return NotImplemented + + return method(self._key, other._key) + + +class LegacyVersion(_BaseVersion): + + def __init__(self, version): + self._version = str(version) + self._key = _legacy_cmpkey(self._version) + + def __str__(self): + return self._version + + def __repr__(self): + return "<LegacyVersion({0})>".format(repr(str(self))) + + @property + def public(self): + return self._version + + @property + def base_version(self): + return self._version + + @property + def local(self): + return None + + @property + def is_prerelease(self): + return False + + @property + def is_postrelease(self): + return False + + +_legacy_version_component_re = re.compile( + r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, +) + +_legacy_version_replacement_map = { + "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", +} + + +def _parse_version_parts(s): + for part in _legacy_version_component_re.split(s): + part = _legacy_version_replacement_map.get(part, part) + + if not part or part == ".": + continue + + if part[:1] in "0123456789": + # pad for numeric comparison + yield part.zfill(8) + else: + yield "*" + part + + # ensure that alpha/beta/candidate are before final + yield "*final" + + +def _legacy_cmpkey(version): + # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch + # greater than or equal to 0. This will effectively put the LegacyVersion, + # which uses the defacto standard originally implemented by setuptools, + # as before all PEP 440 versions. + epoch = -1 + + # This scheme is taken from pkg_resources.parse_version setuptools prior to + # it's adoption of the packaging library. + parts = [] + for part in _parse_version_parts(version.lower()): + if part.startswith("*"): + # remove "-" before a prerelease tag + if part < "*final": + while parts and parts[-1] == "*final-": + parts.pop() + + # remove trailing zeros from each series of numeric parts + while parts and parts[-1] == "00000000": + parts.pop() + + parts.append(part) + parts = tuple(parts) + + return epoch, parts + +# Deliberately not anchored to the start and end of the string, to make it +# easier for 3rd party code to reuse +VERSION_PATTERN = r""" + v? + (?: + (?:(?P<epoch>[0-9]+)!)? # epoch + (?P<release>[0-9]+(?:\.[0-9]+)*) # release segment + (?P<pre> # pre-release + [-_\.]? + (?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview)) + [-_\.]? + (?P<pre_n>[0-9]+)? + )? + (?P<post> # post release + (?:-(?P<post_n1>[0-9]+)) + | + (?: + [-_\.]? + (?P<post_l>post|rev|r) + [-_\.]? + (?P<post_n2>[0-9]+)? + ) + )? + (?P<dev> # dev release + [-_\.]? + (?P<dev_l>dev) + [-_\.]? + (?P<dev_n>[0-9]+)? + )? + ) + (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version +""" + + +class Version(_BaseVersion): + + _regex = re.compile( + r"^\s*" + VERSION_PATTERN + r"\s*$", + re.VERBOSE | re.IGNORECASE, + ) + + def __init__(self, version): + # Validate the version and parse it into pieces + match = self._regex.search(version) + if not match: + raise InvalidVersion("Invalid version: '{0}'".format(version)) + + # Store the parsed out pieces of the version + self._version = _Version( + epoch=int(match.group("epoch")) if match.group("epoch") else 0, + release=tuple(int(i) for i in match.group("release").split(".")), + pre=_parse_letter_version( + match.group("pre_l"), + match.group("pre_n"), + ), + post=_parse_letter_version( + match.group("post_l"), + match.group("post_n1") or match.group("post_n2"), + ), + dev=_parse_letter_version( + match.group("dev_l"), + match.group("dev_n"), + ), + local=_parse_local_version(match.group("local")), + ) + + # Generate a key which will be used for sorting + self._key = _cmpkey( + self._version.epoch, + self._version.release, + self._version.pre, + self._version.post, + self._version.dev, + self._version.local, + ) + + def __repr__(self): + return "<Version({0})>".format(repr(str(self))) + + def __str__(self): + parts = [] + + # Epoch + if self._version.epoch != 0: + parts.append("{0}!".format(self._version.epoch)) + + # Release segment + parts.append(".".join(str(x) for x in self._version.release)) + + # Pre-release + if self._version.pre is not None: + parts.append("".join(str(x) for x in self._version.pre)) + + # Post-release + if self._version.post is not None: + parts.append(".post{0}".format(self._version.post[1])) + + # Development release + if self._version.dev is not None: + parts.append(".dev{0}".format(self._version.dev[1])) + + # Local version segment + if self._version.local is not None: + parts.append( + "+{0}".format(".".join(str(x) for x in self._version.local)) + ) + + return "".join(parts) + + @property + def public(self): + return str(self).split("+", 1)[0] + + @property + def base_version(self): + parts = [] + + # Epoch + if self._version.epoch != 0: + parts.append("{0}!".format(self._version.epoch)) + + # Release segment + parts.append(".".join(str(x) for x in self._version.release)) + + return "".join(parts) + + @property + def local(self): + version_string = str(self) + if "+" in version_string: + return version_string.split("+", 1)[1] + + @property + def is_prerelease(self): + return bool(self._version.dev or self._version.pre) + + @property + def is_postrelease(self): + return bool(self._version.post) + + +def _parse_letter_version(letter, number): + if letter: + # We consider there to be an implicit 0 in a pre-release if there is + # not a numeral associated with it. + if number is None: + number = 0 + + # We normalize any letters to their lower case form + letter = letter.lower() + + # We consider some words to be alternate spellings of other words and + # in those cases we want to normalize the spellings to our preferred + # spelling. + if letter == "alpha": + letter = "a" + elif letter == "beta": + letter = "b" + elif letter in ["c", "pre", "preview"]: + letter = "rc" + elif letter in ["rev", "r"]: + letter = "post" + + return letter, int(number) + if not letter and number: + # We assume if we are given a number, but we are not given a letter + # then this is using the implicit post release syntax (e.g. 1.0-1) + letter = "post" + + return letter, int(number) + + +_local_version_seperators = re.compile(r"[\._-]") + + +def _parse_local_version(local): + """ + Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve"). + """ + if local is not None: + return tuple( + part.lower() if not part.isdigit() else int(part) + for part in _local_version_seperators.split(local) + ) + + +def _cmpkey(epoch, release, pre, post, dev, local): + # When we compare a release version, we want to compare it with all of the + # trailing zeros removed. So we'll use a reverse the list, drop all the now + # leading zeros until we come to something non zero, then take the rest + # re-reverse it back into the correct order and make it a tuple and use + # that for our sorting key. + release = tuple( + reversed(list( + itertools.dropwhile( + lambda x: x == 0, + reversed(release), + ) + )) + ) + + # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. + # We'll do this by abusing the pre segment, but we _only_ want to do this + # if there is not a pre or a post segment. If we have one of those then + # the normal sorting rules will handle this case correctly. + if pre is None and post is None and dev is not None: + pre = -Infinity + # Versions without a pre-release (except as noted above) should sort after + # those with one. + elif pre is None: + pre = Infinity + + # Versions without a post segment should sort before those with one. + if post is None: + post = -Infinity + + # Versions without a development segment should sort after those with one. + if dev is None: + dev = Infinity + + if local is None: + # Versions without a local segment should sort before those with one. + local = -Infinity + else: + # Versions with a local segment need that segment parsed to implement + # the sorting rules in PEP440. + # - Alpha numeric segments sort before numeric segments + # - Alpha numeric segments sort lexicographically + # - Numeric segments sort numerically + # - Shorter versions sort before longer versions when the prefixes + # match exactly + local = tuple( + (i, "") if isinstance(i, int) else (-Infinity, i) + for i in local + ) + + return epoch, release, pre, post, dev, local diff --git a/setuptools/_vendor/pyparsing.py b/setuptools/_vendor/pyparsing.py new file mode 100644 index 00000000..cf75e1e5 --- /dev/null +++ b/setuptools/_vendor/pyparsing.py @@ -0,0 +1,5742 @@ +# module pyparsing.py
+#
+# Copyright (c) 2003-2018 Paul T. McGuire
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__doc__ = \
+"""
+pyparsing module - Classes and methods to define and execute parsing grammars
+=============================================================================
+
+The pyparsing module is an alternative approach to creating and executing simple grammars,
+vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you
+don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
+provides a library of classes that you use to construct the grammar directly in Python.
+
+Here is a program to parse "Hello, World!" (or any greeting of the form
+C{"<salutation>, <addressee>!"}), built up using L{Word}, L{Literal}, and L{And} elements
+(L{'+'<ParserElement.__add__>} operator gives L{And} expressions, strings are auto-converted to
+L{Literal} expressions)::
+
+ from pyparsing import Word, alphas
+
+ # define grammar of a greeting
+ greet = Word(alphas) + "," + Word(alphas) + "!"
+
+ hello = "Hello, World!"
+ print (hello, "->", greet.parseString(hello))
+
+The program outputs the following::
+
+ Hello, World! -> ['Hello', ',', 'World', '!']
+
+The Python representation of the grammar is quite readable, owing to the self-explanatory
+class names, and the use of '+', '|' and '^' operators.
+
+The L{ParseResults} object returned from L{ParserElement.parseString<ParserElement.parseString>} can be accessed as a nested list, a dictionary, or an
+object with named attributes.
+
+The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
+ - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.)
+ - quoted strings
+ - embedded comments
+
+
+Getting Started -
+-----------------
+Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing
+classes inherit from. Use the docstrings for examples of how to:
+ - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes
+ - construct character word-group expressions using the L{Word} class
+ - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes
+ - use L{'+'<And>}, L{'|'<MatchFirst>}, L{'^'<Or>}, and L{'&'<Each>} operators to combine simple expressions into more complex ones
+ - associate names with your parsed results using L{ParserElement.setResultsName}
+ - find some helpful expression short-cuts like L{delimitedList} and L{oneOf}
+ - find more useful common expressions in the L{pyparsing_common} namespace class
+"""
+
+__version__ = "2.2.1"
+__versionTime__ = "18 Sep 2018 00:49 UTC"
+__author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"
+
+import string
+from weakref import ref as wkref
+import copy
+import sys
+import warnings
+import re
+import sre_constants
+import collections
+import pprint
+import traceback
+import types
+from datetime import datetime
+
+try:
+ from _thread import RLock
+except ImportError:
+ from threading import RLock
+
+try:
+ # Python 3
+ from collections.abc import Iterable
+ from collections.abc import MutableMapping
+except ImportError:
+ # Python 2.7
+ from collections import Iterable
+ from collections import MutableMapping
+
+try:
+ from collections import OrderedDict as _OrderedDict
+except ImportError:
+ try:
+ from ordereddict import OrderedDict as _OrderedDict
+ except ImportError:
+ _OrderedDict = None
+
+#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
+
+__all__ = [
+'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty',
+'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal',
+'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
+'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
+'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
+'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter',
+'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
+'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
+'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
+'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums',
+'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno',
+'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
+'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
+'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity',
+'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
+'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
+'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass',
+'CloseMatch', 'tokenMap', 'pyparsing_common',
+]
+
+system_version = tuple(sys.version_info)[:3]
+PY_3 = system_version[0] == 3
+if PY_3:
+ _MAX_INT = sys.maxsize
+ basestring = str
+ unichr = chr
+ _ustr = str
+
+ # build list of single arg builtins, that can be used as parse actions
+ singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max]
+
+else:
+ _MAX_INT = sys.maxint
+ range = xrange
+
+ def _ustr(obj):
+ """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
+ str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
+ then < returns the unicode object | encodes it with the default encoding | ... >.
+ """
+ if isinstance(obj,unicode):
+ return obj
+
+ try:
+ # If this works, then _ustr(obj) has the same behaviour as str(obj), so
+ # it won't break any existing code.
+ return str(obj)
+
+ except UnicodeEncodeError:
+ # Else encode it
+ ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
+ xmlcharref = Regex(r'&#\d+;')
+ xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
+ return xmlcharref.transformString(ret)
+
+ # build list of single arg builtins, tolerant of Python version, that can be used as parse actions
+ singleArgBuiltins = []
+ import __builtin__
+ for fname in "sum len sorted reversed list tuple set any all min max".split():
+ try:
+ singleArgBuiltins.append(getattr(__builtin__,fname))
+ except AttributeError:
+ continue
+
+_generatorType = type((y for y in range(1)))
+
+def _xml_escape(data):
+ """Escape &, <, >, ", ', etc. in a string of data."""
+
+ # ampersand must be replaced first
+ from_symbols = '&><"\''
+ to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split())
+ for from_,to_ in zip(from_symbols, to_symbols):
+ data = data.replace(from_, to_)
+ return data
+
+class _Constants(object):
+ pass
+
+alphas = string.ascii_uppercase + string.ascii_lowercase
+nums = "0123456789"
+hexnums = nums + "ABCDEFabcdef"
+alphanums = alphas + nums
+_bslash = chr(92)
+printables = "".join(c for c in string.printable if c not in string.whitespace)
+
+class ParseBaseException(Exception):
+ """base exception class for all parsing runtime exceptions"""
+ # Performance tuning: we construct a *lot* of these, so keep this
+ # constructor as small and fast as possible
+ def __init__( self, pstr, loc=0, msg=None, elem=None ):
+ self.loc = loc
+ if msg is None:
+ self.msg = pstr
+ self.pstr = ""
+ else:
+ self.msg = msg
+ self.pstr = pstr
+ self.parserElement = elem
+ self.args = (pstr, loc, msg)
+
+ @classmethod
+ def _from_exception(cls, pe):
+ """
+ internal factory method to simplify creating one type of ParseException
+ from another - avoids having __init__ signature conflicts among subclasses
+ """
+ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement)
+
+ def __getattr__( self, aname ):
+ """supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+ """
+ if( aname == "lineno" ):
+ return lineno( self.loc, self.pstr )
+ elif( aname in ("col", "column") ):
+ return col( self.loc, self.pstr )
+ elif( aname == "line" ):
+ return line( self.loc, self.pstr )
+ else:
+ raise AttributeError(aname)
+
+ def __str__( self ):
+ return "%s (at char %d), (line:%d, col:%d)" % \
+ ( self.msg, self.loc, self.lineno, self.column )
+ def __repr__( self ):
+ return _ustr(self)
+ def markInputline( self, markerString = ">!<" ):
+ """Extracts the exception line from the input string, and marks
+ the location of the exception with a special symbol.
+ """
+ line_str = self.line
+ line_column = self.column - 1
+ if markerString:
+ line_str = "".join((line_str[:line_column],
+ markerString, line_str[line_column:]))
+ return line_str.strip()
+ def __dir__(self):
+ return "lineno col line".split() + dir(type(self))
+
+class ParseException(ParseBaseException):
+ """
+ Exception thrown when parse expressions don't match class;
+ supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+
+ Example::
+ try:
+ Word(nums).setName("integer").parseString("ABC")
+ except ParseException as pe:
+ print(pe)
+ print("column: {}".format(pe.col))
+
+ prints::
+ Expected integer (at char 0), (line:1, col:1)
+ column: 1
+ """
+ pass
+
+class ParseFatalException(ParseBaseException):
+ """user-throwable exception thrown when inconsistent parse content
+ is found; stops all parsing immediately"""
+ pass
+
+class ParseSyntaxException(ParseFatalException):
+ """just like L{ParseFatalException}, but thrown internally when an
+ L{ErrorStop<And._ErrorStop>} ('-' operator) indicates that parsing is to stop
+ immediately because an unbacktrackable syntax error has been found"""
+ pass
+
+#~ class ReparseException(ParseBaseException):
+ #~ """Experimental class - parse actions can raise this exception to cause
+ #~ pyparsing to reparse the input string:
+ #~ - with a modified input string, and/or
+ #~ - with a modified start location
+ #~ Set the values of the ReparseException in the constructor, and raise the
+ #~ exception in a parse action to cause pyparsing to use the new string/location.
+ #~ Setting the values as None causes no change to be made.
+ #~ """
+ #~ def __init_( self, newstring, restartLoc ):
+ #~ self.newParseText = newstring
+ #~ self.reparseLoc = restartLoc
+
+class RecursiveGrammarException(Exception):
+ """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive"""
+ def __init__( self, parseElementList ):
+ self.parseElementTrace = parseElementList
+
+ def __str__( self ):
+ return "RecursiveGrammarException: %s" % self.parseElementTrace
+
+class _ParseResultsWithOffset(object):
+ def __init__(self,p1,p2):
+ self.tup = (p1,p2)
+ def __getitem__(self,i):
+ return self.tup[i]
+ def __repr__(self):
+ return repr(self.tup[0])
+ def setOffset(self,i):
+ self.tup = (self.tup[0],i)
+
+class ParseResults(object):
+ """
+ Structured parse results, to provide multiple means of access to the parsed data:
+ - as a list (C{len(results)})
+ - by list index (C{results[0], results[1]}, etc.)
+ - by attribute (C{results.<resultsName>} - see L{ParserElement.setResultsName})
+
+ Example::
+ integer = Word(nums)
+ date_str = (integer.setResultsName("year") + '/'
+ + integer.setResultsName("month") + '/'
+ + integer.setResultsName("day"))
+ # equivalent form:
+ # date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ # parseString returns a ParseResults object
+ result = date_str.parseString("1999/12/31")
+
+ def test(s, fn=repr):
+ print("%s -> %s" % (s, fn(eval(s))))
+ test("list(result)")
+ test("result[0]")
+ test("result['month']")
+ test("result.day")
+ test("'month' in result")
+ test("'minutes' in result")
+ test("result.dump()", str)
+ prints::
+ list(result) -> ['1999', '/', '12', '/', '31']
+ result[0] -> '1999'
+ result['month'] -> '12'
+ result.day -> '31'
+ 'month' in result -> True
+ 'minutes' in result -> False
+ result.dump() -> ['1999', '/', '12', '/', '31']
+ - day: 31
+ - month: 12
+ - year: 1999
+ """
+ def __new__(cls, toklist=None, name=None, asList=True, modal=True ):
+ if isinstance(toklist, cls):
+ return toklist
+ retobj = object.__new__(cls)
+ retobj.__doinit = True
+ return retobj
+
+ # Performance tuning: we construct a *lot* of these, so keep this
+ # constructor as small and fast as possible
+ def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance ):
+ if self.__doinit:
+ self.__doinit = False
+ self.__name = None
+ self.__parent = None
+ self.__accumNames = {}
+ self.__asList = asList
+ self.__modal = modal
+ if toklist is None:
+ toklist = []
+ if isinstance(toklist, list):
+ self.__toklist = toklist[:]
+ elif isinstance(toklist, _generatorType):
+ self.__toklist = list(toklist)
+ else:
+ self.__toklist = [toklist]
+ self.__tokdict = dict()
+
+ if name is not None and name:
+ if not modal:
+ self.__accumNames[name] = 0
+ if isinstance(name,int):
+ name = _ustr(name) # will always return a str, but use _ustr for consistency
+ self.__name = name
+ if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])):
+ if isinstance(toklist,basestring):
+ toklist = [ toklist ]
+ if asList:
+ if isinstance(toklist,ParseResults):
+ self[name] = _ParseResultsWithOffset(toklist.copy(),0)
+ else:
+ self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0)
+ self[name].__name = name
+ else:
+ try:
+ self[name] = toklist[0]
+ except (KeyError,TypeError,IndexError):
+ self[name] = toklist
+
+ def __getitem__( self, i ):
+ if isinstance( i, (int,slice) ):
+ return self.__toklist[i]
+ else:
+ if i not in self.__accumNames:
+ return self.__tokdict[i][-1][0]
+ else:
+ return ParseResults([ v[0] for v in self.__tokdict[i] ])
+
+ def __setitem__( self, k, v, isinstance=isinstance ):
+ if isinstance(v,_ParseResultsWithOffset):
+ self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
+ sub = v[0]
+ elif isinstance(k,(int,slice)):
+ self.__toklist[k] = v
+ sub = v
+ else:
+ self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
+ sub = v
+ if isinstance(sub,ParseResults):
+ sub.__parent = wkref(self)
+
+ def __delitem__( self, i ):
+ if isinstance(i,(int,slice)):
+ mylen = len( self.__toklist )
+ del self.__toklist[i]
+
+ # convert int to slice
+ if isinstance(i, int):
+ if i < 0:
+ i += mylen
+ i = slice(i, i+1)
+ # get removed indices
+ removed = list(range(*i.indices(mylen)))
+ removed.reverse()
+ # fixup indices in token dictionary
+ for name,occurrences in self.__tokdict.items():
+ for j in removed:
+ for k, (value, position) in enumerate(occurrences):
+ occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
+ else:
+ del self.__tokdict[i]
+
+ def __contains__( self, k ):
+ return k in self.__tokdict
+
+ def __len__( self ): return len( self.__toklist )
+ def __bool__(self): return ( not not self.__toklist )
+ __nonzero__ = __bool__
+ def __iter__( self ): return iter( self.__toklist )
+ def __reversed__( self ): return iter( self.__toklist[::-1] )
+ def _iterkeys( self ):
+ if hasattr(self.__tokdict, "iterkeys"):
+ return self.__tokdict.iterkeys()
+ else:
+ return iter(self.__tokdict)
+
+ def _itervalues( self ):
+ return (self[k] for k in self._iterkeys())
+
+ def _iteritems( self ):
+ return ((k, self[k]) for k in self._iterkeys())
+
+ if PY_3:
+ keys = _iterkeys
+ """Returns an iterator of all named result keys (Python 3.x only)."""
+
+ values = _itervalues
+ """Returns an iterator of all named result values (Python 3.x only)."""
+
+ items = _iteritems
+ """Returns an iterator of all named result key-value tuples (Python 3.x only)."""
+
+ else:
+ iterkeys = _iterkeys
+ """Returns an iterator of all named result keys (Python 2.x only)."""
+
+ itervalues = _itervalues
+ """Returns an iterator of all named result values (Python 2.x only)."""
+
+ iteritems = _iteritems
+ """Returns an iterator of all named result key-value tuples (Python 2.x only)."""
+
+ def keys( self ):
+ """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x)."""
+ return list(self.iterkeys())
+
+ def values( self ):
+ """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x)."""
+ return list(self.itervalues())
+
+ def items( self ):
+ """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x)."""
+ return list(self.iteritems())
+
+ def haskeys( self ):
+ """Since keys() returns an iterator, this method is helpful in bypassing
+ code that looks for the existence of any defined results names."""
+ return bool(self.__tokdict)
+
+ def pop( self, *args, **kwargs):
+ """
+ Removes and returns item at specified index (default=C{last}).
+ Supports both C{list} and C{dict} semantics for C{pop()}. If passed no
+ argument or an integer argument, it will use C{list} semantics
+ and pop tokens from the list of parsed tokens. If passed a
+ non-integer argument (most likely a string), it will use C{dict}
+ semantics and pop the corresponding value from any defined
+ results names. A second default return value argument is
+ supported, just as in C{dict.pop()}.
+
+ Example::
+ def remove_first(tokens):
+ tokens.pop(0)
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+ print(OneOrMore(Word(nums)).addParseAction(remove_first).parseString("0 123 321")) # -> ['123', '321']
+
+ label = Word(alphas)
+ patt = label("LABEL") + OneOrMore(Word(nums))
+ print(patt.parseString("AAB 123 321").dump())
+
+ # Use pop() in a parse action to remove named result (note that corresponding value is not
+ # removed from list form of results)
+ def remove_LABEL(tokens):
+ tokens.pop("LABEL")
+ return tokens
+ patt.addParseAction(remove_LABEL)
+ print(patt.parseString("AAB 123 321").dump())
+ prints::
+ ['AAB', '123', '321']
+ - LABEL: AAB
+
+ ['AAB', '123', '321']
+ """
+ if not args:
+ args = [-1]
+ for k,v in kwargs.items():
+ if k == 'default':
+ args = (args[0], v)
+ else:
+ raise TypeError("pop() got an unexpected keyword argument '%s'" % k)
+ if (isinstance(args[0], int) or
+ len(args) == 1 or
+ args[0] in self):
+ index = args[0]
+ ret = self[index]
+ del self[index]
+ return ret
+ else:
+ defaultvalue = args[1]
+ return defaultvalue
+
+ def get(self, key, defaultValue=None):
+ """
+ Returns named result matching the given key, or if there is no
+ such name, then returns the given C{defaultValue} or C{None} if no
+ C{defaultValue} is specified.
+
+ Similar to C{dict.get()}.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString("1999/12/31")
+ print(result.get("year")) # -> '1999'
+ print(result.get("hour", "not specified")) # -> 'not specified'
+ print(result.get("hour")) # -> None
+ """
+ if key in self:
+ return self[key]
+ else:
+ return defaultValue
+
+ def insert( self, index, insStr ):
+ """
+ Inserts new element at location index in the list of parsed tokens.
+
+ Similar to C{list.insert()}.
+
+ Example::
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+
+ # use a parse action to insert the parse location in the front of the parsed results
+ def insert_locn(locn, tokens):
+ tokens.insert(0, locn)
+ print(OneOrMore(Word(nums)).addParseAction(insert_locn).parseString("0 123 321")) # -> [0, '0', '123', '321']
+ """
+ self.__toklist.insert(index, insStr)
+ # fixup indices in token dictionary
+ for name,occurrences in self.__tokdict.items():
+ for k, (value, position) in enumerate(occurrences):
+ occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
+
+ def append( self, item ):
+ """
+ Add single element to end of ParseResults list of elements.
+
+ Example::
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+
+ # use a parse action to compute the sum of the parsed integers, and add it to the end
+ def append_sum(tokens):
+ tokens.append(sum(map(int, tokens)))
+ print(OneOrMore(Word(nums)).addParseAction(append_sum).parseString("0 123 321")) # -> ['0', '123', '321', 444]
+ """
+ self.__toklist.append(item)
+
+ def extend( self, itemseq ):
+ """
+ Add sequence of elements to end of ParseResults list of elements.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+
+ # use a parse action to append the reverse of the matched strings, to make a palindrome
+ def make_palindrome(tokens):
+ tokens.extend(reversed([t[::-1] for t in tokens]))
+ return ''.join(tokens)
+ print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl'
+ """
+ if isinstance(itemseq, ParseResults):
+ self += itemseq
+ else:
+ self.__toklist.extend(itemseq)
+
+ def clear( self ):
+ """
+ Clear all elements and results names.
+ """
+ del self.__toklist[:]
+ self.__tokdict.clear()
+
+ def __getattr__( self, name ):
+ try:
+ return self[name]
+ except KeyError:
+ return ""
+
+ if name in self.__tokdict:
+ if name not in self.__accumNames:
+ return self.__tokdict[name][-1][0]
+ else:
+ return ParseResults([ v[0] for v in self.__tokdict[name] ])
+ else:
+ return ""
+
+ def __add__( self, other ):
+ ret = self.copy()
+ ret += other
+ return ret
+
+ def __iadd__( self, other ):
+ if other.__tokdict:
+ offset = len(self.__toklist)
+ addoffset = lambda a: offset if a<0 else a+offset
+ otheritems = other.__tokdict.items()
+ otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
+ for (k,vlist) in otheritems for v in vlist]
+ for k,v in otherdictitems:
+ self[k] = v
+ if isinstance(v[0],ParseResults):
+ v[0].__parent = wkref(self)
+
+ self.__toklist += other.__toklist
+ self.__accumNames.update( other.__accumNames )
+ return self
+
+ def __radd__(self, other):
+ if isinstance(other,int) and other == 0:
+ # useful for merging many ParseResults using sum() builtin
+ return self.copy()
+ else:
+ # this may raise a TypeError - so be it
+ return other + self
+
+ def __repr__( self ):
+ return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
+
+ def __str__( self ):
+ return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']'
+
+ def _asStringList( self, sep='' ):
+ out = []
+ for item in self.__toklist:
+ if out and sep:
+ out.append(sep)
+ if isinstance( item, ParseResults ):
+ out += item._asStringList()
+ else:
+ out.append( _ustr(item) )
+ return out
+
+ def asList( self ):
+ """
+ Returns the parse results as a nested list of matching tokens, all converted to strings.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+ result = patt.parseString("sldkj lsdkj sldkj")
+ # even though the result prints in string-like form, it is actually a pyparsing ParseResults
+ print(type(result), result) # -> <class 'pyparsing.ParseResults'> ['sldkj', 'lsdkj', 'sldkj']
+
+ # Use asList() to create an actual list
+ result_list = result.asList()
+ print(type(result_list), result_list) # -> <class 'list'> ['sldkj', 'lsdkj', 'sldkj']
+ """
+ return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist]
+
+ def asDict( self ):
+ """
+ Returns the named parse results as a nested dictionary.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString('12/31/1999')
+ print(type(result), repr(result)) # -> <class 'pyparsing.ParseResults'> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]})
+
+ result_dict = result.asDict()
+ print(type(result_dict), repr(result_dict)) # -> <class 'dict'> {'day': '1999', 'year': '12', 'month': '31'}
+
+ # even though a ParseResults supports dict-like access, sometime you just need to have a dict
+ import json
+ print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable
+ print(json.dumps(result.asDict())) # -> {"month": "31", "day": "1999", "year": "12"}
+ """
+ if PY_3:
+ item_fn = self.items
+ else:
+ item_fn = self.iteritems
+
+ def toItem(obj):
+ if isinstance(obj, ParseResults):
+ if obj.haskeys():
+ return obj.asDict()
+ else:
+ return [toItem(v) for v in obj]
+ else:
+ return obj
+
+ return dict((k,toItem(v)) for k,v in item_fn())
+
+ def copy( self ):
+ """
+ Returns a new copy of a C{ParseResults} object.
+ """
+ ret = ParseResults( self.__toklist )
+ ret.__tokdict = self.__tokdict.copy()
+ ret.__parent = self.__parent
+ ret.__accumNames.update( self.__accumNames )
+ ret.__name = self.__name
+ return ret
+
+ def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
+ """
+ (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names.
+ """
+ nl = "\n"
+ out = []
+ namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()
+ for v in vlist)
+ nextLevelIndent = indent + " "
+
+ # collapse out indents if formatting is not desired
+ if not formatted:
+ indent = ""
+ nextLevelIndent = ""
+ nl = ""
+
+ selfTag = None
+ if doctag is not None:
+ selfTag = doctag
+ else:
+ if self.__name:
+ selfTag = self.__name
+
+ if not selfTag:
+ if namedItemsOnly:
+ return ""
+ else:
+ selfTag = "ITEM"
+
+ out += [ nl, indent, "<", selfTag, ">" ]
+
+ for i,res in enumerate(self.__toklist):
+ if isinstance(res,ParseResults):
+ if i in namedItems:
+ out += [ res.asXML(namedItems[i],
+ namedItemsOnly and doctag is None,
+ nextLevelIndent,
+ formatted)]
+ else:
+ out += [ res.asXML(None,
+ namedItemsOnly and doctag is None,
+ nextLevelIndent,
+ formatted)]
+ else:
+ # individual token, see if there is a name for it
+ resTag = None
+ if i in namedItems:
+ resTag = namedItems[i]
+ if not resTag:
+ if namedItemsOnly:
+ continue
+ else:
+ resTag = "ITEM"
+ xmlBodyText = _xml_escape(_ustr(res))
+ out += [ nl, nextLevelIndent, "<", resTag, ">",
+ xmlBodyText,
+ "</", resTag, ">" ]
+
+ out += [ nl, indent, "</", selfTag, ">" ]
+ return "".join(out)
+
+ def __lookup(self,sub):
+ for k,vlist in self.__tokdict.items():
+ for v,loc in vlist:
+ if sub is v:
+ return k
+ return None
+
+ def getName(self):
+ r"""
+ Returns the results name for this token expression. Useful when several
+ different expressions might match at a particular location.
+
+ Example::
+ integer = Word(nums)
+ ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d")
+ house_number_expr = Suppress('#') + Word(nums, alphanums)
+ user_data = (Group(house_number_expr)("house_number")
+ | Group(ssn_expr)("ssn")
+ | Group(integer)("age"))
+ user_info = OneOrMore(user_data)
+
+ result = user_info.parseString("22 111-22-3333 #221B")
+ for item in result:
+ print(item.getName(), ':', item[0])
+ prints::
+ age : 22
+ ssn : 111-22-3333
+ house_number : 221B
+ """
+ if self.__name:
+ return self.__name
+ elif self.__parent:
+ par = self.__parent()
+ if par:
+ return par.__lookup(self)
+ else:
+ return None
+ elif (len(self) == 1 and
+ len(self.__tokdict) == 1 and
+ next(iter(self.__tokdict.values()))[0][1] in (0,-1)):
+ return next(iter(self.__tokdict.keys()))
+ else:
+ return None
+
+ def dump(self, indent='', depth=0, full=True):
+ """
+ Diagnostic method for listing out the contents of a C{ParseResults}.
+ Accepts an optional C{indent} argument so that this string can be embedded
+ in a nested display of other data.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString('12/31/1999')
+ print(result.dump())
+ prints::
+ ['12', '/', '31', '/', '1999']
+ - day: 1999
+ - month: 31
+ - year: 12
+ """
+ out = []
+ NL = '\n'
+ out.append( indent+_ustr(self.asList()) )
+ if full:
+ if self.haskeys():
+ items = sorted((str(k), v) for k,v in self.items())
+ for k,v in items:
+ if out:
+ out.append(NL)
+ out.append( "%s%s- %s: " % (indent,(' '*depth), k) )
+ if isinstance(v,ParseResults):
+ if v:
+ out.append( v.dump(indent,depth+1) )
+ else:
+ out.append(_ustr(v))
+ else:
+ out.append(repr(v))
+ elif any(isinstance(vv,ParseResults) for vv in self):
+ v = self
+ for i,vv in enumerate(v):
+ if isinstance(vv,ParseResults):
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) ))
+ else:
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv)))
+
+ return "".join(out)
+
+ def pprint(self, *args, **kwargs):
+ """
+ Pretty-printer for parsed results as a list, using the C{pprint} module.
+ Accepts additional positional or keyword args as defined for the
+ C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})
+
+ Example::
+ ident = Word(alphas, alphanums)
+ num = Word(nums)
+ func = Forward()
+ term = ident | num | Group('(' + func + ')')
+ func <<= ident + Group(Optional(delimitedList(term)))
+ result = func.parseString("fna a,b,(fnb c,d,200),100")
+ result.pprint(width=40)
+ prints::
+ ['fna',
+ ['a',
+ 'b',
+ ['(', 'fnb', ['c', 'd', '200'], ')'],
+ '100']]
+ """
+ pprint.pprint(self.asList(), *args, **kwargs)
+
+ # add support for pickle protocol
+ def __getstate__(self):
+ return ( self.__toklist,
+ ( self.__tokdict.copy(),
+ self.__parent is not None and self.__parent() or None,
+ self.__accumNames,
+ self.__name ) )
+
+ def __setstate__(self,state):
+ self.__toklist = state[0]
+ (self.__tokdict,
+ par,
+ inAccumNames,
+ self.__name) = state[1]
+ self.__accumNames = {}
+ self.__accumNames.update(inAccumNames)
+ if par is not None:
+ self.__parent = wkref(par)
+ else:
+ self.__parent = None
+
+ def __getnewargs__(self):
+ return self.__toklist, self.__name, self.__asList, self.__modal
+
+ def __dir__(self):
+ return (dir(type(self)) + list(self.keys()))
+
+MutableMapping.register(ParseResults)
+
+def col (loc,strg):
+ """Returns current column within a string, counting newlines as line separators.
+ The first column is number 1.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ s = strg
+ return 1 if 0<loc<len(s) and s[loc-1] == '\n' else loc - s.rfind("\n", 0, loc)
+
+def lineno(loc,strg):
+ """Returns current line number within a string, counting newlines as line separators.
+ The first line is number 1.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ return strg.count("\n",0,loc) + 1
+
+def line( loc, strg ):
+ """Returns the line of text containing loc within a string, counting newlines as line separators.
+ """
+ lastCR = strg.rfind("\n", 0, loc)
+ nextCR = strg.find("\n", loc)
+ if nextCR >= 0:
+ return strg[lastCR+1:nextCR]
+ else:
+ return strg[lastCR+1:]
+
+def _defaultStartDebugAction( instring, loc, expr ):
+ print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )))
+
+def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
+ print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
+
+def _defaultExceptionDebugAction( instring, loc, expr, exc ):
+ print ("Exception raised:" + _ustr(exc))
+
+def nullDebugAction(*args):
+ """'Do-nothing' debug action, to suppress debugging output during parsing."""
+ pass
+
+# Only works on Python 3.x - nonlocal is toxic to Python 2 installs
+#~ 'decorator to trim function calls to match the arity of the target'
+#~ def _trim_arity(func, maxargs=3):
+ #~ if func in singleArgBuiltins:
+ #~ return lambda s,l,t: func(t)
+ #~ limit = 0
+ #~ foundArity = False
+ #~ def wrapper(*args):
+ #~ nonlocal limit,foundArity
+ #~ while 1:
+ #~ try:
+ #~ ret = func(*args[limit:])
+ #~ foundArity = True
+ #~ return ret
+ #~ except TypeError:
+ #~ if limit == maxargs or foundArity:
+ #~ raise
+ #~ limit += 1
+ #~ continue
+ #~ return wrapper
+
+# this version is Python 2.x-3.x cross-compatible
+'decorator to trim function calls to match the arity of the target'
+def _trim_arity(func, maxargs=2):
+ if func in singleArgBuiltins:
+ return lambda s,l,t: func(t)
+ limit = [0]
+ foundArity = [False]
+
+ # traceback return data structure changed in Py3.5 - normalize back to plain tuples
+ if system_version[:2] >= (3,5):
+ def extract_stack(limit=0):
+ # special handling for Python 3.5.0 - extra deep call stack by 1
+ offset = -3 if system_version == (3,5,0) else -2
+ frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
+ return [frame_summary[:2]]
+ def extract_tb(tb, limit=0):
+ frames = traceback.extract_tb(tb, limit=limit)
+ frame_summary = frames[-1]
+ return [frame_summary[:2]]
+ else:
+ extract_stack = traceback.extract_stack
+ extract_tb = traceback.extract_tb
+
+ # synthesize what would be returned by traceback.extract_stack at the call to
+ # user's parse action 'func', so that we don't incur call penalty at parse time
+
+ LINE_DIFF = 6
+ # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND
+ # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
+ this_line = extract_stack(limit=2)[-1]
+ pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF)
+
+ def wrapper(*args):
+ while 1:
+ try:
+ ret = func(*args[limit[0]:])
+ foundArity[0] = True
+ return ret
+ except TypeError:
+ # re-raise TypeErrors if they did not come from our arity testing
+ if foundArity[0]:
+ raise
+ else:
+ try:
+ tb = sys.exc_info()[-1]
+ if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth:
+ raise
+ finally:
+ del tb
+
+ if limit[0] <= maxargs:
+ limit[0] += 1
+ continue
+ raise
+
+ # copy func name to wrapper for sensible debug output
+ func_name = "<parse action>"
+ try:
+ func_name = getattr(func, '__name__',
+ getattr(func, '__class__').__name__)
+ except Exception:
+ func_name = str(func)
+ wrapper.__name__ = func_name
+
+ return wrapper
+
+class ParserElement(object):
+ """Abstract base level parser element class."""
+ DEFAULT_WHITE_CHARS = " \n\t\r"
+ verbose_stacktrace = False
+
+ @staticmethod
+ def setDefaultWhitespaceChars( chars ):
+ r"""
+ Overrides the default whitespace chars
+
+ Example::
+ # default whitespace chars are space, <TAB> and newline
+ OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl']
+
+ # change to just treat newline as significant
+ ParserElement.setDefaultWhitespaceChars(" \t")
+ OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def']
+ """
+ ParserElement.DEFAULT_WHITE_CHARS = chars
+
+ @staticmethod
+ def inlineLiteralsUsing(cls):
+ """
+ Set class to be used for inclusion of string literals into a parser.
+
+ Example::
+ # default literal class used is Literal
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31']
+
+
+ # change to Suppress
+ ParserElement.inlineLiteralsUsing(Suppress)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ date_str.parseString("1999/12/31") # -> ['1999', '12', '31']
+ """
+ ParserElement._literalStringClass = cls
+
+ def __init__( self, savelist=False ):
+ self.parseAction = list()
+ self.failAction = None
+ #~ self.name = "<unknown>" # don't define self.name, let subclasses try/except upcall
+ self.strRepr = None
+ self.resultsName = None
+ self.saveAsList = savelist
+ self.skipWhitespace = True
+ self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+ self.copyDefaultWhiteChars = True
+ self.mayReturnEmpty = False # used when checking for left-recursion
+ self.keepTabs = False
+ self.ignoreExprs = list()
+ self.debug = False
+ self.streamlined = False
+ self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
+ self.errmsg = ""
+ self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
+ self.debugActions = ( None, None, None ) #custom debug actions
+ self.re = None
+ self.callPreparse = True # used to avoid redundant calls to preParse
+ self.callDuringTry = False
+
+ def copy( self ):
+ """
+ Make a copy of this C{ParserElement}. Useful for defining different parse actions
+ for the same parsing pattern, using copies of the original parse element.
+
+ Example::
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K")
+ integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+
+ print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M"))
+ prints::
+ [5120, 100, 655360, 268435456]
+ Equivalent form of C{expr.copy()} is just C{expr()}::
+ integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+ """
+ cpy = copy.copy( self )
+ cpy.parseAction = self.parseAction[:]
+ cpy.ignoreExprs = self.ignoreExprs[:]
+ if self.copyDefaultWhiteChars:
+ cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+ return cpy
+
+ def setName( self, name ):
+ """
+ Define name for this expression, makes debugging and exception messages clearer.
+
+ Example::
+ Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1)
+ Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1)
+ """
+ self.name = name
+ self.errmsg = "Expected " + self.name
+ if hasattr(self,"exception"):
+ self.exception.msg = self.errmsg
+ return self
+
+ def setResultsName( self, name, listAllMatches=False ):
+ """
+ Define name for referencing matching tokens as a nested attribute
+ of the returned parse results.
+ NOTE: this returns a *copy* of the original C{ParserElement} object;
+ this is so that the client can define a basic element, such as an
+ integer, and reference it in multiple places with different names.
+
+ You can also set results names using the abbreviated syntax,
+ C{expr("name")} in place of C{expr.setResultsName("name")} -
+ see L{I{__call__}<__call__>}.
+
+ Example::
+ date_str = (integer.setResultsName("year") + '/'
+ + integer.setResultsName("month") + '/'
+ + integer.setResultsName("day"))
+
+ # equivalent form:
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+ """
+ newself = self.copy()
+ if name.endswith("*"):
+ name = name[:-1]
+ listAllMatches=True
+ newself.resultsName = name
+ newself.modalResults = not listAllMatches
+ return newself
+
+ def setBreak(self,breakFlag = True):
+ """Method to invoke the Python pdb debugger when this element is
+ about to be parsed. Set C{breakFlag} to True to enable, False to
+ disable.
+ """
+ if breakFlag:
+ _parseMethod = self._parse
+ def breaker(instring, loc, doActions=True, callPreParse=True):
+ import pdb
+ pdb.set_trace()
+ return _parseMethod( instring, loc, doActions, callPreParse )
+ breaker._originalParseMethod = _parseMethod
+ self._parse = breaker
+ else:
+ if hasattr(self._parse,"_originalParseMethod"):
+ self._parse = self._parse._originalParseMethod
+ return self
+
+ def setParseAction( self, *fns, **kwargs ):
+ """
+ Define one or more actions to perform when successfully matching parse element definition.
+ Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
+ C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
+ - s = the original string being parsed (see note below)
+ - loc = the location of the matching substring
+ - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
+ If the functions in fns modify the tokens, they can return them as the return
+ value from fn, and the modified list of tokens will replace the original.
+ Otherwise, fn does not need to return any value.
+
+ Optional keyword arguments:
+ - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{parseString}<parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer + '/' + integer + '/' + integer
+
+ date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31']
+
+ # use parse action to convert to ints at parse time
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ date_str = integer + '/' + integer + '/' + integer
+
+ # note that integer fields are now ints, not strings
+ date_str.parseString("1999/12/31") # -> [1999, '/', 12, '/', 31]
+ """
+ self.parseAction = list(map(_trim_arity, list(fns)))
+ self.callDuringTry = kwargs.get("callDuringTry", False)
+ return self
+
+ def addParseAction( self, *fns, **kwargs ):
+ """
+ Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}.
+
+ See examples in L{I{copy}<copy>}.
+ """
+ self.parseAction += list(map(_trim_arity, list(fns)))
+ self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+ return self
+
+ def addCondition(self, *fns, **kwargs):
+ """Add a boolean predicate function to expression's list of parse actions. See
+ L{I{setParseAction}<setParseAction>} for function call signatures. Unlike C{setParseAction},
+ functions passed to C{addCondition} need to return boolean success/fail of the condition.
+
+ Optional keyword arguments:
+ - message = define a custom message to be used in the raised exception
+ - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException
+
+ Example::
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ year_int = integer.copy()
+ year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later")
+ date_str = year_int + '/' + integer + '/' + integer
+
+ result = date_str.parseString("1999/12/31") # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1)
+ """
+ msg = kwargs.get("message", "failed user-defined condition")
+ exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException
+ for fn in fns:
+ def pa(s,l,t):
+ if not bool(_trim_arity(fn)(s,l,t)):
+ raise exc_type(s,l,msg)
+ self.parseAction.append(pa)
+ self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+ return self
+
+ def setFailAction( self, fn ):
+ """Define action to perform if parsing fails at this expression.
+ Fail acton fn is a callable function that takes the arguments
+ C{fn(s,loc,expr,err)} where:
+ - s = string being parsed
+ - loc = location where expression match was attempted and failed
+ - expr = the parse expression that failed
+ - err = the exception thrown
+ The function returns no value. It may throw C{L{ParseFatalException}}
+ if it is desired to stop parsing immediately."""
+ self.failAction = fn
+ return self
+
+ def _skipIgnorables( self, instring, loc ):
+ exprsFound = True
+ while exprsFound:
+ exprsFound = False
+ for e in self.ignoreExprs:
+ try:
+ while 1:
+ loc,dummy = e._parse( instring, loc )
+ exprsFound = True
+ except ParseException:
+ pass
+ return loc
+
+ def preParse( self, instring, loc ):
+ if self.ignoreExprs:
+ loc = self._skipIgnorables( instring, loc )
+
+ if self.skipWhitespace:
+ wt = self.whiteChars
+ instrlen = len(instring)
+ while loc < instrlen and instring[loc] in wt:
+ loc += 1
+
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ return loc, []
+
+ def postParse( self, instring, loc, tokenlist ):
+ return tokenlist
+
+ #~ @profile
+ def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
+ debugging = ( self.debug ) #and doActions )
+
+ if debugging or self.failAction:
+ #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
+ if (self.debugActions[0] ):
+ self.debugActions[0]( instring, loc, self )
+ if callPreParse and self.callPreparse:
+ preloc = self.preParse( instring, loc )
+ else:
+ preloc = loc
+ tokensStart = preloc
+ try:
+ try:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+ except IndexError:
+ raise ParseException( instring, len(instring), self.errmsg, self )
+ except ParseBaseException as err:
+ #~ print ("Exception raised:", err)
+ if self.debugActions[2]:
+ self.debugActions[2]( instring, tokensStart, self, err )
+ if self.failAction:
+ self.failAction( instring, tokensStart, self, err )
+ raise
+ else:
+ if callPreParse and self.callPreparse:
+ preloc = self.preParse( instring, loc )
+ else:
+ preloc = loc
+ tokensStart = preloc
+ if self.mayIndexError or preloc >= len(instring):
+ try:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+ except IndexError:
+ raise ParseException( instring, len(instring), self.errmsg, self )
+ else:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+
+ tokens = self.postParse( instring, loc, tokens )
+
+ retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
+ if self.parseAction and (doActions or self.callDuringTry):
+ if debugging:
+ try:
+ for fn in self.parseAction:
+ tokens = fn( instring, tokensStart, retTokens )
+ if tokens is not None:
+ retTokens = ParseResults( tokens,
+ self.resultsName,
+ asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+ modal=self.modalResults )
+ except ParseBaseException as err:
+ #~ print "Exception raised in user parse action:", err
+ if (self.debugActions[2] ):
+ self.debugActions[2]( instring, tokensStart, self, err )
+ raise
+ else:
+ for fn in self.parseAction:
+ tokens = fn( instring, tokensStart, retTokens )
+ if tokens is not None:
+ retTokens = ParseResults( tokens,
+ self.resultsName,
+ asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+ modal=self.modalResults )
+ if debugging:
+ #~ print ("Matched",self,"->",retTokens.asList())
+ if (self.debugActions[1] ):
+ self.debugActions[1]( instring, tokensStart, loc, self, retTokens )
+
+ return loc, retTokens
+
+ def tryParse( self, instring, loc ):
+ try:
+ return self._parse( instring, loc, doActions=False )[0]
+ except ParseFatalException:
+ raise ParseException( instring, loc, self.errmsg, self)
+
+ def canParseNext(self, instring, loc):
+ try:
+ self.tryParse(instring, loc)
+ except (ParseException, IndexError):
+ return False
+ else:
+ return True
+
+ class _UnboundedCache(object):
+ def __init__(self):
+ cache = {}
+ self.not_in_cache = not_in_cache = object()
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+
+ def clear(self):
+ cache.clear()
+
+ def cache_len(self):
+ return len(cache)
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
+
+ if _OrderedDict is not None:
+ class _FifoCache(object):
+ def __init__(self, size):
+ self.not_in_cache = not_in_cache = object()
+
+ cache = _OrderedDict()
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+ while len(cache) > size:
+ try:
+ cache.popitem(False)
+ except KeyError:
+ pass
+
+ def clear(self):
+ cache.clear()
+
+ def cache_len(self):
+ return len(cache)
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
+
+ else:
+ class _FifoCache(object):
+ def __init__(self, size):
+ self.not_in_cache = not_in_cache = object()
+
+ cache = {}
+ key_fifo = collections.deque([], size)
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+ while len(key_fifo) > size:
+ cache.pop(key_fifo.popleft(), None)
+ key_fifo.append(key)
+
+ def clear(self):
+ cache.clear()
+ key_fifo.clear()
+
+ def cache_len(self):
+ return len(cache)
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
+
+ # argument cache for optimizing repeated calls when backtracking through recursive expressions
+ packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail
+ packrat_cache_lock = RLock()
+ packrat_cache_stats = [0, 0]
+
+ # this method gets repeatedly called during backtracking with the same arguments -
+ # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
+ def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
+ HIT, MISS = 0, 1
+ lookup = (self, instring, loc, callPreParse, doActions)
+ with ParserElement.packrat_cache_lock:
+ cache = ParserElement.packrat_cache
+ value = cache.get(lookup)
+ if value is cache.not_in_cache:
+ ParserElement.packrat_cache_stats[MISS] += 1
+ try:
+ value = self._parseNoCache(instring, loc, doActions, callPreParse)
+ except ParseBaseException as pe:
+ # cache a copy of the exception, without the traceback
+ cache.set(lookup, pe.__class__(*pe.args))
+ raise
+ else:
+ cache.set(lookup, (value[0], value[1].copy()))
+ return value
+ else:
+ ParserElement.packrat_cache_stats[HIT] += 1
+ if isinstance(value, Exception):
+ raise value
+ return (value[0], value[1].copy())
+
+ _parse = _parseNoCache
+
+ @staticmethod
+ def resetCache():
+ ParserElement.packrat_cache.clear()
+ ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats)
+
+ _packratEnabled = False
+ @staticmethod
+ def enablePackrat(cache_size_limit=128):
+ """Enables "packrat" parsing, which adds memoizing to the parsing logic.
+ Repeated parse attempts at the same string location (which happens
+ often in many complex grammars) can immediately return a cached value,
+ instead of re-executing parsing/validating code. Memoizing is done of
+ both valid results and parsing exceptions.
+
+ Parameters:
+ - cache_size_limit - (default=C{128}) - if an integer value is provided
+ will limit the size of the packrat cache; if None is passed, then
+ the cache size will be unbounded; if 0 is passed, the cache will
+ be effectively disabled.
+
+ This speedup may break existing programs that use parse actions that
+ have side-effects. For this reason, packrat parsing is disabled when
+ you first import pyparsing. To activate the packrat feature, your
+ program must call the class method C{ParserElement.enablePackrat()}. If
+ your program uses C{psyco} to "compile as you go", you must call
+ C{enablePackrat} before calling C{psyco.full()}. If you do not do this,
+ Python will crash. For best results, call C{enablePackrat()} immediately
+ after importing pyparsing.
+
+ Example::
+ import pyparsing
+ pyparsing.ParserElement.enablePackrat()
+ """
+ if not ParserElement._packratEnabled:
+ ParserElement._packratEnabled = True
+ if cache_size_limit is None:
+ ParserElement.packrat_cache = ParserElement._UnboundedCache()
+ else:
+ ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit)
+ ParserElement._parse = ParserElement._parseCache
+
+ def parseString( self, instring, parseAll=False ):
+ """
+ Execute the parse expression with the given string.
+ This is the main interface to the client code, once the complete
+ expression has been built.
+
+ If you want the grammar to require that the entire input string be
+ successfully parsed, then set C{parseAll} to True (equivalent to ending
+ the grammar with C{L{StringEnd()}}).
+
+ Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
+ in order to report proper column numbers in parse actions.
+ If the input string contains tabs and
+ the grammar uses parse actions that use the C{loc} argument to index into the
+ string being parsed, you can ensure you have a consistent view of the input
+ string by:
+ - calling C{parseWithTabs} on your grammar before calling C{parseString}
+ (see L{I{parseWithTabs}<parseWithTabs>})
+ - define your parse action using the full C{(s,loc,toks)} signature, and
+ reference the input string using the parse action's C{s} argument
+ - explictly expand the tabs in your input string before calling
+ C{parseString}
+
+ Example::
+ Word('a').parseString('aaaaabaaa') # -> ['aaaaa']
+ Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text
+ """
+ ParserElement.resetCache()
+ if not self.streamlined:
+ self.streamline()
+ #~ self.saveAsList = True
+ for e in self.ignoreExprs:
+ e.streamline()
+ if not self.keepTabs:
+ instring = instring.expandtabs()
+ try:
+ loc, tokens = self._parse( instring, 0 )
+ if parseAll:
+ loc = self.preParse( instring, loc )
+ se = Empty() + StringEnd()
+ se._parse( instring, loc )
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+ else:
+ return tokens
+
+ def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ):
+ """
+ Scan the input string for expression matches. Each match will return the
+ matching tokens, start location, and end location. May be called with optional
+ C{maxMatches} argument, to clip scanning after 'n' matches are found. If
+ C{overlap} is specified, then overlapping matches will be reported.
+
+ Note that the start and end locations are reported relative to the string
+ being parsed. See L{I{parseString}<parseString>} for more information on parsing
+ strings with embedded tabs.
+
+ Example::
+ source = "sldjf123lsdjjkf345sldkjf879lkjsfd987"
+ print(source)
+ for tokens,start,end in Word(alphas).scanString(source):
+ print(' '*start + '^'*(end-start))
+ print(' '*start + tokens[0])
+
+ prints::
+
+ sldjf123lsdjjkf345sldkjf879lkjsfd987
+ ^^^^^
+ sldjf
+ ^^^^^^^
+ lsdjjkf
+ ^^^^^^
+ sldkjf
+ ^^^^^^
+ lkjsfd
+ """
+ if not self.streamlined:
+ self.streamline()
+ for e in self.ignoreExprs:
+ e.streamline()
+
+ if not self.keepTabs:
+ instring = _ustr(instring).expandtabs()
+ instrlen = len(instring)
+ loc = 0
+ preparseFn = self.preParse
+ parseFn = self._parse
+ ParserElement.resetCache()
+ matches = 0
+ try:
+ while loc <= instrlen and matches < maxMatches:
+ try:
+ preloc = preparseFn( instring, loc )
+ nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
+ except ParseException:
+ loc = preloc+1
+ else:
+ if nextLoc > loc:
+ matches += 1
+ yield tokens, preloc, nextLoc
+ if overlap:
+ nextloc = preparseFn( instring, loc )
+ if nextloc > loc:
+ loc = nextLoc
+ else:
+ loc += 1
+ else:
+ loc = nextLoc
+ else:
+ loc = preloc+1
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def transformString( self, instring ):
+ """
+ Extension to C{L{scanString}}, to modify matching text with modified tokens that may
+ be returned from a parse action. To use C{transformString}, define a grammar and
+ attach a parse action to it that modifies the returned token list.
+ Invoking C{transformString()} on a target string will then scan for matches,
+ and replace the matched text patterns according to the logic in the parse
+ action. C{transformString()} returns the resulting transformed string.
+
+ Example::
+ wd = Word(alphas)
+ wd.setParseAction(lambda toks: toks[0].title())
+
+ print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york."))
+ Prints::
+ Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York.
+ """
+ out = []
+ lastE = 0
+ # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
+ # keep string locs straight between transformString and scanString
+ self.keepTabs = True
+ try:
+ for t,s,e in self.scanString( instring ):
+ out.append( instring[lastE:s] )
+ if t:
+ if isinstance(t,ParseResults):
+ out += t.asList()
+ elif isinstance(t,list):
+ out += t
+ else:
+ out.append(t)
+ lastE = e
+ out.append(instring[lastE:])
+ out = [o for o in out if o]
+ return "".join(map(_ustr,_flatten(out)))
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def searchString( self, instring, maxMatches=_MAX_INT ):
+ """
+ Another extension to C{L{scanString}}, simplifying the access to the tokens found
+ to match the given parse expression. May be called with optional
+ C{maxMatches} argument, to clip searching after 'n' matches are found.
+
+ Example::
+ # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters
+ cap_word = Word(alphas.upper(), alphas.lower())
+
+ print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))
+
+ # the sum() builtin can be used to merge results into a single ParseResults object
+ print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")))
+ prints::
+ [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']]
+ ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity']
+ """
+ try:
+ return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False):
+ """
+ Generator method to split a string using the given expression as a separator.
+ May be called with optional C{maxsplit} argument, to limit the number of splits;
+ and the optional C{includeSeparators} argument (default=C{False}), if the separating
+ matching text should be included in the split results.
+
+ Example::
+ punc = oneOf(list(".,;:/-!?"))
+ print(list(punc.split("This, this?, this sentence, is badly punctuated!")))
+ prints::
+ ['This', ' this', '', ' this sentence', ' is badly punctuated', '']
+ """
+ splits = 0
+ last = 0
+ for t,s,e in self.scanString(instring, maxMatches=maxsplit):
+ yield instring[last:s]
+ if includeSeparators:
+ yield t[0]
+ last = e
+ yield instring[last:]
+
+ def __add__(self, other ):
+ """
+ Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement
+ converts them to L{Literal}s by default.
+
+ Example::
+ greet = Word(alphas) + "," + Word(alphas) + "!"
+ hello = "Hello, World!"
+ print (hello, "->", greet.parseString(hello))
+ Prints::
+ Hello, World! -> ['Hello', ',', 'World', '!']
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return And( [ self, other ] )
+
+ def __radd__(self, other ):
+ """
+ Implementation of + operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other + self
+
+ def __sub__(self, other):
+ """
+ Implementation of - operator, returns C{L{And}} with error stop
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return self + And._ErrorStop() + other
+
+ def __rsub__(self, other ):
+ """
+ Implementation of - operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other - self
+
+ def __mul__(self,other):
+ """
+ Implementation of * operator, allows use of C{expr * 3} in place of
+ C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer
+ tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples
+ may also include C{None} as in:
+ - C{expr*(n,None)} or C{expr*(n,)} is equivalent
+ to C{expr*n + L{ZeroOrMore}(expr)}
+ (read as "at least n instances of C{expr}")
+ - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
+ (read as "0 to n instances of C{expr}")
+ - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
+ - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
+
+ Note that C{expr*(None,n)} does not raise an exception if
+ more than n exprs exist in the input stream; that is,
+ C{expr*(None,n)} does not enforce a maximum number of expr
+ occurrences. If this behavior is desired, then write
+ C{expr*(None,n) + ~expr}
+ """
+ if isinstance(other,int):
+ minElements, optElements = other,0
+ elif isinstance(other,tuple):
+ other = (other + (None, None))[:2]
+ if other[0] is None:
+ other = (0, other[1])
+ if isinstance(other[0],int) and other[1] is None:
+ if other[0] == 0:
+ return ZeroOrMore(self)
+ if other[0] == 1:
+ return OneOrMore(self)
+ else:
+ return self*other[0] + ZeroOrMore(self)
+ elif isinstance(other[0],int) and isinstance(other[1],int):
+ minElements, optElements = other
+ optElements -= minElements
+ else:
+ raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
+ else:
+ raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))
+
+ if minElements < 0:
+ raise ValueError("cannot multiply ParserElement by negative value")
+ if optElements < 0:
+ raise ValueError("second tuple value must be greater or equal to first tuple value")
+ if minElements == optElements == 0:
+ raise ValueError("cannot multiply ParserElement by 0 or (0,0)")
+
+ if (optElements):
+ def makeOptionalList(n):
+ if n>1:
+ return Optional(self + makeOptionalList(n-1))
+ else:
+ return Optional(self)
+ if minElements:
+ if minElements == 1:
+ ret = self + makeOptionalList(optElements)
+ else:
+ ret = And([self]*minElements) + makeOptionalList(optElements)
+ else:
+ ret = makeOptionalList(optElements)
+ else:
+ if minElements == 1:
+ ret = self
+ else:
+ ret = And([self]*minElements)
+ return ret
+
+ def __rmul__(self, other):
+ return self.__mul__(other)
+
+ def __or__(self, other ):
+ """
+ Implementation of | operator - returns C{L{MatchFirst}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return MatchFirst( [ self, other ] )
+
+ def __ror__(self, other ):
+ """
+ Implementation of | operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other | self
+
+ def __xor__(self, other ):
+ """
+ Implementation of ^ operator - returns C{L{Or}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return Or( [ self, other ] )
+
+ def __rxor__(self, other ):
+ """
+ Implementation of ^ operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other ^ self
+
+ def __and__(self, other ):
+ """
+ Implementation of & operator - returns C{L{Each}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return Each( [ self, other ] )
+
+ def __rand__(self, other ):
+ """
+ Implementation of & operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other & self
+
+ def __invert__( self ):
+ """
+ Implementation of ~ operator - returns C{L{NotAny}}
+ """
+ return NotAny( self )
+
+ def __call__(self, name=None):
+ """
+ Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}.
+
+ If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
+ passed as C{True}.
+
+ If C{name} is omitted, same as calling C{L{copy}}.
+
+ Example::
+ # these are equivalent
+ userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
+ userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
+ """
+ if name is not None:
+ return self.setResultsName(name)
+ else:
+ return self.copy()
+
+ def suppress( self ):
+ """
+ Suppresses the output of this C{ParserElement}; useful to keep punctuation from
+ cluttering up returned output.
+ """
+ return Suppress( self )
+
+ def leaveWhitespace( self ):
+ """
+ Disables the skipping of whitespace before matching the characters in the
+ C{ParserElement}'s defined pattern. This is normally only used internally by
+ the pyparsing module, but may be needed in some whitespace-sensitive grammars.
+ """
+ self.skipWhitespace = False
+ return self
+
+ def setWhitespaceChars( self, chars ):
+ """
+ Overrides the default whitespace chars
+ """
+ self.skipWhitespace = True
+ self.whiteChars = chars
+ self.copyDefaultWhiteChars = False
+ return self
+
+ def parseWithTabs( self ):
+ """
+ Overrides default behavior to expand C{<TAB>}s to spaces before parsing the input string.
+ Must be called before C{parseString} when the input grammar contains elements that
+ match C{<TAB>} characters.
+ """
+ self.keepTabs = True
+ return self
+
+ def ignore( self, other ):
+ """
+ Define expression to be ignored (e.g., comments) while doing pattern
+ matching; may be called repeatedly, to define multiple comment or other
+ ignorable patterns.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+ patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj']
+
+ patt.ignore(cStyleComment)
+ patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd']
+ """
+ if isinstance(other, basestring):
+ other = Suppress(other)
+
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ self.ignoreExprs.append(other)
+ else:
+ self.ignoreExprs.append( Suppress( other.copy() ) )
+ return self
+
+ def setDebugActions( self, startAction, successAction, exceptionAction ):
+ """
+ Enable display of debugging messages while doing pattern matching.
+ """
+ self.debugActions = (startAction or _defaultStartDebugAction,
+ successAction or _defaultSuccessDebugAction,
+ exceptionAction or _defaultExceptionDebugAction)
+ self.debug = True
+ return self
+
+ def setDebug( self, flag=True ):
+ """
+ Enable display of debugging messages while doing pattern matching.
+ Set C{flag} to True to enable, False to disable.
+
+ Example::
+ wd = Word(alphas).setName("alphaword")
+ integer = Word(nums).setName("numword")
+ term = wd | integer
+
+ # turn on debugging for wd
+ wd.setDebug()
+
+ OneOrMore(term).parseString("abc 123 xyz 890")
+
+ prints::
+ Match alphaword at loc 0(1,1)
+ Matched alphaword -> ['abc']
+ Match alphaword at loc 3(1,4)
+ Exception raised:Expected alphaword (at char 4), (line:1, col:5)
+ Match alphaword at loc 7(1,8)
+ Matched alphaword -> ['xyz']
+ Match alphaword at loc 11(1,12)
+ Exception raised:Expected alphaword (at char 12), (line:1, col:13)
+ Match alphaword at loc 15(1,16)
+ Exception raised:Expected alphaword (at char 15), (line:1, col:16)
+
+ The output shown is that produced by the default debug actions - custom debug actions can be
+ specified using L{setDebugActions}. Prior to attempting
+ to match the C{wd} expression, the debugging message C{"Match <exprname> at loc <n>(<line>,<col>)"}
+ is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"}
+ message is shown. Also note the use of L{setName} to assign a human-readable name to the expression,
+ which makes debugging and exception messages easier to understand - for instance, the default
+ name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}.
+ """
+ if flag:
+ self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
+ else:
+ self.debug = False
+ return self
+
+ def __str__( self ):
+ return self.name
+
+ def __repr__( self ):
+ return _ustr(self)
+
+ def streamline( self ):
+ self.streamlined = True
+ self.strRepr = None
+ return self
+
+ def checkRecursion( self, parseElementList ):
+ pass
+
+ def validate( self, validateTrace=[] ):
+ """
+ Check defined expressions for valid structure, check for infinite recursive definitions.
+ """
+ self.checkRecursion( [] )
+
+ def parseFile( self, file_or_filename, parseAll=False ):
+ """
+ Execute the parse expression on the given file or filename.
+ If a filename is specified (instead of a file object),
+ the entire file is opened, read, and closed before parsing.
+ """
+ try:
+ file_contents = file_or_filename.read()
+ except AttributeError:
+ with open(file_or_filename, "r") as f:
+ file_contents = f.read()
+ try:
+ return self.parseString(file_contents, parseAll)
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def __eq__(self,other):
+ if isinstance(other, ParserElement):
+ return self is other or vars(self) == vars(other)
+ elif isinstance(other, basestring):
+ return self.matches(other)
+ else:
+ return super(ParserElement,self)==other
+
+ def __ne__(self,other):
+ return not (self == other)
+
+ def __hash__(self):
+ return hash(id(self))
+
+ def __req__(self,other):
+ return self == other
+
+ def __rne__(self,other):
+ return not (self == other)
+
+ def matches(self, testString, parseAll=True):
+ """
+ Method for quick testing of a parser against a test string. Good for simple
+ inline microtests of sub expressions while building up larger parser.
+
+ Parameters:
+ - testString - to test against this expression for a match
+ - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests
+
+ Example::
+ expr = Word(nums)
+ assert expr.matches("100")
+ """
+ try:
+ self.parseString(_ustr(testString), parseAll=parseAll)
+ return True
+ except ParseBaseException:
+ return False
+
+ def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False):
+ """
+ Execute the parse expression on a series of test strings, showing each
+ test, the parsed results or where the parse failed. Quick and easy way to
+ run a parse expression against a list of sample strings.
+
+ Parameters:
+ - tests - a list of separate test strings, or a multiline string of test strings
+ - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests
+ - comment - (default=C{'#'}) - expression for indicating embedded comments in the test
+ string; pass None to disable comment filtering
+ - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline;
+ if False, only dump nested list
+ - printResults - (default=C{True}) prints test output to stdout
+ - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing
+
+ Returns: a (success, results) tuple, where success indicates that all tests succeeded
+ (or failed if C{failureTests} is True), and the results contain a list of lines of each
+ test's output
+
+ Example::
+ number_expr = pyparsing_common.number.copy()
+
+ result = number_expr.runTests('''
+ # unsigned integer
+ 100
+ # negative integer
+ -100
+ # float with scientific notation
+ 6.02e23
+ # integer with scientific notation
+ 1e-12
+ ''')
+ print("Success" if result[0] else "Failed!")
+
+ result = number_expr.runTests('''
+ # stray character
+ 100Z
+ # missing leading digit before '.'
+ -.100
+ # too many '.'
+ 3.14.159
+ ''', failureTests=True)
+ print("Success" if result[0] else "Failed!")
+ prints::
+ # unsigned integer
+ 100
+ [100]
+
+ # negative integer
+ -100
+ [-100]
+
+ # float with scientific notation
+ 6.02e23
+ [6.02e+23]
+
+ # integer with scientific notation
+ 1e-12
+ [1e-12]
+
+ Success
+
+ # stray character
+ 100Z
+ ^
+ FAIL: Expected end of text (at char 3), (line:1, col:4)
+
+ # missing leading digit before '.'
+ -.100
+ ^
+ FAIL: Expected {real number with scientific notation | real number | signed integer} (at char 0), (line:1, col:1)
+
+ # too many '.'
+ 3.14.159
+ ^
+ FAIL: Expected end of text (at char 4), (line:1, col:5)
+
+ Success
+
+ Each test string must be on a single line. If you want to test a string that spans multiple
+ lines, create a test like this::
+
+ expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines")
+
+ (Note that this is a raw string literal, you must include the leading 'r'.)
+ """
+ if isinstance(tests, basestring):
+ tests = list(map(str.strip, tests.rstrip().splitlines()))
+ if isinstance(comment, basestring):
+ comment = Literal(comment)
+ allResults = []
+ comments = []
+ success = True
+ for t in tests:
+ if comment is not None and comment.matches(t, False) or comments and not t:
+ comments.append(t)
+ continue
+ if not t:
+ continue
+ out = ['\n'.join(comments), t]
+ comments = []
+ try:
+ t = t.replace(r'\n','\n')
+ result = self.parseString(t, parseAll=parseAll)
+ out.append(result.dump(full=fullDump))
+ success = success and not failureTests
+ except ParseBaseException as pe:
+ fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else ""
+ if '\n' in t:
+ out.append(line(pe.loc, t))
+ out.append(' '*(col(pe.loc,t)-1) + '^' + fatal)
+ else:
+ out.append(' '*pe.loc + '^' + fatal)
+ out.append("FAIL: " + str(pe))
+ success = success and failureTests
+ result = pe
+ except Exception as exc:
+ out.append("FAIL-EXCEPTION: " + str(exc))
+ success = success and failureTests
+ result = exc
+
+ if printResults:
+ if fullDump:
+ out.append('')
+ print('\n'.join(out))
+
+ allResults.append((t, result))
+
+ return success, allResults
+
+
+class Token(ParserElement):
+ """
+ Abstract C{ParserElement} subclass, for defining atomic matching patterns.
+ """
+ def __init__( self ):
+ super(Token,self).__init__( savelist=False )
+
+
+class Empty(Token):
+ """
+ An empty token, will always match.
+ """
+ def __init__( self ):
+ super(Empty,self).__init__()
+ self.name = "Empty"
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+
+
+class NoMatch(Token):
+ """
+ A token that will never match.
+ """
+ def __init__( self ):
+ super(NoMatch,self).__init__()
+ self.name = "NoMatch"
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+ self.errmsg = "Unmatchable token"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Literal(Token):
+ """
+ Token to exactly match a specified string.
+
+ Example::
+ Literal('blah').parseString('blah') # -> ['blah']
+ Literal('blah').parseString('blahfooblah') # -> ['blah']
+ Literal('blah').parseString('bla') # -> Exception: Expected "blah"
+
+ For case-insensitive matching, use L{CaselessLiteral}.
+
+ For keyword matching (force word break before and after the matched string),
+ use L{Keyword} or L{CaselessKeyword}.
+ """
+ def __init__( self, matchString ):
+ super(Literal,self).__init__()
+ self.match = matchString
+ self.matchLen = len(matchString)
+ try:
+ self.firstMatchChar = matchString[0]
+ except IndexError:
+ warnings.warn("null string passed to Literal; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+ self.__class__ = Empty
+ self.name = '"%s"' % _ustr(self.match)
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = False
+ self.mayIndexError = False
+
+ # Performance tuning: this routine gets called a *lot*
+ # if this is a single character match string and the first character matches,
+ # short-circuit as quickly as possible, and avoid calling startswith
+ #~ @profile
+ def parseImpl( self, instring, loc, doActions=True ):
+ if (instring[loc] == self.firstMatchChar and
+ (self.matchLen==1 or instring.startswith(self.match,loc)) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+_L = Literal
+ParserElement._literalStringClass = Literal
+
+class Keyword(Token):
+ """
+ Token to exactly match a specified string as a keyword, that is, it must be
+ immediately followed by a non-keyword character. Compare with C{L{Literal}}:
+ - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}.
+ - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
+ Accepts two optional constructor arguments in addition to the keyword string:
+ - C{identChars} is a string of characters that would be valid identifier characters,
+ defaulting to all alphanumerics + "_" and "$"
+ - C{caseless} allows case-insensitive matching, default is C{False}.
+
+ Example::
+ Keyword("start").parseString("start") # -> ['start']
+ Keyword("start").parseString("starting") # -> Exception
+
+ For case-insensitive matching, use L{CaselessKeyword}.
+ """
+ DEFAULT_KEYWORD_CHARS = alphanums+"_$"
+
+ def __init__( self, matchString, identChars=None, caseless=False ):
+ super(Keyword,self).__init__()
+ if identChars is None:
+ identChars = Keyword.DEFAULT_KEYWORD_CHARS
+ self.match = matchString
+ self.matchLen = len(matchString)
+ try:
+ self.firstMatchChar = matchString[0]
+ except IndexError:
+ warnings.warn("null string passed to Keyword; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+ self.name = '"%s"' % self.match
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = False
+ self.mayIndexError = False
+ self.caseless = caseless
+ if caseless:
+ self.caselessmatch = matchString.upper()
+ identChars = identChars.upper()
+ self.identChars = set(identChars)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.caseless:
+ if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
+ (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ else:
+ if (instring[loc] == self.firstMatchChar and
+ (self.matchLen==1 or instring.startswith(self.match,loc)) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
+ (loc == 0 or instring[loc-1] not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ def copy(self):
+ c = super(Keyword,self).copy()
+ c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
+ return c
+
+ @staticmethod
+ def setDefaultKeywordChars( chars ):
+ """Overrides the default Keyword chars
+ """
+ Keyword.DEFAULT_KEYWORD_CHARS = chars
+
+class CaselessLiteral(Literal):
+ """
+ Token to match a specified string, ignoring case of letters.
+ Note: the matched results will always be in the case of the given
+ match string, NOT the case of the input text.
+
+ Example::
+ OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD']
+
+ (Contrast with example for L{CaselessKeyword}.)
+ """
+ def __init__( self, matchString ):
+ super(CaselessLiteral,self).__init__( matchString.upper() )
+ # Preserve the defining literal.
+ self.returnString = matchString
+ self.name = "'%s'" % self.returnString
+ self.errmsg = "Expected " + self.name
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if instring[ loc:loc+self.matchLen ].upper() == self.match:
+ return loc+self.matchLen, self.returnString
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class CaselessKeyword(Keyword):
+ """
+ Caseless version of L{Keyword}.
+
+ Example::
+ OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD']
+
+ (Contrast with example for L{CaselessLiteral}.)
+ """
+ def __init__( self, matchString, identChars=None ):
+ super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class CloseMatch(Token):
+ """
+ A variation on L{Literal} which matches "close" matches, that is,
+ strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters:
+ - C{match_string} - string to be matched
+ - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match
+
+ The results from a successful parse will contain the matched text from the input string and the following named results:
+ - C{mismatches} - a list of the positions within the match_string where mismatches were found
+ - C{original} - the original match_string used to compare against the input string
+
+ If C{mismatches} is an empty list, then the match was an exact match.
+
+ Example::
+ patt = CloseMatch("ATCATCGAATGGA")
+ patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']})
+ patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1)
+
+ # exact match
+ patt.parseString("ATCATCGAATGGA") # -> (['ATCATCGAATGGA'], {'mismatches': [[]], 'original': ['ATCATCGAATGGA']})
+
+ # close match allowing up to 2 mismatches
+ patt = CloseMatch("ATCATCGAATGGA", maxMismatches=2)
+ patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']})
+ """
+ def __init__(self, match_string, maxMismatches=1):
+ super(CloseMatch,self).__init__()
+ self.name = match_string
+ self.match_string = match_string
+ self.maxMismatches = maxMismatches
+ self.errmsg = "Expected %r (with up to %d mismatches)" % (self.match_string, self.maxMismatches)
+ self.mayIndexError = False
+ self.mayReturnEmpty = False
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ start = loc
+ instrlen = len(instring)
+ maxloc = start + len(self.match_string)
+
+ if maxloc <= instrlen:
+ match_string = self.match_string
+ match_stringloc = 0
+ mismatches = []
+ maxMismatches = self.maxMismatches
+
+ for match_stringloc,s_m in enumerate(zip(instring[loc:maxloc], self.match_string)):
+ src,mat = s_m
+ if src != mat:
+ mismatches.append(match_stringloc)
+ if len(mismatches) > maxMismatches:
+ break
+ else:
+ loc = match_stringloc + 1
+ results = ParseResults([instring[start:loc]])
+ results['original'] = self.match_string
+ results['mismatches'] = mismatches
+ return loc, results
+
+ raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Word(Token):
+ """
+ Token for matching words composed of allowed character sets.
+ Defined with string containing all allowed initial characters,
+ an optional string containing allowed body characters (if omitted,
+ defaults to the initial character set), and an optional minimum,
+ maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction. An optional
+ C{excludeChars} parameter can list characters that might be found in
+ the input C{bodyChars} string; useful to define a word of all printables
+ except for one or two characters, for instance.
+
+ L{srange} is useful for defining custom character set strings for defining
+ C{Word} expressions, using range notation from regular expression character sets.
+
+ A common mistake is to use C{Word} to match a specific literal string, as in
+ C{Word("Address")}. Remember that C{Word} uses the string argument to define
+ I{sets} of matchable characters. This expression would match "Add", "AAA",
+ "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'.
+ To match an exact literal string, use L{Literal} or L{Keyword}.
+
+ pyparsing includes helper strings for building Words:
+ - L{alphas}
+ - L{nums}
+ - L{alphanums}
+ - L{hexnums}
+ - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.)
+ - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.)
+ - L{printables} (any non-whitespace character)
+
+ Example::
+ # a word composed of digits
+ integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9"))
+
+ # a word with a leading capital, and zero or more lowercase
+ capital_word = Word(alphas.upper(), alphas.lower())
+
+ # hostnames are alphanumeric, with leading alpha, and '-'
+ hostname = Word(alphas, alphanums+'-')
+
+ # roman numeral (not a strict parser, accepts invalid mix of characters)
+ roman = Word("IVXLCDM")
+
+ # any string of non-whitespace characters, except for ','
+ csv_value = Word(printables, excludeChars=",")
+ """
+ def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ):
+ super(Word,self).__init__()
+ if excludeChars:
+ initChars = ''.join(c for c in initChars if c not in excludeChars)
+ if bodyChars:
+ bodyChars = ''.join(c for c in bodyChars if c not in excludeChars)
+ self.initCharsOrig = initChars
+ self.initChars = set(initChars)
+ if bodyChars :
+ self.bodyCharsOrig = bodyChars
+ self.bodyChars = set(bodyChars)
+ else:
+ self.bodyCharsOrig = initChars
+ self.bodyChars = set(initChars)
+
+ self.maxSpecified = max > 0
+
+ if min < 1:
+ raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted")
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.asKeyword = asKeyword
+
+ if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
+ if self.bodyCharsOrig == self.initCharsOrig:
+ self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
+ elif len(self.initCharsOrig) == 1:
+ self.reString = "%s[%s]*" % \
+ (re.escape(self.initCharsOrig),
+ _escapeRegexRangeChars(self.bodyCharsOrig),)
+ else:
+ self.reString = "[%s][%s]*" % \
+ (_escapeRegexRangeChars(self.initCharsOrig),
+ _escapeRegexRangeChars(self.bodyCharsOrig),)
+ if self.asKeyword:
+ self.reString = r"\b"+self.reString+r"\b"
+ try:
+ self.re = re.compile( self.reString )
+ except Exception:
+ self.re = None
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.re:
+ result = self.re.match(instring,loc)
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ return loc, result.group()
+
+ if not(instring[ loc ] in self.initChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ start = loc
+ loc += 1
+ instrlen = len(instring)
+ bodychars = self.bodyChars
+ maxloc = start + self.maxLen
+ maxloc = min( maxloc, instrlen )
+ while loc < maxloc and instring[loc] in bodychars:
+ loc += 1
+
+ throwException = False
+ if loc - start < self.minLen:
+ throwException = True
+ if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
+ throwException = True
+ if self.asKeyword:
+ if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars):
+ throwException = True
+
+ if throwException:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+ def __str__( self ):
+ try:
+ return super(Word,self).__str__()
+ except Exception:
+ pass
+
+
+ if self.strRepr is None:
+
+ def charsAsStr(s):
+ if len(s)>4:
+ return s[:4]+"..."
+ else:
+ return s
+
+ if ( self.initCharsOrig != self.bodyCharsOrig ):
+ self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
+ else:
+ self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)
+
+ return self.strRepr
+
+
+class Regex(Token):
+ r"""
+ Token for matching strings that match a given regular expression.
+ Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
+ If the given regex contains named groups (defined using C{(?P<name>...)}), these will be preserved as
+ named parse results.
+
+ Example::
+ realnum = Regex(r"[+-]?\d+\.\d*")
+ date = Regex(r'(?P<year>\d{4})-(?P<month>\d\d?)-(?P<day>\d\d?)')
+ # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression
+ roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})")
+ """
+ compiledREtype = type(re.compile("[A-Z]"))
+ def __init__( self, pattern, flags=0):
+ """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags."""
+ super(Regex,self).__init__()
+
+ if isinstance(pattern, basestring):
+ if not pattern:
+ warnings.warn("null string passed to Regex; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+
+ self.pattern = pattern
+ self.flags = flags
+
+ try:
+ self.re = re.compile(self.pattern, self.flags)
+ self.reString = self.pattern
+ except sre_constants.error:
+ warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
+ SyntaxWarning, stacklevel=2)
+ raise
+
+ elif isinstance(pattern, Regex.compiledREtype):
+ self.re = pattern
+ self.pattern = \
+ self.reString = str(pattern)
+ self.flags = flags
+
+ else:
+ raise ValueError("Regex may only be constructed with a string or a compiled RE object")
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ result = self.re.match(instring,loc)
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ d = result.groupdict()
+ ret = ParseResults(result.group())
+ if d:
+ for k in d:
+ ret[k] = d[k]
+ return loc,ret
+
+ def __str__( self ):
+ try:
+ return super(Regex,self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "Re:(%s)" % repr(self.pattern)
+
+ return self.strRepr
+
+
+class QuotedString(Token):
+ r"""
+ Token for matching strings that are delimited by quoting characters.
+
+ Defined with the following parameters:
+ - quoteChar - string of one or more characters defining the quote delimiting string
+ - escChar - character to escape quotes, typically backslash (default=C{None})
+ - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None})
+ - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
+ - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
+ - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
+ - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True})
+
+ Example::
+ qs = QuotedString('"')
+ print(qs.searchString('lsjdf "This is the quote" sldjf'))
+ complex_qs = QuotedString('{{', endQuoteChar='}}')
+ print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf'))
+ sql_qs = QuotedString('"', escQuote='""')
+ print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf'))
+ prints::
+ [['This is the quote']]
+ [['This is the "quote"']]
+ [['This is the quote with "embedded" quotes']]
+ """
+ def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True):
+ super(QuotedString,self).__init__()
+
+ # remove white space from quote chars - wont work anyway
+ quoteChar = quoteChar.strip()
+ if not quoteChar:
+ warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+ raise SyntaxError()
+
+ if endQuoteChar is None:
+ endQuoteChar = quoteChar
+ else:
+ endQuoteChar = endQuoteChar.strip()
+ if not endQuoteChar:
+ warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+ raise SyntaxError()
+
+ self.quoteChar = quoteChar
+ self.quoteCharLen = len(quoteChar)
+ self.firstQuoteChar = quoteChar[0]
+ self.endQuoteChar = endQuoteChar
+ self.endQuoteCharLen = len(endQuoteChar)
+ self.escChar = escChar
+ self.escQuote = escQuote
+ self.unquoteResults = unquoteResults
+ self.convertWhitespaceEscapes = convertWhitespaceEscapes
+
+ if multiline:
+ self.flags = re.MULTILINE | re.DOTALL
+ self.pattern = r'%s(?:[^%s%s]' % \
+ ( re.escape(self.quoteChar),
+ _escapeRegexRangeChars(self.endQuoteChar[0]),
+ (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+ else:
+ self.flags = 0
+ self.pattern = r'%s(?:[^%s\n\r%s]' % \
+ ( re.escape(self.quoteChar),
+ _escapeRegexRangeChars(self.endQuoteChar[0]),
+ (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+ if len(self.endQuoteChar) > 1:
+ self.pattern += (
+ '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
+ _escapeRegexRangeChars(self.endQuoteChar[i]))
+ for i in range(len(self.endQuoteChar)-1,0,-1)) + ')'
+ )
+ if escQuote:
+ self.pattern += (r'|(?:%s)' % re.escape(escQuote))
+ if escChar:
+ self.pattern += (r'|(?:%s.)' % re.escape(escChar))
+ self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
+ self.pattern += (r')*%s' % re.escape(self.endQuoteChar))
+
+ try:
+ self.re = re.compile(self.pattern, self.flags)
+ self.reString = self.pattern
+ except sre_constants.error:
+ warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
+ SyntaxWarning, stacklevel=2)
+ raise
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ ret = result.group()
+
+ if self.unquoteResults:
+
+ # strip off quotes
+ ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
+
+ if isinstance(ret,basestring):
+ # replace escaped whitespace
+ if '\\' in ret and self.convertWhitespaceEscapes:
+ ws_map = {
+ r'\t' : '\t',
+ r'\n' : '\n',
+ r'\f' : '\f',
+ r'\r' : '\r',
+ }
+ for wslit,wschar in ws_map.items():
+ ret = ret.replace(wslit, wschar)
+
+ # replace escaped characters
+ if self.escChar:
+ ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret)
+
+ # replace escaped quotes
+ if self.escQuote:
+ ret = ret.replace(self.escQuote, self.endQuoteChar)
+
+ return loc, ret
+
+ def __str__( self ):
+ try:
+ return super(QuotedString,self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)
+
+ return self.strRepr
+
+
+class CharsNotIn(Token):
+ """
+ Token for matching words composed of characters I{not} in a given set (will
+ include whitespace in matched characters if not listed in the provided exclusion set - see example).
+ Defined with string containing all disallowed characters, and an optional
+ minimum, maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction.
+
+ Example::
+ # define a comma-separated-value as anything that is not a ','
+ csv_value = CharsNotIn(',')
+ print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213"))
+ prints::
+ ['dkls', 'lsdkjf', 's12 34', '@!#', '213']
+ """
+ def __init__( self, notChars, min=1, max=0, exact=0 ):
+ super(CharsNotIn,self).__init__()
+ self.skipWhitespace = False
+ self.notChars = notChars
+
+ if min < 1:
+ raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted")
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = ( self.minLen == 0 )
+ self.mayIndexError = False
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if instring[loc] in self.notChars:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ start = loc
+ loc += 1
+ notchars = self.notChars
+ maxlen = min( start+self.maxLen, len(instring) )
+ while loc < maxlen and \
+ (instring[loc] not in notchars):
+ loc += 1
+
+ if loc - start < self.minLen:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+ def __str__( self ):
+ try:
+ return super(CharsNotIn, self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None:
+ if len(self.notChars) > 4:
+ self.strRepr = "!W:(%s...)" % self.notChars[:4]
+ else:
+ self.strRepr = "!W:(%s)" % self.notChars
+
+ return self.strRepr
+
+class White(Token):
+ """
+ Special matching class for matching whitespace. Normally, whitespace is ignored
+ by pyparsing grammars. This class is included when some whitespace structures
+ are significant. Define with a string containing the whitespace characters to be
+ matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments,
+ as defined for the C{L{Word}} class.
+ """
+ whiteStrs = {
+ " " : "<SPC>",
+ "\t": "<TAB>",
+ "\n": "<LF>",
+ "\r": "<CR>",
+ "\f": "<FF>",
+ }
+ def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
+ super(White,self).__init__()
+ self.matchWhite = ws
+ self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) )
+ #~ self.leaveWhitespace()
+ self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite))
+ self.mayReturnEmpty = True
+ self.errmsg = "Expected " + self.name
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if not(instring[ loc ] in self.matchWhite):
+ raise ParseException(instring, loc, self.errmsg, self)
+ start = loc
+ loc += 1
+ maxloc = start + self.maxLen
+ maxloc = min( maxloc, len(instring) )
+ while loc < maxloc and instring[loc] in self.matchWhite:
+ loc += 1
+
+ if loc - start < self.minLen:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+
+class _PositionToken(Token):
+ def __init__( self ):
+ super(_PositionToken,self).__init__()
+ self.name=self.__class__.__name__
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+
+class GoToColumn(_PositionToken):
+ """
+ Token to advance to a specific column of input text; useful for tabular report scraping.
+ """
+ def __init__( self, colno ):
+ super(GoToColumn,self).__init__()
+ self.col = colno
+
+ def preParse( self, instring, loc ):
+ if col(loc,instring) != self.col:
+ instrlen = len(instring)
+ if self.ignoreExprs:
+ loc = self._skipIgnorables( instring, loc )
+ while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
+ loc += 1
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ thiscol = col( loc, instring )
+ if thiscol > self.col:
+ raise ParseException( instring, loc, "Text not in expected column", self )
+ newloc = loc + self.col - thiscol
+ ret = instring[ loc: newloc ]
+ return newloc, ret
+
+
+class LineStart(_PositionToken):
+ """
+ Matches if current position is at the beginning of a line within the parse string
+
+ Example::
+
+ test = '''\
+ AAA this line
+ AAA and this line
+ AAA but not this one
+ B AAA and definitely not this one
+ '''
+
+ for t in (LineStart() + 'AAA' + restOfLine).searchString(test):
+ print(t)
+
+ Prints::
+ ['AAA', ' this line']
+ ['AAA', ' and this line']
+
+ """
+ def __init__( self ):
+ super(LineStart,self).__init__()
+ self.errmsg = "Expected start of line"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if col(loc, instring) == 1:
+ return loc, []
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class LineEnd(_PositionToken):
+ """
+ Matches if current position is at the end of a line within the parse string
+ """
+ def __init__( self ):
+ super(LineEnd,self).__init__()
+ self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
+ self.errmsg = "Expected end of line"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc<len(instring):
+ if instring[loc] == "\n":
+ return loc+1, "\n"
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+ elif loc == len(instring):
+ return loc+1, []
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class StringStart(_PositionToken):
+ """
+ Matches if current position is at the beginning of the parse string
+ """
+ def __init__( self ):
+ super(StringStart,self).__init__()
+ self.errmsg = "Expected start of text"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc != 0:
+ # see if entire string up to here is just whitespace and ignoreables
+ if loc != self.preParse( instring, 0 ):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class StringEnd(_PositionToken):
+ """
+ Matches if current position is at the end of the parse string
+ """
+ def __init__( self ):
+ super(StringEnd,self).__init__()
+ self.errmsg = "Expected end of text"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc < len(instring):
+ raise ParseException(instring, loc, self.errmsg, self)
+ elif loc == len(instring):
+ return loc+1, []
+ elif loc > len(instring):
+ return loc, []
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class WordStart(_PositionToken):
+ """
+ Matches if the current position is at the beginning of a Word, and
+ is not preceded by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of
+ the string being parsed, or at the beginning of a line.
+ """
+ def __init__(self, wordChars = printables):
+ super(WordStart,self).__init__()
+ self.wordChars = set(wordChars)
+ self.errmsg = "Not at the start of a word"
+
+ def parseImpl(self, instring, loc, doActions=True ):
+ if loc != 0:
+ if (instring[loc-1] in self.wordChars or
+ instring[loc] not in self.wordChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class WordEnd(_PositionToken):
+ """
+ Matches if the current position is at the end of a Word, and
+ is not followed by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of
+ the string being parsed, or at the end of a line.
+ """
+ def __init__(self, wordChars = printables):
+ super(WordEnd,self).__init__()
+ self.wordChars = set(wordChars)
+ self.skipWhitespace = False
+ self.errmsg = "Not at the end of a word"
+
+ def parseImpl(self, instring, loc, doActions=True ):
+ instrlen = len(instring)
+ if instrlen>0 and loc<instrlen:
+ if (instring[loc] in self.wordChars or
+ instring[loc-1] not in self.wordChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+
+class ParseExpression(ParserElement):
+ """
+ Abstract subclass of ParserElement, for combining and post-processing parsed tokens.
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(ParseExpression,self).__init__(savelist)
+ if isinstance( exprs, _generatorType ):
+ exprs = list(exprs)
+
+ if isinstance( exprs, basestring ):
+ self.exprs = [ ParserElement._literalStringClass( exprs ) ]
+ elif isinstance( exprs, Iterable ):
+ exprs = list(exprs)
+ # if sequence of strings provided, wrap with Literal
+ if all(isinstance(expr, basestring) for expr in exprs):
+ exprs = map(ParserElement._literalStringClass, exprs)
+ self.exprs = list(exprs)
+ else:
+ try:
+ self.exprs = list( exprs )
+ except TypeError:
+ self.exprs = [ exprs ]
+ self.callPreparse = False
+
+ def __getitem__( self, i ):
+ return self.exprs[i]
+
+ def append( self, other ):
+ self.exprs.append( other )
+ self.strRepr = None
+ return self
+
+ def leaveWhitespace( self ):
+ """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on
+ all contained expressions."""
+ self.skipWhitespace = False
+ self.exprs = [ e.copy() for e in self.exprs ]
+ for e in self.exprs:
+ e.leaveWhitespace()
+ return self
+
+ def ignore( self, other ):
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ super( ParseExpression, self).ignore( other )
+ for e in self.exprs:
+ e.ignore( self.ignoreExprs[-1] )
+ else:
+ super( ParseExpression, self).ignore( other )
+ for e in self.exprs:
+ e.ignore( self.ignoreExprs[-1] )
+ return self
+
+ def __str__( self ):
+ try:
+ return super(ParseExpression,self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) )
+ return self.strRepr
+
+ def streamline( self ):
+ super(ParseExpression,self).streamline()
+
+ for e in self.exprs:
+ e.streamline()
+
+ # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d )
+ # but only if there are no parse actions or resultsNames on the nested And's
+ # (likewise for Or's and MatchFirst's)
+ if ( len(self.exprs) == 2 ):
+ other = self.exprs[0]
+ if ( isinstance( other, self.__class__ ) and
+ not(other.parseAction) and
+ other.resultsName is None and
+ not other.debug ):
+ self.exprs = other.exprs[:] + [ self.exprs[1] ]
+ self.strRepr = None
+ self.mayReturnEmpty |= other.mayReturnEmpty
+ self.mayIndexError |= other.mayIndexError
+
+ other = self.exprs[-1]
+ if ( isinstance( other, self.__class__ ) and
+ not(other.parseAction) and
+ other.resultsName is None and
+ not other.debug ):
+ self.exprs = self.exprs[:-1] + other.exprs[:]
+ self.strRepr = None
+ self.mayReturnEmpty |= other.mayReturnEmpty
+ self.mayIndexError |= other.mayIndexError
+
+ self.errmsg = "Expected " + _ustr(self)
+
+ return self
+
+ def setResultsName( self, name, listAllMatches=False ):
+ ret = super(ParseExpression,self).setResultsName(name,listAllMatches)
+ return ret
+
+ def validate( self, validateTrace=[] ):
+ tmp = validateTrace[:]+[self]
+ for e in self.exprs:
+ e.validate(tmp)
+ self.checkRecursion( [] )
+
+ def copy(self):
+ ret = super(ParseExpression,self).copy()
+ ret.exprs = [e.copy() for e in self.exprs]
+ return ret
+
+class And(ParseExpression):
+ """
+ Requires all given C{ParseExpression}s to be found in the given order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'+'} operator.
+ May also be constructed using the C{'-'} operator, which will suppress backtracking.
+
+ Example::
+ integer = Word(nums)
+ name_expr = OneOrMore(Word(alphas))
+
+ expr = And([integer("id"),name_expr("name"),integer("age")])
+ # more easily written as:
+ expr = integer("id") + name_expr("name") + integer("age")
+ """
+
+ class _ErrorStop(Empty):
+ def __init__(self, *args, **kwargs):
+ super(And._ErrorStop,self).__init__(*args, **kwargs)
+ self.name = '-'
+ self.leaveWhitespace()
+
+ def __init__( self, exprs, savelist = True ):
+ super(And,self).__init__(exprs, savelist)
+ self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+ self.setWhitespaceChars( self.exprs[0].whiteChars )
+ self.skipWhitespace = self.exprs[0].skipWhitespace
+ self.callPreparse = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ # pass False as last arg to _parse for first element, since we already
+ # pre-parsed the string as part of our And pre-parsing
+ loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False )
+ errorStop = False
+ for e in self.exprs[1:]:
+ if isinstance(e, And._ErrorStop):
+ errorStop = True
+ continue
+ if errorStop:
+ try:
+ loc, exprtokens = e._parse( instring, loc, doActions )
+ except ParseSyntaxException:
+ raise
+ except ParseBaseException as pe:
+ pe.__traceback__ = None
+ raise ParseSyntaxException._from_exception(pe)
+ except IndexError:
+ raise ParseSyntaxException(instring, len(instring), self.errmsg, self)
+ else:
+ loc, exprtokens = e._parse( instring, loc, doActions )
+ if exprtokens or exprtokens.haskeys():
+ resultlist += exprtokens
+ return loc, resultlist
+
+ def __iadd__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ return self.append( other ) #And( [ self, other ] )
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+ if not e.mayReturnEmpty:
+ break
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+
+class Or(ParseExpression):
+ """
+ Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the expression that matches the longest string will be used.
+ May be constructed using the C{'^'} operator.
+
+ Example::
+ # construct Or using '^' operator
+
+ number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums))
+ print(number.searchString("123 3.1416 789"))
+ prints::
+ [['123'], ['3.1416'], ['789']]
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(Or,self).__init__(exprs, savelist)
+ if self.exprs:
+ self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+ else:
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ maxExcLoc = -1
+ maxException = None
+ matches = []
+ for e in self.exprs:
+ try:
+ loc2 = e.tryParse( instring, loc )
+ except ParseException as err:
+ err.__traceback__ = None
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+ except IndexError:
+ if len(instring) > maxExcLoc:
+ maxException = ParseException(instring,len(instring),e.errmsg,self)
+ maxExcLoc = len(instring)
+ else:
+ # save match among all matches, to retry longest to shortest
+ matches.append((loc2, e))
+
+ if matches:
+ matches.sort(key=lambda x: -x[0])
+ for _,e in matches:
+ try:
+ return e._parse( instring, loc, doActions )
+ except ParseException as err:
+ err.__traceback__ = None
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+
+ if maxException is not None:
+ maxException.msg = self.errmsg
+ raise maxException
+ else:
+ raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+
+ def __ixor__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ return self.append( other ) #Or( [ self, other ] )
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class MatchFirst(ParseExpression):
+ """
+ Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the first one listed is the one that will match.
+ May be constructed using the C{'|'} operator.
+
+ Example::
+ # construct MatchFirst using '|' operator
+
+ # watch the order of expressions to match
+ number = Word(nums) | Combine(Word(nums) + '.' + Word(nums))
+ print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']]
+
+ # put more selective expression first
+ number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums)
+ print(number.searchString("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']]
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(MatchFirst,self).__init__(exprs, savelist)
+ if self.exprs:
+ self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+ else:
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ maxExcLoc = -1
+ maxException = None
+ for e in self.exprs:
+ try:
+ ret = e._parse( instring, loc, doActions )
+ return ret
+ except ParseException as err:
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+ except IndexError:
+ if len(instring) > maxExcLoc:
+ maxException = ParseException(instring,len(instring),e.errmsg,self)
+ maxExcLoc = len(instring)
+
+ # only got here if no expression matched, raise exception for match that made it the furthest
+ else:
+ if maxException is not None:
+ maxException.msg = self.errmsg
+ raise maxException
+ else:
+ raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+ def __ior__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ return self.append( other ) #MatchFirst( [ self, other ] )
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class Each(ParseExpression):
+ """
+ Requires all given C{ParseExpression}s to be found, but in any order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'&'} operator.
+
+ Example::
+ color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN")
+ shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON")
+ integer = Word(nums)
+ shape_attr = "shape:" + shape_type("shape")
+ posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn")
+ color_attr = "color:" + color("color")
+ size_attr = "size:" + integer("size")
+
+ # use Each (using operator '&') to accept attributes in any order
+ # (shape and posn are required, color and size are optional)
+ shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr)
+
+ shape_spec.runTests('''
+ shape: SQUARE color: BLACK posn: 100, 120
+ shape: CIRCLE size: 50 color: BLUE posn: 50,80
+ color:GREEN size:20 shape:TRIANGLE posn:20,40
+ '''
+ )
+ prints::
+ shape: SQUARE color: BLACK posn: 100, 120
+ ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']]
+ - color: BLACK
+ - posn: ['100', ',', '120']
+ - x: 100
+ - y: 120
+ - shape: SQUARE
+
+
+ shape: CIRCLE size: 50 color: BLUE posn: 50,80
+ ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE', 'posn:', ['50', ',', '80']]
+ - color: BLUE
+ - posn: ['50', ',', '80']
+ - x: 50
+ - y: 80
+ - shape: CIRCLE
+ - size: 50
+
+
+ color: GREEN size: 20 shape: TRIANGLE posn: 20,40
+ ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE', 'posn:', ['20', ',', '40']]
+ - color: GREEN
+ - posn: ['20', ',', '40']
+ - x: 20
+ - y: 40
+ - shape: TRIANGLE
+ - size: 20
+ """
+ def __init__( self, exprs, savelist = True ):
+ super(Each,self).__init__(exprs, savelist)
+ self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+ self.skipWhitespace = True
+ self.initExprGroups = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.initExprGroups:
+ self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional))
+ opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ]
+ opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)]
+ self.optionals = opt1 + opt2
+ self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ]
+ self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ]
+ self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
+ self.required += self.multirequired
+ self.initExprGroups = False
+ tmpLoc = loc
+ tmpReqd = self.required[:]
+ tmpOpt = self.optionals[:]
+ matchOrder = []
+
+ keepMatching = True
+ while keepMatching:
+ tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
+ failed = []
+ for e in tmpExprs:
+ try:
+ tmpLoc = e.tryParse( instring, tmpLoc )
+ except ParseException:
+ failed.append(e)
+ else:
+ matchOrder.append(self.opt1map.get(id(e),e))
+ if e in tmpReqd:
+ tmpReqd.remove(e)
+ elif e in tmpOpt:
+ tmpOpt.remove(e)
+ if len(failed) == len(tmpExprs):
+ keepMatching = False
+
+ if tmpReqd:
+ missing = ", ".join(_ustr(e) for e in tmpReqd)
+ raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
+
+ # add any unmatched Optionals, in case they have default values defined
+ matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt]
+
+ resultlist = []
+ for e in matchOrder:
+ loc,results = e._parse(instring,loc,doActions)
+ resultlist.append(results)
+
+ finalResults = sum(resultlist, ParseResults([]))
+ return loc, finalResults
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class ParseElementEnhance(ParserElement):
+ """
+ Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens.
+ """
+ def __init__( self, expr, savelist=False ):
+ super(ParseElementEnhance,self).__init__(savelist)
+ if isinstance( expr, basestring ):
+ if issubclass(ParserElement._literalStringClass, Token):
+ expr = ParserElement._literalStringClass(expr)
+ else:
+ expr = ParserElement._literalStringClass(Literal(expr))
+ self.expr = expr
+ self.strRepr = None
+ if expr is not None:
+ self.mayIndexError = expr.mayIndexError
+ self.mayReturnEmpty = expr.mayReturnEmpty
+ self.setWhitespaceChars( expr.whiteChars )
+ self.skipWhitespace = expr.skipWhitespace
+ self.saveAsList = expr.saveAsList
+ self.callPreparse = expr.callPreparse
+ self.ignoreExprs.extend(expr.ignoreExprs)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.expr is not None:
+ return self.expr._parse( instring, loc, doActions, callPreParse=False )
+ else:
+ raise ParseException("",loc,self.errmsg,self)
+
+ def leaveWhitespace( self ):
+ self.skipWhitespace = False
+ self.expr = self.expr.copy()
+ if self.expr is not None:
+ self.expr.leaveWhitespace()
+ return self
+
+ def ignore( self, other ):
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ super( ParseElementEnhance, self).ignore( other )
+ if self.expr is not None:
+ self.expr.ignore( self.ignoreExprs[-1] )
+ else:
+ super( ParseElementEnhance, self).ignore( other )
+ if self.expr is not None:
+ self.expr.ignore( self.ignoreExprs[-1] )
+ return self
+
+ def streamline( self ):
+ super(ParseElementEnhance,self).streamline()
+ if self.expr is not None:
+ self.expr.streamline()
+ return self
+
+ def checkRecursion( self, parseElementList ):
+ if self in parseElementList:
+ raise RecursiveGrammarException( parseElementList+[self] )
+ subRecCheckList = parseElementList[:] + [ self ]
+ if self.expr is not None:
+ self.expr.checkRecursion( subRecCheckList )
+
+ def validate( self, validateTrace=[] ):
+ tmp = validateTrace[:]+[self]
+ if self.expr is not None:
+ self.expr.validate(tmp)
+ self.checkRecursion( [] )
+
+ def __str__( self ):
+ try:
+ return super(ParseElementEnhance,self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None and self.expr is not None:
+ self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
+ return self.strRepr
+
+
+class FollowedBy(ParseElementEnhance):
+ """
+ Lookahead matching of the given parse expression. C{FollowedBy}
+ does I{not} advance the parsing position within the input string, it only
+ verifies that the specified parse expression matches at the current
+ position. C{FollowedBy} always returns a null token list.
+
+ Example::
+ # use FollowedBy to match a label only if it is followed by a ':'
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+
+ OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint()
+ prints::
+ [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']]
+ """
+ def __init__( self, expr ):
+ super(FollowedBy,self).__init__(expr)
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ self.expr.tryParse( instring, loc )
+ return loc, []
+
+
+class NotAny(ParseElementEnhance):
+ """
+ Lookahead to disallow matching with the given parse expression. C{NotAny}
+ does I{not} advance the parsing position within the input string, it only
+ verifies that the specified parse expression does I{not} match at the current
+ position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny}
+ always returns a null token list. May be constructed using the '~' operator.
+
+ Example::
+
+ """
+ def __init__( self, expr ):
+ super(NotAny,self).__init__(expr)
+ #~ self.leaveWhitespace()
+ self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
+ self.mayReturnEmpty = True
+ self.errmsg = "Found unwanted token, "+_ustr(self.expr)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.expr.canParseNext(instring, loc):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "~{" + _ustr(self.expr) + "}"
+
+ return self.strRepr
+
+class _MultipleMatch(ParseElementEnhance):
+ def __init__( self, expr, stopOn=None):
+ super(_MultipleMatch, self).__init__(expr)
+ self.saveAsList = True
+ ender = stopOn
+ if isinstance(ender, basestring):
+ ender = ParserElement._literalStringClass(ender)
+ self.not_ender = ~ender if ender is not None else None
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ self_expr_parse = self.expr._parse
+ self_skip_ignorables = self._skipIgnorables
+ check_ender = self.not_ender is not None
+ if check_ender:
+ try_not_ender = self.not_ender.tryParse
+
+ # must be at least one (but first see if we are the stopOn sentinel;
+ # if so, fail)
+ if check_ender:
+ try_not_ender(instring, loc)
+ loc, tokens = self_expr_parse( instring, loc, doActions, callPreParse=False )
+ try:
+ hasIgnoreExprs = (not not self.ignoreExprs)
+ while 1:
+ if check_ender:
+ try_not_ender(instring, loc)
+ if hasIgnoreExprs:
+ preloc = self_skip_ignorables( instring, loc )
+ else:
+ preloc = loc
+ loc, tmptokens = self_expr_parse( instring, preloc, doActions )
+ if tmptokens or tmptokens.haskeys():
+ tokens += tmptokens
+ except (ParseException,IndexError):
+ pass
+
+ return loc, tokens
+
+class OneOrMore(_MultipleMatch):
+ """
+ Repetition of one or more of the given expression.
+
+ Parameters:
+ - expr - expression that must match one or more times
+ - stopOn - (default=C{None}) - expression for a terminating sentinel
+ (only required if the sentinel would ordinarily match the repetition
+ expression)
+
+ Example::
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+ text = "shape: SQUARE posn: upper left color: BLACK"
+ OneOrMore(attr_expr).parseString(text).pprint() # Fail! read 'color' as data instead of next label -> [['shape', 'SQUARE color']]
+
+ # use stopOn attribute for OneOrMore to avoid reading label string as part of the data
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+ OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
+
+ # could also be written as
+ (attr_expr * (1,)).parseString(text).pprint()
+ """
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + _ustr(self.expr) + "}..."
+
+ return self.strRepr
+
+class ZeroOrMore(_MultipleMatch):
+ """
+ Optional repetition of zero or more of the given expression.
+
+ Parameters:
+ - expr - expression that must match zero or more times
+ - stopOn - (default=C{None}) - expression for a terminating sentinel
+ (only required if the sentinel would ordinarily match the repetition
+ expression)
+
+ Example: similar to L{OneOrMore}
+ """
+ def __init__( self, expr, stopOn=None):
+ super(ZeroOrMore,self).__init__(expr, stopOn=stopOn)
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ try:
+ return super(ZeroOrMore, self).parseImpl(instring, loc, doActions)
+ except (ParseException,IndexError):
+ return loc, []
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "[" + _ustr(self.expr) + "]..."
+
+ return self.strRepr
+
+class _NullToken(object):
+ def __bool__(self):
+ return False
+ __nonzero__ = __bool__
+ def __str__(self):
+ return ""
+
+_optionalNotMatched = _NullToken()
+class Optional(ParseElementEnhance):
+ """
+ Optional matching of the given expression.
+
+ Parameters:
+ - expr - expression that must match zero or more times
+ - default (optional) - value to be returned if the optional expression is not found.
+
+ Example::
+ # US postal code can be a 5-digit zip, plus optional 4-digit qualifier
+ zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4)))
+ zip.runTests('''
+ # traditional ZIP code
+ 12345
+
+ # ZIP+4 form
+ 12101-0001
+
+ # invalid ZIP
+ 98765-
+ ''')
+ prints::
+ # traditional ZIP code
+ 12345
+ ['12345']
+
+ # ZIP+4 form
+ 12101-0001
+ ['12101-0001']
+
+ # invalid ZIP
+ 98765-
+ ^
+ FAIL: Expected end of text (at char 5), (line:1, col:6)
+ """
+ def __init__( self, expr, default=_optionalNotMatched ):
+ super(Optional,self).__init__( expr, savelist=False )
+ self.saveAsList = self.expr.saveAsList
+ self.defaultValue = default
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ try:
+ loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+ except (ParseException,IndexError):
+ if self.defaultValue is not _optionalNotMatched:
+ if self.expr.resultsName:
+ tokens = ParseResults([ self.defaultValue ])
+ tokens[self.expr.resultsName] = self.defaultValue
+ else:
+ tokens = [ self.defaultValue ]
+ else:
+ tokens = []
+ return loc, tokens
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "[" + _ustr(self.expr) + "]"
+
+ return self.strRepr
+
+class SkipTo(ParseElementEnhance):
+ """
+ Token for skipping over all undefined text until the matched expression is found.
+
+ Parameters:
+ - expr - target expression marking the end of the data to be skipped
+ - include - (default=C{False}) if True, the target expression is also parsed
+ (the skipped text and target expression are returned as a 2-element list).
+ - ignore - (default=C{None}) used to define grammars (typically quoted strings and
+ comments) that might contain false matches to the target expression
+ - failOn - (default=C{None}) define expressions that are not allowed to be
+ included in the skipped test; if found before the target expression is found,
+ the SkipTo is not a match
+
+ Example::
+ report = '''
+ Outstanding Issues Report - 1 Jan 2000
+
+ # | Severity | Description | Days Open
+ -----+----------+-------------------------------------------+-----------
+ 101 | Critical | Intermittent system crash | 6
+ 94 | Cosmetic | Spelling error on Login ('log|n') | 14
+ 79 | Minor | System slow when running too many reports | 47
+ '''
+ integer = Word(nums)
+ SEP = Suppress('|')
+ # use SkipTo to simply match everything up until the next SEP
+ # - ignore quoted strings, so that a '|' character inside a quoted string does not match
+ # - parse action will call token.strip() for each matched token, i.e., the description body
+ string_data = SkipTo(SEP, ignore=quotedString)
+ string_data.setParseAction(tokenMap(str.strip))
+ ticket_expr = (integer("issue_num") + SEP
+ + string_data("sev") + SEP
+ + string_data("desc") + SEP
+ + integer("days_open"))
+
+ for tkt in ticket_expr.searchString(report):
+ print tkt.dump()
+ prints::
+ ['101', 'Critical', 'Intermittent system crash', '6']
+ - days_open: 6
+ - desc: Intermittent system crash
+ - issue_num: 101
+ - sev: Critical
+ ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14']
+ - days_open: 14
+ - desc: Spelling error on Login ('log|n')
+ - issue_num: 94
+ - sev: Cosmetic
+ ['79', 'Minor', 'System slow when running too many reports', '47']
+ - days_open: 47
+ - desc: System slow when running too many reports
+ - issue_num: 79
+ - sev: Minor
+ """
+ def __init__( self, other, include=False, ignore=None, failOn=None ):
+ super( SkipTo, self ).__init__( other )
+ self.ignoreExpr = ignore
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+ self.includeMatch = include
+ self.asList = False
+ if isinstance(failOn, basestring):
+ self.failOn = ParserElement._literalStringClass(failOn)
+ else:
+ self.failOn = failOn
+ self.errmsg = "No match found for "+_ustr(self.expr)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ startloc = loc
+ instrlen = len(instring)
+ expr = self.expr
+ expr_parse = self.expr._parse
+ self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None
+ self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None
+
+ tmploc = loc
+ while tmploc <= instrlen:
+ if self_failOn_canParseNext is not None:
+ # break if failOn expression matches
+ if self_failOn_canParseNext(instring, tmploc):
+ break
+
+ if self_ignoreExpr_tryParse is not None:
+ # advance past ignore expressions
+ while 1:
+ try:
+ tmploc = self_ignoreExpr_tryParse(instring, tmploc)
+ except ParseBaseException:
+ break
+
+ try:
+ expr_parse(instring, tmploc, doActions=False, callPreParse=False)
+ except (ParseException, IndexError):
+ # no match, advance loc in string
+ tmploc += 1
+ else:
+ # matched skipto expr, done
+ break
+
+ else:
+ # ran off the end of the input string without matching skipto expr, fail
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ # build up return values
+ loc = tmploc
+ skiptext = instring[startloc:loc]
+ skipresult = ParseResults(skiptext)
+
+ if self.includeMatch:
+ loc, mat = expr_parse(instring,loc,doActions,callPreParse=False)
+ skipresult += mat
+
+ return loc, skipresult
+
+class Forward(ParseElementEnhance):
+ """
+ Forward declaration of an expression to be defined later -
+ used for recursive grammars, such as algebraic infix notation.
+ When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator.
+
+ Note: take care when assigning to C{Forward} not to overlook precedence of operators.
+ Specifically, '|' has a lower precedence than '<<', so that::
+ fwdExpr << a | b | c
+ will actually be evaluated as::
+ (fwdExpr << a) | b | c
+ thereby leaving b and c out as parseable alternatives. It is recommended that you
+ explicitly group the values inserted into the C{Forward}::
+ fwdExpr << (a | b | c)
+ Converting to use the '<<=' operator instead will avoid this problem.
+
+ See L{ParseResults.pprint} for an example of a recursive parser created using
+ C{Forward}.
+ """
+ def __init__( self, other=None ):
+ super(Forward,self).__init__( other, savelist=False )
+
+ def __lshift__( self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass(other)
+ self.expr = other
+ self.strRepr = None
+ self.mayIndexError = self.expr.mayIndexError
+ self.mayReturnEmpty = self.expr.mayReturnEmpty
+ self.setWhitespaceChars( self.expr.whiteChars )
+ self.skipWhitespace = self.expr.skipWhitespace
+ self.saveAsList = self.expr.saveAsList
+ self.ignoreExprs.extend(self.expr.ignoreExprs)
+ return self
+
+ def __ilshift__(self, other):
+ return self << other
+
+ def leaveWhitespace( self ):
+ self.skipWhitespace = False
+ return self
+
+ def streamline( self ):
+ if not self.streamlined:
+ self.streamlined = True
+ if self.expr is not None:
+ self.expr.streamline()
+ return self
+
+ def validate( self, validateTrace=[] ):
+ if self not in validateTrace:
+ tmp = validateTrace[:]+[self]
+ if self.expr is not None:
+ self.expr.validate(tmp)
+ self.checkRecursion([])
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+ return self.__class__.__name__ + ": ..."
+
+ # stubbed out for now - creates awful memory and perf issues
+ self._revertClass = self.__class__
+ self.__class__ = _ForwardNoRecurse
+ try:
+ if self.expr is not None:
+ retString = _ustr(self.expr)
+ else:
+ retString = "None"
+ finally:
+ self.__class__ = self._revertClass
+ return self.__class__.__name__ + ": " + retString
+
+ def copy(self):
+ if self.expr is not None:
+ return super(Forward,self).copy()
+ else:
+ ret = Forward()
+ ret <<= self
+ return ret
+
+class _ForwardNoRecurse(Forward):
+ def __str__( self ):
+ return "..."
+
+class TokenConverter(ParseElementEnhance):
+ """
+ Abstract subclass of C{ParseExpression}, for converting parsed results.
+ """
+ def __init__( self, expr, savelist=False ):
+ super(TokenConverter,self).__init__( expr )#, savelist )
+ self.saveAsList = False
+
+class Combine(TokenConverter):
+ """
+ Converter to concatenate all matching tokens to a single string.
+ By default, the matching patterns must also be contiguous in the input string;
+ this can be disabled by specifying C{'adjacent=False'} in the constructor.
+
+ Example::
+ real = Word(nums) + '.' + Word(nums)
+ print(real.parseString('3.1416')) # -> ['3', '.', '1416']
+ # will also erroneously match the following
+ print(real.parseString('3. 1416')) # -> ['3', '.', '1416']
+
+ real = Combine(Word(nums) + '.' + Word(nums))
+ print(real.parseString('3.1416')) # -> ['3.1416']
+ # no match when there are internal spaces
+ print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...)
+ """
+ def __init__( self, expr, joinString="", adjacent=True ):
+ super(Combine,self).__init__( expr )
+ # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
+ if adjacent:
+ self.leaveWhitespace()
+ self.adjacent = adjacent
+ self.skipWhitespace = True
+ self.joinString = joinString
+ self.callPreparse = True
+
+ def ignore( self, other ):
+ if self.adjacent:
+ ParserElement.ignore(self, other)
+ else:
+ super( Combine, self).ignore( other )
+ return self
+
+ def postParse( self, instring, loc, tokenlist ):
+ retToks = tokenlist.copy()
+ del retToks[:]
+ retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
+
+ if self.resultsName and retToks.haskeys():
+ return [ retToks ]
+ else:
+ return retToks
+
+class Group(TokenConverter):
+ """
+ Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions.
+
+ Example::
+ ident = Word(alphas)
+ num = Word(nums)
+ term = ident | num
+ func = ident + Optional(delimitedList(term))
+ print(func.parseString("fn a,b,100")) # -> ['fn', 'a', 'b', '100']
+
+ func = ident + Group(Optional(delimitedList(term)))
+ print(func.parseString("fn a,b,100")) # -> ['fn', ['a', 'b', '100']]
+ """
+ def __init__( self, expr ):
+ super(Group,self).__init__( expr )
+ self.saveAsList = True
+
+ def postParse( self, instring, loc, tokenlist ):
+ return [ tokenlist ]
+
+class Dict(TokenConverter):
+ """
+ Converter to return a repetitive expression as a list, but also as a dictionary.
+ Each element can also be referenced using the first token in the expression as its key.
+ Useful for tabular report scraping when the first column can be used as a item key.
+
+ Example::
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+ text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+ attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+
+ # print attributes as plain groups
+ print(OneOrMore(attr_expr).parseString(text).dump())
+
+ # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names
+ result = Dict(OneOrMore(Group(attr_expr))).parseString(text)
+ print(result.dump())
+
+ # access named fields as dict entries, or output as dict
+ print(result['shape'])
+ print(result.asDict())
+ prints::
+ ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap']
+
+ [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+ - color: light blue
+ - posn: upper left
+ - shape: SQUARE
+ - texture: burlap
+ SQUARE
+ {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'}
+ See more examples at L{ParseResults} of accessing fields by results name.
+ """
+ def __init__( self, expr ):
+ super(Dict,self).__init__( expr )
+ self.saveAsList = True
+
+ def postParse( self, instring, loc, tokenlist ):
+ for i,tok in enumerate(tokenlist):
+ if len(tok) == 0:
+ continue
+ ikey = tok[0]
+ if isinstance(ikey,int):
+ ikey = _ustr(tok[0]).strip()
+ if len(tok)==1:
+ tokenlist[ikey] = _ParseResultsWithOffset("",i)
+ elif len(tok)==2 and not isinstance(tok[1],ParseResults):
+ tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
+ else:
+ dictvalue = tok.copy() #ParseResults(i)
+ del dictvalue[0]
+ if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()):
+ tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
+ else:
+ tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
+
+ if self.resultsName:
+ return [ tokenlist ]
+ else:
+ return tokenlist
+
+
+class Suppress(TokenConverter):
+ """
+ Converter for ignoring the results of a parsed expression.
+
+ Example::
+ source = "a, b, c,d"
+ wd = Word(alphas)
+ wd_list1 = wd + ZeroOrMore(',' + wd)
+ print(wd_list1.parseString(source))
+
+ # often, delimiters that are useful during parsing are just in the
+ # way afterward - use Suppress to keep them out of the parsed output
+ wd_list2 = wd + ZeroOrMore(Suppress(',') + wd)
+ print(wd_list2.parseString(source))
+ prints::
+ ['a', ',', 'b', ',', 'c', ',', 'd']
+ ['a', 'b', 'c', 'd']
+ (See also L{delimitedList}.)
+ """
+ def postParse( self, instring, loc, tokenlist ):
+ return []
+
+ def suppress( self ):
+ return self
+
+
+class OnlyOnce(object):
+ """
+ Wrapper for parse actions, to ensure they are only called once.
+ """
+ def __init__(self, methodCall):
+ self.callable = _trim_arity(methodCall)
+ self.called = False
+ def __call__(self,s,l,t):
+ if not self.called:
+ results = self.callable(s,l,t)
+ self.called = True
+ return results
+ raise ParseException(s,l,"")
+ def reset(self):
+ self.called = False
+
+def traceParseAction(f):
+ """
+ Decorator for debugging parse actions.
+
+ When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".}
+ When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised.
+
+ Example::
+ wd = Word(alphas)
+
+ @traceParseAction
+ def remove_duplicate_chars(tokens):
+ return ''.join(sorted(set(''.join(tokens))))
+
+ wds = OneOrMore(wd).setParseAction(remove_duplicate_chars)
+ print(wds.parseString("slkdjs sld sldd sdlf sdljf"))
+ prints::
+ >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {}))
+ <<leaving remove_duplicate_chars (ret: 'dfjkls')
+ ['dfjkls']
+ """
+ f = _trim_arity(f)
+ def z(*paArgs):
+ thisFunc = f.__name__
+ s,l,t = paArgs[-3:]
+ if len(paArgs)>3:
+ thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
+ sys.stderr.write( ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc,line(l,s),l,t) )
+ try:
+ ret = f(*paArgs)
+ except Exception as exc:
+ sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
+ raise
+ sys.stderr.write( "<<leaving %s (ret: %r)\n" % (thisFunc,ret) )
+ return ret
+ try:
+ z.__name__ = f.__name__
+ except AttributeError:
+ pass
+ return z
+
+#
+# global helpers
+#
+def delimitedList( expr, delim=",", combine=False ):
+ """
+ Helper to define a delimited list of expressions - the delimiter defaults to ','.
+ By default, the list elements and delimiters can have intervening whitespace, and
+ comments, but this can be overridden by passing C{combine=True} in the constructor.
+ If C{combine} is set to C{True}, the matching tokens are returned as a single token
+ string, with the delimiters included; otherwise, the matching tokens are returned
+ as a list of tokens, with the delimiters suppressed.
+
+ Example::
+ delimitedList(Word(alphas)).parseString("aa,bb,cc") # -> ['aa', 'bb', 'cc']
+ delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE']
+ """
+ dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
+ if combine:
+ return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
+ else:
+ return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
+
+def countedArray( expr, intExpr=None ):
+ """
+ Helper to define a counted list of expressions.
+ This helper defines a pattern of the form::
+ integer expr expr expr...
+ where the leading integer tells how many expr expressions follow.
+ The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
+
+ If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value.
+
+ Example::
+ countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd']
+
+ # in this parser, the leading integer value is given in binary,
+ # '10' indicating that 2 values are in the array
+ binaryConstant = Word('01').setParseAction(lambda t: int(t[0], 2))
+ countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef') # -> ['ab', 'cd']
+ """
+ arrayExpr = Forward()
+ def countFieldParseAction(s,l,t):
+ n = t[0]
+ arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
+ return []
+ if intExpr is None:
+ intExpr = Word(nums).setParseAction(lambda t:int(t[0]))
+ else:
+ intExpr = intExpr.copy()
+ intExpr.setName("arrayLen")
+ intExpr.addParseAction(countFieldParseAction, callDuringTry=True)
+ return ( intExpr + arrayExpr ).setName('(len) ' + _ustr(expr) + '...')
+
+def _flatten(L):
+ ret = []
+ for i in L:
+ if isinstance(i,list):
+ ret.extend(_flatten(i))
+ else:
+ ret.append(i)
+ return ret
+
+def matchPreviousLiteral(expr):
+ """
+ Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousLiteral(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches a
+ previous literal, will also match the leading C{"1:1"} in C{"1:10"}.
+ If this is not desired, use C{matchPreviousExpr}.
+ Do I{not} use with packrat parsing enabled.
+ """
+ rep = Forward()
+ def copyTokenToRepeater(s,l,t):
+ if t:
+ if len(t) == 1:
+ rep << t[0]
+ else:
+ # flatten t tokens
+ tflat = _flatten(t.asList())
+ rep << And(Literal(tt) for tt in tflat)
+ else:
+ rep << Empty()
+ expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ rep.setName('(prev) ' + _ustr(expr))
+ return rep
+
+def matchPreviousExpr(expr):
+ """
+ Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousExpr(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches by
+ expressions, will I{not} match the leading C{"1:1"} in C{"1:10"};
+ the expressions are evaluated first, and then compared, so
+ C{"1"} is compared with C{"10"}.
+ Do I{not} use with packrat parsing enabled.
+ """
+ rep = Forward()
+ e2 = expr.copy()
+ rep <<= e2
+ def copyTokenToRepeater(s,l,t):
+ matchTokens = _flatten(t.asList())
+ def mustMatchTheseTokens(s,l,t):
+ theseTokens = _flatten(t.asList())
+ if theseTokens != matchTokens:
+ raise ParseException("",0,"")
+ rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
+ expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ rep.setName('(prev) ' + _ustr(expr))
+ return rep
+
+def _escapeRegexRangeChars(s):
+ #~ escape these chars: ^-]
+ for c in r"\^-]":
+ s = s.replace(c,_bslash+c)
+ s = s.replace("\n",r"\n")
+ s = s.replace("\t",r"\t")
+ return _ustr(s)
+
+def oneOf( strs, caseless=False, useRegex=True ):
+ """
+ Helper to quickly define a set of alternative Literals, and makes sure to do
+ longest-first testing when there is a conflict, regardless of the input order,
+ but returns a C{L{MatchFirst}} for best performance.
+
+ Parameters:
+ - strs - a string of space-delimited literals, or a collection of string literals
+ - caseless - (default=C{False}) - treat all literals as caseless
+ - useRegex - (default=C{True}) - as an optimization, will generate a Regex
+ object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or
+ if creating a C{Regex} raises an exception)
+
+ Example::
+ comp_oper = oneOf("< = > <= >= !=")
+ var = Word(alphas)
+ number = Word(nums)
+ term = var | number
+ comparison_expr = term + comp_oper + term
+ print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12"))
+ prints::
+ [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']]
+ """
+ if caseless:
+ isequal = ( lambda a,b: a.upper() == b.upper() )
+ masks = ( lambda a,b: b.upper().startswith(a.upper()) )
+ parseElementClass = CaselessLiteral
+ else:
+ isequal = ( lambda a,b: a == b )
+ masks = ( lambda a,b: b.startswith(a) )
+ parseElementClass = Literal
+
+ symbols = []
+ if isinstance(strs,basestring):
+ symbols = strs.split()
+ elif isinstance(strs, Iterable):
+ symbols = list(strs)
+ else:
+ warnings.warn("Invalid argument to oneOf, expected string or iterable",
+ SyntaxWarning, stacklevel=2)
+ if not symbols:
+ return NoMatch()
+
+ i = 0
+ while i < len(symbols)-1:
+ cur = symbols[i]
+ for j,other in enumerate(symbols[i+1:]):
+ if ( isequal(other, cur) ):
+ del symbols[i+j+1]
+ break
+ elif ( masks(cur, other) ):
+ del symbols[i+j+1]
+ symbols.insert(i,other)
+ cur = other
+ break
+ else:
+ i += 1
+
+ if not caseless and useRegex:
+ #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
+ try:
+ if len(symbols)==len("".join(symbols)):
+ return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) ).setName(' | '.join(symbols))
+ else:
+ return Regex( "|".join(re.escape(sym) for sym in symbols) ).setName(' | '.join(symbols))
+ except Exception:
+ warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
+ SyntaxWarning, stacklevel=2)
+
+
+ # last resort, just use MatchFirst
+ return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols))
+
+def dictOf( key, value ):
+ """
+ Helper to easily and clearly define a dictionary by specifying the respective patterns
+ for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
+ in the proper order. The key pattern can include delimiting markers or punctuation,
+ as long as they are suppressed, thereby leaving the significant key text. The value
+ pattern can include named results, so that the C{Dict} results can include named token
+ fields.
+
+ Example::
+ text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+ attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+ print(OneOrMore(attr_expr).parseString(text).dump())
+
+ attr_label = label
+ attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)
+
+ # similar to Dict, but simpler call format
+ result = dictOf(attr_label, attr_value).parseString(text)
+ print(result.dump())
+ print(result['shape'])
+ print(result.shape) # object attribute access works too
+ print(result.asDict())
+ prints::
+ [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+ - color: light blue
+ - posn: upper left
+ - shape: SQUARE
+ - texture: burlap
+ SQUARE
+ SQUARE
+ {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'}
+ """
+ return Dict( ZeroOrMore( Group ( key + value ) ) )
+
+def originalTextFor(expr, asString=True):
+ """
+ Helper to return the original, untokenized text for a given expression. Useful to
+ restore the parsed fields of an HTML start tag into the raw tag text itself, or to
+ revert separate tokens with intervening whitespace back to the original matching
+ input text. By default, returns astring containing the original parsed text.
+
+ If the optional C{asString} argument is passed as C{False}, then the return value is a
+ C{L{ParseResults}} containing any results names that were originally matched, and a
+ single token containing the original matched text from the input string. So if
+ the expression passed to C{L{originalTextFor}} contains expressions with defined
+ results names, you must set C{asString} to C{False} if you want to preserve those
+ results name values.
+
+ Example::
+ src = "this is test <b> bold <i>text</i> </b> normal text "
+ for tag in ("b","i"):
+ opener,closer = makeHTMLTags(tag)
+ patt = originalTextFor(opener + SkipTo(closer) + closer)
+ print(patt.searchString(src)[0])
+ prints::
+ ['<b> bold <i>text</i> </b>']
+ ['<i>text</i>']
+ """
+ locMarker = Empty().setParseAction(lambda s,loc,t: loc)
+ endlocMarker = locMarker.copy()
+ endlocMarker.callPreparse = False
+ matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end")
+ if asString:
+ extractText = lambda s,l,t: s[t._original_start:t._original_end]
+ else:
+ def extractText(s,l,t):
+ t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]]
+ matchExpr.setParseAction(extractText)
+ matchExpr.ignoreExprs = expr.ignoreExprs
+ return matchExpr
+
+def ungroup(expr):
+ """
+ Helper to undo pyparsing's default grouping of And expressions, even
+ if all but one are non-empty.
+ """
+ return TokenConverter(expr).setParseAction(lambda t:t[0])
+
+def locatedExpr(expr):
+ """
+ Helper to decorate a returned token with its starting and ending locations in the input string.
+ This helper adds the following results names:
+ - locn_start = location where matched expression begins
+ - locn_end = location where matched expression ends
+ - value = the actual parsed results
+
+ Be careful if the input text contains C{<TAB>} characters, you may want to call
+ C{L{ParserElement.parseWithTabs}}
+
+ Example::
+ wd = Word(alphas)
+ for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"):
+ print(match)
+ prints::
+ [[0, 'ljsdf', 5]]
+ [[8, 'lksdjjf', 15]]
+ [[18, 'lkkjj', 23]]
+ """
+ locator = Empty().setParseAction(lambda s,l,t: l)
+ return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end"))
+
+
+# convenience constants for positional expressions
+empty = Empty().setName("empty")
+lineStart = LineStart().setName("lineStart")
+lineEnd = LineEnd().setName("lineEnd")
+stringStart = StringStart().setName("stringStart")
+stringEnd = StringEnd().setName("stringEnd")
+
+_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
+_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
+_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1)
+_charRange = Group(_singleChar + Suppress("-") + _singleChar)
+_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
+
+def srange(s):
+ r"""
+ Helper to easily define string ranges for use in Word construction. Borrows
+ syntax from regexp '[]' string range definitions::
+ srange("[0-9]") -> "0123456789"
+ srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
+ srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
+ The input string must be enclosed in []'s, and the returned string is the expanded
+ character set joined into a single string.
+ The values enclosed in the []'s may be:
+ - a single character
+ - an escaped character with a leading backslash (such as C{\-} or C{\]})
+ - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character)
+ (C{\0x##} is also supported for backwards compatibility)
+ - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character)
+ - a range of any of the above, separated by a dash (C{'a-z'}, etc.)
+ - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.)
+ """
+ _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1))
+ try:
+ return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body)
+ except Exception:
+ return ""
+
+def matchOnlyAtCol(n):
+ """
+ Helper method for defining parse actions that require matching at a specific
+ column in the input text.
+ """
+ def verifyCol(strg,locn,toks):
+ if col(locn,strg) != n:
+ raise ParseException(strg,locn,"matched token not at column %d" % n)
+ return verifyCol
+
+def replaceWith(replStr):
+ """
+ Helper method for common parse actions that simply return a literal value. Especially
+ useful when used with C{L{transformString<ParserElement.transformString>}()}.
+
+ Example::
+ num = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ na = oneOf("N/A NA").setParseAction(replaceWith(math.nan))
+ term = na | num
+
+ OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234]
+ """
+ return lambda s,l,t: [replStr]
+
+def removeQuotes(s,l,t):
+ """
+ Helper parse action for removing quotation marks from parsed quoted strings.
+
+ Example::
+ # by default, quotation marks are included in parsed results
+ quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"]
+
+ # use removeQuotes to strip quotation marks from parsed results
+ quotedString.setParseAction(removeQuotes)
+ quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["Now is the Winter of our Discontent"]
+ """
+ return t[0][1:-1]
+
+def tokenMap(func, *args):
+ """
+ Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional
+ args are passed, they are forwarded to the given function as additional arguments after
+ the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the
+ parsed data to an integer using base 16.
+
+ Example (compare the last to example in L{ParserElement.transformString}::
+ hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16))
+ hex_ints.runTests('''
+ 00 11 22 aa FF 0a 0d 1a
+ ''')
+
+ upperword = Word(alphas).setParseAction(tokenMap(str.upper))
+ OneOrMore(upperword).runTests('''
+ my kingdom for a horse
+ ''')
+
+ wd = Word(alphas).setParseAction(tokenMap(str.title))
+ OneOrMore(wd).setParseAction(' '.join).runTests('''
+ now is the winter of our discontent made glorious summer by this sun of york
+ ''')
+ prints::
+ 00 11 22 aa FF 0a 0d 1a
+ [0, 17, 34, 170, 255, 10, 13, 26]
+
+ my kingdom for a horse
+ ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE']
+
+ now is the winter of our discontent made glorious summer by this sun of york
+ ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York']
+ """
+ def pa(s,l,t):
+ return [func(tokn, *args) for tokn in t]
+
+ try:
+ func_name = getattr(func, '__name__',
+ getattr(func, '__class__').__name__)
+ except Exception:
+ func_name = str(func)
+ pa.__name__ = func_name
+
+ return pa
+
+upcaseTokens = tokenMap(lambda t: _ustr(t).upper())
+"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}"""
+
+downcaseTokens = tokenMap(lambda t: _ustr(t).lower())
+"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}"""
+
+def _makeTags(tagStr, xml):
+ """Internal helper to construct opening and closing tag expressions, given a tag name"""
+ if isinstance(tagStr,basestring):
+ resname = tagStr
+ tagStr = Keyword(tagStr, caseless=not xml)
+ else:
+ resname = tagStr.name
+
+ tagAttrName = Word(alphas,alphanums+"_-:")
+ if (xml):
+ tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
+ openTag = Suppress("<") + tagStr("tag") + \
+ Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
+ Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+ else:
+ printablesLessRAbrack = "".join(c for c in printables if c not in ">")
+ tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
+ openTag = Suppress("<") + tagStr("tag") + \
+ Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
+ Optional( Suppress("=") + tagAttrValue ) ))) + \
+ Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+ closeTag = Combine(_L("</") + tagStr + ">")
+
+ openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % resname)
+ closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % resname)
+ openTag.tag = resname
+ closeTag.tag = resname
+ return openTag, closeTag
+
+def makeHTMLTags(tagStr):
+ """
+ Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches
+ tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values.
+
+ Example::
+ text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>'
+ # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple
+ a,a_end = makeHTMLTags("A")
+ link_expr = a + SkipTo(a_end)("link_text") + a_end
+
+ for link in link_expr.searchString(text):
+ # attributes in the <A> tag (like "href" shown here) are also accessible as named results
+ print(link.link_text, '->', link.href)
+ prints::
+ pyparsing -> http://pyparsing.wikispaces.com
+ """
+ return _makeTags( tagStr, False )
+
+def makeXMLTags(tagStr):
+ """
+ Helper to construct opening and closing tag expressions for XML, given a tag name. Matches
+ tags only in the given upper/lower case.
+
+ Example: similar to L{makeHTMLTags}
+ """
+ return _makeTags( tagStr, True )
+
+def withAttribute(*args,**attrDict):
+ """
+ Helper to create a validating parse action to be used with start tags created
+ with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
+ with a required attribute value, to avoid false matches on common tags such as
+ C{<TD>} or C{<DIV>}.
+
+ Call C{withAttribute} with a series of attribute names and values. Specify the list
+ of filter attributes names and values as:
+ - keyword arguments, as in C{(align="right")}, or
+ - as an explicit dict with C{**} operator, when an attribute name is also a Python
+ reserved word, as in C{**{"class":"Customer", "align":"right"}}
+ - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
+ For attribute names with a namespace prefix, you must use the second form. Attribute
+ names are matched insensitive to upper/lower case.
+
+ If just testing for C{class} (with or without a namespace), use C{L{withClass}}.
+
+ To verify that the attribute exists, but without specifying a value, pass
+ C{withAttribute.ANY_VALUE} as the value.
+
+ Example::
+ html = '''
+ <div>
+ Some text
+ <div type="grid">1 4 0 1 0</div>
+ <div type="graph">1,3 2,3 1,1</div>
+ <div>this has no type</div>
+ </div>
+
+ '''
+ div,div_end = makeHTMLTags("div")
+
+ # only match div tag having a type attribute with value "grid"
+ div_grid = div().setParseAction(withAttribute(type="grid"))
+ grid_expr = div_grid + SkipTo(div | div_end)("body")
+ for grid_header in grid_expr.searchString(html):
+ print(grid_header.body)
+
+ # construct a match with any div tag having a type attribute, regardless of the value
+ div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE))
+ div_expr = div_any_type + SkipTo(div | div_end)("body")
+ for div_header in div_expr.searchString(html):
+ print(div_header.body)
+ prints::
+ 1 4 0 1 0
+
+ 1 4 0 1 0
+ 1,3 2,3 1,1
+ """
+ if args:
+ attrs = args[:]
+ else:
+ attrs = attrDict.items()
+ attrs = [(k,v) for k,v in attrs]
+ def pa(s,l,tokens):
+ for attrName,attrValue in attrs:
+ if attrName not in tokens:
+ raise ParseException(s,l,"no matching attribute " + attrName)
+ if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
+ raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
+ (attrName, tokens[attrName], attrValue))
+ return pa
+withAttribute.ANY_VALUE = object()
+
+def withClass(classname, namespace=''):
+ """
+ Simplified version of C{L{withAttribute}} when matching on a div class - made
+ difficult because C{class} is a reserved word in Python.
+
+ Example::
+ html = '''
+ <div>
+ Some text
+ <div class="grid">1 4 0 1 0</div>
+ <div class="graph">1,3 2,3 1,1</div>
+ <div>this <div> has no class</div>
+ </div>
+
+ '''
+ div,div_end = makeHTMLTags("div")
+ div_grid = div().setParseAction(withClass("grid"))
+
+ grid_expr = div_grid + SkipTo(div | div_end)("body")
+ for grid_header in grid_expr.searchString(html):
+ print(grid_header.body)
+
+ div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE))
+ div_expr = div_any_type + SkipTo(div | div_end)("body")
+ for div_header in div_expr.searchString(html):
+ print(div_header.body)
+ prints::
+ 1 4 0 1 0
+
+ 1 4 0 1 0
+ 1,3 2,3 1,1
+ """
+ classattr = "%s:class" % namespace if namespace else "class"
+ return withAttribute(**{classattr : classname})
+
+opAssoc = _Constants()
+opAssoc.LEFT = object()
+opAssoc.RIGHT = object()
+
+def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ):
+ """
+ Helper method for constructing grammars of expressions made up of
+ operators working in a precedence hierarchy. Operators may be unary or
+ binary, left- or right-associative. Parse actions can also be attached
+ to operator expressions. The generated parser will also recognize the use
+ of parentheses to override operator precedences (see example below).
+
+ Note: if you define a deep operator list, you may see performance issues
+ when using infixNotation. See L{ParserElement.enablePackrat} for a
+ mechanism to potentially improve your parser performance.
+
+ Parameters:
+ - baseExpr - expression representing the most basic element for the nested
+ - opList - list of tuples, one for each operator precedence level in the
+ expression grammar; each tuple is of the form
+ (opExpr, numTerms, rightLeftAssoc, parseAction), where:
+ - opExpr is the pyparsing expression for the operator;
+ may also be a string, which will be converted to a Literal;
+ if numTerms is 3, opExpr is a tuple of two expressions, for the
+ two operators separating the 3 terms
+ - numTerms is the number of terms for this operator (must
+ be 1, 2, or 3)
+ - rightLeftAssoc is the indicator whether the operator is
+ right or left associative, using the pyparsing-defined
+ constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
+ - parseAction is the parse action to be associated with
+ expressions matching this operator expression (the
+ parse action tuple member may be omitted); if the parse action
+ is passed a tuple or list of functions, this is equivalent to
+ calling C{setParseAction(*fn)} (L{ParserElement.setParseAction})
+ - lpar - expression for matching left-parentheses (default=C{Suppress('(')})
+ - rpar - expression for matching right-parentheses (default=C{Suppress(')')})
+
+ Example::
+ # simple example of four-function arithmetic with ints and variable names
+ integer = pyparsing_common.signed_integer
+ varname = pyparsing_common.identifier
+
+ arith_expr = infixNotation(integer | varname,
+ [
+ ('-', 1, opAssoc.RIGHT),
+ (oneOf('* /'), 2, opAssoc.LEFT),
+ (oneOf('+ -'), 2, opAssoc.LEFT),
+ ])
+
+ arith_expr.runTests('''
+ 5+3*6
+ (5+3)*6
+ -2--11
+ ''', fullDump=False)
+ prints::
+ 5+3*6
+ [[5, '+', [3, '*', 6]]]
+
+ (5+3)*6
+ [[[5, '+', 3], '*', 6]]
+
+ -2--11
+ [[['-', 2], '-', ['-', 11]]]
+ """
+ ret = Forward()
+ lastExpr = baseExpr | ( lpar + ret + rpar )
+ for i,operDef in enumerate(opList):
+ opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
+ termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr
+ if arity == 3:
+ if opExpr is None or len(opExpr) != 2:
+ raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
+ opExpr1, opExpr2 = opExpr
+ thisExpr = Forward().setName(termName)
+ if rightLeftAssoc == opAssoc.LEFT:
+ if arity == 1:
+ matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
+ elif arity == 2:
+ if opExpr is not None:
+ matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) )
+ else:
+ matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) )
+ elif arity == 3:
+ matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \
+ Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr )
+ else:
+ raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+ elif rightLeftAssoc == opAssoc.RIGHT:
+ if arity == 1:
+ # try to avoid LR with this extra test
+ if not isinstance(opExpr, Optional):
+ opExpr = Optional(opExpr)
+ matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr )
+ elif arity == 2:
+ if opExpr is not None:
+ matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) )
+ else:
+ matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) )
+ elif arity == 3:
+ matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \
+ Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr )
+ else:
+ raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+ else:
+ raise ValueError("operator must indicate right or left associativity")
+ if pa:
+ if isinstance(pa, (tuple, list)):
+ matchExpr.setParseAction(*pa)
+ else:
+ matchExpr.setParseAction(pa)
+ thisExpr <<= ( matchExpr.setName(termName) | lastExpr )
+ lastExpr = thisExpr
+ ret <<= lastExpr
+ return ret
+
+operatorPrecedence = infixNotation
+"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release."""
+
+dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes")
+sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes")
+quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"'|
+ Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("quotedString using single or double quotes")
+unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal")
+
+def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()):
+ """
+ Helper method for defining nested lists enclosed in opening and closing
+ delimiters ("(" and ")" are the default).
+
+ Parameters:
+ - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression
+ - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression
+ - content - expression for items within the nested lists (default=C{None})
+ - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString})
+
+ If an expression is not provided for the content argument, the nested
+ expression will capture all whitespace-delimited content between delimiters
+ as a list of separate values.
+
+ Use the C{ignoreExpr} argument to define expressions that may contain
+ opening or closing characters that should not be treated as opening
+ or closing characters for nesting, such as quotedString or a comment
+ expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}.
+ The default is L{quotedString}, but if no expressions are to be ignored,
+ then pass C{None} for this argument.
+
+ Example::
+ data_type = oneOf("void int short long char float double")
+ decl_data_type = Combine(data_type + Optional(Word('*')))
+ ident = Word(alphas+'_', alphanums+'_')
+ number = pyparsing_common.number
+ arg = Group(decl_data_type + ident)
+ LPAR,RPAR = map(Suppress, "()")
+
+ code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment))
+
+ c_function = (decl_data_type("type")
+ + ident("name")
+ + LPAR + Optional(delimitedList(arg), [])("args") + RPAR
+ + code_body("body"))
+ c_function.ignore(cStyleComment)
+
+ source_code = '''
+ int is_odd(int x) {
+ return (x%2);
+ }
+
+ int dec_to_hex(char hchar) {
+ if (hchar >= '0' && hchar <= '9') {
+ return (ord(hchar)-ord('0'));
+ } else {
+ return (10+ord(hchar)-ord('A'));
+ }
+ }
+ '''
+ for func in c_function.searchString(source_code):
+ print("%(name)s (%(type)s) args: %(args)s" % func)
+
+ prints::
+ is_odd (int) args: [['int', 'x']]
+ dec_to_hex (int) args: [['char', 'hchar']]
+ """
+ if opener == closer:
+ raise ValueError("opening and closing strings cannot be the same")
+ if content is None:
+ if isinstance(opener,basestring) and isinstance(closer,basestring):
+ if len(opener) == 1 and len(closer)==1:
+ if ignoreExpr is not None:
+ content = (Combine(OneOrMore(~ignoreExpr +
+ CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ if ignoreExpr is not None:
+ content = (Combine(OneOrMore(~ignoreExpr +
+ ~Literal(opener) + ~Literal(closer) +
+ CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) +
+ CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ raise ValueError("opening and closing arguments must be strings if no content expression is given")
+ ret = Forward()
+ if ignoreExpr is not None:
+ ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
+ else:
+ ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) )
+ ret.setName('nested %s%s expression' % (opener,closer))
+ return ret
+
+def indentedBlock(blockStatementExpr, indentStack, indent=True):
+ """
+ Helper method for defining space-delimited indentation blocks, such as
+ those used to define block statements in Python source code.
+
+ Parameters:
+ - blockStatementExpr - expression defining syntax of statement that
+ is repeated within the indented block
+ - indentStack - list created by caller to manage indentation stack
+ (multiple statementWithIndentedBlock expressions within a single grammar
+ should share a common indentStack)
+ - indent - boolean indicating whether block must be indented beyond the
+ the current level; set to False for block of left-most statements
+ (default=C{True})
+
+ A valid block must contain at least one C{blockStatement}.
+
+ Example::
+ data = '''
+ def A(z):
+ A1
+ B = 100
+ G = A2
+ A2
+ A3
+ B
+ def BB(a,b,c):
+ BB1
+ def BBA():
+ bba1
+ bba2
+ bba3
+ C
+ D
+ def spam(x,y):
+ def eggs(z):
+ pass
+ '''
+
+
+ indentStack = [1]
+ stmt = Forward()
+
+ identifier = Word(alphas, alphanums)
+ funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":")
+ func_body = indentedBlock(stmt, indentStack)
+ funcDef = Group( funcDecl + func_body )
+
+ rvalue = Forward()
+ funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")")
+ rvalue << (funcCall | identifier | Word(nums))
+ assignment = Group(identifier + "=" + rvalue)
+ stmt << ( funcDef | assignment | identifier )
+
+ module_body = OneOrMore(stmt)
+
+ parseTree = module_body.parseString(data)
+ parseTree.pprint()
+ prints::
+ [['def',
+ 'A',
+ ['(', 'z', ')'],
+ ':',
+ [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]],
+ 'B',
+ ['def',
+ 'BB',
+ ['(', 'a', 'b', 'c', ')'],
+ ':',
+ [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]],
+ 'C',
+ 'D',
+ ['def',
+ 'spam',
+ ['(', 'x', 'y', ')'],
+ ':',
+ [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]]
+ """
+ def checkPeerIndent(s,l,t):
+ if l >= len(s): return
+ curCol = col(l,s)
+ if curCol != indentStack[-1]:
+ if curCol > indentStack[-1]:
+ raise ParseFatalException(s,l,"illegal nesting")
+ raise ParseException(s,l,"not a peer entry")
+
+ def checkSubIndent(s,l,t):
+ curCol = col(l,s)
+ if curCol > indentStack[-1]:
+ indentStack.append( curCol )
+ else:
+ raise ParseException(s,l,"not a subentry")
+
+ def checkUnindent(s,l,t):
+ if l >= len(s): return
+ curCol = col(l,s)
+ if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):
+ raise ParseException(s,l,"not an unindent")
+ indentStack.pop()
+
+ NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
+ INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT')
+ PEER = Empty().setParseAction(checkPeerIndent).setName('')
+ UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT')
+ if indent:
+ smExpr = Group( Optional(NL) +
+ #~ FollowedBy(blockStatementExpr) +
+ INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)
+ else:
+ smExpr = Group( Optional(NL) +
+ (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
+ blockStatementExpr.ignore(_bslash + LineEnd())
+ return smExpr.setName('indented block')
+
+alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
+punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
+
+anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:").setName('any tag'))
+_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(),'><& "\''))
+commonHTMLEntity = Regex('&(?P<entity>' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity")
+def replaceHTMLEntity(t):
+ """Helper parser action to replace common HTML entities with their special characters"""
+ return _htmlEntityMap.get(t.entity)
+
+# it's easy to get these comment structures wrong - they're very common, so may as well make them available
+cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment")
+"Comment of the form C{/* ... */}"
+
+htmlComment = Regex(r"<!--[\s\S]*?-->").setName("HTML comment")
+"Comment of the form C{<!-- ... -->}"
+
+restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line")
+dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment")
+"Comment of the form C{// ... (to end of line)}"
+
+cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment")
+"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}"
+
+javaStyleComment = cppStyleComment
+"Same as C{L{cppStyleComment}}"
+
+pythonStyleComment = Regex(r"#.*").setName("Python style comment")
+"Comment of the form C{# ... (to end of line)}"
+
+_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') +
+ Optional( Word(" \t") +
+ ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
+commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList")
+"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas.
+ This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}."""
+
+# some other useful expressions - using lower-case class name since we are really using this as a namespace
+class pyparsing_common:
+ """
+ Here are some common low-level expressions that may be useful in jump-starting parser development:
+ - numeric forms (L{integers<integer>}, L{reals<real>}, L{scientific notation<sci_real>})
+ - common L{programming identifiers<identifier>}
+ - network addresses (L{MAC<mac_address>}, L{IPv4<ipv4_address>}, L{IPv6<ipv6_address>})
+ - ISO8601 L{dates<iso8601_date>} and L{datetime<iso8601_datetime>}
+ - L{UUID<uuid>}
+ - L{comma-separated list<comma_separated_list>}
+ Parse actions:
+ - C{L{convertToInteger}}
+ - C{L{convertToFloat}}
+ - C{L{convertToDate}}
+ - C{L{convertToDatetime}}
+ - C{L{stripHTMLTags}}
+ - C{L{upcaseTokens}}
+ - C{L{downcaseTokens}}
+
+ Example::
+ pyparsing_common.number.runTests('''
+ # any int or real number, returned as the appropriate type
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ ''')
+
+ pyparsing_common.fnumber.runTests('''
+ # any int or real number, returned as float
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ ''')
+
+ pyparsing_common.hex_integer.runTests('''
+ # hex numbers
+ 100
+ FF
+ ''')
+
+ pyparsing_common.fraction.runTests('''
+ # fractions
+ 1/2
+ -3/4
+ ''')
+
+ pyparsing_common.mixed_integer.runTests('''
+ # mixed fractions
+ 1
+ 1/2
+ -3/4
+ 1-3/4
+ ''')
+
+ import uuid
+ pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+ pyparsing_common.uuid.runTests('''
+ # uuid
+ 12345678-1234-5678-1234-567812345678
+ ''')
+ prints::
+ # any int or real number, returned as the appropriate type
+ 100
+ [100]
+
+ -100
+ [-100]
+
+ +100
+ [100]
+
+ 3.14159
+ [3.14159]
+
+ 6.02e23
+ [6.02e+23]
+
+ 1e-12
+ [1e-12]
+
+ # any int or real number, returned as float
+ 100
+ [100.0]
+
+ -100
+ [-100.0]
+
+ +100
+ [100.0]
+
+ 3.14159
+ [3.14159]
+
+ 6.02e23
+ [6.02e+23]
+
+ 1e-12
+ [1e-12]
+
+ # hex numbers
+ 100
+ [256]
+
+ FF
+ [255]
+
+ # fractions
+ 1/2
+ [0.5]
+
+ -3/4
+ [-0.75]
+
+ # mixed fractions
+ 1
+ [1]
+
+ 1/2
+ [0.5]
+
+ -3/4
+ [-0.75]
+
+ 1-3/4
+ [1.75]
+
+ # uuid
+ 12345678-1234-5678-1234-567812345678
+ [UUID('12345678-1234-5678-1234-567812345678')]
+ """
+
+ convertToInteger = tokenMap(int)
+ """
+ Parse action for converting parsed integers to Python int
+ """
+
+ convertToFloat = tokenMap(float)
+ """
+ Parse action for converting parsed numbers to Python float
+ """
+
+ integer = Word(nums).setName("integer").setParseAction(convertToInteger)
+ """expression that parses an unsigned integer, returns an int"""
+
+ hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int,16))
+ """expression that parses a hexadecimal integer, returns an int"""
+
+ signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger)
+ """expression that parses an integer with optional leading sign, returns an int"""
+
+ fraction = (signed_integer().setParseAction(convertToFloat) + '/' + signed_integer().setParseAction(convertToFloat)).setName("fraction")
+ """fractional expression of an integer divided by an integer, returns a float"""
+ fraction.addParseAction(lambda t: t[0]/t[-1])
+
+ mixed_integer = (fraction | signed_integer + Optional(Optional('-').suppress() + fraction)).setName("fraction or mixed integer-fraction")
+ """mixed integer of the form 'integer - fraction', with optional leading integer, returns float"""
+ mixed_integer.addParseAction(sum)
+
+ real = Regex(r'[+-]?\d+\.\d*').setName("real number").setParseAction(convertToFloat)
+ """expression that parses a floating point number and returns a float"""
+
+ sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat)
+ """expression that parses a floating point number with optional scientific notation and returns a float"""
+
+ # streamlining this expression makes the docs nicer-looking
+ number = (sci_real | real | signed_integer).streamline()
+ """any numeric expression, returns the corresponding Python type"""
+
+ fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat)
+ """any int or real number, returned as float"""
+
+ identifier = Word(alphas+'_', alphanums+'_').setName("identifier")
+ """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')"""
+
+ ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address")
+ "IPv4 address (C{0.0.0.0 - 255.255.255.255})"
+
+ _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer")
+ _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address")
+ _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part)*(0,6)) + "::" + Optional(_ipv6_part + (':' + _ipv6_part)*(0,6))).setName("short IPv6 address")
+ _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8)
+ _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address")
+ ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address")
+ "IPv6 address (long, short, or mixed form)"
+
+ mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address")
+ "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)"
+
+ @staticmethod
+ def convertToDate(fmt="%Y-%m-%d"):
+ """
+ Helper to create a parse action for converting parsed date string to Python datetime.date
+
+ Params -
+ - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"})
+
+ Example::
+ date_expr = pyparsing_common.iso8601_date.copy()
+ date_expr.setParseAction(pyparsing_common.convertToDate())
+ print(date_expr.parseString("1999-12-31"))
+ prints::
+ [datetime.date(1999, 12, 31)]
+ """
+ def cvt_fn(s,l,t):
+ try:
+ return datetime.strptime(t[0], fmt).date()
+ except ValueError as ve:
+ raise ParseException(s, l, str(ve))
+ return cvt_fn
+
+ @staticmethod
+ def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"):
+ """
+ Helper to create a parse action for converting parsed datetime string to Python datetime.datetime
+
+ Params -
+ - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"})
+
+ Example::
+ dt_expr = pyparsing_common.iso8601_datetime.copy()
+ dt_expr.setParseAction(pyparsing_common.convertToDatetime())
+ print(dt_expr.parseString("1999-12-31T23:59:59.999"))
+ prints::
+ [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)]
+ """
+ def cvt_fn(s,l,t):
+ try:
+ return datetime.strptime(t[0], fmt)
+ except ValueError as ve:
+ raise ParseException(s, l, str(ve))
+ return cvt_fn
+
+ iso8601_date = Regex(r'(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?').setName("ISO8601 date")
+ "ISO8601 date (C{yyyy-mm-dd})"
+
+ iso8601_datetime = Regex(r'(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime")
+ "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}"
+
+ uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID")
+ "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})"
+
+ _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress()
+ @staticmethod
+ def stripHTMLTags(s, l, tokens):
+ """
+ Parse action to remove HTML tags from web page HTML source
+
+ Example::
+ # strip HTML links from normal text
+ text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>'
+ td,td_end = makeHTMLTags("TD")
+ table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end
+
+ print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page'
+ """
+ return pyparsing_common._html_stripper.transformString(tokens[0])
+
+ _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',')
+ + Optional( White(" \t") ) ) ).streamline().setName("commaItem")
+ comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list")
+ """Predefined expression of 1 or more printable words or quoted strings, separated by commas."""
+
+ upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper()))
+ """Parse action to convert tokens to upper case."""
+
+ downcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).lower()))
+ """Parse action to convert tokens to lower case."""
+
+
+if __name__ == "__main__":
+
+ selectToken = CaselessLiteral("select")
+ fromToken = CaselessLiteral("from")
+
+ ident = Word(alphas, alphanums + "_$")
+
+ columnName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+ columnNameList = Group(delimitedList(columnName)).setName("columns")
+ columnSpec = ('*' | columnNameList)
+
+ tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+ tableNameList = Group(delimitedList(tableName)).setName("tables")
+
+ simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables")
+
+ # demo runTests method, including embedded comments in test string
+ simpleSQL.runTests("""
+ # '*' as column list and dotted table name
+ select * from SYS.XYZZY
+
+ # caseless match on "SELECT", and casts back to "select"
+ SELECT * from XYZZY, ABC
+
+ # list of column names, and mixed case SELECT keyword
+ Select AA,BB,CC from Sys.dual
+
+ # multiple tables
+ Select A, B, C from Sys.dual, Table2
+
+ # invalid SELECT keyword - should fail
+ Xelect A, B, C from Sys.dual
+
+ # incomplete command - should fail
+ Select
+
+ # invalid column name - should fail
+ Select ^^^ frox Sys.dual
+
+ """)
+
+ pyparsing_common.number.runTests("""
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ """)
+
+ # any int or real number, returned as float
+ pyparsing_common.fnumber.runTests("""
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ """)
+
+ pyparsing_common.hex_integer.runTests("""
+ 100
+ FF
+ """)
+
+ import uuid
+ pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+ pyparsing_common.uuid.runTests("""
+ 12345678-1234-5678-1234-567812345678
+ """)
diff --git a/setuptools/_vendor/six.py b/setuptools/_vendor/six.py new file mode 100644 index 00000000..190c0239 --- /dev/null +++ b/setuptools/_vendor/six.py @@ -0,0 +1,868 @@ +"""Utilities for writing code that runs on Python 2 and 3""" + +# Copyright (c) 2010-2015 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson <benjamin@python.org>" +__version__ = "1.10.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") + + +if sys.version_info[:2] == (3, 2): + exec_("""def raise_from(value, from_value): + if from_value is None: + raise value + raise value from from_value +""") +elif sys.version_info[:2] > (3, 2): + exec_("""def raise_from(value, from_value): + raise value from from_value +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + def wrapper(f): + f = functools.wraps(wrapped, assigned, updated)(f) + f.__wrapped__ = wrapped + return f + return wrapper +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/setuptools/_vendor/vendored.txt b/setuptools/_vendor/vendored.txt new file mode 100644 index 00000000..7d77863d --- /dev/null +++ b/setuptools/_vendor/vendored.txt @@ -0,0 +1,3 @@ +packaging==16.8 +pyparsing==2.2.1 +six==1.10.0 diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index cc82b3da..81436044 100755..100644 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -8,7 +8,7 @@ import posixpath import contextlib from distutils.errors import DistutilsError -from pkg_resources import ensure_directory, ContextualZipFile +from pkg_resources import ensure_directory __all__ = [ "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", @@ -98,7 +98,7 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): if not zipfile.is_zipfile(filename): raise UnrecognizedFormat("%s is not a zip file" % (filename,)) - with ContextualZipFile(filename) as z: + with zipfile.ZipFile(filename) as z: for info in z.infolist(): name = info.filename diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index 54f2987b..c883d92f 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -61,14 +61,28 @@ class Distribution(setuptools.dist.Distribution): distutils.core.Distribution = orig +def _to_str(s): + """ + Convert a filename to a string (on Python 2, explicitly + a byte string, not Unicode) as distutils checks for the + exact type str. + """ + if sys.version_info[0] == 2 and not isinstance(s, str): + # Assume it's Unicode, as that's what the PEP says + # should be provided. + return s.encode(sys.getfilesystemencoding()) + return s + + def _run_setup(setup_script='setup.py'): # Note that we can reuse our build directory between calls # Correctness comes first, then optimization later __file__ = setup_script + __name__ = '__main__' f = getattr(tokenize, 'open', open)(__file__) code = f.read().replace('\\r\\n', '\\n') f.close() - exec(compile(code, __file__, 'exec')) + exec(compile(code, __file__, 'exec'), locals()) def _fix_config(config_settings): @@ -77,9 +91,8 @@ def _fix_config(config_settings): return config_settings -def _get_build_requires(config_settings): +def _get_build_requires(config_settings, requirements): config_settings = _fix_config(config_settings) - requirements = ['setuptools', 'wheel'] sys.argv = sys.argv[:1] + ['egg_info'] + \ config_settings["--global-option"] @@ -92,27 +105,59 @@ def _get_build_requires(config_settings): return requirements +def _get_immediate_subdirectories(a_dir): + return [name for name in os.listdir(a_dir) + if os.path.isdir(os.path.join(a_dir, name))] + + def get_requires_for_build_wheel(config_settings=None): config_settings = _fix_config(config_settings) - return _get_build_requires(config_settings) + return _get_build_requires(config_settings, requirements=['wheel']) def get_requires_for_build_sdist(config_settings=None): config_settings = _fix_config(config_settings) - return _get_build_requires(config_settings) + return _get_build_requires(config_settings, requirements=[]) def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): - sys.argv = sys.argv[:1] + ['dist_info', '--egg-base', metadata_directory] + sys.argv = sys.argv[:1] + ['dist_info', '--egg-base', _to_str(metadata_directory)] _run_setup() - dist_infos = [f for f in os.listdir(metadata_directory) - if f.endswith('.dist-info')] + dist_info_directory = metadata_directory + while True: + dist_infos = [f for f in os.listdir(dist_info_directory) + if f.endswith('.dist-info')] + + if len(dist_infos) == 0 and \ + len(_get_immediate_subdirectories(dist_info_directory)) == 1: + dist_info_directory = os.path.join( + dist_info_directory, os.listdir(dist_info_directory)[0]) + continue + + assert len(dist_infos) == 1 + break + + # PEP 517 requires that the .dist-info directory be placed in the + # metadata_directory. To comply, we MUST copy the directory to the root + if dist_info_directory != metadata_directory: + shutil.move( + os.path.join(dist_info_directory, dist_infos[0]), + metadata_directory) + shutil.rmtree(dist_info_directory, ignore_errors=True) - assert len(dist_infos) == 1 return dist_infos[0] +def _file_with_extension(directory, extension): + matching = ( + f for f in os.listdir(directory) + if f.endswith(extension) + ) + file, = matching + return file + + def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): config_settings = _fix_config(config_settings) @@ -124,25 +169,15 @@ def build_wheel(wheel_directory, config_settings=None, shutil.rmtree(wheel_directory) shutil.copytree('dist', wheel_directory) - wheels = [f for f in os.listdir(wheel_directory) - if f.endswith('.whl')] - - assert len(wheels) == 1 - return wheels[0] + return _file_with_extension(wheel_directory, '.whl') def build_sdist(sdist_directory, config_settings=None): config_settings = _fix_config(config_settings) sdist_directory = os.path.abspath(sdist_directory) - sys.argv = sys.argv[:1] + ['sdist'] + \ - config_settings["--global-option"] + sys.argv = sys.argv[:1] + ['sdist', '--formats', 'gztar'] + \ + config_settings["--global-option"] + \ + ["--dist-dir", sdist_directory] _run_setup() - if sdist_directory != 'dist': - shutil.rmtree(sdist_directory) - shutil.copytree('dist', sdist_directory) - - sdists = [f for f in os.listdir(sdist_directory) - if f.endswith('.tar.gz')] - assert len(sdists) == 1 - return sdists[0] + return _file_with_extension(sdist_directory, '.tar.gz') diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py index 4fe3bb56..fe619e2e 100644 --- a/setuptools/command/__init__.py +++ b/setuptools/command/__init__.py @@ -2,7 +2,8 @@ __all__ = [ 'alias', 'bdist_egg', 'bdist_rpm', 'build_ext', 'build_py', 'develop', 'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts', 'sdist', 'setopt', 'test', 'install_egg_info', 'install_scripts', - 'register', 'bdist_wininst', 'upload_docs', 'upload', 'build_clib', 'dist_info', + 'register', 'bdist_wininst', 'upload_docs', 'upload', 'build_clib', + 'dist_info', ] from distutils.command.bdist import bdist diff --git a/setuptools/command/alias.py b/setuptools/command/alias.py index 4532b1cc..4532b1cc 100755..100644 --- a/setuptools/command/alias.py +++ b/setuptools/command/alias.py diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 51755d52..9f8df917 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -8,6 +8,7 @@ from distutils import log from types import CodeType import sys import os +import re import textwrap import marshal @@ -38,6 +39,7 @@ def strip_module(filename): filename = filename[:-6] return filename + def sorted_walk(dir): """Do os.walk in a reproducible way, independent of indeterministic filesystem readdir order @@ -47,6 +49,7 @@ def sorted_walk(dir): files.sort() yield base, dirs, files + def write_stub(resource, pyfile): _stub_template = textwrap.dedent(""" def __bootstrap__(): @@ -240,11 +243,28 @@ class bdist_egg(Command): log.info("Removing .py files from temporary directory") for base, dirs, files in walk_egg(self.bdist_dir): for name in files: + path = os.path.join(base, name) + if name.endswith('.py'): - path = os.path.join(base, name) log.debug("Deleting %s", path) os.unlink(path) + if base.endswith('__pycache__'): + path_old = path + + pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc' + m = re.match(pattern, name) + path_new = os.path.join( + base, os.pardir, m.group('name') + '.pyc') + log.info( + "Renaming file from [%s] to [%s]" + % (path_old, path_new)) + try: + os.remove(path_new) + except OSError: + pass + os.rename(path_old, path_new) + def zip_safe(self): safe = getattr(self.distribution, 'zip_safe', None) if safe is not None: @@ -391,10 +411,12 @@ def scan_module(egg_dir, base, name, stubs): return True # Extension module pkg = base[len(egg_dir) + 1:].replace(os.sep, '.') module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0] - if sys.version_info < (3, 3): + if six.PY2: skip = 8 # skip magic & date - else: + elif sys.version_info < (3, 7): skip = 12 # skip magic & date & file size + else: + skip = 16 # skip magic & reserved? & date & file size f = open(filename, 'rb') f.read(skip) code = marshal.load(f) diff --git a/setuptools/command/bdist_rpm.py b/setuptools/command/bdist_rpm.py index 70730927..70730927 100755..100644 --- a/setuptools/command/bdist_rpm.py +++ b/setuptools/command/bdist_rpm.py diff --git a/setuptools/command/bdist_wininst.py b/setuptools/command/bdist_wininst.py index 073de97b..073de97b 100755..100644 --- a/setuptools/command/bdist_wininst.py +++ b/setuptools/command/bdist_wininst.py diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 36f53f0d..60a8a32f 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -15,6 +15,9 @@ from setuptools.extern import six try: # Attempt to use Cython for building extensions, if available from Cython.Distutils.build_ext import build_ext as _build_ext + # Additionally, assert that the compiler module will load + # also. Ref #1229. + __import__('Cython.Compiler.Main') except ImportError: _build_ext = _du_build_ext @@ -109,7 +112,7 @@ class build_ext(_build_ext): and get_abi3_suffix() ) if use_abi3: - so_ext = _get_config_var_837('EXT_SUFFIX') + so_ext = get_config_var('EXT_SUFFIX') filename = filename[:-len(so_ext)] filename = filename + get_abi3_suffix() if isinstance(ext, Library): @@ -316,13 +319,3 @@ else: self.create_static_lib( objects, basename, output_dir, debug, target_lang ) - - -def _get_config_var_837(name): - """ - In https://github.com/pypa/setuptools/pull/837, we discovered - Python 3.3.0 exposes the extension suffix under the name 'SO'. - """ - if sys.version_info < (3, 3, 1): - name = 'SO' - return get_config_var(name) diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index 85b23c60..009e4f93 100755..100644 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -7,11 +7,13 @@ import io from setuptools.extern import six -from pkg_resources import Distribution, PathMetadata, normalize_path +import pkg_resources from setuptools.command.easy_install import easy_install from setuptools import namespaces import setuptools +__metaclass__ = type + class develop(namespaces.DevelopInstaller, easy_install): """Set up package for development""" @@ -63,9 +65,9 @@ class develop(namespaces.DevelopInstaller, easy_install): if self.egg_path is None: self.egg_path = os.path.abspath(ei.egg_base) - target = normalize_path(self.egg_base) - egg_path = normalize_path(os.path.join(self.install_dir, - self.egg_path)) + target = pkg_resources.normalize_path(self.egg_base) + egg_path = pkg_resources.normalize_path( + os.path.join(self.install_dir, self.egg_path)) if egg_path != target: raise DistutilsOptionError( "--egg-path must be a relative path from the install" @@ -73,9 +75,9 @@ class develop(namespaces.DevelopInstaller, easy_install): ) # Make a distribution for the package's source - self.dist = Distribution( + self.dist = pkg_resources.Distribution( target, - PathMetadata(target, os.path.abspath(ei.egg_info)), + pkg_resources.PathMetadata(target, os.path.abspath(ei.egg_info)), project_name=ei.egg_name ) @@ -95,11 +97,14 @@ class develop(namespaces.DevelopInstaller, easy_install): path_to_setup = egg_base.replace(os.sep, '/').rstrip('/') if path_to_setup != os.curdir: path_to_setup = '../' * (path_to_setup.count('/') + 1) - resolved = normalize_path(os.path.join(install_dir, egg_path, path_to_setup)) - if resolved != normalize_path(os.curdir): + resolved = pkg_resources.normalize_path( + os.path.join(install_dir, egg_path, path_to_setup) + ) + if resolved != pkg_resources.normalize_path(os.curdir): raise DistutilsOptionError( "Can't get a consistent path to setup script from" - " installation directory", resolved, normalize_path(os.curdir)) + " installation directory", resolved, + pkg_resources.normalize_path(os.curdir)) return path_to_setup def install_for_development(self): @@ -110,7 +115,7 @@ class develop(namespaces.DevelopInstaller, easy_install): self.reinitialize_command('build_py', inplace=0) self.run_command('build_py') bpy_cmd = self.get_finalized_command("build_py") - build_path = normalize_path(bpy_cmd.build_lib) + build_path = pkg_resources.normalize_path(bpy_cmd.build_lib) # Build extensions self.reinitialize_command('egg_info', egg_base=build_path) @@ -124,7 +129,8 @@ class develop(namespaces.DevelopInstaller, easy_install): self.egg_path = build_path self.dist.location = build_path # XXX - self.dist._provider = PathMetadata(build_path, ei_cmd.egg_info) + self.dist._provider = pkg_resources.PathMetadata( + build_path, ei_cmd.egg_info) else: # Without 2to3 inplace works fine: self.run_command('egg_info') @@ -190,12 +196,13 @@ class develop(namespaces.DevelopInstaller, easy_install): return easy_install.install_wrapper_scripts(self, dist) -class VersionlessRequirement(object): +class VersionlessRequirement: """ Adapt a pkg_resources.Distribution to simply return the project name as the 'requirement' so that scripts will work across multiple versions. + >>> from pkg_resources import Distribution >>> dist = Distribution(project_name='foo', version='1.0') >>> str(dist.as_requirement()) 'foo==1.0' diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index c6c6dacb..c45258fa 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -4,7 +4,6 @@ As defined in the wheel specification """ import os -import shutil from distutils.core import Command from distutils import log @@ -27,14 +26,11 @@ class dist_info(Command): def run(self): egg_info = self.get_finalized_command('egg_info') + egg_info.egg_base = self.egg_base + egg_info.finalize_options() egg_info.run() dist_info_dir = egg_info.egg_info[:-len('.egg-info')] + '.dist-info' log.info("creating '{}'".format(os.path.abspath(dist_info_dir))) bdist_wheel = self.get_finalized_command('bdist_wheel') bdist_wheel.egg2dist(egg_info.egg_info, dist_info_dir) - - if self.egg_base: - destination = os.path.join(self.egg_base, dist_info_dir) - log.info("creating '{}'".format(os.path.abspath(destination))) - shutil.move(dist_info_dir, destination) diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 8fba7b41..06c98271 100755..100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -40,12 +40,16 @@ import subprocess import shlex import io + +from sysconfig import get_config_vars, get_path + +from setuptools import SetuptoolsDeprecationWarning + from setuptools.extern import six from setuptools.extern.six.moves import configparser, map from setuptools import Command from setuptools.sandbox import run_setup -from setuptools.py31compat import get_path, get_config_vars from setuptools.py27compat import rmtree_safe from setuptools.command import setopt from setuptools.archive_util import unpack_archive @@ -53,6 +57,7 @@ from setuptools.package_index import ( PackageIndex, parse_requirement_arg, URL_SCHEME, ) from setuptools.command import bdist_egg, egg_info +from setuptools.wheel import Wheel from pkg_resources import ( yield_lines, normalize_path, resource_string, ensure_directory, get_distribution, find_distributions, Environment, Requirement, @@ -61,6 +66,8 @@ from pkg_resources import ( ) import pkg_resources.py31compat +__metaclass__ = type + # Turn on PEP440Warnings warnings.filterwarnings("default", category=pkg_resources.PEP440Warning) @@ -92,7 +99,7 @@ def samefile(p1, p2): if six.PY2: - def _to_ascii(s): + def _to_bytes(s): return s def isascii(s): @@ -103,8 +110,8 @@ if six.PY2: return False else: - def _to_ascii(s): - return s.encode('ascii') + def _to_bytes(s): + return s.encode('utf8') def isascii(s): try: @@ -318,7 +325,7 @@ class easy_install(Command): self.all_site_dirs.append(normalize_path(d)) if not self.editable: self.check_site_dir() - self.index_url = self.index_url or "https://pypi.python.org/simple" + self.index_url = self.index_url or "https://pypi.org/simple/" self.shadow_path = self.all_site_dirs[:] for path_item in self.install_dir, normalize_path(self.script_dir): if path_item not in self.shadow_path: @@ -628,7 +635,7 @@ class easy_install(Command): @contextlib.contextmanager def _tmpdir(self): - tmpdir = tempfile.mkdtemp(prefix=six.u("easy_install-")) + tmpdir = tempfile.mkdtemp(prefix=u"easy_install-") try: # cast to str as workaround for #709 and #710 and #712 yield str(tmpdir) @@ -801,7 +808,7 @@ class easy_install(Command): if is_script: body = self._load_template(dev_path) % locals() script_text = ScriptWriter.get_header(script_text) + body - self.write_script(script_name, _to_ascii(script_text), 'b') + self.write_script(script_name, _to_bytes(script_text), 'b') @staticmethod def _load_template(dev_path): @@ -827,14 +834,16 @@ class easy_install(Command): target = os.path.join(self.script_dir, script_name) self.add_output(target) + if self.dry_run: + return + mask = current_umask() - if not self.dry_run: - ensure_directory(target) - if os.path.exists(target): - os.unlink(target) - with open(target, "w" + mode) as f: - f.write(contents) - chmod(target, 0o777 - mask) + ensure_directory(target) + if os.path.exists(target): + os.unlink(target) + with open(target, "w" + mode) as f: + f.write(contents) + chmod(target, 0o777 - mask) def install_eggs(self, spec, dist_filename, tmpdir): # .egg dirs or files are already built, so just return them @@ -842,6 +851,8 @@ class easy_install(Command): return [self.install_egg(dist_filename, tmpdir)] elif dist_filename.lower().endswith('.exe'): return [self.install_exe(dist_filename, tmpdir)] + elif dist_filename.lower().endswith('.whl'): + return [self.install_wheel(dist_filename, tmpdir)] # Anything else, try to extract and build setup_base = tmpdir @@ -1038,6 +1049,35 @@ class easy_install(Command): f.write('\n'.join(locals()[name]) + '\n') f.close() + def install_wheel(self, wheel_path, tmpdir): + wheel = Wheel(wheel_path) + assert wheel.is_compatible() + destination = os.path.join(self.install_dir, wheel.egg_name()) + destination = os.path.abspath(destination) + if not self.dry_run: + ensure_directory(destination) + if os.path.isdir(destination) and not os.path.islink(destination): + dir_util.remove_tree(destination, dry_run=self.dry_run) + elif os.path.exists(destination): + self.execute( + os.unlink, + (destination,), + "Removing " + destination, + ) + try: + self.execute( + wheel.install_as_egg, + (destination,), + ("Installing %s to %s") % ( + os.path.basename(wheel_path), + os.path.dirname(destination) + ), + ) + finally: + update_dist_caches(destination, fix_zipimporter_caches=False) + self.add_output(destination) + return self.egg_distribution(destination) + __mv_warning = textwrap.dedent(""" Because this distribution was installed --multi-version, before you can import modules from this package in an application, you will need to @@ -1216,7 +1256,6 @@ class easy_install(Command): def byte_compile(self, to_compile): if sys.dont_write_bytecode: - self.warn('byte-compiling is disabled, skipping.') return from distutils.util import byte_compile @@ -1817,7 +1856,7 @@ def _update_zipimporter_cache(normalized_path, cache, updater=None): # get/del patterns instead. For more detailed information see the # following links: # https://github.com/pypa/setuptools/issues/202#issuecomment-202913420 - # https://bitbucket.org/pypy/pypy/src/dd07756a34a41f674c0cacfbc8ae1d4cc9ea2ae4/pypy/module/zipimport/interp_zipimport.py#cl-99 + # http://bit.ly/2h9itJX old_entry = cache[p] del cache[p] new_entry = updater and updater(p, old_entry) @@ -2016,7 +2055,7 @@ class WindowsCommandSpec(CommandSpec): split_args = dict(posix=False) -class ScriptWriter(object): +class ScriptWriter: """ Encapsulates behavior around writing entry point scripts for console and gui apps. @@ -2041,7 +2080,7 @@ class ScriptWriter(object): @classmethod def get_script_args(cls, dist, executable=None, wininst=False): # for backward compatibility - warnings.warn("Use get_args", DeprecationWarning) + warnings.warn("Use get_args", EasyInstallDeprecationWarning) writer = (WindowsScriptWriter if wininst else ScriptWriter).best() header = cls.get_script_header("", executable, wininst) return writer.get_args(dist, header) @@ -2049,12 +2088,10 @@ class ScriptWriter(object): @classmethod def get_script_header(cls, script_text, executable=None, wininst=False): # for backward compatibility - warnings.warn("Use get_header", DeprecationWarning) + warnings.warn("Use get_header", EasyInstallDeprecationWarning, stacklevel=2) if wininst: executable = "python.exe" - cmd = cls.command_spec_class.best().from_param(executable) - cmd.install_options(script_text) - return cmd.as_header() + return cls.get_header(script_text, executable) @classmethod def get_args(cls, dist, header=None): @@ -2086,7 +2123,7 @@ class ScriptWriter(object): @classmethod def get_writer(cls, force_windows): # for backward compatibility - warnings.warn("Use best", DeprecationWarning) + warnings.warn("Use best", EasyInstallDeprecationWarning) return WindowsScriptWriter.best() if force_windows else cls.best() @classmethod @@ -2118,7 +2155,7 @@ class WindowsScriptWriter(ScriptWriter): @classmethod def get_writer(cls): # for backward compatibility - warnings.warn("Use best", DeprecationWarning) + warnings.warn("Use best", EasyInstallDeprecationWarning) return cls.best() @classmethod @@ -2299,3 +2336,7 @@ def _patch_usage(): yield finally: distutils.core.gen_usage = saved + +class EasyInstallDeprecationWarning(SetuptoolsDeprecationWarning): + """Class for warning about deprecations in EasyInstall in SetupTools. Not ignored by default, unlike DeprecationWarning.""" + diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index a183d15d..d9fe3da3 100755..100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -30,8 +30,8 @@ from pkg_resources import ( import setuptools.unicode_utils as unicode_utils from setuptools.glob import glob -from pkg_resources.extern import packaging - +from setuptools.extern import packaging +from setuptools import SetuptoolsDeprecationWarning def translate_pattern(glob): """ @@ -116,7 +116,33 @@ def translate_pattern(glob): return re.compile(pat, flags=re.MULTILINE|re.DOTALL) -class egg_info(Command): +class InfoCommon: + tag_build = None + tag_date = None + + @property + def name(self): + return safe_name(self.distribution.get_name()) + + def tagged_version(self): + version = self.distribution.get_version() + # egg_info may be called more than once for a distribution, + # in which case the version string already contains all tags. + if self.vtags and version.endswith(self.vtags): + return safe_version(version) + return safe_version(version + self.vtags) + + def tags(self): + version = '' + if self.tag_build: + version += self.tag_build + if self.tag_date: + version += time.strftime("-%Y%m%d") + return version + vtags = property(tags) + + +class egg_info(InfoCommon, Command): description = "create a distribution's .egg-info directory" user_options = [ @@ -133,14 +159,11 @@ class egg_info(Command): } def initialize_options(self): - self.egg_name = None - self.egg_version = None self.egg_base = None + self.egg_name = None self.egg_info = None - self.tag_build = None - self.tag_date = 0 + self.egg_version = None self.broken_egg_info = False - self.vtags = None #################################### # allow the 'tag_svn_revision' to be detected and @@ -160,9 +183,7 @@ class egg_info(Command): build tag. Install build keys in a deterministic order to avoid arbitrary reordering on subsequent builds. """ - # python 2.6 compatibility - odict = getattr(collections, 'OrderedDict', dict) - egg_info = odict() + egg_info = collections.OrderedDict() # follow the order these keys would have been added # when PYTHONHASHSEED=0 egg_info['tag_build'] = self.tags() @@ -170,10 +191,12 @@ class egg_info(Command): edit_config(filename, dict(egg_info=egg_info)) def finalize_options(self): - self.egg_name = safe_name(self.distribution.get_name()) - self.vtags = self.tags() + # Note: we need to capture the current value returned + # by `self.tagged_version()`, so we can later update + # `self.distribution.metadata.version` without + # repercussions. + self.egg_name = self.name self.egg_version = self.tagged_version() - parsed_version = parse_version(self.egg_version) try: @@ -256,16 +279,9 @@ class egg_info(Command): if not self.dry_run: os.unlink(filename) - def tagged_version(self): - version = self.distribution.get_version() - # egg_info may be called more than once for a distribution, - # in which case the version string already contains all tags. - if self.vtags and version.endswith(self.vtags): - return safe_version(version) - return safe_version(version + self.vtags) - def run(self): self.mkpath(self.egg_info) + os.utime(self.egg_info, None) installer = self.distribution.fetch_build_egg for ep in iter_entry_points('egg_info.writers'): ep.require(installer=installer) @@ -279,14 +295,6 @@ class egg_info(Command): self.find_sources() - def tags(self): - version = '' - if self.tag_build: - version += self.tag_build - if self.tag_date: - version += time.strftime("-%Y%m%d") - return version - def find_sources(self): """Generate SOURCES.txt manifest file""" manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") @@ -567,6 +575,12 @@ class manifest_maker(sdist): self.filelist.extend(rcfiles) elif os.path.exists(self.manifest): self.read_manifest() + + if os.path.exists("setup.py"): + # setup.py should be included by default, even if it's not + # the script called to create the sdist + self.filelist.append("setup.py") + ei_cmd = self.get_finalized_command('egg_info') self.filelist.graft(ei_cmd.egg_info) @@ -599,10 +613,7 @@ def write_pkg_info(cmd, basename, filename): metadata = cmd.distribution.metadata metadata.version, oldver = cmd.egg_version, metadata.version metadata.name, oldname = cmd.egg_name, metadata.name - metadata.long_description_content_type = getattr( - cmd.distribution, - 'long_description_content_type' - ) + try: # write unescaped data to PKG-INFO, so older pkg_resources # can still parse it @@ -642,7 +653,7 @@ def write_requirements(cmd, basename, filename): def write_setup_requirements(cmd, basename, filename): - data = StringIO() + data = io.StringIO() _write_requirements(data, cmd.distribution.setup_requires) cmd.write_or_delete_file("setup-requirements", filename, data.getvalue()) @@ -691,7 +702,7 @@ def get_pkg_info_revision(): Get a -r### off of PKG-INFO Version in case this is an sdist of a subversion revision. """ - warnings.warn("get_pkg_info_revision is deprecated.", DeprecationWarning) + warnings.warn("get_pkg_info_revision is deprecated.", EggInfoDeprecationWarning) if os.path.exists('PKG-INFO'): with io.open('PKG-INFO') as f: for line in f: @@ -699,3 +710,7 @@ def get_pkg_info_revision(): if match: return int(match.group(1)) return 0 + + +class EggInfoDeprecationWarning(SetuptoolsDeprecationWarning): + """Class for warning about deprecations in eggInfo in setupTools. Not ignored by default, unlike DeprecationWarning.""" diff --git a/setuptools/command/install_egg_info.py b/setuptools/command/install_egg_info.py index edc4718b..edc4718b 100755..100644 --- a/setuptools/command/install_egg_info.py +++ b/setuptools/command/install_egg_info.py diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py index 16234273..16234273 100755..100644 --- a/setuptools/command/install_scripts.py +++ b/setuptools/command/install_scripts.py diff --git a/setuptools/command/register.py b/setuptools/command/register.py index 8d6336a1..98bc0156 100755..100644 --- a/setuptools/command/register.py +++ b/setuptools/command/register.py @@ -1,3 +1,4 @@ +from distutils import log import distutils.command.register as orig @@ -5,6 +6,13 @@ class register(orig.register): __doc__ = orig.register.__doc__ def run(self): - # Make sure that we are using valid current name/version info - self.run_command('egg_info') - orig.register.run(self) + try: + # Make sure that we are using valid current name/version info + self.run_command('egg_info') + orig.register.run(self) + finally: + self.announce( + "WARNING: Registering is deprecated, use twine to " + "upload instead (https://pypi.org/p/twine/)", + log.WARN + ) diff --git a/setuptools/command/rotate.py b/setuptools/command/rotate.py index b89353f5..b89353f5 100755..100644 --- a/setuptools/command/rotate.py +++ b/setuptools/command/rotate.py diff --git a/setuptools/command/saveopts.py b/setuptools/command/saveopts.py index 611cec55..611cec55 100755..100644 --- a/setuptools/command/saveopts.py +++ b/setuptools/command/saveopts.py diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 508148e0..bcfae4d8 100755..100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -51,13 +51,6 @@ class sdist(sdist_add_defaults, orig.sdist): for cmd_name in self.get_sub_commands(): self.run_command(cmd_name) - # Call check_metadata only if no 'check' command - # (distutils <= 2.6) - import distutils.command - - if 'check' not in distutils.command.__all__: - self.check_metadata() - self.make_distribution() dist_files = getattr(self.distribution, 'dist_files', []) diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index 7e57cc02..7e57cc02 100755..100644 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py diff --git a/setuptools/command/test.py b/setuptools/command/test.py index 638d0c56..dde0118c 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -3,6 +3,7 @@ import operator import sys import contextlib import itertools +import unittest from distutils.errors import DistutilsError, DistutilsOptionError from distutils import log from unittest import TestLoader @@ -14,10 +15,16 @@ from pkg_resources import (resource_listdir, resource_exists, normalize_path, working_set, _namespace_packages, evaluate_marker, add_activation_listener, require, EntryPoint) from setuptools import Command -from setuptools.py31compat import unittest_main + +__metaclass__ = type class ScanningLoader(TestLoader): + + def __init__(self): + TestLoader.__init__(self) + self._visited = set() + def loadTestsFromModule(self, module, pattern=None): """Return a suite of all tests cases contained in the given module @@ -25,6 +32,10 @@ class ScanningLoader(TestLoader): If the module has an ``additional_tests`` function, call it and add the return value to the tests. """ + if module in self._visited: + return None + self._visited.add(module) + tests = [] tests.append(TestLoader.loadTestsFromModule(self, module)) @@ -49,7 +60,7 @@ class ScanningLoader(TestLoader): # adapted from jaraco.classes.properties:NonDataProperty -class NonDataProperty(object): +class NonDataProperty: def __init__(self, fget): self.fget = fget @@ -101,6 +112,8 @@ class test(Command): return list(self._test_args()) def _test_args(self): + if not self.test_suite and sys.version_info >= (2, 7): + yield 'discover' if self.verbose: yield '--verbose' if self.test_suite: @@ -230,12 +243,11 @@ class test(Command): del_modules.append(name) list(map(sys.modules.__delitem__, del_modules)) - exit_kwarg = {} if sys.version_info < (2, 7) else {"exit": False} - test = unittest_main( + test = unittest.main( None, None, self._argv, testLoader=self._resolve_as_ep(self.test_loader), testRunner=self._resolve_as_ep(self.test_runner), - **exit_kwarg + exit=False, ) if not test.result.wasSuccessful(): msg = 'Test failed: %s' % test.result diff --git a/setuptools/command/upload.py b/setuptools/command/upload.py index a44173a9..6db8888b 100644 --- a/setuptools/command/upload.py +++ b/setuptools/command/upload.py @@ -1,5 +1,19 @@ +import io +import os +import hashlib import getpass + +from base64 import standard_b64encode + +from distutils import log from distutils.command import upload as orig +from distutils.spawn import spawn + +from distutils.errors import DistutilsError + +from setuptools.extern.six.moves.urllib.request import urlopen, Request +from setuptools.extern.six.moves.urllib.error import HTTPError +from setuptools.extern.six.moves.urllib.parse import urlparse class upload(orig.upload): @@ -7,6 +21,15 @@ class upload(orig.upload): Override default upload behavior to obtain password in a variety of different ways. """ + def run(self): + try: + orig.upload.run(self) + finally: + self.announce( + "WARNING: Uploading via this command is deprecated, use twine " + "to upload instead (https://pypi.org/p/twine/)", + log.WARN + ) def finalize_options(self): orig.upload.finalize_options(self) @@ -22,6 +45,137 @@ class upload(orig.upload): self._prompt_for_password() ) + def upload_file(self, command, pyversion, filename): + # Makes sure the repository URL is compliant + schema, netloc, url, params, query, fragments = \ + urlparse(self.repository) + if params or query or fragments: + raise AssertionError("Incompatible url %s" % self.repository) + + if schema not in ('http', 'https'): + raise AssertionError("unsupported schema " + schema) + + # Sign if requested + if self.sign: + gpg_args = ["gpg", "--detach-sign", "-a", filename] + if self.identity: + gpg_args[2:2] = ["--local-user", self.identity] + spawn(gpg_args, + dry_run=self.dry_run) + + # Fill in the data - send all the meta-data in case we need to + # register a new release + with open(filename, 'rb') as f: + content = f.read() + + meta = self.distribution.metadata + + data = { + # action + ':action': 'file_upload', + 'protocol_version': '1', + + # identify release + 'name': meta.get_name(), + 'version': meta.get_version(), + + # file content + 'content': (os.path.basename(filename), content), + 'filetype': command, + 'pyversion': pyversion, + 'md5_digest': hashlib.md5(content).hexdigest(), + + # additional meta-data + 'metadata_version': str(meta.get_metadata_version()), + 'summary': meta.get_description(), + 'home_page': meta.get_url(), + 'author': meta.get_contact(), + 'author_email': meta.get_contact_email(), + 'license': meta.get_licence(), + 'description': meta.get_long_description(), + 'keywords': meta.get_keywords(), + 'platform': meta.get_platforms(), + 'classifiers': meta.get_classifiers(), + 'download_url': meta.get_download_url(), + # PEP 314 + 'provides': meta.get_provides(), + 'requires': meta.get_requires(), + 'obsoletes': meta.get_obsoletes(), + } + + data['comment'] = '' + + if self.sign: + data['gpg_signature'] = (os.path.basename(filename) + ".asc", + open(filename+".asc", "rb").read()) + + # set up the authentication + user_pass = (self.username + ":" + self.password).encode('ascii') + # The exact encoding of the authentication string is debated. + # Anyway PyPI only accepts ascii for both username or password. + auth = "Basic " + standard_b64encode(user_pass).decode('ascii') + + # Build up the MIME payload for the POST data + boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' + sep_boundary = b'\r\n--' + boundary.encode('ascii') + end_boundary = sep_boundary + b'--\r\n' + body = io.BytesIO() + for key, value in data.items(): + title = '\r\nContent-Disposition: form-data; name="%s"' % key + # handle multiple entries for the same name + if not isinstance(value, list): + value = [value] + for value in value: + if type(value) is tuple: + title += '; filename="%s"' % value[0] + value = value[1] + else: + value = str(value).encode('utf-8') + body.write(sep_boundary) + body.write(title.encode('utf-8')) + body.write(b"\r\n\r\n") + body.write(value) + body.write(end_boundary) + body = body.getvalue() + + msg = "Submitting %s to %s" % (filename, self.repository) + self.announce(msg, log.INFO) + + # build the Request + headers = { + 'Content-type': 'multipart/form-data; boundary=%s' % boundary, + 'Content-length': str(len(body)), + 'Authorization': auth, + } + + request = Request(self.repository, data=body, + headers=headers) + # send the data + try: + result = urlopen(request) + status = result.getcode() + reason = result.msg + except HTTPError as e: + status = e.code + reason = e.msg + except OSError as e: + self.announce(str(e), log.ERROR) + raise + + if status == 200: + self.announce('Server response (%s): %s' % (status, reason), + log.INFO) + if self.show_response: + text = getattr(self, '_read_pypi_response', + lambda x: None)(result) + if text is not None: + msg = '\n'.join(('-' * 75, text, '-' * 75)) + self.announce(msg, log.INFO) + else: + msg = 'Upload failed (%s): %s' % (status, reason) + self.announce(msg, log.ERROR) + raise DistutilsError(msg) + def _load_password_from_keyring(self): """ Attempt to load password from keyring. Suppress Exceptions. diff --git a/setuptools/config.py b/setuptools/config.py index 9a62e2ec..b6626043 100644 --- a/setuptools/config.py +++ b/setuptools/config.py @@ -2,12 +2,20 @@ from __future__ import absolute_import, unicode_literals import io import os import sys + +import warnings +import functools from collections import defaultdict from functools import partial +from functools import wraps +from importlib import import_module from distutils.errors import DistutilsOptionError, DistutilsFileError -from setuptools.py26compat import import_module -from setuptools.extern.six import string_types +from setuptools.extern.packaging.version import LegacyVersion, parse +from setuptools.extern.six import string_types, PY3 + + +__metaclass__ = type def read_configuration( @@ -57,6 +65,18 @@ def read_configuration( return configuration_to_dict(handlers) +def _get_option(target_obj, key): + """ + Given a target object and option key, get that option from + the target object, either through a get_{key} method or + from an attribute directly. + """ + getter_name = 'get_{key}'.format(**locals()) + by_attribute = functools.partial(getattr, target_obj, key) + getter = getattr(target_obj, getter_name, by_attribute) + return getter() + + def configuration_to_dict(handlers): """Returns configuration data gathered by given handlers as a dict. @@ -68,20 +88,9 @@ def configuration_to_dict(handlers): config_dict = defaultdict(dict) for handler in handlers: - - obj_alias = handler.section_prefix - target_obj = handler.target_obj - for option in handler.set_options: - getter = getattr(target_obj, 'get_%s' % option, None) - - if getter is None: - value = getattr(target_obj, option) - - else: - value = getter() - - config_dict[obj_alias][option] = value + value = _get_option(handler.target_obj, option) + config_dict[handler.section_prefix][option] = value return config_dict @@ -101,18 +110,19 @@ def parse_configuration( If False exceptions are propagated as expected. :rtype: list """ - meta = ConfigMetadataHandler( - distribution.metadata, command_options, ignore_option_errors) - meta.parse() - options = ConfigOptionsHandler( distribution, command_options, ignore_option_errors) options.parse() - return [meta, options] + meta = ConfigMetadataHandler( + distribution.metadata, command_options, ignore_option_errors, + distribution.package_dir) + meta.parse() + + return meta, options -class ConfigHandler(object): +class ConfigHandler: """Handles metadata supplied in configuration files.""" section_prefix = None @@ -237,6 +247,26 @@ class ConfigHandler(object): return value in ('1', 'true', 'yes') @classmethod + def _exclude_files_parser(cls, key): + """Returns a parser function to make sure field inputs + are not files. + + Parses a value after getting the key so error messages are + more informative. + + :param key: + :rtype: callable + """ + def parser(value): + exclude_directive = 'file:' + if value.startswith(exclude_directive): + raise ValueError( + 'Only strings are accepted for the {0} field, ' + 'files are not accepted'.format(key)) + return value + return parser + + @classmethod def _parse_file(cls, value): """Represents value as a string, allowing including text from nearest files using `file:` directive. @@ -245,7 +275,6 @@ class ConfigHandler(object): directory with setup.py. Examples: - file: LICENSE file: README.rst, CHANGELOG.md, src/file.txt :param str value: @@ -280,7 +309,7 @@ class ConfigHandler(object): return f.read() @classmethod - def _parse_attr(cls, value): + def _parse_attr(cls, value, package_dir=None): """Represents value as a module attribute. Examples: @@ -300,7 +329,21 @@ class ConfigHandler(object): module_name = '.'.join(attrs_path) module_name = module_name or '__init__' - sys.path.insert(0, os.getcwd()) + parent_path = os.getcwd() + if package_dir: + if attrs_path[0] in package_dir: + # A custom path was specified for the module we want to import + custom_path = package_dir[attrs_path[0]] + parts = custom_path.rsplit('/', 1) + if len(parts) > 1: + parent_path = os.path.join(os.getcwd(), parts[0]) + module_name = parts[1] + else: + module_name = custom_path + elif '' in package_dir: + # A custom parent directory was specified for all root modules + parent_path = os.path.join(os.getcwd(), package_dir['']) + sys.path.insert(0, parent_path) try: module = import_module(module_name) value = getattr(module, attr_name) @@ -370,7 +413,7 @@ class ConfigHandler(object): section_parser_method = getattr( self, - # Dots in section names are tranlsated into dunderscores. + # Dots in section names are translated into dunderscores. ('parse_section%s' % method_postfix).replace('.', '__'), None) @@ -381,6 +424,20 @@ class ConfigHandler(object): section_parser_method(section_options) + def _deprecated_config_handler(self, func, msg, warning_class): + """ this function will wrap around parameters that are deprecated + + :param msg: deprecation message + :param warning_class: class of warning exception to be raised + :param func: function to be wrapped around + """ + @wraps(func) + def config_handler(*args, **kwargs): + warnings.warn(msg, warning_class) + return func(*args, **kwargs) + + return config_handler + class ConfigMetadataHandler(ConfigHandler): @@ -399,23 +456,36 @@ class ConfigMetadataHandler(ConfigHandler): """ + def __init__(self, target_obj, options, ignore_option_errors=False, + package_dir=None): + super(ConfigMetadataHandler, self).__init__(target_obj, options, + ignore_option_errors) + self.package_dir = package_dir + @property def parsers(self): """Metadata item name to parser function mapping.""" parse_list = self._parse_list parse_file = self._parse_file + parse_dict = self._parse_dict + exclude_files_parser = self._exclude_files_parser return { 'platforms': parse_list, 'keywords': parse_list, 'provides': parse_list, - 'requires': parse_list, + 'requires': self._deprecated_config_handler( + parse_list, + "The requires parameter is deprecated, please use " + "install_requires for runtime dependencies.", + DeprecationWarning), 'obsoletes': parse_list, 'classifiers': self._get_parser_compound(parse_file, parse_list), - 'license': parse_file, + 'license': exclude_files_parser('license'), 'description': parse_file, 'long_description': parse_file, 'version': self._parse_version, + 'project_urls': parse_dict, } def _parse_version(self, value): @@ -425,7 +495,22 @@ class ConfigMetadataHandler(ConfigHandler): :rtype: str """ - version = self._parse_attr(value) + version = self._parse_file(value) + + if version != value: + version = version.strip() + # Be strict about versions loaded from file because it's easy to + # accidentally include newlines and other unintended content + if isinstance(parse(version), LegacyVersion): + tmpl = ( + 'Version loaded from {value} does not ' + 'comply with PEP 440: {version}' + ) + raise DistutilsOptionError(tmpl.format(**locals())) + + return version + + version = self._parse_attr(value, self.package_dir) if callable(version): version = version() @@ -477,16 +562,25 @@ class ConfigOptionsHandler(ConfigHandler): :param value: :rtype: list """ - find_directive = 'find:' + find_directives = ['find:', 'find_namespace:'] + trimmed_value = value.strip() - if not value.startswith(find_directive): + if trimmed_value not in find_directives: return self._parse_list(value) + findns = trimmed_value == find_directives[1] + if findns and not PY3: + raise DistutilsOptionError( + 'find_namespace: directive is unsupported on Python < 3.3') + # Read function arguments from a dedicated section. find_kwargs = self.parse_section_packages__find( self.sections.get('packages.find', {})) - from setuptools import find_packages + if findns: + from setuptools import find_namespace_packages as find_packages + else: + from setuptools import find_packages return find_packages(**find_kwargs) @@ -552,3 +646,11 @@ class ConfigOptionsHandler(ConfigHandler): parse_list = partial(self._parse_list, separator=';') self['extras_require'] = self._parse_section_to_dict( section_options, parse_list) + + def parse_section_data_files(self, section_options): + """Parses `data_files` configuration file section. + + :param dict section_options: + """ + parsed = self._parse_section_to_dict(section_options, self._parse_list) + self['data_files'] = [(k, v) for k, v in parsed.items()] diff --git a/setuptools/dist.py b/setuptools/dist.py index b10bd6f7..a2600711 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- __all__ = ['Distribution'] import re @@ -9,15 +10,22 @@ import distutils.core import distutils.cmd import distutils.dist import itertools + + from collections import defaultdict +from email import message_from_file + from distutils.errors import ( DistutilsOptionError, DistutilsPlatformError, DistutilsSetupError, ) from distutils.util import rfc822_escape +from distutils.version import StrictVersion from setuptools.extern import six +from setuptools.extern import packaging from setuptools.extern.six.moves import map, filter, filterfalse -from pkg_resources.extern import packaging + +from . import SetuptoolsDeprecationWarning from setuptools.depends import Require from setuptools import windows_support @@ -26,53 +34,145 @@ from setuptools.config import parse_configuration import pkg_resources from .py36compat import Distribution_parse_config_files -__import__('pkg_resources.extern.packaging.specifiers') -__import__('pkg_resources.extern.packaging.version') +__import__('setuptools.extern.packaging.specifiers') +__import__('setuptools.extern.packaging.version') def _get_unpatched(cls): - warnings.warn("Do not call this function", DeprecationWarning) + warnings.warn("Do not call this function", DistDeprecationWarning) return get_unpatched(cls) +def get_metadata_version(self): + mv = getattr(self, 'metadata_version', None) + + if mv is None: + if self.long_description_content_type or self.provides_extras: + mv = StrictVersion('2.1') + elif (self.maintainer is not None or + self.maintainer_email is not None or + getattr(self, 'python_requires', None) is not None): + mv = StrictVersion('1.2') + elif (self.provides or self.requires or self.obsoletes or + self.classifiers or self.download_url): + mv = StrictVersion('1.1') + else: + mv = StrictVersion('1.0') + + self.metadata_version = mv + + return mv + + +def read_pkg_file(self, file): + """Reads the metadata values from a file object.""" + msg = message_from_file(file) + + def _read_field(name): + value = msg[name] + if value == 'UNKNOWN': + return None + return value + + def _read_list(name): + values = msg.get_all(name, None) + if values == []: + return None + return values + + self.metadata_version = StrictVersion(msg['metadata-version']) + self.name = _read_field('name') + self.version = _read_field('version') + self.description = _read_field('summary') + # we are filling author only. + self.author = _read_field('author') + self.maintainer = None + self.author_email = _read_field('author-email') + self.maintainer_email = None + self.url = _read_field('home-page') + self.license = _read_field('license') + + if 'download-url' in msg: + self.download_url = _read_field('download-url') + else: + self.download_url = None + + self.long_description = _read_field('description') + self.description = _read_field('summary') + + if 'keywords' in msg: + self.keywords = _read_field('keywords').split(',') + + self.platforms = _read_list('platform') + self.classifiers = _read_list('classifier') + + # PEP 314 - these fields only exist in 1.1 + if self.metadata_version == StrictVersion('1.1'): + self.requires = _read_list('requires') + self.provides = _read_list('provides') + self.obsoletes = _read_list('obsoletes') + else: + self.requires = None + self.provides = None + self.obsoletes = None + + # Based on Python 3.5 version def write_pkg_file(self, file): """Write the PKG-INFO format data to a file object. """ - version = '1.0' - if (self.provides or self.requires or self.obsoletes or - self.classifiers or self.download_url): - version = '1.1' - # Setuptools specific for PEP 345 - if hasattr(self, 'python_requires'): - version = '1.2' - - file.write('Metadata-Version: %s\n' % version) - file.write('Name: %s\n' % self.get_name()) - file.write('Version: %s\n' % self.get_version()) - file.write('Summary: %s\n' % self.get_description()) - file.write('Home-page: %s\n' % self.get_url()) - file.write('Author: %s\n' % self.get_contact()) - file.write('Author-email: %s\n' % self.get_contact_email()) - file.write('License: %s\n' % self.get_license()) - if self.download_url: - file.write('Download-URL: %s\n' % self.download_url) + version = self.get_metadata_version() + + if six.PY2: + def write_field(key, value): + file.write("%s: %s\n" % (key, self._encode_field(value))) + else: + def write_field(key, value): + file.write("%s: %s\n" % (key, value)) + + + write_field('Metadata-Version', str(version)) + write_field('Name', self.get_name()) + write_field('Version', self.get_version()) + write_field('Summary', self.get_description()) + write_field('Home-page', self.get_url()) + + if version < StrictVersion('1.2'): + write_field('Author', self.get_contact()) + write_field('Author-email', self.get_contact_email()) + else: + optional_fields = ( + ('Author', 'author'), + ('Author-email', 'author_email'), + ('Maintainer', 'maintainer'), + ('Maintainer-email', 'maintainer_email'), + ) + + for field, attr in optional_fields: + attr_val = getattr(self, attr) + + if attr_val is not None: + write_field(field, attr_val) - long_desc_content_type = getattr( - self, - 'long_description_content_type', - None - ) or 'UNKNOWN' - file.write('Description-Content-Type: %s\n' % long_desc_content_type) + write_field('License', self.get_license()) + if self.download_url: + write_field('Download-URL', self.download_url) + for project_url in self.project_urls.items(): + write_field('Project-URL', '%s, %s' % project_url) long_desc = rfc822_escape(self.get_long_description()) - file.write('Description: %s\n' % long_desc) + write_field('Description', long_desc) keywords = ','.join(self.get_keywords()) if keywords: - file.write('Keywords: %s\n' % keywords) + write_field('Keywords', keywords) + + if version >= StrictVersion('1.2'): + for platform in self.get_platforms(): + write_field('Platform', platform) + else: + self._write_list(file, 'Platform', self.get_platforms()) - self._write_list(file, 'Platform', self.get_platforms()) self._write_list(file, 'Classifier', self.get_classifiers()) # PEP 314 @@ -82,16 +182,17 @@ def write_pkg_file(self, file): # Setuptools specific for PEP 345 if hasattr(self, 'python_requires'): - file.write('Requires-Python: %s\n' % self.python_requires) + write_field('Requires-Python', self.python_requires) - -# from Python 3.4 -def write_pkg_info(self, base_dir): - """Write the PKG-INFO file into the release tree. - """ - with open(os.path.join(base_dir, 'PKG-INFO'), 'w', - encoding='UTF-8') as pkg_info: - self.write_pkg_file(pkg_info) + # PEP 566 + if self.long_description_content_type: + write_field( + 'Description-Content-Type', + self.long_description_content_type + ) + if self.provides_extras: + for extra in self.provides_extras: + write_field('Provides-Extra', extra) sequence = tuple, list @@ -166,6 +267,8 @@ def check_requirements(dist, attr, value): """Verify that install_requires is a valid requirements list""" try: list(pkg_resources.parse_requirements(value)) + if isinstance(value, (dict, set)): + raise TypeError("Unordered types are not allowed") except (TypeError, ValueError) as error: tmpl = ( "{attr!r} must be a string or list of strings " @@ -297,6 +400,12 @@ class Distribution(Distribution_parse_config_files, _Distribution): distribution for the included and excluded features. """ + _DISTUTILS_UNSUPPORTED_METADATA = { + 'long_description_content_type': None, + 'project_urls': dict, + 'provides_extras': set, + } + _patched_dist = None def patch_missing_pkg_info(self, attrs): @@ -316,26 +425,36 @@ class Distribution(Distribution_parse_config_files, _Distribution): have_package_data = hasattr(self, "package_data") if not have_package_data: self.package_data = {} - _attrs_dict = attrs or {} - if 'features' in _attrs_dict or 'require_features' in _attrs_dict: + attrs = attrs or {} + if 'features' in attrs or 'require_features' in attrs: Feature.warn_deprecated() self.require_features = [] self.features = {} self.dist_files = [] - self.src_root = attrs and attrs.pop("src_root", None) + # Filter-out setuptools' specific options. + self.src_root = attrs.pop("src_root", None) self.patch_missing_pkg_info(attrs) - self.long_description_content_type = _attrs_dict.get( - 'long_description_content_type' - ) - # Make sure we have any eggs needed to interpret 'attrs' - if attrs is not None: - self.dependency_links = attrs.pop('dependency_links', []) - assert_string_list(self, 'dependency_links', self.dependency_links) - if attrs and 'setup_requires' in attrs: - self.fetch_build_eggs(attrs['setup_requires']) + self.dependency_links = attrs.pop('dependency_links', []) + self.setup_requires = attrs.pop('setup_requires', []) for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): vars(self).setdefault(ep.name, None) - _Distribution.__init__(self, attrs) + _Distribution.__init__(self, { + k: v for k, v in attrs.items() + if k not in self._DISTUTILS_UNSUPPORTED_METADATA + }) + + # Fill-in missing metadata fields not supported by distutils. + # Note some fields may have been set by other tools (e.g. pbr) + # above; they are taken preferrentially to setup() arguments + for option, default in self._DISTUTILS_UNSUPPORTED_METADATA.items(): + for source in self.metadata.__dict__, attrs: + if option in source: + value = source[option] + break + else: + value = default() if default else None + setattr(self.metadata, option, value) + if isinstance(self.metadata.version, numbers.Number): # Some people apparently take "version number" too literally :) self.metadata.version = str(self.metadata.version) @@ -368,6 +487,16 @@ class Distribution(Distribution_parse_config_files, _Distribution): """ if getattr(self, 'python_requires', None): self.metadata.python_requires = self.python_requires + + if getattr(self, 'extras_require', None): + for extra in self.extras_require.keys(): + # Since this gets called multiple times at points where the + # keys have become 'converted' extras, ensure that we are only + # truly adding extras we haven't seen before here. + extra = extra.split(':')[0] + if extra: + self.metadata.provides_extras.add(extra) + self._convert_extras_requirements() self._move_install_requirements_markers() @@ -427,14 +556,15 @@ class Distribution(Distribution_parse_config_files, _Distribution): req.marker = None return req - def parse_config_files(self, filenames=None): + def parse_config_files(self, filenames=None, ignore_option_errors=False): """Parses configuration files from various levels and loads configuration. """ Distribution_parse_config_files.parse_config_files(self, filenames=filenames) - parse_configuration(self, self.command_options) + parse_configuration(self, self.command_options, + ignore_option_errors=ignore_option_errors) self._finalize_requires() def parse_command_line(self): @@ -497,19 +627,20 @@ class Distribution(Distribution_parse_config_files, _Distribution): """Fetch an egg needed for building""" from setuptools.command.easy_install import easy_install dist = self.__class__({'script_args': ['easy_install']}) - dist.parse_config_files() opts = dist.get_option_dict('easy_install') - keep = ( - 'find_links', 'site_dirs', 'index_url', 'optimize', - 'site_dirs', 'allow_hosts' - ) - for key in list(opts): - if key not in keep: - del opts[key] # don't use any other settings + opts.clear() + opts.update( + (k, v) + for k, v in self.get_option_dict('easy_install').items() + if k in ( + # don't use any other settings + 'find_links', 'site_dirs', 'index_url', + 'optimize', 'site_dirs', 'allow_hosts', + )) if self.dependency_links: links = self.dependency_links[:] if 'find_links' in opts: - links = opts['find_links'][1].split() + links + links = opts['find_links'][1] + links opts['find_links'] = ('setup', links) install_dir = self.get_egg_cache_dir() cmd = easy_install( @@ -921,7 +1052,7 @@ class Feature: "Features are deprecated and will be removed in a future " "version. See https://github.com/pypa/setuptools/issues/65." ) - warnings.warn(msg, DeprecationWarning, stacklevel=3) + warnings.warn(msg, DistDeprecationWarning, stacklevel=3) def __init__( self, description, standard=False, available=True, @@ -1010,3 +1141,7 @@ class Feature: " doesn't contain any packages or modules under %s" % (self.description, item, item) ) + + +class DistDeprecationWarning(SetuptoolsDeprecationWarning): + """Class for warning about deprecations in dist in setuptools. Not ignored by default, unlike DeprecationWarning.""" diff --git a/setuptools/extern/__init__.py b/setuptools/extern/__init__.py index 2cd08b7e..cb2fa329 100644 --- a/setuptools/extern/__init__.py +++ b/setuptools/extern/__init__.py @@ -1,4 +1,73 @@ -from pkg_resources.extern import VendorImporter +import sys -names = 'six', -VendorImporter(__name__, names, 'pkg_resources._vendor').install() + +class VendorImporter: + """ + A PEP 302 meta path importer for finding optionally-vendored + or otherwise naturally-installed packages from root_name. + """ + + def __init__(self, root_name, vendored_names=(), vendor_pkg=None): + self.root_name = root_name + self.vendored_names = set(vendored_names) + self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor') + + @property + def search_path(self): + """ + Search first the vendor package then as a natural package. + """ + yield self.vendor_pkg + '.' + yield '' + + def find_module(self, fullname, path=None): + """ + Return self when fullname starts with root_name and the + target module is one vendored through this importer. + """ + root, base, target = fullname.partition(self.root_name + '.') + if root: + return + if not any(map(target.startswith, self.vendored_names)): + return + return self + + def load_module(self, fullname): + """ + Iterate over the search path to locate and load fullname. + """ + root, base, target = fullname.partition(self.root_name + '.') + for prefix in self.search_path: + try: + extant = prefix + target + __import__(extant) + mod = sys.modules[extant] + sys.modules[fullname] = mod + # mysterious hack: + # Remove the reference to the extant package/module + # on later Python versions to cause relative imports + # in the vendor package to resolve the same modules + # as those going through this importer. + if sys.version_info >= (3, ): + del sys.modules[extant] + return mod + except ImportError: + pass + else: + raise ImportError( + "The '{target}' package is required; " + "normally this is bundled with this package so if you get " + "this warning, consult the packager of your " + "distribution.".format(**locals()) + ) + + def install(self): + """ + Install this importer into sys.meta_path if not already present. + """ + if self not in sys.meta_path: + sys.meta_path.append(self) + + +names = 'six', 'packaging', 'pyparsing', +VendorImporter(__name__, names, 'setuptools._vendor').install() diff --git a/setuptools/glibc.py b/setuptools/glibc.py new file mode 100644 index 00000000..a134591c --- /dev/null +++ b/setuptools/glibc.py @@ -0,0 +1,86 @@ +# This file originally from pip: +# https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/utils/glibc.py +from __future__ import absolute_import + +import ctypes +import re +import warnings + + +def glibc_version_string(): + "Returns glibc version string, or None if not using glibc." + + # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen + # manpage says, "If filename is NULL, then the returned handle is for the + # main program". This way we can let the linker do the work to figure out + # which libc our process is actually using. + process_namespace = ctypes.CDLL(None) + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return None + + # Call gnu_get_libc_version, which returns a string like "2.5" + gnu_get_libc_version.restype = ctypes.c_char_p + version_str = gnu_get_libc_version() + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") + + return version_str + + +# Separated out from have_compatible_glibc for easier unit testing +def check_glibc_version(version_str, required_major, minimum_minor): + # Parse string and check against requested version. + # + # We use a regexp instead of str.split because we want to discard any + # random junk that might come after the minor version -- this might happen + # in patched/forked versions of glibc (e.g. Linaro's version of glibc + # uses version strings like "2.20-2014.11"). See gh-3588. + m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str) + if not m: + warnings.warn("Expected glibc version with 2 components major.minor," + " got: %s" % version_str, RuntimeWarning) + return False + return (int(m.group("major")) == required_major and + int(m.group("minor")) >= minimum_minor) + + +def have_compatible_glibc(required_major, minimum_minor): + version_str = glibc_version_string() + if version_str is None: + return False + return check_glibc_version(version_str, required_major, minimum_minor) + + +# platform.libc_ver regularly returns completely nonsensical glibc +# versions. E.g. on my computer, platform says: +# +# ~$ python2.7 -c 'import platform; print(platform.libc_ver())' +# ('glibc', '2.7') +# ~$ python3.5 -c 'import platform; print(platform.libc_ver())' +# ('glibc', '2.9') +# +# But the truth is: +# +# ~$ ldd --version +# ldd (Debian GLIBC 2.22-11) 2.22 +# +# This is unfortunate, because it means that the linehaul data on libc +# versions that was generated by pip 8.1.2 and earlier is useless and +# misleading. Solution: instead of using platform, use our code that actually +# works. +def libc_ver(): + """Try to determine the glibc version + + Returns a tuple of strings (lib, version) which default to empty strings + in case the lookup fails. + """ + glibc_version = glibc_version_string() + if glibc_version is None: + return ("", "") + else: + return ("glibc", glibc_version) diff --git a/setuptools/glob.py b/setuptools/glob.py index 6c781de3..9d7cbc5d 100644 --- a/setuptools/glob.py +++ b/setuptools/glob.py @@ -3,14 +3,12 @@ Filename globbing utility. Mostly a copy of `glob` from Python 3.5. Changes include: * `yield from` and PEP3102 `*` removed. - * `bytes` changed to `six.binary_type`. * Hidden files are not ignored. """ import os import re import fnmatch -from setuptools.extern.six import binary_type __all__ = ["glob", "iglob", "escape"] @@ -92,7 +90,7 @@ def _iglob(pathname, recursive): def glob1(dirname, pattern): if not dirname: - if isinstance(pattern, binary_type): + if isinstance(pattern, bytes): dirname = os.curdir.encode('ASCII') else: dirname = os.curdir @@ -129,8 +127,8 @@ def glob2(dirname, pattern): # Recursively yields relative pathnames inside a literal directory. def _rlistdir(dirname): if not dirname: - if isinstance(dirname, binary_type): - dirname = binary_type(os.curdir, 'ASCII') + if isinstance(dirname, bytes): + dirname = os.curdir.encode('ASCII') else: dirname = os.curdir try: @@ -149,7 +147,7 @@ magic_check_bytes = re.compile(b'([*?[])') def has_magic(s): - if isinstance(s, binary_type): + if isinstance(s, bytes): match = magic_check_bytes.search(s) else: match = magic_check.search(s) @@ -157,7 +155,7 @@ def has_magic(s): def _isrecursive(pattern): - if isinstance(pattern, binary_type): + if isinstance(pattern, bytes): return pattern == b'**' else: return pattern == '**' @@ -169,7 +167,7 @@ def escape(pathname): # Escaping is done by wrapping any of "*?[" between square brackets. # Metacharacters do not work in the drive part and shouldn't be escaped. drive, pathname = os.path.splitdrive(pathname) - if isinstance(pathname, binary_type): + if isinstance(pathname, bytes): pathname = magic_check_bytes.sub(br'[\1]', pathname) else: pathname = magic_check.sub(r'[\1]', pathname) diff --git a/setuptools/monkey.py b/setuptools/monkey.py index 6d3711ec..3c77f8cf 100644 --- a/setuptools/monkey.py +++ b/setuptools/monkey.py @@ -7,9 +7,9 @@ import distutils.filelist import platform import types import functools +from importlib import import_module import inspect -from .py26compat import import_module from setuptools.extern import six import setuptools @@ -75,8 +75,6 @@ def patch_all(): needs_warehouse = ( sys.version_info < (2, 7, 13) or - (3, 0) < sys.version_info < (3, 3, 7) - or (3, 4) < sys.version_info < (3, 4, 6) or (3, 5) < sys.version_info <= (3, 5, 3) @@ -86,8 +84,7 @@ def patch_all(): warehouse = 'https://upload.pypi.org/legacy/' distutils.config.PyPIRCCommand.DEFAULT_REPOSITORY = warehouse - _patch_distribution_metadata_write_pkg_file() - _patch_distribution_metadata_write_pkg_info() + _patch_distribution_metadata() # Install Distribution throughout the distutils for module in distutils.dist, distutils.core, distutils.cmd: @@ -104,26 +101,11 @@ def patch_all(): patch_for_msvc_specialized_compiler() -def _patch_distribution_metadata_write_pkg_file(): - """Patch write_pkg_file to also write Requires-Python/Requires-External""" - distutils.dist.DistributionMetadata.write_pkg_file = ( - setuptools.dist.write_pkg_file - ) - - -def _patch_distribution_metadata_write_pkg_info(): - """ - Workaround issue #197 - Python 3 prior to 3.2.2 uses an environment-local - encoding to save the pkg_info. Monkey-patch its write_pkg_info method to - correct this undesirable behavior. - """ - environment_local = (3,) <= sys.version_info[:3] < (3, 2, 2) - if not environment_local: - return - - distutils.dist.DistributionMetadata.write_pkg_info = ( - setuptools.dist.write_pkg_info - ) +def _patch_distribution_metadata(): + """Patch write_pkg_file and read_pkg_file for higher metadata standards""" + for attr in ('write_pkg_file', 'read_pkg_file', 'get_metadata_version'): + new_val = getattr(setuptools.dist, attr) + setattr(distutils.dist.DistributionMetadata, attr, new_val) def patch_func(replacement, target_mod, func_name): diff --git a/setuptools/msvc.py b/setuptools/msvc.py index 8e3b638f..b9c472f1 100644 --- a/setuptools/msvc.py +++ b/setuptools/msvc.py @@ -22,7 +22,7 @@ import sys import platform import itertools import distutils.errors -from pkg_resources.extern.packaging.version import LegacyVersion +from setuptools.extern.packaging.version import LegacyVersion from setuptools.extern.six.moves import filterfalse @@ -48,7 +48,7 @@ else: _msvc9_suppress_errors = ( # msvc9compiler isn't available on some platforms ImportError, - + # msvc9compiler raises DistutilsPlatformError in some # environments. See #1118. distutils.errors.DistutilsPlatformError, @@ -232,8 +232,7 @@ def _augment_exception(exc, version, arch=''): elif version >= 14.0: # For VC++ 14.0 Redirect user to Visual C++ Build Tools message += (' Get it with "Microsoft Visual C++ Build Tools": ' - r'http://landinghub.visualstudio.com/' - 'visual-cpp-build-tools') + r'https://visualstudio.microsoft.com/downloads/') exc.args = (message, ) diff --git a/setuptools/namespaces.py b/setuptools/namespaces.py index dc16106d..dc16106d 100755..100644 --- a/setuptools/namespaces.py +++ b/setuptools/namespaces.py diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a6363b18..7e9517ce 100755..100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -7,13 +7,9 @@ import socket import base64 import hashlib import itertools +import warnings from functools import wraps -try: - from urllib.parse import splituser -except ImportError: - from urllib2 import splituser - from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client, configparser, map @@ -21,21 +17,23 @@ import setuptools from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, Environment, find_distributions, safe_name, safe_version, - to_filename, Requirement, DEVELOP_DIST, + to_filename, Requirement, DEVELOP_DIST, EGG_DIST, ) from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError from fnmatch import translate -from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers +from setuptools.py33compat import unescape +from setuptools.wheel import Wheel + +__metaclass__ = type EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') -HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) -# this is here to fix emacs' cruddy broken syntax highlighting +HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) PYPI_MD5 = re.compile( - '<a href="([^"#]+)">([^<]+)</a>\n\\s+\\(<a (?:title="MD5 hash"\n\\s+)' - 'href="[^?]+\\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\\)' + r'<a href="([^"#]+)">([^<]+)</a>\n\s+\(<a (?:title="MD5 hash"\n\s+)' + r'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\)' ) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -116,6 +114,17 @@ def distros_for_location(location, basename, metadata=None): if basename.endswith('.egg') and '-' in basename: # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] + if basename.endswith('.whl') and '-' in basename: + wheel = Wheel(basename) + if not wheel.is_compatible(): + return [] + return [Distribution( + location=location, + project_name=wheel.project_name, + version=wheel.version, + # Increase priority over eggs. + precedence=EGG_DIST + 1, + )] if basename.endswith('.exe'): win_base, py_ver, platform = parse_bdist_wininst(basename) if win_base is not None: @@ -141,7 +150,7 @@ def distros_for_filename(filename, metadata=None): def interpret_distro_name( location, basename, metadata, py_version=None, precedence=SOURCE_DIST, platform=None - ): +): """Generate alternative interpretations of a source distro name Note: if `location` is a filesystem filename, you should call @@ -228,7 +237,7 @@ def find_external_links(url, page): yield urllib.parse.urljoin(url, htmldecode(match.group(1))) -class ContentChecker(object): +class ContentChecker: """ A null content checker that defines the interface for checking content """ @@ -290,9 +299,9 @@ class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" def __init__( - self, index_url="https://pypi.python.org/simple", hosts=('*',), + self, index_url="https://pypi.org/simple/", hosts=('*',), ca_bundle=None, verify_ssl=True, *args, **kw - ): + ): Environment.__init__(self, *args, **kw) self.index_url = index_url + "/" [:not index_url.endswith('/')] self.scanned_urls = {} @@ -346,7 +355,8 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. + if not isinstance(page, str): + # In Python 3 and got bytes but want str. if isinstance(f, urllib.error.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' @@ -381,8 +391,9 @@ class PackageIndex(Environment): is_file = s and s.group(1).lower() == 'file' if is_file or self.allows(urllib.parse.urlparse(url)[1]): return True - msg = ("\nNote: Bypassing %s (disallowed host; see " - "http://bit.ly/1dg9ijs for details).\n") + msg = ( + "\nNote: Bypassing %s (disallowed host; see " + "http://bit.ly/2hrImnY for details).\n") if fatal: raise DistutilsError(msg % url) else: @@ -500,15 +511,16 @@ class PackageIndex(Environment): """ checker is a ContentChecker """ - checker.report(self.debug, + checker.report( + self.debug, "Validating %%s checksum for %s" % filename) if not checker.is_valid(): tfp.close() os.unlink(filename) raise DistutilsError( "%s validation failed for %s; " - "possible download problem?" % ( - checker.hash.name, os.path.basename(filename)) + "possible download problem?" + % (checker.hash.name, os.path.basename(filename)) ) def add_find_links(self, urls): @@ -536,7 +548,8 @@ class PackageIndex(Environment): if self[requirement.key]: # we've seen at least one distro meth, msg = self.info, "Couldn't retrieve index page for %r" else: # no distros seen for this name, might be misspelled - meth, msg = (self.warn, + meth, msg = ( + self.warn, "Couldn't find index page for %r (maybe misspelled?)") meth(msg, requirement.unsafe_name) self.scan_all() @@ -577,8 +590,7 @@ class PackageIndex(Environment): def fetch_distribution( self, requirement, tmpdir, force_scan=False, source=False, - develop_ok=False, local_index=None - ): + develop_ok=False, local_index=None): """Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. @@ -609,12 +621,19 @@ class PackageIndex(Environment): if dist.precedence == DEVELOP_DIST and not develop_ok: if dist not in skipped: - self.warn("Skipping development or system egg: %s", dist) + self.warn( + "Skipping development or system egg: %s", dist, + ) skipped[dist] = 1 continue - if dist in req and (dist.precedence <= SOURCE_DIST or not source): - dist.download_location = self.download(dist.location, tmpdir) + test = ( + dist in req + and (dist.precedence <= SOURCE_DIST or not source) + ) + if test: + loc = self.download(dist.location, tmpdir) + dist.download_location = loc if os.path.exists(dist.download_location): return dist @@ -704,10 +723,10 @@ class PackageIndex(Environment): def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file - fp, info = None, None + fp = None try: checker = HashChecker.from_url(url) - fp = self.open_url(strip_fragment(url)) + fp = self.open_url(url) if isinstance(fp, urllib.error.HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code, fp.msg) @@ -830,6 +849,10 @@ class PackageIndex(Environment): raise DistutilsError("Unexpected HTML page found at " + url) def _download_svn(self, url, filename): + warnings.warn("SVN download support is deprecated", UserWarning) + def splituser(host): + user, delim, host = host.rpartition('@') + return user, host url = url.split('#', 1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: @@ -915,27 +938,20 @@ class PackageIndex(Environment): entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub -def uchr(c): - if not isinstance(c, int): - return c - if c > 255: - return six.unichr(c) - return chr(c) - - def decode_entity(match): - what = match.group(1) - if what.startswith('#x'): - what = int(what[2:], 16) - elif what.startswith('#'): - what = int(what[1:]) - else: - what = six.moves.html_entities.name2codepoint.get(what, match.group(0)) - return uchr(what) + what = match.group(0) + return unescape(what) def htmldecode(text): - """Decode HTML entities in the given text.""" + """ + Decode HTML entities in the given text. + + >>> htmldecode( + ... 'https://../package_name-0.1.2.tar.gz' + ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') + 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' + """ return entity_sub(decode_entity, text) @@ -969,15 +985,14 @@ def _encode_auth(auth): auth_s = urllib.parse.unquote(auth) # convert to bytes auth_bytes = auth_s.encode() - # use the legacy interface for Python 2.3 support - encoded_bytes = base64.encodestring(auth_bytes) + encoded_bytes = base64.b64encode(auth_bytes) # convert back to a string encoded = encoded_bytes.decode() # strip the trailing carriage return return encoded.replace('\n', '') -class Credential(object): +class Credential: """ A username/password pair. Use like a namedtuple. """ @@ -1035,15 +1050,16 @@ class PyPIConfig(configparser.RawConfigParser): def open_with_auth(url, opener=urllib.request.urlopen): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url) + parsed = urllib.parse.urlparse(url) + scheme, netloc, path, params, query, frag = parsed # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): raise http_client.InvalidURL("nonnumeric port: ''") - if scheme in ('http', 'https'): - auth, host = splituser(netloc) + if scheme in ('http', 'https') and parsed.username: + auth = ':'.join((parsed.username, parsed.password)) else: auth = None @@ -1056,7 +1072,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): if auth: auth = "Basic " + _encode_auth(auth) - parts = scheme, host, path, params, query, frag + parts = scheme, parsed.hostname, path, params, query, frag new_url = urllib.parse.urlunparse(parts) request = urllib.request.Request(new_url) request.add_header("Authorization", auth) @@ -1070,7 +1086,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) - if s2 == scheme and h2 == host: + if s2 == scheme and h2 == parsed.hostname: parts = s2, netloc, path2, param2, query2, frag2 fp.url = urllib.parse.urlunparse(parts) @@ -1103,7 +1119,8 @@ def local_open(url): f += '/' files.append('<a href="{name}">{name}</a>'.format(name=f)) else: - tmpl = ("<html><head><title>{url}</title>" + tmpl = ( + "<html><head><title>{url}</title>" "</head><body>{files}</body></html>") body = tmpl.format(url=url, files='\n'.join(files)) status, message = 200, "OK" diff --git a/setuptools/pep425tags.py b/setuptools/pep425tags.py new file mode 100644 index 00000000..48745a29 --- /dev/null +++ b/setuptools/pep425tags.py @@ -0,0 +1,319 @@ +# This file originally from pip: +# https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/pep425tags.py +"""Generate and work with PEP 425 Compatibility Tags.""" +from __future__ import absolute_import + +import distutils.util +from distutils import log +import platform +import re +import sys +import sysconfig +import warnings +from collections import OrderedDict + +from .extern import six + +from . import glibc + +_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') + + +def get_config_var(var): + try: + return sysconfig.get_config_var(var) + except IOError as e: # Issue #1074 + warnings.warn("{}".format(e), RuntimeWarning) + return None + + +def get_abbr_impl(): + """Return abbreviated implementation name.""" + if hasattr(sys, 'pypy_version_info'): + pyimpl = 'pp' + elif sys.platform.startswith('java'): + pyimpl = 'jy' + elif sys.platform == 'cli': + pyimpl = 'ip' + else: + pyimpl = 'cp' + return pyimpl + + +def get_impl_ver(): + """Return implementation version.""" + impl_ver = get_config_var("py_version_nodot") + if not impl_ver or get_abbr_impl() == 'pp': + impl_ver = ''.join(map(str, get_impl_version_info())) + return impl_ver + + +def get_impl_version_info(): + """Return sys.version_info-like tuple for use in decrementing the minor + version.""" + if get_abbr_impl() == 'pp': + # as per https://github.com/pypa/pip/issues/2882 + return (sys.version_info[0], sys.pypy_version_info.major, + sys.pypy_version_info.minor) + else: + return sys.version_info[0], sys.version_info[1] + + +def get_impl_tag(): + """ + Returns the Tag for this specific implementation. + """ + return "{}{}".format(get_abbr_impl(), get_impl_ver()) + + +def get_flag(var, fallback, expected=True, warn=True): + """Use a fallback method for determining SOABI flags if the needed config + var is unset or unavailable.""" + val = get_config_var(var) + if val is None: + if warn: + log.debug("Config variable '%s' is unset, Python ABI tag may " + "be incorrect", var) + return fallback() + return val == expected + + +def get_abi_tag(): + """Return the ABI tag based on SOABI (if available) or emulate SOABI + (CPython 2, PyPy).""" + soabi = get_config_var('SOABI') + impl = get_abbr_impl() + if not soabi and impl in {'cp', 'pp'} and hasattr(sys, 'maxunicode'): + d = '' + m = '' + u = '' + if get_flag('Py_DEBUG', + lambda: hasattr(sys, 'gettotalrefcount'), + warn=(impl == 'cp')): + d = 'd' + if get_flag('WITH_PYMALLOC', + lambda: impl == 'cp', + warn=(impl == 'cp')): + m = 'm' + if get_flag('Py_UNICODE_SIZE', + lambda: sys.maxunicode == 0x10ffff, + expected=4, + warn=(impl == 'cp' and + six.PY2)) \ + and six.PY2: + u = 'u' + abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u) + elif soabi and soabi.startswith('cpython-'): + abi = 'cp' + soabi.split('-')[1] + elif soabi: + abi = soabi.replace('.', '_').replace('-', '_') + else: + abi = None + return abi + + +def _is_running_32bit(): + return sys.maxsize == 2147483647 + + +def get_platform(): + """Return our platform name 'win32', 'linux_x86_64'""" + if sys.platform == 'darwin': + # distutils.util.get_platform() returns the release based on the value + # of MACOSX_DEPLOYMENT_TARGET on which Python was built, which may + # be significantly older than the user's current machine. + release, _, machine = platform.mac_ver() + split_ver = release.split('.') + + if machine == "x86_64" and _is_running_32bit(): + machine = "i386" + elif machine == "ppc64" and _is_running_32bit(): + machine = "ppc" + + return 'macosx_{}_{}_{}'.format(split_ver[0], split_ver[1], machine) + + # XXX remove distutils dependency + result = distutils.util.get_platform().replace('.', '_').replace('-', '_') + if result == "linux_x86_64" and _is_running_32bit(): + # 32 bit Python program (running on a 64 bit Linux): pip should only + # install and run 32 bit compiled extensions in that case. + result = "linux_i686" + + return result + + +def is_manylinux1_compatible(): + # Only Linux, and only x86-64 / i686 + if get_platform() not in {"linux_x86_64", "linux_i686"}: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux1_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 5 uses glibc 2.5. + return glibc.have_compatible_glibc(2, 5) + + +def get_darwin_arches(major, minor, machine): + """Return a list of supported arches (including group arches) for + the given major, minor and machine architecture of a macOS machine. + """ + arches = [] + + def _supports_arch(major, minor, arch): + # Looking at the application support for macOS versions in the chart + # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears + # our timeline looks roughly like: + # + # 10.0 - Introduces ppc support. + # 10.4 - Introduces ppc64, i386, and x86_64 support, however the ppc64 + # and x86_64 support is CLI only, and cannot be used for GUI + # applications. + # 10.5 - Extends ppc64 and x86_64 support to cover GUI applications. + # 10.6 - Drops support for ppc64 + # 10.7 - Drops support for ppc + # + # Given that we do not know if we're installing a CLI or a GUI + # application, we must be conservative and assume it might be a GUI + # application and behave as if ppc64 and x86_64 support did not occur + # until 10.5. + # + # Note: The above information is taken from the "Application support" + # column in the chart not the "Processor support" since I believe + # that we care about what instruction sets an application can use + # not which processors the OS supports. + if arch == 'ppc': + return (major, minor) <= (10, 5) + if arch == 'ppc64': + return (major, minor) == (10, 5) + if arch == 'i386': + return (major, minor) >= (10, 4) + if arch == 'x86_64': + return (major, minor) >= (10, 5) + if arch in groups: + for garch in groups[arch]: + if _supports_arch(major, minor, garch): + return True + return False + + groups = OrderedDict([ + ("fat", ("i386", "ppc")), + ("intel", ("x86_64", "i386")), + ("fat64", ("x86_64", "ppc64")), + ("fat32", ("x86_64", "i386", "ppc")), + ]) + + if _supports_arch(major, minor, machine): + arches.append(machine) + + for garch in groups: + if machine in groups[garch] and _supports_arch(major, minor, garch): + arches.append(garch) + + arches.append('universal') + + return arches + + +def get_supported(versions=None, noarch=False, platform=None, + impl=None, abi=None): + """Return a list of supported tags for each version specified in + `versions`. + + :param versions: a list of string versions, of the form ["33", "32"], + or None. The first version will be assumed to support our ABI. + :param platform: specify the exact platform you want valid + tags for, or None. If None, use the local system platform. + :param impl: specify the exact implementation you want valid + tags for, or None. If None, use the local interpreter impl. + :param abi: specify the exact abi you want valid + tags for, or None. If None, use the local interpreter abi. + """ + supported = [] + + # Versions must be given with respect to the preference + if versions is None: + versions = [] + version_info = get_impl_version_info() + major = version_info[:-1] + # Support all previous minor Python versions. + for minor in range(version_info[-1], -1, -1): + versions.append(''.join(map(str, major + (minor,)))) + + impl = impl or get_abbr_impl() + + abis = [] + + abi = abi or get_abi_tag() + if abi: + abis[0:0] = [abi] + + abi3s = set() + import imp + for suffix in imp.get_suffixes(): + if suffix[0].startswith('.abi'): + abi3s.add(suffix[0].split('.', 2)[1]) + + abis.extend(sorted(list(abi3s))) + + abis.append('none') + + if not noarch: + arch = platform or get_platform() + if arch.startswith('macosx'): + # support macosx-10.6-intel on macosx-10.9-x86_64 + match = _osx_arch_pat.match(arch) + if match: + name, major, minor, actual_arch = match.groups() + tpl = '{}_{}_%i_%s'.format(name, major) + arches = [] + for m in reversed(range(int(minor) + 1)): + for a in get_darwin_arches(int(major), m, actual_arch): + arches.append(tpl % (m, a)) + else: + # arch pattern didn't match (?!) + arches = [arch] + elif platform is None and is_manylinux1_compatible(): + arches = [arch.replace('linux', 'manylinux1'), arch] + else: + arches = [arch] + + # Current version, current API (built specifically for our Python): + for abi in abis: + for arch in arches: + supported.append(('%s%s' % (impl, versions[0]), abi, arch)) + + # abi3 modules compatible with older version of Python + for version in versions[1:]: + # abi3 was introduced in Python 3.2 + if version in {'31', '30'}: + break + for abi in abi3s: # empty set if not Python 3 + for arch in arches: + supported.append(("%s%s" % (impl, version), abi, arch)) + + # Has binaries, does not use the Python API: + for arch in arches: + supported.append(('py%s' % (versions[0][0]), 'none', arch)) + + # No abi / arch, but requires our implementation: + supported.append(('%s%s' % (impl, versions[0]), 'none', 'any')) + # Tagged specifically as being cross-version compatible + # (with just the major version specified) + supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any')) + + # No abi / arch, generic Python + for i, version in enumerate(versions): + supported.append(('py%s' % (version,), 'none', 'any')) + if i == 0: + supported.append(('py%s' % (version[0]), 'none', 'any')) + + return supported + + +implementation_tag = get_impl_tag() diff --git a/setuptools/py26compat.py b/setuptools/py26compat.py deleted file mode 100644 index 4d3add8c..00000000 --- a/setuptools/py26compat.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Compatibility Support for Python 2.6 and earlier -""" - -import sys - -try: - from urllib.parse import splittag -except ImportError: - from urllib import splittag - - -def strip_fragment(url): - """ - In `Python 8280 <http://bugs.python.org/issue8280>`_, Python 2.7 and - later was patched to disregard the fragment when making URL requests. - Do the same for Python 2.6 and earlier. - """ - url, fragment = splittag(url) - return url - - -if sys.version_info >= (2, 7): - strip_fragment = lambda x: x - -try: - from importlib import import_module -except ImportError: - - def import_module(module_name): - return __import__(module_name, fromlist=['__name__']) diff --git a/setuptools/py31compat.py b/setuptools/py31compat.py index 44b025d4..1a0705ec 100644 --- a/setuptools/py31compat.py +++ b/setuptools/py31compat.py @@ -1,18 +1,6 @@ -import sys -import unittest +__all__ = [] -__all__ = ['get_config_vars', 'get_path'] - -try: - # Python 2.7 or >=3.2 - from sysconfig import get_config_vars, get_path -except ImportError: - from distutils.sysconfig import get_config_vars, get_python_lib - - def get_path(name): - if name not in ('platlib', 'purelib'): - raise ValueError("Name must be purelib or platlib") - return get_python_lib(name == 'platlib') +__metaclass__ = type try: @@ -22,7 +10,7 @@ except ImportError: import shutil import tempfile - class TemporaryDirectory(object): + class TemporaryDirectory: """ Very simple temporary directory context manager. Will try to delete afterward, but will also ignore OS and similar @@ -42,15 +30,3 @@ except ImportError: except OSError: # removal errors are not the only possible pass self.name = None - - -unittest_main = unittest.main - -_PY31 = (3, 1) <= sys.version_info[:2] < (3, 2) -if _PY31: - # on Python 3.1, translate testRunner==None to TextTestRunner - # for compatibility with Python 2.6, 2.7, and 3.2+ - def unittest_main(*args, **kwargs): - if 'testRunner' in kwargs and kwargs['testRunner'] is None: - kwargs['testRunner'] = unittest.TextTestRunner - return unittest.main(*args, **kwargs) diff --git a/setuptools/py33compat.py b/setuptools/py33compat.py index af64d5d1..87cf5398 100644 --- a/setuptools/py33compat.py +++ b/setuptools/py33compat.py @@ -2,13 +2,20 @@ import dis import array import collections +try: + import html +except ImportError: + html = None + from setuptools.extern import six +from setuptools.extern.six.moves import html_parser +__metaclass__ = type OpArg = collections.namedtuple('OpArg', 'opcode arg') -class Bytecode_compat(object): +class Bytecode_compat: def __init__(self, code): self.code = code @@ -43,3 +50,6 @@ class Bytecode_compat(object): Bytecode = getattr(dis, 'Bytecode', Bytecode_compat) + + +unescape = getattr(html, 'unescape', html_parser.HTMLParser().unescape) diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 1d981f49..685f3f72 100755..100644 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -39,10 +39,6 @@ def _execfile(filename, globals, locals=None): mode = 'rb' with open(filename, mode) as stream: script = stream.read() - # compile() function in Python 2.6 and 3.1 requires LF line endings. - if sys.version_info[:2] < (2, 7) or sys.version_info[:2] >= (3, 0) and sys.version_info[:2] < (3, 2): - script = script.replace(b'\r\n', b'\n') - script = script.replace(b'\r', b'\n') if locals is None: locals = globals code = compile(script, filename, 'exec') diff --git a/setuptools/script (dev).tmpl b/setuptools/script (dev).tmpl index d58b1bb5..39a24b04 100644 --- a/setuptools/script (dev).tmpl +++ b/setuptools/script (dev).tmpl @@ -2,4 +2,5 @@ __requires__ = %(spec)r __import__('pkg_resources').require(%(spec)r) __file__ = %(dev_path)r -exec(compile(open(__file__).read(), __file__, 'exec')) +with open(__file__) as f: + exec(compile(f.read(), __file__, 'exec')) diff --git a/setuptools/site-patch.py b/setuptools/site-patch.py index 0d2d2ff8..40b00de0 100644 --- a/setuptools/site-patch.py +++ b/setuptools/site-patch.py @@ -23,7 +23,7 @@ def __boot(): break else: try: - import imp # Avoid import loop in Python >= 3.3 + import imp # Avoid import loop in Python 3 stream, path, descr = imp.find_module('site', [item]) except ImportError: continue diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py index 72b18ef2..226db694 100644 --- a/setuptools/ssl_support.py +++ b/setuptools/ssl_support.py @@ -59,7 +59,7 @@ if not match_hostname: def _dnsname_match(dn, hostname, max_wildcards=1): """Matching according to RFC 6125, section 6.4.3 - http://tools.ietf.org/html/rfc6125#section-6.4.3 + https://tools.ietf.org/html/rfc6125#section-6.4.3 """ pats = [] if not dn: @@ -186,9 +186,14 @@ class VerifyingHTTPSConn(HTTPSConnection): else: actual_host = self.host - self.sock = ssl.wrap_socket( - sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_bundle - ) + if hasattr(ssl, 'create_default_context'): + ctx = ssl.create_default_context(cafile=self.ca_bundle) + self.sock = ctx.wrap_socket(sock, server_hostname=actual_host) + else: + # This is for python < 2.7.9 and < 3.4? + self.sock = ssl.wrap_socket( + sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_bundle + ) try: match_hostname(self.sock.getpeercert(), actual_host) except CertificateError: diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index 54dd7d2b..5f4a1c29 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -2,5 +2,17 @@ import locale import pytest +from setuptools.extern.six import PY2, PY3 + + +__all__ = [ + 'fail_on_ascii', 'py2_only', 'py3_only' +] + + is_ascii = locale.getpreferredencoding() == 'ANSI_X3.4-1968' fail_on_ascii = pytest.mark.xfail(is_ascii, reason="Test fails in this locale") + + +py2_only = pytest.mark.skipif(not PY2, reason="Test runs on Python 2 only") +py3_only = pytest.mark.skipif(not PY3, reason="Test runs on Python 3 only") diff --git a/setuptools/tests/files.py b/setuptools/tests/files.py index 98de9fc3..465a6b41 100644 --- a/setuptools/tests/files.py +++ b/setuptools/tests/files.py @@ -30,5 +30,9 @@ def build_files(file_defs, prefix=""): pkg_resources.py31compat.makedirs(full_name, exist_ok=True) build_files(contents, prefix=full_name) else: - with open(full_name, 'w') as f: - f.write(contents) + if isinstance(contents, bytes): + with open(full_name, 'wb') as f: + f.write(contents) + else: + with open(full_name, 'w') as f: + f.write(contents) diff --git a/setuptools/tests/py26compat.py b/setuptools/tests/py26compat.py deleted file mode 100644 index 18cece05..00000000 --- a/setuptools/tests/py26compat.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys -import tarfile -import contextlib - - -def _tarfile_open_ex(*args, **kwargs): - """ - Extend result as a context manager. - """ - return contextlib.closing(tarfile.open(*args, **kwargs)) - - -if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 2): - tarfile_open = _tarfile_open_ex -else: - tarfile_open = tarfile.open diff --git a/setuptools/tests/test_bdist_egg.py b/setuptools/tests/test_bdist_egg.py index d24aa366..54742aa6 100644 --- a/setuptools/tests/test_bdist_egg.py +++ b/setuptools/tests/test_bdist_egg.py @@ -2,6 +2,7 @@ """ import os import re +import zipfile import pytest @@ -16,7 +17,7 @@ setup(name='foo', py_modules=['hi']) """ -@pytest.yield_fixture +@pytest.fixture(scope='function') def setup_context(tmpdir): with (tmpdir / 'setup.py').open('w') as f: f.write(SETUP_PY) @@ -32,7 +33,7 @@ class Test: script_name='setup.py', script_args=['bdist_egg'], name='foo', - py_modules=['hi'] + py_modules=['hi'], )) os.makedirs(os.path.join('build', 'src')) with contexts.quiet(): @@ -42,3 +43,24 @@ class Test: # let's see if we got our egg link at the right place [content] = os.listdir('dist') assert re.match(r'foo-0.0.0-py[23].\d.egg$', content) + + @pytest.mark.xfail( + os.environ.get('PYTHONDONTWRITEBYTECODE'), + reason="Byte code disabled", + ) + def test_exclude_source_files(self, setup_context, user_override): + dist = Distribution(dict( + script_name='setup.py', + script_args=['bdist_egg', '--exclude-source-files'], + name='foo', + py_modules=['hi'], + )) + with contexts.quiet(): + dist.parse_command_line() + dist.run_commands() + [dist_name] = os.listdir('dist') + dist_filename = os.path.join('dist', dist_name) + zip = zipfile.ZipFile(dist_filename) + names = list(zi.filename for zi in zip.filelist) + assert 'hi.pyc' in names + assert 'hi.py' not in names diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 69a700c2..82b44c89 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -1,16 +1,24 @@ +from __future__ import unicode_literals + import os +import shutil +import tarfile import pytest +from setuptools.build_meta import build_sdist from .files import build_files from .textwrap import DALS +from . import py2_only +__metaclass__ = type -futures = pytest.importorskip('concurrent.futures') -importlib = pytest.importorskip('importlib') +# Backports on Python 2.7 +import importlib +from concurrent import futures -class BuildBackendBase(object): +class BuildBackendBase: def __init__(self, cwd=None, env={}, backend_name='setuptools.build_meta'): self.cwd = cwd self.env = env @@ -19,12 +27,14 @@ class BuildBackendBase(object): class BuildBackend(BuildBackendBase): """PEP 517 Build Backend""" + def __init__(self, *args, **kwargs): super(BuildBackend, self).__init__(*args, **kwargs) self.pool = futures.ProcessPoolExecutor() def __getattr__(self, name): """Handles aribrary function invocations on the build backend.""" + def method(*args, **kw): root = os.path.abspath(self.cwd) caller = BuildBackendCaller(root, self.env, self.backend_name) @@ -42,12 +52,12 @@ class BuildBackendCaller(BuildBackendBase): return getattr(mod, name)(*args, **kw) -@pytest.fixture -def build_backend(tmpdir): - defn = { +defns = [ + { 'setup.py': DALS(""" __import__('setuptools').setup( name='foo', + version='0.0.0', py_modules=['hello'], setup_requires=['six'], ) @@ -56,15 +66,59 @@ def build_backend(tmpdir): def run(): print('hello') """), - } - build_files(defn, prefix=str(tmpdir)) + }, + { + 'setup.py': DALS(""" + assert __name__ == '__main__' + __import__('setuptools').setup( + name='foo', + version='0.0.0', + py_modules=['hello'], + setup_requires=['six'], + ) + """), + 'hello.py': DALS(""" + def run(): + print('hello') + """), + }, + { + 'setup.py': DALS(""" + variable = True + def function(): + return variable + assert variable + __import__('setuptools').setup( + name='foo', + version='0.0.0', + py_modules=['hello'], + setup_requires=['six'], + ) + """), + 'hello.py': DALS(""" + def run(): + print('hello') + """), + }, +] + + +@pytest.fixture(params=defns) +def build_backend(tmpdir, request): + build_files(request.param, prefix=str(tmpdir)) with tmpdir.as_cwd(): yield BuildBackend(cwd='.') def test_get_requires_for_build_wheel(build_backend): actual = build_backend.get_requires_for_build_wheel() - expected = ['six', 'setuptools', 'wheel'] + expected = ['six', 'wheel'] + assert sorted(actual) == sorted(expected) + + +def test_get_requires_for_build_sdist(build_backend): + actual = build_backend.get_requires_for_build_sdist() + expected = ['six'] assert sorted(actual) == sorted(expected) @@ -91,3 +145,94 @@ def test_prepare_metadata_for_build_wheel(build_backend): dist_info = build_backend.prepare_metadata_for_build_wheel(dist_dir) assert os.path.isfile(os.path.join(dist_dir, dist_info, 'METADATA')) + + +@py2_only +def test_prepare_metadata_for_build_wheel_with_str(build_backend): + dist_dir = os.path.abspath(str('pip-dist-info')) + os.makedirs(dist_dir) + + dist_info = build_backend.prepare_metadata_for_build_wheel(dist_dir) + + assert os.path.isfile(os.path.join(dist_dir, dist_info, 'METADATA')) + + +def test_build_sdist_explicit_dist(build_backend): + # explicitly specifying the dist folder should work + # the folder sdist_directory and the ``--dist-dir`` can be the same + dist_dir = os.path.abspath('dist') + sdist_name = build_backend.build_sdist(dist_dir) + assert os.path.isfile(os.path.join(dist_dir, sdist_name)) + + +def test_build_sdist_version_change(build_backend): + sdist_into_directory = os.path.abspath("out_sdist") + os.makedirs(sdist_into_directory) + + sdist_name = build_backend.build_sdist(sdist_into_directory) + assert os.path.isfile(os.path.join(sdist_into_directory, sdist_name)) + + # if the setup.py changes subsequent call of the build meta + # should still succeed, given the + # sdist_directory the frontend specifies is empty + with open(os.path.abspath("setup.py"), 'rt') as file_handler: + content = file_handler.read() + with open(os.path.abspath("setup.py"), 'wt') as file_handler: + file_handler.write( + content.replace("version='0.0.0'", "version='0.0.1'")) + + shutil.rmtree(sdist_into_directory) + os.makedirs(sdist_into_directory) + + sdist_name = build_backend.build_sdist("out_sdist") + assert os.path.isfile( + os.path.join(os.path.abspath("out_sdist"), sdist_name)) + + +def test_build_sdist_setup_py_exists(tmpdir_cwd): + # If build_sdist is called from a script other than setup.py, + # ensure setup.py is include + build_files(defns[0]) + targz_path = build_sdist("temp") + with tarfile.open(os.path.join("temp", targz_path)) as tar: + assert any('setup.py' in name for name in tar.getnames()) + + +def test_build_sdist_setup_py_manifest_excluded(tmpdir_cwd): + # Ensure that MANIFEST.in can exclude setup.py + files = { + 'setup.py': DALS(""" + __import__('setuptools').setup( + name='foo', + version='0.0.0', + py_modules=['hello'] + )"""), + 'hello.py': '', + 'MANIFEST.in': DALS(""" + exclude setup.py + """) + } + + build_files(files) + targz_path = build_sdist("temp") + with tarfile.open(os.path.join("temp", targz_path)) as tar: + assert not any('setup.py' in name for name in tar.getnames()) + + +def test_build_sdist_builds_targz_even_if_zip_indicated(tmpdir_cwd): + files = { + 'setup.py': DALS(""" + __import__('setuptools').setup( + name='foo', + version='0.0.0', + py_modules=['hello'] + )"""), + 'hello.py': '', + 'setup.cfg': DALS(""" + [sdist] + formats=zip + """) + } + + build_files(files) + build_sdist("temp") diff --git a/setuptools/tests/test_config.py b/setuptools/tests/test_config.py index 89fde257..0b8ae158 100644 --- a/setuptools/tests/test_config.py +++ b/setuptools/tests/test_config.py @@ -3,25 +3,32 @@ from __future__ import unicode_literals import contextlib import pytest + from distutils.errors import DistutilsOptionError, DistutilsFileError -from setuptools.dist import Distribution +from mock import patch +from setuptools.dist import Distribution, _Distribution from setuptools.config import ConfigHandler, read_configuration from setuptools.extern.six.moves.configparser import InterpolationMissingOptionError from setuptools.tests import is_ascii - +from . import py2_only, py3_only +from .textwrap import DALS class ErrConfigHandler(ConfigHandler): """Erroneous handler. Fails to implement required methods.""" -def make_package_dir(name, base_dir): - dir_package = base_dir.mkdir(name) - init_file = dir_package.join('__init__.py') - init_file.write('') +def make_package_dir(name, base_dir, ns=False): + dir_package = base_dir + for dir_name in name.split('/'): + dir_package = dir_package.mkdir(dir_name) + init_file = None + if not ns: + init_file = dir_package.join('__init__.py') + init_file.write('') return dir_package, init_file -def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii'): +def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii', package_path='fake_package'): if setup_py is None: setup_py = ( @@ -33,7 +40,7 @@ def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii'): config = tmpdir.join('setup.cfg') config.write(setup_cfg.encode(encoding), mode='wb') - package_dir, init_file = make_package_dir('fake_package', tmpdir) + package_dir, init_file = make_package_dir(package_path, tmpdir) init_file.write( 'VERSION = (1, 2, 3)\n' @@ -115,6 +122,7 @@ class TestMetadata: '[metadata]\n' 'version = 10.1.1\n' 'description = Some description\n' + 'long_description_content_type = text/something\n' 'long_description = file: README\n' 'name = fake_name\n' 'keywords = one, two\n' @@ -136,6 +144,7 @@ class TestMetadata: assert metadata.version == '10.1.1' assert metadata.description == 'Some description' + assert metadata.long_description_content_type == 'text/something' assert metadata.long_description == 'readme contents\nline2' assert metadata.provides == ['package', 'package.sub'] assert metadata.license == 'BSD 3-Clause License' @@ -144,6 +153,24 @@ class TestMetadata: assert metadata.download_url == 'http://test.test.com/test/' assert metadata.maintainer_email == 'test@test.com' + def test_license_cfg(self, tmpdir): + fake_env( + tmpdir, + DALS(""" + [metadata] + name=foo + version=0.0.1 + license=Apache 2.0 + """) + ) + + with get_dist(tmpdir) as dist: + metadata = dist.metadata + + assert metadata.name == "foo" + assert metadata.version == "0.0.1" + assert metadata.license == "Apache 2.0" + def test_file_mixed(self, tmpdir): fake_env( @@ -220,6 +247,22 @@ class TestMetadata: 'Programming Language :: Python :: 3.5', ] + def test_dict(self, tmpdir): + + fake_env( + tmpdir, + '[metadata]\n' + 'project_urls =\n' + ' Link One = https://example.com/one/\n' + ' Link Two = https://example.com/two/\n' + ) + with get_dist(tmpdir) as dist: + metadata = dist.metadata + assert metadata.project_urls == { + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + } + def test_version(self, tmpdir): _, config = fake_env( @@ -255,6 +298,68 @@ class TestMetadata: with get_dist(tmpdir) as dist: assert dist.metadata.version == '2016.11.26' + def test_version_file(self, tmpdir): + + _, config = fake_env( + tmpdir, + '[metadata]\n' + 'version = file: fake_package/version.txt\n' + ) + tmpdir.join('fake_package', 'version.txt').write('1.2.3\n') + + with get_dist(tmpdir) as dist: + assert dist.metadata.version == '1.2.3' + + tmpdir.join('fake_package', 'version.txt').write('1.2.3\n4.5.6\n') + with pytest.raises(DistutilsOptionError): + with get_dist(tmpdir) as dist: + _ = dist.metadata.version + + def test_version_with_package_dir_simple(self, tmpdir): + + _, config = fake_env( + tmpdir, + '[metadata]\n' + 'version = attr: fake_package_simple.VERSION\n' + '[options]\n' + 'package_dir =\n' + ' = src\n', + package_path='src/fake_package_simple' + ) + + with get_dist(tmpdir) as dist: + assert dist.metadata.version == '1.2.3' + + def test_version_with_package_dir_rename(self, tmpdir): + + _, config = fake_env( + tmpdir, + '[metadata]\n' + 'version = attr: fake_package_rename.VERSION\n' + '[options]\n' + 'package_dir =\n' + ' fake_package_rename = fake_dir\n', + package_path='fake_dir' + ) + + with get_dist(tmpdir) as dist: + assert dist.metadata.version == '1.2.3' + + def test_version_with_package_dir_complex(self, tmpdir): + + _, config = fake_env( + tmpdir, + '[metadata]\n' + 'version = attr: fake_package_complex.VERSION\n' + '[options]\n' + 'package_dir =\n' + ' fake_package_complex = src/fake_dir\n', + package_path='src/fake_dir' + ) + + with get_dist(tmpdir) as dist: + assert dist.metadata.version == '1.2.3' + def test_unknown_meta_item(self, tmpdir): fake_env( @@ -311,6 +416,23 @@ class TestMetadata: with get_dist(tmpdir) as dist: assert set(dist.metadata.classifiers) == expected + def test_deprecated_config_handlers(self, tmpdir): + fake_env( + tmpdir, + '[metadata]\n' + 'version = 10.1.1\n' + 'description = Some description\n' + 'requires = some, requirement\n' + ) + + with pytest.deprecated_call(): + with get_dist(tmpdir) as dist: + metadata = dist.metadata + + assert metadata.version == '10.1.1' + assert metadata.description == 'Some description' + assert metadata.requires == ['some', 'requirement'] + def test_interpolation(self, tmpdir): fake_env( tmpdir, @@ -584,6 +706,60 @@ class TestOptions: assert set(dist.packages) == set( ['fake_package', 'fake_package.sub_two']) + @py2_only + def test_find_namespace_directive_fails_on_py2(self, tmpdir): + dir_package, config = fake_env( + tmpdir, + '[options]\n' + 'packages = find_namespace:\n' + ) + + with pytest.raises(DistutilsOptionError): + with get_dist(tmpdir) as dist: + dist.parse_config_files() + + @py3_only + def test_find_namespace_directive(self, tmpdir): + dir_package, config = fake_env( + tmpdir, + '[options]\n' + 'packages = find_namespace:\n' + ) + + dir_sub_one, _ = make_package_dir('sub_one', dir_package) + dir_sub_two, _ = make_package_dir('sub_two', dir_package, ns=True) + + with get_dist(tmpdir) as dist: + assert set(dist.packages) == { + 'fake_package', 'fake_package.sub_two', 'fake_package.sub_one' + } + + config.write( + '[options]\n' + 'packages = find_namespace:\n' + '\n' + '[options.packages.find]\n' + 'where = .\n' + 'include =\n' + ' fake_package.sub_one\n' + ' two\n' + ) + with get_dist(tmpdir) as dist: + assert dist.packages == ['fake_package.sub_one'] + + config.write( + '[options]\n' + 'packages = find_namespace:\n' + '\n' + '[options.packages.find]\n' + 'exclude =\n' + ' fake_package.sub_one\n' + ) + with get_dist(tmpdir) as dist: + assert set(dist.packages) == { + 'fake_package', 'fake_package.sub_two' + } + def test_extras_require(self, tmpdir): fake_env( tmpdir, @@ -599,6 +775,7 @@ class TestOptions: 'pdf': ['ReportLab>=1.2', 'RXP'], 'rest': ['docutils>=0.3', 'pack==1.1,==1.3'] } + assert dist.metadata.provides_extras == set(['pdf', 'rest']) def test_entry_points(self, tmpdir): _, config = fake_env( @@ -633,3 +810,60 @@ class TestOptions: with get_dist(tmpdir) as dist: assert dist.entry_points == expected + + def test_data_files(self, tmpdir): + fake_env( + tmpdir, + '[options.data_files]\n' + 'cfg =\n' + ' a/b.conf\n' + ' c/d.conf\n' + 'data = e/f.dat, g/h.dat\n' + ) + + with get_dist(tmpdir) as dist: + expected = [ + ('cfg', ['a/b.conf', 'c/d.conf']), + ('data', ['e/f.dat', 'g/h.dat']), + ] + assert sorted(dist.data_files) == sorted(expected) + +saved_dist_init = _Distribution.__init__ +class TestExternalSetters: + # During creation of the setuptools Distribution() object, we call + # the init of the parent distutils Distribution object via + # _Distribution.__init__ (). + # + # It's possible distutils calls out to various keyword + # implementations (i.e. distutils.setup_keywords entry points) + # that may set a range of variables. + # + # This wraps distutil's Distribution.__init__ and simulates + # pbr or something else setting these values. + def _fake_distribution_init(self, dist, attrs): + saved_dist_init(dist, attrs) + # see self._DISTUTUILS_UNSUPPORTED_METADATA + setattr(dist.metadata, 'long_description_content_type', + 'text/something') + # Test overwrite setup() args + setattr(dist.metadata, 'project_urls', { + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + }) + return None + + @patch.object(_Distribution, '__init__', autospec=True) + def test_external_setters(self, mock_parent_init, tmpdir): + mock_parent_init.side_effect = self._fake_distribution_init + + dist = Distribution(attrs={ + 'project_urls': { + 'will_be': 'ignored' + } + }) + + assert dist.metadata.long_description_content_type == 'text/something' + assert dist.metadata.project_urls == { + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + } diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index cb4ff4b4..00d4bd9a 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -8,6 +8,7 @@ import site import sys import io import subprocess +import platform from setuptools.extern import six from setuptools.command import test @@ -61,7 +62,8 @@ class TestDevelop: in_virtualenv = hasattr(sys, 'real_prefix') in_venv = hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix - @pytest.mark.skipif(in_virtualenv or in_venv, + @pytest.mark.skipif( + in_virtualenv or in_venv, reason="Cannot run when invoked in a virtualenv or venv") def test_2to3_user_mode(self, test_env): settings = dict( @@ -101,7 +103,8 @@ class TestDevelop: Test that console scripts are installed and that they reference only the project by name and not the current version. """ - pytest.skip("TODO: needs a fixture to cause 'develop' " + pytest.skip( + "TODO: needs a fixture to cause 'develop' " "to be invoked without mutating environment.") settings = dict( name='foo', @@ -153,8 +156,14 @@ class TestNamespaces: with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(develop_cmd) - @pytest.mark.skipif(bool(os.environ.get("APPVEYOR")), - reason="https://github.com/pypa/setuptools/issues/851") + @pytest.mark.skipif( + bool(os.environ.get("APPVEYOR")), + reason="https://github.com/pypa/setuptools/issues/851", + ) + @pytest.mark.skipif( + platform.python_implementation() == 'PyPy' and six.PY3, + reason="https://github.com/pypa/setuptools/issues/1202", + ) def test_namespace_package_importable(self, tmpdir): """ Installing two packages sharing the same namespace, one installed @@ -169,7 +178,7 @@ class TestNamespaces: install_cmd = [ sys.executable, '-m', - 'pip.__main__', + 'pip', 'install', str(pkg_A), '-t', str(target), diff --git a/setuptools/tests/test_dist.py b/setuptools/tests/test_dist.py index 435ffec0..cf830b43 100644 --- a/setuptools/tests/test_dist.py +++ b/setuptools/tests/test_dist.py @@ -1,10 +1,18 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +import io +from setuptools.dist import DistDeprecationWarning, _get_unpatched from setuptools import Distribution from setuptools.extern.six.moves.urllib.request import pathname2url from setuptools.extern.six.moves.urllib_parse import urljoin +from setuptools.extern import six from .textwrap import DALS from .test_easy_install import make_nspkg_sdist +import pytest def test_dist_fetch_build_egg(tmpdir): """ @@ -12,6 +20,7 @@ def test_dist_fetch_build_egg(tmpdir): """ index = tmpdir.mkdir('index') index_url = urljoin('file://', pathname2url(str(index))) + def sdist_with_index(distname, version): dist_dir = index.mkdir(distname) dist_sdist = '%s-%s.tar.gz' % (distname, version) @@ -39,8 +48,215 @@ def test_dist_fetch_build_egg(tmpdir): '''.split() with tmpdir.as_cwd(): dist = Distribution() + dist.parse_config_files() resolved_dists = [ dist.fetch_build_egg(r) for r in reqs ] assert [dist.key for dist in resolved_dists if dist] == reqs + + +def test_dist__get_unpatched_deprecated(): + pytest.warns(DistDeprecationWarning, _get_unpatched, [""]) + + +def __read_test_cases(): + # Metadata version 1.0 + base_attrs = { + "name": "package", + "version": "0.0.1", + "author": "Foo Bar", + "author_email": "foo@bar.net", + "long_description": "Long\ndescription", + "description": "Short description", + "keywords": ["one", "two"] + } + + def merge_dicts(d1, d2): + d1 = d1.copy() + d1.update(d2) + + return d1 + + test_cases = [ + ('Metadata version 1.0', base_attrs.copy()), + ('Metadata version 1.1: Provides', merge_dicts(base_attrs, { + 'provides': ['package'] + })), + ('Metadata version 1.1: Obsoletes', merge_dicts(base_attrs, { + 'obsoletes': ['foo'] + })), + ('Metadata version 1.1: Classifiers', merge_dicts(base_attrs, { + 'classifiers': [ + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'License :: OSI Approved :: MIT License' + ]})), + ('Metadata version 1.1: Download URL', merge_dicts(base_attrs, { + 'download_url': 'https://example.com' + })), + ('Metadata Version 1.2: Requires-Python', merge_dicts(base_attrs, { + 'python_requires': '>=3.7' + })), + pytest.param('Metadata Version 1.2: Project-Url', + merge_dicts(base_attrs, { + 'project_urls': { + 'Foo': 'https://example.bar' + } + }), marks=pytest.mark.xfail( + reason="Issue #1578: project_urls not read" + )), + ('Metadata Version 2.1: Long Description Content Type', + merge_dicts(base_attrs, { + 'long_description_content_type': 'text/x-rst; charset=UTF-8' + })), + pytest.param('Metadata Version 2.1: Provides Extra', + merge_dicts(base_attrs, { + 'provides_extras': ['foo', 'bar'] + }), marks=pytest.mark.xfail(reason="provides_extras not read")), + ('Missing author, missing author e-mail', + {'name': 'foo', 'version': '1.0.0'}), + ('Missing author', + {'name': 'foo', + 'version': '1.0.0', + 'author_email': 'snorri@sturluson.name'}), + ('Missing author e-mail', + {'name': 'foo', + 'version': '1.0.0', + 'author': 'Snorri Sturluson'}), + ('Missing author', + {'name': 'foo', + 'version': '1.0.0', + 'author': 'Snorri Sturluson'}), + ] + + return test_cases + + +@pytest.mark.parametrize('name,attrs', __read_test_cases()) +def test_read_metadata(name, attrs): + dist = Distribution(attrs) + metadata_out = dist.metadata + dist_class = metadata_out.__class__ + + # Write to PKG_INFO and then load into a new metadata object + if six.PY2: + PKG_INFO = io.BytesIO() + else: + PKG_INFO = io.StringIO() + + metadata_out.write_pkg_file(PKG_INFO) + + PKG_INFO.seek(0) + metadata_in = dist_class() + metadata_in.read_pkg_file(PKG_INFO) + + tested_attrs = [ + ('name', dist_class.get_name), + ('version', dist_class.get_version), + ('author', dist_class.get_contact), + ('author_email', dist_class.get_contact_email), + ('metadata_version', dist_class.get_metadata_version), + ('provides', dist_class.get_provides), + ('description', dist_class.get_description), + ('download_url', dist_class.get_download_url), + ('keywords', dist_class.get_keywords), + ('platforms', dist_class.get_platforms), + ('obsoletes', dist_class.get_obsoletes), + ('requires', dist_class.get_requires), + ('classifiers', dist_class.get_classifiers), + ('project_urls', lambda s: getattr(s, 'project_urls', {})), + ('provides_extras', lambda s: getattr(s, 'provides_extras', set())), + ] + + for attr, getter in tested_attrs: + assert getter(metadata_in) == getter(metadata_out) + + +def __maintainer_test_cases(): + attrs = {"name": "package", + "version": "1.0", + "description": "xxx"} + + def merge_dicts(d1, d2): + d1 = d1.copy() + d1.update(d2) + + return d1 + + test_cases = [ + ('No author, no maintainer', attrs.copy()), + ('Author (no e-mail), no maintainer', merge_dicts( + attrs, + {'author': 'Author Name'})), + ('Author (e-mail), no maintainer', merge_dicts( + attrs, + {'author': 'Author Name', + 'author_email': 'author@name.com'})), + ('No author, maintainer (no e-mail)', merge_dicts( + attrs, + {'maintainer': 'Maintainer Name'})), + ('No author, maintainer (e-mail)', merge_dicts( + attrs, + {'maintainer': 'Maintainer Name', + 'maintainer_email': 'maintainer@name.com'})), + ('Author (no e-mail), Maintainer (no-email)', merge_dicts( + attrs, + {'author': 'Author Name', + 'maintainer': 'Maintainer Name'})), + ('Author (e-mail), Maintainer (e-mail)', merge_dicts( + attrs, + {'author': 'Author Name', + 'author_email': 'author@name.com', + 'maintainer': 'Maintainer Name', + 'maintainer_email': 'maintainer@name.com'})), + ('No author (e-mail), no maintainer (e-mail)', merge_dicts( + attrs, + {'author_email': 'author@name.com', + 'maintainer_email': 'maintainer@name.com'})), + ('Author unicode', merge_dicts( + attrs, + {'author': '鉄沢寛'})), + ('Maintainer unicode', merge_dicts( + attrs, + {'maintainer': 'Jan Łukasiewicz'})), + ] + + return test_cases + + +@pytest.mark.parametrize('name,attrs', __maintainer_test_cases()) +def test_maintainer_author(name, attrs, tmpdir): + tested_keys = { + 'author': 'Author', + 'author_email': 'Author-email', + 'maintainer': 'Maintainer', + 'maintainer_email': 'Maintainer-email', + } + + # Generate a PKG-INFO file + dist = Distribution(attrs) + fn = tmpdir.mkdir('pkg_info') + fn_s = str(fn) + + dist.metadata.write_pkg_info(fn_s) + + with io.open(str(fn.join('PKG-INFO')), 'r', encoding='utf-8') as f: + raw_pkg_lines = f.readlines() + + # Drop blank lines + pkg_lines = list(filter(None, raw_pkg_lines)) + + pkg_lines_set = set(pkg_lines) + + # Duplicate lines should not be generated + assert len(pkg_lines) == len(pkg_lines_set) + + for fkey, dkey in tested_keys.items(): + val = attrs.get(dkey, None) + if val is None: + for line in pkg_lines: + assert not line.startswith(fkey + ':') + else: + line = '%s: %s' % (fkey, val) + assert line in pkg_lines_set diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index c9d396f4..2cf65ae7 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Easy install Tests """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import sys import os @@ -15,8 +15,9 @@ import distutils.errors import io import zipfile import mock - +from setuptools.command.easy_install import EasyInstallDeprecationWarning, ScriptWriter, WindowsScriptWriter import time +from setuptools.extern import six from setuptools.extern.six.moves import urllib import pytest @@ -33,16 +34,17 @@ import setuptools.tests.server from setuptools.tests import fail_on_ascii import pkg_resources -from .py26compat import tarfile_open from . import contexts from .textwrap import DALS +__metaclass__ = type + -class FakeDist(object): +class FakeDist: def get_entry_map(self, group): if group != 'console_scripts': return {} - return {'name': 'ep'} + return {str('name'): 'ep'} def as_requirement(self): return 'spec' @@ -78,7 +80,7 @@ class TestEasyInstallTest: sys.exit( load_entry_point('spec', 'console_scripts', 'name')() ) - """) + """) # noqa: E501 dist = FakeDist() args = next(ei.ScriptWriter.get_args(dist)) @@ -125,7 +127,9 @@ class TestEasyInstallTest: site.getsitepackages. """ path = normalize_path('/setuptools/test/site-packages') - mock_gsp = lambda: [path] + + def mock_gsp(): + return [path] monkeypatch.setattr(site, 'getsitepackages', mock_gsp, raising=False) assert path in ei.get_site_dirs() @@ -153,7 +157,7 @@ class TestEasyInstallTest: "", ), ( - u'mypkg/\u2603.txt', + 'mypkg/☃.txt', "", ), ] @@ -168,7 +172,8 @@ class TestEasyInstallTest: return str(sdist) @fail_on_ascii - def test_unicode_filename_in_sdist(self, sdist_unicode, tmpdir, monkeypatch): + def test_unicode_filename_in_sdist( + self, sdist_unicode, tmpdir, monkeypatch): """ The install command should execute correctly even if the package has unicode filenames. @@ -184,7 +189,122 @@ class TestEasyInstallTest: cmd.ensure_finalized() cmd.easy_install(sdist_unicode) + @pytest.fixture + def sdist_unicode_in_script(self, tmpdir): + files = [ + ( + "setup.py", + DALS(""" + import setuptools + setuptools.setup( + name="setuptools-test-unicode", + version="1.0", + packages=["mypkg"], + include_package_data=True, + scripts=['mypkg/unicode_in_script'], + ) + """), + ), + ("mypkg/__init__.py", ""), + ( + "mypkg/unicode_in_script", + DALS( + """ + #!/bin/sh + # á + + non_python_fn() { + } + """), + ), + ] + sdist_name = "setuptools-test-unicode-script-1.0.zip" + sdist = tmpdir / sdist_name + # can't use make_sdist, because the issue only occurs + # with zip sdists. + sdist_zip = zipfile.ZipFile(str(sdist), "w") + for filename, content in files: + sdist_zip.writestr(filename, content.encode('utf-8')) + sdist_zip.close() + return str(sdist) + + @fail_on_ascii + def test_unicode_content_in_sdist( + self, sdist_unicode_in_script, tmpdir, monkeypatch): + """ + The install command should execute correctly even if + the package has unicode in scripts. + """ + dist = Distribution({"script_args": ["easy_install"]}) + target = (tmpdir / "target").ensure_dir() + cmd = ei.easy_install(dist, install_dir=str(target), args=["x"]) + monkeypatch.setitem(os.environ, "PYTHONPATH", str(target)) + cmd.ensure_finalized() + cmd.easy_install(sdist_unicode_in_script) + + @pytest.fixture + def sdist_script(self, tmpdir): + files = [ + ( + 'setup.py', + DALS(""" + import setuptools + setuptools.setup( + name="setuptools-test-script", + version="1.0", + scripts=["mypkg_script"], + ) + """), + ), + ( + 'mypkg_script', + DALS(""" + #/usr/bin/python + print('mypkg_script') + """), + ), + ] + sdist_name = 'setuptools-test-script-1.0.zip' + sdist = str(tmpdir / sdist_name) + make_sdist(sdist, files) + return sdist + + @pytest.mark.skipif(not sys.platform.startswith('linux'), + reason="Test can only be run on Linux") + def test_script_install(self, sdist_script, tmpdir, monkeypatch): + """ + Check scripts are installed. + """ + dist = Distribution({'script_args': ['easy_install']}) + target = (tmpdir / 'target').ensure_dir() + cmd = ei.easy_install( + dist, + install_dir=str(target), + args=['x'], + ) + monkeypatch.setitem(os.environ, 'PYTHONPATH', str(target)) + cmd.ensure_finalized() + cmd.easy_install(sdist_script) + assert (target / 'mypkg_script').exists() + + + def test_dist_get_script_args_deprecated(self): + with pytest.warns(EasyInstallDeprecationWarning): + ScriptWriter.get_script_args(None, None) + + def test_dist_get_script_header_deprecated(self): + with pytest.warns(EasyInstallDeprecationWarning): + ScriptWriter.get_script_header("") + def test_dist_get_writer_deprecated(self): + with pytest.warns(EasyInstallDeprecationWarning): + ScriptWriter.get_writer(None) + + def test_dist_WindowsScriptWriter_get_writer_deprecated(self): + with pytest.warns(EasyInstallDeprecationWarning): + WindowsScriptWriter.get_writer() + +@pytest.mark.filterwarnings('ignore:Unbuilt egg') class TestPTHFileWriter: def test_add_from_cwd_site_sets_dirty(self): '''a pth file manager should set dirty @@ -352,8 +472,8 @@ class TestSetupRequires: dist_file, ] with sandbox.save_argv(['easy_install']): - # attempt to install the dist. It should fail because - # it doesn't exist. + # attempt to install the dist. It should + # fail because it doesn't exist. with pytest.raises(SystemExit): easy_install_pkg.main(ei_params) # there should have been two or three requests to the server @@ -381,7 +501,15 @@ class TestSetupRequires: """))]) yield dist_path - def test_setup_requires_overrides_version_conflict(self): + use_setup_cfg = ( + (), + ('dependency_links',), + ('setup_requires',), + ('dependency_links', 'setup_requires'), + ) + + @pytest.mark.parametrize('use_setup_cfg', use_setup_cfg) + def test_setup_requires_overrides_version_conflict(self, use_setup_cfg): """ Regression test for distribution issue 323: https://bitbucket.org/tarek/distribute/issues/323 @@ -397,18 +525,20 @@ class TestSetupRequires: with contexts.save_pkg_resources_state(): with contexts.tempdir() as temp_dir: - test_pkg = create_setup_requires_package(temp_dir) + test_pkg = create_setup_requires_package( + temp_dir, use_setup_cfg=use_setup_cfg) test_setup_py = os.path.join(test_pkg, 'setup.py') with contexts.quiet() as (stdout, stderr): # Don't even need to install the package, just # running the setup.py at all is sufficient - run_setup(test_setup_py, ['--name']) + run_setup(test_setup_py, [str('--name')]) lines = stdout.readlines() assert len(lines) > 0 - assert lines[-1].strip(), 'test_pkg' + assert lines[-1].strip() == 'test_pkg' - def test_setup_requires_override_nspkg(self): + @pytest.mark.parametrize('use_setup_cfg', use_setup_cfg) + def test_setup_requires_override_nspkg(self, use_setup_cfg): """ Like ``test_setup_requires_overrides_version_conflict`` but where the ``setup_requires`` package is part of a namespace package that has @@ -423,7 +553,7 @@ class TestSetupRequires: # extracted path to sys.path so foo.bar v0.1 is importable foobar_1_dir = os.path.join(temp_dir, 'foo.bar-0.1') os.mkdir(foobar_1_dir) - with tarfile_open(foobar_1_archive) as tf: + with tarfile.open(foobar_1_archive) as tf: tf.extractall(foobar_1_dir) sys.path.insert(1, foobar_1_dir) @@ -446,7 +576,8 @@ class TestSetupRequires: """) test_pkg = create_setup_requires_package( - temp_dir, 'foo.bar', '0.2', make_nspkg_sdist, template) + temp_dir, 'foo.bar', '0.2', make_nspkg_sdist, template, + use_setup_cfg=use_setup_cfg) test_setup_py = os.path.join(test_pkg, 'setup.py') @@ -454,9 +585,10 @@ class TestSetupRequires: try: # Don't even need to install the package, just # running the setup.py at all is sufficient - run_setup(test_setup_py, ['--name']) + run_setup(test_setup_py, [str('--name')]) except pkg_resources.VersionConflict: - self.fail('Installing setup.py requirements ' + self.fail( + 'Installing setup.py requirements ' 'caused a VersionConflict') assert 'FAIL' not in stdout.getvalue() @@ -464,6 +596,40 @@ class TestSetupRequires: assert len(lines) > 0 assert lines[-1].strip() == 'test_pkg' + @pytest.mark.parametrize('use_setup_cfg', use_setup_cfg) + def test_setup_requires_with_attr_version(self, use_setup_cfg): + def make_dependency_sdist(dist_path, distname, version): + files = [( + 'setup.py', + DALS(""" + import setuptools + setuptools.setup( + name={name!r}, + version={version!r}, + py_modules=[{name!r}], + ) + """.format(name=distname, version=version)), + ), ( + distname + '.py', + DALS(""" + version = 42 + """), + )] + make_sdist(dist_path, files) + with contexts.save_pkg_resources_state(): + with contexts.tempdir() as temp_dir: + test_pkg = create_setup_requires_package( + temp_dir, setup_attrs=dict(version='attr: foobar.version'), + make_package=make_dependency_sdist, + use_setup_cfg=use_setup_cfg+('version',), + ) + test_setup_py = os.path.join(test_pkg, 'setup.py') + with contexts.quiet() as (stdout, stderr): + run_setup(test_setup_py, [str('--version')]) + lines = stdout.readlines() + assert len(lines) > 0 + assert lines[-1].strip() == '42' + def make_trivial_sdist(dist_path, distname, version): """ @@ -521,7 +687,7 @@ def make_sdist(dist_path, files): listed in ``files`` as ``(filename, content)`` tuples. """ - with tarfile_open(dist_path, 'w:gz') as dist: + with tarfile.open(dist_path, 'w:gz') as dist: for filename, content in files: file_bytes = io.BytesIO(content.encode('utf-8')) file_info = tarfile.TarInfo(name=filename) @@ -532,7 +698,8 @@ def make_sdist(dist_path, files): def create_setup_requires_package(path, distname='foobar', version='0.1', make_package=make_trivial_sdist, - setup_py_template=None): + setup_py_template=None, setup_attrs={}, + use_setup_cfg=()): """Creates a source tree under path for a trivial test package that has a single requirement in setup_requires--a tarball for that requirement is also created and added to the dependency_links argument. @@ -547,17 +714,44 @@ def create_setup_requires_package(path, distname='foobar', version='0.1', 'setup_requires': ['%s==%s' % (distname, version)], 'dependency_links': [os.path.abspath(path)] } + test_setup_attrs.update(setup_attrs) test_pkg = os.path.join(path, 'test_pkg') - test_setup_py = os.path.join(test_pkg, 'setup.py') os.mkdir(test_pkg) + if use_setup_cfg: + test_setup_cfg = os.path.join(test_pkg, 'setup.cfg') + options = [] + metadata = [] + for name in use_setup_cfg: + value = test_setup_attrs.pop(name) + if name in 'name version'.split(): + section = metadata + else: + section = options + if isinstance(value, (tuple, list)): + value = ';'.join(value) + section.append('%s: %s' % (name, value)) + with open(test_setup_cfg, 'w') as f: + f.write(DALS( + """ + [metadata] + {metadata} + [options] + {options} + """ + ).format( + options='\n'.join(options), + metadata='\n'.join(metadata), + )) + + test_setup_py = os.path.join(test_pkg, 'setup.py') + if setup_py_template is None: setup_py_template = DALS("""\ import setuptools setuptools.setup(**%r) """) - with open(test_setup_py, 'w') as f: f.write(setup_py_template % test_setup_attrs) @@ -573,27 +767,31 @@ def create_setup_requires_package(path, distname='foobar', version='0.1', ) class TestScriptHeader: non_ascii_exe = '/Users/José/bin/python' - exe_with_spaces = r'C:\Program Files\Python33\python.exe' + if six.PY2: + non_ascii_exe = non_ascii_exe.encode('utf-8') + exe_with_spaces = r'C:\Program Files\Python36\python.exe' def test_get_script_header(self): expected = '#!%s\n' % ei.nt_quote_arg(os.path.normpath(sys.executable)) - actual = ei.ScriptWriter.get_script_header('#!/usr/local/bin/python') + actual = ei.ScriptWriter.get_header('#!/usr/local/bin/python') assert actual == expected def test_get_script_header_args(self): - expected = '#!%s -x\n' % ei.nt_quote_arg(os.path.normpath - (sys.executable)) - actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python -x') + expected = '#!%s -x\n' % ei.nt_quote_arg( + os.path.normpath(sys.executable)) + actual = ei.ScriptWriter.get_header('#!/usr/bin/python -x') assert actual == expected def test_get_script_header_non_ascii_exe(self): - actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', + actual = ei.ScriptWriter.get_header( + '#!/usr/bin/python', executable=self.non_ascii_exe) - expected = '#!%s -x\n' % self.non_ascii_exe + expected = str('#!%s -x\n') % self.non_ascii_exe assert actual == expected def test_get_script_header_exe_with_spaces(self): - actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', + actual = ei.ScriptWriter.get_header( + '#!/usr/bin/python', executable='"' + self.exe_with_spaces + '"') expected = '#!"%s"\n' % self.exe_with_spaces assert actual == expected @@ -636,7 +834,7 @@ class TestCommandSpec: class TestWindowsScriptWriter: def test_header(self): - hdr = ei.WindowsScriptWriter.get_script_header('') + hdr = ei.WindowsScriptWriter.get_header('') assert hdr.startswith('#!') assert hdr.endswith('\n') hdr = hdr.lstrip('#!') diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index 5196f32e..5eb81621 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -1,11 +1,13 @@ +import datetime +import sys import ast import os import glob import re import stat -import sys +import time -from setuptools.command.egg_info import egg_info, manifest_maker +from setuptools.command.egg_info import egg_info, manifest_maker, EggInfoDeprecationWarning, get_pkg_info_revision from setuptools.dist import Distribution from setuptools.extern.six.moves import map @@ -16,12 +18,14 @@ from .files import build_files from .textwrap import DALS from . import contexts +__metaclass__ = type + class Environment(str): pass -class TestEggInfo(object): +class TestEggInfo: setup_script = DALS(""" from setuptools import setup @@ -64,12 +68,6 @@ class TestEggInfo(object): }) yield env - dict_order_fails = pytest.mark.skipif( - sys.version_info < (2, 7), - reason="Intermittent failures on Python 2.6", - ) - - @dict_order_fails def test_egg_info_save_version_info_setup_empty(self, tmpdir_cwd, env): """ When the egg_info section is empty or not present, running @@ -104,7 +102,6 @@ class TestEggInfo(object): flags = re.MULTILINE | re.DOTALL assert re.search(pattern, content, flags) - @dict_order_fails def test_egg_info_save_version_info_setup_defaults(self, tmpdir_cwd, env): """ When running save_version_info on an existing setup.cfg @@ -135,11 +132,11 @@ class TestEggInfo(object): self._validate_content_order(content, expected_order) - def test_egg_base_installed_egg_info(self, tmpdir_cwd, env): + def test_expected_files_produced(self, tmpdir_cwd, env): self._create_project() - self._run_install_command(tmpdir_cwd, env) - actual = self._find_egg_info_files(env.paths['lib']) + self._run_egg_info_command(tmpdir_cwd, env) + actual = os.listdir('foo.egg-info') expected = [ 'PKG-INFO', @@ -151,6 +148,52 @@ class TestEggInfo(object): ] assert sorted(actual) == expected + def test_license_is_a_string(self, tmpdir_cwd, env): + setup_config = DALS(""" + [metadata] + name=foo + version=0.0.1 + license=file:MIT + """) + + setup_script = DALS(""" + from setuptools import setup + + setup() + """) + + build_files({'setup.py': setup_script, + 'setup.cfg': setup_config}) + + # This command should fail with a ValueError, but because it's + # currently configured to use a subprocess, the actual traceback + # object is lost and we need to parse it from stderr + with pytest.raises(AssertionError) as exc: + self._run_egg_info_command(tmpdir_cwd, env) + + # Hopefully this is not too fragile: the only argument to the + # assertion error should be a traceback, ending with: + # ValueError: .... + # + # assert not 1 + tb = exc.value.args[0].split('\n') + assert tb[-3].lstrip().startswith('ValueError') + + def test_rebuilt(self, tmpdir_cwd, env): + """Ensure timestamps are updated when the command is re-run.""" + self._create_project() + + self._run_egg_info_command(tmpdir_cwd, env) + timestamp_a = os.path.getmtime('foo.egg-info') + + # arbitrary sleep just to handle *really* fast systems + time.sleep(.001) + + self._run_egg_info_command(tmpdir_cwd, env) + timestamp_b = os.path.getmtime('foo.egg-info') + + assert timestamp_a != timestamp_b + def test_manifest_template_is_read(self, tmpdir_cwd, env): self._create_project() build_files({ @@ -161,8 +204,8 @@ class TestEggInfo(object): 'usage.rst': "Run 'hi'", } }) - self._run_install_command(tmpdir_cwd, env) - egg_info_dir = self._find_egg_info_files(env.paths['lib']).base + self._run_egg_info_command(tmpdir_cwd, env) + egg_info_dir = os.path.join('.', 'foo.egg-info') sources_txt = os.path.join(egg_info_dir, 'SOURCES.txt') with open(sources_txt) as f: assert 'docs/usage.rst' in f.read().split('\n') @@ -188,7 +231,7 @@ class TestEggInfo(object): ) invalid_marker = "<=>++" - class RequiresTestHelper(object): + class RequiresTestHelper: @staticmethod def parametrize(*test_list, **format_dict): @@ -198,7 +241,8 @@ class TestEggInfo(object): test_params = test.lstrip().split('\n\n', 3) name_kwargs = test_params.pop(0).split('\n') if len(name_kwargs) > 1: - install_cmd_kwargs = ast.literal_eval(name_kwargs[1].strip()) + val = name_kwargs[1].strip() + install_cmd_kwargs = ast.literal_eval(val) else: install_cmd_kwargs = {} name = name_kwargs[0].strip() @@ -218,9 +262,11 @@ class TestEggInfo(object): expected_requires, install_cmd_kwargs, marks=marks)) - return pytest.mark.parametrize('requires,use_setup_cfg,' - 'expected_requires,install_cmd_kwargs', - argvalues, ids=idlist) + return pytest.mark.parametrize( + 'requires,use_setup_cfg,' + 'expected_requires,install_cmd_kwargs', + argvalues, ids=idlist, + ) @RequiresTestHelper.parametrize( # Format of a test: @@ -235,6 +281,32 @@ class TestEggInfo(object): # expected contents of requires.txt ''' + install_requires_deterministic + + install_requires=["wheel>=0.5", "pytest"] + + [options] + install_requires = + wheel>=0.5 + pytest + + wheel>=0.5 + pytest + ''', + + ''' + install_requires_ordered + + install_requires=["pytest>=3.0.2,!=10.9999"] + + [options] + install_requires = + pytest>=3.0.2,!=10.9999 + + pytest!=10.9999,>=3.0.2 + ''', + + ''' install_requires_with_marker install_requires=["barbazquux;{mismatch_marker}"], @@ -368,11 +440,11 @@ class TestEggInfo(object): mismatch_marker=mismatch_marker, mismatch_marker_alternate=mismatch_marker_alternate, ) - def test_requires(self, tmpdir_cwd, env, - requires, use_setup_cfg, - expected_requires, install_cmd_kwargs): + def test_requires( + self, tmpdir_cwd, env, requires, use_setup_cfg, + expected_requires, install_cmd_kwargs): self._setup_script_with_requires(requires, use_setup_cfg) - self._run_install_command(tmpdir_cwd, env, **install_cmd_kwargs) + self._run_egg_info_command(tmpdir_cwd, env, **install_cmd_kwargs) egg_info_dir = os.path.join('.', 'foo.egg-info') requires_txt = os.path.join(egg_info_dir, 'requires.txt') if os.path.exists(requires_txt): @@ -383,12 +455,23 @@ class TestEggInfo(object): assert install_requires.lstrip() == expected_requires assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + def test_install_requires_unordered_disallowed(self, tmpdir_cwd, env): + """ + Packages that pass unordered install_requires sequences + should be rejected as they produce non-deterministic + builds. See #458. + """ + req = 'install_requires={"fake-factory==0.5.2", "pytz"}' + self._setup_script_with_requires(req) + with pytest.raises(AssertionError): + self._run_egg_info_command(tmpdir_cwd, env) + def test_extras_require_with_invalid_marker(self, tmpdir_cwd, env): tmpl = 'extras_require={{":{marker}": ["barbazquux"]}},' req = tmpl.format(marker=self.invalid_marker) self._setup_script_with_requires(req) with pytest.raises(AssertionError): - self._run_install_command(tmpdir_cwd, env) + self._run_egg_info_command(tmpdir_cwd, env) assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] def test_extras_require_with_invalid_marker_in_req(self, tmpdir_cwd, env): @@ -396,9 +479,44 @@ class TestEggInfo(object): req = tmpl.format(marker=self.invalid_marker) self._setup_script_with_requires(req) with pytest.raises(AssertionError): - self._run_install_command(tmpdir_cwd, env) + self._run_egg_info_command(tmpdir_cwd, env) assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + def test_provides_extra(self, tmpdir_cwd, env): + self._setup_script_with_requires( + 'extras_require={"foobar": ["barbazquux"]},') + environ = os.environ.copy().update( + HOME=env.paths['home'], + ) + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + env=environ, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + assert 'Provides-Extra: foobar' in pkg_info_lines + assert 'Metadata-Version: 2.1' in pkg_info_lines + + def test_doesnt_provides_extra(self, tmpdir_cwd, env): + self._setup_script_with_requires( + '''install_requires=["spam ; python_version<'3.6'"]''') + environ = os.environ.copy().update( + HOME=env.paths['home'], + ) + environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + env=environ, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_text = pkginfo_file.read() + assert 'Provides-Extra:' not in pkg_info_text + def test_long_description_content_type(self, tmpdir_cwd, env): # Test that specifying a `long_description_content_type` keyword arg to # the `setup` function results in writing a `Description-Content-Type` @@ -423,6 +541,37 @@ class TestEggInfo(object): pkg_info_lines = pkginfo_file.read().split('\n') expected_line = 'Description-Content-Type: text/markdown' assert expected_line in pkg_info_lines + assert 'Metadata-Version: 2.1' in pkg_info_lines + + def test_project_urls(self, tmpdir_cwd, env): + # Test that specifying a `project_urls` dict to the `setup` + # function results in writing multiple `Project-URL` lines to + # the `PKG-INFO` file in the `<distribution>.egg-info` + # directory. + # `Project-URL` is described at https://packaging.python.org + # /specifications/core-metadata/#project-url-multiple-use + + self._setup_script_with_requires( + """project_urls={ + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + },""") + environ = os.environ.copy().update( + HOME=env.paths['home'], + ) + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + env=environ, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + expected_line = 'Project-URL: Link One, https://example.com/one/' + assert expected_line in pkg_info_lines + expected_line = 'Project-URL: Link Two, https://example.com/two/' + assert expected_line in pkg_info_lines def test_python_requires_egg_info(self, tmpdir_cwd, env): self._setup_script_with_requires( @@ -442,15 +591,6 @@ class TestEggInfo(object): assert 'Requires-Python: >=2.7.12' in pkg_info_lines assert 'Metadata-Version: 1.2' in pkg_info_lines - def test_python_requires_install(self, tmpdir_cwd, env): - self._setup_script_with_requires( - """python_requires='>=1.2.3',""") - self._run_install_command(tmpdir_cwd, env) - egg_info_dir = self._find_egg_info_files(env.paths['lib']).base - pkginfo = os.path.join(egg_info_dir, 'PKG-INFO') - with open(pkginfo) as f: - assert 'Requires-Python: >=1.2.3' in f.read().split('\n') - def test_manifest_maker_warning_suppression(self): fixtures = [ "standard file not found: should have one of foo.py, bar.py", @@ -460,17 +600,27 @@ class TestEggInfo(object): for msg in fixtures: assert manifest_maker._should_suppress_warning(msg) - def _run_install_command(self, tmpdir_cwd, env, cmd=None, output=None): + def test_egg_info_includes_setup_py(self, tmpdir_cwd): + self._create_project() + dist = Distribution({"name": "foo", "version": "0.0.1"}) + dist.script_name = "non_setup.py" + egg_info_instance = egg_info(dist) + egg_info_instance.finalize_options() + egg_info_instance.run() + + assert 'setup.py' in egg_info_instance.filelist.files + + with open(egg_info_instance.egg_info + "/SOURCES.txt") as f: + sources = f.read().split('\n') + assert 'setup.py' in sources + + def _run_egg_info_command(self, tmpdir_cwd, env, cmd=None, output=None): environ = os.environ.copy().update( HOME=env.paths['home'], ) if cmd is None: cmd = [ - 'install', - '--home', env.paths['home'], - '--install-lib', env.paths['lib'], - '--install-scripts', env.paths['scripts'], - '--install-data', env.paths['data'], + 'egg_info', ] code, data = environment.run_setup_py( cmd=cmd, @@ -478,25 +628,29 @@ class TestEggInfo(object): data_stream=1, env=environ, ) - if code: - raise AssertionError(data) + assert not code, data + if output: assert output in data - def _find_egg_info_files(self, root): - class DirList(list): - def __init__(self, files, base): - super(DirList, self).__init__(files) - self.base = base + def test_egg_info_tag_only_once(self, tmpdir_cwd, env): + self._create_project() + build_files({ + 'setup.cfg': DALS(""" + [egg_info] + tag_build = dev + tag_date = 0 + tag_svn_revision = 0 + """), + }) + self._run_egg_info_command(tmpdir_cwd, env) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + assert 'Version: 0.0.0.dev0' in pkg_info_lines - results = ( - DirList(filenames, dirpath) - for dirpath, dirnames, filenames in os.walk(root) - if os.path.basename(dirpath) == 'EGG-INFO' - ) - # expect exactly one result - result, = results - return result + def test_get_pkg_info_revision_deprecated(self): + pytest.warns(EggInfoDeprecationWarning, get_pkg_info_revision) EGG_INFO_TESTS = ( # Check for issue #1136: invalid string type when diff --git a/setuptools/tests/test_find_packages.py b/setuptools/tests/test_find_packages.py index a6023de9..b08f91c7 100644 --- a/setuptools/tests/test_find_packages.py +++ b/setuptools/tests/test_find_packages.py @@ -7,14 +7,15 @@ import platform import pytest -import setuptools -from setuptools import find_packages +from . import py3_only -find_420_packages = setuptools.PEP420PackageFinder.find +from setuptools.extern.six import PY3 +from setuptools import find_packages +if PY3: + from setuptools import find_namespace_packages # modeled after CPython's test.support.can_symlink - def can_symlink(): TESTFN = tempfile.mktemp() symlink_path = TESTFN + "can_symlink" @@ -153,30 +154,35 @@ class TestFindPackages: def _assert_packages(self, actual, expected): assert set(actual) == set(expected) + @py3_only def test_pep420_ns_package(self): - packages = find_420_packages( + packages = find_namespace_packages( self.dist_dir, include=['pkg*'], exclude=['pkg.subpkg.assets']) self._assert_packages(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg']) + @py3_only def test_pep420_ns_package_no_includes(self): - packages = find_420_packages( + packages = find_namespace_packages( self.dist_dir, exclude=['pkg.subpkg.assets']) self._assert_packages(packages, ['docs', 'pkg', 'pkg.nspkg', 'pkg.subpkg']) + @py3_only def test_pep420_ns_package_no_includes_or_excludes(self): - packages = find_420_packages(self.dist_dir) - expected = [ - 'docs', 'pkg', 'pkg.nspkg', 'pkg.subpkg', 'pkg.subpkg.assets'] + packages = find_namespace_packages(self.dist_dir) + expected = ['docs', 'pkg', 'pkg.nspkg', 'pkg.subpkg', 'pkg.subpkg.assets'] self._assert_packages(packages, expected) + @py3_only def test_regular_package_with_nested_pep420_ns_packages(self): self._touch('__init__.py', self.pkg_dir) - packages = find_420_packages( + packages = find_namespace_packages( self.dist_dir, exclude=['docs', 'pkg.subpkg.assets']) self._assert_packages(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg']) + @py3_only def test_pep420_ns_package_no_non_package_dirs(self): shutil.rmtree(self.docs_dir) shutil.rmtree(os.path.join(self.dist_dir, 'pkg/subpkg/assets')) - packages = find_420_packages(self.dist_dir) + packages = find_namespace_packages(self.dist_dir) self._assert_packages(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg']) + diff --git a/setuptools/tests/test_glibc.py b/setuptools/tests/test_glibc.py new file mode 100644 index 00000000..795fdc56 --- /dev/null +++ b/setuptools/tests/test_glibc.py @@ -0,0 +1,52 @@ +import warnings + +import pytest + +from setuptools.glibc import check_glibc_version + +__metaclass__ = type + + +@pytest.fixture(params=[ + "2.20", + # used by "linaro glibc", see gh-3588 + "2.20-2014.11", + # weird possibilities that I just made up + "2.20+dev", + "2.20-custom", + "2.20.1", + ]) +def two_twenty(request): + return request.param + + +@pytest.fixture(params=["asdf", "", "foo.bar"]) +def bad_string(request): + return request.param + + +class TestGlibc: + def test_manylinux1_check_glibc_version(self, two_twenty): + """ + Test that the check_glibc_version function is robust against weird + glibc version strings. + """ + assert check_glibc_version(two_twenty, 2, 15) + assert check_glibc_version(two_twenty, 2, 20) + assert not check_glibc_version(two_twenty, 2, 21) + assert not check_glibc_version(two_twenty, 3, 15) + assert not check_glibc_version(two_twenty, 1, 15) + + def test_bad_versions(self, bad_string): + """ + For unparseable strings, warn and return False + """ + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings("always") + assert not check_glibc_version(bad_string, 2, 5) + for w in ws: + if "Expected glibc version with" in str(w.message): + break + else: + # Didn't find the warning we were expecting + assert False diff --git a/setuptools/tests/test_glob.py b/setuptools/tests/test_glob.py new file mode 100644 index 00000000..a0728c5d --- /dev/null +++ b/setuptools/tests/test_glob.py @@ -0,0 +1,35 @@ +import pytest + +from setuptools.glob import glob + +from .files import build_files + + +@pytest.mark.parametrize('tree, pattern, matches', ( + ('', b'', []), + ('', '', []), + (''' + appveyor.yml + CHANGES.rst + LICENSE + MANIFEST.in + pyproject.toml + README.rst + setup.cfg + setup.py + ''', '*.rst', ('CHANGES.rst', 'README.rst')), + (''' + appveyor.yml + CHANGES.rst + LICENSE + MANIFEST.in + pyproject.toml + README.rst + setup.cfg + setup.py + ''', b'*.rst', (b'CHANGES.rst', b'README.rst')), +)) +def test_glob(monkeypatch, tmpdir, tree, pattern, matches): + monkeypatch.chdir(tmpdir) + build_files({name: '' for name in tree.split()}) + assert list(sorted(glob(pattern))) == list(sorted(matches)) diff --git a/setuptools/tests/test_install_scripts.py b/setuptools/tests/test_install_scripts.py index 7393241f..727ad65b 100644 --- a/setuptools/tests/test_install_scripts.py +++ b/setuptools/tests/test_install_scripts.py @@ -19,7 +19,7 @@ class TestInstallScripts: ) unix_exe = '/usr/dummy-test-path/local/bin/python' unix_spaces_exe = '/usr/bin/env dummy-test-python' - win32_exe = 'C:\\Dummy Test Path\\Program Files\\Python 3.3\\python.exe' + win32_exe = 'C:\\Dummy Test Path\\Program Files\\Python 3.6\\python.exe' def _run_install_scripts(self, install_dir, executable=None): dist = Distribution(self.settings) diff --git a/setuptools/tests/test_manifest.py b/setuptools/tests/test_manifest.py index 65eec7d9..5edfbea0 100644 --- a/setuptools/tests/test_manifest.py +++ b/setuptools/tests/test_manifest.py @@ -15,10 +15,11 @@ from setuptools.command.egg_info import FileList, egg_info, translate_pattern from setuptools.dist import Distribution from setuptools.extern import six from setuptools.tests.textwrap import DALS +from . import py3_only import pytest -py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") +__metaclass__ = type def make_local_path(s): @@ -157,7 +158,7 @@ def test_translated_pattern_mismatch(pattern_mismatch): assert not translate_pattern(pattern).match(target) -class TempDirTestCase(object): +class TempDirTestCase: def setup_method(self, method): self.temp_dir = tempfile.mkdtemp() self.old_cwd = os.getcwd() diff --git a/setuptools/tests/test_namespaces.py b/setuptools/tests/test_namespaces.py index 1ac1b35e..670ccee9 100644 --- a/setuptools/tests/test_namespaces.py +++ b/setuptools/tests/test_namespaces.py @@ -12,10 +12,10 @@ from setuptools.command import test class TestNamespaces: - @pytest.mark.xfail(sys.version_info < (3, 5), - reason="Requires importlib.util.module_from_spec") - @pytest.mark.skipif(bool(os.environ.get("APPVEYOR")), - reason="https://github.com/pypa/setuptools/issues/851") + @pytest.mark.skipif( + sys.version_info < (3, 5), + reason="Requires importlib.util.module_from_spec", + ) def test_mixed_site_and_non_site(self, tmpdir): """ Installing two packages sharing the same namespace, one installed @@ -55,8 +55,6 @@ class TestNamespaces: with test.test.paths_on_pythonpath(map(str, targets)): subprocess.check_call(try_import) - @pytest.mark.skipif(bool(os.environ.get("APPVEYOR")), - reason="https://github.com/pypa/setuptools/issues/851") def test_pkg_resources_import(self, tmpdir): """ Ensure that a namespace package doesn't break on import @@ -81,8 +79,6 @@ class TestNamespaces: with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(try_import) - @pytest.mark.skipif(bool(os.environ.get("APPVEYOR")), - reason="https://github.com/pypa/setuptools/issues/851") def test_namespace_package_installed_and_cwd(self, tmpdir): """ Installing a namespace packages but also having it in the current @@ -109,3 +105,32 @@ class TestNamespaces: ] with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(pkg_resources_imp, cwd=str(pkg_A)) + + def test_packages_in_the_same_namespace_installed_and_cwd(self, tmpdir): + """ + Installing one namespace package and also have another in the same + namespace in the current working directory, both of them must be + importable. + """ + pkg_A = namespaces.build_namespace_package(tmpdir, 'myns.pkgA') + pkg_B = namespaces.build_namespace_package(tmpdir, 'myns.pkgB') + target = tmpdir / 'packages' + # use pip to install to the target directory + install_cmd = [ + sys.executable, + '-m', + 'pip.__main__', + 'install', + str(pkg_A), + '-t', str(target), + ] + subprocess.check_call(install_cmd) + namespaces.make_site_dir(target) + + # ensure that all packages import and pkg_resources imports + pkg_resources_imp = [ + sys.executable, + '-c', 'import pkg_resources; import myns.pkgA; import myns.pkgB', + ] + with test.test.paths_on_pythonpath([str(target)]): + subprocess.check_call(pkg_resources_imp, cwd=str(pkg_B)) diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py index 53e20d44..13cffb7e 100644 --- a/setuptools/tests/test_packageindex.py +++ b/setuptools/tests/test_packageindex.py @@ -6,6 +6,8 @@ import distutils.errors from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client +import mock +import pytest import pkg_resources import setuptools.package_index @@ -223,6 +225,61 @@ class TestPackageIndex: assert dists[0].version == '' assert dists[1].version == vc + def test_download_git_with_rev(self, tmpdir): + url = 'git+https://github.example/group/project@master#egg=foo' + index = setuptools.package_index.PackageIndex() + + with mock.patch("os.system") as os_system_mock: + result = index.download(url, str(tmpdir)) + + os_system_mock.assert_called() + + expected_dir = str(tmpdir / 'project@master') + expected = ( + 'git clone --quiet ' + 'https://github.example/group/project {expected_dir}' + ).format(**locals()) + first_call_args = os_system_mock.call_args_list[0][0] + assert first_call_args == (expected,) + + tmpl = '(cd {expected_dir} && git checkout --quiet master)' + expected = tmpl.format(**locals()) + assert os_system_mock.call_args_list[1][0] == (expected,) + assert result == expected_dir + + def test_download_git_no_rev(self, tmpdir): + url = 'git+https://github.example/group/project#egg=foo' + index = setuptools.package_index.PackageIndex() + + with mock.patch("os.system") as os_system_mock: + result = index.download(url, str(tmpdir)) + + os_system_mock.assert_called() + + expected_dir = str(tmpdir / 'project') + expected = ( + 'git clone --quiet ' + 'https://github.example/group/project {expected_dir}' + ).format(**locals()) + os_system_mock.assert_called_once_with(expected) + + def test_download_svn(self, tmpdir): + url = 'svn+https://svn.example/project#egg=foo' + index = setuptools.package_index.PackageIndex() + + with pytest.warns(UserWarning): + with mock.patch("os.system") as os_system_mock: + result = index.download(url, str(tmpdir)) + + os_system_mock.assert_called() + + expected_dir = str(tmpdir / 'project') + expected = ( + 'svn checkout -q ' + 'svn+https://svn.example/project {expected_dir}' + ).format(**locals()) + os_system_mock.assert_called_once_with(expected) + class TestContentCheckers: def test_md5(self): @@ -265,11 +322,11 @@ class TestPyPIConfig: with pypirc.open('w') as strm: strm.write(DALS(""" [pypi] - repository=https://pypi.python.org + repository=https://pypi.org username=jaraco password=pity% """)) cfg = setuptools.package_index.PyPIConfig() - cred = cfg.creds_by_repository['https://pypi.python.org'] + cred = cfg.creds_by_repository['https://pypi.org'] assert cred.username == 'jaraco' assert cred.password == 'pity%' diff --git a/setuptools/tests/test_pep425tags.py b/setuptools/tests/test_pep425tags.py new file mode 100644 index 00000000..f558a0d8 --- /dev/null +++ b/setuptools/tests/test_pep425tags.py @@ -0,0 +1,168 @@ +import sys + +import pytest +from mock import patch + +from setuptools import pep425tags + +__metaclass__ = type + + +class TestPEP425Tags: + + def mock_get_config_var(self, **kwd): + """ + Patch sysconfig.get_config_var for arbitrary keys. + """ + get_config_var = pep425tags.sysconfig.get_config_var + + def _mock_get_config_var(var): + if var in kwd: + return kwd[var] + return get_config_var(var) + return _mock_get_config_var + + def abi_tag_unicode(self, flags, config_vars): + """ + Used to test ABI tags, verify correct use of the `u` flag + """ + config_vars.update({'SOABI': None}) + base = pep425tags.get_abbr_impl() + pep425tags.get_impl_ver() + + if sys.version_info < (3, 3): + config_vars.update({'Py_UNICODE_SIZE': 2}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('setuptools.pep425tags.sysconfig.get_config_var', mock_gcf): + abi_tag = pep425tags.get_abi_tag() + assert abi_tag == base + flags + + config_vars.update({'Py_UNICODE_SIZE': 4}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('setuptools.pep425tags.sysconfig.get_config_var', + mock_gcf): + abi_tag = pep425tags.get_abi_tag() + assert abi_tag == base + flags + 'u' + + else: + # On Python >= 3.3, UCS-4 is essentially permanently enabled, and + # Py_UNICODE_SIZE is None. SOABI on these builds does not include + # the 'u' so manual SOABI detection should not do so either. + config_vars.update({'Py_UNICODE_SIZE': None}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('setuptools.pep425tags.sysconfig.get_config_var', + mock_gcf): + abi_tag = pep425tags.get_abi_tag() + assert abi_tag == base + flags + + def test_broken_sysconfig(self): + """ + Test that pep425tags still works when sysconfig is broken. + Can be a problem on Python 2.7 + Issue #1074. + """ + def raises_ioerror(var): + raise IOError("I have the wrong path!") + + with patch('setuptools.pep425tags.sysconfig.get_config_var', + raises_ioerror): + with pytest.warns(RuntimeWarning): + assert len(pep425tags.get_supported()) + + def test_no_hyphen_tag(self): + """ + Test that no tag contains a hyphen. + """ + mock_gcf = self.mock_get_config_var(SOABI='cpython-35m-darwin') + + with patch('setuptools.pep425tags.sysconfig.get_config_var', + mock_gcf): + supported = pep425tags.get_supported() + + for (py, abi, plat) in supported: + assert '-' not in py + assert '-' not in abi + assert '-' not in plat + + def test_manual_abi_noflags(self): + """ + Test that no flags are set on a non-PyDebug, non-Pymalloc ABI tag. + """ + self.abi_tag_unicode('', {'Py_DEBUG': False, 'WITH_PYMALLOC': False}) + + def test_manual_abi_d_flag(self): + """ + Test that the `d` flag is set on a PyDebug, non-Pymalloc ABI tag. + """ + self.abi_tag_unicode('d', {'Py_DEBUG': True, 'WITH_PYMALLOC': False}) + + def test_manual_abi_m_flag(self): + """ + Test that the `m` flag is set on a non-PyDebug, Pymalloc ABI tag. + """ + self.abi_tag_unicode('m', {'Py_DEBUG': False, 'WITH_PYMALLOC': True}) + + def test_manual_abi_dm_flags(self): + """ + Test that the `dm` flags are set on a PyDebug, Pymalloc ABI tag. + """ + self.abi_tag_unicode('dm', {'Py_DEBUG': True, 'WITH_PYMALLOC': True}) + + +class TestManylinux1Tags: + + @patch('setuptools.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: True) + def test_manylinux1_compatible_on_linux_x86_64(self): + """ + Test that manylinux1 is enabled on linux_x86_64 + """ + assert pep425tags.is_manylinux1_compatible() + + @patch('setuptools.pep425tags.get_platform', lambda: 'linux_i686') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: True) + def test_manylinux1_compatible_on_linux_i686(self): + """ + Test that manylinux1 is enabled on linux_i686 + """ + assert pep425tags.is_manylinux1_compatible() + + @patch('setuptools.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: False) + def test_manylinux1_2(self): + """ + Test that manylinux1 is disabled with incompatible glibc + """ + assert not pep425tags.is_manylinux1_compatible() + + @patch('setuptools.pep425tags.get_platform', lambda: 'arm6vl') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: True) + def test_manylinux1_3(self): + """ + Test that manylinux1 is disabled on arm6vl + """ + assert not pep425tags.is_manylinux1_compatible() + + @patch('setuptools.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: True) + @patch('sys.platform', 'linux2') + def test_manylinux1_tag_is_first(self): + """ + Test that the more specific tag manylinux1 comes first. + """ + groups = {} + for pyimpl, abi, arch in pep425tags.get_supported(): + groups.setdefault((pyimpl, abi), []).append(arch) + + for arches in groups.values(): + if arches == ['any']: + continue + # Expect the most specific arch first: + if len(arches) == 3: + assert arches == ['manylinux1_x86_64', 'linux_x86_64', 'any'] + else: + assert arches == ['manylinux1_x86_64', 'linux_x86_64'] diff --git a/setuptools/tests/test_register.py b/setuptools/tests/test_register.py new file mode 100644 index 00000000..96114595 --- /dev/null +++ b/setuptools/tests/test_register.py @@ -0,0 +1,43 @@ +import mock +from distutils import log + +import pytest + +from setuptools.command.register import register +from setuptools.dist import Distribution + + +class TestRegisterTest: + def test_warns_deprecation(self): + dist = Distribution() + + cmd = register(dist) + cmd.run_command = mock.Mock() + cmd.send_metadata = mock.Mock() + cmd.announce = mock.Mock() + + cmd.run() + + cmd.announce.assert_called_with( + "WARNING: Registering is deprecated, use twine to upload instead " + "(https://pypi.org/p/twine/)", + log.WARN + ) + + def test_warns_deprecation_when_raising(self): + dist = Distribution() + + cmd = register(dist) + cmd.run_command = mock.Mock() + cmd.send_metadata = mock.Mock() + cmd.send_metadata.side_effect = Exception + cmd.announce = mock.Mock() + + with pytest.raises(Exception): + cmd.run() + + cmd.announce.assert_called_with( + "WARNING: Registering is deprecated, use twine to upload instead " + "(https://pypi.org/p/twine/)", + log.WARN + ) diff --git a/setuptools/tests/test_sandbox.py b/setuptools/tests/test_sandbox.py index a3f1206d..d8675422 100644 --- a/setuptools/tests/test_sandbox.py +++ b/setuptools/tests/test_sandbox.py @@ -75,6 +75,8 @@ class TestExceptionSaver: def test_unpickleable_exception(self): class CantPickleThis(Exception): "This Exception is unpickleable because it's not in globals" + def __repr__(self): + return 'CantPickleThis%r' % (self.args,) with setuptools.sandbox.ExceptionSaver() as saved_exc: raise CantPickleThis('detail') diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index 02222da5..d2c4e0cf 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -20,8 +20,8 @@ from setuptools.command.egg_info import manifest_maker from setuptools.dist import Distribution from setuptools.tests import fail_on_ascii from .text import Filenames +from . import py3_only -py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") SETUP_ATTRS = { 'name': 'sdist_test', @@ -92,9 +92,8 @@ fail_on_latin1_encoded_filenames = pytest.mark.xfail( class TestSdistTest: def setup_method(self, method): self.temp_dir = tempfile.mkdtemp() - f = open(os.path.join(self.temp_dir, 'setup.py'), 'w') - f.write(SETUP_PY) - f.close() + with open(os.path.join(self.temp_dir, 'setup.py'), 'w') as f: + f.write(SETUP_PY) # Set up the rest of the test package test_pkg = os.path.join(self.temp_dir, 'sdist_test') @@ -135,6 +134,47 @@ class TestSdistTest: assert os.path.join('sdist_test', 'c.rst') not in manifest assert os.path.join('d', 'e.dat') in manifest + def test_setup_py_exists(self): + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'foo.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + with quiet(): + cmd.run() + + manifest = cmd.filelist.files + assert 'setup.py' in manifest + + def test_setup_py_missing(self): + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'foo.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + if os.path.exists("setup.py"): + os.remove("setup.py") + with quiet(): + cmd.run() + + manifest = cmd.filelist.files + assert 'setup.py' not in manifest + + def test_setup_py_excluded(self): + with open("MANIFEST.in", "w") as manifest_file: + manifest_file.write("exclude setup.py") + + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'foo.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + with quiet(): + cmd.run() + + manifest = cmd.filelist.files + assert 'setup.py' not in manifest + def test_defaults_case_sensitivity(self): """ Make sure default files (README.*, etc.) are added in a case-sensitive diff --git a/setuptools/tests/test_setuptools.py b/setuptools/tests/test_setuptools.py index 26e37a6c..7aae3a16 100644 --- a/setuptools/tests/test_setuptools.py +++ b/setuptools/tests/test_setuptools.py @@ -210,6 +210,7 @@ class TestDistro: self.dist.exclude(package_dir=['q']) +@pytest.mark.filterwarnings('ignore:Features are deprecated') class TestFeatures: def setup_method(self, method): self.req = Require('Distutils', '1.0.3', 'distutils') diff --git a/setuptools/tests/test_test.py b/setuptools/tests/test_test.py index 7ea43c57..8d1425e1 100644 --- a/setuptools/tests/test_test.py +++ b/setuptools/tests/test_test.py @@ -2,9 +2,9 @@ from __future__ import unicode_literals +from distutils import log import os -import site -from distutils.errors import DistutilsError +import sys import pytest @@ -66,26 +66,62 @@ def sample_test(tmpdir_cwd): f.write(TEST_PY) -@pytest.mark.skipif('hasattr(sys, "real_prefix")') -@pytest.mark.usefixtures('user_override') -@pytest.mark.usefixtures('sample_test') -class TestTestTest: - def test_test(self): - params = dict( - name='foo', - packages=['name', 'name.space', 'name.space.tests'], - namespace_packages=['name'], - test_suite='name.space.tests.test_suite', - use_2to3=True, - ) - dist = Distribution(params) - dist.script_name = 'setup.py' - cmd = test(dist) - cmd.user = 1 - cmd.ensure_finalized() - cmd.install_dir = site.USER_SITE - cmd.user = 1 - with contexts.quiet(): - # The test runner calls sys.exit - with contexts.suppress_exceptions(SystemExit): - cmd.run() +@pytest.fixture +def quiet_log(): + # Running some of the other tests will automatically + # change the log level to info, messing our output. + log.set_verbosity(0) + + +@pytest.mark.usefixtures('sample_test', 'quiet_log') +def test_test(capfd): + params = dict( + name='foo', + packages=['name', 'name.space', 'name.space.tests'], + namespace_packages=['name'], + test_suite='name.space.tests.test_suite', + use_2to3=True, + ) + dist = Distribution(params) + dist.script_name = 'setup.py' + cmd = test(dist) + cmd.ensure_finalized() + # The test runner calls sys.exit + with contexts.suppress_exceptions(SystemExit): + cmd.run() + out, err = capfd.readouterr() + assert out == 'Foo\n' + + +@pytest.mark.usefixtures('tmpdir_cwd', 'quiet_log') +def test_tests_are_run_once(capfd): + params = dict( + name='foo', + packages=['dummy'], + ) + with open('setup.py', 'wt') as f: + f.write('from setuptools import setup; setup(\n') + for k, v in sorted(params.items()): + f.write(' %s=%r,\n' % (k, v)) + f.write(')\n') + os.makedirs('dummy') + with open('dummy/__init__.py', 'wt'): + pass + with open('dummy/test_dummy.py', 'wt') as f: + f.write(DALS( + """ + from __future__ import print_function + import unittest + class TestTest(unittest.TestCase): + def test_test(self): + print('Foo') + """)) + dist = Distribution(params) + dist.script_name = 'setup.py' + cmd = test(dist) + cmd.ensure_finalized() + # The test runner calls sys.exit + with contexts.suppress_exceptions(SystemExit): + cmd.run() + out, err = capfd.readouterr() + assert out == 'Foo\n' diff --git a/setuptools/tests/test_upload.py b/setuptools/tests/test_upload.py new file mode 100644 index 00000000..320c6959 --- /dev/null +++ b/setuptools/tests/test_upload.py @@ -0,0 +1,213 @@ +import mock +import os +import re + +from distutils import log +from distutils.errors import DistutilsError + +import pytest + +from setuptools.command.upload import upload +from setuptools.dist import Distribution +from setuptools.extern import six + + +def _parse_upload_body(body): + boundary = u'\r\n----------------GHSKFJDLGDS7543FJKLFHRE75642756743254' + entries = [] + name_re = re.compile(u'^Content-Disposition: form-data; name="([^\"]+)"') + + for entry in body.split(boundary): + pair = entry.split(u'\r\n\r\n') + if not len(pair) == 2: + continue + + key, value = map(six.text_type.strip, pair) + m = name_re.match(key) + if m is not None: + key = m.group(1) + + entries.append((key, value)) + + return entries + + +@pytest.fixture +def patched_upload(tmpdir): + class Fix: + def __init__(self, cmd, urlopen): + self.cmd = cmd + self.urlopen = urlopen + + def __iter__(self): + return iter((self.cmd, self.urlopen)) + + def get_uploaded_metadata(self): + request = self.urlopen.call_args_list[0][0][0] + body = request.data.decode('utf-8') + entries = dict(_parse_upload_body(body)) + + return entries + + class ResponseMock(mock.Mock): + def getheader(self, name, default=None): + """Mocked getheader method for response object""" + return { + 'content-type': 'text/plain; charset=utf-8', + }.get(name.lower(), default) + + with mock.patch('setuptools.command.upload.urlopen') as urlopen: + urlopen.return_value = ResponseMock() + urlopen.return_value.getcode.return_value = 200 + urlopen.return_value.read.return_value = b'' + + content = os.path.join(str(tmpdir), "content_data") + + with open(content, 'w') as f: + f.write("Some content") + + dist = Distribution() + dist.dist_files = [('sdist', '3.7.0', content)] + + cmd = upload(dist) + cmd.announce = mock.Mock() + cmd.username = 'user' + cmd.password = 'hunter2' + + yield Fix(cmd, urlopen) + + +class TestUploadTest: + def test_upload_metadata(self, patched_upload): + cmd, patch = patched_upload + + # Set the metadata version to 2.1 + cmd.distribution.metadata.metadata_version = '2.1' + + # Run the command + cmd.ensure_finalized() + cmd.run() + + # Make sure we did the upload + patch.assert_called_once() + + # Make sure the metadata version is correct in the headers + entries = patched_upload.get_uploaded_metadata() + assert entries['metadata_version'] == '2.1' + + def test_warns_deprecation(self): + dist = Distribution() + dist.dist_files = [(mock.Mock(), mock.Mock(), mock.Mock())] + + cmd = upload(dist) + cmd.upload_file = mock.Mock() + cmd.announce = mock.Mock() + + cmd.run() + + cmd.announce.assert_called_once_with( + "WARNING: Uploading via this command is deprecated, use twine to " + "upload instead (https://pypi.org/p/twine/)", + log.WARN + ) + + def test_warns_deprecation_when_raising(self): + dist = Distribution() + dist.dist_files = [(mock.Mock(), mock.Mock(), mock.Mock())] + + cmd = upload(dist) + cmd.upload_file = mock.Mock() + cmd.upload_file.side_effect = Exception + cmd.announce = mock.Mock() + + with pytest.raises(Exception): + cmd.run() + + cmd.announce.assert_called_once_with( + "WARNING: Uploading via this command is deprecated, use twine to " + "upload instead (https://pypi.org/p/twine/)", + log.WARN + ) + + @pytest.mark.parametrize('url', [ + 'https://example.com/a;parameter', # Has parameters + 'https://example.com/a?query', # Has query + 'https://example.com/a#fragment', # Has fragment + 'ftp://example.com', # Invalid scheme + + ]) + def test_upload_file_invalid_url(self, url, patched_upload): + patched_upload.urlopen.side_effect = Exception("Should not be reached") + + cmd = patched_upload.cmd + cmd.repository = url + + cmd.ensure_finalized() + with pytest.raises(AssertionError): + cmd.run() + + def test_upload_file_http_error(self, patched_upload): + patched_upload.urlopen.side_effect = six.moves.urllib.error.HTTPError( + 'https://example.com', + 404, + 'File not found', + None, + None + ) + + cmd = patched_upload.cmd + cmd.ensure_finalized() + + with pytest.raises(DistutilsError): + cmd.run() + + cmd.announce.assert_any_call( + 'Upload failed (404): File not found', + log.ERROR) + + def test_upload_file_os_error(self, patched_upload): + patched_upload.urlopen.side_effect = OSError("Invalid") + + cmd = patched_upload.cmd + cmd.ensure_finalized() + + with pytest.raises(OSError): + cmd.run() + + cmd.announce.assert_any_call('Invalid', log.ERROR) + + @mock.patch('setuptools.command.upload.spawn') + def test_upload_file_gpg(self, spawn, patched_upload): + cmd, urlopen = patched_upload + + cmd.sign = True + cmd.identity = "Alice" + cmd.dry_run = True + content_fname = cmd.distribution.dist_files[0][2] + signed_file = content_fname + '.asc' + + with open(signed_file, 'wb') as f: + f.write("signed-data".encode('utf-8')) + + cmd.ensure_finalized() + cmd.run() + + # Make sure that GPG was called + spawn.assert_called_once_with([ + "gpg", "--detach-sign", "--local-user", "Alice", "-a", + content_fname + ], dry_run=True) + + # Read the 'signed' data that was transmitted + entries = patched_upload.get_uploaded_metadata() + assert entries['gpg_signature'] == 'signed-data' + + def test_show_response_no_error(self, patched_upload): + # This test is just that show_response doesn't throw an error + # It is not really important what the printed response looks like + # in a deprecated command, but we don't want to introduce new + # errors when importing this function from distutils + + patched_upload.cmd.show_response = True + patched_upload.cmd.ensure_finalized() + patched_upload.cmd.run() diff --git a/setuptools/tests/test_virtualenv.py b/setuptools/tests/test_virtualenv.py index 9dbd3c86..7b5fea17 100644 --- a/setuptools/tests/test_virtualenv.py +++ b/setuptools/tests/test_virtualenv.py @@ -1,7 +1,9 @@ +import distutils.command import glob import os import sys +import pytest from pytest import yield_fixture from pytest_fixture_config import yield_requires_config @@ -11,6 +13,20 @@ from .textwrap import DALS from .test_easy_install import make_nspkg_sdist +@pytest.fixture(autouse=True) +def pytest_virtualenv_works(virtualenv): + """ + pytest_virtualenv may not work. if it doesn't, skip these + tests. See #1284. + """ + venv_prefix = virtualenv.run( + 'python -c "import sys; print(sys.prefix)"', + capture=True, + ).strip() + if venv_prefix == sys.prefix: + pytest.skip("virtualenv is broken (see pypa/setuptools#1284)") + + @yield_requires_config(pytest_virtualenv.CONFIG, ['virtualenv_executable']) @yield_fixture(scope='function') def bare_virtualenv(): @@ -26,6 +42,7 @@ def bare_virtualenv(): SOURCE_DIR = os.path.join(os.path.dirname(__file__), '../..') + def test_clean_env_install(bare_virtualenv): """ Check setuptools can be installed in a clean environment. @@ -35,14 +52,12 @@ def test_clean_env_install(bare_virtualenv): 'python setup.py install', )).format(source=SOURCE_DIR)) + def test_pip_upgrade_from_source(virtualenv): """ Check pip can upgrade setuptools from source. """ dist_dir = virtualenv.workspace - if sys.version_info < (2, 7): - # Python 2.6 support was dropped in wheel 0.30.0. - virtualenv.run('pip install -U "wheel<0.30.0"') # Generate source distribution / wheel. virtualenv.run(' && '.join(( 'cd {source}', @@ -56,6 +71,7 @@ def test_pip_upgrade_from_source(virtualenv): # And finally try to upgrade from source. virtualenv.run('pip install --no-cache-dir --upgrade ' + sdist) + def test_test_command_install_requirements(bare_virtualenv, tmpdir): """ Check the test command will install all required dependencies. @@ -64,6 +80,7 @@ def test_test_command_install_requirements(bare_virtualenv, tmpdir): 'cd {source}', 'python setup.py develop', )).format(source=SOURCE_DIR)) + def sdist(distname, version): dist_path = tmpdir.join('%s-%s.tar.gz' % (distname, version)) make_nspkg_sdist(str(dist_path), distname, version) @@ -118,3 +135,14 @@ def test_test_command_install_requirements(bare_virtualenv, tmpdir): 'python setup.py test -s test', )).format(tmpdir=tmpdir)) assert tmpdir.join('success').check() + + +def test_no_missing_dependencies(bare_virtualenv): + """ + Quick and dirty test to ensure all external dependencies are vendored. + """ + for command in ('upload',):#sorted(distutils.command.__all__): + bare_virtualenv.run(' && '.join(( + 'cd {source}', + 'python setup.py {command} -h', + )).format(command=command, source=SOURCE_DIR)) diff --git a/setuptools/tests/test_wheel.py b/setuptools/tests/test_wheel.py new file mode 100644 index 00000000..6db5fa11 --- /dev/null +++ b/setuptools/tests/test_wheel.py @@ -0,0 +1,543 @@ +# -*- coding: utf-8 -*- + +"""wheel tests +""" + +from distutils.sysconfig import get_config_var +from distutils.util import get_platform +import contextlib +import glob +import inspect +import os +import shutil +import subprocess +import sys +import zipfile + +import pytest + +from pkg_resources import Distribution, PathMetadata, PY_MAJOR +from setuptools.extern.packaging.utils import canonicalize_name +from setuptools.wheel import Wheel + +from .contexts import tempdir +from .files import build_files +from .textwrap import DALS + +__metaclass__ = type + + +WHEEL_INFO_TESTS = ( + ('invalid.whl', ValueError), + ('simplewheel-2.0-1-py2.py3-none-any.whl', { + 'project_name': 'simplewheel', + 'version': '2.0', + 'build': '1', + 'py_version': 'py2.py3', + 'abi': 'none', + 'platform': 'any', + }), + ('simple.dist-0.1-py2.py3-none-any.whl', { + 'project_name': 'simple.dist', + 'version': '0.1', + 'build': None, + 'py_version': 'py2.py3', + 'abi': 'none', + 'platform': 'any', + }), + ('example_pkg_a-1-py3-none-any.whl', { + 'project_name': 'example_pkg_a', + 'version': '1', + 'build': None, + 'py_version': 'py3', + 'abi': 'none', + 'platform': 'any', + }), + ('PyQt5-5.9-5.9.1-cp35.cp36.cp37-abi3-manylinux1_x86_64.whl', { + 'project_name': 'PyQt5', + 'version': '5.9', + 'build': '5.9.1', + 'py_version': 'cp35.cp36.cp37', + 'abi': 'abi3', + 'platform': 'manylinux1_x86_64', + }), +) + +@pytest.mark.parametrize( + ('filename', 'info'), WHEEL_INFO_TESTS, + ids=[t[0] for t in WHEEL_INFO_TESTS] +) +def test_wheel_info(filename, info): + if inspect.isclass(info): + with pytest.raises(info): + Wheel(filename) + return + w = Wheel(filename) + assert {k: getattr(w, k) for k in info.keys()} == info + + +@contextlib.contextmanager +def build_wheel(extra_file_defs=None, **kwargs): + file_defs = { + 'setup.py': (DALS( + ''' + # -*- coding: utf-8 -*- + from setuptools import setup + import setuptools + setup(**%r) + ''' + ) % kwargs).encode('utf-8'), + } + if extra_file_defs: + file_defs.update(extra_file_defs) + with tempdir() as source_dir: + build_files(file_defs, source_dir) + subprocess.check_call((sys.executable, 'setup.py', + '-q', 'bdist_wheel'), cwd=source_dir) + yield glob.glob(os.path.join(source_dir, 'dist', '*.whl'))[0] + + +def tree_set(root): + contents = set() + for dirpath, dirnames, filenames in os.walk(root): + for filename in filenames: + contents.add(os.path.join(os.path.relpath(dirpath, root), + filename)) + return contents + + +def flatten_tree(tree): + """Flatten nested dicts and lists into a full list of paths""" + output = set() + for node, contents in tree.items(): + if isinstance(contents, dict): + contents = flatten_tree(contents) + + for elem in contents: + if isinstance(elem, dict): + output |= {os.path.join(node, val) + for val in flatten_tree(elem)} + else: + output.add(os.path.join(node, elem)) + return output + + +def format_install_tree(tree): + return {x.format( + py_version=PY_MAJOR, + platform=get_platform(), + shlib_ext=get_config_var('EXT_SUFFIX') or get_config_var('SO')) + for x in tree} + + +def _check_wheel_install(filename, install_dir, install_tree_includes, + project_name, version, requires_txt): + w = Wheel(filename) + egg_path = os.path.join(install_dir, w.egg_name()) + w.install_as_egg(egg_path) + if install_tree_includes is not None: + install_tree = format_install_tree(install_tree_includes) + exp = tree_set(install_dir) + assert install_tree.issubset(exp), (install_tree - exp) + + metadata = PathMetadata(egg_path, os.path.join(egg_path, 'EGG-INFO')) + dist = Distribution.from_filename(egg_path, metadata=metadata) + assert dist.project_name == project_name + assert dist.version == version + if requires_txt is None: + assert not dist.has_metadata('requires.txt') + else: + assert requires_txt == dist.get_metadata('requires.txt').lstrip() + + +class Record: + + def __init__(self, id, **kwargs): + self._id = id + self._fields = kwargs + + def __repr__(self): + return '%s(**%r)' % (self._id, self._fields) + + +WHEEL_INSTALL_TESTS = ( + + dict( + id='basic', + file_defs={ + 'foo': { + '__init__.py': '' + } + }, + setup_kwargs=dict( + packages=['foo'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt' + ], + 'foo': ['__init__.py'] + } + }), + ), + + dict( + id='utf-8', + setup_kwargs=dict( + description='Description accentuée', + ) + ), + + dict( + id='data', + file_defs={ + 'data.txt': DALS( + ''' + Some data... + ''' + ), + }, + setup_kwargs=dict( + data_files=[('data_dir', ['data.txt'])], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt' + ], + 'data_dir': [ + 'data.txt' + ] + } + }), + ), + + dict( + id='extension', + file_defs={ + 'extension.c': DALS( + ''' + #include "Python.h" + + #if PY_MAJOR_VERSION >= 3 + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "extension", + NULL, + 0, + NULL, + NULL, + NULL, + NULL, + NULL + }; + + #define INITERROR return NULL + + PyMODINIT_FUNC PyInit_extension(void) + + #else + + #define INITERROR return + + void initextension(void) + + #endif + { + #if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); + #else + PyObject *module = Py_InitModule("extension", NULL); + #endif + if (module == NULL) + INITERROR; + #if PY_MAJOR_VERSION >= 3 + return module; + #endif + } + ''' + ), + }, + setup_kwargs=dict( + ext_modules=[ + Record('setuptools.Extension', + name='extension', + sources=['extension.c']) + ], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}-{platform}.egg': [ + 'extension{shlib_ext}', + {'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt', + ]}, + ] + }), + ), + + dict( + id='header', + file_defs={ + 'header.h': DALS( + ''' + ''' + ), + }, + setup_kwargs=dict( + headers=['header.h'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': [ + 'header.h', + {'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt', + ]}, + ] + }), + ), + + dict( + id='script', + file_defs={ + 'script.py': DALS( + ''' + #/usr/bin/python + print('hello world!') + ''' + ), + 'script.sh': DALS( + ''' + #/bin/sh + echo 'hello world!' + ''' + ), + }, + setup_kwargs=dict( + scripts=['script.py', 'script.sh'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt', + {'scripts': [ + 'script.py', + 'script.sh' + ]} + + ] + } + }) + ), + + dict( + id='requires1', + install_requires='foobar==2.0', + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'requires.txt', + 'top_level.txt', + ] + } + }), + requires_txt=DALS( + ''' + foobar==2.0 + ''' + ), + ), + + dict( + id='requires2', + install_requires=''' + bar + foo<=2.0; %r in sys_platform + ''' % sys.platform, + requires_txt=DALS( + ''' + bar + foo<=2.0 + ''' + ), + ), + + dict( + id='requires3', + install_requires=''' + bar; %r != sys_platform + ''' % sys.platform, + ), + + dict( + id='requires4', + install_requires=''' + foo + ''', + extras_require={ + 'extra': 'foobar>3', + }, + requires_txt=DALS( + ''' + foo + + [extra] + foobar>3 + ''' + ), + ), + + dict( + id='requires5', + extras_require={ + 'extra': 'foobar; %r != sys_platform' % sys.platform, + }, + requires_txt=DALS( + ''' + [extra] + ''' + ), + ), + + dict( + id='namespace_package', + file_defs={ + 'foo': { + 'bar': { + '__init__.py': '' + }, + }, + }, + setup_kwargs=dict( + namespace_packages=['foo'], + packages=['foo.bar'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': [ + 'foo-1.0-py{py_version}-nspkg.pth', + {'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'namespace_packages.txt', + 'top_level.txt', + ]}, + {'foo': [ + '__init__.py', + {'bar': ['__init__.py']}, + ]}, + ] + }), + ), + + dict( + id='data_in_package', + file_defs={ + 'foo': { + '__init__.py': '', + 'data_dir': { + 'data.txt': DALS( + ''' + Some data... + ''' + ), + } + } + }, + setup_kwargs=dict( + packages=['foo'], + data_files=[('foo/data_dir', ['foo/data_dir/data.txt'])], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt', + ], + 'foo': [ + '__init__.py', + {'data_dir': [ + 'data.txt', + ]} + ] + } + }), + ), + +) + +@pytest.mark.parametrize( + 'params', WHEEL_INSTALL_TESTS, + ids=list(params['id'] for params in WHEEL_INSTALL_TESTS), +) +def test_wheel_install(params): + project_name = params.get('name', 'foo') + version = params.get('version', '1.0') + install_requires = params.get('install_requires', []) + extras_require = params.get('extras_require', {}) + requires_txt = params.get('requires_txt', None) + install_tree = params.get('install_tree') + file_defs = params.get('file_defs', {}) + setup_kwargs = params.get('setup_kwargs', {}) + with build_wheel( + name=project_name, + version=version, + install_requires=install_requires, + extras_require=extras_require, + extra_file_defs=file_defs, + **setup_kwargs + ) as filename, tempdir() as install_dir: + _check_wheel_install(filename, install_dir, + install_tree, project_name, + version, requires_txt) + + +def test_wheel_install_pep_503(): + project_name = 'Foo_Bar' # PEP 503 canonicalized name is "foo-bar" + version = '1.0' + with build_wheel( + name=project_name, + version=version, + ) as filename, tempdir() as install_dir: + new_filename = filename.replace(project_name, + canonicalize_name(project_name)) + shutil.move(filename, new_filename) + _check_wheel_install(new_filename, install_dir, None, + canonicalize_name(project_name), + version, None) + + +def test_wheel_no_dist_dir(): + project_name = 'nodistinfo' + version = '1.0' + wheel_name = '{0}-{1}-py2.py3-none-any.whl'.format(project_name, version) + with tempdir() as source_dir: + wheel_path = os.path.join(source_dir, wheel_name) + # create an empty zip file + zipfile.ZipFile(wheel_path, 'w').close() + with tempdir() as install_dir: + with pytest.raises(ValueError): + _check_wheel_install(wheel_path, install_dir, None, + project_name, + version, None) diff --git a/setuptools/wheel.py b/setuptools/wheel.py new file mode 100644 index 00000000..e11f0a1d --- /dev/null +++ b/setuptools/wheel.py @@ -0,0 +1,211 @@ +"""Wheels support.""" + +from distutils.util import get_platform +import email +import itertools +import os +import posixpath +import re +import zipfile + +import pkg_resources +import setuptools +from pkg_resources import parse_version +from setuptools.extern.packaging.utils import canonicalize_name +from setuptools.extern.six import PY3 +from setuptools import pep425tags +from setuptools.command.egg_info import write_requirements + + +__metaclass__ = type + + +WHEEL_NAME = re.compile( + r"""^(?P<project_name>.+?)-(?P<version>\d.*?) + ((-(?P<build>\d.*?))?-(?P<py_version>.+?)-(?P<abi>.+?)-(?P<platform>.+?) + )\.whl$""", + re.VERBOSE).match + +NAMESPACE_PACKAGE_INIT = '''\ +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + __path__ = __import__('pkgutil').extend_path(__path__, __name__) +''' + + +def unpack(src_dir, dst_dir): + '''Move everything under `src_dir` to `dst_dir`, and delete the former.''' + for dirpath, dirnames, filenames in os.walk(src_dir): + subdir = os.path.relpath(dirpath, src_dir) + for f in filenames: + src = os.path.join(dirpath, f) + dst = os.path.join(dst_dir, subdir, f) + os.renames(src, dst) + for n, d in reversed(list(enumerate(dirnames))): + src = os.path.join(dirpath, d) + dst = os.path.join(dst_dir, subdir, d) + if not os.path.exists(dst): + # Directory does not exist in destination, + # rename it and prune it from os.walk list. + os.renames(src, dst) + del dirnames[n] + # Cleanup. + for dirpath, dirnames, filenames in os.walk(src_dir, topdown=True): + assert not filenames + os.rmdir(dirpath) + + +class Wheel: + + def __init__(self, filename): + match = WHEEL_NAME(os.path.basename(filename)) + if match is None: + raise ValueError('invalid wheel name: %r' % filename) + self.filename = filename + for k, v in match.groupdict().items(): + setattr(self, k, v) + + def tags(self): + '''List tags (py_version, abi, platform) supported by this wheel.''' + return itertools.product( + self.py_version.split('.'), + self.abi.split('.'), + self.platform.split('.'), + ) + + def is_compatible(self): + '''Is the wheel is compatible with the current platform?''' + supported_tags = pep425tags.get_supported() + return next((True for t in self.tags() if t in supported_tags), False) + + def egg_name(self): + return pkg_resources.Distribution( + project_name=self.project_name, version=self.version, + platform=(None if self.platform == 'any' else get_platform()), + ).egg_name() + '.egg' + + def get_dist_info(self, zf): + # find the correct name of the .dist-info dir in the wheel file + for member in zf.namelist(): + dirname = posixpath.dirname(member) + if (dirname.endswith('.dist-info') and + canonicalize_name(dirname).startswith( + canonicalize_name(self.project_name))): + return dirname + raise ValueError("unsupported wheel format. .dist-info not found") + + def install_as_egg(self, destination_eggdir): + '''Install wheel as an egg directory.''' + with zipfile.ZipFile(self.filename) as zf: + self._install_as_egg(destination_eggdir, zf) + + def _install_as_egg(self, destination_eggdir, zf): + dist_basename = '%s-%s' % (self.project_name, self.version) + dist_info = self.get_dist_info(zf) + dist_data = '%s.data' % dist_basename + egg_info = os.path.join(destination_eggdir, 'EGG-INFO') + + self._convert_metadata(zf, destination_eggdir, dist_info, egg_info) + self._move_data_entries(destination_eggdir, dist_data) + self._fix_namespace_packages(egg_info, destination_eggdir) + + @staticmethod + def _convert_metadata(zf, destination_eggdir, dist_info, egg_info): + def get_metadata(name): + with zf.open(posixpath.join(dist_info, name)) as fp: + value = fp.read().decode('utf-8') if PY3 else fp.read() + return email.parser.Parser().parsestr(value) + + wheel_metadata = get_metadata('WHEEL') + # Check wheel format version is supported. + wheel_version = parse_version(wheel_metadata.get('Wheel-Version')) + wheel_v1 = ( + parse_version('1.0') <= wheel_version < parse_version('2.0dev0') + ) + if not wheel_v1: + raise ValueError( + 'unsupported wheel format version: %s' % wheel_version) + # Extract to target directory. + os.mkdir(destination_eggdir) + zf.extractall(destination_eggdir) + # Convert metadata. + dist_info = os.path.join(destination_eggdir, dist_info) + dist = pkg_resources.Distribution.from_location( + destination_eggdir, dist_info, + metadata=pkg_resources.PathMetadata(destination_eggdir, dist_info), + ) + + # Note: Evaluate and strip markers now, + # as it's difficult to convert back from the syntax: + # foobar; "linux" in sys_platform and extra == 'test' + def raw_req(req): + req.marker = None + return str(req) + install_requires = list(sorted(map(raw_req, dist.requires()))) + extras_require = { + extra: sorted( + req + for req in map(raw_req, dist.requires((extra,))) + if req not in install_requires + ) + for extra in dist.extras + } + os.rename(dist_info, egg_info) + os.rename( + os.path.join(egg_info, 'METADATA'), + os.path.join(egg_info, 'PKG-INFO'), + ) + setup_dist = setuptools.Distribution( + attrs=dict( + install_requires=install_requires, + extras_require=extras_require, + ), + ) + write_requirements( + setup_dist.get_command_obj('egg_info'), + None, + os.path.join(egg_info, 'requires.txt'), + ) + + @staticmethod + def _move_data_entries(destination_eggdir, dist_data): + """Move data entries to their correct location.""" + dist_data = os.path.join(destination_eggdir, dist_data) + dist_data_scripts = os.path.join(dist_data, 'scripts') + if os.path.exists(dist_data_scripts): + egg_info_scripts = os.path.join( + destination_eggdir, 'EGG-INFO', 'scripts') + os.mkdir(egg_info_scripts) + for entry in os.listdir(dist_data_scripts): + # Remove bytecode, as it's not properly handled + # during easy_install scripts install phase. + if entry.endswith('.pyc'): + os.unlink(os.path.join(dist_data_scripts, entry)) + else: + os.rename( + os.path.join(dist_data_scripts, entry), + os.path.join(egg_info_scripts, entry), + ) + os.rmdir(dist_data_scripts) + for subdir in filter(os.path.exists, ( + os.path.join(dist_data, d) + for d in ('data', 'headers', 'purelib', 'platlib') + )): + unpack(subdir, destination_eggdir) + if os.path.exists(dist_data): + os.rmdir(dist_data) + + @staticmethod + def _fix_namespace_packages(egg_info, destination_eggdir): + namespace_packages = os.path.join( + egg_info, 'namespace_packages.txt') + if os.path.exists(namespace_packages): + with open(namespace_packages) as fp: + namespace_packages = fp.read().split() + for mod in namespace_packages: + mod_dir = os.path.join(destination_eggdir, *mod.split('.')) + mod_init = os.path.join(mod_dir, '__init__.py') + if os.path.exists(mod_dir) and not os.path.exists(mod_init): + with open(mod_init, 'w') as fp: + fp.write(NAMESPACE_PACKAGE_INIT) |