diff options
134 files changed, 14662 insertions, 2031 deletions
diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000..69cb7601 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1 @@ +comment: false diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..aa550807 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,12 @@ +<!-- First time contributors: Take a moment to review https://setuptools.readthedocs.io/en/latest/developer-guide.html! --> +<!-- Remove sections if not applicable --> + +## Summary of changes + +<!-- Summary goes here --> + +Closes <!-- issue number here --> + +### Pull Request Checklist +- [ ] Changes have tests +- [ ] News fragment added in changelog.d. See [documentation](http://setuptools.readthedocs.io/en/latest/developer-guide.html#making-a-pull-request) for details @@ -2,6 +2,7 @@ bin build dist +docs/build include lib distribute.egg-info @@ -9,9 +10,12 @@ setuptools.egg-info .coverage .eggs .tox +.venv *.egg *.py[cod] *.swp *~ .hg* .cache +.idea/ +.pytest_cache/ diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 00000000..7f0df535 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,11 @@ +pull_request_rules: +- name: auto-merge + conditions: + - base=master + - label=auto-merge + - status-success=continuous-integration/appveyor/pr + - status-success=continuous-integration/travis-ci/pr + - status-success=deploy/netlify + actions: + merge: + method: merge diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..3aef6b6b --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,4 @@ +python: + version: 3 + requirements_file: docs/requirements.txt + pip_install: false diff --git a/.travis.yml b/.travis.yml index 60f5815f..d1febccb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,38 @@ dist: trusty -sudo: false language: python + jobs: fast_finish: true include: - - python: 2.6 - - python: &latest_py2 2.7 - - python: 3.3 - - python: 3.4 - - python: 3.5 - - python: &latest_py3 3.6 - - python: nightly + - &latest_py2 + python: 2.7 + - <<: *latest_py2 + env: LANG=C - python: pypy + env: DISABLE_COVERAGE=1 # Don't run coverage on pypy (too slow). - python: pypy3 - - python: *latest_py3 - env: LANG=C - - python: *latest_py2 + env: DISABLE_COVERAGE=1 + - python: 3.4 + - python: 3.5 + - &default_py + python: 3.6 + - &latest_py3 + python: 3.7 + dist: xenial + - <<: *latest_py3 env: LANG=C - - stage: deploy (to PyPI for tagged commits) + - python: 3.8-dev + dist: xenial + env: DISABLE_COVERAGE=1 # Ignore invalid coverage data. + - <<: *default_py + stage: deploy (to PyPI for tagged commits) if: tag IS present - python: *latest_py3 install: skip script: skip - before_deploy: python bootstrap.py + after_success: true + before_deploy: + - python bootstrap.py + - "! grep pyc setuptools.egg-info/SOURCES.txt" deploy: provider: pypi on: @@ -38,13 +48,37 @@ jobs: cache: pip install: + +# ensure we have recent pip/setuptools/wheel +- pip install --disable-pip-version-check --upgrade pip setuptools wheel # need tox to get started -- pip install tox +- pip install --upgrade tox tox-venv # Output the env, to verify behavior +- pip freeze --all - env # update egg_info based on setup.py in checkout - python bootstrap.py +- "! grep pyc setuptools.egg-info/SOURCES.txt" + +script: + - | + ( # Run testsuite. + if [ -z "$DISABLE_COVERAGE" ] + then + tox -- --cov + else + tox + fi + ) -script: tox +after_success: + - | + ( # Upload coverage data. + if [ -z "$DISABLE_COVERAGE" ] + then + export TRAVIS_JOB_NAME="${TRAVIS_PYTHON_VERSION} (LANG=$LANG)" CODECOV_ENV=TRAVIS_JOB_NAME + tox -e coverage,codecov + fi + ) diff --git a/CHANGES.rst b/CHANGES.rst index 6b84c03f..48a176a8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,323 @@ +v40.6.3 +------- + +* #1594: PEP 517 backend no longer declares setuptools as a dependency as it can be assumed. + + +v40.6.2 +------- + +* #1592: Fix invalid dependency on external six module (instead of vendored version). + + +v40.6.1 +------- + +* #1590: Fixed regression where packages without ``author`` or ``author_email`` fields generated malformed package metadata. + + +v40.6.0 +------- + +* #1541: Officially deprecated the ``requires`` parameter in ``setup()``. +* #1519: In ``pkg_resources.normalize_path``, additional path normalization is now performed to ensure path values to a directory is always the same, preventing false positives when checking scripts have a consistent prefix to set up on Windows. +* #1545: Changed the warning class of all deprecation warnings; deprecation warning classes are no longer derived from ``DeprecationWarning`` and are thus visible by default. +* #1554: ``build_meta.build_sdist`` now includes ``setup.py`` in source distributions by default. +* #1576: Started monkey-patching ``get_metadata_version`` and ``read_pkg_file`` onto ``distutils.DistributionMetadata`` to retain the correct version on the ``PKG-INFO`` file in the (deprecated) ``upload`` command. +* #1533: Restricted the ``recursive-include setuptools/_vendor`` to contain only .py and .txt files. +* #1395: Changed Pyrex references to Cython in the documentation. +* #1456: Documented that the ``rpmbuild`` packages is required for the ``bdist_rpm`` command. +* #1537: Documented how to use ``setup.cfg`` for ``src/ layouts`` +* #1539: Added minimum version column in ``setup.cfg`` metadata table. +* #1552: Fixed a minor typo in the python 2/3 compatibility documentation. +* #1553: Updated installation instructions to point to ``pip install`` instead of ``ez_setup.py``. +* #1560: Updated ``setuptools`` distribution documentation to remove some outdated information. +* #1564: Documented ``setup.cfg`` minimum version for version and project_urls. +* #1572: Added the ``concurrent.futures`` backport ``futures`` to the Python 2.7 test suite requirements. + + +v40.5.0 +------- + +* #1335: In ``pkg_resources.normalize_path``, fix issue on Cygwin when cwd contains symlinks. +* #1502: Deprecated support for downloads from Subversion in package_index/easy_install. +* #1517: Dropped use of six.u in favor of `u""` literals. +* #1520: Added support for ``data_files`` in ``setup.cfg``. +* #1525: Fixed rendering of the deprecation warning in easy_install doc. + + +v40.4.3 +------- + +* #1480: Bump vendored pyparsing in pkg_resources to 2.2.1. + + +v40.4.2 +------- + +* #1497: Updated gitignore in repo. + + +v40.4.1 +------- + +* #1480: Bump vendored pyparsing to 2.2.1. + + +v40.4.0 +------- + +* #1481: Join the sdist ``--dist-dir`` and the ``build_meta`` sdist directory argument to point to the same target (meaning the build frontend no longer needs to clean manually the dist dir to avoid multiple sdist presence, and setuptools no longer needs to handle conflicts between the two). + + +v40.3.0 +------- + +* #1402: Fixed a bug with namespace packages under Python 3.6 when one package in + current directory hides another which is installed. +* #1427: Set timestamp of ``.egg-info`` directory whenever ``egg_info`` command is run. +* #1474: ``build_meta.get_requires_for_build_sdist`` now does not include the ``wheel`` package anymore. +* #1486: Suppress warnings in pkg_resources.handle_ns. +* #1479: Remove internal use of six.binary_type. + + +v40.2.0 +------- + +* #1466: Fix handling of Unicode arguments in PEP 517 backend + + +v40.1.1 +-------- + +* #1465: Fix regression with `egg_info` command when tagging is used. + + +v40.1.0 +------- + +* #1410: Deprecated ``upload`` and ``register`` commands. +* #1312: Introduced find_namespace_packages() to find PEP 420 namespace packages. +* #1420: Added find_namespace: directive to config parser. +* #1418: Solved race in when creating egg cache directories. +* #1450: Upgraded vendored PyParsing from 2.1.10 to 2.2.0. +* #1451: Upgraded vendored appdirs from 1.4.0 to 1.4.3. +* #1388: Fixed "Microsoft Visual C++ Build Tools" link in exception when Visual C++ not found. +* #1389: Added support for scripts which have unicode content. +* #1416: Moved several Python version checks over to using ``six.PY2`` and ``six.PY3``. +* #1441: Removed spurious executable permissions from files that don't need them. + + +v40.0.0 +------- + +* #1342: Drop support for Python 3.3. +* #1366: In package_index, fixed handling of encoded entities in URLs. +* #1383: In pkg_resources VendorImporter, avoid removing packages imported from the root. +* #1379: Minor doc fixes after actually using the new release process. +* #1385: Removed section on non-package data files. +* #1403: Fix developer's guide. +* #1404: Fix PEP 518 configuration: set build requirements in ``pyproject.toml`` to ``["wheel"]``. + + +v39.2.0 +------- + +* #1359: Support using "file:" to load a PEP 440-compliant package version from + a text file. +* #1360: Fixed issue with a mismatch between the name of the package and the + name of the .dist-info file in wheel files +* #1364: Add `__dir__()` implementation to `pkg_resources.Distribution()` that + includes the attributes in the `_provider` instance variable. +* #1365: Take the package_dir option into account when loading the version from + a module attribute. +* #1353: Added coverage badge to README. +* #1356: Made small fixes to the developer guide documentation. +* #1357: Fixed warnings in documentation builds and started enforcing that the + docs build without warnings in tox. +* #1376: Updated release process docs. +* #1343: The ``setuptools`` specific ``long_description_content_type``, + ``project_urls`` and ``provides_extras`` fields are now set consistently + after any ``distutils`` ``setup_keywords`` calls, allowing them to override + values. +* #1352: Added ``tox`` environment for documentation builds. +* #1354: Added ``towncrier`` for changelog management. +* #1355: Add PR template. +* #1368: Fixed tests which failed without network connectivity. +* #1369: Added unit tests for PEP 425 compatibility tags support. +* #1372: Stop testing Python 3.3 in Travis CI, now that the latest version of + ``wheel`` no longer installs on it. + +v39.1.0 +------- + +* #1340: Update all PyPI URLs to reflect the switch to the + new Warehouse codebase. +* #1337: In ``pkg_resources``, now support loading resources + for modules loaded by the ``SourcelessFileLoader``. +* #1332: Silence spurious wheel related warnings on Windows. + +v39.0.1 +------- + +* #1297: Restore Unicode handling for Maintainer fields in + metadata. + +v39.0.0 +------- + +* #1296: Setuptools now vendors its own direct dependencies, no + longer relying on the dependencies as vendored by pkg_resources. + +* #296: Removed long-deprecated support for iteration on + Version objects as returned by ``pkg_resources.parse_version``. + Removed the ``SetuptoolsVersion`` and + ``SetuptoolsLegacyVersion`` names as well. They should not + have been used, but if they were, replace with + ``Version`` and ``LegacyVersion`` from ``packaging.version``. + +v38.7.0 +------- + +* #1288: Add support for maintainer in PKG-INFO. + +v38.6.1 +------- + +* #1292: Avoid generating ``Provides-Extra`` in metadata when + no extra is present (but environment markers are). + +v38.6.0 +------- + +* #1286: Add support for Metadata 2.1 (PEP 566). + +v38.5.2 +------- + +* #1285: Fixed RuntimeError in pkg_resources.parse_requirements + on Python 3.7 (stemming from PEP 479). + +v38.5.1 +------- + +* #1271: Revert to Cython legacy ``build_ext`` behavior for + compatibility. + +v38.5.0 +------- + +* #1229: Expand imports in ``build_ext`` to refine detection of + Cython availability. + +* #1270: When Cython is available, ``build_ext`` now uses the + new_build_ext. + +v38.4.1 +------- + +* #1257: In bdist_egg.scan_module, fix ValueError on Python 3.7. + +v38.4.0 +------- + +* #1231: Removed warning when PYTHONDONTWRITEBYTECODE is enabled. + +v38.3.0 +------- + +* #1210: Add support for PEP 345 Project-URL metadata. +* #1207: Add support for ``long_description_type`` to setup.cfg + declarative config as intended and documented. + +v38.2.5 +------- + +* #1232: Fix trailing slash handling in ``pkg_resources.ZipProvider``. + +v38.2.4 +------- + +* #1220: Fix `data_files` handling when installing from wheel. + +v38.2.3 +------- + +* fix Travis' Python 3.3 job. + +v38.2.2 +------- + +* #1214: fix handling of namespace packages when installing + from a wheel. + +v38.2.1 +------- + +* #1212: fix encoding handling of metadata when installing + from a wheel. + +v38.2.0 +------- + +* #1200: easy_install now support installing from wheels: + they will be installed as standalone unzipped eggs. + +v38.1.0 +------- + +* #1208: Improve error message when failing to locate scripts + in egg-info metadata. + +v38.0.0 +------- + +* #458: In order to support deterministic builds, Setuptools no + longer allows packages to declare ``install_requires`` as + unordered sequences (sets or dicts). + +v37.0.0 +------- + +* #878: Drop support for Python 2.6. Python 2.6 users should + rely on 'setuptools < 37dev'. + +v36.8.0 +------- + +* #1190: In SSL support for package index operations, use SNI + where available. + +v36.7.3 +------- + +* #1175: Bug fixes to ``build_meta`` module. + +v36.7.2 +------- + +* #701: Fixed duplicate test discovery on Python 3. + +v36.7.1 +------- + +* #1193: Avoid test failures in bdist_egg when + PYTHONDONTWRITEBYTECODE is set. + +v36.7.0 +------- + +* #1054: Support ``setup_requires`` in ``setup.cfg`` files. + v36.6.1 ------- * #1132: Removed redundant and costly serialization/parsing step in ``EntryPoint.__init__``. +* #844: ``bdist_egg --exclude-source-files`` now tested and works + on Python 3. + v36.6.0 ------- @@ -1647,7 +1961,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Issue #313: Removed built-in support for subversion. Projects wishing to retain support for subversion will need to use a third party library. The extant implementation is being ported to `setuptools_svn - <https://pypi.python.org/pypi/setuptools_svn>`_. + <https://pypi.org/project/setuptools_svn/>`_. * Issue #315: Updated setuptools to hide its own loaded modules during installation of another package. This change will enable setuptools to upgrade (or downgrade) itself even when its own metadata and implementation @@ -2152,7 +2466,7 @@ process to fail and PyPI uploads no longer accept files for 13.0. * Address security vulnerability in SSL match_hostname check as reported in Python #17997. * Prefer `backports.ssl_match_hostname - <https://pypi.python.org/pypi/backports.ssl_match_hostname>`_ for backport + <https://pypi.org/project/backports.ssl_match_hostname/>`_ for backport implementation if present. * Correct NameError in ``ssl_support`` module (``socket.error``). @@ -2979,6 +3293,37 @@ easy_install gracefully under Google App Engine (with an ``ImportError`` loading the C-based module, instead of getting a ``NameError``). + * Fixed ``win32.exe`` support for .pth files, so unnecessary directory nesting + is flattened out in the resulting egg. (There was a case-sensitivity + problem that affected some distributions, notably ``pywin32``.) + + * Prevent ``--help-commands`` and other junk from showing under Python 2.5 + when running ``easy_install --help``. + + * Fixed GUI scripts sometimes not executing on Windows + + * Fixed not picking up dependency links from recursive dependencies. + + * Only make ``.py``, ``.dll`` and ``.so`` files executable when unpacking eggs + + * Changes for Jython compatibility + + * Improved error message when a requirement is also a directory name, but the + specified directory is not a source package. + + * Fixed ``--allow-hosts`` option blocking ``file:`` URLs + + * Fixed HTTP SVN detection failing when the page title included a project + name (e.g. on SourceForge-hosted SVN) + + * Fix Jython script installation to handle ``#!`` lines better when + ``sys.executable`` is a script. + + * Removed use of deprecated ``md5`` module if ``hashlib`` is available + + * Keep site directories (e.g. ``site-packages``) from being included in + ``.pth`` files. + 0.6c7 ----- @@ -2989,6 +3334,11 @@ easy_install ``--root`` or ``--single-version-externally-managed``, due to the parent package not having the child package as an attribute. + * ``ftp:`` download URLs now work correctly. + + * The default ``--index-url`` is now ``https://pypi.python.org/simple``, to use + the Python Package Index's new simpler (and faster!) REST API. + 0.6c6 ----- @@ -3012,6 +3362,18 @@ easy_install * Fix ``find_packages()`` treating ``ez_setup`` and directories with ``.`` in their names as packages. + * EasyInstall no longer aborts the installation process if a URL it wants to + retrieve can't be downloaded, unless the URL is an actual package download. + Instead, it issues a warning and tries to keep going. + + * Fixed distutils-style scripts originally built on Windows having their line + endings doubled when installed on any platform. + + * Added ``--local-snapshots-ok`` flag, to allow building eggs from projects + installed using ``setup.py develop``. + + * Fixed not HTML-decoding URLs scraped from web pages + 0.6c5 ----- @@ -3021,6 +3383,9 @@ easy_install * Fix uploaded ``bdist_wininst`` packages being described as suitable for "any" version by Python 2.5, even if a ``--target-version`` was specified. + * Fixed ``.dll`` files on Cygwin not having executable permissions when an egg + is installed unzipped. + 0.6c4 ----- @@ -3050,11 +3415,34 @@ easy_install listed a namespace package ``foo.bar`` without explicitly listing ``foo`` as a namespace package. + * Added support for HTTP "Basic" authentication using ``http://user:pass@host`` + URLs. If a password-protected page contains links to the same host (and + protocol), those links will inherit the credentials used to access the + original page. + + * Removed all special support for Sourceforge mirrors, as Sourceforge's + mirror system now works well for non-browser downloads. + + * Fixed not recognizing ``win32.exe`` installers that included a custom + bitmap. + + * Fixed not allowing ``os.open()`` of paths outside the sandbox, even if they + are opened read-only (e.g. reading ``/dev/urandom`` for random numbers, as + is done by ``os.urandom()`` on some platforms). + + * Fixed a problem with ``.pth`` testing on Windows when ``sys.executable`` + has a space in it (e.g., the user installed Python to a ``Program Files`` + directory). + 0.6c3 ----- * Fixed breakages caused by Subversion 1.4's new "working copy" format + * You can once again use "python -m easy_install" with Python 2.4 and above. + + * Python 2.5 compatibility fixes added. + 0.6c2 ----- @@ -3065,6 +3453,19 @@ easy_install * Running ``setup.py develop`` on a setuptools-using project will now install setuptools if needed, instead of only downloading the egg. + * Windows script wrappers now support quoted arguments and arguments + containing spaces. (Patch contributed by Jim Fulton.) + + * The ``ez_setup.py`` script now actually works when you put a setuptools + ``.egg`` alongside it for bootstrapping an offline machine. + + * A writable installation directory on ``sys.path`` is no longer required to + download and extract a source distribution using ``--editable``. + + * Generated scripts now use ``-x`` on the ``#!`` line when ``sys.executable`` + contains non-ASCII characters, to prevent deprecation warnings about an + unspecified encoding when the script is run. + 0.6c1 ----- @@ -3088,6 +3489,9 @@ easy_install the version was overridden on the command line that built the source distribution.) + * EasyInstall now includes setuptools version information in the + ``User-Agent`` string sent to websites it visits. + 0.6b4 ----- @@ -3100,6 +3504,27 @@ easy_install * Fixed redundant warnings about missing ``README`` file(s); it should now appear only if you are actually a source distribution. + * Fix creating Python wrappers for non-Python scripts + + * Fix ``ftp://`` directory listing URLs from causing a crash when used in the + "Home page" or "Download URL" slots on PyPI. + + * Fix ``sys.path_importer_cache`` not being updated when an existing zipfile + or directory is deleted/overwritten. + + * Fix not recognizing HTML 404 pages from package indexes. + + * Allow ``file://`` URLs to be used as a package index. URLs that refer to + directories will use an internally-generated directory listing if there is + no ``index.html`` file in the directory. + + * Allow external links in a package index to be specified using + ``rel="homepage"`` or ``rel="download"``, without needing the old + PyPI-specific visible markup. + + * Suppressed warning message about possibly-misspelled project name, if an egg + or link for that project name has already been seen. + 0.6b3 ----- @@ -3110,6 +3535,28 @@ easy_install ``include_package_data`` and ``package_data`` are used to refer to the same files. + * Fix local ``--find-links`` eggs not being copied except with + ``--always-copy``. + + * Fix sometimes not detecting local packages installed outside of "site" + directories. + + * Fix mysterious errors during initial ``setuptools`` install, caused by + ``ez_setup`` trying to run ``easy_install`` twice, due to a code fallthru + after deleting the egg from which it's running. + +0.6b2 +----- + + * Don't install or update a ``site.py`` patch when installing to a + ``PYTHONPATH`` directory with ``--multi-version``, unless an + ``easy-install.pth`` file is already in use there. + + * Construct ``.pth`` file paths in such a way that installing an egg whose + name begins with ``import`` doesn't cause a syntax error. + + * Fixed a bogus warning message that wasn't updated since the 0.5 versions. + 0.6b1 ----- @@ -3117,6 +3564,21 @@ easy_install the name of a ``.py`` loader/wrapper. (Python's import machinery ignores this suffix when searching for an extension module.) + * Better ambiguity management: accept ``#egg`` name/version even if processing + what appears to be a correctly-named distutils file, and ignore ``.egg`` + files with no ``-``, since valid Python ``.egg`` files always have a version + number (but Scheme eggs often don't). + + * Support ``file://`` links to directories in ``--find-links``, so that + easy_install can build packages from local source checkouts. + + * Added automatic retry for Sourceforge mirrors. The new download process is + to first just try dl.sourceforge.net, then randomly select mirror IPs and + remove ones that fail, until something works. The removed IPs stay removed + for the remainder of the run. + + * Ignore bdist_dumb distributions when looking at download URLs. + 0.6a11 ------ @@ -3150,11 +3612,75 @@ easy_install it. Previously, the file could be left open and the actual error would be masked by problems trying to remove the open file on Windows systems. + * Process ``dependency_links.txt`` if found in a distribution, by adding the + URLs to the list for scanning. + + * Use relative paths in ``.pth`` files when eggs are being installed to the + same directory as the ``.pth`` file. This maximizes portability of the + target directory when building applications that contain eggs. + + * Added ``easy_install-N.N`` script(s) for convenience when using multiple + Python versions. + + * Added automatic handling of installation conflicts. Eggs are now shifted to + the front of sys.path, in an order consistent with where they came from, + making EasyInstall seamlessly co-operate with system package managers. + + The ``--delete-conflicting`` and ``--ignore-conflicts-at-my-risk`` options + are now no longer necessary, and will generate warnings at the end of a + run if you use them. + + * Don't recursively traverse subdirectories given to ``--find-links``. + 0.6a10 ------ * Fixed the ``develop`` command ignoring ``--find-links``. + * Added exhaustive testing of the install directory, including a spawn test + for ``.pth`` file support, and directory writability/existence checks. This + should virtually eliminate the need to set or configure ``--site-dirs``. + + * Added ``--prefix`` option for more do-what-I-mean-ishness in the absence of + RTFM-ing. :) + + * Enhanced ``PYTHONPATH`` support so that you don't have to put any eggs on it + manually to make it work. ``--multi-version`` is no longer a silent + default; you must explicitly use it if installing to a non-PYTHONPATH, + non-"site" directory. + + * Expand ``$variables`` used in the ``--site-dirs``, ``--build-directory``, + ``--install-dir``, and ``--script-dir`` options, whether on the command line + or in configuration files. + + * Improved SourceForge mirror processing to work faster and be less affected + by transient HTML changes made by SourceForge. + + * PyPI searches now use the exact spelling of requirements specified on the + command line or in a project's ``install_requires``. Previously, a + normalized form of the name was used, which could lead to unnecessary + full-index searches when a project's name had an underscore (``_``) in it. + + * EasyInstall can now download bare ``.py`` files and wrap them in an egg, + as long as you include an ``#egg=name-version`` suffix on the URL, or if + the ``.py`` file is listed as the "Download URL" on the project's PyPI page. + This allows third parties to "package" trivial Python modules just by + linking to them (e.g. from within their own PyPI page or download links + page). + + * The ``--always-copy`` option now skips "system" and "development" eggs since + they can't be reliably copied. Note that this may cause EasyInstall to + choose an older version of a package than what you expected, or it may cause + downloading and installation of a fresh version of what's already installed. + + * The ``--find-links`` option previously scanned all supplied URLs and + directories as early as possible, but now only directories and direct + archive links are scanned immediately. URLs are not retrieved unless a + package search was already going to go online due to a package not being + available locally, or due to the use of the ``--update`` or ``-U`` option. + + * Fixed the annoying ``--help-commands`` wart. + 0.6a9 ----- @@ -3205,6 +3731,22 @@ easy_install and entry-point wrapper scripts), and ``easy_install`` can turn the .exe back into an ``.egg`` file or directory and install it as such. + * Fixed ``.pth`` file processing picking up nested eggs (i.e. ones inside + "baskets") when they weren't explicitly listed in the ``.pth`` file. + + * If more than one URL appears to describe the exact same distribution, prefer + the shortest one. This helps to avoid "table of contents" CGI URLs like the + ones on effbot.org. + + * Quote arguments to python.exe (including python's path) to avoid problems + when Python (or a script) is installed in a directory whose name contains + spaces on Windows. + + * Support full roundtrip translation of eggs to and from ``bdist_wininst`` + format. Running ``bdist_wininst`` on a setuptools-based package wraps the + egg in an .exe that will safely install it as an egg (i.e., with metadata + and entry-point wrapper scripts), and ``easy_install`` can turn the .exe + back into an ``.egg`` file or directory and install it as such. 0.6a8 ----- @@ -3233,6 +3775,45 @@ easy_install metadata cache to pretend that the egg has valid version information, until it has a chance to make it actually be so (via the ``egg_info`` command). + * Update for changed SourceForge mirror format + + * Fixed not installing dependencies for some packages fetched via Subversion + + * Fixed dependency installation with ``--always-copy`` not using the same + dependency resolution procedure as other operations. + + * Fixed not fully removing temporary directories on Windows, if a Subversion + checkout left read-only files behind + + * Fixed some problems building extensions when Pyrex was installed, especially + with Python 2.4 and/or packages using SWIG. + +0.6a7 +----- + + * Fixed not being able to install Windows script wrappers using Python 2.3 + +0.6a6 +----- + + * Added support for "traditional" PYTHONPATH-based non-root installation, and + also the convenient ``virtual-python.py`` script, based on a contribution + by Ian Bicking. The setuptools egg now contains a hacked ``site`` module + that makes the PYTHONPATH-based approach work with .pth files, so that you + can get the full EasyInstall feature set on such installations. + + * Added ``--no-deps`` and ``--allow-hosts`` options. + + * Improved Windows ``.exe`` script wrappers so that the script can have the + same name as a module without confusing Python. + + * Changed dependency processing so that it's breadth-first, allowing a + depender's preferences to override those of a dependee, to prevent conflicts + when a lower version is acceptable to the dependee, but not the depender. + Also, ensure that currently installed/selected packages aren't given + precedence over ones desired by a package being installed, which could + cause conflict errors. + 0.6a5 ----- @@ -3245,6 +3826,17 @@ easy_install on Windows and other platforms. (The special handling is only for Windows; other platforms are treated the same as for ``console_scripts``.) + * Improved error message when trying to use old ways of running + ``easy_install``. Removed the ability to run via ``python -m`` or by + running ``easy_install.py``; ``easy_install`` is the command to run on all + supported platforms. + + * Improved wrapper script generation and runtime initialization so that a + VersionConflict doesn't occur if you later install a competing version of a + needed package as the default version of that package. + + * Fixed a problem parsing version numbers in ``#egg=`` links. + 0.6a2 ----- @@ -3253,6 +3845,15 @@ easy_install scripts get an ``.exe`` wrapper so you can just type their name. On other platforms, the scripts are written without a file extension. + * EasyInstall can now install "console_scripts" defined by packages that use + ``setuptools`` and define appropriate entry points. On Windows, console + scripts get an ``.exe`` wrapper so you can just type their name. On other + platforms, the scripts are installed without a file extension. + + * Using ``python -m easy_install`` or running ``easy_install.py`` is now + DEPRECATED, since an ``easy_install`` wrapper is now available on all + platforms. + 0.6a1 ----- @@ -3298,6 +3899,55 @@ easy_install or documented, and never would have worked without EasyInstall - which it pre-dated and was never compatible with. + * EasyInstall now does MD5 validation of downloads from PyPI, or from any link + that has an "#md5=..." trailer with a 32-digit lowercase hex md5 digest. + + * EasyInstall now handles symlinks in target directories by removing the link, + rather than attempting to overwrite the link's destination. This makes it + easier to set up an alternate Python "home" directory (as described in + the Non-Root Installation section of the docs). + + * Added support for handling MacOS platform information in ``.egg`` filenames, + based on a contribution by Kevin Dangoor. You may wish to delete and + reinstall any eggs whose filename includes "darwin" and "Power_Macintosh", + because the format for this platform information has changed so that minor + OS X upgrades (such as 10.4.1 to 10.4.2) do not cause eggs built with a + previous OS version to become obsolete. + + * easy_install's dependency processing algorithms have changed. When using + ``--always-copy``, it now ensures that dependencies are copied too. When + not using ``--always-copy``, it tries to use a single resolution loop, + rather than recursing. + + * Fixed installing extra ``.pyc`` or ``.pyo`` files for scripts with ``.py`` + extensions. + + * Added ``--site-dirs`` option to allow adding custom "site" directories. + Made ``easy-install.pth`` work in platform-specific alternate site + directories (e.g. ``~/Library/Python/2.x/site-packages`` on Mac OS X). + + * If you manually delete the current version of a package, the next run of + EasyInstall against the target directory will now remove the stray entry + from the ``easy-install.pth`` file. + + * EasyInstall now recognizes URLs with a ``#egg=project_name`` fragment ID + as pointing to the named project's source checkout. Such URLs have a lower + match precedence than any other kind of distribution, so they'll only be + used if they have a higher version number than any other available + distribution, or if you use the ``--editable`` option. The ``#egg`` + fragment can contain a version if it's formatted as ``#egg=proj-ver``, + where ``proj`` is the project name, and ``ver`` is the version number. You + *must* use the format for these values that the ``bdist_egg`` command uses; + i.e., all non-alphanumeric runs must be condensed to single underscore + characters. + + * Added the ``--editable`` option; see Editing and Viewing Source Packages + in the docs. Also, slightly changed the behavior of the + ``--build-directory`` option. + + * Fixed the setup script sandbox facility not recognizing certain paths as + valid on case-insensitive platforms. + 0.5a12 ------ @@ -3305,12 +3955,28 @@ easy_install ``python -m``, and marks them as unsafe for zipping, since Python 2.4 can't handle ``-m`` on zipped modules. + * Fix ``python -m easy_install`` not working due to setuptools being installed + as a zipfile. Update safety scanner to check for modules that might be used + as ``python -m`` scripts. + + * Misc. fixes for win32.exe support, including changes to support Python 2.4's + changed ``bdist_wininst`` format. + 0.5a11 ------ * Fix breakage of the "develop" command that was caused by the addition of ``--always-unzip`` to the ``easy_install`` command. +0.5a10 +------ + + * Put the ``easy_install`` module back in as a module, as it's needed for + ``python -m`` to run it! + + * Allow ``--find-links/-f`` to accept local directories or filenames as well + as URLs. + 0.5a9 ----- @@ -3345,6 +4011,31 @@ easy_install * Fixed the swapped ``-d`` and ``-b`` options of ``bdist_egg``. + * EasyInstall now automatically detects when an "unmanaged" package or + module is going to be on ``sys.path`` ahead of a package you're installing, + thereby preventing the newer version from being imported. By default, it + will abort installation to alert you of the problem, but there are also + new options (``--delete-conflicting`` and ``--ignore-conflicts-at-my-risk``) + available to change the default behavior. (Note: this new feature doesn't + take effect for egg files that were built with older ``setuptools`` + versions, because they lack the new metadata file required to implement it.) + + * The ``easy_install`` distutils command now uses ``DistutilsError`` as its + base error type for errors that should just issue a message to stderr and + exit the program without a traceback. + + * EasyInstall can now be given a path to a directory containing a setup + script, and it will attempt to build and install the package there. + + * EasyInstall now performs a safety analysis on module contents to determine + whether a package is likely to run in zipped form, and displays + information about what modules may be doing introspection that would break + when running as a zipfile. + + * Added the ``--always-unzip/-Z`` option, to force unzipping of packages that + would ordinarily be considered safe to unzip, and changed the meaning of + ``--zip-ok/-z`` to "always leave everything zipped". + 0.5a8 ----- @@ -3372,6 +4063,9 @@ easy_install * Added a "setopt" command that sets a single option in a specified distutils configuration file. + * There is now a separate documentation page for setuptools; revision + history that's not specific to EasyInstall has been moved to that page. + 0.5a7 ----- @@ -3439,6 +4133,39 @@ easy_install * Setup scripts using setuptools now always install using ``easy_install`` internally, for ease of uninstallation and upgrading. + * Added ``--always-copy/-a`` option to always copy needed packages to the + installation directory, even if they're already present elsewhere on + sys.path. (In previous versions, this was the default behavior, but now + you must request it.) + + * Added ``--upgrade/-U`` option to force checking PyPI for latest available + version(s) of all packages requested by name and version, even if a matching + version is available locally. + + * Added automatic installation of dependencies declared by a distribution + being installed. These dependencies must be listed in the distribution's + ``EGG-INFO`` directory, so the distribution has to have declared its + dependencies by using setuptools. If a package has requirements it didn't + declare, you'll still have to deal with them yourself. (E.g., by asking + EasyInstall to find and install them.) + + * Added the ``--record`` option to ``easy_install`` for the benefit of tools + that run ``setup.py install --record=filename`` on behalf of another + packaging system.) + +0.5a3 +----- + + * Fixed not setting script permissions to allow execution. + + * Improved sandboxing so that setup scripts that want a temporary directory + (e.g. pychecker) can still run in the sandbox. + +0.5a2 +----- + + * Fix stupid stupid refactoring-at-the-last-minute typos. :( + 0.5a1 ----- @@ -3453,6 +4180,29 @@ easy_install from setuptools import setup # etc... + * Added support for converting ``.win32.exe`` installers to eggs on the fly. + EasyInstall will now recognize such files by name and install them. + + * Fixed a problem with picking the "best" version to install (versions were + being sorted as strings, rather than as parsed values) + +0.4a4 +----- + + * Added support for the distutils "verbose/quiet" and "dry-run" options, as + well as the "optimize" flag. + + * Support downloading packages that were uploaded to PyPI (by scanning all + links on package pages, not just the homepage/download links). + +0.4a3 +----- + + * Add progress messages to the search/download process so that you can tell + what URLs it's reading to find download links. (Hopefully, this will help + people report out-of-date and broken links to package authors, and to tell + when they've asked for a package that doesn't exist.) + 0.4a2 ----- @@ -3480,6 +4230,44 @@ easy_install their ``command_consumes_arguments`` attribute to ``True`` in order to receive an ``args`` option containing the rest of the command line. + * Added support for installing scripts + + * Added support for setting options via distutils configuration files, and + using distutils' default options as a basis for EasyInstall's defaults. + + * Renamed ``--scan-url/-s`` to ``--find-links/-f`` to free up ``-s`` for the + script installation directory option. + + * Use ``urllib2`` instead of ``urllib``, to allow use of ``https:`` URLs if + Python includes SSL support. + +0.4a1 +----- + + * Added ``--scan-url`` and ``--index-url`` options, to scan download pages + and search PyPI for needed packages. + +0.3a4 +----- + + * Restrict ``--build-directory=DIR/-b DIR`` option to only be used with single + URL installs, to avoid running the wrong setup.py. + +0.3a3 +----- + + * Added ``--build-directory=DIR/-b DIR`` option. + + * Added "installation report" that explains how to use 'require()' when doing + a multiversion install or alternate installation directory. + + * Added SourceForge mirror auto-select (Contributed by Ian Bicking) + + * Added "sandboxing" that stops a setup script from running if it attempts to + write to the filesystem outside of the build area + + * Added more workarounds for packages with quirky ``install_data`` hacks + 0.3a2 ----- @@ -3487,6 +4275,9 @@ easy_install with a subversion revision number, the current date, or an explicit tag value. Run ``setup.py bdist_egg --help`` to get more information. + * Added subversion download support for ``svn:`` and ``svn+`` URLs, as well as + automatic recognition of HTTP subversion URLs (Contributed by Ian Bicking) + * Misc. bug fixes 0.3a1 diff --git a/MANIFEST.in b/MANIFEST.in index 325bbed8..9cce3c90 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,7 @@ recursive-include setuptools *.py *.exe *.xml recursive-include tests *.py recursive-include setuptools/tests *.html recursive-include docs *.py *.txt *.conf *.css *.css_t Makefile indexsidebar.html -recursive-include setuptools/_vendor * +recursive-include setuptools/_vendor *.py *.txt recursive-include pkg_resources *.py *.txt include *.py include *.rst diff --git a/README.rst b/README.rst index 1c6ee603..0454f2ed 100755..100644 --- a/README.rst +++ b/README.rst @@ -5,20 +5,24 @@ :target: https://setuptools.readthedocs.io .. image:: https://img.shields.io/travis/pypa/setuptools/master.svg?label=Linux%20build%20%40%20Travis%20CI - :target: http://travis-ci.org/pypa/setuptools + :target: https://travis-ci.org/pypa/setuptools -.. image:: https://img.shields.io/appveyor/ci/jaraco/setuptools/master.svg?label=Windows%20build%20%40%20Appveyor - :target: https://ci.appveyor.com/project/jaraco/setuptools/branch/master +.. image:: https://img.shields.io/appveyor/ci/pypa/setuptools/master.svg?label=Windows%20build%20%40%20Appveyor + :target: https://ci.appveyor.com/project/pypa/setuptools/branch/master + +.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg + :target: https://codecov.io/gh/pypa/setuptools .. image:: https://img.shields.io/pypi/pyversions/setuptools.svg +.. image:: https://tidelift.com/badges/github/pypa/setuptools + :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme + See the `Installation Instructions <https://packaging.python.org/installing/>`_ in the Python Packaging User's Guide for instructions on installing, upgrading, and uninstalling Setuptools. -The project is `maintained at GitHub <https://github.com/pypa/setuptools>`_. - Questions and comments should be directed to the `distutils-sig mailing list <http://mail.python.org/pipermail/distutils-sig/>`_. Bug reports and especially tested patches may be diff --git a/appveyor.yml b/appveyor.yml index 9313a482..ef4a9f7e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,10 +1,15 @@ +clone_depth: 50 + environment: - APPVEYOR: true + APPVEYOR: True + CODECOV_ENV: APPVEYOR_JOB_NAME matrix: - - PYTHON: "C:\\Python35-x64" - - PYTHON: "C:\\Python27-x64" + - APPVEYOR_JOB_NAME: "python36-x64" + PYTHON: "C:\\Python36-x64" + - APPVEYOR_JOB_NAME: "python27-x64" + PYTHON: "C:\\Python27-x64" install: # symlink python from a directory with a space @@ -14,7 +19,18 @@ install: build: off +cache: + - '%LOCALAPPDATA%\pip\Cache' + test_script: - - "python bootstrap.py" - - "python -m pip install tox" - - "tox" + - python --version + - python -m pip install --disable-pip-version-check --upgrade pip setuptools wheel + - pip install --upgrade tox tox-venv + - pip freeze --all + - python bootstrap.py + - tox -- --cov + +after_test: + - tox -e coverage,codecov + +version: '{build}' diff --git a/changelog.d/.gitignore b/changelog.d/.gitignore new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/changelog.d/.gitignore diff --git a/changelog.d/1499.change.rst b/changelog.d/1499.change.rst new file mode 100644 index 00000000..10e4db68 --- /dev/null +++ b/changelog.d/1499.change.rst @@ -0,0 +1 @@ +``setuptools.package_index`` no longer relies on the deprecated ``urllib.parse.splituser`` per Python #27485. diff --git a/changelog.d/1544.change.rst b/changelog.d/1544.change.rst new file mode 100644 index 00000000..748b64e1 --- /dev/null +++ b/changelog.d/1544.change.rst @@ -0,0 +1 @@ +Added tests for PackageIndex.download (for git URLs). diff --git a/changelog.d/1551.breaking.rst b/changelog.d/1551.breaking.rst new file mode 100644 index 00000000..c0e477ce --- /dev/null +++ b/changelog.d/1551.breaking.rst @@ -0,0 +1 @@ +File inputs for the `license` field in `setup.cfg` files now explicitly raise an error. diff --git a/changelog.d/1625.change.rst b/changelog.d/1625.change.rst new file mode 100644 index 00000000..9125ac0c --- /dev/null +++ b/changelog.d/1625.change.rst @@ -0,0 +1 @@ +In PEP 517 build_meta builder, ensure that sdists are built as gztar per the spec. diff --git a/conftest.py b/conftest.py index 3cccfe1a..0d7b274c 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,6 @@ +import sys + + pytest_plugins = 'setuptools.tests.fixtures' @@ -6,3 +9,17 @@ def pytest_addoption(parser): "--package_name", action="append", default=[], help="list of package_name to pass to test functions", ) + + +collect_ignore = [ + 'tests/manual_test.py', + 'setuptools/tests/mod_with_constant.py', +] + + +if sys.version_info < (3,): + collect_ignore.append('setuptools/lib2to3_ex.py') + + +if sys.version_info < (3, 6): + collect_ignore.append('pavement.py') diff --git a/docs/_templates/indexsidebar.html b/docs/_templates/indexsidebar.html index 3b127602..b8c6148e 100644 --- a/docs/_templates/indexsidebar.html +++ b/docs/_templates/indexsidebar.html @@ -1,8 +1,15 @@ <h3>Download</h3> <p>Current version: <b>{{ version }}</b></p> -<p>Get Setuptools from the <a href="https://pypi.python.org/pypi/setuptools"> Python Package Index</a> +<p>Get Setuptools from the <a href="https://pypi.org/project/setuptools/"> Python Package Index</a> <h3>Questions? Suggestions? Contributions?</h3> <p>Visit the <a href="https://github.com/pypa/setuptools">Setuptools project page</a> </p> + +<h3 class="donation">Professional support</h3> + +<p> +Professionally-supported {{ project }} is available with the +<a href="https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme">Tidelift Subscription</a>. +</p> diff --git a/docs/conf.py b/docs/conf.py index f7d02303..c7eb6d3f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,7 +34,7 @@ import os # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['jaraco.packaging.sphinx', 'rst.linker', 'sphinx.ext.autosectionlabel'] +extensions = ['jaraco.packaging.sphinx', 'rst.linker'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -45,6 +45,9 @@ source_suffix = '.txt' # The master toctree document. master_doc = 'index' +# A list of glob-style patterns that should be excluded when looking for source files. +exclude_patterns = ['requirements.txt'] + # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = [] diff --git a/docs/developer-guide.txt b/docs/developer-guide.txt index 8a136380..a5942c8b 100644 --- a/docs/developer-guide.txt +++ b/docs/developer-guide.txt @@ -57,40 +57,59 @@ Setuptools makes extensive use of hyperlinks to tickets in the changelog so that system integrators and other users can get a quick summary, but then jump to the in-depth discussion about any subject referenced. ------------ -Source Code ------------ +--------------------- +Making a pull request +--------------------- + +When making a pull request, please include a short summary of the changes +and a reference to any issue tickets that the PR is intended to solve. +All PRs with code changes should include tests. All changes should include a +changelog entry. + +``setuptools`` uses `towncrier <https://pypi.org/project/towncrier/>`_ +for changelog management, so when making a PR, please add a news fragment in the +``changelog.d/`` folder. Changelog files are written in reStructuredText and +should be a 1 or 2 sentence description of the substantive changes in the PR. +They should be named ``<pr_number>.<category>.rst``, where the categories are: + +- ``change``: Any backwards compatible code change +- ``breaking``: Any backwards-compatibility breaking change +- ``doc``: A change to the documentation +- ``misc``: Changes internal to the repo like CI, test and build changes +- ``deprecation``: For deprecations of an existing feature or behavior + +A pull request may have more than one of these components, for example a code +change may introduce a new feature that deprecates an old feature, in which +case two fragments should be added. It is not necessary to make a separate +documentation fragment for documentation changes accompanying the relevant +code changes. See the following for an example news fragment: + +.. code-block:: bash + + $ cat changelog.d/1288.change.rst + Add support for maintainer in PKG-INFO -Grab the code at Github:: - - $ git checkout https://github.com/pypa/setuptools - -If you want to contribute changes, we recommend you fork the repository on -Github, commit the changes to your repository, and then make a pull request -on Github. If you make some changes, don't forget to: - -- add a note in CHANGES.rst - -Please commit all changes in the 'master' branch against the latest available -commit or for bug-fixes, against an earlier commit or release in which the -bug occurred. - -If you find yourself working on more than one issue at a time, Setuptools -generally prefers Git-style branches, so use Mercurial bookmarks or Git -branches or multiple forks to maintain separate efforts. +------------------- +Auto-Merge Requests +------------------- -The Continuous Integration tests that validate every release are run -from this repository. +To support running all code through CI, even lightweight contributions, +the project employs Mergify to auto-merge pull requests tagged as +auto-merge. -For posterity, the old `Bitbucket mirror -<https://bitbucket.org/pypa/setuptools>`_ is available. +Use ``hub pull-request -l auto-merge`` to create such a pull request +from the command line after pushing a new branch. ------- Testing ------- -The primary tests are run using tox. To run the tests, first make -sure you have tox installed, then invoke it:: +The primary tests are run using tox. To run the tests, first create the metadata +needed to run the tests:: + + $ python bootstrap.py + +Then make sure you have tox installed, and invoke it:: $ tox @@ -109,10 +128,12 @@ Setuptools follows ``semver``. Building Documentation ---------------------- -Setuptools relies on the Sphinx system for building documentation. -To accommodate RTD, docs must be built from the docs/ directory. +Setuptools relies on the `Sphinx`_ system for building documentation. +The `published documentation`_ is hosted on Read the Docs. + +To build the docs locally, use tox:: -To build them, you need to have installed the requirements specified -in docs/requirements.txt. One way to do this is to use rwt: + $ tox -e docs - setuptools/docs$ python -m rwt -r requirements.txt -- -m sphinx . html +.. _Sphinx: http://www.sphinx-doc.org/en/master/ +.. _published documentation: https://setuptools.readthedocs.io/en/latest/ diff --git a/docs/easy_install.txt b/docs/easy_install.txt index bd9f0e86..aa11f890 100644 --- a/docs/easy_install.txt +++ b/docs/easy_install.txt @@ -2,6 +2,12 @@ Easy Install ============ +.. warning:: + Easy Install is deprecated. Do not use it. Instead use pip. If + you think you need Easy Install, please reach out to the PyPA + team (a ticket to pip or setuptools is fine), describing your + use-case. + Easy Install is a python module (``easy_install``) bundled with ``setuptools`` that lets you automatically download, build, install, and manage Python packages. @@ -17,7 +23,7 @@ bug -- and then do so via list discussion first.) (Also, if you'd like to learn about how you can use ``setuptools`` to make your own packages work better with EasyInstall, or provide EasyInstall-like features without requiring your users to use EasyInstall directly, you'll probably want -to check out the full `setuptools`_ documentation as well.) +to check out the full documentation as well.) .. contents:: **Table of Contents** @@ -31,11 +37,11 @@ Using "Easy Install" Installing "Easy Install" ------------------------- -Please see the `setuptools PyPI page <https://pypi.python.org/pypi/setuptools>`_ +Please see the `setuptools PyPI page <https://pypi.org/project/setuptools/>`_ for download links and basic installation instructions for each of the supported platforms. -You will need at least Python 2.6. An ``easy_install`` script will be +You will need at least Python 3.4 or 2.7. An ``easy_install`` script will be installed in the normal location for Python scripts on your platform. Note that the instructions on the setuptools PyPI page assume that you are @@ -305,8 +311,7 @@ Regardless of the technique used, the script(s) will be installed to a Scripts directory (by default in the Python installation directory). It is recommended for EasyInstall that you ensure this directory is in the PATH environment variable. The easiest way to ensure the Scripts directory is in the PATH is -to run ``Tools\Scripts\win_add2path.py`` from the Python directory (requires -Python 2.6 or later). +to run ``Tools\Scripts\win_add2path.py`` from the Python directory. Note that instead of changing your ``PATH`` to include the Python scripts directory, you can also retarget the installation location for scripts so they @@ -832,7 +837,7 @@ Command-Line Options ``--index-url=URL, -i URL`` (New in 0.4a1; default changed in 0.6c7) Specifies the base URL of the Python Package Index. The default is - https://pypi.python.org/simple if not specified. When a package is requested + https://pypi.org/simple/ if not specified. When a package is requested that is not locally available or linked from a ``--find-links`` download page, the package index will be searched for download pages for the needed package, and those download pages will be searched for links to download @@ -987,21 +992,20 @@ The following section lists only the easiest and most relevant approaches [1]_. `Use "virtualenv"`_ -.. [1] There are older ways to achieve custom installation using various ``easy_install`` and ``setup.py install`` options, combined with ``PYTHONPATH`` and/or ``PYTHONUSERBASE`` alterations, but all of these are effectively deprecated by the User scheme brought in by `PEP-370`_ in Python 2.6. +.. [1] There are older ways to achieve custom installation using various ``easy_install`` and ``setup.py install`` options, combined with ``PYTHONPATH`` and/or ``PYTHONUSERBASE`` alterations, but all of these are effectively deprecated by the User scheme brought in by `PEP-370`_. .. _PEP-370: http://www.python.org/dev/peps/pep-0370/ Use the "--user" option ~~~~~~~~~~~~~~~~~~~~~~~ -With Python 2.6 came the User scheme for installation, which means that all -python distributions support an alternative install location that is specific to a user [2]_ [3]_. +Python provides a User scheme for installation, which means that all +python distributions support an alternative install location that is specific to a user [3]_. The Default location for each OS is explained in the python documentation for the ``site.USER_BASE`` variable. This mode of installation can be turned on by specifying the ``--user`` option to ``setup.py install`` or ``easy_install``. This approach serves the need to have a user-specific stash of packages. -.. [2] Prior to Python2.6, Mac OS X offered a form of the User scheme. That is now subsumed into the User scheme introduced in Python 2.6. .. [3] Prior to the User scheme, there was the Home scheme, which is still available, but requires more effort than the User scheme to get packages recognized. Use the "--user" option and customize "PYTHONUSERBASE" @@ -1021,7 +1025,7 @@ that the User installation scheme alone does not provide, e.g. the ability to hi Please refer to the `virtualenv`_ documentation for more details. -.. _virtualenv: https://pypi.python.org/pypi/virtualenv +.. _virtualenv: https://pypi.org/project/virtualenv/ @@ -1079,547 +1083,3 @@ EasyInstall to be able to look up and download packages: 8. If a package index is accessed via a ``file://`` URL, then EasyInstall will automatically use ``index.html`` files, if present, when trying to read a directory with a trailing ``/`` on the URL. - - -Backward Compatibility -~~~~~~~~~~~~~~~~~~~~~~ - -Package indexes that wish to support setuptools versions prior to 0.6b4 should -also follow these rules: - -* Homepage and download links must be preceded with ``"<th>Home Page"`` or - ``"<th>Download URL"``, in addition to (or instead of) the ``rel=""`` - attributes on the actual links. These marker strings do not need to be - visible, or uncommented, however! For example, the following is a valid - homepage link that will work with any version of setuptools:: - - <li> - <strong>Home Page:</strong> - <!-- <th>Home Page --> - <a rel="homepage" href="http://sqlobject.org">http://sqlobject.org</a> - </li> - - Even though the marker string is in an HTML comment, older versions of - EasyInstall will still "see" it and know that the link that follows is the - project's home page URL. - -* The pages described by paragraph 3(b) of the preceding section *must* - contain the string ``"Index of Packages</title>"`` somewhere in their text. - This can be inside of an HTML comment, if desired, and it can be anywhere - in the page. (Note: this string MUST NOT appear on normal project pages, as - described in paragraphs 2 and 3(a)!) - -In addition, for compatibility with PyPI versions that do not use ``#md5=`` -fragment IDs, EasyInstall uses the following regular expression to match PyPI's -displayed MD5 info (broken onto two lines for readability):: - - <a href="([^"#]+)">([^<]+)</a>\n\s+\(<a href="[^?]+\?:action=show_md5 - &digest=([0-9a-f]{32})">md5</a>\) - -History -======= - -0.6c9 - * Fixed ``win32.exe`` support for .pth files, so unnecessary directory nesting - is flattened out in the resulting egg. (There was a case-sensitivity - problem that affected some distributions, notably ``pywin32``.) - - * Prevent ``--help-commands`` and other junk from showing under Python 2.5 - when running ``easy_install --help``. - - * Fixed GUI scripts sometimes not executing on Windows - - * Fixed not picking up dependency links from recursive dependencies. - - * Only make ``.py``, ``.dll`` and ``.so`` files executable when unpacking eggs - - * Changes for Jython compatibility - - * Improved error message when a requirement is also a directory name, but the - specified directory is not a source package. - - * Fixed ``--allow-hosts`` option blocking ``file:`` URLs - - * Fixed HTTP SVN detection failing when the page title included a project - name (e.g. on SourceForge-hosted SVN) - - * Fix Jython script installation to handle ``#!`` lines better when - ``sys.executable`` is a script. - - * Removed use of deprecated ``md5`` module if ``hashlib`` is available - - * Keep site directories (e.g. ``site-packages``) from being included in - ``.pth`` files. - -0.6c7 - * ``ftp:`` download URLs now work correctly. - - * The default ``--index-url`` is now ``https://pypi.python.org/simple``, to use - the Python Package Index's new simpler (and faster!) REST API. - -0.6c6 - * EasyInstall no longer aborts the installation process if a URL it wants to - retrieve can't be downloaded, unless the URL is an actual package download. - Instead, it issues a warning and tries to keep going. - - * Fixed distutils-style scripts originally built on Windows having their line - endings doubled when installed on any platform. - - * Added ``--local-snapshots-ok`` flag, to allow building eggs from projects - installed using ``setup.py develop``. - - * Fixed not HTML-decoding URLs scraped from web pages - -0.6c5 - * Fixed ``.dll`` files on Cygwin not having executable permissions when an egg - is installed unzipped. - -0.6c4 - * Added support for HTTP "Basic" authentication using ``http://user:pass@host`` - URLs. If a password-protected page contains links to the same host (and - protocol), those links will inherit the credentials used to access the - original page. - - * Removed all special support for Sourceforge mirrors, as Sourceforge's - mirror system now works well for non-browser downloads. - - * Fixed not recognizing ``win32.exe`` installers that included a custom - bitmap. - - * Fixed not allowing ``os.open()`` of paths outside the sandbox, even if they - are opened read-only (e.g. reading ``/dev/urandom`` for random numbers, as - is done by ``os.urandom()`` on some platforms). - - * Fixed a problem with ``.pth`` testing on Windows when ``sys.executable`` - has a space in it (e.g., the user installed Python to a ``Program Files`` - directory). - -0.6c3 - * You can once again use "python -m easy_install" with Python 2.4 and above. - - * Python 2.5 compatibility fixes added. - -0.6c2 - * Windows script wrappers now support quoted arguments and arguments - containing spaces. (Patch contributed by Jim Fulton.) - - * The ``ez_setup.py`` script now actually works when you put a setuptools - ``.egg`` alongside it for bootstrapping an offline machine. - - * A writable installation directory on ``sys.path`` is no longer required to - download and extract a source distribution using ``--editable``. - - * Generated scripts now use ``-x`` on the ``#!`` line when ``sys.executable`` - contains non-ASCII characters, to prevent deprecation warnings about an - unspecified encoding when the script is run. - -0.6c1 - * EasyInstall now includes setuptools version information in the - ``User-Agent`` string sent to websites it visits. - -0.6b4 - * Fix creating Python wrappers for non-Python scripts - - * Fix ``ftp://`` directory listing URLs from causing a crash when used in the - "Home page" or "Download URL" slots on PyPI. - - * Fix ``sys.path_importer_cache`` not being updated when an existing zipfile - or directory is deleted/overwritten. - - * Fix not recognizing HTML 404 pages from package indexes. - - * Allow ``file://`` URLs to be used as a package index. URLs that refer to - directories will use an internally-generated directory listing if there is - no ``index.html`` file in the directory. - - * Allow external links in a package index to be specified using - ``rel="homepage"`` or ``rel="download"``, without needing the old - PyPI-specific visible markup. - - * Suppressed warning message about possibly-misspelled project name, if an egg - or link for that project name has already been seen. - -0.6b3 - * Fix local ``--find-links`` eggs not being copied except with - ``--always-copy``. - - * Fix sometimes not detecting local packages installed outside of "site" - directories. - - * Fix mysterious errors during initial ``setuptools`` install, caused by - ``ez_setup`` trying to run ``easy_install`` twice, due to a code fallthru - after deleting the egg from which it's running. - -0.6b2 - * Don't install or update a ``site.py`` patch when installing to a - ``PYTHONPATH`` directory with ``--multi-version``, unless an - ``easy-install.pth`` file is already in use there. - - * Construct ``.pth`` file paths in such a way that installing an egg whose - name begins with ``import`` doesn't cause a syntax error. - - * Fixed a bogus warning message that wasn't updated since the 0.5 versions. - -0.6b1 - * Better ambiguity management: accept ``#egg`` name/version even if processing - what appears to be a correctly-named distutils file, and ignore ``.egg`` - files with no ``-``, since valid Python ``.egg`` files always have a version - number (but Scheme eggs often don't). - - * Support ``file://`` links to directories in ``--find-links``, so that - easy_install can build packages from local source checkouts. - - * Added automatic retry for Sourceforge mirrors. The new download process is - to first just try dl.sourceforge.net, then randomly select mirror IPs and - remove ones that fail, until something works. The removed IPs stay removed - for the remainder of the run. - - * Ignore bdist_dumb distributions when looking at download URLs. - -0.6a11 - * Process ``dependency_links.txt`` if found in a distribution, by adding the - URLs to the list for scanning. - - * Use relative paths in ``.pth`` files when eggs are being installed to the - same directory as the ``.pth`` file. This maximizes portability of the - target directory when building applications that contain eggs. - - * Added ``easy_install-N.N`` script(s) for convenience when using multiple - Python versions. - - * Added automatic handling of installation conflicts. Eggs are now shifted to - the front of sys.path, in an order consistent with where they came from, - making EasyInstall seamlessly co-operate with system package managers. - - The ``--delete-conflicting`` and ``--ignore-conflicts-at-my-risk`` options - are now no longer necessary, and will generate warnings at the end of a - run if you use them. - - * Don't recursively traverse subdirectories given to ``--find-links``. - -0.6a10 - * Added exhaustive testing of the install directory, including a spawn test - for ``.pth`` file support, and directory writability/existence checks. This - should virtually eliminate the need to set or configure ``--site-dirs``. - - * Added ``--prefix`` option for more do-what-I-mean-ishness in the absence of - RTFM-ing. :) - - * Enhanced ``PYTHONPATH`` support so that you don't have to put any eggs on it - manually to make it work. ``--multi-version`` is no longer a silent - default; you must explicitly use it if installing to a non-PYTHONPATH, - non-"site" directory. - - * Expand ``$variables`` used in the ``--site-dirs``, ``--build-directory``, - ``--install-dir``, and ``--script-dir`` options, whether on the command line - or in configuration files. - - * Improved SourceForge mirror processing to work faster and be less affected - by transient HTML changes made by SourceForge. - - * PyPI searches now use the exact spelling of requirements specified on the - command line or in a project's ``install_requires``. Previously, a - normalized form of the name was used, which could lead to unnecessary - full-index searches when a project's name had an underscore (``_``) in it. - - * EasyInstall can now download bare ``.py`` files and wrap them in an egg, - as long as you include an ``#egg=name-version`` suffix on the URL, or if - the ``.py`` file is listed as the "Download URL" on the project's PyPI page. - This allows third parties to "package" trivial Python modules just by - linking to them (e.g. from within their own PyPI page or download links - page). - - * The ``--always-copy`` option now skips "system" and "development" eggs since - they can't be reliably copied. Note that this may cause EasyInstall to - choose an older version of a package than what you expected, or it may cause - downloading and installation of a fresh version of what's already installed. - - * The ``--find-links`` option previously scanned all supplied URLs and - directories as early as possible, but now only directories and direct - archive links are scanned immediately. URLs are not retrieved unless a - package search was already going to go online due to a package not being - available locally, or due to the use of the ``--update`` or ``-U`` option. - - * Fixed the annoying ``--help-commands`` wart. - -0.6a9 - * Fixed ``.pth`` file processing picking up nested eggs (i.e. ones inside - "baskets") when they weren't explicitly listed in the ``.pth`` file. - - * If more than one URL appears to describe the exact same distribution, prefer - the shortest one. This helps to avoid "table of contents" CGI URLs like the - ones on effbot.org. - - * Quote arguments to python.exe (including python's path) to avoid problems - when Python (or a script) is installed in a directory whose name contains - spaces on Windows. - - * Support full roundtrip translation of eggs to and from ``bdist_wininst`` - format. Running ``bdist_wininst`` on a setuptools-based package wraps the - egg in an .exe that will safely install it as an egg (i.e., with metadata - and entry-point wrapper scripts), and ``easy_install`` can turn the .exe - back into an ``.egg`` file or directory and install it as such. - -0.6a8 - * Update for changed SourceForge mirror format - - * Fixed not installing dependencies for some packages fetched via Subversion - - * Fixed dependency installation with ``--always-copy`` not using the same - dependency resolution procedure as other operations. - - * Fixed not fully removing temporary directories on Windows, if a Subversion - checkout left read-only files behind - - * Fixed some problems building extensions when Pyrex was installed, especially - with Python 2.4 and/or packages using SWIG. - -0.6a7 - * Fixed not being able to install Windows script wrappers using Python 2.3 - -0.6a6 - * Added support for "traditional" PYTHONPATH-based non-root installation, and - also the convenient ``virtual-python.py`` script, based on a contribution - by Ian Bicking. The setuptools egg now contains a hacked ``site`` module - that makes the PYTHONPATH-based approach work with .pth files, so that you - can get the full EasyInstall feature set on such installations. - - * Added ``--no-deps`` and ``--allow-hosts`` options. - - * Improved Windows ``.exe`` script wrappers so that the script can have the - same name as a module without confusing Python. - - * Changed dependency processing so that it's breadth-first, allowing a - depender's preferences to override those of a dependee, to prevent conflicts - when a lower version is acceptable to the dependee, but not the depender. - Also, ensure that currently installed/selected packages aren't given - precedence over ones desired by a package being installed, which could - cause conflict errors. - -0.6a3 - * Improved error message when trying to use old ways of running - ``easy_install``. Removed the ability to run via ``python -m`` or by - running ``easy_install.py``; ``easy_install`` is the command to run on all - supported platforms. - - * Improved wrapper script generation and runtime initialization so that a - VersionConflict doesn't occur if you later install a competing version of a - needed package as the default version of that package. - - * Fixed a problem parsing version numbers in ``#egg=`` links. - -0.6a2 - * EasyInstall can now install "console_scripts" defined by packages that use - ``setuptools`` and define appropriate entry points. On Windows, console - scripts get an ``.exe`` wrapper so you can just type their name. On other - platforms, the scripts are installed without a file extension. - - * Using ``python -m easy_install`` or running ``easy_install.py`` is now - DEPRECATED, since an ``easy_install`` wrapper is now available on all - platforms. - -0.6a1 - * EasyInstall now does MD5 validation of downloads from PyPI, or from any link - that has an "#md5=..." trailer with a 32-digit lowercase hex md5 digest. - - * EasyInstall now handles symlinks in target directories by removing the link, - rather than attempting to overwrite the link's destination. This makes it - easier to set up an alternate Python "home" directory (as described above in - the `Non-Root Installation`_ section). - - * Added support for handling MacOS platform information in ``.egg`` filenames, - based on a contribution by Kevin Dangoor. You may wish to delete and - reinstall any eggs whose filename includes "darwin" and "Power_Macintosh", - because the format for this platform information has changed so that minor - OS X upgrades (such as 10.4.1 to 10.4.2) do not cause eggs built with a - previous OS version to become obsolete. - - * easy_install's dependency processing algorithms have changed. When using - ``--always-copy``, it now ensures that dependencies are copied too. When - not using ``--always-copy``, it tries to use a single resolution loop, - rather than recursing. - - * Fixed installing extra ``.pyc`` or ``.pyo`` files for scripts with ``.py`` - extensions. - - * Added ``--site-dirs`` option to allow adding custom "site" directories. - Made ``easy-install.pth`` work in platform-specific alternate site - directories (e.g. ``~/Library/Python/2.x/site-packages`` on Mac OS X). - - * If you manually delete the current version of a package, the next run of - EasyInstall against the target directory will now remove the stray entry - from the ``easy-install.pth`` file. - - * EasyInstall now recognizes URLs with a ``#egg=project_name`` fragment ID - as pointing to the named project's source checkout. Such URLs have a lower - match precedence than any other kind of distribution, so they'll only be - used if they have a higher version number than any other available - distribution, or if you use the ``--editable`` option. The ``#egg`` - fragment can contain a version if it's formatted as ``#egg=proj-ver``, - where ``proj`` is the project name, and ``ver`` is the version number. You - *must* use the format for these values that the ``bdist_egg`` command uses; - i.e., all non-alphanumeric runs must be condensed to single underscore - characters. - - * Added the ``--editable`` option; see `Editing and Viewing Source Packages`_ - above for more info. Also, slightly changed the behavior of the - ``--build-directory`` option. - - * Fixed the setup script sandbox facility not recognizing certain paths as - valid on case-insensitive platforms. - -0.5a12 - * Fix ``python -m easy_install`` not working due to setuptools being installed - as a zipfile. Update safety scanner to check for modules that might be used - as ``python -m`` scripts. - - * Misc. fixes for win32.exe support, including changes to support Python 2.4's - changed ``bdist_wininst`` format. - -0.5a10 - * Put the ``easy_install`` module back in as a module, as it's needed for - ``python -m`` to run it! - - * Allow ``--find-links/-f`` to accept local directories or filenames as well - as URLs. - -0.5a9 - * EasyInstall now automatically detects when an "unmanaged" package or - module is going to be on ``sys.path`` ahead of a package you're installing, - thereby preventing the newer version from being imported. By default, it - will abort installation to alert you of the problem, but there are also - new options (``--delete-conflicting`` and ``--ignore-conflicts-at-my-risk``) - available to change the default behavior. (Note: this new feature doesn't - take effect for egg files that were built with older ``setuptools`` - versions, because they lack the new metadata file required to implement it.) - - * The ``easy_install`` distutils command now uses ``DistutilsError`` as its - base error type for errors that should just issue a message to stderr and - exit the program without a traceback. - - * EasyInstall can now be given a path to a directory containing a setup - script, and it will attempt to build and install the package there. - - * EasyInstall now performs a safety analysis on module contents to determine - whether a package is likely to run in zipped form, and displays - information about what modules may be doing introspection that would break - when running as a zipfile. - - * Added the ``--always-unzip/-Z`` option, to force unzipping of packages that - would ordinarily be considered safe to unzip, and changed the meaning of - ``--zip-ok/-z`` to "always leave everything zipped". - -0.5a8 - * There is now a separate documentation page for `setuptools`_; revision - history that's not specific to EasyInstall has been moved to that page. - - .. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools - -0.5a5 - * Made ``easy_install`` a standard ``setuptools`` command, moving it from - the ``easy_install`` module to ``setuptools.command.easy_install``. Note - that if you were importing or extending it, you must now change your imports - accordingly. ``easy_install.py`` is still installed as a script, but not as - a module. - -0.5a4 - * Added ``--always-copy/-a`` option to always copy needed packages to the - installation directory, even if they're already present elsewhere on - sys.path. (In previous versions, this was the default behavior, but now - you must request it.) - - * Added ``--upgrade/-U`` option to force checking PyPI for latest available - version(s) of all packages requested by name and version, even if a matching - version is available locally. - - * Added automatic installation of dependencies declared by a distribution - being installed. These dependencies must be listed in the distribution's - ``EGG-INFO`` directory, so the distribution has to have declared its - dependencies by using setuptools. If a package has requirements it didn't - declare, you'll still have to deal with them yourself. (E.g., by asking - EasyInstall to find and install them.) - - * Added the ``--record`` option to ``easy_install`` for the benefit of tools - that run ``setup.py install --record=filename`` on behalf of another - packaging system.) - -0.5a3 - * Fixed not setting script permissions to allow execution. - - * Improved sandboxing so that setup scripts that want a temporary directory - (e.g. pychecker) can still run in the sandbox. - -0.5a2 - * Fix stupid stupid refactoring-at-the-last-minute typos. :( - -0.5a1 - * Added support for converting ``.win32.exe`` installers to eggs on the fly. - EasyInstall will now recognize such files by name and install them. - - * Fixed a problem with picking the "best" version to install (versions were - being sorted as strings, rather than as parsed values) - -0.4a4 - * Added support for the distutils "verbose/quiet" and "dry-run" options, as - well as the "optimize" flag. - - * Support downloading packages that were uploaded to PyPI (by scanning all - links on package pages, not just the homepage/download links). - -0.4a3 - * Add progress messages to the search/download process so that you can tell - what URLs it's reading to find download links. (Hopefully, this will help - people report out-of-date and broken links to package authors, and to tell - when they've asked for a package that doesn't exist.) - -0.4a2 - * Added support for installing scripts - - * Added support for setting options via distutils configuration files, and - using distutils' default options as a basis for EasyInstall's defaults. - - * Renamed ``--scan-url/-s`` to ``--find-links/-f`` to free up ``-s`` for the - script installation directory option. - - * Use ``urllib2`` instead of ``urllib``, to allow use of ``https:`` URLs if - Python includes SSL support. - -0.4a1 - * Added ``--scan-url`` and ``--index-url`` options, to scan download pages - and search PyPI for needed packages. - -0.3a4 - * Restrict ``--build-directory=DIR/-b DIR`` option to only be used with single - URL installs, to avoid running the wrong setup.py. - -0.3a3 - * Added ``--build-directory=DIR/-b DIR`` option. - - * Added "installation report" that explains how to use 'require()' when doing - a multiversion install or alternate installation directory. - - * Added SourceForge mirror auto-select (Contributed by Ian Bicking) - - * Added "sandboxing" that stops a setup script from running if it attempts to - write to the filesystem outside of the build area - - * Added more workarounds for packages with quirky ``install_data`` hacks - -0.3a2 - * Added subversion download support for ``svn:`` and ``svn+`` URLs, as well as - automatic recognition of HTTP subversion URLs (Contributed by Ian Bicking) - - * Misc. bug fixes - -0.3a1 - * Initial release. - - -Future Plans -============ - -* Additional utilities to list/remove/verify packages -* Signature checking? SSL? Ability to suppress PyPI search? -* Display byte progress meter when downloading distributions and long pages? -* Redirect stdout/stderr to log during run_setup? - diff --git a/docs/ez_setup.txt b/docs/ez_setup.txt new file mode 100644 index 00000000..0126fee3 --- /dev/null +++ b/docs/ez_setup.txt @@ -0,0 +1,195 @@ +:orphan: + +``ez_setup`` distribution guide +=============================== + +Using ``setuptools``... Without bundling it! +--------------------------------------------- + +.. warning:: **ez_setup** is deprecated in favor of PIP with **PEP-518** support. + +.. _ez_setup.py: https://bootstrap.pypa.io/ez_setup.py + +.. _EasyInstall Installation Instructions: easy_install.html + +.. _Custom Installation Locations: easy_install.html + +Your users might not have ``setuptools`` installed on their machines, or even +if they do, it might not be the right version. Fixing this is easy; just +download `ez_setup.py`_, and put it in the same directory as your ``setup.py`` +script. (Be sure to add it to your revision control system, too.) Then add +these two lines to the very top of your setup script, before the script imports +anything from setuptools: + +.. code-block:: python + + import ez_setup + ez_setup.use_setuptools() + +That's it. The ``ez_setup`` module will automatically download a matching +version of ``setuptools`` from PyPI, if it isn't present on the target system. +Whenever you install an updated version of setuptools, you should also update +your projects' ``ez_setup.py`` files, so that a matching version gets installed +on the target machine(s). + +(By the way, if you need to distribute a specific version of ``setuptools``, +you can specify the exact version and base download URL as parameters to the +``use_setuptools()`` function. See the function's docstring for details.) + + +What Your Users Should Know +--------------------------- + +In general, a setuptools-based project looks just like any distutils-based +project -- as long as your users have an internet connection and are installing +to ``site-packages``, that is. But for some users, these conditions don't +apply, and they may become frustrated if this is their first encounter with +a setuptools-based project. To keep these users happy, you should review the +following topics in your project's installation instructions, if they are +relevant to your project and your target audience isn't already familiar with +setuptools and ``easy_install``. + +Network Access + If your project is using ``ez_setup``, you should inform users of the + need to either have network access, or to preinstall the correct version of + setuptools using the `EasyInstall installation instructions`_. Those + instructions also have tips for dealing with firewalls as well as how to + manually download and install setuptools. + +Custom Installation Locations + You should inform your users that if they are installing your project to + somewhere other than the main ``site-packages`` directory, they should + first install setuptools using the instructions for `Custom Installation + Locations`_, before installing your project. + +Your Project's Dependencies + If your project depends on other projects that may need to be downloaded + from PyPI or elsewhere, you should list them in your installation + instructions, or tell users how to find out what they are. While most + users will not need this information, any users who don't have unrestricted + internet access may have to find, download, and install the other projects + manually. (Note, however, that they must still install those projects + using ``easy_install``, or your project will not know they are installed, + and your setup script will try to download them again.) + + If you want to be especially friendly to users with limited network access, + you may wish to build eggs for your project and its dependencies, making + them all available for download from your site, or at least create a page + with links to all of the needed eggs. In this way, users with limited + network access can manually download all the eggs to a single directory, + then use the ``-f`` option of ``easy_install`` to specify the directory + to find eggs in. Users who have full network access can just use ``-f`` + with the URL of your download page, and ``easy_install`` will find all the + needed eggs using your links directly. This is also useful when your + target audience isn't able to compile packages (e.g. most Windows users) + and your package or some of its dependencies include C code. + +Revision Control System Users and Co-Developers + Users and co-developers who are tracking your in-development code using + a revision control system should probably read this manual's sections + regarding such development. Alternately, you may wish to create a + quick-reference guide containing the tips from this manual that apply to + your particular situation. For example, if you recommend that people use + ``setup.py develop`` when tracking your in-development code, you should let + them know that this needs to be run after every update or commit. + + Similarly, if you remove modules or data files from your project, you + should remind them to run ``setup.py clean --all`` and delete any obsolete + ``.pyc`` or ``.pyo``. (This tip applies to the distutils in general, not + just setuptools, but not everybody knows about them; be kind to your users + by spelling out your project's best practices rather than leaving them + guessing.) + +Creating System Packages + Some users want to manage all Python packages using a single package + manager, and sometimes that package manager isn't ``easy_install``! + Setuptools currently supports ``bdist_rpm``, ``bdist_wininst``, and + ``bdist_dumb`` formats for system packaging. If a user has a locally- + installed "bdist" packaging tool that internally uses the distutils + ``install`` command, it should be able to work with ``setuptools``. Some + examples of "bdist" formats that this should work with include the + ``bdist_nsi`` and ``bdist_msi`` formats for Windows. + + However, packaging tools that build binary distributions by running + ``setup.py install`` on the command line or as a subprocess will require + modification to work with setuptools. They should use the + ``--single-version-externally-managed`` option to the ``install`` command, + combined with the standard ``--root`` or ``--record`` options. + See the `install command`_ documentation below for more details. The + ``bdist_deb`` command is an example of a command that currently requires + this kind of patching to work with setuptools. + + Please note that building system packages may require you to install + some system software, for example ``bdist_rpm`` requires the ``rpmbuild`` + command to be installed. + + If you or your users have a problem building a usable system package for + your project, please report the problem via the mailing list so that + either the "bdist" tool in question or setuptools can be modified to + resolve the issue. + +Your users might not have ``setuptools`` installed on their machines, or even +if they do, it might not be the right version. Fixing this is easy; just +download `ez_setup.py`_, and put it in the same directory as your ``setup.py`` +script. (Be sure to add it to your revision control system, too.) Then add +these two lines to the very top of your setup script, before the script imports +anything from setuptools: + +.. code-block:: python + + import ez_setup + ez_setup.use_setuptools() + +That's it. The ``ez_setup`` module will automatically download a matching +version of ``setuptools`` from PyPI, if it isn't present on the target system. +Whenever you install an updated version of setuptools, you should also update +your projects' ``ez_setup.py`` files, so that a matching version gets installed +on the target machine(s). + +(By the way, if you need to distribute a specific version of ``setuptools``, +you can specify the exact version and base download URL as parameters to the +``use_setuptools()`` function. See the function's docstring for details.) + +.. _install command: + +``install`` - Run ``easy_install`` or old-style installation +============================================================ + +The setuptools ``install`` command is basically a shortcut to run the +``easy_install`` command on the current project. However, for convenience +in creating "system packages" of setuptools-based projects, you can also +use this option: + +``--single-version-externally-managed`` + This boolean option tells the ``install`` command to perform an "old style" + installation, with the addition of an ``.egg-info`` directory so that the + installed project will still have its metadata available and operate + normally. If you use this option, you *must* also specify the ``--root`` + or ``--record`` options (or both), because otherwise you will have no way + to identify and remove the installed files. + +This option is automatically in effect when ``install`` is invoked by another +distutils command, so that commands like ``bdist_wininst`` and ``bdist_rpm`` +will create system packages of eggs. It is also automatically in effect if +you specify the ``--root`` option. + + +``install_egg_info`` - Install an ``.egg-info`` directory in ``site-packages`` +============================================================================== + +Setuptools runs this command as part of ``install`` operations that use the +``--single-version-externally-managed`` options. You should not invoke it +directly; it is documented here for completeness and so that distutils +extensions such as system package builders can make use of it. This command +has only one option: + +``--install-dir=DIR, -d DIR`` + The parent directory where the ``.egg-info`` directory will be placed. + Defaults to the same as the ``--install-dir`` option specified for the + ``install_lib`` command, which is usually the system ``site-packages`` + directory. + +This command assumes that the ``egg_info`` command has been given valid options +via the command line or ``setup.cfg``, as it will invoke the ``egg_info`` +command and use its options to locate the project's source ``.egg-info`` +directory. diff --git a/docs/formats.txt b/docs/formats.txt index 9e6fe727..a182eb99 100644 --- a/docs/formats.txt +++ b/docs/formats.txt @@ -110,7 +110,7 @@ the need to create a directory just to store one file. This option is other metadata. (In fact, setuptools itself never generates ``.egg-info`` files, either; the support for using files was added so that the requirement could easily be satisfied by other tools, such -as the distutils in Python 2.5). +as distutils). In addition to the ``PKG-INFO`` file, an egg's metadata directory may also include files and directories representing various forms of diff --git a/docs/history.txt b/docs/history.txt index 8fd1dc65..385cfa7e 100644 --- a/docs/history.txt +++ b/docs/history.txt @@ -40,7 +40,6 @@ Credits re-invigorated the community on the project, encouraged renewed innovation, and addressed many defects. -* Since the merge with Distribute, Jason R. Coombs is the - maintainer of setuptools. The project is maintained in coordination with - the Python Packaging Authority (PyPA) and the larger Python community. - +* Jason R. Coombs performed the merge with Distribute, maintaining the + project for several years in coordination with the Python Packaging + Authority (PyPA). diff --git a/docs/index.txt b/docs/index.txt index 74aabb5e..13a46e74 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -17,9 +17,9 @@ Documentation content: :maxdepth: 2 setuptools - easy_install pkg_resources python3 development roadmap + Deprecated: Easy Install <easy_install> history diff --git a/docs/pkg_resources.txt b/docs/pkg_resources.txt index 487320ce..0c9fb5f2 100644 --- a/docs/pkg_resources.txt +++ b/docs/pkg_resources.txt @@ -620,8 +620,8 @@ Requirements Parsing The "markers" in a requirement are used to specify when a requirement should be installed -- the requirement will be installed if the marker evaluates as true in the current environment. For example, specifying - ``argparse;python_version<"2.7"`` will not install in an Python 2.7 or 3.3 - environment, but will in a Python 2.6 environment. + ``argparse;python_version<"3.0"`` will not install in an Python 3 + environment, but will in a Python 2 environment. ``Requirement`` Methods and Attributes -------------------------------------- @@ -638,7 +638,7 @@ Requirements Parsing sorted into ascending version order, and used to establish what ranges of versions are acceptable. Adjacent redundant conditions are effectively consolidated (e.g. ``">1, >2"`` produces the same results as ``">2"``, and - ``"<2,<3"`` produces the same results as``"<2"``). ``"!="`` versions are + ``"<2,<3"`` produces the same results as ``"<2"``). ``"!="`` versions are excised from the ranges they fall within. The version being tested for acceptability is then checked for membership in the resulting ranges. @@ -703,7 +703,7 @@ entry point group and look for entry points named "pre_process" and To advertise an entry point, a project needs to use ``setuptools`` and provide an ``entry_points`` argument to ``setup()`` in its setup script, so that the entry points will be included in the distribution's metadata. For more -details, see the ``setuptools`` documentation. (XXX link here to setuptools) +details, see the [``setuptools`` documentation](https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins). Each project distribution can advertise at most one entry point of a given name within the same entry point group. For example, a distutils extension @@ -1087,6 +1087,7 @@ so that supporting custom importers or new distribution formats can be done simply by creating an appropriate `IResourceProvider`_ implementation; see the section below on `Supporting Custom Importers`_ for more details. +.. _ResourceManager API: ``ResourceManager`` API ======================= @@ -1658,19 +1659,7 @@ PEP 302 Utilities ----------------- ``get_importer(path_item)`` - Retrieve a PEP 302 "importer" for the given path item (which need not - actually be on ``sys.path``). This routine simulates the PEP 302 protocol - for obtaining an "importer" object. It first checks for an importer for - the path item in ``sys.path_importer_cache``, and if not found it calls - each of the ``sys.path_hooks`` and caches the result if a good importer is - found. If no importer is found, this routine returns an ``ImpWrapper`` - instance that wraps the builtin import machinery as a PEP 302-compliant - "importer" object. This ``ImpWrapper`` is *not* cached; instead a new - instance is returned each time. - - (Note: When run under Python 2.5, this function is simply an alias for - ``pkgutil.get_importer()``, and instead of ``pkg_resources.ImpWrapper`` - instances, it may return ``pkgutil.ImpImporter`` instances.) + A deprecated alias for ``pkgutil.get_importer()`` File/Path Utilities diff --git a/docs/python3.txt b/docs/python3.txt index d550cb68..6b55fe78 100644 --- a/docs/python3.txt +++ b/docs/python3.txt @@ -9,9 +9,9 @@ code. Setuptools provides a facility to invoke 2to3 on the code as a part of the build process, by setting the keyword parameter ``use_2to3`` to True, but -the Setuptools strongly recommends instead developing a unified codebase -using `six <https://pypi.python.org/pypi/six>`_, -`future <https://pypi.python.org/pypi/future>`_, or another compatibility +the Setuptools project strongly recommends instead developing a unified codebase +using `six <https://pypi.org/project/six/>`_, +`future <https://pypi.org/project/future/>`_, or another compatibility library. diff --git a/docs/releases.txt b/docs/releases.txt index 30ea084f..98ba39e8 100644 --- a/docs/releases.txt +++ b/docs/releases.txt @@ -7,20 +7,35 @@ mechanical technique for releases, enacted by Travis following a successful build of a tagged release per `PyPI deployment <https://docs.travis-ci.com/user/deployment/pypi>`_. -Prior to cutting a release, please check that the CHANGES.rst reflects -the summary of changes since the last release. -Ideally, these changelog entries would have been added -along with the changes, but it's always good to check. -Think about it from the -perspective of a user not involved with the development--what would -that person want to know about what has changed--or from the -perspective of your future self wanting to know when a particular -change landed. - -To cut a release, install and run ``bump2version {part}`` where ``part`` +Prior to cutting a release, please use `towncrier`_ to update +``CHANGES.rst`` to summarize the changes since the last release. +To update the changelog: + +1. Install towncrier via ``pip install towncrier`` if not already installed. +2. Preview the new ``CHANGES.rst`` entry by running + ``towncrier --draft --version {new.version.number}`` (enter the desired + version number for the next release). If any changes are needed, make + them and generate a new preview until the output is acceptable. Run + ``git add`` for any modified files. +3. Run ``towncrier --version {new.version.number}`` to stage the changelog + updates in git. +4. Verify that there are no remaining ``changelog.d/*.rst`` files. If a + file was named incorrectly, it may be ignored by towncrier. +5. Review the updated ``CHANGES.rst`` file. If any changes are needed, + make the edits and stage them via ``git add CHANGES.rst``. + +Once the changelog edits are staged and ready to commit, cut a release by +installing and running ``bump2version --allow-dirty {part}`` where ``part`` is major, minor, or patch based on the scope of the changes in the -release. Then, push the commits to the master branch. If tests pass, -the release will be uploaded to PyPI (from the Python 3.6 tests). +release. Then, push the commits to the master branch:: + + $ git push origin master + $ git push --tags + +If tests pass, the release will be uploaded to PyPI (from the Python 3.6 +tests). + +.. _towncrier: https://pypi.org/project/towncrier/ Release Frequency ----------------- diff --git a/docs/requirements.txt b/docs/requirements.txt index 2138c884..c6d594e8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx +sphinx!=1.8.0 rst.linker>=1.9 jaraco.packaging>=3.2 diff --git a/docs/roadmap.txt b/docs/roadmap.txt index 8f175b9f..147288f3 100644 --- a/docs/roadmap.txt +++ b/docs/roadmap.txt @@ -2,5 +2,6 @@ Roadmap ======= -Setuptools is primarily in maintenance mode. The project attempts to address -user issues, concerns, and feature requests in a timely fashion. +Setuptools maintains a series of `milestones +<https://github.com/pypa/setuptools/milestones>`_ to track +a roadmap of large-scale goals. diff --git a/docs/setuptools.txt b/docs/setuptools.txt index f255cfd2..cf166e9c 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -3,7 +3,7 @@ Building and Distributing Packages with Setuptools ================================================== ``Setuptools`` is a collection of enhancements to the Python ``distutils`` -(for Python 2.6 and up) that allow developers to more easily build and +that allow developers to more easily build and distribute Python packages, especially ones that have dependencies on other packages. @@ -41,15 +41,13 @@ Feature Highlights: files for any number of "main" functions in your project. (Note: this is not a py2exe replacement; the .exe files rely on the local Python installation.) -* Transparent Pyrex support, so that your setup.py can list ``.pyx`` files and - still work even when the end-user doesn't have Pyrex installed (as long as - you include the Pyrex-generated C in your source distribution) +* Transparent Cython support, so that your setup.py can list ``.pyx`` files and + still work even when the end-user doesn't have Cython installed (as long as + you include the Cython-generated C in your source distribution) * Command aliases - create project-specific, per-user, or site-wide shortcut names for commonly used commands and options -* PyPI upload support - upload your source distributions and eggs to PyPI - * Deploy your project in "development mode", such that it's available on ``sys.path``, yet can still be edited directly from its source checkout. @@ -59,6 +57,9 @@ Feature Highlights: * Create extensible applications and frameworks that automatically discover extensions, using simple "entry points" declared in a project's setup script. +* Full support for PEP 420 via ``find_namespace_packages()``, which is also backwards + compatible to the existing ``find_packages()`` for Python >= 3.3. + .. contents:: **Table of Contents** .. _ez_setup.py: `bootstrap module`_ @@ -72,23 +73,17 @@ Developer's Guide Installing ``setuptools`` ========================= -Please follow the `EasyInstall Installation Instructions`_ to install the -current stable version of setuptools. In particular, be sure to read the -section on `Custom Installation Locations`_ if you are installing anywhere -other than Python's ``site-packages`` directory. - -.. _EasyInstall Installation Instructions: easy_install.html#installation-instructions +.. _EasyInstall Installation Instructions: easy_install.html -.. _Custom Installation Locations: easy_install.html#custom-installation-locations +.. _Custom Installation Locations: easy_install.html -If you want the current in-development version of setuptools, you should first -install a stable version, and then run:: +.. _Installing Packages: https://packaging.python.org/tutorials/installing-packages/ - ez_setup.py setuptools==dev +To install the latest version of setuptools, use:: -This will download and install the latest development (i.e. unstable) version -of setuptools from the Python Subversion sandbox. + pip install -U setuptools +Refer to `Installing Packages`_ guide for more information. Basic Use ========= @@ -107,11 +102,10 @@ As you can see, it doesn't take much to use setuptools in a project. Run that script in your project folder, alongside the Python packages you have developed. -Invoke that script to produce eggs, upload to -PyPI, and automatically include all packages in the directory where the -setup.py lives. See the `Command Reference`_ section below to see what -commands you can give to this setup script. For example, -to produce a source distribution, simply invoke:: +Invoke that script to produce distributions and automatically include all +packages in the directory where the setup.py lives. See the `Command +Reference`_ section below to see what commands you can give to this setup +script. For example, to produce a source distribution, simply invoke:: python setup.py sdist @@ -138,13 +132,18 @@ dependencies, and perhaps some data files and scripts:: 'hello': ['*.msg'], }, - # metadata for upload to PyPI + # metadata to display on PyPI author="Me", author_email="me@example.com", description="This is an Example Package", license="PSF", keywords="hello world example examples", url="http://example.com/HelloWorld/", # project home page, if any + project_urls={ + "Bug Tracker": "https://bugs.example.com/HelloWorld/", + "Documentation": "https://docs.example.com/HelloWorld/", + "Source Code": "https://code.example.com/HelloWorld/", + } # could also include long_description, download_url, classifiers, etc. ) @@ -237,7 +236,6 @@ have setuptools automatically tag your in-development releases with various pre- or post-release tags. See the following sections for more details: * `Tagging and "Daily Build" or "Snapshot" Releases`_ -* `Managing "Continuous Releases" Using Subversion`_ * The `egg_info`_ command @@ -408,6 +406,11 @@ unless you need the associated ``setuptools`` feature. A list of modules to search for additional fixers to be used during the 2to3 conversion. See :doc:`python3` for more details. +``project_urls`` + An arbitrary map of URL names to hyperlinks, allowing more extensible + documentation of where various resources can be found than the simple + ``url`` and ``download_url`` options provide. + Using ``find_packages()`` ------------------------- @@ -427,9 +430,9 @@ such projects also need something like ``package_dir={'':'src'}`` in their ``setup()`` arguments, but that's just a normal distutils thing.) Anyway, ``find_packages()`` walks the target directory, filtering by inclusion -patterns, and finds Python packages (any directory). On Python 3.2 and -earlier, packages are only recognized if they include an ``__init__.py`` file. -Finally, exclusion patterns are applied to remove matching packages. +patterns, and finds Python packages (any directory). Packages are only +recognized if they include an ``__init__.py`` file. Finally, exclusion +patterns are applied to remove matching packages. Inclusion and exclusion patterns are package names, optionally including wildcards. For @@ -453,6 +456,67 @@ argument in your setup script. Especially since it frees you from having to remember to modify your setup script whenever your project grows additional top-level packages or subpackages. +``find_namespace_packages()`` +----------------------------- +In Python 3.3+, ``setuptools`` also provides the ``find_namespace_packages`` variant +of ``find_packages``, which has the same function signature as +``find_packages``, but works with `PEP 420`_ compliant implicit namespace +packages. Here is a minimal setup script using ``find_namespace_packages``:: + + from setuptools import setup, find_namespace_packages + setup( + name="HelloWorld", + version="0.1", + packages=find_namespace_packages(), + ) + + +Keep in mind that according to PEP 420, you may have to either re-organize your +codebase a bit or define a few exclusions, as the definition of an implicit +namespace package is quite lenient, so for a project organized like so:: + + + ├── namespace + │ └── mypackage + │ ├── __init__.py + │ └── mod1.py + ├── setup.py + └── tests + └── test_mod1.py + +A naive ``find_namespace_packages()`` would install both ``namespace.mypackage`` and a +top-level package called ``tests``! One way to avoid this problem is to use the +``include`` keyword to whitelist the packages to include, like so:: + + from setuptools import setup, find_namespace_packages + + setup( + name="namespace.mypackage", + version="0.1", + packages=find_namespace_packages(include=['namespace.*']) + ) + +Another option is to use the "src" layout, where all package code is placed in +the ``src`` directory, like so:: + + + ├── setup.py + ├── src + │ └── namespace + │ └── mypackage + │ ├── __init__.py + │ └── mod1.py + └── tests + └── test_mod1.py + +With this layout, the package directory is specified as ``src``, as such:: + + setup(name="namespace.mypackage", + version="0.1", + package_dir={'': 'src'}, + packages=find_namespace_packages(where='src')) + +.. _PEP 420: https://www.python.org/dev/peps/pep-0420/ Automatic Script Creation ========================= @@ -561,7 +625,7 @@ project name or version identifier must be replaced with ``-``. Version specifiers for a given project are internally sorted into ascending version order, and used to establish what ranges of versions are acceptable. Adjacent redundant conditions are also consolidated (e.g. ``">1, >2"`` becomes -``">1"``, and ``"<2,<3"`` becomes ``"<3"``). ``"!="`` versions are excised from +``">2"``, and ``"<2,<3"`` becomes ``"<2"``). ``"!="`` versions are excised from the ranges they fall within. A project's version is then checked for membership in the resulting ranges. (Note that providing conflicting conditions for the same version (e.g. "<2,>=2" or "==2,!=2") is meaningless and may @@ -609,9 +673,8 @@ using ``setup.py develop``.) Dependencies that aren't in PyPI -------------------------------- -If your project depends on packages that aren't registered in PyPI, you may -still be able to depend on them, as long as they are available for download -as: +If your project depends on packages that don't exist on PyPI, you may still be +able to depend on them, as long as they are available for download as: - an egg, in the standard distutils ``sdist`` format, - a single ``.py`` file, or @@ -875,6 +938,14 @@ Also notice that if you use paths, you *must* use a forward slash (``/``) as the path separator, even if you are on Windows. Setuptools automatically converts slashes to appropriate platform-specific separators at build time. +If datafiles are contained in a subdirectory of a package that isn't a package +itself (no ``__init__.py``), then the subdirectory names (or ``*``) are required +in the ``package_data`` argument (as shown above with ``'data/*.dat'``). + +When building an ``sdist``, the datafiles are also drawn from the +``package_name.egg-info/SOURCES.txt`` file, so make sure that this is removed if +the ``setup.py`` ``package_data`` list is updated before calling ``setup.py``. + (Note: although the ``package_data`` argument was previously only available in ``setuptools``, it was also added to the Python ``distutils`` package as of Python 2.4; there is `some documentation for the feature`__ available on the @@ -916,7 +987,7 @@ In summary, the three options allow you to: Accept all data files and directories matched by ``MANIFEST.in``. ``package_data`` - Specify additional patterns to match files and directories that may or may + Specify additional patterns to match files that may or may not be matched by ``MANIFEST.in`` or found in source control. ``exclude_package_data`` @@ -953,35 +1024,14 @@ a quick example of converting code that uses ``__file__`` to use Non-Package Data Files ---------------------- -The ``distutils`` normally install general "data files" to a platform-specific -location (e.g. ``/usr/share``). This feature intended to be used for things -like documentation, example configuration files, and the like. ``setuptools`` -does not install these data files in a separate location, however. They are -bundled inside the egg file or directory, alongside the Python modules and -packages. The data files can also be accessed using the :ref:`ResourceManager -API`, by specifying a ``Requirement`` instead of a package name:: - - from pkg_resources import Requirement, resource_filename - filename = resource_filename(Requirement.parse("MyProject"),"sample.conf") - -The above code will obtain the filename of the "sample.conf" file in the data -root of the "MyProject" distribution. - -Note, by the way, that this encapsulation of data files means that you can't -actually install data files to some arbitrary location on a user's machine; -this is a feature, not a bug. You can always include a script in your -distribution that extracts and copies your the documentation or data files to -a user-specified location, at their discretion. If you put related data files -in a single directory, you can use ``resource_filename()`` with the directory -name to get a filesystem directory that then can be copied with the ``shutil`` -module. (Even if your package is installed as a zipfile, calling -``resource_filename()`` on a directory will return an actual filesystem -directory, whose contents will be that entire subtree of your distribution.) - -(Of course, if you're writing a new package, you can just as easily place your -data files or directories inside one of your packages, rather than using the -distutils' approach. However, if you're updating an existing application, it -may be simpler not to change the way it currently specifies these data files.) +Historically, ``setuptools`` by way of ``easy_install`` would encapsulate data +files from the distribution into the egg (see `the old docs +<https://github.com/pypa/setuptools/blob/52aacd5b276fedd6849c3a648a0014f5da563e93/docs/setuptools.txt#L970-L1001>`_). As eggs are deprecated and pip-based installs +fall back to the platform-specific location for installing data files, there is +no supported facility to reliably retrieve these resources. + +Instead, the PyPA recommends that any data files you wish to be accessible at +run time be included in the package. Automatic Resource Extraction @@ -1173,130 +1223,53 @@ the quoted part. Distributing a ``setuptools``-based project =========================================== -Using ``setuptools``... Without bundling it! ---------------------------------------------- +Detailed instructions to distribute a setuptools project can be found at +`Packaging project tutorials`_. -.. warning:: **ez_setup** is deprecated in favor of PIP with **PEP-518** support. +.. _Packaging project tutorials: https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives -Your users might not have ``setuptools`` installed on their machines, or even -if they do, it might not be the right version. Fixing this is easy; just -download `ez_setup.py`_, and put it in the same directory as your ``setup.py`` -script. (Be sure to add it to your revision control system, too.) Then add -these two lines to the very top of your setup script, before the script imports -anything from setuptools: +Before you begin, make sure you have the latest versions of setuptools and wheel:: -.. code-block:: python + python3 -m pip install --user --upgrade setuptools wheel - import ez_setup - ez_setup.use_setuptools() +To build a setuptools project, run this command from the same directory where +setup.py is located:: -That's it. The ``ez_setup`` module will automatically download a matching -version of ``setuptools`` from PyPI, if it isn't present on the target system. -Whenever you install an updated version of setuptools, you should also update -your projects' ``ez_setup.py`` files, so that a matching version gets installed -on the target machine(s). + python3 setup.py sdist bdist_wheel -By the way, setuptools supports the new PyPI "upload" command, so you can use -``setup.py sdist upload`` or ``setup.py bdist_egg upload`` to upload your -source or egg distributions respectively. Your project's current version must -be registered with PyPI first, of course; you can use ``setup.py register`` to -do that. Or you can do it all in one step, e.g. ``setup.py register sdist -bdist_egg upload`` will register the package, build source and egg -distributions, and then upload them both to PyPI, where they'll be easily -found by other projects that depend on them. +This will generate distribution archives in the `dist` directory. -(By the way, if you need to distribute a specific version of ``setuptools``, -you can specify the exact version and base download URL as parameters to the -``use_setuptools()`` function. See the function's docstring for details.) +Before you upload the generated archives make sure you're registered on +https://test.pypi.org/account/register/. You will also need to verify your email +to be able to upload any packages. +You should install twine to be able to upload packages:: + python3 -m pip install --user --upgrade setuptools wheel -What Your Users Should Know ---------------------------- +Now, to upload these archives, run:: + + twine upload --repository-url https://test.pypi.org/legacy/ dist/* + +To install your newly uploaded package ``example_pkg``, you can use pip:: + + python3 -m pip install --index-url https://test.pypi.org/simple/ example_pkg + +If you have issues at any point, please refer to `Packaging project tutorials`_ +for clarification. + +Distributing legacy ``setuptools`` projects using ez_setup.py +------------------------------------------------------------- + +.. warning:: **ez_setup** is deprecated in favor of PIP with **PEP-518** support. + +Distributing packages using the legacy ``ez_setup.py`` and ``easy_install`` is +deprecated in favor of PIP. Please consider migrating to using pip and twine based +distribution. -In general, a setuptools-based project looks just like any distutils-based -project -- as long as your users have an internet connection and are installing -to ``site-packages``, that is. But for some users, these conditions don't -apply, and they may become frustrated if this is their first encounter with -a setuptools-based project. To keep these users happy, you should review the -following topics in your project's installation instructions, if they are -relevant to your project and your target audience isn't already familiar with -setuptools and ``easy_install``. - -Network Access - If your project is using ``ez_setup``, you should inform users of the - need to either have network access, or to preinstall the correct version of - setuptools using the `EasyInstall installation instructions`_. Those - instructions also have tips for dealing with firewalls as well as how to - manually download and install setuptools. - -Custom Installation Locations - You should inform your users that if they are installing your project to - somewhere other than the main ``site-packages`` directory, they should - first install setuptools using the instructions for `Custom Installation - Locations`_, before installing your project. - -Your Project's Dependencies - If your project depends on other projects that may need to be downloaded - from PyPI or elsewhere, you should list them in your installation - instructions, or tell users how to find out what they are. While most - users will not need this information, any users who don't have unrestricted - internet access may have to find, download, and install the other projects - manually. (Note, however, that they must still install those projects - using ``easy_install``, or your project will not know they are installed, - and your setup script will try to download them again.) - - If you want to be especially friendly to users with limited network access, - you may wish to build eggs for your project and its dependencies, making - them all available for download from your site, or at least create a page - with links to all of the needed eggs. In this way, users with limited - network access can manually download all the eggs to a single directory, - then use the ``-f`` option of ``easy_install`` to specify the directory - to find eggs in. Users who have full network access can just use ``-f`` - with the URL of your download page, and ``easy_install`` will find all the - needed eggs using your links directly. This is also useful when your - target audience isn't able to compile packages (e.g. most Windows users) - and your package or some of its dependencies include C code. - -Revision Control System Users and Co-Developers - Users and co-developers who are tracking your in-development code using - a revision control system should probably read this manual's sections - regarding such development. Alternately, you may wish to create a - quick-reference guide containing the tips from this manual that apply to - your particular situation. For example, if you recommend that people use - ``setup.py develop`` when tracking your in-development code, you should let - them know that this needs to be run after every update or commit. - - Similarly, if you remove modules or data files from your project, you - should remind them to run ``setup.py clean --all`` and delete any obsolete - ``.pyc`` or ``.pyo``. (This tip applies to the distutils in general, not - just setuptools, but not everybody knows about them; be kind to your users - by spelling out your project's best practices rather than leaving them - guessing.) - -Creating System Packages - Some users want to manage all Python packages using a single package - manager, and sometimes that package manager isn't ``easy_install``! - Setuptools currently supports ``bdist_rpm``, ``bdist_wininst``, and - ``bdist_dumb`` formats for system packaging. If a user has a locally- - installed "bdist" packaging tool that internally uses the distutils - ``install`` command, it should be able to work with ``setuptools``. Some - examples of "bdist" formats that this should work with include the - ``bdist_nsi`` and ``bdist_msi`` formats for Windows. - - However, packaging tools that build binary distributions by running - ``setup.py install`` on the command line or as a subprocess will require - modification to work with setuptools. They should use the - ``--single-version-externally-managed`` option to the ``install`` command, - combined with the standard ``--root`` or ``--record`` options. - See the `install command`_ documentation below for more details. The - ``bdist_deb`` command is an example of a command that currently requires - this kind of patching to work with setuptools. - - If you or your users have a problem building a usable system package for - your project, please report the problem via the mailing list so that - either the "bdist" tool in question or setuptools can be modified to - resolve the issue. +However, if you still have any ``ez_setup`` based packages, documentation for +ez_setup based distributions can be found at `ez_setup distribution guide`_. +.. _ez_setup distribution guide: ez_setup.html Setting the ``zip_safe`` flag ----------------------------- @@ -1348,6 +1321,7 @@ then make an explicit declaration of ``True`` or ``False`` for the ``zip_safe`` flag, so that it will not be necessary for ``bdist_egg`` or ``EasyInstall`` to try to guess whether your project can work as a zipfile. +.. _Namespace Packages: Namespace Packages ------------------ @@ -1520,22 +1494,11 @@ you use any option at all. Making your package available for EasyInstall --------------------------------------------- -If you use the ``register`` command (``setup.py register``) to register your -package with PyPI, that's most of the battle right there. (See the -`docs for the register command`_ for more details.) - -.. _docs for the register command: http://docs.python.org/dist/package-index.html - -If you also use the `upload`_ command to upload actual distributions of your -package, that's even better, because EasyInstall will be able to find and -download them directly from your project's PyPI page. - -However, there may be reasons why you don't want to upload distributions to +There may be reasons why you don't want to upload distributions to PyPI, and just want your existing distributions (or perhaps a Subversion checkout) to be used instead. -So here's what you need to do before running the ``register`` command. There -are three ``setup()`` arguments that affect EasyInstall: +There are three ``setup()`` arguments that affect EasyInstall: ``url`` and ``download_url`` These become links on your project's PyPI page. EasyInstall will examine @@ -1593,13 +1556,12 @@ tagging the release, so the trunk will still produce development snapshots. Alternately, if you are not branching for releases, you can override the default version options on the command line, using something like:: - python setup.py egg_info -Db "" sdist bdist_egg register upload + python setup.py egg_info -Db "" sdist bdist_egg The first part of this command (``egg_info -Db ""``) will override the -configured tag information, before creating source and binary eggs, registering -the project with PyPI, and uploading the files. Thus, these commands will use -the plain version from your ``setup.py``, without adding the build designation -string. +configured tag information, before creating source and binary eggs. Thus, these +commands will use the plain version from your ``setup.py``, without adding the +build designation string. Of course, if you will be doing this a lot, you may wish to create a personal alias for this operation, e.g.:: @@ -1608,36 +1570,50 @@ alias for this operation, e.g.:: You can then use it like this:: - python setup.py release sdist bdist_egg register upload + python setup.py release sdist bdist_egg Or of course you can create more elaborate aliases that do all of the above. See the sections below on the `egg_info`_ and `alias`_ commands for more ideas. -Distributing Extensions compiled with Pyrex -------------------------------------------- +Distributing Extensions compiled with Cython +-------------------------------------------- + +``setuptools`` will detect at build time whether Cython is installed or not. +If Cython is not found ``setuptools`` will ignore pyx files. + +To ensure Cython is available, include Cython in the build-requires section +of your pyproject.toml:: + + [build-system] + requires=[..., 'cython'] -``setuptools`` includes transparent support for building Pyrex extensions, as -long as you define your extensions using ``setuptools.Extension``, *not* -``distutils.Extension``. You must also not import anything from Pyrex in -your setup script. +Built with pip 10 or later, that declaration is sufficient to include Cython +in the build. For broader compatibility, declare the dependency in your +setup-requires of setup.cfg:: + + [options] + setup_requires = + ... + cython + +As long as Cython is present in the build environment, ``setuptools`` includes +transparent support for building Cython extensions, as +long as extensions are defined using ``setuptools.Extension``. If you follow these rules, you can safely list ``.pyx`` files as the source -of your ``Extension`` objects in the setup script. ``setuptools`` will detect -at build time whether Pyrex is installed or not. If it is, then ``setuptools`` -will use it. If not, then ``setuptools`` will silently change the -``Extension`` objects to refer to the ``.c`` counterparts of the ``.pyx`` -files, so that the normal distutils C compilation process will occur. +of your ``Extension`` objects in the setup script. If it is, then ``setuptools`` +will use it. Of course, for this to work, your source distributions must include the C -code generated by Pyrex, as well as your original ``.pyx`` files. This means +code generated by Cython, as well as your original ``.pyx`` files. This means that you will probably want to include current ``.c`` files in your revision control system, rebuilding them whenever you check changes in for the ``.pyx`` source files. This will ensure that people tracking your project in a revision -control system will be able to build it even if they don't have Pyrex +control system will be able to build it even if they don't have Cython installed, and that your source releases will be similarly usable with or -without Pyrex. +without Cython. ----------------- @@ -1924,11 +1900,11 @@ This command performs two operations: it updates a project's ``.egg-info`` metadata directory (used by the ``bdist_egg``, ``develop``, and ``test`` commands), and it allows you to temporarily change a project's version string, to support "daily builds" or "snapshot" releases. It is run automatically by -the ``sdist``, ``bdist_egg``, ``develop``, ``register``, and ``test`` commands -in order to update the project's metadata, but you can also specify it -explicitly in order to temporarily change the project's version string while -executing other commands. (It also generates the``.egg-info/SOURCES.txt`` -manifest file, which is used when you are building source distributions.) +the ``sdist``, ``bdist_egg``, ``develop``, and ``test`` commands in order to +update the project's metadata, but you can also specify it explicitly in order +to temporarily change the project's version string while executing other +commands. (It also generates the``.egg-info/SOURCES.txt`` manifest file, which +is used when you are building source distributions.) In addition to writing the core egg metadata defined by ``setuptools`` and required by ``pkg_resources``, this command can be extended to write other @@ -2002,60 +1978,14 @@ Creating a dated "nightly build" snapshot egg:: python setup.py egg_info --tag-date --tag-build=DEV bdist_egg -Creating and uploading a release with no version tags, even if some default -tags are specified in ``setup.cfg``:: +Creating a release with no version tags, even if some default tags are +specified in ``setup.cfg``:: - python setup.py egg_info -RDb "" sdist bdist_egg register upload + python setup.py egg_info -RDb "" sdist bdist_egg (Notice that ``egg_info`` must always appear on the command line *before* any commands that you want the version changes to apply to.) - -.. _install command: - -``install`` - Run ``easy_install`` or old-style installation -============================================================ - -The setuptools ``install`` command is basically a shortcut to run the -``easy_install`` command on the current project. However, for convenience -in creating "system packages" of setuptools-based projects, you can also -use this option: - -``--single-version-externally-managed`` - This boolean option tells the ``install`` command to perform an "old style" - installation, with the addition of an ``.egg-info`` directory so that the - installed project will still have its metadata available and operate - normally. If you use this option, you *must* also specify the ``--root`` - or ``--record`` options (or both), because otherwise you will have no way - to identify and remove the installed files. - -This option is automatically in effect when ``install`` is invoked by another -distutils command, so that commands like ``bdist_wininst`` and ``bdist_rpm`` -will create system packages of eggs. It is also automatically in effect if -you specify the ``--root`` option. - - -``install_egg_info`` - Install an ``.egg-info`` directory in ``site-packages`` -============================================================================== - -Setuptools runs this command as part of ``install`` operations that use the -``--single-version-externally-managed`` options. You should not invoke it -directly; it is documented here for completeness and so that distutils -extensions such as system package builders can make use of it. This command -has only one option: - -``--install-dir=DIR, -d DIR`` - The parent directory where the ``.egg-info`` directory will be placed. - Defaults to the same as the ``--install-dir`` option specified for the - ``install_lib`` command, which is usually the system ``site-packages`` - directory. - -This command assumes that the ``egg_info`` command has been given valid options -via the command line or ``setup.cfg``, as it will invoke the ``egg_info`` -command and use its options to locate the project's source ``.egg-info`` -directory. - - .. _rotate: ``rotate`` - Delete outdated distribution files @@ -2254,25 +2184,16 @@ available: ``upload`` - Upload source and/or egg distributions to PyPI =========================================================== +.. warning:: + **upload** is deprecated in favor of using `twine + <https://pypi.org/p/twine>`_ + The ``upload`` command is implemented and `documented <https://docs.python.org/3.1/distutils/uploading.html>`_ in distutils. -Setuptools augments the ``upload`` command with support -for `keyring <https://pypi.python.org/pypi/keyring>`_, -allowing the password to be stored in a secure -location and not in plaintext in the .pypirc file. To use -keyring, first install keyring and set the password for -the relevant repository, e.g.:: - - python -m keyring set <repository> <username> - Password for '<username>' in '<repository>': ******** - -Then, in .pypirc, set the repository configuration as normal, -but omit the password. Thereafter, uploads will use the -password from the keyring. - New in 20.1: Added keyring support. +New in 40.0: Deprecated the upload command. ----------------------------------------- @@ -2287,7 +2208,7 @@ Configuring setup() using setup.cfg files ``Setuptools`` allows using configuration files (usually :file:`setup.cfg`) to define a package’s metadata and other options that are normally supplied -to the ``setup()`` function. +to the ``setup()`` function (declarative config). This approach not only allows automation scenarios but also reduces boilerplate code in some cases. @@ -2323,6 +2244,9 @@ boilerplate code in some cases. scripts = bin/first.py bin/second.py + install_requires = + requests + importlib; python_version == "2.6" [options.package_data] * = *.txt, *.rst @@ -2337,6 +2261,11 @@ boilerplate code in some cases. src.subpackage1 src.subpackage2 + [options.data_files] + /etc/my_package = + site.d/00_default.conf + host.d/00_default.conf + data = data/img/logo.png, data/svg/icon.svg Metadata and options are set in the config sections of the same name. @@ -2359,12 +2288,41 @@ Metadata and options are set in the config sections of the same name. * In some cases, complex values can be provided in dedicated subsections for clarity. -* Some keys allow ``file:``, ``attr:``, and ``find:`` directives in order to - cover common usecases. +* Some keys allow ``file:``, ``attr:``, and ``find:`` and ``find_namespace:`` directives in + order to cover common usecases. * Unknown keys are ignored. +Using a ``src/`` layout +======================= + +One commonly used package configuration has all the module source code in a +subdirectory (often called the ``src/`` layout), like this:: + + ├── src + │ └── mypackage + │ ├── __init__.py + │ └── mod1.py + ├── setup.py + └── setup.cfg + +You can set up your ``setup.cfg`` to automatically find all your packages in +the subdirectory like this: + +.. code-block:: ini + + # This example contains just the necessary options for a src-layout, set up + # the rest of the file as described above. + + [options] + package_dir= + =src + packages=find: + + [options.packages.find] + where=src + Specifying values ================= @@ -2399,36 +2357,44 @@ Metadata The aliases given below are supported for compatibility reasons, but their use is not advised. -============================== ================= ===== -Key Aliases Type -============================== ================= ===== +============================== ================= ================= =============== ===== +Key Aliases Type Minimum Version Notes +============================== ================= ================= =============== ===== name str -version attr:, str +version attr:, file:, str 39.2.0 (1) url home-page str download_url download-url str +project_urls dict 38.3.0 author str author_email author-email str maintainer str maintainer_email maintainer-email str classifiers classifier file:, list-comma -license file:, str +license str description summary file:, str long_description long-description file:, str -long_description_content_type str +long_description_content_type str 38.6.0 keywords list-comma platforms platform list-comma provides list-comma requires list-comma obsoletes list-comma -============================== ================= ===== +============================== ================= ================= =============== ===== +.. note:: + A version loaded using the ``file:`` directive must comply with PEP 440. + It is easy to accidentally put something other than a valid version + string in such a file, so validation is stricter in this case. + +Notes: +1. The `version` file attribute has only been supported since 39.2.0. Options ------- -======================= ===== -Key Type -======================= ===== +======================= =================================== =============== ===== +Key Type Minimum Version Notes +======================= =================================== =============== ===== zip_safe bool setup_requires list-semi install_requires list-semi @@ -2444,21 +2410,25 @@ eager_resources list-comma dependency_links list-comma tests_require list-semi include_package_data bool -packages find:, list-comma +packages find:, find_namespace:, list-comma package_dir dict package_data section exclude_package_data section namespace_packages list-comma py_modules list-comma -======================= ===== +data_files dict 40.6.0 +======================= =================================== =============== ===== .. note:: - **packages** - The ``find:`` directive can be further configured + **packages** - The ``find:`` and ``find_namespace:`` directive can be further configured in a dedicated subsection ``options.packages.find``. This subsection - accepts the same keys as the `setuptools.find` function: + accepts the same keys as the `setuptools.find_packages` and the + `setuptools.find_namespace_packages` function: ``where``, ``include``, and ``exclude``. + **find_namespace directive** - The ``find_namespace:`` directive is supported since Python >=3.3. + Configuration API ================= @@ -2652,8 +2622,8 @@ Adding Support for Revision Control Systems If the files you want to include in the source distribution are tracked using Git, Mercurial or SVN, you can use the following packages to achieve that: -- Git and Mercurial: `setuptools_scm <https://pypi.python.org/pypi/setuptools_scm>`_ -- SVN: `setuptools_svn <https://pypi.python.org/pypi/setuptools_svn>`_ +- Git and Mercurial: `setuptools_scm <https://pypi.org/project/setuptools_scm/>`_ +- SVN: `setuptools_svn <https://pypi.org/project/setuptools_svn/>`_ If you would like to create a plugin for ``setuptools`` to find files tracked by another revision control system, you can do so by adding an entry point to diff --git a/easy_install.py b/easy_install.py index d87e9840..d87e9840 100755..100644 --- a/easy_install.py +++ b/easy_install.py diff --git a/launcher.c b/launcher.c index be69f0c6..be69f0c6 100755..100644 --- a/launcher.c +++ b/launcher.c diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 00000000..ec21e7be --- /dev/null +++ b/netlify.toml @@ -0,0 +1,5 @@ +# Configuration for pull request documentation previews via Netlify + +[build] + publish = "docs/build/html" + command = "pip install tox && tox -e docs" diff --git a/pavement.py b/pavement.py index f85617d4..b5220d10 100644 --- a/pavement.py +++ b/pavement.py @@ -1,7 +1,8 @@ import re +import sys +import subprocess from paver.easy import task, path as Path -import pip def remove_all(paths): @@ -11,22 +12,54 @@ def remove_all(paths): @task def update_vendored(): - vendor = Path('pkg_resources/_vendor') - # pip uninstall doesn't support -t, so do it manually - remove_all(vendor.glob('packaging*')) - remove_all(vendor.glob('six*')) - remove_all(vendor.glob('pyparsing*')) - remove_all(vendor.glob('appdirs*')) + update_pkg_resources() + update_setuptools() + + +def rewrite_packaging(pkg_files, new_root): + """ + Rewrite imports in packaging to redirect to vendored copies. + """ + for file in pkg_files.glob('*.py'): + text = file.text() + text = re.sub(r' (pyparsing|six)', rf' {new_root}.\1', text) + file.write_text(text) + + +def clean(vendor): + """ + Remove all files out of the vendor directory except the meta + data (as pip uninstall doesn't support -t). + """ + remove_all( + path + for path in vendor.glob('*') + if path.basename() != 'vendored.txt' + ) + + +def install(vendor): + clean(vendor) install_args = [ + sys.executable, + '-m', 'pip', 'install', '-r', str(vendor / 'vendored.txt'), '-t', str(vendor), ] - pip.main(install_args) - packaging = vendor / 'packaging' - for file in packaging.glob('*.py'): - text = file.text() - text = re.sub(r' (pyparsing|six)', r' pkg_resources.extern.\1', text) - file.write_text(text) + subprocess.check_call(install_args) remove_all(vendor.glob('*.dist-info')) remove_all(vendor.glob('*.egg-info')) + (vendor / '__init__.py').write_text('') + + +def update_pkg_resources(): + vendor = Path('pkg_resources/_vendor') + install(vendor) + rewrite_packaging(vendor / 'packaging', 'pkg_resources.extern') + + +def update_setuptools(): + vendor = Path('setuptools/_vendor') + install(vendor) + rewrite_packaging(vendor / 'packaging', 'setuptools.extern') diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index 6f1071fb..6ca68daa 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -47,6 +47,11 @@ except ImportError: # Python 3.2 compatibility import imp as _imp +try: + FileExistsError +except NameError: + FileExistsError = OSError + from pkg_resources.extern import six from pkg_resources.extern.six.moves import urllib, map, filter @@ -78,8 +83,11 @@ __import__('pkg_resources.extern.packaging.requirements') __import__('pkg_resources.extern.packaging.markers') -if (3, 0) < sys.version_info < (3, 3): - raise RuntimeError("Python 3.3 or later is required") +__metaclass__ = type + + +if (3, 0) < sys.version_info < (3, 4): + raise RuntimeError("Python 3.4 or later is required") if six.PY2: # Those builtin exceptions are only defined in Python 3 @@ -90,6 +98,21 @@ if six.PY2: # satisfy the linters. require = None working_set = None +add_activation_listener = None +resources_stream = None +cleanup_resources = None +resource_dir = None +resource_stream = None +set_extraction_path = None +resource_isdir = None +resource_string = None +iter_entry_points = None +resource_listdir = None +resource_filename = None +resource_exists = None +_distribution_finders = None +_namespace_handlers = None +_namespace_packages = None class PEP440Warning(RuntimeWarning): @@ -99,118 +122,11 @@ class PEP440Warning(RuntimeWarning): """ -class _SetuptoolsVersionMixin(object): - def __hash__(self): - return super(_SetuptoolsVersionMixin, self).__hash__() - - def __lt__(self, other): - if isinstance(other, tuple): - return tuple(self) < other - else: - return super(_SetuptoolsVersionMixin, self).__lt__(other) - - def __le__(self, other): - if isinstance(other, tuple): - return tuple(self) <= other - else: - return super(_SetuptoolsVersionMixin, self).__le__(other) - - def __eq__(self, other): - if isinstance(other, tuple): - return tuple(self) == other - else: - return super(_SetuptoolsVersionMixin, self).__eq__(other) - - def __ge__(self, other): - if isinstance(other, tuple): - return tuple(self) >= other - else: - return super(_SetuptoolsVersionMixin, self).__ge__(other) - - def __gt__(self, other): - if isinstance(other, tuple): - return tuple(self) > other - else: - return super(_SetuptoolsVersionMixin, self).__gt__(other) - - def __ne__(self, other): - if isinstance(other, tuple): - return tuple(self) != other - else: - return super(_SetuptoolsVersionMixin, self).__ne__(other) - - def __getitem__(self, key): - return tuple(self)[key] - - def __iter__(self): - component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE) - replace = { - 'pre': 'c', - 'preview': 'c', - '-': 'final-', - 'rc': 'c', - 'dev': '@', - }.get - - def _parse_version_parts(s): - for part in component_re.split(s): - part = replace(part, part) - if not part or part == '.': - continue - if part[:1] in '0123456789': - # pad for numeric comparison - yield part.zfill(8) - else: - yield '*' + part - - # ensure that alpha/beta/candidate are before final - yield '*final' - - def old_parse_version(s): - parts = [] - for part in _parse_version_parts(s.lower()): - if part.startswith('*'): - # remove '-' before a prerelease tag - if part < '*final': - while parts and parts[-1] == '*final-': - parts.pop() - # remove trailing zeros from each series of numeric parts - while parts and parts[-1] == '00000000': - parts.pop() - parts.append(part) - return tuple(parts) - - # Warn for use of this function - warnings.warn( - "You have iterated over the result of " - "pkg_resources.parse_version. This is a legacy behavior which is " - "inconsistent with the new version class introduced in setuptools " - "8.0. In most cases, conversion to a tuple is unnecessary. For " - "comparison of versions, sort the Version instances directly. If " - "you have another use case requiring the tuple, please file a " - "bug with the setuptools project describing that need.", - RuntimeWarning, - stacklevel=1, - ) - - for part in old_parse_version(str(self)): - yield part - - -class SetuptoolsVersion(_SetuptoolsVersionMixin, packaging.version.Version): - pass - - -class SetuptoolsLegacyVersion(_SetuptoolsVersionMixin, - packaging.version.LegacyVersion): - pass - - def parse_version(v): try: - return SetuptoolsVersion(v) + return packaging.version.Version(v) except packaging.version.InvalidVersion: - return SetuptoolsLegacyVersion(v) + return packaging.version.LegacyVersion(v) _state_vars = {} @@ -322,6 +238,9 @@ __all__ = [ 'register_finder', 'register_namespace_handler', 'register_loader_type', 'fixup_namespace_packages', 'get_importer', + # Warnings + 'PkgResourcesDeprecationWarning', + # Deprecated/backward compatibility only 'run_main', 'AvailableDistributions', ] @@ -469,11 +388,7 @@ def get_build_platform(): XXX Currently this is the same as ``distutils.util.get_platform()``, but it needs some hacks for Linux and Mac OS X. """ - try: - # Python 2.7 or >=3.2 - from sysconfig import get_platform - except ImportError: - from distutils.util import get_platform + from sysconfig import get_platform plat = get_platform() if sys.platform == "darwin" and not plat.startswith('macosx-'): @@ -633,7 +548,7 @@ class IResourceProvider(IMetadataProvider): """List of resource names in the directory (like ``os.listdir()``)""" -class WorkingSet(object): +class WorkingSet: """A collection of active distributions on sys.path (or a similar list)""" def __init__(self, entries=None): @@ -733,13 +648,12 @@ class WorkingSet(object): distributions in the working set, otherwise only ones matching both `group` and `name` are yielded (in distribution order). """ - for dist in self: - entries = dist.get_entry_map(group) - if name is None: - for ep in entries.values(): - yield ep - elif name in entries: - yield entries[name] + return ( + entry + for dist in self + for entry in dist.get_entry_map(group).values() + if name is None or name == entry.name + ) def run_script(self, requires, script_name): """Locate distribution for `requires` and run `script_name` script""" @@ -1040,7 +954,7 @@ class _ReqExtras(dict): return not req.marker or any(extra_evals) -class Environment(object): +class Environment: """Searchable snapshot of distributions on a search path""" def __init__( @@ -1055,7 +969,7 @@ class Environment(object): `platform` is an optional string specifying the name of the platform that platform-specific distributions must be compatible with. If unspecified, it defaults to the current platform. `python` is an - optional string naming the desired version of Python (e.g. ``'3.3'``); + optional string naming the desired version of Python (e.g. ``'3.6'``); it defaults to the current version. You may explicitly set `platform` (and/or `python`) to ``None`` if you @@ -1074,9 +988,12 @@ class Environment(object): requirements specified when this environment was created, or False is returned. """ - return (self.python is None or dist.py_version is None - or dist.py_version == self.python) \ - and compatible_platforms(dist.platform, self.platform) + py_compat = ( + self.python is None + or dist.py_version is None + or dist.py_version == self.python + ) + return py_compat and compatible_platforms(dist.platform, self.platform) def remove(self, dist): """Remove `dist` from the environment""" @@ -1289,7 +1206,7 @@ class ResourceManager: target_path = os.path.join(extract_path, archive_name + '-tmp', *names) try: _bypass_ensure_directory(target_path) - except: + except Exception: self.extraction_error() self._warn_unsafe_extraction_path(extract_path) @@ -1513,7 +1430,10 @@ class NullProvider: def run_script(self, script_name, namespace): script = 'scripts/' + script_name if not self.has_metadata(script): - raise ResolutionError("No script named %r" % script_name) + raise ResolutionError( + "Script {script!r} not found in metadata at {self.egg_info!r}" + .format(**locals()), + ) script_text = self.get_metadata(script).replace('\r\n', '\n') script_text = script_text.replace('\r', '\n') script_filename = self._fn(self.egg_info, script) @@ -1604,12 +1524,10 @@ class DefaultProvider(EggProvider): @classmethod def _register(cls): - loader_cls = getattr( - importlib_machinery, - 'SourceFileLoader', - type(None), - ) - register_loader_type(loader_cls, cls) + loader_names = 'SourceFileLoader', 'SourcelessFileLoader', + for name in loader_names: + loader_cls = getattr(importlib_machinery, name, type(None)) + register_loader_type(loader_cls, cls) DefaultProvider._register() @@ -1618,11 +1536,16 @@ DefaultProvider._register() class EmptyProvider(NullProvider): """Provider that returns nothing for all requests""" - _isdir = _has = lambda self, path: False - _get = lambda self, path: '' - _listdir = lambda self, path: [] module_path = None + _isdir = _has = lambda self, path: False + + def _get(self, path): + return '' + + def _listdir(self, path): + return [] + def __init__(self): pass @@ -1644,7 +1567,7 @@ class ZipManifests(dict): Use a platform-specific path separator (os.sep) for the path keys for compatibility with pypy on Windows. """ - with ContextualZipFile(path) as zfile: + with zipfile.ZipFile(path) as zfile: items = ( ( name.replace('/', os.sep), @@ -1677,26 +1600,6 @@ class MemoizedZipManifests(ZipManifests): return self[path].manifest -class ContextualZipFile(zipfile.ZipFile): - """ - Supplement ZipFile class to support context manager for Python 2.6 - """ - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def __new__(cls, *args, **kwargs): - """ - Construct a ZipFile or ContextualZipFile as appropriate - """ - if hasattr(zipfile.ZipFile, '__exit__'): - return zipfile.ZipFile(*args, **kwargs) - return super(ContextualZipFile, cls).__new__(cls) - - class ZipProvider(EggProvider): """Resource support for zips and eggs""" @@ -1710,6 +1613,9 @@ class ZipProvider(EggProvider): def _zipinfo_name(self, fspath): # Convert a virtual filename (full path to file) into a zipfile subpath # usable with the zipimport directory cache for our target archive + fspath = fspath.rstrip(os.sep) + if fspath == self.loader.archive: + return '' if fspath.startswith(self.zip_pre): return fspath[len(self.zip_pre):] raise AssertionError( @@ -1897,7 +1803,7 @@ class FileMetadata(EmptyProvider): return metadata def _warn_on_replacement(self, metadata): - # Python 2.6 and 3.2 compat for: replacement_char = '�' + # Python 2.7 compat for: replacement_char = '�' replacement_char = b'\xef\xbf\xbd'.decode('utf-8') if replacement_char in metadata: tmpl = "{self.path} could not be properly decoded in UTF-8" @@ -2191,7 +2097,12 @@ def _handle_ns(packageName, path_item): importer = get_importer(path_item) if importer is None: return None - loader = importer.find_module(packageName) + + # capture warnings due to #1111 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + loader = importer.find_module(packageName) + if loader is None: return None module = sys.modules.get(packageName) @@ -2236,12 +2147,13 @@ def _rebuild_mod_path(orig_path, package_name, module): parts = path_parts[:-module_parts] return safe_sys_path_index(_normalize_cached(os.sep.join(parts))) - if not isinstance(orig_path, list): - # Is this behavior useful when module.__path__ is not a list? - return + new_path = sorted(orig_path, key=position_in_sys_path) + new_path = [_normalize_cached(p) for p in new_path] - orig_path.sort(key=position_in_sys_path) - module.__path__[:] = [_normalize_cached(p) for p in orig_path] + if isinstance(module.__path__, list): + module.__path__[:] = new_path + else: + module.__path__ = new_path def declare_namespace(packageName): @@ -2252,9 +2164,10 @@ def declare_namespace(packageName): if packageName in _namespace_packages: return - path, parent = sys.path, None - if '.' in packageName: - parent = '.'.join(packageName.split('.')[:-1]) + path = sys.path + parent, _, _ = packageName.rpartition('.') + + if parent: declare_namespace(parent) if parent not in _namespace_packages: __import__(parent) @@ -2265,7 +2178,7 @@ def declare_namespace(packageName): # Track what packages are namespaces, so when new path items are added, # they can be updated - _namespace_packages.setdefault(parent, []).append(packageName) + _namespace_packages.setdefault(parent or None, []).append(packageName) _namespace_packages.setdefault(packageName, []) for path_item in path: @@ -2318,7 +2231,18 @@ register_namespace_handler(object, null_ns_handler) def normalize_path(filename): """Normalize a file/dir name for comparison purposes""" - return os.path.normcase(os.path.realpath(filename)) + return os.path.normcase(os.path.realpath(os.path.normpath(_cygwin_patch(filename)))) + + +def _cygwin_patch(filename): # pragma: nocover + """ + Contrary to POSIX 2008, on Cygwin, getcwd (3) contains + symlink components. Using + os.path.abspath() works around this limitation. A fix in os.getcwd() + would probably better, in Cygwin even more so, except + that this seems to be by design... + """ + return os.path.abspath(filename) if sys.platform == 'cygwin' else filename def _normalize_cached(filename, _cache={}): @@ -2383,7 +2307,7 @@ EGG_NAME = re.compile( ).match -class EntryPoint(object): +class EntryPoint: """Object representing an advertised importable object""" def __init__(self, name, module_name, attrs=(), extras=(), dist=None): @@ -2414,7 +2338,7 @@ class EntryPoint(object): warnings.warn( "Parameters to load are deprecated. Call .resolve and " ".require separately.", - DeprecationWarning, + PkgResourcesDeprecationWarning, stacklevel=2, ) if require: @@ -2529,14 +2453,15 @@ def _version_from_file(lines): Given an iterable of lines from a Metadata file, return the value of the Version field, if present, or None otherwise. """ - is_version_line = lambda line: line.lower().startswith('version:') + def is_version_line(line): + return line.lower().startswith('version:') version_lines = filter(is_version_line, lines) line = next(iter(version_lines), '') _, _, value = line.partition(':') return safe_version(value.strip()) or None -class Distribution(object): +class Distribution: """Wrap an actual or potential sys.path entry w/metadata""" PKG_INFO = 'PKG-INFO' @@ -2666,23 +2591,44 @@ class Distribution(object): @property def _dep_map(self): + """ + A map of extra to its list of (direct) requirements + for this distribution, including the null extra. + """ try: return self.__dep_map except AttributeError: - dm = self.__dep_map = {None: []} - for name in 'requires.txt', 'depends.txt': - for extra, reqs in split_sections(self._get_metadata(name)): - if extra: - if ':' in extra: - extra, marker = extra.split(':', 1) - if invalid_marker(marker): - # XXX warn - reqs = [] - elif not evaluate_marker(marker): - reqs = [] - extra = safe_extra(extra) or None - dm.setdefault(extra, []).extend(parse_requirements(reqs)) - return dm + self.__dep_map = self._filter_extras(self._build_dep_map()) + return self.__dep_map + + @staticmethod + def _filter_extras(dm): + """ + Given a mapping of extras to dependencies, strip off + environment markers and filter out any dependencies + not matching the markers. + """ + for extra in list(filter(None, dm)): + new_extra = extra + reqs = dm.pop(extra) + new_extra, _, marker = extra.partition(':') + fails_marker = marker and ( + invalid_marker(marker) + or not evaluate_marker(marker) + ) + if fails_marker: + reqs = [] + new_extra = safe_extra(new_extra) or None + + dm.setdefault(new_extra, []).extend(reqs) + return dm + + def _build_dep_map(self): + dm = {} + for name in 'requires.txt', 'depends.txt': + for extra, reqs in split_sections(self._get_metadata(name)): + dm.setdefault(extra, []).extend(parse_requirements(reqs)) + return dm def requires(self, extras=()): """List of Requirements needed for this distro if `extras` are used""" @@ -2745,6 +2691,19 @@ class Distribution(object): raise AttributeError(attr) return getattr(self._provider, attr) + def __dir__(self): + return list( + set(super(Distribution, self).__dir__()) + | set( + attr for attr in self._provider.__dir__() + if not attr.startswith('_') + ) + ) + + if not hasattr(object, '__dir__'): + # python 2.7 not supported + del __dir__ + @classmethod def from_filename(cls, filename, metadata=None, **kw): return cls.from_location( @@ -3004,7 +2963,10 @@ def parse_requirements(strs): # If there is a line continuation, drop it, and append the next line. if line.endswith('\\'): line = line[:-2].strip() - line += next(lines) + try: + line += next(lines) + except StopIteration: + return yield Requirement(line) @@ -3093,7 +3055,10 @@ def _bypass_ensure_directory(path): dirname, filename = split(path) if dirname and filename and not isdir(dirname): _bypass_ensure_directory(dirname) - mkdir(dirname, 0o755) + try: + mkdir(dirname, 0o755) + except FileExistsError: + pass def split_sections(s): @@ -3196,3 +3161,11 @@ def _initialize_master_working_set(): # match order list(map(working_set.add_entry, sys.path)) globals().update(locals()) + +class PkgResourcesDeprecationWarning(Warning): + """ + Base class for warning about deprecations in ``pkg_resources`` + + This class is not derived from ``DeprecationWarning``, and as such is + visible by default. + """ diff --git a/pkg_resources/_vendor/appdirs.py b/pkg_resources/_vendor/appdirs.py index f4dba095..ae67001a 100644 --- a/pkg_resources/_vendor/appdirs.py +++ b/pkg_resources/_vendor/appdirs.py @@ -13,7 +13,7 @@ See <http://github.com/ActiveState/appdirs> for details and usage. # - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html # - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html -__version_info__ = (1, 4, 0) +__version_info__ = (1, 4, 3) __version__ = '.'.join(map(str, __version_info__)) @@ -98,7 +98,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): - """Return full path to the user-shared data dir for this application. + r"""Return full path to the user-shared data dir for this application. "appname" is the name of application. If None, just the system directory is returned. @@ -117,7 +117,7 @@ def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): returned, or '/usr/local/share/<AppName>', if XDG_DATA_DIRS is not set - Typical user data directories are: + Typical site data directories are: Mac OS X: /Library/Application Support/<AppName> Unix: /usr/local/share/<AppName> or /usr/share/<AppName> Win XP: C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName> @@ -184,13 +184,13 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> for a discussion of issues. - Typical user data directories are: + Typical user config directories are: Mac OS X: same as user_data_dir Unix: ~/.config/<AppName> # or in $XDG_CONFIG_HOME, if defined Win *: same as user_data_dir For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME. - That means, by deafult "~/.config/<AppName>". + That means, by default "~/.config/<AppName>". """ if system in ["win32", "darwin"]: path = user_data_dir(appname, appauthor, None, roaming) @@ -204,7 +204,7 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): - """Return full path to the user-shared data dir for this application. + r"""Return full path to the user-shared data dir for this application. "appname" is the name of application. If None, just the system directory is returned. @@ -222,7 +222,7 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) returned. By default, the first item from XDG_CONFIG_DIRS is returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set - Typical user data directories are: + Typical site config directories are: Mac OS X: same as site_data_dir Unix: /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in $XDG_CONFIG_DIRS @@ -311,6 +311,48 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): return path +def user_state_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific state dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> + for a discussion of issues. + + Typical user state directories are: + Mac OS X: same as user_data_dir + Unix: ~/.local/state/<AppName> # or in $XDG_STATE_HOME, if defined + Win *: same as user_data_dir + + For Unix, we follow this Debian proposal <https://wiki.debian.org/XDGBaseDirectorySpecification#state> + to extend the XDG spec and support $XDG_STATE_HOME. + + That means, by default "~/.local/state/<AppName>". + """ + if system in ["win32", "darwin"]: + path = user_data_dir(appname, appauthor, None, roaming) + else: + path = os.getenv('XDG_STATE_HOME', os.path.expanduser("~/.local/state")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): r"""Return full path to the user-specific log dir for this application. @@ -329,7 +371,7 @@ def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): "Logs" to the base app data dir for Windows, and "log" to the base cache dir for Unix. See discussion below. - Typical user cache directories are: + Typical user log directories are: Mac OS X: ~/Library/Logs/<AppName> Unix: ~/.cache/<AppName>/log # or under $XDG_CACHE_HOME if defined Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs @@ -364,8 +406,8 @@ def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): class AppDirs(object): """Convenience wrapper for getting application dirs.""" - def __init__(self, appname, appauthor=None, version=None, roaming=False, - multipath=False): + def __init__(self, appname=None, appauthor=None, version=None, + roaming=False, multipath=False): self.appname = appname self.appauthor = appauthor self.version = version @@ -398,6 +440,11 @@ class AppDirs(object): version=self.version) @property + def user_state_dir(self): + return user_state_dir(self.appname, self.appauthor, + version=self.version) + + @property def user_log_dir(self): return user_log_dir(self.appname, self.appauthor, version=self.version) @@ -410,7 +457,10 @@ def _get_win_folder_from_registry(csidl_name): registry for this guarantees us the correct answer for all CSIDL_* names. """ - import _winreg + if PY3: + import winreg as _winreg + else: + import _winreg shell_folder_name = { "CSIDL_APPDATA": "AppData", @@ -500,7 +550,7 @@ def _get_win_folder_with_jna(csidl_name): if has_high_char: buf = array.zeros('c', buf_size) kernel = win32.Kernel32.INSTANCE - if kernal.GetShortPathName(dir, buf, buf_size): + if kernel.GetShortPathName(dir, buf, buf_size): dir = jna.Native.toString(buf.tostring()).rstrip("\0") return dir @@ -527,9 +577,15 @@ if __name__ == "__main__": appname = "MyApp" appauthor = "MyCompany" - props = ("user_data_dir", "site_data_dir", - "user_config_dir", "site_config_dir", - "user_cache_dir", "user_log_dir") + props = ("user_data_dir", + "user_config_dir", + "user_cache_dir", + "user_state_dir", + "user_log_dir", + "site_data_dir", + "site_config_dir") + + print("-- app dirs %s --" % __version__) print("-- app dirs (with optional 'version')") dirs = AppDirs(appname, appauthor, version="1.0") diff --git a/pkg_resources/_vendor/pyparsing.py b/pkg_resources/_vendor/pyparsing.py index a2122435..cf75e1e5 100644 --- a/pkg_resources/_vendor/pyparsing.py +++ b/pkg_resources/_vendor/pyparsing.py @@ -1,6 +1,6 @@ # module pyparsing.py
#
-# Copyright (c) 2003-2016 Paul T. McGuire
+# Copyright (c) 2003-2018 Paul T. McGuire
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -25,6 +25,7 @@ __doc__ = \
"""
pyparsing module - Classes and methods to define and execute parsing grammars
+=============================================================================
The pyparsing module is an alternative approach to creating and executing simple grammars,
vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you
@@ -58,10 +59,23 @@ The pyparsing module handles some of the problems that are typically vexing when - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.)
- quoted strings
- embedded comments
+
+
+Getting Started -
+-----------------
+Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing
+classes inherit from. Use the docstrings for examples of how to:
+ - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes
+ - construct character word-group expressions using the L{Word} class
+ - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes
+ - use L{'+'<And>}, L{'|'<MatchFirst>}, L{'^'<Or>}, and L{'&'<Each>} operators to combine simple expressions into more complex ones
+ - associate names with your parsed results using L{ParserElement.setResultsName}
+ - find some helpful expression short-cuts like L{delimitedList} and L{oneOf}
+ - find more useful common expressions in the L{pyparsing_common} namespace class
"""
-__version__ = "2.1.10"
-__versionTime__ = "07 Oct 2016 01:31 UTC"
+__version__ = "2.2.1"
+__versionTime__ = "18 Sep 2018 00:49 UTC"
__author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"
import string
@@ -83,6 +97,15 @@ except ImportError: from threading import RLock
try:
+ # Python 3
+ from collections.abc import Iterable
+ from collections.abc import MutableMapping
+except ImportError:
+ # Python 2.7
+ from collections import Iterable
+ from collections import MutableMapping
+
+try:
from collections import OrderedDict as _OrderedDict
except ImportError:
try:
@@ -144,7 +167,7 @@ else: except UnicodeEncodeError:
# Else encode it
ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
- xmlcharref = Regex('&#\d+;')
+ xmlcharref = Regex(r'&#\d+;')
xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
return xmlcharref.transformString(ret)
@@ -809,7 +832,7 @@ class ParseResults(object): return None
def getName(self):
- """
+ r"""
Returns the results name for this token expression. Useful when several
different expressions might match at a particular location.
@@ -940,7 +963,7 @@ class ParseResults(object): def __dir__(self):
return (dir(type(self)) + list(self.keys()))
-collections.MutableMapping.register(ParseResults)
+MutableMapping.register(ParseResults)
def col (loc,strg):
"""Returns current column within a string, counting newlines as line separators.
@@ -1025,11 +1048,11 @@ def _trim_arity(func, maxargs=2): # special handling for Python 3.5.0 - extra deep call stack by 1
offset = -3 if system_version == (3,5,0) else -2
frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
- return [(frame_summary.filename, frame_summary.lineno)]
+ return [frame_summary[:2]]
def extract_tb(tb, limit=0):
frames = traceback.extract_tb(tb, limit=limit)
frame_summary = frames[-1]
- return [(frame_summary.filename, frame_summary.lineno)]
+ return [frame_summary[:2]]
else:
extract_stack = traceback.extract_stack
extract_tb = traceback.extract_tb
@@ -1226,7 +1249,7 @@ class ParserElement(object): def setParseAction( self, *fns, **kwargs ):
"""
- Define action to perform when successfully matching parse element definition.
+ Define one or more actions to perform when successfully matching parse element definition.
Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
- s = the original string being parsed (see note below)
@@ -1264,7 +1287,7 @@ class ParserElement(object): def addParseAction( self, *fns, **kwargs ):
"""
- Add parse action to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}.
+ Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}.
See examples in L{I{copy}<copy>}.
"""
@@ -1374,7 +1397,7 @@ class ParserElement(object): else:
preloc = loc
tokensStart = preloc
- if self.mayIndexError or loc >= len(instring):
+ if self.mayIndexError or preloc >= len(instring):
try:
loc,tokens = self.parseImpl( instring, preloc, doActions )
except IndexError:
@@ -1408,7 +1431,6 @@ class ParserElement(object): self.resultsName,
asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
modal=self.modalResults )
-
if debugging:
#~ print ("Matched",self,"->",retTokens.asList())
if (self.debugActions[1] ):
@@ -1443,10 +1465,14 @@ class ParserElement(object): def clear(self):
cache.clear()
+
+ def cache_len(self):
+ return len(cache)
self.get = types.MethodType(get, self)
self.set = types.MethodType(set, self)
self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
if _OrderedDict is not None:
class _FifoCache(object):
@@ -1460,15 +1486,22 @@ class ParserElement(object): def set(self, key, value):
cache[key] = value
- if len(cache) > size:
- cache.popitem(False)
+ while len(cache) > size:
+ try:
+ cache.popitem(False)
+ except KeyError:
+ pass
def clear(self):
cache.clear()
+ def cache_len(self):
+ return len(cache)
+
self.get = types.MethodType(get, self)
self.set = types.MethodType(set, self)
self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
else:
class _FifoCache(object):
@@ -1483,7 +1516,7 @@ class ParserElement(object): def set(self, key, value):
cache[key] = value
- if len(cache) > size:
+ while len(key_fifo) > size:
cache.pop(key_fifo.popleft(), None)
key_fifo.append(key)
@@ -1491,9 +1524,13 @@ class ParserElement(object): cache.clear()
key_fifo.clear()
+ def cache_len(self):
+ return len(cache)
+
self.get = types.MethodType(get, self)
self.set = types.MethodType(set, self)
self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
# argument cache for optimizing repeated calls when backtracking through recursive expressions
packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail
@@ -1743,8 +1780,12 @@ class ParserElement(object): cap_word = Word(alphas.upper(), alphas.lower())
print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))
+
+ # the sum() builtin can be used to merge results into a single ParseResults object
+ print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")))
prints::
- ['More', 'Iron', 'Lead', 'Gold', 'I']
+ [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']]
+ ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity']
"""
try:
return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
@@ -1819,7 +1860,7 @@ class ParserElement(object): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
SyntaxWarning, stacklevel=2)
return None
- return And( [ self, And._ErrorStop(), other ] )
+ return self + And._ErrorStop() + other
def __rsub__(self, other ):
"""
@@ -2722,7 +2763,7 @@ class Word(Token): class Regex(Token):
- """
+ r"""
Token for matching strings that match a given regular expression.
Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
If the given regex contains named groups (defined using C{(?P<name>...)}), these will be preserved as
@@ -2911,7 +2952,7 @@ class QuotedString(Token): # replace escaped characters
if self.escChar:
- ret = re.sub(self.escCharReplacePattern,"\g<1>",ret)
+ ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret)
# replace escaped quotes
if self.escQuote:
@@ -3223,7 +3264,7 @@ class ParseExpression(ParserElement): if isinstance( exprs, basestring ):
self.exprs = [ ParserElement._literalStringClass( exprs ) ]
- elif isinstance( exprs, collections.Iterable ):
+ elif isinstance( exprs, Iterable ):
exprs = list(exprs)
# if sequence of strings provided, wrap with Literal
if all(isinstance(expr, basestring) for expr in exprs):
@@ -4374,7 +4415,7 @@ def traceParseAction(f): @traceParseAction
def remove_duplicate_chars(tokens):
- return ''.join(sorted(set(''.join(tokens)))
+ return ''.join(sorted(set(''.join(tokens))))
wds = OneOrMore(wd).setParseAction(remove_duplicate_chars)
print(wds.parseString("slkdjs sld sldd sdlf sdljf"))
@@ -4564,7 +4605,7 @@ def oneOf( strs, caseless=False, useRegex=True ): symbols = []
if isinstance(strs,basestring):
symbols = strs.split()
- elif isinstance(strs, collections.Iterable):
+ elif isinstance(strs, Iterable):
symbols = list(strs)
else:
warnings.warn("Invalid argument to oneOf, expected string or iterable",
@@ -4715,7 +4756,7 @@ stringEnd = StringEnd().setName("stringEnd") _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
-_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE)
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1)
_charRange = Group(_singleChar + Suppress("-") + _singleChar)
_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
@@ -5020,7 +5061,9 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
- parseAction is the parse action to be associated with
expressions matching this operator expression (the
- parse action tuple member may be omitted)
+ parse action tuple member may be omitted); if the parse action
+ is passed a tuple or list of functions, this is equivalent to
+ calling C{setParseAction(*fn)} (L{ParserElement.setParseAction})
- lpar - expression for matching left-parentheses (default=C{Suppress('(')})
- rpar - expression for matching right-parentheses (default=C{Suppress(')')})
@@ -5093,7 +5136,10 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): else:
raise ValueError("operator must indicate right or left associativity")
if pa:
- matchExpr.setParseAction( pa )
+ if isinstance(pa, (tuple, list)):
+ matchExpr.setParseAction(*pa)
+ else:
+ matchExpr.setParseAction(pa)
thisExpr <<= ( matchExpr.setName(termName) | lastExpr )
lastExpr = thisExpr
ret <<= lastExpr
diff --git a/pkg_resources/_vendor/vendored.txt b/pkg_resources/_vendor/vendored.txt index 9a94c5bc..7f4f4087 100644 --- a/pkg_resources/_vendor/vendored.txt +++ b/pkg_resources/_vendor/vendored.txt @@ -1,4 +1,4 @@ packaging==16.8 -pyparsing==2.1.10 +pyparsing==2.2.1 six==1.10.0 -appdirs==1.4.0 +appdirs==1.4.3 diff --git a/pkg_resources/api_tests.txt b/pkg_resources/api_tests.txt index 4fbd3d23..0a75170e 100644 --- a/pkg_resources/api_tests.txt +++ b/pkg_resources/api_tests.txt @@ -385,10 +385,10 @@ Environment Markers >>> em("sys_platform=='win32'") == (sys.platform=='win32') True - >>> em("python_version >= '2.6'") + >>> em("python_version >= '2.7'") True - >>> em("python_version > '2.5'") + >>> em("python_version > '2.6'") True >>> im("implementation_name=='cpython'") diff --git a/pkg_resources/extern/__init__.py b/pkg_resources/extern/__init__.py index b4156fec..c1eb9e99 100644 --- a/pkg_resources/extern/__init__.py +++ b/pkg_resources/extern/__init__.py @@ -48,7 +48,7 @@ class VendorImporter: # on later Python versions to cause relative imports # in the vendor package to resolve the same modules # as those going through this importer. - if sys.version_info > (3, 3): + if prefix and sys.version_info > (3, 3): del sys.modules[extant] return mod except ImportError: diff --git a/pkg_resources/py31compat.py b/pkg_resources/py31compat.py index 331a51bb..a381c424 100644 --- a/pkg_resources/py31compat.py +++ b/pkg_resources/py31compat.py @@ -2,6 +2,8 @@ import os import errno import sys +from .extern import six + def _makedirs_31(path, exist_ok=False): try: @@ -15,8 +17,7 @@ def _makedirs_31(path, exist_ok=False): # and exists_ok considerations are disentangled. # See https://github.com/pypa/setuptools/pull/1083#issuecomment-315168663 needs_makedirs = ( - sys.version_info < (3, 2, 5) or - (3, 3) <= sys.version_info < (3, 3, 6) or + six.PY2 or (3, 4) <= sys.version_info < (3, 4, 1) ) makedirs = _makedirs_31 if needs_makedirs else os.makedirs diff --git a/pkg_resources/tests/test_find_distributions.py b/pkg_resources/tests/test_find_distributions.py index 97999b33..d735c590 100644 --- a/pkg_resources/tests/test_find_distributions.py +++ b/pkg_resources/tests/test_find_distributions.py @@ -13,6 +13,7 @@ setuptools.setup( ) """.lstrip() + class TestFindDistributions: @pytest.fixture diff --git a/pkg_resources/tests/test_pkg_resources.py b/pkg_resources/tests/test_pkg_resources.py index c6a7ac97..416f9aff 100644 --- a/pkg_resources/tests/test_pkg_resources.py +++ b/pkg_resources/tests/test_pkg_resources.py @@ -12,16 +12,19 @@ import stat import distutils.dist import distutils.command.install_egg_info +try: + from unittest import mock +except ImportError: + import mock + from pkg_resources.extern.six.moves import map +from pkg_resources.extern.six import text_type, string_types import pytest import pkg_resources -try: - unicode -except NameError: - unicode = str +__metaclass__ = type def timestamp(dt): @@ -35,7 +38,7 @@ def timestamp(dt): return time.mktime(dt.timetuple()) -class EggRemover(unicode): +class EggRemover(text_type): def __call__(self): if self in sys.path: sys.path.remove(self) @@ -43,7 +46,7 @@ class EggRemover(unicode): os.remove(self) -class TestZipProvider(object): +class TestZipProvider: finalizers = [] ref_time = datetime.datetime(2013, 5, 12, 13, 25, 0) @@ -62,10 +65,21 @@ class TestZipProvider(object): zip_info.filename = 'data.dat' zip_info.date_time = cls.ref_time.timetuple() zip_egg.writestr(zip_info, 'hello, world!') + zip_info = zipfile.ZipInfo() + zip_info.filename = 'subdir/mod2.py' + zip_info.date_time = cls.ref_time.timetuple() + zip_egg.writestr(zip_info, 'x = 6\n') + zip_info = zipfile.ZipInfo() + zip_info.filename = 'subdir/data2.dat' + zip_info.date_time = cls.ref_time.timetuple() + zip_egg.writestr(zip_info, 'goodbye, world!') zip_egg.close() egg.close() sys.path.append(egg.name) + subdir = os.path.join(egg.name, 'subdir') + sys.path.append(subdir) + cls.finalizers.append(EggRemover(subdir)) cls.finalizers.append(EggRemover(egg.name)) @classmethod @@ -73,6 +87,30 @@ class TestZipProvider(object): for finalizer in cls.finalizers: finalizer() + def test_resource_listdir(self): + import mod + zp = pkg_resources.ZipProvider(mod) + + expected_root = ['data.dat', 'mod.py', 'subdir'] + assert sorted(zp.resource_listdir('')) == expected_root + assert sorted(zp.resource_listdir('/')) == expected_root + + expected_subdir = ['data2.dat', 'mod2.py'] + assert sorted(zp.resource_listdir('subdir')) == expected_subdir + assert sorted(zp.resource_listdir('subdir/')) == expected_subdir + + assert zp.resource_listdir('nonexistent') == [] + assert zp.resource_listdir('nonexistent/') == [] + + import mod2 + zp2 = pkg_resources.ZipProvider(mod2) + + assert sorted(zp2.resource_listdir('')) == expected_subdir + assert sorted(zp2.resource_listdir('/')) == expected_subdir + + assert zp2.resource_listdir('subdir') == [] + assert zp2.resource_listdir('subdir/') == [] + def test_resource_filename_rewrites_on_change(self): """ If a previous call to get_resource_filename has saved the file, but @@ -97,16 +135,42 @@ class TestZipProvider(object): manager.cleanup_resources() -class TestResourceManager(object): +class TestResourceManager: def test_get_cache_path(self): mgr = pkg_resources.ResourceManager() path = mgr.get_cache_path('foo') type_ = str(type(path)) message = "Unexpected type from get_cache_path: " + type_ - assert isinstance(path, (unicode, str)), message + assert isinstance(path, string_types), message + + def test_get_cache_path_race(self, tmpdir): + # Patch to os.path.isdir to create a race condition + def patched_isdir(dirname, unpatched_isdir=pkg_resources.isdir): + patched_isdir.dirnames.append(dirname) + + was_dir = unpatched_isdir(dirname) + if not was_dir: + os.makedirs(dirname) + return was_dir + + patched_isdir.dirnames = [] + + # Get a cache path with a "race condition" + mgr = pkg_resources.ResourceManager() + mgr.set_extraction_path(str(tmpdir)) + + archive_name = os.sep.join(('foo', 'bar', 'baz')) + with mock.patch.object(pkg_resources, 'isdir', new=patched_isdir): + mgr.get_cache_path(archive_name) + # Because this test relies on the implementation details of this + # function, these assertions are a sentinel to ensure that the + # test suite will not fail silently if the implementation changes. + called_dirnames = patched_isdir.dirnames + assert len(called_dirnames) == 2 + assert called_dirnames[0].split(os.sep)[-2:] == ['foo', 'bar'] + assert called_dirnames[1].split(os.sep)[-1:] == ['foo'] -class TestIndependence: """ Tests to ensure that pkg_resources runs independently from setuptools. """ @@ -119,14 +183,16 @@ class TestIndependence: lines = ( 'import pkg_resources', 'import sys', - 'assert "setuptools" not in sys.modules, ' - '"setuptools was imported"', + ( + 'assert "setuptools" not in sys.modules, ' + '"setuptools was imported"' + ), ) cmd = [sys.executable, '-c', '; '.join(lines)] subprocess.check_call(cmd) -class TestDeepVersionLookupDistutils(object): +class TestDeepVersionLookupDistutils: @pytest.fixture def env(self, tmpdir): """ @@ -170,3 +236,56 @@ class TestDeepVersionLookupDistutils(object): req = pkg_resources.Requirement.parse('foo>=1.9') dist = pkg_resources.WorkingSet([env.paths['lib']]).find(req) assert dist.version == version + + @pytest.mark.parametrize( + 'unnormalized, normalized', + [ + ('foo', 'foo'), + ('foo/', 'foo'), + ('foo/bar', 'foo/bar'), + ('foo/bar/', 'foo/bar'), + ], + ) + def test_normalize_path_trailing_sep(self, unnormalized, normalized): + """Ensure the trailing slash is cleaned for path comparison. + + See pypa/setuptools#1519. + """ + result_from_unnormalized = pkg_resources.normalize_path(unnormalized) + result_from_normalized = pkg_resources.normalize_path(normalized) + assert result_from_unnormalized == result_from_normalized + + @pytest.mark.skipif( + os.path.normcase('A') != os.path.normcase('a'), + reason='Testing case-insensitive filesystems.', + ) + @pytest.mark.parametrize( + 'unnormalized, normalized', + [ + ('MiXeD/CasE', 'mixed/case'), + ], + ) + def test_normalize_path_normcase(self, unnormalized, normalized): + """Ensure mixed case is normalized on case-insensitive filesystems. + """ + result_from_unnormalized = pkg_resources.normalize_path(unnormalized) + result_from_normalized = pkg_resources.normalize_path(normalized) + assert result_from_unnormalized == result_from_normalized + + @pytest.mark.skipif( + os.path.sep != '\\', + reason='Testing systems using backslashes as path separators.', + ) + @pytest.mark.parametrize( + 'unnormalized, expected', + [ + ('forward/slash', 'forward\\slash'), + ('forward/slash/', 'forward\\slash'), + ('backward\\slash\\', 'backward\\slash'), + ], + ) + def test_normalize_path_backslash_sep(self, unnormalized, expected): + """Ensure path seps are cleaned on backslash path sep systems. + """ + result = pkg_resources.normalize_path(unnormalized) + assert result.endswith(expected) diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index 0b05343c..86afcf74 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -4,6 +4,7 @@ import os import sys import string import platform +import itertools from pkg_resources.extern.six.moves import map @@ -11,9 +12,18 @@ import pytest from pkg_resources.extern import packaging import pkg_resources -from pkg_resources import (parse_requirements, VersionConflict, parse_version, +from pkg_resources import ( + parse_requirements, VersionConflict, parse_version, Distribution, EntryPoint, Requirement, safe_version, safe_name, - WorkingSet) + WorkingSet, PkgResourcesDeprecationWarning) + + +# from Python 3.6 docs. +def pairwise(iterable): + "s -> (s0,s1), (s1,s2), (s2, s3), ..." + a, b = itertools.tee(iterable) + next(b, None) + return zip(a, b) class Metadata(pkg_resources.EmptyProvider): @@ -51,7 +61,8 @@ class TestDistro: assert list(ad) == ['foopkg'] # Distributions sort by version - assert [dist.version for dist in ad['FooPkg']] == ['1.4', '1.3-1', '1.2'] + expected = ['1.4', '1.3-1', '1.2'] + assert [dist.version for dist in ad['FooPkg']] == expected # Removing a distribution leaves sequence alone ad.remove(ad['FooPkg'][1]) @@ -97,7 +108,10 @@ class TestDistro: def testDistroBasics(self): d = Distribution( "/some/path", - project_name="FooPkg", version="1.3-1", py_version="2.4", platform="win32" + project_name="FooPkg", + version="1.3-1", + py_version="2.4", + platform="win32", ) self.checkFooPkg(d) @@ -113,10 +127,11 @@ class TestDistro: def testDistroMetadata(self): d = Distribution( - "/some/path", project_name="FooPkg", py_version="2.4", platform="win32", + "/some/path", project_name="FooPkg", + py_version="2.4", platform="win32", metadata=Metadata( ('PKG-INFO', "Metadata-Version: 1.0\nVersion: 1.3-1\n") - ) + ), ) self.checkFooPkg(d) @@ -130,6 +145,35 @@ class TestDistro: for v in "Twisted>=1.5", "Twisted>=1.5\nZConfig>=2.0": self.checkRequires(self.distRequires(v), v) + needs_object_dir = pytest.mark.skipif( + not hasattr(object, '__dir__'), + reason='object.__dir__ necessary for self.__dir__ implementation', + ) + + def test_distribution_dir(self): + d = pkg_resources.Distribution() + dir(d) + + @needs_object_dir + def test_distribution_dir_includes_provider_dir(self): + d = pkg_resources.Distribution() + before = d.__dir__() + assert 'test_attr' not in before + d._provider.test_attr = None + after = d.__dir__() + assert len(after) == len(before) + 1 + assert 'test_attr' in after + + @needs_object_dir + def test_distribution_dir_ignores_provider_dir_leading_underscore(self): + d = pkg_resources.Distribution() + before = d.__dir__() + assert '_test_attr' not in before + d._provider._test_attr = None + after = d.__dir__() + assert len(after) == len(before) + assert '_test_attr' not in after + def testResolve(self): ad = pkg_resources.Environment([]) ws = WorkingSet([]) @@ -164,7 +208,10 @@ class TestDistro: ad.add(Baz) # Activation list now includes resolved dependency - assert list(ws.resolve(parse_requirements("Foo[bar]"), ad)) == [Foo, Baz] + assert ( + list(ws.resolve(parse_requirements("Foo[bar]"), ad)) + == [Foo, Baz] + ) # Requests for conflicting versions produce VersionConflict with pytest.raises(VersionConflict) as vc: ws.resolve(parse_requirements("Foo==1.2\nFoo!=1.2"), ad) @@ -206,12 +253,10 @@ class TestDistro: """Extras are also evaluated as markers at resolution time.""" ad = pkg_resources.Environment([]) ws = WorkingSet([]) - # Metadata needs to be native strings due to cStringIO behaviour in - # 2.6, so use str(). Foo = Distribution.from_filename( "/foo_dir/Foo-1.2.dist-info", - metadata=Metadata(("METADATA", str("Provides-Extra: baz\n" - "Requires-Dist: quux; extra=='baz'"))) + metadata=Metadata(("METADATA", "Provides-Extra: baz\n" + "Requires-Dist: quux; extra=='baz'")) ) ad.add(Foo) assert list(ws.resolve(parse_requirements("Foo"), ad)) == [Foo] @@ -224,12 +269,10 @@ class TestDistro: """Extras are also evaluated as markers at resolution time.""" ad = pkg_resources.Environment([]) ws = WorkingSet([]) - # Metadata needs to be native strings due to cStringIO behaviour in - # 2.6, so use str(). Foo = Distribution.from_filename( "/foo_dir/Foo-1.2.dist-info", - metadata=Metadata(("METADATA", str("Provides-Extra: baz-lightyear\n" - "Requires-Dist: quux; extra=='baz-lightyear'"))) + metadata=Metadata(("METADATA", "Provides-Extra: baz-lightyear\n" + "Requires-Dist: quux; extra=='baz-lightyear'")) ) ad.add(Foo) assert list(ws.resolve(parse_requirements("Foo"), ad)) == [Foo] @@ -241,14 +284,12 @@ class TestDistro: def test_marker_evaluation_with_multiple_extras(self): ad = pkg_resources.Environment([]) ws = WorkingSet([]) - # Metadata needs to be native strings due to cStringIO behaviour in - # 2.6, so use str(). Foo = Distribution.from_filename( "/foo_dir/Foo-1.2.dist-info", - metadata=Metadata(("METADATA", str("Provides-Extra: baz\n" + metadata=Metadata(("METADATA", "Provides-Extra: baz\n" "Requires-Dist: quux; extra=='baz'\n" "Provides-Extra: bar\n" - "Requires-Dist: fred; extra=='bar'\n"))) + "Requires-Dist: fred; extra=='bar'\n")) ) ad.add(Foo) quux = Distribution.from_filename("/foo_dir/quux-1.0.dist-info") @@ -261,22 +302,20 @@ class TestDistro: def test_marker_evaluation_with_extras_loop(self): ad = pkg_resources.Environment([]) ws = WorkingSet([]) - # Metadata needs to be native strings due to cStringIO behaviour in - # 2.6, so use str(). a = Distribution.from_filename( "/foo_dir/a-0.2.dist-info", - metadata=Metadata(("METADATA", str("Requires-Dist: c[a]"))) + metadata=Metadata(("METADATA", "Requires-Dist: c[a]")) ) b = Distribution.from_filename( "/foo_dir/b-0.3.dist-info", - metadata=Metadata(("METADATA", str("Requires-Dist: c[b]"))) + metadata=Metadata(("METADATA", "Requires-Dist: c[b]")) ) c = Distribution.from_filename( "/foo_dir/c-1.0.dist-info", - metadata=Metadata(("METADATA", str("Provides-Extra: a\n" + metadata=Metadata(("METADATA", "Provides-Extra: a\n" "Requires-Dist: b;extra=='a'\n" "Provides-Extra: b\n" - "Requires-Dist: foo;extra=='b'"))) + "Requires-Dist: foo;extra=='b'")) ) foo = Distribution.from_filename("/foo_dir/foo-0.1.dist-info") for dist in (a, b, c, foo): @@ -423,7 +462,8 @@ class TestEntryPoints: submap_expect = dict( feature1=EntryPoint('feature1', 'somemodule', ['somefunction']), - feature2=EntryPoint('feature2', 'another.module', ['SomeClass'], ['extra1', 'extra2']), + feature2=EntryPoint( + 'feature2', 'another.module', ['SomeClass'], ['extra1', 'extra2']), feature3=EntryPoint('feature3', 'this.module', extras=['something']) ) submap_str = """ @@ -452,6 +492,15 @@ class TestEntryPoints: with pytest.raises(ValueError): EntryPoint.parse_map(self.submap_str) + def testDeprecationWarnings(self): + ep = EntryPoint( + "foo", "pkg_resources.tests.test_resources", ["TestEntryPoints"], + ["x"] + ) + with pytest.warns(pkg_resources.PkgResourcesDeprecationWarning): + ep.load(require=False) + + class TestRequirements: def testBasics(self): @@ -526,11 +575,17 @@ class TestRequirements: Requirement.parse('setuptools').project_name == 'setuptools') # setuptools 0.7 and higher means setuptools. assert ( - Requirement.parse('setuptools == 0.7').project_name == 'setuptools') + Requirement.parse('setuptools == 0.7').project_name + == 'setuptools' + ) assert ( - Requirement.parse('setuptools == 0.7a1').project_name == 'setuptools') + Requirement.parse('setuptools == 0.7a1').project_name + == 'setuptools' + ) assert ( - Requirement.parse('setuptools >= 0.7').project_name == 'setuptools') + Requirement.parse('setuptools >= 0.7').project_name + == 'setuptools' + ) class TestParsing: @@ -560,7 +615,7 @@ class TestParsing: """ assert ( list(pkg_resources.split_sections(sample)) - == + == [ (None, ["x"]), ("Y", ["z", "a"]), @@ -622,7 +677,7 @@ class TestParsing: assert ( Requirement.parse("name==1.1;python_version=='2.7'") != - Requirement.parse("name==1.1;python_version=='3.3'") + Requirement.parse("name==1.1;python_version=='3.6'") ) assert ( Requirement.parse("name==1.0;python_version=='2.7'") @@ -630,9 +685,9 @@ class TestParsing: Requirement.parse("name==1.2;python_version=='2.7'") ) assert ( - Requirement.parse("name[foo]==1.0;python_version=='3.3'") + Requirement.parse("name[foo]==1.0;python_version=='3.6'") != - Requirement.parse("name[foo,bar]==1.0;python_version=='3.3'") + Requirement.parse("name[foo,bar]==1.0;python_version=='3.6'") ) def test_local_version(self): @@ -642,95 +697,53 @@ class TestParsing: req, = parse_requirements('foo>=1.0, <3') req, = parse_requirements('foo >= 1.0, < 3') - def testVersionEquality(self): - def c(s1, s2): - p1, p2 = parse_version(s1), parse_version(s2) - assert p1 == p2, (s1, s2, p1, p2) - - c('1.2-rc1', '1.2rc1') - c('0.4', '0.4.0') - c('0.4.0.0', '0.4.0') - c('0.4.0-0', '0.4-0') - c('0post1', '0.0post1') - c('0pre1', '0.0c1') - c('0.0.0preview1', '0c1') - c('0.0c1', '0-rc1') - c('1.2a1', '1.2.a.1') - c('1.2.a', '1.2a') - - def testVersionOrdering(self): - def c(s1, s2): - p1, p2 = parse_version(s1), parse_version(s2) - assert p1 < p2, (s1, s2, p1, p2) - - c('2.1', '2.1.1') - c('2a1', '2b0') - c('2a1', '2.1') - c('2.3a1', '2.3') - c('2.1-1', '2.1-2') - c('2.1-1', '2.1.1') - c('2.1', '2.1post4') - c('2.1a0-20040501', '2.1') - c('1.1', '02.1') - c('3.2', '3.2.post0') - c('3.2post1', '3.2post2') - c('0.4', '4.0') - c('0.0.4', '0.4.0') - c('0post1', '0.4post1') - c('2.1.0-rc1', '2.1.0') - c('2.1dev', '2.1a0') - - torture = """ + @pytest.mark.parametrize( + ['lower', 'upper'], + [ + ('1.2-rc1', '1.2rc1'), + ('0.4', '0.4.0'), + ('0.4.0.0', '0.4.0'), + ('0.4.0-0', '0.4-0'), + ('0post1', '0.0post1'), + ('0pre1', '0.0c1'), + ('0.0.0preview1', '0c1'), + ('0.0c1', '0-rc1'), + ('1.2a1', '1.2.a.1'), + ('1.2.a', '1.2a'), + ], + ) + def testVersionEquality(self, lower, upper): + assert parse_version(lower) == parse_version(upper) + + torture = """ 0.80.1-3 0.80.1-2 0.80.1-1 0.79.9999+0.80.0pre4-1 0.79.9999+0.80.0pre2-3 0.79.9999+0.80.0pre2-2 0.77.2-1 0.77.1-1 0.77.0-1 - """.split() - - for p, v1 in enumerate(torture): - for v2 in torture[p + 1:]: - c(v2, v1) - - def testVersionBuildout(self): - """ - Buildout has a function in it's bootstrap.py that inspected the return - value of parse_version. The new parse_version returns a Version class - which needs to support this behavior, at least for now. - """ - - def buildout(parsed_version): - _final_parts = '*final-', '*final' - - def _final_version(parsed_version): - for part in parsed_version: - if (part[:1] == '*') and (part not in _final_parts): - return False - return True - - return _final_version(parsed_version) - - assert buildout(parse_version("1.0")) - assert not buildout(parse_version("1.0a1")) - - def testVersionIndexable(self): """ - Some projects were doing things like parse_version("v")[0], so we'll - support indexing the same as we support iterating. - """ - assert parse_version("1.0")[0] == "00000001" - def testVersionTupleSort(self): - """ - Some projects expected to be able to sort tuples against the return - value of parse_version. So again we'll add a warning enabled shim to - make this possible. - """ - assert parse_version("1.0") < tuple(parse_version("2.0")) - assert parse_version("1.0") <= tuple(parse_version("2.0")) - assert parse_version("1.0") == tuple(parse_version("1.0")) - assert parse_version("3.0") > tuple(parse_version("2.0")) - assert parse_version("3.0") >= tuple(parse_version("2.0")) - assert parse_version("3.0") != tuple(parse_version("2.0")) - assert not (parse_version("3.0") != tuple(parse_version("3.0"))) + @pytest.mark.parametrize( + ['lower', 'upper'], + [ + ('2.1', '2.1.1'), + ('2a1', '2b0'), + ('2a1', '2.1'), + ('2.3a1', '2.3'), + ('2.1-1', '2.1-2'), + ('2.1-1', '2.1.1'), + ('2.1', '2.1post4'), + ('2.1a0-20040501', '2.1'), + ('1.1', '02.1'), + ('3.2', '3.2.post0'), + ('3.2post1', '3.2post2'), + ('0.4', '4.0'), + ('0.0.4', '0.4.0'), + ('0post1', '0.4post1'), + ('2.1.0-rc1', '2.1.0'), + ('2.1dev', '2.1a0'), + ] + list(pairwise(reversed(torture.split()))), + ) + def testVersionOrdering(self, lower, upper): + assert parse_version(lower) < parse_version(upper) def testVersionHashable(self): """ @@ -846,7 +859,8 @@ class TestNamespaces: subpkg = nspkg / 'subpkg' subpkg.ensure_dir() (nspkg / '__init__.py').write_text(self.ns_str, encoding='utf-8') - (subpkg / '__init__.py').write_text(vers_str % number, encoding='utf-8') + (subpkg / '__init__.py').write_text( + vers_str % number, encoding='utf-8') import nspkg.subpkg import nspkg diff --git a/pkg_resources/tests/test_working_set.py b/pkg_resources/tests/test_working_set.py index 422a7283..b3ca4ea8 100644 --- a/pkg_resources/tests/test_working_set.py +++ b/pkg_resources/tests/test_working_set.py @@ -1,6 +1,7 @@ import inspect import re import textwrap +import functools import pytest @@ -8,6 +9,8 @@ import pkg_resources from .test_resources import Metadata +__metaclass__ = type + def strip_comments(s): return '\n'.join( @@ -15,6 +18,7 @@ def strip_comments(s): if l.strip() and not l.strip().startswith('#') ) + def parse_distributions(s): ''' Parse a series of distribution specs of the form: @@ -31,10 +35,11 @@ def parse_distributions(s): yield 2 distributions: - project_name=foo, version=0.2 - - project_name=bar, version=1.0, requires=['foo>=3.0', 'baz; extra=="feature"'] + - project_name=bar, version=1.0, + requires=['foo>=3.0', 'baz; extra=="feature"'] ''' s = s.strip() - for spec in re.split('\n(?=[^\s])', s): + for spec in re.split(r'\n(?=[^\s])', s): if not spec: continue fields = spec.split('\n', 1) @@ -42,7 +47,7 @@ def parse_distributions(s): name, version = fields.pop(0).split('-') if fields: requires = textwrap.dedent(fields.pop(0)) - metadata=Metadata(('requires.txt', requires)) + metadata = Metadata(('requires.txt', requires)) else: metadata = None dist = pkg_resources.Distribution(project_name=name, @@ -51,7 +56,7 @@ def parse_distributions(s): yield dist -class FakeInstaller(object): +class FakeInstaller: def __init__(self, installable_dists): self._installable_dists = installable_dists @@ -84,7 +89,7 @@ def parametrize_test_working_set_resolve(*test_list): ): idlist.append(id_) expected = strip_comments(expected.strip()) - if re.match('\w+$', expected): + if re.match(r'\w+$', expected): expected = getattr(pkg_resources, expected) assert issubclass(expected, Exception) else: @@ -467,7 +472,8 @@ def test_working_set_resolve(installed_dists, installable_dists, requirements, replace_conflicting, resolved_dists_or_exception): ws = pkg_resources.WorkingSet([]) list(map(ws.add, installed_dists)) - resolve_call = lambda: ws.resolve( + resolve_call = functools.partial( + ws.resolve, requirements, installer=FakeInstaller(installable_dists), replace_conflicting=replace_conflicting, ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..4ef804ec --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["wheel"] +build-backend = "setuptools.build_meta" + +[tool.towncrier] + package = "setuptools" + package_dir = "setuptools" + filename = "CHANGES.rst" + directory = "changelog.d" + title_format = "v{version}" + issue_format = "#{issue}" + template = "towncrier_template.rst" + underlines = ["-"] + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "change" + name = "Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "doc" + name = "Documentation changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "misc" + name = "Misc" + showcontent = true diff --git a/pytest.ini b/pytest.ini index 16fdc5af..1c5b6b09 100755..100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] -addopts=--doctest-modules --ignore release.py --ignore setuptools/lib2to3_ex.py --ignore tests/manual_test.py --ignore tests/test_pypi.py --ignore tests/shlib_test --doctest-glob=pkg_resources/api_tests.txt --ignore scripts/upload-old-releases-as-zip.py --ignore pavement.py --ignore setuptools/tests/mod_with_constant.py -rsxX +addopts=--doctest-modules --doctest-glob=pkg_resources/api_tests.txt -rsxX norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern .* flake8-ignore = setuptools/site-patch.py F821 setuptools/py*compat.py F811 +doctest_optionflags=ELLIPSIS ALLOW_UNICODE diff --git a/setup.cfg b/setup.cfg index 8422a269..dd404469 100755..100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 36.6.0 +current_version = 40.6.3 commit = True tag = True @@ -45,8 +45,8 @@ def _gen_console_scripts(): ) if any(os.environ.get(var) not in (None, "", "0") for var in var_names): return - yield ("easy_install-{shortver} = setuptools.command.easy_install:main" - .format(shortver=sys.version[:3])) + tmpl = "easy_install-{shortver} = setuptools.command.easy_install:main" + yield tmpl.format(shortver=sys.version[:3]) readme_path = os.path.join(here, 'README.rst') @@ -89,15 +89,20 @@ def pypi_link(pkg_filename): setup_params = dict( name="setuptools", - version="36.6.0", - description="Easily download, build, install, upgrade, and uninstall " - "Python packages", + version="40.6.3", + description=( + "Easily download, build, install, upgrade, and uninstall " + "Python packages" + ), author="Python Packaging Authority", author_email="distutils-sig@python.org", long_description=long_description, long_description_content_type='text/x-rst; charset=UTF-8', keywords="CPAN PyPI distutils eggs package management", url="https://github.com/pypa/setuptools", + project_urls={ + "Documentation": "https://setuptools.readthedocs.io/", + }, src_root=None, packages=setuptools.find_packages(exclude=['*.tests']), package_data=package_data, @@ -136,7 +141,10 @@ setup_params = dict( "requires.txt = setuptools.command.egg_info:write_requirements", "entry_points.txt = setuptools.command.egg_info:write_entries", "eager_resources.txt = setuptools.command.egg_info:overwrite_arg", - "namespace_packages.txt = setuptools.command.egg_info:overwrite_arg", + ( + "namespace_packages.txt = " + "setuptools.command.egg_info:overwrite_arg" + ), "top_level.txt = setuptools.command.egg_info:write_toplevel_names", "depends.txt = setuptools.command.egg_info:warn_depends_obsolete", "dependency_links.txt = setuptools.command.egg_info:overwrite_arg", @@ -151,19 +159,18 @@ setup_params = dict( License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python :: 2 - Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 Topic :: Software Development :: Libraries :: Python Modules Topic :: System :: Archiving :: Packaging Topic :: System :: Systems Administration Topic :: Utilities """).strip().splitlines(), - python_requires='>=2.6,!=3.0.*,!=3.1.*,!=3.2.*', + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*', extras_require={ "ssl:sys_platform=='win32'": "wincertstore==0.2", "certs": "certifi==2016.9.26", diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 77b4a374..a71b2bbd 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -1,6 +1,7 @@ """Extensions to the 'distutils' for large or complex distributions""" import os +import sys import functools import distutils.core import distutils.filelist @@ -9,7 +10,9 @@ from distutils.errors import DistutilsOptionError from distutils.util import convert_path from fnmatch import fnmatchcase -from setuptools.extern.six import string_types +from ._deprecation_warning import SetuptoolsDeprecationWarning + +from setuptools.extern.six import PY3, string_types from setuptools.extern.six.moves import filter, map import setuptools.version @@ -18,11 +21,18 @@ from setuptools.dist import Distribution, Feature from setuptools.depends import Require from . import monkey +__metaclass__ = type + + __all__ = [ 'setup', 'Distribution', 'Feature', 'Command', 'Extension', 'Require', - 'find_packages', + 'SetuptoolsDeprecationWarning', + 'find_packages' ] +if PY3: + __all__.append('find_namespace_packages') + __version__ = setuptools.version.__version__ bootstrap_install_from = None @@ -34,7 +44,7 @@ run_2to3_on_doctests = True lib2to3_fixer_packages = ['lib2to3.fixes'] -class PackageFinder(object): +class PackageFinder: """ Generate a list of all Python packages found within a directory """ @@ -112,7 +122,30 @@ class PEP420PackageFinder(PackageFinder): find_packages = PackageFinder.find -setup = distutils.core.setup +if PY3: + find_namespace_packages = PEP420PackageFinder.find + + +def _install_setup_requires(attrs): + # Note: do not use `setuptools.Distribution` directly, as + # our PEP 517 backend patch `distutils.core.Distribution`. + dist = distutils.core.Distribution(dict( + (k, v) for k, v in attrs.items() + if k in ('dependency_links', 'setup_requires') + )) + # Honor setup.cfg's options. + dist.parse_config_files(ignore_option_errors=True) + if dist.setup_requires: + dist.fetch_build_eggs(dist.setup_requires) + + +def setup(**attrs): + # Make sure we have any requirements needed to interpret 'attrs'. + _install_setup_requires(attrs) + return distutils.core.setup(**attrs) + +setup.__doc__ = distutils.core.setup.__doc__ + _Command = monkey.get_unpatched(distutils.core.Command) @@ -191,4 +224,5 @@ def findall(dir=os.curdir): return list(files) +# Apply monkey patches monkey.patch_all() diff --git a/setuptools/_deprecation_warning.py b/setuptools/_deprecation_warning.py new file mode 100644 index 00000000..086b64dd --- /dev/null +++ b/setuptools/_deprecation_warning.py @@ -0,0 +1,7 @@ +class SetuptoolsDeprecationWarning(Warning): + """ + Base class for warning deprecations in ``setuptools`` + + This class is not derived from ``DeprecationWarning``, and as such is + visible by default. + """ diff --git a/setuptools/_vendor/__init__.py b/setuptools/_vendor/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/setuptools/_vendor/__init__.py diff --git a/setuptools/_vendor/packaging/__about__.py b/setuptools/_vendor/packaging/__about__.py new file mode 100644 index 00000000..95d330ef --- /dev/null +++ b/setuptools/_vendor/packaging/__about__.py @@ -0,0 +1,21 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +__all__ = [ + "__title__", "__summary__", "__uri__", "__version__", "__author__", + "__email__", "__license__", "__copyright__", +] + +__title__ = "packaging" +__summary__ = "Core utilities for Python packages" +__uri__ = "https://github.com/pypa/packaging" + +__version__ = "16.8" + +__author__ = "Donald Stufft and individual contributors" +__email__ = "donald@stufft.io" + +__license__ = "BSD or Apache License, Version 2.0" +__copyright__ = "Copyright 2014-2016 %s" % __author__ diff --git a/setuptools/_vendor/packaging/__init__.py b/setuptools/_vendor/packaging/__init__.py new file mode 100644 index 00000000..5ee62202 --- /dev/null +++ b/setuptools/_vendor/packaging/__init__.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +from .__about__ import ( + __author__, __copyright__, __email__, __license__, __summary__, __title__, + __uri__, __version__ +) + +__all__ = [ + "__title__", "__summary__", "__uri__", "__version__", "__author__", + "__email__", "__license__", "__copyright__", +] diff --git a/setuptools/_vendor/packaging/_compat.py b/setuptools/_vendor/packaging/_compat.py new file mode 100644 index 00000000..210bb80b --- /dev/null +++ b/setuptools/_vendor/packaging/_compat.py @@ -0,0 +1,30 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import sys + + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +# flake8: noqa + +if PY3: + string_types = str, +else: + string_types = basestring, + + +def with_metaclass(meta, *bases): + """ + Create a base class with a metaclass. + """ + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) diff --git a/setuptools/_vendor/packaging/_structures.py b/setuptools/_vendor/packaging/_structures.py new file mode 100644 index 00000000..ccc27861 --- /dev/null +++ b/setuptools/_vendor/packaging/_structures.py @@ -0,0 +1,68 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + + +class Infinity(object): + + def __repr__(self): + return "Infinity" + + def __hash__(self): + return hash(repr(self)) + + def __lt__(self, other): + return False + + def __le__(self, other): + return False + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __ne__(self, other): + return not isinstance(other, self.__class__) + + def __gt__(self, other): + return True + + def __ge__(self, other): + return True + + def __neg__(self): + return NegativeInfinity + +Infinity = Infinity() + + +class NegativeInfinity(object): + + def __repr__(self): + return "-Infinity" + + def __hash__(self): + return hash(repr(self)) + + def __lt__(self, other): + return True + + def __le__(self, other): + return True + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __ne__(self, other): + return not isinstance(other, self.__class__) + + def __gt__(self, other): + return False + + def __ge__(self, other): + return False + + def __neg__(self): + return Infinity + +NegativeInfinity = NegativeInfinity() diff --git a/setuptools/_vendor/packaging/markers.py b/setuptools/_vendor/packaging/markers.py new file mode 100644 index 00000000..031332a3 --- /dev/null +++ b/setuptools/_vendor/packaging/markers.py @@ -0,0 +1,301 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import operator +import os +import platform +import sys + +from setuptools.extern.pyparsing import ParseException, ParseResults, stringStart, stringEnd +from setuptools.extern.pyparsing import ZeroOrMore, Group, Forward, QuotedString +from setuptools.extern.pyparsing import Literal as L # noqa + +from ._compat import string_types +from .specifiers import Specifier, InvalidSpecifier + + +__all__ = [ + "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", + "Marker", "default_environment", +] + + +class InvalidMarker(ValueError): + """ + An invalid marker was found, users should refer to PEP 508. + """ + + +class UndefinedComparison(ValueError): + """ + An invalid operation was attempted on a value that doesn't support it. + """ + + +class UndefinedEnvironmentName(ValueError): + """ + A name was attempted to be used that does not exist inside of the + environment. + """ + + +class Node(object): + + def __init__(self, value): + self.value = value + + def __str__(self): + return str(self.value) + + def __repr__(self): + return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) + + def serialize(self): + raise NotImplementedError + + +class Variable(Node): + + def serialize(self): + return str(self) + + +class Value(Node): + + def serialize(self): + return '"{0}"'.format(self) + + +class Op(Node): + + def serialize(self): + return str(self) + + +VARIABLE = ( + L("implementation_version") | + L("platform_python_implementation") | + L("implementation_name") | + L("python_full_version") | + L("platform_release") | + L("platform_version") | + L("platform_machine") | + L("platform_system") | + L("python_version") | + L("sys_platform") | + L("os_name") | + L("os.name") | # PEP-345 + L("sys.platform") | # PEP-345 + L("platform.version") | # PEP-345 + L("platform.machine") | # PEP-345 + L("platform.python_implementation") | # PEP-345 + L("python_implementation") | # undocumented setuptools legacy + L("extra") +) +ALIASES = { + 'os.name': 'os_name', + 'sys.platform': 'sys_platform', + 'platform.version': 'platform_version', + 'platform.machine': 'platform_machine', + 'platform.python_implementation': 'platform_python_implementation', + 'python_implementation': 'platform_python_implementation' +} +VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) + +VERSION_CMP = ( + L("===") | + L("==") | + L(">=") | + L("<=") | + L("!=") | + L("~=") | + L(">") | + L("<") +) + +MARKER_OP = VERSION_CMP | L("not in") | L("in") +MARKER_OP.setParseAction(lambda s, l, t: Op(t[0])) + +MARKER_VALUE = QuotedString("'") | QuotedString('"') +MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) + +BOOLOP = L("and") | L("or") + +MARKER_VAR = VARIABLE | MARKER_VALUE + +MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) + +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() + +MARKER_EXPR = Forward() +MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) +MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) + +MARKER = stringStart + MARKER_EXPR + stringEnd + + +def _coerce_parse_result(results): + if isinstance(results, ParseResults): + return [_coerce_parse_result(i) for i in results] + else: + return results + + +def _format_marker(marker, first=True): + assert isinstance(marker, (list, tuple, string_types)) + + # Sometimes we have a structure like [[...]] which is a single item list + # where the single item is itself it's own list. In that case we want skip + # the rest of this function so that we don't get extraneous () on the + # outside. + if (isinstance(marker, list) and len(marker) == 1 and + isinstance(marker[0], (list, tuple))): + return _format_marker(marker[0]) + + if isinstance(marker, list): + inner = (_format_marker(m, first=False) for m in marker) + if first: + return " ".join(inner) + else: + return "(" + " ".join(inner) + ")" + elif isinstance(marker, tuple): + return " ".join([m.serialize() for m in marker]) + else: + return marker + + +_operators = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _eval_op(lhs, op, rhs): + try: + spec = Specifier("".join([op.serialize(), rhs])) + except InvalidSpecifier: + pass + else: + return spec.contains(lhs) + + oper = _operators.get(op.serialize()) + if oper is None: + raise UndefinedComparison( + "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs) + ) + + return oper(lhs, rhs) + + +_undefined = object() + + +def _get_env(environment, name): + value = environment.get(name, _undefined) + + if value is _undefined: + raise UndefinedEnvironmentName( + "{0!r} does not exist in evaluation environment.".format(name) + ) + + return value + + +def _evaluate_markers(markers, environment): + groups = [[]] + + for marker in markers: + assert isinstance(marker, (list, tuple, string_types)) + + if isinstance(marker, list): + groups[-1].append(_evaluate_markers(marker, environment)) + elif isinstance(marker, tuple): + lhs, op, rhs = marker + + if isinstance(lhs, Variable): + lhs_value = _get_env(environment, lhs.value) + rhs_value = rhs.value + else: + lhs_value = lhs.value + rhs_value = _get_env(environment, rhs.value) + + groups[-1].append(_eval_op(lhs_value, op, rhs_value)) + else: + assert marker in ["and", "or"] + if marker == "or": + groups.append([]) + + return any(all(item) for item in groups) + + +def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + +def default_environment(): + if hasattr(sys, 'implementation'): + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + else: + iver = '0' + implementation_name = '' + + return { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": platform.python_version()[:3], + "sys_platform": sys.platform, + } + + +class Marker(object): + + def __init__(self, marker): + try: + self._markers = _coerce_parse_result(MARKER.parseString(marker)) + except ParseException as e: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e.loc:e.loc + 8]) + raise InvalidMarker(err_str) + + def __str__(self): + return _format_marker(self._markers) + + def __repr__(self): + return "<Marker({0!r})>".format(str(self)) + + def evaluate(self, environment=None): + """Evaluate a marker. + + Return the boolean from evaluating the given marker against the + environment. environment is an optional argument to override all or + part of the determined environment. + + The environment is determined from the current Python process. + """ + current_environment = default_environment() + if environment is not None: + current_environment.update(environment) + + return _evaluate_markers(self._markers, current_environment) diff --git a/setuptools/_vendor/packaging/requirements.py b/setuptools/_vendor/packaging/requirements.py new file mode 100644 index 00000000..5b493416 --- /dev/null +++ b/setuptools/_vendor/packaging/requirements.py @@ -0,0 +1,127 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import string +import re + +from setuptools.extern.pyparsing import stringStart, stringEnd, originalTextFor, ParseException +from setuptools.extern.pyparsing import ZeroOrMore, Word, Optional, Regex, Combine +from setuptools.extern.pyparsing import Literal as L # noqa +from setuptools.extern.six.moves.urllib import parse as urlparse + +from .markers import MARKER_EXPR, Marker +from .specifiers import LegacySpecifier, Specifier, SpecifierSet + + +class InvalidRequirement(ValueError): + """ + An invalid requirement was found, users should refer to PEP 508. + """ + + +ALPHANUM = Word(string.ascii_letters + string.digits) + +LBRACKET = L("[").suppress() +RBRACKET = L("]").suppress() +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() +COMMA = L(",").suppress() +SEMICOLON = L(";").suppress() +AT = L("@").suppress() + +PUNCTUATION = Word("-_.") +IDENTIFIER_END = ALPHANUM | (ZeroOrMore(PUNCTUATION) + ALPHANUM) +IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) + +NAME = IDENTIFIER("name") +EXTRA = IDENTIFIER + +URI = Regex(r'[^ ]+')("url") +URL = (AT + URI) + +EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) +EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") + +VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) +VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) + +VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY +VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), + joinString=",", adjacent=False)("_raw_spec") +_VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') + +VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") +VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) + +MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") +MARKER_EXPR.setParseAction( + lambda s, l, t: Marker(s[t._original_start:t._original_end]) +) +MARKER_SEPERATOR = SEMICOLON +MARKER = MARKER_SEPERATOR + MARKER_EXPR + +VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) +URL_AND_MARKER = URL + Optional(MARKER) + +NAMED_REQUIREMENT = \ + NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) + +REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd + + +class Requirement(object): + """Parse a requirement. + + Parse a given requirement string into its parts, such as name, specifier, + URL, and extras. Raises InvalidRequirement on a badly-formed requirement + string. + """ + + # TODO: Can we test whether something is contained within a requirement? + # If so how do we do that? Do we need to test against the _name_ of + # the thing as well as the version? What about the markers? + # TODO: Can we normalize the name and extra name? + + def __init__(self, requirement_string): + try: + req = REQUIREMENT.parseString(requirement_string) + except ParseException as e: + raise InvalidRequirement( + "Invalid requirement, parse error at \"{0!r}\"".format( + requirement_string[e.loc:e.loc + 8])) + + self.name = req.name + if req.url: + parsed_url = urlparse.urlparse(req.url) + if not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc): + raise InvalidRequirement("Invalid URL given") + self.url = req.url + else: + self.url = None + self.extras = set(req.extras.asList() if req.extras else []) + self.specifier = SpecifierSet(req.specifier) + self.marker = req.marker if req.marker else None + + def __str__(self): + parts = [self.name] + + if self.extras: + parts.append("[{0}]".format(",".join(sorted(self.extras)))) + + if self.specifier: + parts.append(str(self.specifier)) + + if self.url: + parts.append("@ {0}".format(self.url)) + + if self.marker: + parts.append("; {0}".format(self.marker)) + + return "".join(parts) + + def __repr__(self): + return "<Requirement({0!r})>".format(str(self)) diff --git a/setuptools/_vendor/packaging/specifiers.py b/setuptools/_vendor/packaging/specifiers.py new file mode 100644 index 00000000..7f5a76cf --- /dev/null +++ b/setuptools/_vendor/packaging/specifiers.py @@ -0,0 +1,774 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import abc +import functools +import itertools +import re + +from ._compat import string_types, with_metaclass +from .version import Version, LegacyVersion, parse + + +class InvalidSpecifier(ValueError): + """ + An invalid specifier was found, users should refer to PEP 440. + """ + + +class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): + + @abc.abstractmethod + def __str__(self): + """ + Returns the str representation of this Specifier like object. This + should be representative of the Specifier itself. + """ + + @abc.abstractmethod + def __hash__(self): + """ + Returns a hash value for this Specifier like object. + """ + + @abc.abstractmethod + def __eq__(self, other): + """ + Returns a boolean representing whether or not the two Specifier like + objects are equal. + """ + + @abc.abstractmethod + def __ne__(self, other): + """ + Returns a boolean representing whether or not the two Specifier like + objects are not equal. + """ + + @abc.abstractproperty + def prereleases(self): + """ + Returns whether or not pre-releases as a whole are allowed by this + specifier. + """ + + @prereleases.setter + def prereleases(self, value): + """ + Sets whether or not pre-releases as a whole are allowed by this + specifier. + """ + + @abc.abstractmethod + def contains(self, item, prereleases=None): + """ + Determines if the given item is contained within this specifier. + """ + + @abc.abstractmethod + def filter(self, iterable, prereleases=None): + """ + Takes an iterable of items and filters them so that only items which + are contained within this specifier are allowed in it. + """ + + +class _IndividualSpecifier(BaseSpecifier): + + _operators = {} + + def __init__(self, spec="", prereleases=None): + match = self._regex.search(spec) + if not match: + raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) + + self._spec = ( + match.group("operator").strip(), + match.group("version").strip(), + ) + + # Store whether or not this Specifier should accept prereleases + self._prereleases = prereleases + + def __repr__(self): + pre = ( + ", prereleases={0!r}".format(self.prereleases) + if self._prereleases is not None + else "" + ) + + return "<{0}({1!r}{2})>".format( + self.__class__.__name__, + str(self), + pre, + ) + + def __str__(self): + return "{0}{1}".format(*self._spec) + + def __hash__(self): + return hash(self._spec) + + def __eq__(self, other): + if isinstance(other, string_types): + try: + other = self.__class__(other) + except InvalidSpecifier: + return NotImplemented + elif not isinstance(other, self.__class__): + return NotImplemented + + return self._spec == other._spec + + def __ne__(self, other): + if isinstance(other, string_types): + try: + other = self.__class__(other) + except InvalidSpecifier: + return NotImplemented + elif not isinstance(other, self.__class__): + return NotImplemented + + return self._spec != other._spec + + def _get_operator(self, op): + return getattr(self, "_compare_{0}".format(self._operators[op])) + + def _coerce_version(self, version): + if not isinstance(version, (LegacyVersion, Version)): + version = parse(version) + return version + + @property + def operator(self): + return self._spec[0] + + @property + def version(self): + return self._spec[1] + + @property + def prereleases(self): + return self._prereleases + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + def __contains__(self, item): + return self.contains(item) + + def contains(self, item, prereleases=None): + # Determine if prereleases are to be allowed or not. + if prereleases is None: + prereleases = self.prereleases + + # Normalize item to a Version or LegacyVersion, this allows us to have + # a shortcut for ``"2.0" in Specifier(">=2") + item = self._coerce_version(item) + + # Determine if we should be supporting prereleases in this specifier + # or not, if we do not support prereleases than we can short circuit + # logic if this version is a prereleases. + if item.is_prerelease and not prereleases: + return False + + # Actually do the comparison to determine if this item is contained + # within this Specifier or not. + return self._get_operator(self.operator)(item, self.version) + + def filter(self, iterable, prereleases=None): + yielded = False + found_prereleases = [] + + kw = {"prereleases": prereleases if prereleases is not None else True} + + # Attempt to iterate over all the values in the iterable and if any of + # them match, yield them. + for version in iterable: + parsed_version = self._coerce_version(version) + + if self.contains(parsed_version, **kw): + # If our version is a prerelease, and we were not set to allow + # prereleases, then we'll store it for later incase nothing + # else matches this specifier. + if (parsed_version.is_prerelease and not + (prereleases or self.prereleases)): + found_prereleases.append(version) + # Either this is not a prerelease, or we should have been + # accepting prereleases from the begining. + else: + yielded = True + yield version + + # Now that we've iterated over everything, determine if we've yielded + # any values, and if we have not and we have any prereleases stored up + # then we will go ahead and yield the prereleases. + if not yielded and found_prereleases: + for version in found_prereleases: + yield version + + +class LegacySpecifier(_IndividualSpecifier): + + _regex_str = ( + r""" + (?P<operator>(==|!=|<=|>=|<|>)) + \s* + (?P<version> + [^,;\s)]* # Since this is a "legacy" specifier, and the version + # string can be just about anything, we match everything + # except for whitespace, a semi-colon for marker support, + # a closing paren since versions can be enclosed in + # them, and a comma since it's a version separator. + ) + """ + ) + + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + + _operators = { + "==": "equal", + "!=": "not_equal", + "<=": "less_than_equal", + ">=": "greater_than_equal", + "<": "less_than", + ">": "greater_than", + } + + def _coerce_version(self, version): + if not isinstance(version, LegacyVersion): + version = LegacyVersion(str(version)) + return version + + def _compare_equal(self, prospective, spec): + return prospective == self._coerce_version(spec) + + def _compare_not_equal(self, prospective, spec): + return prospective != self._coerce_version(spec) + + def _compare_less_than_equal(self, prospective, spec): + return prospective <= self._coerce_version(spec) + + def _compare_greater_than_equal(self, prospective, spec): + return prospective >= self._coerce_version(spec) + + def _compare_less_than(self, prospective, spec): + return prospective < self._coerce_version(spec) + + def _compare_greater_than(self, prospective, spec): + return prospective > self._coerce_version(spec) + + +def _require_version_compare(fn): + @functools.wraps(fn) + def wrapped(self, prospective, spec): + if not isinstance(prospective, Version): + return False + return fn(self, prospective, spec) + return wrapped + + +class Specifier(_IndividualSpecifier): + + _regex_str = ( + r""" + (?P<operator>(~=|==|!=|<=|>=|<|>|===)) + (?P<version> + (?: + # The identity operators allow for an escape hatch that will + # do an exact string match of the version you wish to install. + # This will not be parsed by PEP 440 and we cannot determine + # any semantic meaning from it. This operator is discouraged + # but included entirely as an escape hatch. + (?<====) # Only match for the identity operator + \s* + [^\s]* # We just match everything, except for whitespace + # since we are only testing for strict identity. + ) + | + (?: + # The (non)equality operators allow for wild card and local + # versions to be specified so we have to define these two + # operators separately to enable that. + (?<===|!=) # Only match for equals and not equals + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)* # release + (?: # pre release + [-_\.]? + (a|b|c|rc|alpha|beta|pre|preview) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + + # You cannot use a wild card and a dev or local version + # together so group them with a | and make them optional. + (?: + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local + | + \.\* # Wild card syntax of .* + )? + ) + | + (?: + # The compatible operator requires at least two digits in the + # release segment. + (?<=~=) # Only match for the compatible operator + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) + (?: # pre release + [-_\.]? + (a|b|c|rc|alpha|beta|pre|preview) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + ) + | + (?: + # All other operators only allow a sub set of what the + # (non)equality operators do. Specifically they do not allow + # local versions to be specified nor do they allow the prefix + # matching wild cards. + (?<!==|!=|~=) # We have special cases for these + # operators so we want to make sure they + # don't match here. + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)* # release + (?: # pre release + [-_\.]? + (a|b|c|rc|alpha|beta|pre|preview) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + ) + ) + """ + ) + + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + + _operators = { + "~=": "compatible", + "==": "equal", + "!=": "not_equal", + "<=": "less_than_equal", + ">=": "greater_than_equal", + "<": "less_than", + ">": "greater_than", + "===": "arbitrary", + } + + @_require_version_compare + def _compare_compatible(self, prospective, spec): + # Compatible releases have an equivalent combination of >= and ==. That + # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to + # implement this in terms of the other specifiers instead of + # implementing it ourselves. The only thing we need to do is construct + # the other specifiers. + + # We want everything but the last item in the version, but we want to + # ignore post and dev releases and we want to treat the pre-release as + # it's own separate segment. + prefix = ".".join( + list( + itertools.takewhile( + lambda x: (not x.startswith("post") and not + x.startswith("dev")), + _version_split(spec), + ) + )[:-1] + ) + + # Add the prefix notation to the end of our string + prefix += ".*" + + return (self._get_operator(">=")(prospective, spec) and + self._get_operator("==")(prospective, prefix)) + + @_require_version_compare + def _compare_equal(self, prospective, spec): + # We need special logic to handle prefix matching + if spec.endswith(".*"): + # In the case of prefix matching we want to ignore local segment. + prospective = Version(prospective.public) + # Split the spec out by dots, and pretend that there is an implicit + # dot in between a release segment and a pre-release segment. + spec = _version_split(spec[:-2]) # Remove the trailing .* + + # Split the prospective version out by dots, and pretend that there + # is an implicit dot in between a release segment and a pre-release + # segment. + prospective = _version_split(str(prospective)) + + # Shorten the prospective version to be the same length as the spec + # so that we can determine if the specifier is a prefix of the + # prospective version or not. + prospective = prospective[:len(spec)] + + # Pad out our two sides with zeros so that they both equal the same + # length. + spec, prospective = _pad_version(spec, prospective) + else: + # Convert our spec string into a Version + spec = Version(spec) + + # If the specifier does not have a local segment, then we want to + # act as if the prospective version also does not have a local + # segment. + if not spec.local: + prospective = Version(prospective.public) + + return prospective == spec + + @_require_version_compare + def _compare_not_equal(self, prospective, spec): + return not self._compare_equal(prospective, spec) + + @_require_version_compare + def _compare_less_than_equal(self, prospective, spec): + return prospective <= Version(spec) + + @_require_version_compare + def _compare_greater_than_equal(self, prospective, spec): + return prospective >= Version(spec) + + @_require_version_compare + def _compare_less_than(self, prospective, spec): + # Convert our spec to a Version instance, since we'll want to work with + # it as a version. + spec = Version(spec) + + # Check to see if the prospective version is less than the spec + # version. If it's not we can short circuit and just return False now + # instead of doing extra unneeded work. + if not prospective < spec: + return False + + # This special case is here so that, unless the specifier itself + # includes is a pre-release version, that we do not accept pre-release + # versions for the version mentioned in the specifier (e.g. <3.1 should + # not match 3.1.dev0, but should match 3.0.dev0). + if not spec.is_prerelease and prospective.is_prerelease: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # If we've gotten to here, it means that prospective version is both + # less than the spec version *and* it's not a pre-release of the same + # version in the spec. + return True + + @_require_version_compare + def _compare_greater_than(self, prospective, spec): + # Convert our spec to a Version instance, since we'll want to work with + # it as a version. + spec = Version(spec) + + # Check to see if the prospective version is greater than the spec + # version. If it's not we can short circuit and just return False now + # instead of doing extra unneeded work. + if not prospective > spec: + return False + + # This special case is here so that, unless the specifier itself + # includes is a post-release version, that we do not accept + # post-release versions for the version mentioned in the specifier + # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). + if not spec.is_postrelease and prospective.is_postrelease: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # Ensure that we do not allow a local version of the version mentioned + # in the specifier, which is techincally greater than, to match. + if prospective.local is not None: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # If we've gotten to here, it means that prospective version is both + # greater than the spec version *and* it's not a pre-release of the + # same version in the spec. + return True + + def _compare_arbitrary(self, prospective, spec): + return str(prospective).lower() == str(spec).lower() + + @property + def prereleases(self): + # If there is an explicit prereleases set for this, then we'll just + # blindly use that. + if self._prereleases is not None: + return self._prereleases + + # Look at all of our specifiers and determine if they are inclusive + # operators, and if they are if they are including an explicit + # prerelease. + operator, version = self._spec + if operator in ["==", ">=", "<=", "~=", "==="]: + # The == specifier can include a trailing .*, if it does we + # want to remove before parsing. + if operator == "==" and version.endswith(".*"): + version = version[:-2] + + # Parse the version, and if it is a pre-release than this + # specifier allows pre-releases. + if parse(version).is_prerelease: + return True + + return False + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + +_prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") + + +def _version_split(version): + result = [] + for item in version.split("."): + match = _prefix_regex.search(item) + if match: + result.extend(match.groups()) + else: + result.append(item) + return result + + +def _pad_version(left, right): + left_split, right_split = [], [] + + # Get the release segment of our versions + left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) + right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) + + # Get the rest of our versions + left_split.append(left[len(left_split[0]):]) + right_split.append(right[len(right_split[0]):]) + + # Insert our padding + left_split.insert( + 1, + ["0"] * max(0, len(right_split[0]) - len(left_split[0])), + ) + right_split.insert( + 1, + ["0"] * max(0, len(left_split[0]) - len(right_split[0])), + ) + + return ( + list(itertools.chain(*left_split)), + list(itertools.chain(*right_split)), + ) + + +class SpecifierSet(BaseSpecifier): + + def __init__(self, specifiers="", prereleases=None): + # Split on , to break each indidivual specifier into it's own item, and + # strip each item to remove leading/trailing whitespace. + specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] + + # Parsed each individual specifier, attempting first to make it a + # Specifier and falling back to a LegacySpecifier. + parsed = set() + for specifier in specifiers: + try: + parsed.add(Specifier(specifier)) + except InvalidSpecifier: + parsed.add(LegacySpecifier(specifier)) + + # Turn our parsed specifiers into a frozen set and save them for later. + self._specs = frozenset(parsed) + + # Store our prereleases value so we can use it later to determine if + # we accept prereleases or not. + self._prereleases = prereleases + + def __repr__(self): + pre = ( + ", prereleases={0!r}".format(self.prereleases) + if self._prereleases is not None + else "" + ) + + return "<SpecifierSet({0!r}{1})>".format(str(self), pre) + + def __str__(self): + return ",".join(sorted(str(s) for s in self._specs)) + + def __hash__(self): + return hash(self._specs) + + def __and__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + specifier = SpecifierSet() + specifier._specs = frozenset(self._specs | other._specs) + + if self._prereleases is None and other._prereleases is not None: + specifier._prereleases = other._prereleases + elif self._prereleases is not None and other._prereleases is None: + specifier._prereleases = self._prereleases + elif self._prereleases == other._prereleases: + specifier._prereleases = self._prereleases + else: + raise ValueError( + "Cannot combine SpecifierSets with True and False prerelease " + "overrides." + ) + + return specifier + + def __eq__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif isinstance(other, _IndividualSpecifier): + other = SpecifierSet(str(other)) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + return self._specs == other._specs + + def __ne__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif isinstance(other, _IndividualSpecifier): + other = SpecifierSet(str(other)) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + return self._specs != other._specs + + def __len__(self): + return len(self._specs) + + def __iter__(self): + return iter(self._specs) + + @property + def prereleases(self): + # If we have been given an explicit prerelease modifier, then we'll + # pass that through here. + if self._prereleases is not None: + return self._prereleases + + # If we don't have any specifiers, and we don't have a forced value, + # then we'll just return None since we don't know if this should have + # pre-releases or not. + if not self._specs: + return None + + # Otherwise we'll see if any of the given specifiers accept + # prereleases, if any of them do we'll return True, otherwise False. + return any(s.prereleases for s in self._specs) + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + def __contains__(self, item): + return self.contains(item) + + def contains(self, item, prereleases=None): + # Ensure that our item is a Version or LegacyVersion instance. + if not isinstance(item, (LegacyVersion, Version)): + item = parse(item) + + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + + # We can determine if we're going to allow pre-releases by looking to + # see if any of the underlying items supports them. If none of them do + # and this item is a pre-release then we do not allow it and we can + # short circuit that here. + # Note: This means that 1.0.dev1 would not be contained in something + # like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0 + if not prereleases and item.is_prerelease: + return False + + # We simply dispatch to the underlying specs here to make sure that the + # given version is contained within all of them. + # Note: This use of all() here means that an empty set of specifiers + # will always return True, this is an explicit design decision. + return all( + s.contains(item, prereleases=prereleases) + for s in self._specs + ) + + def filter(self, iterable, prereleases=None): + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + + # If we have any specifiers, then we want to wrap our iterable in the + # filter method for each one, this will act as a logical AND amongst + # each specifier. + if self._specs: + for spec in self._specs: + iterable = spec.filter(iterable, prereleases=bool(prereleases)) + return iterable + # If we do not have any specifiers, then we need to have a rough filter + # which will filter out any pre-releases, unless there are no final + # releases, and which will filter out LegacyVersion in general. + else: + filtered = [] + found_prereleases = [] + + for item in iterable: + # Ensure that we some kind of Version class for this item. + if not isinstance(item, (LegacyVersion, Version)): + parsed_version = parse(item) + else: + parsed_version = item + + # Filter out any item which is parsed as a LegacyVersion + if isinstance(parsed_version, LegacyVersion): + continue + + # Store any item which is a pre-release for later unless we've + # already found a final version or we are accepting prereleases + if parsed_version.is_prerelease and not prereleases: + if not filtered: + found_prereleases.append(item) + else: + filtered.append(item) + + # If we've found no items except for pre-releases, then we'll go + # ahead and use the pre-releases + if not filtered and found_prereleases and prereleases is None: + return found_prereleases + + return filtered diff --git a/setuptools/_vendor/packaging/utils.py b/setuptools/_vendor/packaging/utils.py new file mode 100644 index 00000000..942387ce --- /dev/null +++ b/setuptools/_vendor/packaging/utils.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import re + + +_canonicalize_regex = re.compile(r"[-_.]+") + + +def canonicalize_name(name): + # This is taken from PEP 503. + return _canonicalize_regex.sub("-", name).lower() diff --git a/setuptools/_vendor/packaging/version.py b/setuptools/_vendor/packaging/version.py new file mode 100644 index 00000000..83b5ee8c --- /dev/null +++ b/setuptools/_vendor/packaging/version.py @@ -0,0 +1,393 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import collections +import itertools +import re + +from ._structures import Infinity + + +__all__ = [ + "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" +] + + +_Version = collections.namedtuple( + "_Version", + ["epoch", "release", "dev", "pre", "post", "local"], +) + + +def parse(version): + """ + Parse the given version string and return either a :class:`Version` object + or a :class:`LegacyVersion` object depending on if the given version is + a valid PEP 440 version or a legacy version. + """ + try: + return Version(version) + except InvalidVersion: + return LegacyVersion(version) + + +class InvalidVersion(ValueError): + """ + An invalid version was found, users should refer to PEP 440. + """ + + +class _BaseVersion(object): + + def __hash__(self): + return hash(self._key) + + def __lt__(self, other): + return self._compare(other, lambda s, o: s < o) + + def __le__(self, other): + return self._compare(other, lambda s, o: s <= o) + + def __eq__(self, other): + return self._compare(other, lambda s, o: s == o) + + def __ge__(self, other): + return self._compare(other, lambda s, o: s >= o) + + def __gt__(self, other): + return self._compare(other, lambda s, o: s > o) + + def __ne__(self, other): + return self._compare(other, lambda s, o: s != o) + + def _compare(self, other, method): + if not isinstance(other, _BaseVersion): + return NotImplemented + + return method(self._key, other._key) + + +class LegacyVersion(_BaseVersion): + + def __init__(self, version): + self._version = str(version) + self._key = _legacy_cmpkey(self._version) + + def __str__(self): + return self._version + + def __repr__(self): + return "<LegacyVersion({0})>".format(repr(str(self))) + + @property + def public(self): + return self._version + + @property + def base_version(self): + return self._version + + @property + def local(self): + return None + + @property + def is_prerelease(self): + return False + + @property + def is_postrelease(self): + return False + + +_legacy_version_component_re = re.compile( + r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, +) + +_legacy_version_replacement_map = { + "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", +} + + +def _parse_version_parts(s): + for part in _legacy_version_component_re.split(s): + part = _legacy_version_replacement_map.get(part, part) + + if not part or part == ".": + continue + + if part[:1] in "0123456789": + # pad for numeric comparison + yield part.zfill(8) + else: + yield "*" + part + + # ensure that alpha/beta/candidate are before final + yield "*final" + + +def _legacy_cmpkey(version): + # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch + # greater than or equal to 0. This will effectively put the LegacyVersion, + # which uses the defacto standard originally implemented by setuptools, + # as before all PEP 440 versions. + epoch = -1 + + # This scheme is taken from pkg_resources.parse_version setuptools prior to + # it's adoption of the packaging library. + parts = [] + for part in _parse_version_parts(version.lower()): + if part.startswith("*"): + # remove "-" before a prerelease tag + if part < "*final": + while parts and parts[-1] == "*final-": + parts.pop() + + # remove trailing zeros from each series of numeric parts + while parts and parts[-1] == "00000000": + parts.pop() + + parts.append(part) + parts = tuple(parts) + + return epoch, parts + +# Deliberately not anchored to the start and end of the string, to make it +# easier for 3rd party code to reuse +VERSION_PATTERN = r""" + v? + (?: + (?:(?P<epoch>[0-9]+)!)? # epoch + (?P<release>[0-9]+(?:\.[0-9]+)*) # release segment + (?P<pre> # pre-release + [-_\.]? + (?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview)) + [-_\.]? + (?P<pre_n>[0-9]+)? + )? + (?P<post> # post release + (?:-(?P<post_n1>[0-9]+)) + | + (?: + [-_\.]? + (?P<post_l>post|rev|r) + [-_\.]? + (?P<post_n2>[0-9]+)? + ) + )? + (?P<dev> # dev release + [-_\.]? + (?P<dev_l>dev) + [-_\.]? + (?P<dev_n>[0-9]+)? + )? + ) + (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version +""" + + +class Version(_BaseVersion): + + _regex = re.compile( + r"^\s*" + VERSION_PATTERN + r"\s*$", + re.VERBOSE | re.IGNORECASE, + ) + + def __init__(self, version): + # Validate the version and parse it into pieces + match = self._regex.search(version) + if not match: + raise InvalidVersion("Invalid version: '{0}'".format(version)) + + # Store the parsed out pieces of the version + self._version = _Version( + epoch=int(match.group("epoch")) if match.group("epoch") else 0, + release=tuple(int(i) for i in match.group("release").split(".")), + pre=_parse_letter_version( + match.group("pre_l"), + match.group("pre_n"), + ), + post=_parse_letter_version( + match.group("post_l"), + match.group("post_n1") or match.group("post_n2"), + ), + dev=_parse_letter_version( + match.group("dev_l"), + match.group("dev_n"), + ), + local=_parse_local_version(match.group("local")), + ) + + # Generate a key which will be used for sorting + self._key = _cmpkey( + self._version.epoch, + self._version.release, + self._version.pre, + self._version.post, + self._version.dev, + self._version.local, + ) + + def __repr__(self): + return "<Version({0})>".format(repr(str(self))) + + def __str__(self): + parts = [] + + # Epoch + if self._version.epoch != 0: + parts.append("{0}!".format(self._version.epoch)) + + # Release segment + parts.append(".".join(str(x) for x in self._version.release)) + + # Pre-release + if self._version.pre is not None: + parts.append("".join(str(x) for x in self._version.pre)) + + # Post-release + if self._version.post is not None: + parts.append(".post{0}".format(self._version.post[1])) + + # Development release + if self._version.dev is not None: + parts.append(".dev{0}".format(self._version.dev[1])) + + # Local version segment + if self._version.local is not None: + parts.append( + "+{0}".format(".".join(str(x) for x in self._version.local)) + ) + + return "".join(parts) + + @property + def public(self): + return str(self).split("+", 1)[0] + + @property + def base_version(self): + parts = [] + + # Epoch + if self._version.epoch != 0: + parts.append("{0}!".format(self._version.epoch)) + + # Release segment + parts.append(".".join(str(x) for x in self._version.release)) + + return "".join(parts) + + @property + def local(self): + version_string = str(self) + if "+" in version_string: + return version_string.split("+", 1)[1] + + @property + def is_prerelease(self): + return bool(self._version.dev or self._version.pre) + + @property + def is_postrelease(self): + return bool(self._version.post) + + +def _parse_letter_version(letter, number): + if letter: + # We consider there to be an implicit 0 in a pre-release if there is + # not a numeral associated with it. + if number is None: + number = 0 + + # We normalize any letters to their lower case form + letter = letter.lower() + + # We consider some words to be alternate spellings of other words and + # in those cases we want to normalize the spellings to our preferred + # spelling. + if letter == "alpha": + letter = "a" + elif letter == "beta": + letter = "b" + elif letter in ["c", "pre", "preview"]: + letter = "rc" + elif letter in ["rev", "r"]: + letter = "post" + + return letter, int(number) + if not letter and number: + # We assume if we are given a number, but we are not given a letter + # then this is using the implicit post release syntax (e.g. 1.0-1) + letter = "post" + + return letter, int(number) + + +_local_version_seperators = re.compile(r"[\._-]") + + +def _parse_local_version(local): + """ + Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve"). + """ + if local is not None: + return tuple( + part.lower() if not part.isdigit() else int(part) + for part in _local_version_seperators.split(local) + ) + + +def _cmpkey(epoch, release, pre, post, dev, local): + # When we compare a release version, we want to compare it with all of the + # trailing zeros removed. So we'll use a reverse the list, drop all the now + # leading zeros until we come to something non zero, then take the rest + # re-reverse it back into the correct order and make it a tuple and use + # that for our sorting key. + release = tuple( + reversed(list( + itertools.dropwhile( + lambda x: x == 0, + reversed(release), + ) + )) + ) + + # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. + # We'll do this by abusing the pre segment, but we _only_ want to do this + # if there is not a pre or a post segment. If we have one of those then + # the normal sorting rules will handle this case correctly. + if pre is None and post is None and dev is not None: + pre = -Infinity + # Versions without a pre-release (except as noted above) should sort after + # those with one. + elif pre is None: + pre = Infinity + + # Versions without a post segment should sort before those with one. + if post is None: + post = -Infinity + + # Versions without a development segment should sort after those with one. + if dev is None: + dev = Infinity + + if local is None: + # Versions without a local segment should sort before those with one. + local = -Infinity + else: + # Versions with a local segment need that segment parsed to implement + # the sorting rules in PEP440. + # - Alpha numeric segments sort before numeric segments + # - Alpha numeric segments sort lexicographically + # - Numeric segments sort numerically + # - Shorter versions sort before longer versions when the prefixes + # match exactly + local = tuple( + (i, "") if isinstance(i, int) else (-Infinity, i) + for i in local + ) + + return epoch, release, pre, post, dev, local diff --git a/setuptools/_vendor/pyparsing.py b/setuptools/_vendor/pyparsing.py new file mode 100644 index 00000000..cf75e1e5 --- /dev/null +++ b/setuptools/_vendor/pyparsing.py @@ -0,0 +1,5742 @@ +# module pyparsing.py
+#
+# Copyright (c) 2003-2018 Paul T. McGuire
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__doc__ = \
+"""
+pyparsing module - Classes and methods to define and execute parsing grammars
+=============================================================================
+
+The pyparsing module is an alternative approach to creating and executing simple grammars,
+vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you
+don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
+provides a library of classes that you use to construct the grammar directly in Python.
+
+Here is a program to parse "Hello, World!" (or any greeting of the form
+C{"<salutation>, <addressee>!"}), built up using L{Word}, L{Literal}, and L{And} elements
+(L{'+'<ParserElement.__add__>} operator gives L{And} expressions, strings are auto-converted to
+L{Literal} expressions)::
+
+ from pyparsing import Word, alphas
+
+ # define grammar of a greeting
+ greet = Word(alphas) + "," + Word(alphas) + "!"
+
+ hello = "Hello, World!"
+ print (hello, "->", greet.parseString(hello))
+
+The program outputs the following::
+
+ Hello, World! -> ['Hello', ',', 'World', '!']
+
+The Python representation of the grammar is quite readable, owing to the self-explanatory
+class names, and the use of '+', '|' and '^' operators.
+
+The L{ParseResults} object returned from L{ParserElement.parseString<ParserElement.parseString>} can be accessed as a nested list, a dictionary, or an
+object with named attributes.
+
+The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
+ - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.)
+ - quoted strings
+ - embedded comments
+
+
+Getting Started -
+-----------------
+Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing
+classes inherit from. Use the docstrings for examples of how to:
+ - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes
+ - construct character word-group expressions using the L{Word} class
+ - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes
+ - use L{'+'<And>}, L{'|'<MatchFirst>}, L{'^'<Or>}, and L{'&'<Each>} operators to combine simple expressions into more complex ones
+ - associate names with your parsed results using L{ParserElement.setResultsName}
+ - find some helpful expression short-cuts like L{delimitedList} and L{oneOf}
+ - find more useful common expressions in the L{pyparsing_common} namespace class
+"""
+
+__version__ = "2.2.1"
+__versionTime__ = "18 Sep 2018 00:49 UTC"
+__author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"
+
+import string
+from weakref import ref as wkref
+import copy
+import sys
+import warnings
+import re
+import sre_constants
+import collections
+import pprint
+import traceback
+import types
+from datetime import datetime
+
+try:
+ from _thread import RLock
+except ImportError:
+ from threading import RLock
+
+try:
+ # Python 3
+ from collections.abc import Iterable
+ from collections.abc import MutableMapping
+except ImportError:
+ # Python 2.7
+ from collections import Iterable
+ from collections import MutableMapping
+
+try:
+ from collections import OrderedDict as _OrderedDict
+except ImportError:
+ try:
+ from ordereddict import OrderedDict as _OrderedDict
+ except ImportError:
+ _OrderedDict = None
+
+#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
+
+__all__ = [
+'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty',
+'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal',
+'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
+'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
+'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
+'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter',
+'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
+'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
+'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
+'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums',
+'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno',
+'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
+'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
+'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity',
+'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
+'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
+'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass',
+'CloseMatch', 'tokenMap', 'pyparsing_common',
+]
+
+system_version = tuple(sys.version_info)[:3]
+PY_3 = system_version[0] == 3
+if PY_3:
+ _MAX_INT = sys.maxsize
+ basestring = str
+ unichr = chr
+ _ustr = str
+
+ # build list of single arg builtins, that can be used as parse actions
+ singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max]
+
+else:
+ _MAX_INT = sys.maxint
+ range = xrange
+
+ def _ustr(obj):
+ """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
+ str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
+ then < returns the unicode object | encodes it with the default encoding | ... >.
+ """
+ if isinstance(obj,unicode):
+ return obj
+
+ try:
+ # If this works, then _ustr(obj) has the same behaviour as str(obj), so
+ # it won't break any existing code.
+ return str(obj)
+
+ except UnicodeEncodeError:
+ # Else encode it
+ ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
+ xmlcharref = Regex(r'&#\d+;')
+ xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
+ return xmlcharref.transformString(ret)
+
+ # build list of single arg builtins, tolerant of Python version, that can be used as parse actions
+ singleArgBuiltins = []
+ import __builtin__
+ for fname in "sum len sorted reversed list tuple set any all min max".split():
+ try:
+ singleArgBuiltins.append(getattr(__builtin__,fname))
+ except AttributeError:
+ continue
+
+_generatorType = type((y for y in range(1)))
+
+def _xml_escape(data):
+ """Escape &, <, >, ", ', etc. in a string of data."""
+
+ # ampersand must be replaced first
+ from_symbols = '&><"\''
+ to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split())
+ for from_,to_ in zip(from_symbols, to_symbols):
+ data = data.replace(from_, to_)
+ return data
+
+class _Constants(object):
+ pass
+
+alphas = string.ascii_uppercase + string.ascii_lowercase
+nums = "0123456789"
+hexnums = nums + "ABCDEFabcdef"
+alphanums = alphas + nums
+_bslash = chr(92)
+printables = "".join(c for c in string.printable if c not in string.whitespace)
+
+class ParseBaseException(Exception):
+ """base exception class for all parsing runtime exceptions"""
+ # Performance tuning: we construct a *lot* of these, so keep this
+ # constructor as small and fast as possible
+ def __init__( self, pstr, loc=0, msg=None, elem=None ):
+ self.loc = loc
+ if msg is None:
+ self.msg = pstr
+ self.pstr = ""
+ else:
+ self.msg = msg
+ self.pstr = pstr
+ self.parserElement = elem
+ self.args = (pstr, loc, msg)
+
+ @classmethod
+ def _from_exception(cls, pe):
+ """
+ internal factory method to simplify creating one type of ParseException
+ from another - avoids having __init__ signature conflicts among subclasses
+ """
+ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement)
+
+ def __getattr__( self, aname ):
+ """supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+ """
+ if( aname == "lineno" ):
+ return lineno( self.loc, self.pstr )
+ elif( aname in ("col", "column") ):
+ return col( self.loc, self.pstr )
+ elif( aname == "line" ):
+ return line( self.loc, self.pstr )
+ else:
+ raise AttributeError(aname)
+
+ def __str__( self ):
+ return "%s (at char %d), (line:%d, col:%d)" % \
+ ( self.msg, self.loc, self.lineno, self.column )
+ def __repr__( self ):
+ return _ustr(self)
+ def markInputline( self, markerString = ">!<" ):
+ """Extracts the exception line from the input string, and marks
+ the location of the exception with a special symbol.
+ """
+ line_str = self.line
+ line_column = self.column - 1
+ if markerString:
+ line_str = "".join((line_str[:line_column],
+ markerString, line_str[line_column:]))
+ return line_str.strip()
+ def __dir__(self):
+ return "lineno col line".split() + dir(type(self))
+
+class ParseException(ParseBaseException):
+ """
+ Exception thrown when parse expressions don't match class;
+ supported attributes by name are:
+ - lineno - returns the line number of the exception text
+ - col - returns the column number of the exception text
+ - line - returns the line containing the exception text
+
+ Example::
+ try:
+ Word(nums).setName("integer").parseString("ABC")
+ except ParseException as pe:
+ print(pe)
+ print("column: {}".format(pe.col))
+
+ prints::
+ Expected integer (at char 0), (line:1, col:1)
+ column: 1
+ """
+ pass
+
+class ParseFatalException(ParseBaseException):
+ """user-throwable exception thrown when inconsistent parse content
+ is found; stops all parsing immediately"""
+ pass
+
+class ParseSyntaxException(ParseFatalException):
+ """just like L{ParseFatalException}, but thrown internally when an
+ L{ErrorStop<And._ErrorStop>} ('-' operator) indicates that parsing is to stop
+ immediately because an unbacktrackable syntax error has been found"""
+ pass
+
+#~ class ReparseException(ParseBaseException):
+ #~ """Experimental class - parse actions can raise this exception to cause
+ #~ pyparsing to reparse the input string:
+ #~ - with a modified input string, and/or
+ #~ - with a modified start location
+ #~ Set the values of the ReparseException in the constructor, and raise the
+ #~ exception in a parse action to cause pyparsing to use the new string/location.
+ #~ Setting the values as None causes no change to be made.
+ #~ """
+ #~ def __init_( self, newstring, restartLoc ):
+ #~ self.newParseText = newstring
+ #~ self.reparseLoc = restartLoc
+
+class RecursiveGrammarException(Exception):
+ """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive"""
+ def __init__( self, parseElementList ):
+ self.parseElementTrace = parseElementList
+
+ def __str__( self ):
+ return "RecursiveGrammarException: %s" % self.parseElementTrace
+
+class _ParseResultsWithOffset(object):
+ def __init__(self,p1,p2):
+ self.tup = (p1,p2)
+ def __getitem__(self,i):
+ return self.tup[i]
+ def __repr__(self):
+ return repr(self.tup[0])
+ def setOffset(self,i):
+ self.tup = (self.tup[0],i)
+
+class ParseResults(object):
+ """
+ Structured parse results, to provide multiple means of access to the parsed data:
+ - as a list (C{len(results)})
+ - by list index (C{results[0], results[1]}, etc.)
+ - by attribute (C{results.<resultsName>} - see L{ParserElement.setResultsName})
+
+ Example::
+ integer = Word(nums)
+ date_str = (integer.setResultsName("year") + '/'
+ + integer.setResultsName("month") + '/'
+ + integer.setResultsName("day"))
+ # equivalent form:
+ # date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ # parseString returns a ParseResults object
+ result = date_str.parseString("1999/12/31")
+
+ def test(s, fn=repr):
+ print("%s -> %s" % (s, fn(eval(s))))
+ test("list(result)")
+ test("result[0]")
+ test("result['month']")
+ test("result.day")
+ test("'month' in result")
+ test("'minutes' in result")
+ test("result.dump()", str)
+ prints::
+ list(result) -> ['1999', '/', '12', '/', '31']
+ result[0] -> '1999'
+ result['month'] -> '12'
+ result.day -> '31'
+ 'month' in result -> True
+ 'minutes' in result -> False
+ result.dump() -> ['1999', '/', '12', '/', '31']
+ - day: 31
+ - month: 12
+ - year: 1999
+ """
+ def __new__(cls, toklist=None, name=None, asList=True, modal=True ):
+ if isinstance(toklist, cls):
+ return toklist
+ retobj = object.__new__(cls)
+ retobj.__doinit = True
+ return retobj
+
+ # Performance tuning: we construct a *lot* of these, so keep this
+ # constructor as small and fast as possible
+ def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance ):
+ if self.__doinit:
+ self.__doinit = False
+ self.__name = None
+ self.__parent = None
+ self.__accumNames = {}
+ self.__asList = asList
+ self.__modal = modal
+ if toklist is None:
+ toklist = []
+ if isinstance(toklist, list):
+ self.__toklist = toklist[:]
+ elif isinstance(toklist, _generatorType):
+ self.__toklist = list(toklist)
+ else:
+ self.__toklist = [toklist]
+ self.__tokdict = dict()
+
+ if name is not None and name:
+ if not modal:
+ self.__accumNames[name] = 0
+ if isinstance(name,int):
+ name = _ustr(name) # will always return a str, but use _ustr for consistency
+ self.__name = name
+ if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])):
+ if isinstance(toklist,basestring):
+ toklist = [ toklist ]
+ if asList:
+ if isinstance(toklist,ParseResults):
+ self[name] = _ParseResultsWithOffset(toklist.copy(),0)
+ else:
+ self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0)
+ self[name].__name = name
+ else:
+ try:
+ self[name] = toklist[0]
+ except (KeyError,TypeError,IndexError):
+ self[name] = toklist
+
+ def __getitem__( self, i ):
+ if isinstance( i, (int,slice) ):
+ return self.__toklist[i]
+ else:
+ if i not in self.__accumNames:
+ return self.__tokdict[i][-1][0]
+ else:
+ return ParseResults([ v[0] for v in self.__tokdict[i] ])
+
+ def __setitem__( self, k, v, isinstance=isinstance ):
+ if isinstance(v,_ParseResultsWithOffset):
+ self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
+ sub = v[0]
+ elif isinstance(k,(int,slice)):
+ self.__toklist[k] = v
+ sub = v
+ else:
+ self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
+ sub = v
+ if isinstance(sub,ParseResults):
+ sub.__parent = wkref(self)
+
+ def __delitem__( self, i ):
+ if isinstance(i,(int,slice)):
+ mylen = len( self.__toklist )
+ del self.__toklist[i]
+
+ # convert int to slice
+ if isinstance(i, int):
+ if i < 0:
+ i += mylen
+ i = slice(i, i+1)
+ # get removed indices
+ removed = list(range(*i.indices(mylen)))
+ removed.reverse()
+ # fixup indices in token dictionary
+ for name,occurrences in self.__tokdict.items():
+ for j in removed:
+ for k, (value, position) in enumerate(occurrences):
+ occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
+ else:
+ del self.__tokdict[i]
+
+ def __contains__( self, k ):
+ return k in self.__tokdict
+
+ def __len__( self ): return len( self.__toklist )
+ def __bool__(self): return ( not not self.__toklist )
+ __nonzero__ = __bool__
+ def __iter__( self ): return iter( self.__toklist )
+ def __reversed__( self ): return iter( self.__toklist[::-1] )
+ def _iterkeys( self ):
+ if hasattr(self.__tokdict, "iterkeys"):
+ return self.__tokdict.iterkeys()
+ else:
+ return iter(self.__tokdict)
+
+ def _itervalues( self ):
+ return (self[k] for k in self._iterkeys())
+
+ def _iteritems( self ):
+ return ((k, self[k]) for k in self._iterkeys())
+
+ if PY_3:
+ keys = _iterkeys
+ """Returns an iterator of all named result keys (Python 3.x only)."""
+
+ values = _itervalues
+ """Returns an iterator of all named result values (Python 3.x only)."""
+
+ items = _iteritems
+ """Returns an iterator of all named result key-value tuples (Python 3.x only)."""
+
+ else:
+ iterkeys = _iterkeys
+ """Returns an iterator of all named result keys (Python 2.x only)."""
+
+ itervalues = _itervalues
+ """Returns an iterator of all named result values (Python 2.x only)."""
+
+ iteritems = _iteritems
+ """Returns an iterator of all named result key-value tuples (Python 2.x only)."""
+
+ def keys( self ):
+ """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x)."""
+ return list(self.iterkeys())
+
+ def values( self ):
+ """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x)."""
+ return list(self.itervalues())
+
+ def items( self ):
+ """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x)."""
+ return list(self.iteritems())
+
+ def haskeys( self ):
+ """Since keys() returns an iterator, this method is helpful in bypassing
+ code that looks for the existence of any defined results names."""
+ return bool(self.__tokdict)
+
+ def pop( self, *args, **kwargs):
+ """
+ Removes and returns item at specified index (default=C{last}).
+ Supports both C{list} and C{dict} semantics for C{pop()}. If passed no
+ argument or an integer argument, it will use C{list} semantics
+ and pop tokens from the list of parsed tokens. If passed a
+ non-integer argument (most likely a string), it will use C{dict}
+ semantics and pop the corresponding value from any defined
+ results names. A second default return value argument is
+ supported, just as in C{dict.pop()}.
+
+ Example::
+ def remove_first(tokens):
+ tokens.pop(0)
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+ print(OneOrMore(Word(nums)).addParseAction(remove_first).parseString("0 123 321")) # -> ['123', '321']
+
+ label = Word(alphas)
+ patt = label("LABEL") + OneOrMore(Word(nums))
+ print(patt.parseString("AAB 123 321").dump())
+
+ # Use pop() in a parse action to remove named result (note that corresponding value is not
+ # removed from list form of results)
+ def remove_LABEL(tokens):
+ tokens.pop("LABEL")
+ return tokens
+ patt.addParseAction(remove_LABEL)
+ print(patt.parseString("AAB 123 321").dump())
+ prints::
+ ['AAB', '123', '321']
+ - LABEL: AAB
+
+ ['AAB', '123', '321']
+ """
+ if not args:
+ args = [-1]
+ for k,v in kwargs.items():
+ if k == 'default':
+ args = (args[0], v)
+ else:
+ raise TypeError("pop() got an unexpected keyword argument '%s'" % k)
+ if (isinstance(args[0], int) or
+ len(args) == 1 or
+ args[0] in self):
+ index = args[0]
+ ret = self[index]
+ del self[index]
+ return ret
+ else:
+ defaultvalue = args[1]
+ return defaultvalue
+
+ def get(self, key, defaultValue=None):
+ """
+ Returns named result matching the given key, or if there is no
+ such name, then returns the given C{defaultValue} or C{None} if no
+ C{defaultValue} is specified.
+
+ Similar to C{dict.get()}.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString("1999/12/31")
+ print(result.get("year")) # -> '1999'
+ print(result.get("hour", "not specified")) # -> 'not specified'
+ print(result.get("hour")) # -> None
+ """
+ if key in self:
+ return self[key]
+ else:
+ return defaultValue
+
+ def insert( self, index, insStr ):
+ """
+ Inserts new element at location index in the list of parsed tokens.
+
+ Similar to C{list.insert()}.
+
+ Example::
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+
+ # use a parse action to insert the parse location in the front of the parsed results
+ def insert_locn(locn, tokens):
+ tokens.insert(0, locn)
+ print(OneOrMore(Word(nums)).addParseAction(insert_locn).parseString("0 123 321")) # -> [0, '0', '123', '321']
+ """
+ self.__toklist.insert(index, insStr)
+ # fixup indices in token dictionary
+ for name,occurrences in self.__tokdict.items():
+ for k, (value, position) in enumerate(occurrences):
+ occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
+
+ def append( self, item ):
+ """
+ Add single element to end of ParseResults list of elements.
+
+ Example::
+ print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+
+ # use a parse action to compute the sum of the parsed integers, and add it to the end
+ def append_sum(tokens):
+ tokens.append(sum(map(int, tokens)))
+ print(OneOrMore(Word(nums)).addParseAction(append_sum).parseString("0 123 321")) # -> ['0', '123', '321', 444]
+ """
+ self.__toklist.append(item)
+
+ def extend( self, itemseq ):
+ """
+ Add sequence of elements to end of ParseResults list of elements.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+
+ # use a parse action to append the reverse of the matched strings, to make a palindrome
+ def make_palindrome(tokens):
+ tokens.extend(reversed([t[::-1] for t in tokens]))
+ return ''.join(tokens)
+ print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl'
+ """
+ if isinstance(itemseq, ParseResults):
+ self += itemseq
+ else:
+ self.__toklist.extend(itemseq)
+
+ def clear( self ):
+ """
+ Clear all elements and results names.
+ """
+ del self.__toklist[:]
+ self.__tokdict.clear()
+
+ def __getattr__( self, name ):
+ try:
+ return self[name]
+ except KeyError:
+ return ""
+
+ if name in self.__tokdict:
+ if name not in self.__accumNames:
+ return self.__tokdict[name][-1][0]
+ else:
+ return ParseResults([ v[0] for v in self.__tokdict[name] ])
+ else:
+ return ""
+
+ def __add__( self, other ):
+ ret = self.copy()
+ ret += other
+ return ret
+
+ def __iadd__( self, other ):
+ if other.__tokdict:
+ offset = len(self.__toklist)
+ addoffset = lambda a: offset if a<0 else a+offset
+ otheritems = other.__tokdict.items()
+ otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
+ for (k,vlist) in otheritems for v in vlist]
+ for k,v in otherdictitems:
+ self[k] = v
+ if isinstance(v[0],ParseResults):
+ v[0].__parent = wkref(self)
+
+ self.__toklist += other.__toklist
+ self.__accumNames.update( other.__accumNames )
+ return self
+
+ def __radd__(self, other):
+ if isinstance(other,int) and other == 0:
+ # useful for merging many ParseResults using sum() builtin
+ return self.copy()
+ else:
+ # this may raise a TypeError - so be it
+ return other + self
+
+ def __repr__( self ):
+ return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
+
+ def __str__( self ):
+ return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']'
+
+ def _asStringList( self, sep='' ):
+ out = []
+ for item in self.__toklist:
+ if out and sep:
+ out.append(sep)
+ if isinstance( item, ParseResults ):
+ out += item._asStringList()
+ else:
+ out.append( _ustr(item) )
+ return out
+
+ def asList( self ):
+ """
+ Returns the parse results as a nested list of matching tokens, all converted to strings.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+ result = patt.parseString("sldkj lsdkj sldkj")
+ # even though the result prints in string-like form, it is actually a pyparsing ParseResults
+ print(type(result), result) # -> <class 'pyparsing.ParseResults'> ['sldkj', 'lsdkj', 'sldkj']
+
+ # Use asList() to create an actual list
+ result_list = result.asList()
+ print(type(result_list), result_list) # -> <class 'list'> ['sldkj', 'lsdkj', 'sldkj']
+ """
+ return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist]
+
+ def asDict( self ):
+ """
+ Returns the named parse results as a nested dictionary.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString('12/31/1999')
+ print(type(result), repr(result)) # -> <class 'pyparsing.ParseResults'> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]})
+
+ result_dict = result.asDict()
+ print(type(result_dict), repr(result_dict)) # -> <class 'dict'> {'day': '1999', 'year': '12', 'month': '31'}
+
+ # even though a ParseResults supports dict-like access, sometime you just need to have a dict
+ import json
+ print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable
+ print(json.dumps(result.asDict())) # -> {"month": "31", "day": "1999", "year": "12"}
+ """
+ if PY_3:
+ item_fn = self.items
+ else:
+ item_fn = self.iteritems
+
+ def toItem(obj):
+ if isinstance(obj, ParseResults):
+ if obj.haskeys():
+ return obj.asDict()
+ else:
+ return [toItem(v) for v in obj]
+ else:
+ return obj
+
+ return dict((k,toItem(v)) for k,v in item_fn())
+
+ def copy( self ):
+ """
+ Returns a new copy of a C{ParseResults} object.
+ """
+ ret = ParseResults( self.__toklist )
+ ret.__tokdict = self.__tokdict.copy()
+ ret.__parent = self.__parent
+ ret.__accumNames.update( self.__accumNames )
+ ret.__name = self.__name
+ return ret
+
+ def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
+ """
+ (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names.
+ """
+ nl = "\n"
+ out = []
+ namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()
+ for v in vlist)
+ nextLevelIndent = indent + " "
+
+ # collapse out indents if formatting is not desired
+ if not formatted:
+ indent = ""
+ nextLevelIndent = ""
+ nl = ""
+
+ selfTag = None
+ if doctag is not None:
+ selfTag = doctag
+ else:
+ if self.__name:
+ selfTag = self.__name
+
+ if not selfTag:
+ if namedItemsOnly:
+ return ""
+ else:
+ selfTag = "ITEM"
+
+ out += [ nl, indent, "<", selfTag, ">" ]
+
+ for i,res in enumerate(self.__toklist):
+ if isinstance(res,ParseResults):
+ if i in namedItems:
+ out += [ res.asXML(namedItems[i],
+ namedItemsOnly and doctag is None,
+ nextLevelIndent,
+ formatted)]
+ else:
+ out += [ res.asXML(None,
+ namedItemsOnly and doctag is None,
+ nextLevelIndent,
+ formatted)]
+ else:
+ # individual token, see if there is a name for it
+ resTag = None
+ if i in namedItems:
+ resTag = namedItems[i]
+ if not resTag:
+ if namedItemsOnly:
+ continue
+ else:
+ resTag = "ITEM"
+ xmlBodyText = _xml_escape(_ustr(res))
+ out += [ nl, nextLevelIndent, "<", resTag, ">",
+ xmlBodyText,
+ "</", resTag, ">" ]
+
+ out += [ nl, indent, "</", selfTag, ">" ]
+ return "".join(out)
+
+ def __lookup(self,sub):
+ for k,vlist in self.__tokdict.items():
+ for v,loc in vlist:
+ if sub is v:
+ return k
+ return None
+
+ def getName(self):
+ r"""
+ Returns the results name for this token expression. Useful when several
+ different expressions might match at a particular location.
+
+ Example::
+ integer = Word(nums)
+ ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d")
+ house_number_expr = Suppress('#') + Word(nums, alphanums)
+ user_data = (Group(house_number_expr)("house_number")
+ | Group(ssn_expr)("ssn")
+ | Group(integer)("age"))
+ user_info = OneOrMore(user_data)
+
+ result = user_info.parseString("22 111-22-3333 #221B")
+ for item in result:
+ print(item.getName(), ':', item[0])
+ prints::
+ age : 22
+ ssn : 111-22-3333
+ house_number : 221B
+ """
+ if self.__name:
+ return self.__name
+ elif self.__parent:
+ par = self.__parent()
+ if par:
+ return par.__lookup(self)
+ else:
+ return None
+ elif (len(self) == 1 and
+ len(self.__tokdict) == 1 and
+ next(iter(self.__tokdict.values()))[0][1] in (0,-1)):
+ return next(iter(self.__tokdict.keys()))
+ else:
+ return None
+
+ def dump(self, indent='', depth=0, full=True):
+ """
+ Diagnostic method for listing out the contents of a C{ParseResults}.
+ Accepts an optional C{indent} argument so that this string can be embedded
+ in a nested display of other data.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ result = date_str.parseString('12/31/1999')
+ print(result.dump())
+ prints::
+ ['12', '/', '31', '/', '1999']
+ - day: 1999
+ - month: 31
+ - year: 12
+ """
+ out = []
+ NL = '\n'
+ out.append( indent+_ustr(self.asList()) )
+ if full:
+ if self.haskeys():
+ items = sorted((str(k), v) for k,v in self.items())
+ for k,v in items:
+ if out:
+ out.append(NL)
+ out.append( "%s%s- %s: " % (indent,(' '*depth), k) )
+ if isinstance(v,ParseResults):
+ if v:
+ out.append( v.dump(indent,depth+1) )
+ else:
+ out.append(_ustr(v))
+ else:
+ out.append(repr(v))
+ elif any(isinstance(vv,ParseResults) for vv in self):
+ v = self
+ for i,vv in enumerate(v):
+ if isinstance(vv,ParseResults):
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) ))
+ else:
+ out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv)))
+
+ return "".join(out)
+
+ def pprint(self, *args, **kwargs):
+ """
+ Pretty-printer for parsed results as a list, using the C{pprint} module.
+ Accepts additional positional or keyword args as defined for the
+ C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})
+
+ Example::
+ ident = Word(alphas, alphanums)
+ num = Word(nums)
+ func = Forward()
+ term = ident | num | Group('(' + func + ')')
+ func <<= ident + Group(Optional(delimitedList(term)))
+ result = func.parseString("fna a,b,(fnb c,d,200),100")
+ result.pprint(width=40)
+ prints::
+ ['fna',
+ ['a',
+ 'b',
+ ['(', 'fnb', ['c', 'd', '200'], ')'],
+ '100']]
+ """
+ pprint.pprint(self.asList(), *args, **kwargs)
+
+ # add support for pickle protocol
+ def __getstate__(self):
+ return ( self.__toklist,
+ ( self.__tokdict.copy(),
+ self.__parent is not None and self.__parent() or None,
+ self.__accumNames,
+ self.__name ) )
+
+ def __setstate__(self,state):
+ self.__toklist = state[0]
+ (self.__tokdict,
+ par,
+ inAccumNames,
+ self.__name) = state[1]
+ self.__accumNames = {}
+ self.__accumNames.update(inAccumNames)
+ if par is not None:
+ self.__parent = wkref(par)
+ else:
+ self.__parent = None
+
+ def __getnewargs__(self):
+ return self.__toklist, self.__name, self.__asList, self.__modal
+
+ def __dir__(self):
+ return (dir(type(self)) + list(self.keys()))
+
+MutableMapping.register(ParseResults)
+
+def col (loc,strg):
+ """Returns current column within a string, counting newlines as line separators.
+ The first column is number 1.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ s = strg
+ return 1 if 0<loc<len(s) and s[loc-1] == '\n' else loc - s.rfind("\n", 0, loc)
+
+def lineno(loc,strg):
+ """Returns current line number within a string, counting newlines as line separators.
+ The first line is number 1.
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+ """
+ return strg.count("\n",0,loc) + 1
+
+def line( loc, strg ):
+ """Returns the line of text containing loc within a string, counting newlines as line separators.
+ """
+ lastCR = strg.rfind("\n", 0, loc)
+ nextCR = strg.find("\n", loc)
+ if nextCR >= 0:
+ return strg[lastCR+1:nextCR]
+ else:
+ return strg[lastCR+1:]
+
+def _defaultStartDebugAction( instring, loc, expr ):
+ print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )))
+
+def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
+ print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
+
+def _defaultExceptionDebugAction( instring, loc, expr, exc ):
+ print ("Exception raised:" + _ustr(exc))
+
+def nullDebugAction(*args):
+ """'Do-nothing' debug action, to suppress debugging output during parsing."""
+ pass
+
+# Only works on Python 3.x - nonlocal is toxic to Python 2 installs
+#~ 'decorator to trim function calls to match the arity of the target'
+#~ def _trim_arity(func, maxargs=3):
+ #~ if func in singleArgBuiltins:
+ #~ return lambda s,l,t: func(t)
+ #~ limit = 0
+ #~ foundArity = False
+ #~ def wrapper(*args):
+ #~ nonlocal limit,foundArity
+ #~ while 1:
+ #~ try:
+ #~ ret = func(*args[limit:])
+ #~ foundArity = True
+ #~ return ret
+ #~ except TypeError:
+ #~ if limit == maxargs or foundArity:
+ #~ raise
+ #~ limit += 1
+ #~ continue
+ #~ return wrapper
+
+# this version is Python 2.x-3.x cross-compatible
+'decorator to trim function calls to match the arity of the target'
+def _trim_arity(func, maxargs=2):
+ if func in singleArgBuiltins:
+ return lambda s,l,t: func(t)
+ limit = [0]
+ foundArity = [False]
+
+ # traceback return data structure changed in Py3.5 - normalize back to plain tuples
+ if system_version[:2] >= (3,5):
+ def extract_stack(limit=0):
+ # special handling for Python 3.5.0 - extra deep call stack by 1
+ offset = -3 if system_version == (3,5,0) else -2
+ frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
+ return [frame_summary[:2]]
+ def extract_tb(tb, limit=0):
+ frames = traceback.extract_tb(tb, limit=limit)
+ frame_summary = frames[-1]
+ return [frame_summary[:2]]
+ else:
+ extract_stack = traceback.extract_stack
+ extract_tb = traceback.extract_tb
+
+ # synthesize what would be returned by traceback.extract_stack at the call to
+ # user's parse action 'func', so that we don't incur call penalty at parse time
+
+ LINE_DIFF = 6
+ # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND
+ # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
+ this_line = extract_stack(limit=2)[-1]
+ pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF)
+
+ def wrapper(*args):
+ while 1:
+ try:
+ ret = func(*args[limit[0]:])
+ foundArity[0] = True
+ return ret
+ except TypeError:
+ # re-raise TypeErrors if they did not come from our arity testing
+ if foundArity[0]:
+ raise
+ else:
+ try:
+ tb = sys.exc_info()[-1]
+ if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth:
+ raise
+ finally:
+ del tb
+
+ if limit[0] <= maxargs:
+ limit[0] += 1
+ continue
+ raise
+
+ # copy func name to wrapper for sensible debug output
+ func_name = "<parse action>"
+ try:
+ func_name = getattr(func, '__name__',
+ getattr(func, '__class__').__name__)
+ except Exception:
+ func_name = str(func)
+ wrapper.__name__ = func_name
+
+ return wrapper
+
+class ParserElement(object):
+ """Abstract base level parser element class."""
+ DEFAULT_WHITE_CHARS = " \n\t\r"
+ verbose_stacktrace = False
+
+ @staticmethod
+ def setDefaultWhitespaceChars( chars ):
+ r"""
+ Overrides the default whitespace chars
+
+ Example::
+ # default whitespace chars are space, <TAB> and newline
+ OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl']
+
+ # change to just treat newline as significant
+ ParserElement.setDefaultWhitespaceChars(" \t")
+ OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def']
+ """
+ ParserElement.DEFAULT_WHITE_CHARS = chars
+
+ @staticmethod
+ def inlineLiteralsUsing(cls):
+ """
+ Set class to be used for inclusion of string literals into a parser.
+
+ Example::
+ # default literal class used is Literal
+ integer = Word(nums)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31']
+
+
+ # change to Suppress
+ ParserElement.inlineLiteralsUsing(Suppress)
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+ date_str.parseString("1999/12/31") # -> ['1999', '12', '31']
+ """
+ ParserElement._literalStringClass = cls
+
+ def __init__( self, savelist=False ):
+ self.parseAction = list()
+ self.failAction = None
+ #~ self.name = "<unknown>" # don't define self.name, let subclasses try/except upcall
+ self.strRepr = None
+ self.resultsName = None
+ self.saveAsList = savelist
+ self.skipWhitespace = True
+ self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+ self.copyDefaultWhiteChars = True
+ self.mayReturnEmpty = False # used when checking for left-recursion
+ self.keepTabs = False
+ self.ignoreExprs = list()
+ self.debug = False
+ self.streamlined = False
+ self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
+ self.errmsg = ""
+ self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
+ self.debugActions = ( None, None, None ) #custom debug actions
+ self.re = None
+ self.callPreparse = True # used to avoid redundant calls to preParse
+ self.callDuringTry = False
+
+ def copy( self ):
+ """
+ Make a copy of this C{ParserElement}. Useful for defining different parse actions
+ for the same parsing pattern, using copies of the original parse element.
+
+ Example::
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K")
+ integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+
+ print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M"))
+ prints::
+ [5120, 100, 655360, 268435456]
+ Equivalent form of C{expr.copy()} is just C{expr()}::
+ integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+ """
+ cpy = copy.copy( self )
+ cpy.parseAction = self.parseAction[:]
+ cpy.ignoreExprs = self.ignoreExprs[:]
+ if self.copyDefaultWhiteChars:
+ cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+ return cpy
+
+ def setName( self, name ):
+ """
+ Define name for this expression, makes debugging and exception messages clearer.
+
+ Example::
+ Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1)
+ Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1)
+ """
+ self.name = name
+ self.errmsg = "Expected " + self.name
+ if hasattr(self,"exception"):
+ self.exception.msg = self.errmsg
+ return self
+
+ def setResultsName( self, name, listAllMatches=False ):
+ """
+ Define name for referencing matching tokens as a nested attribute
+ of the returned parse results.
+ NOTE: this returns a *copy* of the original C{ParserElement} object;
+ this is so that the client can define a basic element, such as an
+ integer, and reference it in multiple places with different names.
+
+ You can also set results names using the abbreviated syntax,
+ C{expr("name")} in place of C{expr.setResultsName("name")} -
+ see L{I{__call__}<__call__>}.
+
+ Example::
+ date_str = (integer.setResultsName("year") + '/'
+ + integer.setResultsName("month") + '/'
+ + integer.setResultsName("day"))
+
+ # equivalent form:
+ date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+ """
+ newself = self.copy()
+ if name.endswith("*"):
+ name = name[:-1]
+ listAllMatches=True
+ newself.resultsName = name
+ newself.modalResults = not listAllMatches
+ return newself
+
+ def setBreak(self,breakFlag = True):
+ """Method to invoke the Python pdb debugger when this element is
+ about to be parsed. Set C{breakFlag} to True to enable, False to
+ disable.
+ """
+ if breakFlag:
+ _parseMethod = self._parse
+ def breaker(instring, loc, doActions=True, callPreParse=True):
+ import pdb
+ pdb.set_trace()
+ return _parseMethod( instring, loc, doActions, callPreParse )
+ breaker._originalParseMethod = _parseMethod
+ self._parse = breaker
+ else:
+ if hasattr(self._parse,"_originalParseMethod"):
+ self._parse = self._parse._originalParseMethod
+ return self
+
+ def setParseAction( self, *fns, **kwargs ):
+ """
+ Define one or more actions to perform when successfully matching parse element definition.
+ Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
+ C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
+ - s = the original string being parsed (see note below)
+ - loc = the location of the matching substring
+ - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
+ If the functions in fns modify the tokens, they can return them as the return
+ value from fn, and the modified list of tokens will replace the original.
+ Otherwise, fn does not need to return any value.
+
+ Optional keyword arguments:
+ - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing
+
+ Note: the default parsing behavior is to expand tabs in the input string
+ before starting the parsing process. See L{I{parseString}<parseString>} for more information
+ on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
+ consistent view of the parsed string, the parse location, and line and column
+ positions within the parsed string.
+
+ Example::
+ integer = Word(nums)
+ date_str = integer + '/' + integer + '/' + integer
+
+ date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31']
+
+ # use parse action to convert to ints at parse time
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ date_str = integer + '/' + integer + '/' + integer
+
+ # note that integer fields are now ints, not strings
+ date_str.parseString("1999/12/31") # -> [1999, '/', 12, '/', 31]
+ """
+ self.parseAction = list(map(_trim_arity, list(fns)))
+ self.callDuringTry = kwargs.get("callDuringTry", False)
+ return self
+
+ def addParseAction( self, *fns, **kwargs ):
+ """
+ Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}.
+
+ See examples in L{I{copy}<copy>}.
+ """
+ self.parseAction += list(map(_trim_arity, list(fns)))
+ self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+ return self
+
+ def addCondition(self, *fns, **kwargs):
+ """Add a boolean predicate function to expression's list of parse actions. See
+ L{I{setParseAction}<setParseAction>} for function call signatures. Unlike C{setParseAction},
+ functions passed to C{addCondition} need to return boolean success/fail of the condition.
+
+ Optional keyword arguments:
+ - message = define a custom message to be used in the raised exception
+ - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException
+
+ Example::
+ integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ year_int = integer.copy()
+ year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later")
+ date_str = year_int + '/' + integer + '/' + integer
+
+ result = date_str.parseString("1999/12/31") # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1)
+ """
+ msg = kwargs.get("message", "failed user-defined condition")
+ exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException
+ for fn in fns:
+ def pa(s,l,t):
+ if not bool(_trim_arity(fn)(s,l,t)):
+ raise exc_type(s,l,msg)
+ self.parseAction.append(pa)
+ self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+ return self
+
+ def setFailAction( self, fn ):
+ """Define action to perform if parsing fails at this expression.
+ Fail acton fn is a callable function that takes the arguments
+ C{fn(s,loc,expr,err)} where:
+ - s = string being parsed
+ - loc = location where expression match was attempted and failed
+ - expr = the parse expression that failed
+ - err = the exception thrown
+ The function returns no value. It may throw C{L{ParseFatalException}}
+ if it is desired to stop parsing immediately."""
+ self.failAction = fn
+ return self
+
+ def _skipIgnorables( self, instring, loc ):
+ exprsFound = True
+ while exprsFound:
+ exprsFound = False
+ for e in self.ignoreExprs:
+ try:
+ while 1:
+ loc,dummy = e._parse( instring, loc )
+ exprsFound = True
+ except ParseException:
+ pass
+ return loc
+
+ def preParse( self, instring, loc ):
+ if self.ignoreExprs:
+ loc = self._skipIgnorables( instring, loc )
+
+ if self.skipWhitespace:
+ wt = self.whiteChars
+ instrlen = len(instring)
+ while loc < instrlen and instring[loc] in wt:
+ loc += 1
+
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ return loc, []
+
+ def postParse( self, instring, loc, tokenlist ):
+ return tokenlist
+
+ #~ @profile
+ def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
+ debugging = ( self.debug ) #and doActions )
+
+ if debugging or self.failAction:
+ #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
+ if (self.debugActions[0] ):
+ self.debugActions[0]( instring, loc, self )
+ if callPreParse and self.callPreparse:
+ preloc = self.preParse( instring, loc )
+ else:
+ preloc = loc
+ tokensStart = preloc
+ try:
+ try:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+ except IndexError:
+ raise ParseException( instring, len(instring), self.errmsg, self )
+ except ParseBaseException as err:
+ #~ print ("Exception raised:", err)
+ if self.debugActions[2]:
+ self.debugActions[2]( instring, tokensStart, self, err )
+ if self.failAction:
+ self.failAction( instring, tokensStart, self, err )
+ raise
+ else:
+ if callPreParse and self.callPreparse:
+ preloc = self.preParse( instring, loc )
+ else:
+ preloc = loc
+ tokensStart = preloc
+ if self.mayIndexError or preloc >= len(instring):
+ try:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+ except IndexError:
+ raise ParseException( instring, len(instring), self.errmsg, self )
+ else:
+ loc,tokens = self.parseImpl( instring, preloc, doActions )
+
+ tokens = self.postParse( instring, loc, tokens )
+
+ retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
+ if self.parseAction and (doActions or self.callDuringTry):
+ if debugging:
+ try:
+ for fn in self.parseAction:
+ tokens = fn( instring, tokensStart, retTokens )
+ if tokens is not None:
+ retTokens = ParseResults( tokens,
+ self.resultsName,
+ asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+ modal=self.modalResults )
+ except ParseBaseException as err:
+ #~ print "Exception raised in user parse action:", err
+ if (self.debugActions[2] ):
+ self.debugActions[2]( instring, tokensStart, self, err )
+ raise
+ else:
+ for fn in self.parseAction:
+ tokens = fn( instring, tokensStart, retTokens )
+ if tokens is not None:
+ retTokens = ParseResults( tokens,
+ self.resultsName,
+ asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+ modal=self.modalResults )
+ if debugging:
+ #~ print ("Matched",self,"->",retTokens.asList())
+ if (self.debugActions[1] ):
+ self.debugActions[1]( instring, tokensStart, loc, self, retTokens )
+
+ return loc, retTokens
+
+ def tryParse( self, instring, loc ):
+ try:
+ return self._parse( instring, loc, doActions=False )[0]
+ except ParseFatalException:
+ raise ParseException( instring, loc, self.errmsg, self)
+
+ def canParseNext(self, instring, loc):
+ try:
+ self.tryParse(instring, loc)
+ except (ParseException, IndexError):
+ return False
+ else:
+ return True
+
+ class _UnboundedCache(object):
+ def __init__(self):
+ cache = {}
+ self.not_in_cache = not_in_cache = object()
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+
+ def clear(self):
+ cache.clear()
+
+ def cache_len(self):
+ return len(cache)
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
+
+ if _OrderedDict is not None:
+ class _FifoCache(object):
+ def __init__(self, size):
+ self.not_in_cache = not_in_cache = object()
+
+ cache = _OrderedDict()
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+ while len(cache) > size:
+ try:
+ cache.popitem(False)
+ except KeyError:
+ pass
+
+ def clear(self):
+ cache.clear()
+
+ def cache_len(self):
+ return len(cache)
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
+
+ else:
+ class _FifoCache(object):
+ def __init__(self, size):
+ self.not_in_cache = not_in_cache = object()
+
+ cache = {}
+ key_fifo = collections.deque([], size)
+
+ def get(self, key):
+ return cache.get(key, not_in_cache)
+
+ def set(self, key, value):
+ cache[key] = value
+ while len(key_fifo) > size:
+ cache.pop(key_fifo.popleft(), None)
+ key_fifo.append(key)
+
+ def clear(self):
+ cache.clear()
+ key_fifo.clear()
+
+ def cache_len(self):
+ return len(cache)
+
+ self.get = types.MethodType(get, self)
+ self.set = types.MethodType(set, self)
+ self.clear = types.MethodType(clear, self)
+ self.__len__ = types.MethodType(cache_len, self)
+
+ # argument cache for optimizing repeated calls when backtracking through recursive expressions
+ packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail
+ packrat_cache_lock = RLock()
+ packrat_cache_stats = [0, 0]
+
+ # this method gets repeatedly called during backtracking with the same arguments -
+ # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
+ def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
+ HIT, MISS = 0, 1
+ lookup = (self, instring, loc, callPreParse, doActions)
+ with ParserElement.packrat_cache_lock:
+ cache = ParserElement.packrat_cache
+ value = cache.get(lookup)
+ if value is cache.not_in_cache:
+ ParserElement.packrat_cache_stats[MISS] += 1
+ try:
+ value = self._parseNoCache(instring, loc, doActions, callPreParse)
+ except ParseBaseException as pe:
+ # cache a copy of the exception, without the traceback
+ cache.set(lookup, pe.__class__(*pe.args))
+ raise
+ else:
+ cache.set(lookup, (value[0], value[1].copy()))
+ return value
+ else:
+ ParserElement.packrat_cache_stats[HIT] += 1
+ if isinstance(value, Exception):
+ raise value
+ return (value[0], value[1].copy())
+
+ _parse = _parseNoCache
+
+ @staticmethod
+ def resetCache():
+ ParserElement.packrat_cache.clear()
+ ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats)
+
+ _packratEnabled = False
+ @staticmethod
+ def enablePackrat(cache_size_limit=128):
+ """Enables "packrat" parsing, which adds memoizing to the parsing logic.
+ Repeated parse attempts at the same string location (which happens
+ often in many complex grammars) can immediately return a cached value,
+ instead of re-executing parsing/validating code. Memoizing is done of
+ both valid results and parsing exceptions.
+
+ Parameters:
+ - cache_size_limit - (default=C{128}) - if an integer value is provided
+ will limit the size of the packrat cache; if None is passed, then
+ the cache size will be unbounded; if 0 is passed, the cache will
+ be effectively disabled.
+
+ This speedup may break existing programs that use parse actions that
+ have side-effects. For this reason, packrat parsing is disabled when
+ you first import pyparsing. To activate the packrat feature, your
+ program must call the class method C{ParserElement.enablePackrat()}. If
+ your program uses C{psyco} to "compile as you go", you must call
+ C{enablePackrat} before calling C{psyco.full()}. If you do not do this,
+ Python will crash. For best results, call C{enablePackrat()} immediately
+ after importing pyparsing.
+
+ Example::
+ import pyparsing
+ pyparsing.ParserElement.enablePackrat()
+ """
+ if not ParserElement._packratEnabled:
+ ParserElement._packratEnabled = True
+ if cache_size_limit is None:
+ ParserElement.packrat_cache = ParserElement._UnboundedCache()
+ else:
+ ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit)
+ ParserElement._parse = ParserElement._parseCache
+
+ def parseString( self, instring, parseAll=False ):
+ """
+ Execute the parse expression with the given string.
+ This is the main interface to the client code, once the complete
+ expression has been built.
+
+ If you want the grammar to require that the entire input string be
+ successfully parsed, then set C{parseAll} to True (equivalent to ending
+ the grammar with C{L{StringEnd()}}).
+
+ Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
+ in order to report proper column numbers in parse actions.
+ If the input string contains tabs and
+ the grammar uses parse actions that use the C{loc} argument to index into the
+ string being parsed, you can ensure you have a consistent view of the input
+ string by:
+ - calling C{parseWithTabs} on your grammar before calling C{parseString}
+ (see L{I{parseWithTabs}<parseWithTabs>})
+ - define your parse action using the full C{(s,loc,toks)} signature, and
+ reference the input string using the parse action's C{s} argument
+ - explictly expand the tabs in your input string before calling
+ C{parseString}
+
+ Example::
+ Word('a').parseString('aaaaabaaa') # -> ['aaaaa']
+ Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text
+ """
+ ParserElement.resetCache()
+ if not self.streamlined:
+ self.streamline()
+ #~ self.saveAsList = True
+ for e in self.ignoreExprs:
+ e.streamline()
+ if not self.keepTabs:
+ instring = instring.expandtabs()
+ try:
+ loc, tokens = self._parse( instring, 0 )
+ if parseAll:
+ loc = self.preParse( instring, loc )
+ se = Empty() + StringEnd()
+ se._parse( instring, loc )
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+ else:
+ return tokens
+
+ def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ):
+ """
+ Scan the input string for expression matches. Each match will return the
+ matching tokens, start location, and end location. May be called with optional
+ C{maxMatches} argument, to clip scanning after 'n' matches are found. If
+ C{overlap} is specified, then overlapping matches will be reported.
+
+ Note that the start and end locations are reported relative to the string
+ being parsed. See L{I{parseString}<parseString>} for more information on parsing
+ strings with embedded tabs.
+
+ Example::
+ source = "sldjf123lsdjjkf345sldkjf879lkjsfd987"
+ print(source)
+ for tokens,start,end in Word(alphas).scanString(source):
+ print(' '*start + '^'*(end-start))
+ print(' '*start + tokens[0])
+
+ prints::
+
+ sldjf123lsdjjkf345sldkjf879lkjsfd987
+ ^^^^^
+ sldjf
+ ^^^^^^^
+ lsdjjkf
+ ^^^^^^
+ sldkjf
+ ^^^^^^
+ lkjsfd
+ """
+ if not self.streamlined:
+ self.streamline()
+ for e in self.ignoreExprs:
+ e.streamline()
+
+ if not self.keepTabs:
+ instring = _ustr(instring).expandtabs()
+ instrlen = len(instring)
+ loc = 0
+ preparseFn = self.preParse
+ parseFn = self._parse
+ ParserElement.resetCache()
+ matches = 0
+ try:
+ while loc <= instrlen and matches < maxMatches:
+ try:
+ preloc = preparseFn( instring, loc )
+ nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
+ except ParseException:
+ loc = preloc+1
+ else:
+ if nextLoc > loc:
+ matches += 1
+ yield tokens, preloc, nextLoc
+ if overlap:
+ nextloc = preparseFn( instring, loc )
+ if nextloc > loc:
+ loc = nextLoc
+ else:
+ loc += 1
+ else:
+ loc = nextLoc
+ else:
+ loc = preloc+1
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def transformString( self, instring ):
+ """
+ Extension to C{L{scanString}}, to modify matching text with modified tokens that may
+ be returned from a parse action. To use C{transformString}, define a grammar and
+ attach a parse action to it that modifies the returned token list.
+ Invoking C{transformString()} on a target string will then scan for matches,
+ and replace the matched text patterns according to the logic in the parse
+ action. C{transformString()} returns the resulting transformed string.
+
+ Example::
+ wd = Word(alphas)
+ wd.setParseAction(lambda toks: toks[0].title())
+
+ print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york."))
+ Prints::
+ Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York.
+ """
+ out = []
+ lastE = 0
+ # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
+ # keep string locs straight between transformString and scanString
+ self.keepTabs = True
+ try:
+ for t,s,e in self.scanString( instring ):
+ out.append( instring[lastE:s] )
+ if t:
+ if isinstance(t,ParseResults):
+ out += t.asList()
+ elif isinstance(t,list):
+ out += t
+ else:
+ out.append(t)
+ lastE = e
+ out.append(instring[lastE:])
+ out = [o for o in out if o]
+ return "".join(map(_ustr,_flatten(out)))
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def searchString( self, instring, maxMatches=_MAX_INT ):
+ """
+ Another extension to C{L{scanString}}, simplifying the access to the tokens found
+ to match the given parse expression. May be called with optional
+ C{maxMatches} argument, to clip searching after 'n' matches are found.
+
+ Example::
+ # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters
+ cap_word = Word(alphas.upper(), alphas.lower())
+
+ print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))
+
+ # the sum() builtin can be used to merge results into a single ParseResults object
+ print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")))
+ prints::
+ [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']]
+ ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity']
+ """
+ try:
+ return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False):
+ """
+ Generator method to split a string using the given expression as a separator.
+ May be called with optional C{maxsplit} argument, to limit the number of splits;
+ and the optional C{includeSeparators} argument (default=C{False}), if the separating
+ matching text should be included in the split results.
+
+ Example::
+ punc = oneOf(list(".,;:/-!?"))
+ print(list(punc.split("This, this?, this sentence, is badly punctuated!")))
+ prints::
+ ['This', ' this', '', ' this sentence', ' is badly punctuated', '']
+ """
+ splits = 0
+ last = 0
+ for t,s,e in self.scanString(instring, maxMatches=maxsplit):
+ yield instring[last:s]
+ if includeSeparators:
+ yield t[0]
+ last = e
+ yield instring[last:]
+
+ def __add__(self, other ):
+ """
+ Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement
+ converts them to L{Literal}s by default.
+
+ Example::
+ greet = Word(alphas) + "," + Word(alphas) + "!"
+ hello = "Hello, World!"
+ print (hello, "->", greet.parseString(hello))
+ Prints::
+ Hello, World! -> ['Hello', ',', 'World', '!']
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return And( [ self, other ] )
+
+ def __radd__(self, other ):
+ """
+ Implementation of + operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other + self
+
+ def __sub__(self, other):
+ """
+ Implementation of - operator, returns C{L{And}} with error stop
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return self + And._ErrorStop() + other
+
+ def __rsub__(self, other ):
+ """
+ Implementation of - operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other - self
+
+ def __mul__(self,other):
+ """
+ Implementation of * operator, allows use of C{expr * 3} in place of
+ C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer
+ tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples
+ may also include C{None} as in:
+ - C{expr*(n,None)} or C{expr*(n,)} is equivalent
+ to C{expr*n + L{ZeroOrMore}(expr)}
+ (read as "at least n instances of C{expr}")
+ - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
+ (read as "0 to n instances of C{expr}")
+ - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
+ - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
+
+ Note that C{expr*(None,n)} does not raise an exception if
+ more than n exprs exist in the input stream; that is,
+ C{expr*(None,n)} does not enforce a maximum number of expr
+ occurrences. If this behavior is desired, then write
+ C{expr*(None,n) + ~expr}
+ """
+ if isinstance(other,int):
+ minElements, optElements = other,0
+ elif isinstance(other,tuple):
+ other = (other + (None, None))[:2]
+ if other[0] is None:
+ other = (0, other[1])
+ if isinstance(other[0],int) and other[1] is None:
+ if other[0] == 0:
+ return ZeroOrMore(self)
+ if other[0] == 1:
+ return OneOrMore(self)
+ else:
+ return self*other[0] + ZeroOrMore(self)
+ elif isinstance(other[0],int) and isinstance(other[1],int):
+ minElements, optElements = other
+ optElements -= minElements
+ else:
+ raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
+ else:
+ raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))
+
+ if minElements < 0:
+ raise ValueError("cannot multiply ParserElement by negative value")
+ if optElements < 0:
+ raise ValueError("second tuple value must be greater or equal to first tuple value")
+ if minElements == optElements == 0:
+ raise ValueError("cannot multiply ParserElement by 0 or (0,0)")
+
+ if (optElements):
+ def makeOptionalList(n):
+ if n>1:
+ return Optional(self + makeOptionalList(n-1))
+ else:
+ return Optional(self)
+ if minElements:
+ if minElements == 1:
+ ret = self + makeOptionalList(optElements)
+ else:
+ ret = And([self]*minElements) + makeOptionalList(optElements)
+ else:
+ ret = makeOptionalList(optElements)
+ else:
+ if minElements == 1:
+ ret = self
+ else:
+ ret = And([self]*minElements)
+ return ret
+
+ def __rmul__(self, other):
+ return self.__mul__(other)
+
+ def __or__(self, other ):
+ """
+ Implementation of | operator - returns C{L{MatchFirst}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return MatchFirst( [ self, other ] )
+
+ def __ror__(self, other ):
+ """
+ Implementation of | operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other | self
+
+ def __xor__(self, other ):
+ """
+ Implementation of ^ operator - returns C{L{Or}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return Or( [ self, other ] )
+
+ def __rxor__(self, other ):
+ """
+ Implementation of ^ operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other ^ self
+
+ def __and__(self, other ):
+ """
+ Implementation of & operator - returns C{L{Each}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return Each( [ self, other ] )
+
+ def __rand__(self, other ):
+ """
+ Implementation of & operator when left operand is not a C{L{ParserElement}}
+ """
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ if not isinstance( other, ParserElement ):
+ warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+ SyntaxWarning, stacklevel=2)
+ return None
+ return other & self
+
+ def __invert__( self ):
+ """
+ Implementation of ~ operator - returns C{L{NotAny}}
+ """
+ return NotAny( self )
+
+ def __call__(self, name=None):
+ """
+ Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}.
+
+ If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
+ passed as C{True}.
+
+ If C{name} is omitted, same as calling C{L{copy}}.
+
+ Example::
+ # these are equivalent
+ userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
+ userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
+ """
+ if name is not None:
+ return self.setResultsName(name)
+ else:
+ return self.copy()
+
+ def suppress( self ):
+ """
+ Suppresses the output of this C{ParserElement}; useful to keep punctuation from
+ cluttering up returned output.
+ """
+ return Suppress( self )
+
+ def leaveWhitespace( self ):
+ """
+ Disables the skipping of whitespace before matching the characters in the
+ C{ParserElement}'s defined pattern. This is normally only used internally by
+ the pyparsing module, but may be needed in some whitespace-sensitive grammars.
+ """
+ self.skipWhitespace = False
+ return self
+
+ def setWhitespaceChars( self, chars ):
+ """
+ Overrides the default whitespace chars
+ """
+ self.skipWhitespace = True
+ self.whiteChars = chars
+ self.copyDefaultWhiteChars = False
+ return self
+
+ def parseWithTabs( self ):
+ """
+ Overrides default behavior to expand C{<TAB>}s to spaces before parsing the input string.
+ Must be called before C{parseString} when the input grammar contains elements that
+ match C{<TAB>} characters.
+ """
+ self.keepTabs = True
+ return self
+
+ def ignore( self, other ):
+ """
+ Define expression to be ignored (e.g., comments) while doing pattern
+ matching; may be called repeatedly, to define multiple comment or other
+ ignorable patterns.
+
+ Example::
+ patt = OneOrMore(Word(alphas))
+ patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj']
+
+ patt.ignore(cStyleComment)
+ patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd']
+ """
+ if isinstance(other, basestring):
+ other = Suppress(other)
+
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ self.ignoreExprs.append(other)
+ else:
+ self.ignoreExprs.append( Suppress( other.copy() ) )
+ return self
+
+ def setDebugActions( self, startAction, successAction, exceptionAction ):
+ """
+ Enable display of debugging messages while doing pattern matching.
+ """
+ self.debugActions = (startAction or _defaultStartDebugAction,
+ successAction or _defaultSuccessDebugAction,
+ exceptionAction or _defaultExceptionDebugAction)
+ self.debug = True
+ return self
+
+ def setDebug( self, flag=True ):
+ """
+ Enable display of debugging messages while doing pattern matching.
+ Set C{flag} to True to enable, False to disable.
+
+ Example::
+ wd = Word(alphas).setName("alphaword")
+ integer = Word(nums).setName("numword")
+ term = wd | integer
+
+ # turn on debugging for wd
+ wd.setDebug()
+
+ OneOrMore(term).parseString("abc 123 xyz 890")
+
+ prints::
+ Match alphaword at loc 0(1,1)
+ Matched alphaword -> ['abc']
+ Match alphaword at loc 3(1,4)
+ Exception raised:Expected alphaword (at char 4), (line:1, col:5)
+ Match alphaword at loc 7(1,8)
+ Matched alphaword -> ['xyz']
+ Match alphaword at loc 11(1,12)
+ Exception raised:Expected alphaword (at char 12), (line:1, col:13)
+ Match alphaword at loc 15(1,16)
+ Exception raised:Expected alphaword (at char 15), (line:1, col:16)
+
+ The output shown is that produced by the default debug actions - custom debug actions can be
+ specified using L{setDebugActions}. Prior to attempting
+ to match the C{wd} expression, the debugging message C{"Match <exprname> at loc <n>(<line>,<col>)"}
+ is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"}
+ message is shown. Also note the use of L{setName} to assign a human-readable name to the expression,
+ which makes debugging and exception messages easier to understand - for instance, the default
+ name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}.
+ """
+ if flag:
+ self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
+ else:
+ self.debug = False
+ return self
+
+ def __str__( self ):
+ return self.name
+
+ def __repr__( self ):
+ return _ustr(self)
+
+ def streamline( self ):
+ self.streamlined = True
+ self.strRepr = None
+ return self
+
+ def checkRecursion( self, parseElementList ):
+ pass
+
+ def validate( self, validateTrace=[] ):
+ """
+ Check defined expressions for valid structure, check for infinite recursive definitions.
+ """
+ self.checkRecursion( [] )
+
+ def parseFile( self, file_or_filename, parseAll=False ):
+ """
+ Execute the parse expression on the given file or filename.
+ If a filename is specified (instead of a file object),
+ the entire file is opened, read, and closed before parsing.
+ """
+ try:
+ file_contents = file_or_filename.read()
+ except AttributeError:
+ with open(file_or_filename, "r") as f:
+ file_contents = f.read()
+ try:
+ return self.parseString(file_contents, parseAll)
+ except ParseBaseException as exc:
+ if ParserElement.verbose_stacktrace:
+ raise
+ else:
+ # catch and re-raise exception from here, clears out pyparsing internal stack trace
+ raise exc
+
+ def __eq__(self,other):
+ if isinstance(other, ParserElement):
+ return self is other or vars(self) == vars(other)
+ elif isinstance(other, basestring):
+ return self.matches(other)
+ else:
+ return super(ParserElement,self)==other
+
+ def __ne__(self,other):
+ return not (self == other)
+
+ def __hash__(self):
+ return hash(id(self))
+
+ def __req__(self,other):
+ return self == other
+
+ def __rne__(self,other):
+ return not (self == other)
+
+ def matches(self, testString, parseAll=True):
+ """
+ Method for quick testing of a parser against a test string. Good for simple
+ inline microtests of sub expressions while building up larger parser.
+
+ Parameters:
+ - testString - to test against this expression for a match
+ - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests
+
+ Example::
+ expr = Word(nums)
+ assert expr.matches("100")
+ """
+ try:
+ self.parseString(_ustr(testString), parseAll=parseAll)
+ return True
+ except ParseBaseException:
+ return False
+
+ def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False):
+ """
+ Execute the parse expression on a series of test strings, showing each
+ test, the parsed results or where the parse failed. Quick and easy way to
+ run a parse expression against a list of sample strings.
+
+ Parameters:
+ - tests - a list of separate test strings, or a multiline string of test strings
+ - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests
+ - comment - (default=C{'#'}) - expression for indicating embedded comments in the test
+ string; pass None to disable comment filtering
+ - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline;
+ if False, only dump nested list
+ - printResults - (default=C{True}) prints test output to stdout
+ - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing
+
+ Returns: a (success, results) tuple, where success indicates that all tests succeeded
+ (or failed if C{failureTests} is True), and the results contain a list of lines of each
+ test's output
+
+ Example::
+ number_expr = pyparsing_common.number.copy()
+
+ result = number_expr.runTests('''
+ # unsigned integer
+ 100
+ # negative integer
+ -100
+ # float with scientific notation
+ 6.02e23
+ # integer with scientific notation
+ 1e-12
+ ''')
+ print("Success" if result[0] else "Failed!")
+
+ result = number_expr.runTests('''
+ # stray character
+ 100Z
+ # missing leading digit before '.'
+ -.100
+ # too many '.'
+ 3.14.159
+ ''', failureTests=True)
+ print("Success" if result[0] else "Failed!")
+ prints::
+ # unsigned integer
+ 100
+ [100]
+
+ # negative integer
+ -100
+ [-100]
+
+ # float with scientific notation
+ 6.02e23
+ [6.02e+23]
+
+ # integer with scientific notation
+ 1e-12
+ [1e-12]
+
+ Success
+
+ # stray character
+ 100Z
+ ^
+ FAIL: Expected end of text (at char 3), (line:1, col:4)
+
+ # missing leading digit before '.'
+ -.100
+ ^
+ FAIL: Expected {real number with scientific notation | real number | signed integer} (at char 0), (line:1, col:1)
+
+ # too many '.'
+ 3.14.159
+ ^
+ FAIL: Expected end of text (at char 4), (line:1, col:5)
+
+ Success
+
+ Each test string must be on a single line. If you want to test a string that spans multiple
+ lines, create a test like this::
+
+ expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines")
+
+ (Note that this is a raw string literal, you must include the leading 'r'.)
+ """
+ if isinstance(tests, basestring):
+ tests = list(map(str.strip, tests.rstrip().splitlines()))
+ if isinstance(comment, basestring):
+ comment = Literal(comment)
+ allResults = []
+ comments = []
+ success = True
+ for t in tests:
+ if comment is not None and comment.matches(t, False) or comments and not t:
+ comments.append(t)
+ continue
+ if not t:
+ continue
+ out = ['\n'.join(comments), t]
+ comments = []
+ try:
+ t = t.replace(r'\n','\n')
+ result = self.parseString(t, parseAll=parseAll)
+ out.append(result.dump(full=fullDump))
+ success = success and not failureTests
+ except ParseBaseException as pe:
+ fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else ""
+ if '\n' in t:
+ out.append(line(pe.loc, t))
+ out.append(' '*(col(pe.loc,t)-1) + '^' + fatal)
+ else:
+ out.append(' '*pe.loc + '^' + fatal)
+ out.append("FAIL: " + str(pe))
+ success = success and failureTests
+ result = pe
+ except Exception as exc:
+ out.append("FAIL-EXCEPTION: " + str(exc))
+ success = success and failureTests
+ result = exc
+
+ if printResults:
+ if fullDump:
+ out.append('')
+ print('\n'.join(out))
+
+ allResults.append((t, result))
+
+ return success, allResults
+
+
+class Token(ParserElement):
+ """
+ Abstract C{ParserElement} subclass, for defining atomic matching patterns.
+ """
+ def __init__( self ):
+ super(Token,self).__init__( savelist=False )
+
+
+class Empty(Token):
+ """
+ An empty token, will always match.
+ """
+ def __init__( self ):
+ super(Empty,self).__init__()
+ self.name = "Empty"
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+
+
+class NoMatch(Token):
+ """
+ A token that will never match.
+ """
+ def __init__( self ):
+ super(NoMatch,self).__init__()
+ self.name = "NoMatch"
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+ self.errmsg = "Unmatchable token"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Literal(Token):
+ """
+ Token to exactly match a specified string.
+
+ Example::
+ Literal('blah').parseString('blah') # -> ['blah']
+ Literal('blah').parseString('blahfooblah') # -> ['blah']
+ Literal('blah').parseString('bla') # -> Exception: Expected "blah"
+
+ For case-insensitive matching, use L{CaselessLiteral}.
+
+ For keyword matching (force word break before and after the matched string),
+ use L{Keyword} or L{CaselessKeyword}.
+ """
+ def __init__( self, matchString ):
+ super(Literal,self).__init__()
+ self.match = matchString
+ self.matchLen = len(matchString)
+ try:
+ self.firstMatchChar = matchString[0]
+ except IndexError:
+ warnings.warn("null string passed to Literal; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+ self.__class__ = Empty
+ self.name = '"%s"' % _ustr(self.match)
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = False
+ self.mayIndexError = False
+
+ # Performance tuning: this routine gets called a *lot*
+ # if this is a single character match string and the first character matches,
+ # short-circuit as quickly as possible, and avoid calling startswith
+ #~ @profile
+ def parseImpl( self, instring, loc, doActions=True ):
+ if (instring[loc] == self.firstMatchChar and
+ (self.matchLen==1 or instring.startswith(self.match,loc)) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+_L = Literal
+ParserElement._literalStringClass = Literal
+
+class Keyword(Token):
+ """
+ Token to exactly match a specified string as a keyword, that is, it must be
+ immediately followed by a non-keyword character. Compare with C{L{Literal}}:
+ - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}.
+ - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
+ Accepts two optional constructor arguments in addition to the keyword string:
+ - C{identChars} is a string of characters that would be valid identifier characters,
+ defaulting to all alphanumerics + "_" and "$"
+ - C{caseless} allows case-insensitive matching, default is C{False}.
+
+ Example::
+ Keyword("start").parseString("start") # -> ['start']
+ Keyword("start").parseString("starting") # -> Exception
+
+ For case-insensitive matching, use L{CaselessKeyword}.
+ """
+ DEFAULT_KEYWORD_CHARS = alphanums+"_$"
+
+ def __init__( self, matchString, identChars=None, caseless=False ):
+ super(Keyword,self).__init__()
+ if identChars is None:
+ identChars = Keyword.DEFAULT_KEYWORD_CHARS
+ self.match = matchString
+ self.matchLen = len(matchString)
+ try:
+ self.firstMatchChar = matchString[0]
+ except IndexError:
+ warnings.warn("null string passed to Keyword; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+ self.name = '"%s"' % self.match
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = False
+ self.mayIndexError = False
+ self.caseless = caseless
+ if caseless:
+ self.caselessmatch = matchString.upper()
+ identChars = identChars.upper()
+ self.identChars = set(identChars)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.caseless:
+ if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
+ (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ else:
+ if (instring[loc] == self.firstMatchChar and
+ (self.matchLen==1 or instring.startswith(self.match,loc)) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
+ (loc == 0 or instring[loc-1] not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ def copy(self):
+ c = super(Keyword,self).copy()
+ c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
+ return c
+
+ @staticmethod
+ def setDefaultKeywordChars( chars ):
+ """Overrides the default Keyword chars
+ """
+ Keyword.DEFAULT_KEYWORD_CHARS = chars
+
+class CaselessLiteral(Literal):
+ """
+ Token to match a specified string, ignoring case of letters.
+ Note: the matched results will always be in the case of the given
+ match string, NOT the case of the input text.
+
+ Example::
+ OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD']
+
+ (Contrast with example for L{CaselessKeyword}.)
+ """
+ def __init__( self, matchString ):
+ super(CaselessLiteral,self).__init__( matchString.upper() )
+ # Preserve the defining literal.
+ self.returnString = matchString
+ self.name = "'%s'" % self.returnString
+ self.errmsg = "Expected " + self.name
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if instring[ loc:loc+self.matchLen ].upper() == self.match:
+ return loc+self.matchLen, self.returnString
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class CaselessKeyword(Keyword):
+ """
+ Caseless version of L{Keyword}.
+
+ Example::
+ OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD']
+
+ (Contrast with example for L{CaselessLiteral}.)
+ """
+ def __init__( self, matchString, identChars=None ):
+ super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+ (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
+ return loc+self.matchLen, self.match
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class CloseMatch(Token):
+ """
+ A variation on L{Literal} which matches "close" matches, that is,
+ strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters:
+ - C{match_string} - string to be matched
+ - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match
+
+ The results from a successful parse will contain the matched text from the input string and the following named results:
+ - C{mismatches} - a list of the positions within the match_string where mismatches were found
+ - C{original} - the original match_string used to compare against the input string
+
+ If C{mismatches} is an empty list, then the match was an exact match.
+
+ Example::
+ patt = CloseMatch("ATCATCGAATGGA")
+ patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']})
+ patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1)
+
+ # exact match
+ patt.parseString("ATCATCGAATGGA") # -> (['ATCATCGAATGGA'], {'mismatches': [[]], 'original': ['ATCATCGAATGGA']})
+
+ # close match allowing up to 2 mismatches
+ patt = CloseMatch("ATCATCGAATGGA", maxMismatches=2)
+ patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']})
+ """
+ def __init__(self, match_string, maxMismatches=1):
+ super(CloseMatch,self).__init__()
+ self.name = match_string
+ self.match_string = match_string
+ self.maxMismatches = maxMismatches
+ self.errmsg = "Expected %r (with up to %d mismatches)" % (self.match_string, self.maxMismatches)
+ self.mayIndexError = False
+ self.mayReturnEmpty = False
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ start = loc
+ instrlen = len(instring)
+ maxloc = start + len(self.match_string)
+
+ if maxloc <= instrlen:
+ match_string = self.match_string
+ match_stringloc = 0
+ mismatches = []
+ maxMismatches = self.maxMismatches
+
+ for match_stringloc,s_m in enumerate(zip(instring[loc:maxloc], self.match_string)):
+ src,mat = s_m
+ if src != mat:
+ mismatches.append(match_stringloc)
+ if len(mismatches) > maxMismatches:
+ break
+ else:
+ loc = match_stringloc + 1
+ results = ParseResults([instring[start:loc]])
+ results['original'] = self.match_string
+ results['mismatches'] = mismatches
+ return loc, results
+
+ raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Word(Token):
+ """
+ Token for matching words composed of allowed character sets.
+ Defined with string containing all allowed initial characters,
+ an optional string containing allowed body characters (if omitted,
+ defaults to the initial character set), and an optional minimum,
+ maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction. An optional
+ C{excludeChars} parameter can list characters that might be found in
+ the input C{bodyChars} string; useful to define a word of all printables
+ except for one or two characters, for instance.
+
+ L{srange} is useful for defining custom character set strings for defining
+ C{Word} expressions, using range notation from regular expression character sets.
+
+ A common mistake is to use C{Word} to match a specific literal string, as in
+ C{Word("Address")}. Remember that C{Word} uses the string argument to define
+ I{sets} of matchable characters. This expression would match "Add", "AAA",
+ "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'.
+ To match an exact literal string, use L{Literal} or L{Keyword}.
+
+ pyparsing includes helper strings for building Words:
+ - L{alphas}
+ - L{nums}
+ - L{alphanums}
+ - L{hexnums}
+ - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.)
+ - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.)
+ - L{printables} (any non-whitespace character)
+
+ Example::
+ # a word composed of digits
+ integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9"))
+
+ # a word with a leading capital, and zero or more lowercase
+ capital_word = Word(alphas.upper(), alphas.lower())
+
+ # hostnames are alphanumeric, with leading alpha, and '-'
+ hostname = Word(alphas, alphanums+'-')
+
+ # roman numeral (not a strict parser, accepts invalid mix of characters)
+ roman = Word("IVXLCDM")
+
+ # any string of non-whitespace characters, except for ','
+ csv_value = Word(printables, excludeChars=",")
+ """
+ def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ):
+ super(Word,self).__init__()
+ if excludeChars:
+ initChars = ''.join(c for c in initChars if c not in excludeChars)
+ if bodyChars:
+ bodyChars = ''.join(c for c in bodyChars if c not in excludeChars)
+ self.initCharsOrig = initChars
+ self.initChars = set(initChars)
+ if bodyChars :
+ self.bodyCharsOrig = bodyChars
+ self.bodyChars = set(bodyChars)
+ else:
+ self.bodyCharsOrig = initChars
+ self.bodyChars = set(initChars)
+
+ self.maxSpecified = max > 0
+
+ if min < 1:
+ raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted")
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.asKeyword = asKeyword
+
+ if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
+ if self.bodyCharsOrig == self.initCharsOrig:
+ self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
+ elif len(self.initCharsOrig) == 1:
+ self.reString = "%s[%s]*" % \
+ (re.escape(self.initCharsOrig),
+ _escapeRegexRangeChars(self.bodyCharsOrig),)
+ else:
+ self.reString = "[%s][%s]*" % \
+ (_escapeRegexRangeChars(self.initCharsOrig),
+ _escapeRegexRangeChars(self.bodyCharsOrig),)
+ if self.asKeyword:
+ self.reString = r"\b"+self.reString+r"\b"
+ try:
+ self.re = re.compile( self.reString )
+ except Exception:
+ self.re = None
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.re:
+ result = self.re.match(instring,loc)
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ return loc, result.group()
+
+ if not(instring[ loc ] in self.initChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ start = loc
+ loc += 1
+ instrlen = len(instring)
+ bodychars = self.bodyChars
+ maxloc = start + self.maxLen
+ maxloc = min( maxloc, instrlen )
+ while loc < maxloc and instring[loc] in bodychars:
+ loc += 1
+
+ throwException = False
+ if loc - start < self.minLen:
+ throwException = True
+ if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
+ throwException = True
+ if self.asKeyword:
+ if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars):
+ throwException = True
+
+ if throwException:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+ def __str__( self ):
+ try:
+ return super(Word,self).__str__()
+ except Exception:
+ pass
+
+
+ if self.strRepr is None:
+
+ def charsAsStr(s):
+ if len(s)>4:
+ return s[:4]+"..."
+ else:
+ return s
+
+ if ( self.initCharsOrig != self.bodyCharsOrig ):
+ self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
+ else:
+ self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)
+
+ return self.strRepr
+
+
+class Regex(Token):
+ r"""
+ Token for matching strings that match a given regular expression.
+ Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
+ If the given regex contains named groups (defined using C{(?P<name>...)}), these will be preserved as
+ named parse results.
+
+ Example::
+ realnum = Regex(r"[+-]?\d+\.\d*")
+ date = Regex(r'(?P<year>\d{4})-(?P<month>\d\d?)-(?P<day>\d\d?)')
+ # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression
+ roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})")
+ """
+ compiledREtype = type(re.compile("[A-Z]"))
+ def __init__( self, pattern, flags=0):
+ """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags."""
+ super(Regex,self).__init__()
+
+ if isinstance(pattern, basestring):
+ if not pattern:
+ warnings.warn("null string passed to Regex; use Empty() instead",
+ SyntaxWarning, stacklevel=2)
+
+ self.pattern = pattern
+ self.flags = flags
+
+ try:
+ self.re = re.compile(self.pattern, self.flags)
+ self.reString = self.pattern
+ except sre_constants.error:
+ warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
+ SyntaxWarning, stacklevel=2)
+ raise
+
+ elif isinstance(pattern, Regex.compiledREtype):
+ self.re = pattern
+ self.pattern = \
+ self.reString = str(pattern)
+ self.flags = flags
+
+ else:
+ raise ValueError("Regex may only be constructed with a string or a compiled RE object")
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ result = self.re.match(instring,loc)
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ d = result.groupdict()
+ ret = ParseResults(result.group())
+ if d:
+ for k in d:
+ ret[k] = d[k]
+ return loc,ret
+
+ def __str__( self ):
+ try:
+ return super(Regex,self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "Re:(%s)" % repr(self.pattern)
+
+ return self.strRepr
+
+
+class QuotedString(Token):
+ r"""
+ Token for matching strings that are delimited by quoting characters.
+
+ Defined with the following parameters:
+ - quoteChar - string of one or more characters defining the quote delimiting string
+ - escChar - character to escape quotes, typically backslash (default=C{None})
+ - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None})
+ - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
+ - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
+ - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
+ - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True})
+
+ Example::
+ qs = QuotedString('"')
+ print(qs.searchString('lsjdf "This is the quote" sldjf'))
+ complex_qs = QuotedString('{{', endQuoteChar='}}')
+ print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf'))
+ sql_qs = QuotedString('"', escQuote='""')
+ print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf'))
+ prints::
+ [['This is the quote']]
+ [['This is the "quote"']]
+ [['This is the quote with "embedded" quotes']]
+ """
+ def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True):
+ super(QuotedString,self).__init__()
+
+ # remove white space from quote chars - wont work anyway
+ quoteChar = quoteChar.strip()
+ if not quoteChar:
+ warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+ raise SyntaxError()
+
+ if endQuoteChar is None:
+ endQuoteChar = quoteChar
+ else:
+ endQuoteChar = endQuoteChar.strip()
+ if not endQuoteChar:
+ warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+ raise SyntaxError()
+
+ self.quoteChar = quoteChar
+ self.quoteCharLen = len(quoteChar)
+ self.firstQuoteChar = quoteChar[0]
+ self.endQuoteChar = endQuoteChar
+ self.endQuoteCharLen = len(endQuoteChar)
+ self.escChar = escChar
+ self.escQuote = escQuote
+ self.unquoteResults = unquoteResults
+ self.convertWhitespaceEscapes = convertWhitespaceEscapes
+
+ if multiline:
+ self.flags = re.MULTILINE | re.DOTALL
+ self.pattern = r'%s(?:[^%s%s]' % \
+ ( re.escape(self.quoteChar),
+ _escapeRegexRangeChars(self.endQuoteChar[0]),
+ (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+ else:
+ self.flags = 0
+ self.pattern = r'%s(?:[^%s\n\r%s]' % \
+ ( re.escape(self.quoteChar),
+ _escapeRegexRangeChars(self.endQuoteChar[0]),
+ (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+ if len(self.endQuoteChar) > 1:
+ self.pattern += (
+ '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
+ _escapeRegexRangeChars(self.endQuoteChar[i]))
+ for i in range(len(self.endQuoteChar)-1,0,-1)) + ')'
+ )
+ if escQuote:
+ self.pattern += (r'|(?:%s)' % re.escape(escQuote))
+ if escChar:
+ self.pattern += (r'|(?:%s.)' % re.escape(escChar))
+ self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
+ self.pattern += (r')*%s' % re.escape(self.endQuoteChar))
+
+ try:
+ self.re = re.compile(self.pattern, self.flags)
+ self.reString = self.pattern
+ except sre_constants.error:
+ warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
+ SyntaxWarning, stacklevel=2)
+ raise
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayIndexError = False
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
+ if not result:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ loc = result.end()
+ ret = result.group()
+
+ if self.unquoteResults:
+
+ # strip off quotes
+ ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
+
+ if isinstance(ret,basestring):
+ # replace escaped whitespace
+ if '\\' in ret and self.convertWhitespaceEscapes:
+ ws_map = {
+ r'\t' : '\t',
+ r'\n' : '\n',
+ r'\f' : '\f',
+ r'\r' : '\r',
+ }
+ for wslit,wschar in ws_map.items():
+ ret = ret.replace(wslit, wschar)
+
+ # replace escaped characters
+ if self.escChar:
+ ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret)
+
+ # replace escaped quotes
+ if self.escQuote:
+ ret = ret.replace(self.escQuote, self.endQuoteChar)
+
+ return loc, ret
+
+ def __str__( self ):
+ try:
+ return super(QuotedString,self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)
+
+ return self.strRepr
+
+
+class CharsNotIn(Token):
+ """
+ Token for matching words composed of characters I{not} in a given set (will
+ include whitespace in matched characters if not listed in the provided exclusion set - see example).
+ Defined with string containing all disallowed characters, and an optional
+ minimum, maximum, and/or exact length. The default value for C{min} is 1 (a
+ minimum value < 1 is not valid); the default values for C{max} and C{exact}
+ are 0, meaning no maximum or exact length restriction.
+
+ Example::
+ # define a comma-separated-value as anything that is not a ','
+ csv_value = CharsNotIn(',')
+ print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213"))
+ prints::
+ ['dkls', 'lsdkjf', 's12 34', '@!#', '213']
+ """
+ def __init__( self, notChars, min=1, max=0, exact=0 ):
+ super(CharsNotIn,self).__init__()
+ self.skipWhitespace = False
+ self.notChars = notChars
+
+ if min < 1:
+ raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted")
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ self.name = _ustr(self)
+ self.errmsg = "Expected " + self.name
+ self.mayReturnEmpty = ( self.minLen == 0 )
+ self.mayIndexError = False
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if instring[loc] in self.notChars:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ start = loc
+ loc += 1
+ notchars = self.notChars
+ maxlen = min( start+self.maxLen, len(instring) )
+ while loc < maxlen and \
+ (instring[loc] not in notchars):
+ loc += 1
+
+ if loc - start < self.minLen:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+ def __str__( self ):
+ try:
+ return super(CharsNotIn, self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None:
+ if len(self.notChars) > 4:
+ self.strRepr = "!W:(%s...)" % self.notChars[:4]
+ else:
+ self.strRepr = "!W:(%s)" % self.notChars
+
+ return self.strRepr
+
+class White(Token):
+ """
+ Special matching class for matching whitespace. Normally, whitespace is ignored
+ by pyparsing grammars. This class is included when some whitespace structures
+ are significant. Define with a string containing the whitespace characters to be
+ matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments,
+ as defined for the C{L{Word}} class.
+ """
+ whiteStrs = {
+ " " : "<SPC>",
+ "\t": "<TAB>",
+ "\n": "<LF>",
+ "\r": "<CR>",
+ "\f": "<FF>",
+ }
+ def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
+ super(White,self).__init__()
+ self.matchWhite = ws
+ self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) )
+ #~ self.leaveWhitespace()
+ self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite))
+ self.mayReturnEmpty = True
+ self.errmsg = "Expected " + self.name
+
+ self.minLen = min
+
+ if max > 0:
+ self.maxLen = max
+ else:
+ self.maxLen = _MAX_INT
+
+ if exact > 0:
+ self.maxLen = exact
+ self.minLen = exact
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if not(instring[ loc ] in self.matchWhite):
+ raise ParseException(instring, loc, self.errmsg, self)
+ start = loc
+ loc += 1
+ maxloc = start + self.maxLen
+ maxloc = min( maxloc, len(instring) )
+ while loc < maxloc and instring[loc] in self.matchWhite:
+ loc += 1
+
+ if loc - start < self.minLen:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ return loc, instring[start:loc]
+
+
+class _PositionToken(Token):
+ def __init__( self ):
+ super(_PositionToken,self).__init__()
+ self.name=self.__class__.__name__
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+
+class GoToColumn(_PositionToken):
+ """
+ Token to advance to a specific column of input text; useful for tabular report scraping.
+ """
+ def __init__( self, colno ):
+ super(GoToColumn,self).__init__()
+ self.col = colno
+
+ def preParse( self, instring, loc ):
+ if col(loc,instring) != self.col:
+ instrlen = len(instring)
+ if self.ignoreExprs:
+ loc = self._skipIgnorables( instring, loc )
+ while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
+ loc += 1
+ return loc
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ thiscol = col( loc, instring )
+ if thiscol > self.col:
+ raise ParseException( instring, loc, "Text not in expected column", self )
+ newloc = loc + self.col - thiscol
+ ret = instring[ loc: newloc ]
+ return newloc, ret
+
+
+class LineStart(_PositionToken):
+ """
+ Matches if current position is at the beginning of a line within the parse string
+
+ Example::
+
+ test = '''\
+ AAA this line
+ AAA and this line
+ AAA but not this one
+ B AAA and definitely not this one
+ '''
+
+ for t in (LineStart() + 'AAA' + restOfLine).searchString(test):
+ print(t)
+
+ Prints::
+ ['AAA', ' this line']
+ ['AAA', ' and this line']
+
+ """
+ def __init__( self ):
+ super(LineStart,self).__init__()
+ self.errmsg = "Expected start of line"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if col(loc, instring) == 1:
+ return loc, []
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class LineEnd(_PositionToken):
+ """
+ Matches if current position is at the end of a line within the parse string
+ """
+ def __init__( self ):
+ super(LineEnd,self).__init__()
+ self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
+ self.errmsg = "Expected end of line"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc<len(instring):
+ if instring[loc] == "\n":
+ return loc+1, "\n"
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+ elif loc == len(instring):
+ return loc+1, []
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class StringStart(_PositionToken):
+ """
+ Matches if current position is at the beginning of the parse string
+ """
+ def __init__( self ):
+ super(StringStart,self).__init__()
+ self.errmsg = "Expected start of text"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc != 0:
+ # see if entire string up to here is just whitespace and ignoreables
+ if loc != self.preParse( instring, 0 ):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class StringEnd(_PositionToken):
+ """
+ Matches if current position is at the end of the parse string
+ """
+ def __init__( self ):
+ super(StringEnd,self).__init__()
+ self.errmsg = "Expected end of text"
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if loc < len(instring):
+ raise ParseException(instring, loc, self.errmsg, self)
+ elif loc == len(instring):
+ return loc+1, []
+ elif loc > len(instring):
+ return loc, []
+ else:
+ raise ParseException(instring, loc, self.errmsg, self)
+
+class WordStart(_PositionToken):
+ """
+ Matches if the current position is at the beginning of a Word, and
+ is not preceded by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of
+ the string being parsed, or at the beginning of a line.
+ """
+ def __init__(self, wordChars = printables):
+ super(WordStart,self).__init__()
+ self.wordChars = set(wordChars)
+ self.errmsg = "Not at the start of a word"
+
+ def parseImpl(self, instring, loc, doActions=True ):
+ if loc != 0:
+ if (instring[loc-1] in self.wordChars or
+ instring[loc] not in self.wordChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+class WordEnd(_PositionToken):
+ """
+ Matches if the current position is at the end of a Word, and
+ is not followed by any character in a given set of C{wordChars}
+ (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+ use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of
+ the string being parsed, or at the end of a line.
+ """
+ def __init__(self, wordChars = printables):
+ super(WordEnd,self).__init__()
+ self.wordChars = set(wordChars)
+ self.skipWhitespace = False
+ self.errmsg = "Not at the end of a word"
+
+ def parseImpl(self, instring, loc, doActions=True ):
+ instrlen = len(instring)
+ if instrlen>0 and loc<instrlen:
+ if (instring[loc] in self.wordChars or
+ instring[loc-1] not in self.wordChars):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+
+class ParseExpression(ParserElement):
+ """
+ Abstract subclass of ParserElement, for combining and post-processing parsed tokens.
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(ParseExpression,self).__init__(savelist)
+ if isinstance( exprs, _generatorType ):
+ exprs = list(exprs)
+
+ if isinstance( exprs, basestring ):
+ self.exprs = [ ParserElement._literalStringClass( exprs ) ]
+ elif isinstance( exprs, Iterable ):
+ exprs = list(exprs)
+ # if sequence of strings provided, wrap with Literal
+ if all(isinstance(expr, basestring) for expr in exprs):
+ exprs = map(ParserElement._literalStringClass, exprs)
+ self.exprs = list(exprs)
+ else:
+ try:
+ self.exprs = list( exprs )
+ except TypeError:
+ self.exprs = [ exprs ]
+ self.callPreparse = False
+
+ def __getitem__( self, i ):
+ return self.exprs[i]
+
+ def append( self, other ):
+ self.exprs.append( other )
+ self.strRepr = None
+ return self
+
+ def leaveWhitespace( self ):
+ """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on
+ all contained expressions."""
+ self.skipWhitespace = False
+ self.exprs = [ e.copy() for e in self.exprs ]
+ for e in self.exprs:
+ e.leaveWhitespace()
+ return self
+
+ def ignore( self, other ):
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ super( ParseExpression, self).ignore( other )
+ for e in self.exprs:
+ e.ignore( self.ignoreExprs[-1] )
+ else:
+ super( ParseExpression, self).ignore( other )
+ for e in self.exprs:
+ e.ignore( self.ignoreExprs[-1] )
+ return self
+
+ def __str__( self ):
+ try:
+ return super(ParseExpression,self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None:
+ self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) )
+ return self.strRepr
+
+ def streamline( self ):
+ super(ParseExpression,self).streamline()
+
+ for e in self.exprs:
+ e.streamline()
+
+ # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d )
+ # but only if there are no parse actions or resultsNames on the nested And's
+ # (likewise for Or's and MatchFirst's)
+ if ( len(self.exprs) == 2 ):
+ other = self.exprs[0]
+ if ( isinstance( other, self.__class__ ) and
+ not(other.parseAction) and
+ other.resultsName is None and
+ not other.debug ):
+ self.exprs = other.exprs[:] + [ self.exprs[1] ]
+ self.strRepr = None
+ self.mayReturnEmpty |= other.mayReturnEmpty
+ self.mayIndexError |= other.mayIndexError
+
+ other = self.exprs[-1]
+ if ( isinstance( other, self.__class__ ) and
+ not(other.parseAction) and
+ other.resultsName is None and
+ not other.debug ):
+ self.exprs = self.exprs[:-1] + other.exprs[:]
+ self.strRepr = None
+ self.mayReturnEmpty |= other.mayReturnEmpty
+ self.mayIndexError |= other.mayIndexError
+
+ self.errmsg = "Expected " + _ustr(self)
+
+ return self
+
+ def setResultsName( self, name, listAllMatches=False ):
+ ret = super(ParseExpression,self).setResultsName(name,listAllMatches)
+ return ret
+
+ def validate( self, validateTrace=[] ):
+ tmp = validateTrace[:]+[self]
+ for e in self.exprs:
+ e.validate(tmp)
+ self.checkRecursion( [] )
+
+ def copy(self):
+ ret = super(ParseExpression,self).copy()
+ ret.exprs = [e.copy() for e in self.exprs]
+ return ret
+
+class And(ParseExpression):
+ """
+ Requires all given C{ParseExpression}s to be found in the given order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'+'} operator.
+ May also be constructed using the C{'-'} operator, which will suppress backtracking.
+
+ Example::
+ integer = Word(nums)
+ name_expr = OneOrMore(Word(alphas))
+
+ expr = And([integer("id"),name_expr("name"),integer("age")])
+ # more easily written as:
+ expr = integer("id") + name_expr("name") + integer("age")
+ """
+
+ class _ErrorStop(Empty):
+ def __init__(self, *args, **kwargs):
+ super(And._ErrorStop,self).__init__(*args, **kwargs)
+ self.name = '-'
+ self.leaveWhitespace()
+
+ def __init__( self, exprs, savelist = True ):
+ super(And,self).__init__(exprs, savelist)
+ self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+ self.setWhitespaceChars( self.exprs[0].whiteChars )
+ self.skipWhitespace = self.exprs[0].skipWhitespace
+ self.callPreparse = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ # pass False as last arg to _parse for first element, since we already
+ # pre-parsed the string as part of our And pre-parsing
+ loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False )
+ errorStop = False
+ for e in self.exprs[1:]:
+ if isinstance(e, And._ErrorStop):
+ errorStop = True
+ continue
+ if errorStop:
+ try:
+ loc, exprtokens = e._parse( instring, loc, doActions )
+ except ParseSyntaxException:
+ raise
+ except ParseBaseException as pe:
+ pe.__traceback__ = None
+ raise ParseSyntaxException._from_exception(pe)
+ except IndexError:
+ raise ParseSyntaxException(instring, len(instring), self.errmsg, self)
+ else:
+ loc, exprtokens = e._parse( instring, loc, doActions )
+ if exprtokens or exprtokens.haskeys():
+ resultlist += exprtokens
+ return loc, resultlist
+
+ def __iadd__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ return self.append( other ) #And( [ self, other ] )
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+ if not e.mayReturnEmpty:
+ break
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+
+class Or(ParseExpression):
+ """
+ Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the expression that matches the longest string will be used.
+ May be constructed using the C{'^'} operator.
+
+ Example::
+ # construct Or using '^' operator
+
+ number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums))
+ print(number.searchString("123 3.1416 789"))
+ prints::
+ [['123'], ['3.1416'], ['789']]
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(Or,self).__init__(exprs, savelist)
+ if self.exprs:
+ self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+ else:
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ maxExcLoc = -1
+ maxException = None
+ matches = []
+ for e in self.exprs:
+ try:
+ loc2 = e.tryParse( instring, loc )
+ except ParseException as err:
+ err.__traceback__ = None
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+ except IndexError:
+ if len(instring) > maxExcLoc:
+ maxException = ParseException(instring,len(instring),e.errmsg,self)
+ maxExcLoc = len(instring)
+ else:
+ # save match among all matches, to retry longest to shortest
+ matches.append((loc2, e))
+
+ if matches:
+ matches.sort(key=lambda x: -x[0])
+ for _,e in matches:
+ try:
+ return e._parse( instring, loc, doActions )
+ except ParseException as err:
+ err.__traceback__ = None
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+
+ if maxException is not None:
+ maxException.msg = self.errmsg
+ raise maxException
+ else:
+ raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+
+ def __ixor__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ return self.append( other ) #Or( [ self, other ] )
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class MatchFirst(ParseExpression):
+ """
+ Requires that at least one C{ParseExpression} is found.
+ If two expressions match, the first one listed is the one that will match.
+ May be constructed using the C{'|'} operator.
+
+ Example::
+ # construct MatchFirst using '|' operator
+
+ # watch the order of expressions to match
+ number = Word(nums) | Combine(Word(nums) + '.' + Word(nums))
+ print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']]
+
+ # put more selective expression first
+ number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums)
+ print(number.searchString("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']]
+ """
+ def __init__( self, exprs, savelist = False ):
+ super(MatchFirst,self).__init__(exprs, savelist)
+ if self.exprs:
+ self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+ else:
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ maxExcLoc = -1
+ maxException = None
+ for e in self.exprs:
+ try:
+ ret = e._parse( instring, loc, doActions )
+ return ret
+ except ParseException as err:
+ if err.loc > maxExcLoc:
+ maxException = err
+ maxExcLoc = err.loc
+ except IndexError:
+ if len(instring) > maxExcLoc:
+ maxException = ParseException(instring,len(instring),e.errmsg,self)
+ maxExcLoc = len(instring)
+
+ # only got here if no expression matched, raise exception for match that made it the furthest
+ else:
+ if maxException is not None:
+ maxException.msg = self.errmsg
+ raise maxException
+ else:
+ raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+ def __ior__(self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass( other )
+ return self.append( other ) #MatchFirst( [ self, other ] )
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class Each(ParseExpression):
+ """
+ Requires all given C{ParseExpression}s to be found, but in any order.
+ Expressions may be separated by whitespace.
+ May be constructed using the C{'&'} operator.
+
+ Example::
+ color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN")
+ shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON")
+ integer = Word(nums)
+ shape_attr = "shape:" + shape_type("shape")
+ posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn")
+ color_attr = "color:" + color("color")
+ size_attr = "size:" + integer("size")
+
+ # use Each (using operator '&') to accept attributes in any order
+ # (shape and posn are required, color and size are optional)
+ shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr)
+
+ shape_spec.runTests('''
+ shape: SQUARE color: BLACK posn: 100, 120
+ shape: CIRCLE size: 50 color: BLUE posn: 50,80
+ color:GREEN size:20 shape:TRIANGLE posn:20,40
+ '''
+ )
+ prints::
+ shape: SQUARE color: BLACK posn: 100, 120
+ ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']]
+ - color: BLACK
+ - posn: ['100', ',', '120']
+ - x: 100
+ - y: 120
+ - shape: SQUARE
+
+
+ shape: CIRCLE size: 50 color: BLUE posn: 50,80
+ ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE', 'posn:', ['50', ',', '80']]
+ - color: BLUE
+ - posn: ['50', ',', '80']
+ - x: 50
+ - y: 80
+ - shape: CIRCLE
+ - size: 50
+
+
+ color: GREEN size: 20 shape: TRIANGLE posn: 20,40
+ ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE', 'posn:', ['20', ',', '40']]
+ - color: GREEN
+ - posn: ['20', ',', '40']
+ - x: 20
+ - y: 40
+ - shape: TRIANGLE
+ - size: 20
+ """
+ def __init__( self, exprs, savelist = True ):
+ super(Each,self).__init__(exprs, savelist)
+ self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+ self.skipWhitespace = True
+ self.initExprGroups = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.initExprGroups:
+ self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional))
+ opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ]
+ opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)]
+ self.optionals = opt1 + opt2
+ self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ]
+ self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ]
+ self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
+ self.required += self.multirequired
+ self.initExprGroups = False
+ tmpLoc = loc
+ tmpReqd = self.required[:]
+ tmpOpt = self.optionals[:]
+ matchOrder = []
+
+ keepMatching = True
+ while keepMatching:
+ tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
+ failed = []
+ for e in tmpExprs:
+ try:
+ tmpLoc = e.tryParse( instring, tmpLoc )
+ except ParseException:
+ failed.append(e)
+ else:
+ matchOrder.append(self.opt1map.get(id(e),e))
+ if e in tmpReqd:
+ tmpReqd.remove(e)
+ elif e in tmpOpt:
+ tmpOpt.remove(e)
+ if len(failed) == len(tmpExprs):
+ keepMatching = False
+
+ if tmpReqd:
+ missing = ", ".join(_ustr(e) for e in tmpReqd)
+ raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
+
+ # add any unmatched Optionals, in case they have default values defined
+ matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt]
+
+ resultlist = []
+ for e in matchOrder:
+ loc,results = e._parse(instring,loc,doActions)
+ resultlist.append(results)
+
+ finalResults = sum(resultlist, ParseResults([]))
+ return loc, finalResults
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}"
+
+ return self.strRepr
+
+ def checkRecursion( self, parseElementList ):
+ subRecCheckList = parseElementList[:] + [ self ]
+ for e in self.exprs:
+ e.checkRecursion( subRecCheckList )
+
+
+class ParseElementEnhance(ParserElement):
+ """
+ Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens.
+ """
+ def __init__( self, expr, savelist=False ):
+ super(ParseElementEnhance,self).__init__(savelist)
+ if isinstance( expr, basestring ):
+ if issubclass(ParserElement._literalStringClass, Token):
+ expr = ParserElement._literalStringClass(expr)
+ else:
+ expr = ParserElement._literalStringClass(Literal(expr))
+ self.expr = expr
+ self.strRepr = None
+ if expr is not None:
+ self.mayIndexError = expr.mayIndexError
+ self.mayReturnEmpty = expr.mayReturnEmpty
+ self.setWhitespaceChars( expr.whiteChars )
+ self.skipWhitespace = expr.skipWhitespace
+ self.saveAsList = expr.saveAsList
+ self.callPreparse = expr.callPreparse
+ self.ignoreExprs.extend(expr.ignoreExprs)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.expr is not None:
+ return self.expr._parse( instring, loc, doActions, callPreParse=False )
+ else:
+ raise ParseException("",loc,self.errmsg,self)
+
+ def leaveWhitespace( self ):
+ self.skipWhitespace = False
+ self.expr = self.expr.copy()
+ if self.expr is not None:
+ self.expr.leaveWhitespace()
+ return self
+
+ def ignore( self, other ):
+ if isinstance( other, Suppress ):
+ if other not in self.ignoreExprs:
+ super( ParseElementEnhance, self).ignore( other )
+ if self.expr is not None:
+ self.expr.ignore( self.ignoreExprs[-1] )
+ else:
+ super( ParseElementEnhance, self).ignore( other )
+ if self.expr is not None:
+ self.expr.ignore( self.ignoreExprs[-1] )
+ return self
+
+ def streamline( self ):
+ super(ParseElementEnhance,self).streamline()
+ if self.expr is not None:
+ self.expr.streamline()
+ return self
+
+ def checkRecursion( self, parseElementList ):
+ if self in parseElementList:
+ raise RecursiveGrammarException( parseElementList+[self] )
+ subRecCheckList = parseElementList[:] + [ self ]
+ if self.expr is not None:
+ self.expr.checkRecursion( subRecCheckList )
+
+ def validate( self, validateTrace=[] ):
+ tmp = validateTrace[:]+[self]
+ if self.expr is not None:
+ self.expr.validate(tmp)
+ self.checkRecursion( [] )
+
+ def __str__( self ):
+ try:
+ return super(ParseElementEnhance,self).__str__()
+ except Exception:
+ pass
+
+ if self.strRepr is None and self.expr is not None:
+ self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
+ return self.strRepr
+
+
+class FollowedBy(ParseElementEnhance):
+ """
+ Lookahead matching of the given parse expression. C{FollowedBy}
+ does I{not} advance the parsing position within the input string, it only
+ verifies that the specified parse expression matches at the current
+ position. C{FollowedBy} always returns a null token list.
+
+ Example::
+ # use FollowedBy to match a label only if it is followed by a ':'
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+
+ OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint()
+ prints::
+ [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']]
+ """
+ def __init__( self, expr ):
+ super(FollowedBy,self).__init__(expr)
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ self.expr.tryParse( instring, loc )
+ return loc, []
+
+
+class NotAny(ParseElementEnhance):
+ """
+ Lookahead to disallow matching with the given parse expression. C{NotAny}
+ does I{not} advance the parsing position within the input string, it only
+ verifies that the specified parse expression does I{not} match at the current
+ position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny}
+ always returns a null token list. May be constructed using the '~' operator.
+
+ Example::
+
+ """
+ def __init__( self, expr ):
+ super(NotAny,self).__init__(expr)
+ #~ self.leaveWhitespace()
+ self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
+ self.mayReturnEmpty = True
+ self.errmsg = "Found unwanted token, "+_ustr(self.expr)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ if self.expr.canParseNext(instring, loc):
+ raise ParseException(instring, loc, self.errmsg, self)
+ return loc, []
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "~{" + _ustr(self.expr) + "}"
+
+ return self.strRepr
+
+class _MultipleMatch(ParseElementEnhance):
+ def __init__( self, expr, stopOn=None):
+ super(_MultipleMatch, self).__init__(expr)
+ self.saveAsList = True
+ ender = stopOn
+ if isinstance(ender, basestring):
+ ender = ParserElement._literalStringClass(ender)
+ self.not_ender = ~ender if ender is not None else None
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ self_expr_parse = self.expr._parse
+ self_skip_ignorables = self._skipIgnorables
+ check_ender = self.not_ender is not None
+ if check_ender:
+ try_not_ender = self.not_ender.tryParse
+
+ # must be at least one (but first see if we are the stopOn sentinel;
+ # if so, fail)
+ if check_ender:
+ try_not_ender(instring, loc)
+ loc, tokens = self_expr_parse( instring, loc, doActions, callPreParse=False )
+ try:
+ hasIgnoreExprs = (not not self.ignoreExprs)
+ while 1:
+ if check_ender:
+ try_not_ender(instring, loc)
+ if hasIgnoreExprs:
+ preloc = self_skip_ignorables( instring, loc )
+ else:
+ preloc = loc
+ loc, tmptokens = self_expr_parse( instring, preloc, doActions )
+ if tmptokens or tmptokens.haskeys():
+ tokens += tmptokens
+ except (ParseException,IndexError):
+ pass
+
+ return loc, tokens
+
+class OneOrMore(_MultipleMatch):
+ """
+ Repetition of one or more of the given expression.
+
+ Parameters:
+ - expr - expression that must match one or more times
+ - stopOn - (default=C{None}) - expression for a terminating sentinel
+ (only required if the sentinel would ordinarily match the repetition
+ expression)
+
+ Example::
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+ text = "shape: SQUARE posn: upper left color: BLACK"
+ OneOrMore(attr_expr).parseString(text).pprint() # Fail! read 'color' as data instead of next label -> [['shape', 'SQUARE color']]
+
+ # use stopOn attribute for OneOrMore to avoid reading label string as part of the data
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+ OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
+
+ # could also be written as
+ (attr_expr * (1,)).parseString(text).pprint()
+ """
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "{" + _ustr(self.expr) + "}..."
+
+ return self.strRepr
+
+class ZeroOrMore(_MultipleMatch):
+ """
+ Optional repetition of zero or more of the given expression.
+
+ Parameters:
+ - expr - expression that must match zero or more times
+ - stopOn - (default=C{None}) - expression for a terminating sentinel
+ (only required if the sentinel would ordinarily match the repetition
+ expression)
+
+ Example: similar to L{OneOrMore}
+ """
+ def __init__( self, expr, stopOn=None):
+ super(ZeroOrMore,self).__init__(expr, stopOn=stopOn)
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ try:
+ return super(ZeroOrMore, self).parseImpl(instring, loc, doActions)
+ except (ParseException,IndexError):
+ return loc, []
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "[" + _ustr(self.expr) + "]..."
+
+ return self.strRepr
+
+class _NullToken(object):
+ def __bool__(self):
+ return False
+ __nonzero__ = __bool__
+ def __str__(self):
+ return ""
+
+_optionalNotMatched = _NullToken()
+class Optional(ParseElementEnhance):
+ """
+ Optional matching of the given expression.
+
+ Parameters:
+ - expr - expression that must match zero or more times
+ - default (optional) - value to be returned if the optional expression is not found.
+
+ Example::
+ # US postal code can be a 5-digit zip, plus optional 4-digit qualifier
+ zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4)))
+ zip.runTests('''
+ # traditional ZIP code
+ 12345
+
+ # ZIP+4 form
+ 12101-0001
+
+ # invalid ZIP
+ 98765-
+ ''')
+ prints::
+ # traditional ZIP code
+ 12345
+ ['12345']
+
+ # ZIP+4 form
+ 12101-0001
+ ['12101-0001']
+
+ # invalid ZIP
+ 98765-
+ ^
+ FAIL: Expected end of text (at char 5), (line:1, col:6)
+ """
+ def __init__( self, expr, default=_optionalNotMatched ):
+ super(Optional,self).__init__( expr, savelist=False )
+ self.saveAsList = self.expr.saveAsList
+ self.defaultValue = default
+ self.mayReturnEmpty = True
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ try:
+ loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+ except (ParseException,IndexError):
+ if self.defaultValue is not _optionalNotMatched:
+ if self.expr.resultsName:
+ tokens = ParseResults([ self.defaultValue ])
+ tokens[self.expr.resultsName] = self.defaultValue
+ else:
+ tokens = [ self.defaultValue ]
+ else:
+ tokens = []
+ return loc, tokens
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+
+ if self.strRepr is None:
+ self.strRepr = "[" + _ustr(self.expr) + "]"
+
+ return self.strRepr
+
+class SkipTo(ParseElementEnhance):
+ """
+ Token for skipping over all undefined text until the matched expression is found.
+
+ Parameters:
+ - expr - target expression marking the end of the data to be skipped
+ - include - (default=C{False}) if True, the target expression is also parsed
+ (the skipped text and target expression are returned as a 2-element list).
+ - ignore - (default=C{None}) used to define grammars (typically quoted strings and
+ comments) that might contain false matches to the target expression
+ - failOn - (default=C{None}) define expressions that are not allowed to be
+ included in the skipped test; if found before the target expression is found,
+ the SkipTo is not a match
+
+ Example::
+ report = '''
+ Outstanding Issues Report - 1 Jan 2000
+
+ # | Severity | Description | Days Open
+ -----+----------+-------------------------------------------+-----------
+ 101 | Critical | Intermittent system crash | 6
+ 94 | Cosmetic | Spelling error on Login ('log|n') | 14
+ 79 | Minor | System slow when running too many reports | 47
+ '''
+ integer = Word(nums)
+ SEP = Suppress('|')
+ # use SkipTo to simply match everything up until the next SEP
+ # - ignore quoted strings, so that a '|' character inside a quoted string does not match
+ # - parse action will call token.strip() for each matched token, i.e., the description body
+ string_data = SkipTo(SEP, ignore=quotedString)
+ string_data.setParseAction(tokenMap(str.strip))
+ ticket_expr = (integer("issue_num") + SEP
+ + string_data("sev") + SEP
+ + string_data("desc") + SEP
+ + integer("days_open"))
+
+ for tkt in ticket_expr.searchString(report):
+ print tkt.dump()
+ prints::
+ ['101', 'Critical', 'Intermittent system crash', '6']
+ - days_open: 6
+ - desc: Intermittent system crash
+ - issue_num: 101
+ - sev: Critical
+ ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14']
+ - days_open: 14
+ - desc: Spelling error on Login ('log|n')
+ - issue_num: 94
+ - sev: Cosmetic
+ ['79', 'Minor', 'System slow when running too many reports', '47']
+ - days_open: 47
+ - desc: System slow when running too many reports
+ - issue_num: 79
+ - sev: Minor
+ """
+ def __init__( self, other, include=False, ignore=None, failOn=None ):
+ super( SkipTo, self ).__init__( other )
+ self.ignoreExpr = ignore
+ self.mayReturnEmpty = True
+ self.mayIndexError = False
+ self.includeMatch = include
+ self.asList = False
+ if isinstance(failOn, basestring):
+ self.failOn = ParserElement._literalStringClass(failOn)
+ else:
+ self.failOn = failOn
+ self.errmsg = "No match found for "+_ustr(self.expr)
+
+ def parseImpl( self, instring, loc, doActions=True ):
+ startloc = loc
+ instrlen = len(instring)
+ expr = self.expr
+ expr_parse = self.expr._parse
+ self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None
+ self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None
+
+ tmploc = loc
+ while tmploc <= instrlen:
+ if self_failOn_canParseNext is not None:
+ # break if failOn expression matches
+ if self_failOn_canParseNext(instring, tmploc):
+ break
+
+ if self_ignoreExpr_tryParse is not None:
+ # advance past ignore expressions
+ while 1:
+ try:
+ tmploc = self_ignoreExpr_tryParse(instring, tmploc)
+ except ParseBaseException:
+ break
+
+ try:
+ expr_parse(instring, tmploc, doActions=False, callPreParse=False)
+ except (ParseException, IndexError):
+ # no match, advance loc in string
+ tmploc += 1
+ else:
+ # matched skipto expr, done
+ break
+
+ else:
+ # ran off the end of the input string without matching skipto expr, fail
+ raise ParseException(instring, loc, self.errmsg, self)
+
+ # build up return values
+ loc = tmploc
+ skiptext = instring[startloc:loc]
+ skipresult = ParseResults(skiptext)
+
+ if self.includeMatch:
+ loc, mat = expr_parse(instring,loc,doActions,callPreParse=False)
+ skipresult += mat
+
+ return loc, skipresult
+
+class Forward(ParseElementEnhance):
+ """
+ Forward declaration of an expression to be defined later -
+ used for recursive grammars, such as algebraic infix notation.
+ When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator.
+
+ Note: take care when assigning to C{Forward} not to overlook precedence of operators.
+ Specifically, '|' has a lower precedence than '<<', so that::
+ fwdExpr << a | b | c
+ will actually be evaluated as::
+ (fwdExpr << a) | b | c
+ thereby leaving b and c out as parseable alternatives. It is recommended that you
+ explicitly group the values inserted into the C{Forward}::
+ fwdExpr << (a | b | c)
+ Converting to use the '<<=' operator instead will avoid this problem.
+
+ See L{ParseResults.pprint} for an example of a recursive parser created using
+ C{Forward}.
+ """
+ def __init__( self, other=None ):
+ super(Forward,self).__init__( other, savelist=False )
+
+ def __lshift__( self, other ):
+ if isinstance( other, basestring ):
+ other = ParserElement._literalStringClass(other)
+ self.expr = other
+ self.strRepr = None
+ self.mayIndexError = self.expr.mayIndexError
+ self.mayReturnEmpty = self.expr.mayReturnEmpty
+ self.setWhitespaceChars( self.expr.whiteChars )
+ self.skipWhitespace = self.expr.skipWhitespace
+ self.saveAsList = self.expr.saveAsList
+ self.ignoreExprs.extend(self.expr.ignoreExprs)
+ return self
+
+ def __ilshift__(self, other):
+ return self << other
+
+ def leaveWhitespace( self ):
+ self.skipWhitespace = False
+ return self
+
+ def streamline( self ):
+ if not self.streamlined:
+ self.streamlined = True
+ if self.expr is not None:
+ self.expr.streamline()
+ return self
+
+ def validate( self, validateTrace=[] ):
+ if self not in validateTrace:
+ tmp = validateTrace[:]+[self]
+ if self.expr is not None:
+ self.expr.validate(tmp)
+ self.checkRecursion([])
+
+ def __str__( self ):
+ if hasattr(self,"name"):
+ return self.name
+ return self.__class__.__name__ + ": ..."
+
+ # stubbed out for now - creates awful memory and perf issues
+ self._revertClass = self.__class__
+ self.__class__ = _ForwardNoRecurse
+ try:
+ if self.expr is not None:
+ retString = _ustr(self.expr)
+ else:
+ retString = "None"
+ finally:
+ self.__class__ = self._revertClass
+ return self.__class__.__name__ + ": " + retString
+
+ def copy(self):
+ if self.expr is not None:
+ return super(Forward,self).copy()
+ else:
+ ret = Forward()
+ ret <<= self
+ return ret
+
+class _ForwardNoRecurse(Forward):
+ def __str__( self ):
+ return "..."
+
+class TokenConverter(ParseElementEnhance):
+ """
+ Abstract subclass of C{ParseExpression}, for converting parsed results.
+ """
+ def __init__( self, expr, savelist=False ):
+ super(TokenConverter,self).__init__( expr )#, savelist )
+ self.saveAsList = False
+
+class Combine(TokenConverter):
+ """
+ Converter to concatenate all matching tokens to a single string.
+ By default, the matching patterns must also be contiguous in the input string;
+ this can be disabled by specifying C{'adjacent=False'} in the constructor.
+
+ Example::
+ real = Word(nums) + '.' + Word(nums)
+ print(real.parseString('3.1416')) # -> ['3', '.', '1416']
+ # will also erroneously match the following
+ print(real.parseString('3. 1416')) # -> ['3', '.', '1416']
+
+ real = Combine(Word(nums) + '.' + Word(nums))
+ print(real.parseString('3.1416')) # -> ['3.1416']
+ # no match when there are internal spaces
+ print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...)
+ """
+ def __init__( self, expr, joinString="", adjacent=True ):
+ super(Combine,self).__init__( expr )
+ # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
+ if adjacent:
+ self.leaveWhitespace()
+ self.adjacent = adjacent
+ self.skipWhitespace = True
+ self.joinString = joinString
+ self.callPreparse = True
+
+ def ignore( self, other ):
+ if self.adjacent:
+ ParserElement.ignore(self, other)
+ else:
+ super( Combine, self).ignore( other )
+ return self
+
+ def postParse( self, instring, loc, tokenlist ):
+ retToks = tokenlist.copy()
+ del retToks[:]
+ retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
+
+ if self.resultsName and retToks.haskeys():
+ return [ retToks ]
+ else:
+ return retToks
+
+class Group(TokenConverter):
+ """
+ Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions.
+
+ Example::
+ ident = Word(alphas)
+ num = Word(nums)
+ term = ident | num
+ func = ident + Optional(delimitedList(term))
+ print(func.parseString("fn a,b,100")) # -> ['fn', 'a', 'b', '100']
+
+ func = ident + Group(Optional(delimitedList(term)))
+ print(func.parseString("fn a,b,100")) # -> ['fn', ['a', 'b', '100']]
+ """
+ def __init__( self, expr ):
+ super(Group,self).__init__( expr )
+ self.saveAsList = True
+
+ def postParse( self, instring, loc, tokenlist ):
+ return [ tokenlist ]
+
+class Dict(TokenConverter):
+ """
+ Converter to return a repetitive expression as a list, but also as a dictionary.
+ Each element can also be referenced using the first token in the expression as its key.
+ Useful for tabular report scraping when the first column can be used as a item key.
+
+ Example::
+ data_word = Word(alphas)
+ label = data_word + FollowedBy(':')
+ attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+ text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+ attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+
+ # print attributes as plain groups
+ print(OneOrMore(attr_expr).parseString(text).dump())
+
+ # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names
+ result = Dict(OneOrMore(Group(attr_expr))).parseString(text)
+ print(result.dump())
+
+ # access named fields as dict entries, or output as dict
+ print(result['shape'])
+ print(result.asDict())
+ prints::
+ ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap']
+
+ [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+ - color: light blue
+ - posn: upper left
+ - shape: SQUARE
+ - texture: burlap
+ SQUARE
+ {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'}
+ See more examples at L{ParseResults} of accessing fields by results name.
+ """
+ def __init__( self, expr ):
+ super(Dict,self).__init__( expr )
+ self.saveAsList = True
+
+ def postParse( self, instring, loc, tokenlist ):
+ for i,tok in enumerate(tokenlist):
+ if len(tok) == 0:
+ continue
+ ikey = tok[0]
+ if isinstance(ikey,int):
+ ikey = _ustr(tok[0]).strip()
+ if len(tok)==1:
+ tokenlist[ikey] = _ParseResultsWithOffset("",i)
+ elif len(tok)==2 and not isinstance(tok[1],ParseResults):
+ tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
+ else:
+ dictvalue = tok.copy() #ParseResults(i)
+ del dictvalue[0]
+ if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()):
+ tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
+ else:
+ tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
+
+ if self.resultsName:
+ return [ tokenlist ]
+ else:
+ return tokenlist
+
+
+class Suppress(TokenConverter):
+ """
+ Converter for ignoring the results of a parsed expression.
+
+ Example::
+ source = "a, b, c,d"
+ wd = Word(alphas)
+ wd_list1 = wd + ZeroOrMore(',' + wd)
+ print(wd_list1.parseString(source))
+
+ # often, delimiters that are useful during parsing are just in the
+ # way afterward - use Suppress to keep them out of the parsed output
+ wd_list2 = wd + ZeroOrMore(Suppress(',') + wd)
+ print(wd_list2.parseString(source))
+ prints::
+ ['a', ',', 'b', ',', 'c', ',', 'd']
+ ['a', 'b', 'c', 'd']
+ (See also L{delimitedList}.)
+ """
+ def postParse( self, instring, loc, tokenlist ):
+ return []
+
+ def suppress( self ):
+ return self
+
+
+class OnlyOnce(object):
+ """
+ Wrapper for parse actions, to ensure they are only called once.
+ """
+ def __init__(self, methodCall):
+ self.callable = _trim_arity(methodCall)
+ self.called = False
+ def __call__(self,s,l,t):
+ if not self.called:
+ results = self.callable(s,l,t)
+ self.called = True
+ return results
+ raise ParseException(s,l,"")
+ def reset(self):
+ self.called = False
+
+def traceParseAction(f):
+ """
+ Decorator for debugging parse actions.
+
+ When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".}
+ When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised.
+
+ Example::
+ wd = Word(alphas)
+
+ @traceParseAction
+ def remove_duplicate_chars(tokens):
+ return ''.join(sorted(set(''.join(tokens))))
+
+ wds = OneOrMore(wd).setParseAction(remove_duplicate_chars)
+ print(wds.parseString("slkdjs sld sldd sdlf sdljf"))
+ prints::
+ >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {}))
+ <<leaving remove_duplicate_chars (ret: 'dfjkls')
+ ['dfjkls']
+ """
+ f = _trim_arity(f)
+ def z(*paArgs):
+ thisFunc = f.__name__
+ s,l,t = paArgs[-3:]
+ if len(paArgs)>3:
+ thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
+ sys.stderr.write( ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc,line(l,s),l,t) )
+ try:
+ ret = f(*paArgs)
+ except Exception as exc:
+ sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
+ raise
+ sys.stderr.write( "<<leaving %s (ret: %r)\n" % (thisFunc,ret) )
+ return ret
+ try:
+ z.__name__ = f.__name__
+ except AttributeError:
+ pass
+ return z
+
+#
+# global helpers
+#
+def delimitedList( expr, delim=",", combine=False ):
+ """
+ Helper to define a delimited list of expressions - the delimiter defaults to ','.
+ By default, the list elements and delimiters can have intervening whitespace, and
+ comments, but this can be overridden by passing C{combine=True} in the constructor.
+ If C{combine} is set to C{True}, the matching tokens are returned as a single token
+ string, with the delimiters included; otherwise, the matching tokens are returned
+ as a list of tokens, with the delimiters suppressed.
+
+ Example::
+ delimitedList(Word(alphas)).parseString("aa,bb,cc") # -> ['aa', 'bb', 'cc']
+ delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE']
+ """
+ dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
+ if combine:
+ return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
+ else:
+ return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
+
+def countedArray( expr, intExpr=None ):
+ """
+ Helper to define a counted list of expressions.
+ This helper defines a pattern of the form::
+ integer expr expr expr...
+ where the leading integer tells how many expr expressions follow.
+ The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
+
+ If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value.
+
+ Example::
+ countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd']
+
+ # in this parser, the leading integer value is given in binary,
+ # '10' indicating that 2 values are in the array
+ binaryConstant = Word('01').setParseAction(lambda t: int(t[0], 2))
+ countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef') # -> ['ab', 'cd']
+ """
+ arrayExpr = Forward()
+ def countFieldParseAction(s,l,t):
+ n = t[0]
+ arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
+ return []
+ if intExpr is None:
+ intExpr = Word(nums).setParseAction(lambda t:int(t[0]))
+ else:
+ intExpr = intExpr.copy()
+ intExpr.setName("arrayLen")
+ intExpr.addParseAction(countFieldParseAction, callDuringTry=True)
+ return ( intExpr + arrayExpr ).setName('(len) ' + _ustr(expr) + '...')
+
+def _flatten(L):
+ ret = []
+ for i in L:
+ if isinstance(i,list):
+ ret.extend(_flatten(i))
+ else:
+ ret.append(i)
+ return ret
+
+def matchPreviousLiteral(expr):
+ """
+ Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousLiteral(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches a
+ previous literal, will also match the leading C{"1:1"} in C{"1:10"}.
+ If this is not desired, use C{matchPreviousExpr}.
+ Do I{not} use with packrat parsing enabled.
+ """
+ rep = Forward()
+ def copyTokenToRepeater(s,l,t):
+ if t:
+ if len(t) == 1:
+ rep << t[0]
+ else:
+ # flatten t tokens
+ tflat = _flatten(t.asList())
+ rep << And(Literal(tt) for tt in tflat)
+ else:
+ rep << Empty()
+ expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ rep.setName('(prev) ' + _ustr(expr))
+ return rep
+
+def matchPreviousExpr(expr):
+ """
+ Helper to define an expression that is indirectly defined from
+ the tokens matched in a previous expression, that is, it looks
+ for a 'repeat' of a previous expression. For example::
+ first = Word(nums)
+ second = matchPreviousExpr(first)
+ matchExpr = first + ":" + second
+ will match C{"1:1"}, but not C{"1:2"}. Because this matches by
+ expressions, will I{not} match the leading C{"1:1"} in C{"1:10"};
+ the expressions are evaluated first, and then compared, so
+ C{"1"} is compared with C{"10"}.
+ Do I{not} use with packrat parsing enabled.
+ """
+ rep = Forward()
+ e2 = expr.copy()
+ rep <<= e2
+ def copyTokenToRepeater(s,l,t):
+ matchTokens = _flatten(t.asList())
+ def mustMatchTheseTokens(s,l,t):
+ theseTokens = _flatten(t.asList())
+ if theseTokens != matchTokens:
+ raise ParseException("",0,"")
+ rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
+ expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+ rep.setName('(prev) ' + _ustr(expr))
+ return rep
+
+def _escapeRegexRangeChars(s):
+ #~ escape these chars: ^-]
+ for c in r"\^-]":
+ s = s.replace(c,_bslash+c)
+ s = s.replace("\n",r"\n")
+ s = s.replace("\t",r"\t")
+ return _ustr(s)
+
+def oneOf( strs, caseless=False, useRegex=True ):
+ """
+ Helper to quickly define a set of alternative Literals, and makes sure to do
+ longest-first testing when there is a conflict, regardless of the input order,
+ but returns a C{L{MatchFirst}} for best performance.
+
+ Parameters:
+ - strs - a string of space-delimited literals, or a collection of string literals
+ - caseless - (default=C{False}) - treat all literals as caseless
+ - useRegex - (default=C{True}) - as an optimization, will generate a Regex
+ object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or
+ if creating a C{Regex} raises an exception)
+
+ Example::
+ comp_oper = oneOf("< = > <= >= !=")
+ var = Word(alphas)
+ number = Word(nums)
+ term = var | number
+ comparison_expr = term + comp_oper + term
+ print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12"))
+ prints::
+ [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']]
+ """
+ if caseless:
+ isequal = ( lambda a,b: a.upper() == b.upper() )
+ masks = ( lambda a,b: b.upper().startswith(a.upper()) )
+ parseElementClass = CaselessLiteral
+ else:
+ isequal = ( lambda a,b: a == b )
+ masks = ( lambda a,b: b.startswith(a) )
+ parseElementClass = Literal
+
+ symbols = []
+ if isinstance(strs,basestring):
+ symbols = strs.split()
+ elif isinstance(strs, Iterable):
+ symbols = list(strs)
+ else:
+ warnings.warn("Invalid argument to oneOf, expected string or iterable",
+ SyntaxWarning, stacklevel=2)
+ if not symbols:
+ return NoMatch()
+
+ i = 0
+ while i < len(symbols)-1:
+ cur = symbols[i]
+ for j,other in enumerate(symbols[i+1:]):
+ if ( isequal(other, cur) ):
+ del symbols[i+j+1]
+ break
+ elif ( masks(cur, other) ):
+ del symbols[i+j+1]
+ symbols.insert(i,other)
+ cur = other
+ break
+ else:
+ i += 1
+
+ if not caseless and useRegex:
+ #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
+ try:
+ if len(symbols)==len("".join(symbols)):
+ return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) ).setName(' | '.join(symbols))
+ else:
+ return Regex( "|".join(re.escape(sym) for sym in symbols) ).setName(' | '.join(symbols))
+ except Exception:
+ warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
+ SyntaxWarning, stacklevel=2)
+
+
+ # last resort, just use MatchFirst
+ return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols))
+
+def dictOf( key, value ):
+ """
+ Helper to easily and clearly define a dictionary by specifying the respective patterns
+ for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
+ in the proper order. The key pattern can include delimiting markers or punctuation,
+ as long as they are suppressed, thereby leaving the significant key text. The value
+ pattern can include named results, so that the C{Dict} results can include named token
+ fields.
+
+ Example::
+ text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+ attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+ print(OneOrMore(attr_expr).parseString(text).dump())
+
+ attr_label = label
+ attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)
+
+ # similar to Dict, but simpler call format
+ result = dictOf(attr_label, attr_value).parseString(text)
+ print(result.dump())
+ print(result['shape'])
+ print(result.shape) # object attribute access works too
+ print(result.asDict())
+ prints::
+ [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+ - color: light blue
+ - posn: upper left
+ - shape: SQUARE
+ - texture: burlap
+ SQUARE
+ SQUARE
+ {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'}
+ """
+ return Dict( ZeroOrMore( Group ( key + value ) ) )
+
+def originalTextFor(expr, asString=True):
+ """
+ Helper to return the original, untokenized text for a given expression. Useful to
+ restore the parsed fields of an HTML start tag into the raw tag text itself, or to
+ revert separate tokens with intervening whitespace back to the original matching
+ input text. By default, returns astring containing the original parsed text.
+
+ If the optional C{asString} argument is passed as C{False}, then the return value is a
+ C{L{ParseResults}} containing any results names that were originally matched, and a
+ single token containing the original matched text from the input string. So if
+ the expression passed to C{L{originalTextFor}} contains expressions with defined
+ results names, you must set C{asString} to C{False} if you want to preserve those
+ results name values.
+
+ Example::
+ src = "this is test <b> bold <i>text</i> </b> normal text "
+ for tag in ("b","i"):
+ opener,closer = makeHTMLTags(tag)
+ patt = originalTextFor(opener + SkipTo(closer) + closer)
+ print(patt.searchString(src)[0])
+ prints::
+ ['<b> bold <i>text</i> </b>']
+ ['<i>text</i>']
+ """
+ locMarker = Empty().setParseAction(lambda s,loc,t: loc)
+ endlocMarker = locMarker.copy()
+ endlocMarker.callPreparse = False
+ matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end")
+ if asString:
+ extractText = lambda s,l,t: s[t._original_start:t._original_end]
+ else:
+ def extractText(s,l,t):
+ t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]]
+ matchExpr.setParseAction(extractText)
+ matchExpr.ignoreExprs = expr.ignoreExprs
+ return matchExpr
+
+def ungroup(expr):
+ """
+ Helper to undo pyparsing's default grouping of And expressions, even
+ if all but one are non-empty.
+ """
+ return TokenConverter(expr).setParseAction(lambda t:t[0])
+
+def locatedExpr(expr):
+ """
+ Helper to decorate a returned token with its starting and ending locations in the input string.
+ This helper adds the following results names:
+ - locn_start = location where matched expression begins
+ - locn_end = location where matched expression ends
+ - value = the actual parsed results
+
+ Be careful if the input text contains C{<TAB>} characters, you may want to call
+ C{L{ParserElement.parseWithTabs}}
+
+ Example::
+ wd = Word(alphas)
+ for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"):
+ print(match)
+ prints::
+ [[0, 'ljsdf', 5]]
+ [[8, 'lksdjjf', 15]]
+ [[18, 'lkkjj', 23]]
+ """
+ locator = Empty().setParseAction(lambda s,l,t: l)
+ return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end"))
+
+
+# convenience constants for positional expressions
+empty = Empty().setName("empty")
+lineStart = LineStart().setName("lineStart")
+lineEnd = LineEnd().setName("lineEnd")
+stringStart = StringStart().setName("stringStart")
+stringEnd = StringEnd().setName("stringEnd")
+
+_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
+_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
+_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1)
+_charRange = Group(_singleChar + Suppress("-") + _singleChar)
+_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
+
+def srange(s):
+ r"""
+ Helper to easily define string ranges for use in Word construction. Borrows
+ syntax from regexp '[]' string range definitions::
+ srange("[0-9]") -> "0123456789"
+ srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
+ srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
+ The input string must be enclosed in []'s, and the returned string is the expanded
+ character set joined into a single string.
+ The values enclosed in the []'s may be:
+ - a single character
+ - an escaped character with a leading backslash (such as C{\-} or C{\]})
+ - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character)
+ (C{\0x##} is also supported for backwards compatibility)
+ - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character)
+ - a range of any of the above, separated by a dash (C{'a-z'}, etc.)
+ - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.)
+ """
+ _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1))
+ try:
+ return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body)
+ except Exception:
+ return ""
+
+def matchOnlyAtCol(n):
+ """
+ Helper method for defining parse actions that require matching at a specific
+ column in the input text.
+ """
+ def verifyCol(strg,locn,toks):
+ if col(locn,strg) != n:
+ raise ParseException(strg,locn,"matched token not at column %d" % n)
+ return verifyCol
+
+def replaceWith(replStr):
+ """
+ Helper method for common parse actions that simply return a literal value. Especially
+ useful when used with C{L{transformString<ParserElement.transformString>}()}.
+
+ Example::
+ num = Word(nums).setParseAction(lambda toks: int(toks[0]))
+ na = oneOf("N/A NA").setParseAction(replaceWith(math.nan))
+ term = na | num
+
+ OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234]
+ """
+ return lambda s,l,t: [replStr]
+
+def removeQuotes(s,l,t):
+ """
+ Helper parse action for removing quotation marks from parsed quoted strings.
+
+ Example::
+ # by default, quotation marks are included in parsed results
+ quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"]
+
+ # use removeQuotes to strip quotation marks from parsed results
+ quotedString.setParseAction(removeQuotes)
+ quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["Now is the Winter of our Discontent"]
+ """
+ return t[0][1:-1]
+
+def tokenMap(func, *args):
+ """
+ Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional
+ args are passed, they are forwarded to the given function as additional arguments after
+ the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the
+ parsed data to an integer using base 16.
+
+ Example (compare the last to example in L{ParserElement.transformString}::
+ hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16))
+ hex_ints.runTests('''
+ 00 11 22 aa FF 0a 0d 1a
+ ''')
+
+ upperword = Word(alphas).setParseAction(tokenMap(str.upper))
+ OneOrMore(upperword).runTests('''
+ my kingdom for a horse
+ ''')
+
+ wd = Word(alphas).setParseAction(tokenMap(str.title))
+ OneOrMore(wd).setParseAction(' '.join).runTests('''
+ now is the winter of our discontent made glorious summer by this sun of york
+ ''')
+ prints::
+ 00 11 22 aa FF 0a 0d 1a
+ [0, 17, 34, 170, 255, 10, 13, 26]
+
+ my kingdom for a horse
+ ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE']
+
+ now is the winter of our discontent made glorious summer by this sun of york
+ ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York']
+ """
+ def pa(s,l,t):
+ return [func(tokn, *args) for tokn in t]
+
+ try:
+ func_name = getattr(func, '__name__',
+ getattr(func, '__class__').__name__)
+ except Exception:
+ func_name = str(func)
+ pa.__name__ = func_name
+
+ return pa
+
+upcaseTokens = tokenMap(lambda t: _ustr(t).upper())
+"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}"""
+
+downcaseTokens = tokenMap(lambda t: _ustr(t).lower())
+"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}"""
+
+def _makeTags(tagStr, xml):
+ """Internal helper to construct opening and closing tag expressions, given a tag name"""
+ if isinstance(tagStr,basestring):
+ resname = tagStr
+ tagStr = Keyword(tagStr, caseless=not xml)
+ else:
+ resname = tagStr.name
+
+ tagAttrName = Word(alphas,alphanums+"_-:")
+ if (xml):
+ tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
+ openTag = Suppress("<") + tagStr("tag") + \
+ Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
+ Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+ else:
+ printablesLessRAbrack = "".join(c for c in printables if c not in ">")
+ tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
+ openTag = Suppress("<") + tagStr("tag") + \
+ Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
+ Optional( Suppress("=") + tagAttrValue ) ))) + \
+ Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+ closeTag = Combine(_L("</") + tagStr + ">")
+
+ openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % resname)
+ closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % resname)
+ openTag.tag = resname
+ closeTag.tag = resname
+ return openTag, closeTag
+
+def makeHTMLTags(tagStr):
+ """
+ Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches
+ tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values.
+
+ Example::
+ text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>'
+ # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple
+ a,a_end = makeHTMLTags("A")
+ link_expr = a + SkipTo(a_end)("link_text") + a_end
+
+ for link in link_expr.searchString(text):
+ # attributes in the <A> tag (like "href" shown here) are also accessible as named results
+ print(link.link_text, '->', link.href)
+ prints::
+ pyparsing -> http://pyparsing.wikispaces.com
+ """
+ return _makeTags( tagStr, False )
+
+def makeXMLTags(tagStr):
+ """
+ Helper to construct opening and closing tag expressions for XML, given a tag name. Matches
+ tags only in the given upper/lower case.
+
+ Example: similar to L{makeHTMLTags}
+ """
+ return _makeTags( tagStr, True )
+
+def withAttribute(*args,**attrDict):
+ """
+ Helper to create a validating parse action to be used with start tags created
+ with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
+ with a required attribute value, to avoid false matches on common tags such as
+ C{<TD>} or C{<DIV>}.
+
+ Call C{withAttribute} with a series of attribute names and values. Specify the list
+ of filter attributes names and values as:
+ - keyword arguments, as in C{(align="right")}, or
+ - as an explicit dict with C{**} operator, when an attribute name is also a Python
+ reserved word, as in C{**{"class":"Customer", "align":"right"}}
+ - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
+ For attribute names with a namespace prefix, you must use the second form. Attribute
+ names are matched insensitive to upper/lower case.
+
+ If just testing for C{class} (with or without a namespace), use C{L{withClass}}.
+
+ To verify that the attribute exists, but without specifying a value, pass
+ C{withAttribute.ANY_VALUE} as the value.
+
+ Example::
+ html = '''
+ <div>
+ Some text
+ <div type="grid">1 4 0 1 0</div>
+ <div type="graph">1,3 2,3 1,1</div>
+ <div>this has no type</div>
+ </div>
+
+ '''
+ div,div_end = makeHTMLTags("div")
+
+ # only match div tag having a type attribute with value "grid"
+ div_grid = div().setParseAction(withAttribute(type="grid"))
+ grid_expr = div_grid + SkipTo(div | div_end)("body")
+ for grid_header in grid_expr.searchString(html):
+ print(grid_header.body)
+
+ # construct a match with any div tag having a type attribute, regardless of the value
+ div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE))
+ div_expr = div_any_type + SkipTo(div | div_end)("body")
+ for div_header in div_expr.searchString(html):
+ print(div_header.body)
+ prints::
+ 1 4 0 1 0
+
+ 1 4 0 1 0
+ 1,3 2,3 1,1
+ """
+ if args:
+ attrs = args[:]
+ else:
+ attrs = attrDict.items()
+ attrs = [(k,v) for k,v in attrs]
+ def pa(s,l,tokens):
+ for attrName,attrValue in attrs:
+ if attrName not in tokens:
+ raise ParseException(s,l,"no matching attribute " + attrName)
+ if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
+ raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
+ (attrName, tokens[attrName], attrValue))
+ return pa
+withAttribute.ANY_VALUE = object()
+
+def withClass(classname, namespace=''):
+ """
+ Simplified version of C{L{withAttribute}} when matching on a div class - made
+ difficult because C{class} is a reserved word in Python.
+
+ Example::
+ html = '''
+ <div>
+ Some text
+ <div class="grid">1 4 0 1 0</div>
+ <div class="graph">1,3 2,3 1,1</div>
+ <div>this <div> has no class</div>
+ </div>
+
+ '''
+ div,div_end = makeHTMLTags("div")
+ div_grid = div().setParseAction(withClass("grid"))
+
+ grid_expr = div_grid + SkipTo(div | div_end)("body")
+ for grid_header in grid_expr.searchString(html):
+ print(grid_header.body)
+
+ div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE))
+ div_expr = div_any_type + SkipTo(div | div_end)("body")
+ for div_header in div_expr.searchString(html):
+ print(div_header.body)
+ prints::
+ 1 4 0 1 0
+
+ 1 4 0 1 0
+ 1,3 2,3 1,1
+ """
+ classattr = "%s:class" % namespace if namespace else "class"
+ return withAttribute(**{classattr : classname})
+
+opAssoc = _Constants()
+opAssoc.LEFT = object()
+opAssoc.RIGHT = object()
+
+def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ):
+ """
+ Helper method for constructing grammars of expressions made up of
+ operators working in a precedence hierarchy. Operators may be unary or
+ binary, left- or right-associative. Parse actions can also be attached
+ to operator expressions. The generated parser will also recognize the use
+ of parentheses to override operator precedences (see example below).
+
+ Note: if you define a deep operator list, you may see performance issues
+ when using infixNotation. See L{ParserElement.enablePackrat} for a
+ mechanism to potentially improve your parser performance.
+
+ Parameters:
+ - baseExpr - expression representing the most basic element for the nested
+ - opList - list of tuples, one for each operator precedence level in the
+ expression grammar; each tuple is of the form
+ (opExpr, numTerms, rightLeftAssoc, parseAction), where:
+ - opExpr is the pyparsing expression for the operator;
+ may also be a string, which will be converted to a Literal;
+ if numTerms is 3, opExpr is a tuple of two expressions, for the
+ two operators separating the 3 terms
+ - numTerms is the number of terms for this operator (must
+ be 1, 2, or 3)
+ - rightLeftAssoc is the indicator whether the operator is
+ right or left associative, using the pyparsing-defined
+ constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
+ - parseAction is the parse action to be associated with
+ expressions matching this operator expression (the
+ parse action tuple member may be omitted); if the parse action
+ is passed a tuple or list of functions, this is equivalent to
+ calling C{setParseAction(*fn)} (L{ParserElement.setParseAction})
+ - lpar - expression for matching left-parentheses (default=C{Suppress('(')})
+ - rpar - expression for matching right-parentheses (default=C{Suppress(')')})
+
+ Example::
+ # simple example of four-function arithmetic with ints and variable names
+ integer = pyparsing_common.signed_integer
+ varname = pyparsing_common.identifier
+
+ arith_expr = infixNotation(integer | varname,
+ [
+ ('-', 1, opAssoc.RIGHT),
+ (oneOf('* /'), 2, opAssoc.LEFT),
+ (oneOf('+ -'), 2, opAssoc.LEFT),
+ ])
+
+ arith_expr.runTests('''
+ 5+3*6
+ (5+3)*6
+ -2--11
+ ''', fullDump=False)
+ prints::
+ 5+3*6
+ [[5, '+', [3, '*', 6]]]
+
+ (5+3)*6
+ [[[5, '+', 3], '*', 6]]
+
+ -2--11
+ [[['-', 2], '-', ['-', 11]]]
+ """
+ ret = Forward()
+ lastExpr = baseExpr | ( lpar + ret + rpar )
+ for i,operDef in enumerate(opList):
+ opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
+ termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr
+ if arity == 3:
+ if opExpr is None or len(opExpr) != 2:
+ raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
+ opExpr1, opExpr2 = opExpr
+ thisExpr = Forward().setName(termName)
+ if rightLeftAssoc == opAssoc.LEFT:
+ if arity == 1:
+ matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
+ elif arity == 2:
+ if opExpr is not None:
+ matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) )
+ else:
+ matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) )
+ elif arity == 3:
+ matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \
+ Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr )
+ else:
+ raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+ elif rightLeftAssoc == opAssoc.RIGHT:
+ if arity == 1:
+ # try to avoid LR with this extra test
+ if not isinstance(opExpr, Optional):
+ opExpr = Optional(opExpr)
+ matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr )
+ elif arity == 2:
+ if opExpr is not None:
+ matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) )
+ else:
+ matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) )
+ elif arity == 3:
+ matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \
+ Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr )
+ else:
+ raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+ else:
+ raise ValueError("operator must indicate right or left associativity")
+ if pa:
+ if isinstance(pa, (tuple, list)):
+ matchExpr.setParseAction(*pa)
+ else:
+ matchExpr.setParseAction(pa)
+ thisExpr <<= ( matchExpr.setName(termName) | lastExpr )
+ lastExpr = thisExpr
+ ret <<= lastExpr
+ return ret
+
+operatorPrecedence = infixNotation
+"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release."""
+
+dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes")
+sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes")
+quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"'|
+ Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("quotedString using single or double quotes")
+unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal")
+
+def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()):
+ """
+ Helper method for defining nested lists enclosed in opening and closing
+ delimiters ("(" and ")" are the default).
+
+ Parameters:
+ - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression
+ - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression
+ - content - expression for items within the nested lists (default=C{None})
+ - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString})
+
+ If an expression is not provided for the content argument, the nested
+ expression will capture all whitespace-delimited content between delimiters
+ as a list of separate values.
+
+ Use the C{ignoreExpr} argument to define expressions that may contain
+ opening or closing characters that should not be treated as opening
+ or closing characters for nesting, such as quotedString or a comment
+ expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}.
+ The default is L{quotedString}, but if no expressions are to be ignored,
+ then pass C{None} for this argument.
+
+ Example::
+ data_type = oneOf("void int short long char float double")
+ decl_data_type = Combine(data_type + Optional(Word('*')))
+ ident = Word(alphas+'_', alphanums+'_')
+ number = pyparsing_common.number
+ arg = Group(decl_data_type + ident)
+ LPAR,RPAR = map(Suppress, "()")
+
+ code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment))
+
+ c_function = (decl_data_type("type")
+ + ident("name")
+ + LPAR + Optional(delimitedList(arg), [])("args") + RPAR
+ + code_body("body"))
+ c_function.ignore(cStyleComment)
+
+ source_code = '''
+ int is_odd(int x) {
+ return (x%2);
+ }
+
+ int dec_to_hex(char hchar) {
+ if (hchar >= '0' && hchar <= '9') {
+ return (ord(hchar)-ord('0'));
+ } else {
+ return (10+ord(hchar)-ord('A'));
+ }
+ }
+ '''
+ for func in c_function.searchString(source_code):
+ print("%(name)s (%(type)s) args: %(args)s" % func)
+
+ prints::
+ is_odd (int) args: [['int', 'x']]
+ dec_to_hex (int) args: [['char', 'hchar']]
+ """
+ if opener == closer:
+ raise ValueError("opening and closing strings cannot be the same")
+ if content is None:
+ if isinstance(opener,basestring) and isinstance(closer,basestring):
+ if len(opener) == 1 and len(closer)==1:
+ if ignoreExpr is not None:
+ content = (Combine(OneOrMore(~ignoreExpr +
+ CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ if ignoreExpr is not None:
+ content = (Combine(OneOrMore(~ignoreExpr +
+ ~Literal(opener) + ~Literal(closer) +
+ CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) +
+ CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+ ).setParseAction(lambda t:t[0].strip()))
+ else:
+ raise ValueError("opening and closing arguments must be strings if no content expression is given")
+ ret = Forward()
+ if ignoreExpr is not None:
+ ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
+ else:
+ ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) )
+ ret.setName('nested %s%s expression' % (opener,closer))
+ return ret
+
+def indentedBlock(blockStatementExpr, indentStack, indent=True):
+ """
+ Helper method for defining space-delimited indentation blocks, such as
+ those used to define block statements in Python source code.
+
+ Parameters:
+ - blockStatementExpr - expression defining syntax of statement that
+ is repeated within the indented block
+ - indentStack - list created by caller to manage indentation stack
+ (multiple statementWithIndentedBlock expressions within a single grammar
+ should share a common indentStack)
+ - indent - boolean indicating whether block must be indented beyond the
+ the current level; set to False for block of left-most statements
+ (default=C{True})
+
+ A valid block must contain at least one C{blockStatement}.
+
+ Example::
+ data = '''
+ def A(z):
+ A1
+ B = 100
+ G = A2
+ A2
+ A3
+ B
+ def BB(a,b,c):
+ BB1
+ def BBA():
+ bba1
+ bba2
+ bba3
+ C
+ D
+ def spam(x,y):
+ def eggs(z):
+ pass
+ '''
+
+
+ indentStack = [1]
+ stmt = Forward()
+
+ identifier = Word(alphas, alphanums)
+ funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":")
+ func_body = indentedBlock(stmt, indentStack)
+ funcDef = Group( funcDecl + func_body )
+
+ rvalue = Forward()
+ funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")")
+ rvalue << (funcCall | identifier | Word(nums))
+ assignment = Group(identifier + "=" + rvalue)
+ stmt << ( funcDef | assignment | identifier )
+
+ module_body = OneOrMore(stmt)
+
+ parseTree = module_body.parseString(data)
+ parseTree.pprint()
+ prints::
+ [['def',
+ 'A',
+ ['(', 'z', ')'],
+ ':',
+ [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]],
+ 'B',
+ ['def',
+ 'BB',
+ ['(', 'a', 'b', 'c', ')'],
+ ':',
+ [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]],
+ 'C',
+ 'D',
+ ['def',
+ 'spam',
+ ['(', 'x', 'y', ')'],
+ ':',
+ [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]]
+ """
+ def checkPeerIndent(s,l,t):
+ if l >= len(s): return
+ curCol = col(l,s)
+ if curCol != indentStack[-1]:
+ if curCol > indentStack[-1]:
+ raise ParseFatalException(s,l,"illegal nesting")
+ raise ParseException(s,l,"not a peer entry")
+
+ def checkSubIndent(s,l,t):
+ curCol = col(l,s)
+ if curCol > indentStack[-1]:
+ indentStack.append( curCol )
+ else:
+ raise ParseException(s,l,"not a subentry")
+
+ def checkUnindent(s,l,t):
+ if l >= len(s): return
+ curCol = col(l,s)
+ if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):
+ raise ParseException(s,l,"not an unindent")
+ indentStack.pop()
+
+ NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
+ INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT')
+ PEER = Empty().setParseAction(checkPeerIndent).setName('')
+ UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT')
+ if indent:
+ smExpr = Group( Optional(NL) +
+ #~ FollowedBy(blockStatementExpr) +
+ INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)
+ else:
+ smExpr = Group( Optional(NL) +
+ (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
+ blockStatementExpr.ignore(_bslash + LineEnd())
+ return smExpr.setName('indented block')
+
+alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
+punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
+
+anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:").setName('any tag'))
+_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(),'><& "\''))
+commonHTMLEntity = Regex('&(?P<entity>' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity")
+def replaceHTMLEntity(t):
+ """Helper parser action to replace common HTML entities with their special characters"""
+ return _htmlEntityMap.get(t.entity)
+
+# it's easy to get these comment structures wrong - they're very common, so may as well make them available
+cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment")
+"Comment of the form C{/* ... */}"
+
+htmlComment = Regex(r"<!--[\s\S]*?-->").setName("HTML comment")
+"Comment of the form C{<!-- ... -->}"
+
+restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line")
+dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment")
+"Comment of the form C{// ... (to end of line)}"
+
+cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment")
+"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}"
+
+javaStyleComment = cppStyleComment
+"Same as C{L{cppStyleComment}}"
+
+pythonStyleComment = Regex(r"#.*").setName("Python style comment")
+"Comment of the form C{# ... (to end of line)}"
+
+_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') +
+ Optional( Word(" \t") +
+ ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
+commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList")
+"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas.
+ This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}."""
+
+# some other useful expressions - using lower-case class name since we are really using this as a namespace
+class pyparsing_common:
+ """
+ Here are some common low-level expressions that may be useful in jump-starting parser development:
+ - numeric forms (L{integers<integer>}, L{reals<real>}, L{scientific notation<sci_real>})
+ - common L{programming identifiers<identifier>}
+ - network addresses (L{MAC<mac_address>}, L{IPv4<ipv4_address>}, L{IPv6<ipv6_address>})
+ - ISO8601 L{dates<iso8601_date>} and L{datetime<iso8601_datetime>}
+ - L{UUID<uuid>}
+ - L{comma-separated list<comma_separated_list>}
+ Parse actions:
+ - C{L{convertToInteger}}
+ - C{L{convertToFloat}}
+ - C{L{convertToDate}}
+ - C{L{convertToDatetime}}
+ - C{L{stripHTMLTags}}
+ - C{L{upcaseTokens}}
+ - C{L{downcaseTokens}}
+
+ Example::
+ pyparsing_common.number.runTests('''
+ # any int or real number, returned as the appropriate type
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ ''')
+
+ pyparsing_common.fnumber.runTests('''
+ # any int or real number, returned as float
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ ''')
+
+ pyparsing_common.hex_integer.runTests('''
+ # hex numbers
+ 100
+ FF
+ ''')
+
+ pyparsing_common.fraction.runTests('''
+ # fractions
+ 1/2
+ -3/4
+ ''')
+
+ pyparsing_common.mixed_integer.runTests('''
+ # mixed fractions
+ 1
+ 1/2
+ -3/4
+ 1-3/4
+ ''')
+
+ import uuid
+ pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+ pyparsing_common.uuid.runTests('''
+ # uuid
+ 12345678-1234-5678-1234-567812345678
+ ''')
+ prints::
+ # any int or real number, returned as the appropriate type
+ 100
+ [100]
+
+ -100
+ [-100]
+
+ +100
+ [100]
+
+ 3.14159
+ [3.14159]
+
+ 6.02e23
+ [6.02e+23]
+
+ 1e-12
+ [1e-12]
+
+ # any int or real number, returned as float
+ 100
+ [100.0]
+
+ -100
+ [-100.0]
+
+ +100
+ [100.0]
+
+ 3.14159
+ [3.14159]
+
+ 6.02e23
+ [6.02e+23]
+
+ 1e-12
+ [1e-12]
+
+ # hex numbers
+ 100
+ [256]
+
+ FF
+ [255]
+
+ # fractions
+ 1/2
+ [0.5]
+
+ -3/4
+ [-0.75]
+
+ # mixed fractions
+ 1
+ [1]
+
+ 1/2
+ [0.5]
+
+ -3/4
+ [-0.75]
+
+ 1-3/4
+ [1.75]
+
+ # uuid
+ 12345678-1234-5678-1234-567812345678
+ [UUID('12345678-1234-5678-1234-567812345678')]
+ """
+
+ convertToInteger = tokenMap(int)
+ """
+ Parse action for converting parsed integers to Python int
+ """
+
+ convertToFloat = tokenMap(float)
+ """
+ Parse action for converting parsed numbers to Python float
+ """
+
+ integer = Word(nums).setName("integer").setParseAction(convertToInteger)
+ """expression that parses an unsigned integer, returns an int"""
+
+ hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int,16))
+ """expression that parses a hexadecimal integer, returns an int"""
+
+ signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger)
+ """expression that parses an integer with optional leading sign, returns an int"""
+
+ fraction = (signed_integer().setParseAction(convertToFloat) + '/' + signed_integer().setParseAction(convertToFloat)).setName("fraction")
+ """fractional expression of an integer divided by an integer, returns a float"""
+ fraction.addParseAction(lambda t: t[0]/t[-1])
+
+ mixed_integer = (fraction | signed_integer + Optional(Optional('-').suppress() + fraction)).setName("fraction or mixed integer-fraction")
+ """mixed integer of the form 'integer - fraction', with optional leading integer, returns float"""
+ mixed_integer.addParseAction(sum)
+
+ real = Regex(r'[+-]?\d+\.\d*').setName("real number").setParseAction(convertToFloat)
+ """expression that parses a floating point number and returns a float"""
+
+ sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat)
+ """expression that parses a floating point number with optional scientific notation and returns a float"""
+
+ # streamlining this expression makes the docs nicer-looking
+ number = (sci_real | real | signed_integer).streamline()
+ """any numeric expression, returns the corresponding Python type"""
+
+ fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat)
+ """any int or real number, returned as float"""
+
+ identifier = Word(alphas+'_', alphanums+'_').setName("identifier")
+ """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')"""
+
+ ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address")
+ "IPv4 address (C{0.0.0.0 - 255.255.255.255})"
+
+ _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer")
+ _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address")
+ _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part)*(0,6)) + "::" + Optional(_ipv6_part + (':' + _ipv6_part)*(0,6))).setName("short IPv6 address")
+ _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8)
+ _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address")
+ ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address")
+ "IPv6 address (long, short, or mixed form)"
+
+ mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address")
+ "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)"
+
+ @staticmethod
+ def convertToDate(fmt="%Y-%m-%d"):
+ """
+ Helper to create a parse action for converting parsed date string to Python datetime.date
+
+ Params -
+ - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"})
+
+ Example::
+ date_expr = pyparsing_common.iso8601_date.copy()
+ date_expr.setParseAction(pyparsing_common.convertToDate())
+ print(date_expr.parseString("1999-12-31"))
+ prints::
+ [datetime.date(1999, 12, 31)]
+ """
+ def cvt_fn(s,l,t):
+ try:
+ return datetime.strptime(t[0], fmt).date()
+ except ValueError as ve:
+ raise ParseException(s, l, str(ve))
+ return cvt_fn
+
+ @staticmethod
+ def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"):
+ """
+ Helper to create a parse action for converting parsed datetime string to Python datetime.datetime
+
+ Params -
+ - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"})
+
+ Example::
+ dt_expr = pyparsing_common.iso8601_datetime.copy()
+ dt_expr.setParseAction(pyparsing_common.convertToDatetime())
+ print(dt_expr.parseString("1999-12-31T23:59:59.999"))
+ prints::
+ [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)]
+ """
+ def cvt_fn(s,l,t):
+ try:
+ return datetime.strptime(t[0], fmt)
+ except ValueError as ve:
+ raise ParseException(s, l, str(ve))
+ return cvt_fn
+
+ iso8601_date = Regex(r'(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?').setName("ISO8601 date")
+ "ISO8601 date (C{yyyy-mm-dd})"
+
+ iso8601_datetime = Regex(r'(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime")
+ "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}"
+
+ uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID")
+ "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})"
+
+ _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress()
+ @staticmethod
+ def stripHTMLTags(s, l, tokens):
+ """
+ Parse action to remove HTML tags from web page HTML source
+
+ Example::
+ # strip HTML links from normal text
+ text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>'
+ td,td_end = makeHTMLTags("TD")
+ table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end
+
+ print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page'
+ """
+ return pyparsing_common._html_stripper.transformString(tokens[0])
+
+ _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',')
+ + Optional( White(" \t") ) ) ).streamline().setName("commaItem")
+ comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list")
+ """Predefined expression of 1 or more printable words or quoted strings, separated by commas."""
+
+ upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper()))
+ """Parse action to convert tokens to upper case."""
+
+ downcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).lower()))
+ """Parse action to convert tokens to lower case."""
+
+
+if __name__ == "__main__":
+
+ selectToken = CaselessLiteral("select")
+ fromToken = CaselessLiteral("from")
+
+ ident = Word(alphas, alphanums + "_$")
+
+ columnName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+ columnNameList = Group(delimitedList(columnName)).setName("columns")
+ columnSpec = ('*' | columnNameList)
+
+ tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+ tableNameList = Group(delimitedList(tableName)).setName("tables")
+
+ simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables")
+
+ # demo runTests method, including embedded comments in test string
+ simpleSQL.runTests("""
+ # '*' as column list and dotted table name
+ select * from SYS.XYZZY
+
+ # caseless match on "SELECT", and casts back to "select"
+ SELECT * from XYZZY, ABC
+
+ # list of column names, and mixed case SELECT keyword
+ Select AA,BB,CC from Sys.dual
+
+ # multiple tables
+ Select A, B, C from Sys.dual, Table2
+
+ # invalid SELECT keyword - should fail
+ Xelect A, B, C from Sys.dual
+
+ # incomplete command - should fail
+ Select
+
+ # invalid column name - should fail
+ Select ^^^ frox Sys.dual
+
+ """)
+
+ pyparsing_common.number.runTests("""
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ """)
+
+ # any int or real number, returned as float
+ pyparsing_common.fnumber.runTests("""
+ 100
+ -100
+ +100
+ 3.14159
+ 6.02e23
+ 1e-12
+ """)
+
+ pyparsing_common.hex_integer.runTests("""
+ 100
+ FF
+ """)
+
+ import uuid
+ pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+ pyparsing_common.uuid.runTests("""
+ 12345678-1234-5678-1234-567812345678
+ """)
diff --git a/setuptools/_vendor/six.py b/setuptools/_vendor/six.py new file mode 100644 index 00000000..190c0239 --- /dev/null +++ b/setuptools/_vendor/six.py @@ -0,0 +1,868 @@ +"""Utilities for writing code that runs on Python 2 and 3""" + +# Copyright (c) 2010-2015 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson <benjamin@python.org>" +__version__ = "1.10.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") + + +if sys.version_info[:2] == (3, 2): + exec_("""def raise_from(value, from_value): + if from_value is None: + raise value + raise value from from_value +""") +elif sys.version_info[:2] > (3, 2): + exec_("""def raise_from(value, from_value): + raise value from from_value +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + def wrapper(f): + f = functools.wraps(wrapped, assigned, updated)(f) + f.__wrapped__ = wrapped + return f + return wrapper +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/setuptools/_vendor/vendored.txt b/setuptools/_vendor/vendored.txt new file mode 100644 index 00000000..7d77863d --- /dev/null +++ b/setuptools/_vendor/vendored.txt @@ -0,0 +1,3 @@ +packaging==16.8 +pyparsing==2.2.1 +six==1.10.0 diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index cc82b3da..81436044 100755..100644 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -8,7 +8,7 @@ import posixpath import contextlib from distutils.errors import DistutilsError -from pkg_resources import ensure_directory, ContextualZipFile +from pkg_resources import ensure_directory __all__ = [ "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", @@ -98,7 +98,7 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): if not zipfile.is_zipfile(filename): raise UnrecognizedFormat("%s is not a zip file" % (filename,)) - with ContextualZipFile(filename) as z: + with zipfile.ZipFile(filename) as z: for info in z.infolist(): name = info.filename diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index 54f2987b..c883d92f 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -61,14 +61,28 @@ class Distribution(setuptools.dist.Distribution): distutils.core.Distribution = orig +def _to_str(s): + """ + Convert a filename to a string (on Python 2, explicitly + a byte string, not Unicode) as distutils checks for the + exact type str. + """ + if sys.version_info[0] == 2 and not isinstance(s, str): + # Assume it's Unicode, as that's what the PEP says + # should be provided. + return s.encode(sys.getfilesystemencoding()) + return s + + def _run_setup(setup_script='setup.py'): # Note that we can reuse our build directory between calls # Correctness comes first, then optimization later __file__ = setup_script + __name__ = '__main__' f = getattr(tokenize, 'open', open)(__file__) code = f.read().replace('\\r\\n', '\\n') f.close() - exec(compile(code, __file__, 'exec')) + exec(compile(code, __file__, 'exec'), locals()) def _fix_config(config_settings): @@ -77,9 +91,8 @@ def _fix_config(config_settings): return config_settings -def _get_build_requires(config_settings): +def _get_build_requires(config_settings, requirements): config_settings = _fix_config(config_settings) - requirements = ['setuptools', 'wheel'] sys.argv = sys.argv[:1] + ['egg_info'] + \ config_settings["--global-option"] @@ -92,27 +105,59 @@ def _get_build_requires(config_settings): return requirements +def _get_immediate_subdirectories(a_dir): + return [name for name in os.listdir(a_dir) + if os.path.isdir(os.path.join(a_dir, name))] + + def get_requires_for_build_wheel(config_settings=None): config_settings = _fix_config(config_settings) - return _get_build_requires(config_settings) + return _get_build_requires(config_settings, requirements=['wheel']) def get_requires_for_build_sdist(config_settings=None): config_settings = _fix_config(config_settings) - return _get_build_requires(config_settings) + return _get_build_requires(config_settings, requirements=[]) def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): - sys.argv = sys.argv[:1] + ['dist_info', '--egg-base', metadata_directory] + sys.argv = sys.argv[:1] + ['dist_info', '--egg-base', _to_str(metadata_directory)] _run_setup() - dist_infos = [f for f in os.listdir(metadata_directory) - if f.endswith('.dist-info')] + dist_info_directory = metadata_directory + while True: + dist_infos = [f for f in os.listdir(dist_info_directory) + if f.endswith('.dist-info')] + + if len(dist_infos) == 0 and \ + len(_get_immediate_subdirectories(dist_info_directory)) == 1: + dist_info_directory = os.path.join( + dist_info_directory, os.listdir(dist_info_directory)[0]) + continue + + assert len(dist_infos) == 1 + break + + # PEP 517 requires that the .dist-info directory be placed in the + # metadata_directory. To comply, we MUST copy the directory to the root + if dist_info_directory != metadata_directory: + shutil.move( + os.path.join(dist_info_directory, dist_infos[0]), + metadata_directory) + shutil.rmtree(dist_info_directory, ignore_errors=True) - assert len(dist_infos) == 1 return dist_infos[0] +def _file_with_extension(directory, extension): + matching = ( + f for f in os.listdir(directory) + if f.endswith(extension) + ) + file, = matching + return file + + def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): config_settings = _fix_config(config_settings) @@ -124,25 +169,15 @@ def build_wheel(wheel_directory, config_settings=None, shutil.rmtree(wheel_directory) shutil.copytree('dist', wheel_directory) - wheels = [f for f in os.listdir(wheel_directory) - if f.endswith('.whl')] - - assert len(wheels) == 1 - return wheels[0] + return _file_with_extension(wheel_directory, '.whl') def build_sdist(sdist_directory, config_settings=None): config_settings = _fix_config(config_settings) sdist_directory = os.path.abspath(sdist_directory) - sys.argv = sys.argv[:1] + ['sdist'] + \ - config_settings["--global-option"] + sys.argv = sys.argv[:1] + ['sdist', '--formats', 'gztar'] + \ + config_settings["--global-option"] + \ + ["--dist-dir", sdist_directory] _run_setup() - if sdist_directory != 'dist': - shutil.rmtree(sdist_directory) - shutil.copytree('dist', sdist_directory) - - sdists = [f for f in os.listdir(sdist_directory) - if f.endswith('.tar.gz')] - assert len(sdists) == 1 - return sdists[0] + return _file_with_extension(sdist_directory, '.tar.gz') diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py index 4fe3bb56..fe619e2e 100644 --- a/setuptools/command/__init__.py +++ b/setuptools/command/__init__.py @@ -2,7 +2,8 @@ __all__ = [ 'alias', 'bdist_egg', 'bdist_rpm', 'build_ext', 'build_py', 'develop', 'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts', 'sdist', 'setopt', 'test', 'install_egg_info', 'install_scripts', - 'register', 'bdist_wininst', 'upload_docs', 'upload', 'build_clib', 'dist_info', + 'register', 'bdist_wininst', 'upload_docs', 'upload', 'build_clib', + 'dist_info', ] from distutils.command.bdist import bdist diff --git a/setuptools/command/alias.py b/setuptools/command/alias.py index 4532b1cc..4532b1cc 100755..100644 --- a/setuptools/command/alias.py +++ b/setuptools/command/alias.py diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 51755d52..9f8df917 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -8,6 +8,7 @@ from distutils import log from types import CodeType import sys import os +import re import textwrap import marshal @@ -38,6 +39,7 @@ def strip_module(filename): filename = filename[:-6] return filename + def sorted_walk(dir): """Do os.walk in a reproducible way, independent of indeterministic filesystem readdir order @@ -47,6 +49,7 @@ def sorted_walk(dir): files.sort() yield base, dirs, files + def write_stub(resource, pyfile): _stub_template = textwrap.dedent(""" def __bootstrap__(): @@ -240,11 +243,28 @@ class bdist_egg(Command): log.info("Removing .py files from temporary directory") for base, dirs, files in walk_egg(self.bdist_dir): for name in files: + path = os.path.join(base, name) + if name.endswith('.py'): - path = os.path.join(base, name) log.debug("Deleting %s", path) os.unlink(path) + if base.endswith('__pycache__'): + path_old = path + + pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc' + m = re.match(pattern, name) + path_new = os.path.join( + base, os.pardir, m.group('name') + '.pyc') + log.info( + "Renaming file from [%s] to [%s]" + % (path_old, path_new)) + try: + os.remove(path_new) + except OSError: + pass + os.rename(path_old, path_new) + def zip_safe(self): safe = getattr(self.distribution, 'zip_safe', None) if safe is not None: @@ -391,10 +411,12 @@ def scan_module(egg_dir, base, name, stubs): return True # Extension module pkg = base[len(egg_dir) + 1:].replace(os.sep, '.') module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0] - if sys.version_info < (3, 3): + if six.PY2: skip = 8 # skip magic & date - else: + elif sys.version_info < (3, 7): skip = 12 # skip magic & date & file size + else: + skip = 16 # skip magic & reserved? & date & file size f = open(filename, 'rb') f.read(skip) code = marshal.load(f) diff --git a/setuptools/command/bdist_rpm.py b/setuptools/command/bdist_rpm.py index 70730927..70730927 100755..100644 --- a/setuptools/command/bdist_rpm.py +++ b/setuptools/command/bdist_rpm.py diff --git a/setuptools/command/bdist_wininst.py b/setuptools/command/bdist_wininst.py index 073de97b..073de97b 100755..100644 --- a/setuptools/command/bdist_wininst.py +++ b/setuptools/command/bdist_wininst.py diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 36f53f0d..60a8a32f 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -15,6 +15,9 @@ from setuptools.extern import six try: # Attempt to use Cython for building extensions, if available from Cython.Distutils.build_ext import build_ext as _build_ext + # Additionally, assert that the compiler module will load + # also. Ref #1229. + __import__('Cython.Compiler.Main') except ImportError: _build_ext = _du_build_ext @@ -109,7 +112,7 @@ class build_ext(_build_ext): and get_abi3_suffix() ) if use_abi3: - so_ext = _get_config_var_837('EXT_SUFFIX') + so_ext = get_config_var('EXT_SUFFIX') filename = filename[:-len(so_ext)] filename = filename + get_abi3_suffix() if isinstance(ext, Library): @@ -316,13 +319,3 @@ else: self.create_static_lib( objects, basename, output_dir, debug, target_lang ) - - -def _get_config_var_837(name): - """ - In https://github.com/pypa/setuptools/pull/837, we discovered - Python 3.3.0 exposes the extension suffix under the name 'SO'. - """ - if sys.version_info < (3, 3, 1): - name = 'SO' - return get_config_var(name) diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index 85b23c60..009e4f93 100755..100644 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -7,11 +7,13 @@ import io from setuptools.extern import six -from pkg_resources import Distribution, PathMetadata, normalize_path +import pkg_resources from setuptools.command.easy_install import easy_install from setuptools import namespaces import setuptools +__metaclass__ = type + class develop(namespaces.DevelopInstaller, easy_install): """Set up package for development""" @@ -63,9 +65,9 @@ class develop(namespaces.DevelopInstaller, easy_install): if self.egg_path is None: self.egg_path = os.path.abspath(ei.egg_base) - target = normalize_path(self.egg_base) - egg_path = normalize_path(os.path.join(self.install_dir, - self.egg_path)) + target = pkg_resources.normalize_path(self.egg_base) + egg_path = pkg_resources.normalize_path( + os.path.join(self.install_dir, self.egg_path)) if egg_path != target: raise DistutilsOptionError( "--egg-path must be a relative path from the install" @@ -73,9 +75,9 @@ class develop(namespaces.DevelopInstaller, easy_install): ) # Make a distribution for the package's source - self.dist = Distribution( + self.dist = pkg_resources.Distribution( target, - PathMetadata(target, os.path.abspath(ei.egg_info)), + pkg_resources.PathMetadata(target, os.path.abspath(ei.egg_info)), project_name=ei.egg_name ) @@ -95,11 +97,14 @@ class develop(namespaces.DevelopInstaller, easy_install): path_to_setup = egg_base.replace(os.sep, '/').rstrip('/') if path_to_setup != os.curdir: path_to_setup = '../' * (path_to_setup.count('/') + 1) - resolved = normalize_path(os.path.join(install_dir, egg_path, path_to_setup)) - if resolved != normalize_path(os.curdir): + resolved = pkg_resources.normalize_path( + os.path.join(install_dir, egg_path, path_to_setup) + ) + if resolved != pkg_resources.normalize_path(os.curdir): raise DistutilsOptionError( "Can't get a consistent path to setup script from" - " installation directory", resolved, normalize_path(os.curdir)) + " installation directory", resolved, + pkg_resources.normalize_path(os.curdir)) return path_to_setup def install_for_development(self): @@ -110,7 +115,7 @@ class develop(namespaces.DevelopInstaller, easy_install): self.reinitialize_command('build_py', inplace=0) self.run_command('build_py') bpy_cmd = self.get_finalized_command("build_py") - build_path = normalize_path(bpy_cmd.build_lib) + build_path = pkg_resources.normalize_path(bpy_cmd.build_lib) # Build extensions self.reinitialize_command('egg_info', egg_base=build_path) @@ -124,7 +129,8 @@ class develop(namespaces.DevelopInstaller, easy_install): self.egg_path = build_path self.dist.location = build_path # XXX - self.dist._provider = PathMetadata(build_path, ei_cmd.egg_info) + self.dist._provider = pkg_resources.PathMetadata( + build_path, ei_cmd.egg_info) else: # Without 2to3 inplace works fine: self.run_command('egg_info') @@ -190,12 +196,13 @@ class develop(namespaces.DevelopInstaller, easy_install): return easy_install.install_wrapper_scripts(self, dist) -class VersionlessRequirement(object): +class VersionlessRequirement: """ Adapt a pkg_resources.Distribution to simply return the project name as the 'requirement' so that scripts will work across multiple versions. + >>> from pkg_resources import Distribution >>> dist = Distribution(project_name='foo', version='1.0') >>> str(dist.as_requirement()) 'foo==1.0' diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index c6c6dacb..c45258fa 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -4,7 +4,6 @@ As defined in the wheel specification """ import os -import shutil from distutils.core import Command from distutils import log @@ -27,14 +26,11 @@ class dist_info(Command): def run(self): egg_info = self.get_finalized_command('egg_info') + egg_info.egg_base = self.egg_base + egg_info.finalize_options() egg_info.run() dist_info_dir = egg_info.egg_info[:-len('.egg-info')] + '.dist-info' log.info("creating '{}'".format(os.path.abspath(dist_info_dir))) bdist_wheel = self.get_finalized_command('bdist_wheel') bdist_wheel.egg2dist(egg_info.egg_info, dist_info_dir) - - if self.egg_base: - destination = os.path.join(self.egg_base, dist_info_dir) - log.info("creating '{}'".format(os.path.abspath(destination))) - shutil.move(dist_info_dir, destination) diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 8fba7b41..06c98271 100755..100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -40,12 +40,16 @@ import subprocess import shlex import io + +from sysconfig import get_config_vars, get_path + +from setuptools import SetuptoolsDeprecationWarning + from setuptools.extern import six from setuptools.extern.six.moves import configparser, map from setuptools import Command from setuptools.sandbox import run_setup -from setuptools.py31compat import get_path, get_config_vars from setuptools.py27compat import rmtree_safe from setuptools.command import setopt from setuptools.archive_util import unpack_archive @@ -53,6 +57,7 @@ from setuptools.package_index import ( PackageIndex, parse_requirement_arg, URL_SCHEME, ) from setuptools.command import bdist_egg, egg_info +from setuptools.wheel import Wheel from pkg_resources import ( yield_lines, normalize_path, resource_string, ensure_directory, get_distribution, find_distributions, Environment, Requirement, @@ -61,6 +66,8 @@ from pkg_resources import ( ) import pkg_resources.py31compat +__metaclass__ = type + # Turn on PEP440Warnings warnings.filterwarnings("default", category=pkg_resources.PEP440Warning) @@ -92,7 +99,7 @@ def samefile(p1, p2): if six.PY2: - def _to_ascii(s): + def _to_bytes(s): return s def isascii(s): @@ -103,8 +110,8 @@ if six.PY2: return False else: - def _to_ascii(s): - return s.encode('ascii') + def _to_bytes(s): + return s.encode('utf8') def isascii(s): try: @@ -318,7 +325,7 @@ class easy_install(Command): self.all_site_dirs.append(normalize_path(d)) if not self.editable: self.check_site_dir() - self.index_url = self.index_url or "https://pypi.python.org/simple" + self.index_url = self.index_url or "https://pypi.org/simple/" self.shadow_path = self.all_site_dirs[:] for path_item in self.install_dir, normalize_path(self.script_dir): if path_item not in self.shadow_path: @@ -628,7 +635,7 @@ class easy_install(Command): @contextlib.contextmanager def _tmpdir(self): - tmpdir = tempfile.mkdtemp(prefix=six.u("easy_install-")) + tmpdir = tempfile.mkdtemp(prefix=u"easy_install-") try: # cast to str as workaround for #709 and #710 and #712 yield str(tmpdir) @@ -801,7 +808,7 @@ class easy_install(Command): if is_script: body = self._load_template(dev_path) % locals() script_text = ScriptWriter.get_header(script_text) + body - self.write_script(script_name, _to_ascii(script_text), 'b') + self.write_script(script_name, _to_bytes(script_text), 'b') @staticmethod def _load_template(dev_path): @@ -827,14 +834,16 @@ class easy_install(Command): target = os.path.join(self.script_dir, script_name) self.add_output(target) + if self.dry_run: + return + mask = current_umask() - if not self.dry_run: - ensure_directory(target) - if os.path.exists(target): - os.unlink(target) - with open(target, "w" + mode) as f: - f.write(contents) - chmod(target, 0o777 - mask) + ensure_directory(target) + if os.path.exists(target): + os.unlink(target) + with open(target, "w" + mode) as f: + f.write(contents) + chmod(target, 0o777 - mask) def install_eggs(self, spec, dist_filename, tmpdir): # .egg dirs or files are already built, so just return them @@ -842,6 +851,8 @@ class easy_install(Command): return [self.install_egg(dist_filename, tmpdir)] elif dist_filename.lower().endswith('.exe'): return [self.install_exe(dist_filename, tmpdir)] + elif dist_filename.lower().endswith('.whl'): + return [self.install_wheel(dist_filename, tmpdir)] # Anything else, try to extract and build setup_base = tmpdir @@ -1038,6 +1049,35 @@ class easy_install(Command): f.write('\n'.join(locals()[name]) + '\n') f.close() + def install_wheel(self, wheel_path, tmpdir): + wheel = Wheel(wheel_path) + assert wheel.is_compatible() + destination = os.path.join(self.install_dir, wheel.egg_name()) + destination = os.path.abspath(destination) + if not self.dry_run: + ensure_directory(destination) + if os.path.isdir(destination) and not os.path.islink(destination): + dir_util.remove_tree(destination, dry_run=self.dry_run) + elif os.path.exists(destination): + self.execute( + os.unlink, + (destination,), + "Removing " + destination, + ) + try: + self.execute( + wheel.install_as_egg, + (destination,), + ("Installing %s to %s") % ( + os.path.basename(wheel_path), + os.path.dirname(destination) + ), + ) + finally: + update_dist_caches(destination, fix_zipimporter_caches=False) + self.add_output(destination) + return self.egg_distribution(destination) + __mv_warning = textwrap.dedent(""" Because this distribution was installed --multi-version, before you can import modules from this package in an application, you will need to @@ -1216,7 +1256,6 @@ class easy_install(Command): def byte_compile(self, to_compile): if sys.dont_write_bytecode: - self.warn('byte-compiling is disabled, skipping.') return from distutils.util import byte_compile @@ -1817,7 +1856,7 @@ def _update_zipimporter_cache(normalized_path, cache, updater=None): # get/del patterns instead. For more detailed information see the # following links: # https://github.com/pypa/setuptools/issues/202#issuecomment-202913420 - # https://bitbucket.org/pypy/pypy/src/dd07756a34a41f674c0cacfbc8ae1d4cc9ea2ae4/pypy/module/zipimport/interp_zipimport.py#cl-99 + # http://bit.ly/2h9itJX old_entry = cache[p] del cache[p] new_entry = updater and updater(p, old_entry) @@ -2016,7 +2055,7 @@ class WindowsCommandSpec(CommandSpec): split_args = dict(posix=False) -class ScriptWriter(object): +class ScriptWriter: """ Encapsulates behavior around writing entry point scripts for console and gui apps. @@ -2041,7 +2080,7 @@ class ScriptWriter(object): @classmethod def get_script_args(cls, dist, executable=None, wininst=False): # for backward compatibility - warnings.warn("Use get_args", DeprecationWarning) + warnings.warn("Use get_args", EasyInstallDeprecationWarning) writer = (WindowsScriptWriter if wininst else ScriptWriter).best() header = cls.get_script_header("", executable, wininst) return writer.get_args(dist, header) @@ -2049,12 +2088,10 @@ class ScriptWriter(object): @classmethod def get_script_header(cls, script_text, executable=None, wininst=False): # for backward compatibility - warnings.warn("Use get_header", DeprecationWarning) + warnings.warn("Use get_header", EasyInstallDeprecationWarning, stacklevel=2) if wininst: executable = "python.exe" - cmd = cls.command_spec_class.best().from_param(executable) - cmd.install_options(script_text) - return cmd.as_header() + return cls.get_header(script_text, executable) @classmethod def get_args(cls, dist, header=None): @@ -2086,7 +2123,7 @@ class ScriptWriter(object): @classmethod def get_writer(cls, force_windows): # for backward compatibility - warnings.warn("Use best", DeprecationWarning) + warnings.warn("Use best", EasyInstallDeprecationWarning) return WindowsScriptWriter.best() if force_windows else cls.best() @classmethod @@ -2118,7 +2155,7 @@ class WindowsScriptWriter(ScriptWriter): @classmethod def get_writer(cls): # for backward compatibility - warnings.warn("Use best", DeprecationWarning) + warnings.warn("Use best", EasyInstallDeprecationWarning) return cls.best() @classmethod @@ -2299,3 +2336,7 @@ def _patch_usage(): yield finally: distutils.core.gen_usage = saved + +class EasyInstallDeprecationWarning(SetuptoolsDeprecationWarning): + """Class for warning about deprecations in EasyInstall in SetupTools. Not ignored by default, unlike DeprecationWarning.""" + diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index a183d15d..d9fe3da3 100755..100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -30,8 +30,8 @@ from pkg_resources import ( import setuptools.unicode_utils as unicode_utils from setuptools.glob import glob -from pkg_resources.extern import packaging - +from setuptools.extern import packaging +from setuptools import SetuptoolsDeprecationWarning def translate_pattern(glob): """ @@ -116,7 +116,33 @@ def translate_pattern(glob): return re.compile(pat, flags=re.MULTILINE|re.DOTALL) -class egg_info(Command): +class InfoCommon: + tag_build = None + tag_date = None + + @property + def name(self): + return safe_name(self.distribution.get_name()) + + def tagged_version(self): + version = self.distribution.get_version() + # egg_info may be called more than once for a distribution, + # in which case the version string already contains all tags. + if self.vtags and version.endswith(self.vtags): + return safe_version(version) + return safe_version(version + self.vtags) + + def tags(self): + version = '' + if self.tag_build: + version += self.tag_build + if self.tag_date: + version += time.strftime("-%Y%m%d") + return version + vtags = property(tags) + + +class egg_info(InfoCommon, Command): description = "create a distribution's .egg-info directory" user_options = [ @@ -133,14 +159,11 @@ class egg_info(Command): } def initialize_options(self): - self.egg_name = None - self.egg_version = None self.egg_base = None + self.egg_name = None self.egg_info = None - self.tag_build = None - self.tag_date = 0 + self.egg_version = None self.broken_egg_info = False - self.vtags = None #################################### # allow the 'tag_svn_revision' to be detected and @@ -160,9 +183,7 @@ class egg_info(Command): build tag. Install build keys in a deterministic order to avoid arbitrary reordering on subsequent builds. """ - # python 2.6 compatibility - odict = getattr(collections, 'OrderedDict', dict) - egg_info = odict() + egg_info = collections.OrderedDict() # follow the order these keys would have been added # when PYTHONHASHSEED=0 egg_info['tag_build'] = self.tags() @@ -170,10 +191,12 @@ class egg_info(Command): edit_config(filename, dict(egg_info=egg_info)) def finalize_options(self): - self.egg_name = safe_name(self.distribution.get_name()) - self.vtags = self.tags() + # Note: we need to capture the current value returned + # by `self.tagged_version()`, so we can later update + # `self.distribution.metadata.version` without + # repercussions. + self.egg_name = self.name self.egg_version = self.tagged_version() - parsed_version = parse_version(self.egg_version) try: @@ -256,16 +279,9 @@ class egg_info(Command): if not self.dry_run: os.unlink(filename) - def tagged_version(self): - version = self.distribution.get_version() - # egg_info may be called more than once for a distribution, - # in which case the version string already contains all tags. - if self.vtags and version.endswith(self.vtags): - return safe_version(version) - return safe_version(version + self.vtags) - def run(self): self.mkpath(self.egg_info) + os.utime(self.egg_info, None) installer = self.distribution.fetch_build_egg for ep in iter_entry_points('egg_info.writers'): ep.require(installer=installer) @@ -279,14 +295,6 @@ class egg_info(Command): self.find_sources() - def tags(self): - version = '' - if self.tag_build: - version += self.tag_build - if self.tag_date: - version += time.strftime("-%Y%m%d") - return version - def find_sources(self): """Generate SOURCES.txt manifest file""" manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") @@ -567,6 +575,12 @@ class manifest_maker(sdist): self.filelist.extend(rcfiles) elif os.path.exists(self.manifest): self.read_manifest() + + if os.path.exists("setup.py"): + # setup.py should be included by default, even if it's not + # the script called to create the sdist + self.filelist.append("setup.py") + ei_cmd = self.get_finalized_command('egg_info') self.filelist.graft(ei_cmd.egg_info) @@ -599,10 +613,7 @@ def write_pkg_info(cmd, basename, filename): metadata = cmd.distribution.metadata metadata.version, oldver = cmd.egg_version, metadata.version metadata.name, oldname = cmd.egg_name, metadata.name - metadata.long_description_content_type = getattr( - cmd.distribution, - 'long_description_content_type' - ) + try: # write unescaped data to PKG-INFO, so older pkg_resources # can still parse it @@ -642,7 +653,7 @@ def write_requirements(cmd, basename, filename): def write_setup_requirements(cmd, basename, filename): - data = StringIO() + data = io.StringIO() _write_requirements(data, cmd.distribution.setup_requires) cmd.write_or_delete_file("setup-requirements", filename, data.getvalue()) @@ -691,7 +702,7 @@ def get_pkg_info_revision(): Get a -r### off of PKG-INFO Version in case this is an sdist of a subversion revision. """ - warnings.warn("get_pkg_info_revision is deprecated.", DeprecationWarning) + warnings.warn("get_pkg_info_revision is deprecated.", EggInfoDeprecationWarning) if os.path.exists('PKG-INFO'): with io.open('PKG-INFO') as f: for line in f: @@ -699,3 +710,7 @@ def get_pkg_info_revision(): if match: return int(match.group(1)) return 0 + + +class EggInfoDeprecationWarning(SetuptoolsDeprecationWarning): + """Class for warning about deprecations in eggInfo in setupTools. Not ignored by default, unlike DeprecationWarning.""" diff --git a/setuptools/command/install_egg_info.py b/setuptools/command/install_egg_info.py index edc4718b..edc4718b 100755..100644 --- a/setuptools/command/install_egg_info.py +++ b/setuptools/command/install_egg_info.py diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py index 16234273..16234273 100755..100644 --- a/setuptools/command/install_scripts.py +++ b/setuptools/command/install_scripts.py diff --git a/setuptools/command/register.py b/setuptools/command/register.py index 8d6336a1..98bc0156 100755..100644 --- a/setuptools/command/register.py +++ b/setuptools/command/register.py @@ -1,3 +1,4 @@ +from distutils import log import distutils.command.register as orig @@ -5,6 +6,13 @@ class register(orig.register): __doc__ = orig.register.__doc__ def run(self): - # Make sure that we are using valid current name/version info - self.run_command('egg_info') - orig.register.run(self) + try: + # Make sure that we are using valid current name/version info + self.run_command('egg_info') + orig.register.run(self) + finally: + self.announce( + "WARNING: Registering is deprecated, use twine to " + "upload instead (https://pypi.org/p/twine/)", + log.WARN + ) diff --git a/setuptools/command/rotate.py b/setuptools/command/rotate.py index b89353f5..b89353f5 100755..100644 --- a/setuptools/command/rotate.py +++ b/setuptools/command/rotate.py diff --git a/setuptools/command/saveopts.py b/setuptools/command/saveopts.py index 611cec55..611cec55 100755..100644 --- a/setuptools/command/saveopts.py +++ b/setuptools/command/saveopts.py diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 508148e0..bcfae4d8 100755..100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -51,13 +51,6 @@ class sdist(sdist_add_defaults, orig.sdist): for cmd_name in self.get_sub_commands(): self.run_command(cmd_name) - # Call check_metadata only if no 'check' command - # (distutils <= 2.6) - import distutils.command - - if 'check' not in distutils.command.__all__: - self.check_metadata() - self.make_distribution() dist_files = getattr(self.distribution, 'dist_files', []) diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index 7e57cc02..7e57cc02 100755..100644 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py diff --git a/setuptools/command/test.py b/setuptools/command/test.py index 638d0c56..dde0118c 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -3,6 +3,7 @@ import operator import sys import contextlib import itertools +import unittest from distutils.errors import DistutilsError, DistutilsOptionError from distutils import log from unittest import TestLoader @@ -14,10 +15,16 @@ from pkg_resources import (resource_listdir, resource_exists, normalize_path, working_set, _namespace_packages, evaluate_marker, add_activation_listener, require, EntryPoint) from setuptools import Command -from setuptools.py31compat import unittest_main + +__metaclass__ = type class ScanningLoader(TestLoader): + + def __init__(self): + TestLoader.__init__(self) + self._visited = set() + def loadTestsFromModule(self, module, pattern=None): """Return a suite of all tests cases contained in the given module @@ -25,6 +32,10 @@ class ScanningLoader(TestLoader): If the module has an ``additional_tests`` function, call it and add the return value to the tests. """ + if module in self._visited: + return None + self._visited.add(module) + tests = [] tests.append(TestLoader.loadTestsFromModule(self, module)) @@ -49,7 +60,7 @@ class ScanningLoader(TestLoader): # adapted from jaraco.classes.properties:NonDataProperty -class NonDataProperty(object): +class NonDataProperty: def __init__(self, fget): self.fget = fget @@ -101,6 +112,8 @@ class test(Command): return list(self._test_args()) def _test_args(self): + if not self.test_suite and sys.version_info >= (2, 7): + yield 'discover' if self.verbose: yield '--verbose' if self.test_suite: @@ -230,12 +243,11 @@ class test(Command): del_modules.append(name) list(map(sys.modules.__delitem__, del_modules)) - exit_kwarg = {} if sys.version_info < (2, 7) else {"exit": False} - test = unittest_main( + test = unittest.main( None, None, self._argv, testLoader=self._resolve_as_ep(self.test_loader), testRunner=self._resolve_as_ep(self.test_runner), - **exit_kwarg + exit=False, ) if not test.result.wasSuccessful(): msg = 'Test failed: %s' % test.result diff --git a/setuptools/command/upload.py b/setuptools/command/upload.py index a44173a9..6db8888b 100644 --- a/setuptools/command/upload.py +++ b/setuptools/command/upload.py @@ -1,5 +1,19 @@ +import io +import os +import hashlib import getpass + +from base64 import standard_b64encode + +from distutils import log from distutils.command import upload as orig +from distutils.spawn import spawn + +from distutils.errors import DistutilsError + +from setuptools.extern.six.moves.urllib.request import urlopen, Request +from setuptools.extern.six.moves.urllib.error import HTTPError +from setuptools.extern.six.moves.urllib.parse import urlparse class upload(orig.upload): @@ -7,6 +21,15 @@ class upload(orig.upload): Override default upload behavior to obtain password in a variety of different ways. """ + def run(self): + try: + orig.upload.run(self) + finally: + self.announce( + "WARNING: Uploading via this command is deprecated, use twine " + "to upload instead (https://pypi.org/p/twine/)", + log.WARN + ) def finalize_options(self): orig.upload.finalize_options(self) @@ -22,6 +45,137 @@ class upload(orig.upload): self._prompt_for_password() ) + def upload_file(self, command, pyversion, filename): + # Makes sure the repository URL is compliant + schema, netloc, url, params, query, fragments = \ + urlparse(self.repository) + if params or query or fragments: + raise AssertionError("Incompatible url %s" % self.repository) + + if schema not in ('http', 'https'): + raise AssertionError("unsupported schema " + schema) + + # Sign if requested + if self.sign: + gpg_args = ["gpg", "--detach-sign", "-a", filename] + if self.identity: + gpg_args[2:2] = ["--local-user", self.identity] + spawn(gpg_args, + dry_run=self.dry_run) + + # Fill in the data - send all the meta-data in case we need to + # register a new release + with open(filename, 'rb') as f: + content = f.read() + + meta = self.distribution.metadata + + data = { + # action + ':action': 'file_upload', + 'protocol_version': '1', + + # identify release + 'name': meta.get_name(), + 'version': meta.get_version(), + + # file content + 'content': (os.path.basename(filename), content), + 'filetype': command, + 'pyversion': pyversion, + 'md5_digest': hashlib.md5(content).hexdigest(), + + # additional meta-data + 'metadata_version': str(meta.get_metadata_version()), + 'summary': meta.get_description(), + 'home_page': meta.get_url(), + 'author': meta.get_contact(), + 'author_email': meta.get_contact_email(), + 'license': meta.get_licence(), + 'description': meta.get_long_description(), + 'keywords': meta.get_keywords(), + 'platform': meta.get_platforms(), + 'classifiers': meta.get_classifiers(), + 'download_url': meta.get_download_url(), + # PEP 314 + 'provides': meta.get_provides(), + 'requires': meta.get_requires(), + 'obsoletes': meta.get_obsoletes(), + } + + data['comment'] = '' + + if self.sign: + data['gpg_signature'] = (os.path.basename(filename) + ".asc", + open(filename+".asc", "rb").read()) + + # set up the authentication + user_pass = (self.username + ":" + self.password).encode('ascii') + # The exact encoding of the authentication string is debated. + # Anyway PyPI only accepts ascii for both username or password. + auth = "Basic " + standard_b64encode(user_pass).decode('ascii') + + # Build up the MIME payload for the POST data + boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' + sep_boundary = b'\r\n--' + boundary.encode('ascii') + end_boundary = sep_boundary + b'--\r\n' + body = io.BytesIO() + for key, value in data.items(): + title = '\r\nContent-Disposition: form-data; name="%s"' % key + # handle multiple entries for the same name + if not isinstance(value, list): + value = [value] + for value in value: + if type(value) is tuple: + title += '; filename="%s"' % value[0] + value = value[1] + else: + value = str(value).encode('utf-8') + body.write(sep_boundary) + body.write(title.encode('utf-8')) + body.write(b"\r\n\r\n") + body.write(value) + body.write(end_boundary) + body = body.getvalue() + + msg = "Submitting %s to %s" % (filename, self.repository) + self.announce(msg, log.INFO) + + # build the Request + headers = { + 'Content-type': 'multipart/form-data; boundary=%s' % boundary, + 'Content-length': str(len(body)), + 'Authorization': auth, + } + + request = Request(self.repository, data=body, + headers=headers) + # send the data + try: + result = urlopen(request) + status = result.getcode() + reason = result.msg + except HTTPError as e: + status = e.code + reason = e.msg + except OSError as e: + self.announce(str(e), log.ERROR) + raise + + if status == 200: + self.announce('Server response (%s): %s' % (status, reason), + log.INFO) + if self.show_response: + text = getattr(self, '_read_pypi_response', + lambda x: None)(result) + if text is not None: + msg = '\n'.join(('-' * 75, text, '-' * 75)) + self.announce(msg, log.INFO) + else: + msg = 'Upload failed (%s): %s' % (status, reason) + self.announce(msg, log.ERROR) + raise DistutilsError(msg) + def _load_password_from_keyring(self): """ Attempt to load password from keyring. Suppress Exceptions. diff --git a/setuptools/config.py b/setuptools/config.py index 9a62e2ec..b6626043 100644 --- a/setuptools/config.py +++ b/setuptools/config.py @@ -2,12 +2,20 @@ from __future__ import absolute_import, unicode_literals import io import os import sys + +import warnings +import functools from collections import defaultdict from functools import partial +from functools import wraps +from importlib import import_module from distutils.errors import DistutilsOptionError, DistutilsFileError -from setuptools.py26compat import import_module -from setuptools.extern.six import string_types +from setuptools.extern.packaging.version import LegacyVersion, parse +from setuptools.extern.six import string_types, PY3 + + +__metaclass__ = type def read_configuration( @@ -57,6 +65,18 @@ def read_configuration( return configuration_to_dict(handlers) +def _get_option(target_obj, key): + """ + Given a target object and option key, get that option from + the target object, either through a get_{key} method or + from an attribute directly. + """ + getter_name = 'get_{key}'.format(**locals()) + by_attribute = functools.partial(getattr, target_obj, key) + getter = getattr(target_obj, getter_name, by_attribute) + return getter() + + def configuration_to_dict(handlers): """Returns configuration data gathered by given handlers as a dict. @@ -68,20 +88,9 @@ def configuration_to_dict(handlers): config_dict = defaultdict(dict) for handler in handlers: - - obj_alias = handler.section_prefix - target_obj = handler.target_obj - for option in handler.set_options: - getter = getattr(target_obj, 'get_%s' % option, None) - - if getter is None: - value = getattr(target_obj, option) - - else: - value = getter() - - config_dict[obj_alias][option] = value + value = _get_option(handler.target_obj, option) + config_dict[handler.section_prefix][option] = value return config_dict @@ -101,18 +110,19 @@ def parse_configuration( If False exceptions are propagated as expected. :rtype: list """ - meta = ConfigMetadataHandler( - distribution.metadata, command_options, ignore_option_errors) - meta.parse() - options = ConfigOptionsHandler( distribution, command_options, ignore_option_errors) options.parse() - return [meta, options] + meta = ConfigMetadataHandler( + distribution.metadata, command_options, ignore_option_errors, + distribution.package_dir) + meta.parse() + + return meta, options -class ConfigHandler(object): +class ConfigHandler: """Handles metadata supplied in configuration files.""" section_prefix = None @@ -237,6 +247,26 @@ class ConfigHandler(object): return value in ('1', 'true', 'yes') @classmethod + def _exclude_files_parser(cls, key): + """Returns a parser function to make sure field inputs + are not files. + + Parses a value after getting the key so error messages are + more informative. + + :param key: + :rtype: callable + """ + def parser(value): + exclude_directive = 'file:' + if value.startswith(exclude_directive): + raise ValueError( + 'Only strings are accepted for the {0} field, ' + 'files are not accepted'.format(key)) + return value + return parser + + @classmethod def _parse_file(cls, value): """Represents value as a string, allowing including text from nearest files using `file:` directive. @@ -245,7 +275,6 @@ class ConfigHandler(object): directory with setup.py. Examples: - file: LICENSE file: README.rst, CHANGELOG.md, src/file.txt :param str value: @@ -280,7 +309,7 @@ class ConfigHandler(object): return f.read() @classmethod - def _parse_attr(cls, value): + def _parse_attr(cls, value, package_dir=None): """Represents value as a module attribute. Examples: @@ -300,7 +329,21 @@ class ConfigHandler(object): module_name = '.'.join(attrs_path) module_name = module_name or '__init__' - sys.path.insert(0, os.getcwd()) + parent_path = os.getcwd() + if package_dir: + if attrs_path[0] in package_dir: + # A custom path was specified for the module we want to import + custom_path = package_dir[attrs_path[0]] + parts = custom_path.rsplit('/', 1) + if len(parts) > 1: + parent_path = os.path.join(os.getcwd(), parts[0]) + module_name = parts[1] + else: + module_name = custom_path + elif '' in package_dir: + # A custom parent directory was specified for all root modules + parent_path = os.path.join(os.getcwd(), package_dir['']) + sys.path.insert(0, parent_path) try: module = import_module(module_name) value = getattr(module, attr_name) @@ -370,7 +413,7 @@ class ConfigHandler(object): section_parser_method = getattr( self, - # Dots in section names are tranlsated into dunderscores. + # Dots in section names are translated into dunderscores. ('parse_section%s' % method_postfix).replace('.', '__'), None) @@ -381,6 +424,20 @@ class ConfigHandler(object): section_parser_method(section_options) + def _deprecated_config_handler(self, func, msg, warning_class): + """ this function will wrap around parameters that are deprecated + + :param msg: deprecation message + :param warning_class: class of warning exception to be raised + :param func: function to be wrapped around + """ + @wraps(func) + def config_handler(*args, **kwargs): + warnings.warn(msg, warning_class) + return func(*args, **kwargs) + + return config_handler + class ConfigMetadataHandler(ConfigHandler): @@ -399,23 +456,36 @@ class ConfigMetadataHandler(ConfigHandler): """ + def __init__(self, target_obj, options, ignore_option_errors=False, + package_dir=None): + super(ConfigMetadataHandler, self).__init__(target_obj, options, + ignore_option_errors) + self.package_dir = package_dir + @property def parsers(self): """Metadata item name to parser function mapping.""" parse_list = self._parse_list parse_file = self._parse_file + parse_dict = self._parse_dict + exclude_files_parser = self._exclude_files_parser return { 'platforms': parse_list, 'keywords': parse_list, 'provides': parse_list, - 'requires': parse_list, + 'requires': self._deprecated_config_handler( + parse_list, + "The requires parameter is deprecated, please use " + "install_requires for runtime dependencies.", + DeprecationWarning), 'obsoletes': parse_list, 'classifiers': self._get_parser_compound(parse_file, parse_list), - 'license': parse_file, + 'license': exclude_files_parser('license'), 'description': parse_file, 'long_description': parse_file, 'version': self._parse_version, + 'project_urls': parse_dict, } def _parse_version(self, value): @@ -425,7 +495,22 @@ class ConfigMetadataHandler(ConfigHandler): :rtype: str """ - version = self._parse_attr(value) + version = self._parse_file(value) + + if version != value: + version = version.strip() + # Be strict about versions loaded from file because it's easy to + # accidentally include newlines and other unintended content + if isinstance(parse(version), LegacyVersion): + tmpl = ( + 'Version loaded from {value} does not ' + 'comply with PEP 440: {version}' + ) + raise DistutilsOptionError(tmpl.format(**locals())) + + return version + + version = self._parse_attr(value, self.package_dir) if callable(version): version = version() @@ -477,16 +562,25 @@ class ConfigOptionsHandler(ConfigHandler): :param value: :rtype: list """ - find_directive = 'find:' + find_directives = ['find:', 'find_namespace:'] + trimmed_value = value.strip() - if not value.startswith(find_directive): + if trimmed_value not in find_directives: return self._parse_list(value) + findns = trimmed_value == find_directives[1] + if findns and not PY3: + raise DistutilsOptionError( + 'find_namespace: directive is unsupported on Python < 3.3') + # Read function arguments from a dedicated section. find_kwargs = self.parse_section_packages__find( self.sections.get('packages.find', {})) - from setuptools import find_packages + if findns: + from setuptools import find_namespace_packages as find_packages + else: + from setuptools import find_packages return find_packages(**find_kwargs) @@ -552,3 +646,11 @@ class ConfigOptionsHandler(ConfigHandler): parse_list = partial(self._parse_list, separator=';') self['extras_require'] = self._parse_section_to_dict( section_options, parse_list) + + def parse_section_data_files(self, section_options): + """Parses `data_files` configuration file section. + + :param dict section_options: + """ + parsed = self._parse_section_to_dict(section_options, self._parse_list) + self['data_files'] = [(k, v) for k, v in parsed.items()] diff --git a/setuptools/dist.py b/setuptools/dist.py index b10bd6f7..a2600711 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- __all__ = ['Distribution'] import re @@ -9,15 +10,22 @@ import distutils.core import distutils.cmd import distutils.dist import itertools + + from collections import defaultdict +from email import message_from_file + from distutils.errors import ( DistutilsOptionError, DistutilsPlatformError, DistutilsSetupError, ) from distutils.util import rfc822_escape +from distutils.version import StrictVersion from setuptools.extern import six +from setuptools.extern import packaging from setuptools.extern.six.moves import map, filter, filterfalse -from pkg_resources.extern import packaging + +from . import SetuptoolsDeprecationWarning from setuptools.depends import Require from setuptools import windows_support @@ -26,53 +34,145 @@ from setuptools.config import parse_configuration import pkg_resources from .py36compat import Distribution_parse_config_files -__import__('pkg_resources.extern.packaging.specifiers') -__import__('pkg_resources.extern.packaging.version') +__import__('setuptools.extern.packaging.specifiers') +__import__('setuptools.extern.packaging.version') def _get_unpatched(cls): - warnings.warn("Do not call this function", DeprecationWarning) + warnings.warn("Do not call this function", DistDeprecationWarning) return get_unpatched(cls) +def get_metadata_version(self): + mv = getattr(self, 'metadata_version', None) + + if mv is None: + if self.long_description_content_type or self.provides_extras: + mv = StrictVersion('2.1') + elif (self.maintainer is not None or + self.maintainer_email is not None or + getattr(self, 'python_requires', None) is not None): + mv = StrictVersion('1.2') + elif (self.provides or self.requires or self.obsoletes or + self.classifiers or self.download_url): + mv = StrictVersion('1.1') + else: + mv = StrictVersion('1.0') + + self.metadata_version = mv + + return mv + + +def read_pkg_file(self, file): + """Reads the metadata values from a file object.""" + msg = message_from_file(file) + + def _read_field(name): + value = msg[name] + if value == 'UNKNOWN': + return None + return value + + def _read_list(name): + values = msg.get_all(name, None) + if values == []: + return None + return values + + self.metadata_version = StrictVersion(msg['metadata-version']) + self.name = _read_field('name') + self.version = _read_field('version') + self.description = _read_field('summary') + # we are filling author only. + self.author = _read_field('author') + self.maintainer = None + self.author_email = _read_field('author-email') + self.maintainer_email = None + self.url = _read_field('home-page') + self.license = _read_field('license') + + if 'download-url' in msg: + self.download_url = _read_field('download-url') + else: + self.download_url = None + + self.long_description = _read_field('description') + self.description = _read_field('summary') + + if 'keywords' in msg: + self.keywords = _read_field('keywords').split(',') + + self.platforms = _read_list('platform') + self.classifiers = _read_list('classifier') + + # PEP 314 - these fields only exist in 1.1 + if self.metadata_version == StrictVersion('1.1'): + self.requires = _read_list('requires') + self.provides = _read_list('provides') + self.obsoletes = _read_list('obsoletes') + else: + self.requires = None + self.provides = None + self.obsoletes = None + + # Based on Python 3.5 version def write_pkg_file(self, file): """Write the PKG-INFO format data to a file object. """ - version = '1.0' - if (self.provides or self.requires or self.obsoletes or - self.classifiers or self.download_url): - version = '1.1' - # Setuptools specific for PEP 345 - if hasattr(self, 'python_requires'): - version = '1.2' - - file.write('Metadata-Version: %s\n' % version) - file.write('Name: %s\n' % self.get_name()) - file.write('Version: %s\n' % self.get_version()) - file.write('Summary: %s\n' % self.get_description()) - file.write('Home-page: %s\n' % self.get_url()) - file.write('Author: %s\n' % self.get_contact()) - file.write('Author-email: %s\n' % self.get_contact_email()) - file.write('License: %s\n' % self.get_license()) - if self.download_url: - file.write('Download-URL: %s\n' % self.download_url) + version = self.get_metadata_version() + + if six.PY2: + def write_field(key, value): + file.write("%s: %s\n" % (key, self._encode_field(value))) + else: + def write_field(key, value): + file.write("%s: %s\n" % (key, value)) + + + write_field('Metadata-Version', str(version)) + write_field('Name', self.get_name()) + write_field('Version', self.get_version()) + write_field('Summary', self.get_description()) + write_field('Home-page', self.get_url()) + + if version < StrictVersion('1.2'): + write_field('Author', self.get_contact()) + write_field('Author-email', self.get_contact_email()) + else: + optional_fields = ( + ('Author', 'author'), + ('Author-email', 'author_email'), + ('Maintainer', 'maintainer'), + ('Maintainer-email', 'maintainer_email'), + ) + + for field, attr in optional_fields: + attr_val = getattr(self, attr) + + if attr_val is not None: + write_field(field, attr_val) - long_desc_content_type = getattr( - self, - 'long_description_content_type', - None - ) or 'UNKNOWN' - file.write('Description-Content-Type: %s\n' % long_desc_content_type) + write_field('License', self.get_license()) + if self.download_url: + write_field('Download-URL', self.download_url) + for project_url in self.project_urls.items(): + write_field('Project-URL', '%s, %s' % project_url) long_desc = rfc822_escape(self.get_long_description()) - file.write('Description: %s\n' % long_desc) + write_field('Description', long_desc) keywords = ','.join(self.get_keywords()) if keywords: - file.write('Keywords: %s\n' % keywords) + write_field('Keywords', keywords) + + if version >= StrictVersion('1.2'): + for platform in self.get_platforms(): + write_field('Platform', platform) + else: + self._write_list(file, 'Platform', self.get_platforms()) - self._write_list(file, 'Platform', self.get_platforms()) self._write_list(file, 'Classifier', self.get_classifiers()) # PEP 314 @@ -82,16 +182,17 @@ def write_pkg_file(self, file): # Setuptools specific for PEP 345 if hasattr(self, 'python_requires'): - file.write('Requires-Python: %s\n' % self.python_requires) + write_field('Requires-Python', self.python_requires) - -# from Python 3.4 -def write_pkg_info(self, base_dir): - """Write the PKG-INFO file into the release tree. - """ - with open(os.path.join(base_dir, 'PKG-INFO'), 'w', - encoding='UTF-8') as pkg_info: - self.write_pkg_file(pkg_info) + # PEP 566 + if self.long_description_content_type: + write_field( + 'Description-Content-Type', + self.long_description_content_type + ) + if self.provides_extras: + for extra in self.provides_extras: + write_field('Provides-Extra', extra) sequence = tuple, list @@ -166,6 +267,8 @@ def check_requirements(dist, attr, value): """Verify that install_requires is a valid requirements list""" try: list(pkg_resources.parse_requirements(value)) + if isinstance(value, (dict, set)): + raise TypeError("Unordered types are not allowed") except (TypeError, ValueError) as error: tmpl = ( "{attr!r} must be a string or list of strings " @@ -297,6 +400,12 @@ class Distribution(Distribution_parse_config_files, _Distribution): distribution for the included and excluded features. """ + _DISTUTILS_UNSUPPORTED_METADATA = { + 'long_description_content_type': None, + 'project_urls': dict, + 'provides_extras': set, + } + _patched_dist = None def patch_missing_pkg_info(self, attrs): @@ -316,26 +425,36 @@ class Distribution(Distribution_parse_config_files, _Distribution): have_package_data = hasattr(self, "package_data") if not have_package_data: self.package_data = {} - _attrs_dict = attrs or {} - if 'features' in _attrs_dict or 'require_features' in _attrs_dict: + attrs = attrs or {} + if 'features' in attrs or 'require_features' in attrs: Feature.warn_deprecated() self.require_features = [] self.features = {} self.dist_files = [] - self.src_root = attrs and attrs.pop("src_root", None) + # Filter-out setuptools' specific options. + self.src_root = attrs.pop("src_root", None) self.patch_missing_pkg_info(attrs) - self.long_description_content_type = _attrs_dict.get( - 'long_description_content_type' - ) - # Make sure we have any eggs needed to interpret 'attrs' - if attrs is not None: - self.dependency_links = attrs.pop('dependency_links', []) - assert_string_list(self, 'dependency_links', self.dependency_links) - if attrs and 'setup_requires' in attrs: - self.fetch_build_eggs(attrs['setup_requires']) + self.dependency_links = attrs.pop('dependency_links', []) + self.setup_requires = attrs.pop('setup_requires', []) for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): vars(self).setdefault(ep.name, None) - _Distribution.__init__(self, attrs) + _Distribution.__init__(self, { + k: v for k, v in attrs.items() + if k not in self._DISTUTILS_UNSUPPORTED_METADATA + }) + + # Fill-in missing metadata fields not supported by distutils. + # Note some fields may have been set by other tools (e.g. pbr) + # above; they are taken preferrentially to setup() arguments + for option, default in self._DISTUTILS_UNSUPPORTED_METADATA.items(): + for source in self.metadata.__dict__, attrs: + if option in source: + value = source[option] + break + else: + value = default() if default else None + setattr(self.metadata, option, value) + if isinstance(self.metadata.version, numbers.Number): # Some people apparently take "version number" too literally :) self.metadata.version = str(self.metadata.version) @@ -368,6 +487,16 @@ class Distribution(Distribution_parse_config_files, _Distribution): """ if getattr(self, 'python_requires', None): self.metadata.python_requires = self.python_requires + + if getattr(self, 'extras_require', None): + for extra in self.extras_require.keys(): + # Since this gets called multiple times at points where the + # keys have become 'converted' extras, ensure that we are only + # truly adding extras we haven't seen before here. + extra = extra.split(':')[0] + if extra: + self.metadata.provides_extras.add(extra) + self._convert_extras_requirements() self._move_install_requirements_markers() @@ -427,14 +556,15 @@ class Distribution(Distribution_parse_config_files, _Distribution): req.marker = None return req - def parse_config_files(self, filenames=None): + def parse_config_files(self, filenames=None, ignore_option_errors=False): """Parses configuration files from various levels and loads configuration. """ Distribution_parse_config_files.parse_config_files(self, filenames=filenames) - parse_configuration(self, self.command_options) + parse_configuration(self, self.command_options, + ignore_option_errors=ignore_option_errors) self._finalize_requires() def parse_command_line(self): @@ -497,19 +627,20 @@ class Distribution(Distribution_parse_config_files, _Distribution): """Fetch an egg needed for building""" from setuptools.command.easy_install import easy_install dist = self.__class__({'script_args': ['easy_install']}) - dist.parse_config_files() opts = dist.get_option_dict('easy_install') - keep = ( - 'find_links', 'site_dirs', 'index_url', 'optimize', - 'site_dirs', 'allow_hosts' - ) - for key in list(opts): - if key not in keep: - del opts[key] # don't use any other settings + opts.clear() + opts.update( + (k, v) + for k, v in self.get_option_dict('easy_install').items() + if k in ( + # don't use any other settings + 'find_links', 'site_dirs', 'index_url', + 'optimize', 'site_dirs', 'allow_hosts', + )) if self.dependency_links: links = self.dependency_links[:] if 'find_links' in opts: - links = opts['find_links'][1].split() + links + links = opts['find_links'][1] + links opts['find_links'] = ('setup', links) install_dir = self.get_egg_cache_dir() cmd = easy_install( @@ -921,7 +1052,7 @@ class Feature: "Features are deprecated and will be removed in a future " "version. See https://github.com/pypa/setuptools/issues/65." ) - warnings.warn(msg, DeprecationWarning, stacklevel=3) + warnings.warn(msg, DistDeprecationWarning, stacklevel=3) def __init__( self, description, standard=False, available=True, @@ -1010,3 +1141,7 @@ class Feature: " doesn't contain any packages or modules under %s" % (self.description, item, item) ) + + +class DistDeprecationWarning(SetuptoolsDeprecationWarning): + """Class for warning about deprecations in dist in setuptools. Not ignored by default, unlike DeprecationWarning.""" diff --git a/setuptools/extern/__init__.py b/setuptools/extern/__init__.py index 2cd08b7e..cb2fa329 100644 --- a/setuptools/extern/__init__.py +++ b/setuptools/extern/__init__.py @@ -1,4 +1,73 @@ -from pkg_resources.extern import VendorImporter +import sys -names = 'six', -VendorImporter(__name__, names, 'pkg_resources._vendor').install() + +class VendorImporter: + """ + A PEP 302 meta path importer for finding optionally-vendored + or otherwise naturally-installed packages from root_name. + """ + + def __init__(self, root_name, vendored_names=(), vendor_pkg=None): + self.root_name = root_name + self.vendored_names = set(vendored_names) + self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor') + + @property + def search_path(self): + """ + Search first the vendor package then as a natural package. + """ + yield self.vendor_pkg + '.' + yield '' + + def find_module(self, fullname, path=None): + """ + Return self when fullname starts with root_name and the + target module is one vendored through this importer. + """ + root, base, target = fullname.partition(self.root_name + '.') + if root: + return + if not any(map(target.startswith, self.vendored_names)): + return + return self + + def load_module(self, fullname): + """ + Iterate over the search path to locate and load fullname. + """ + root, base, target = fullname.partition(self.root_name + '.') + for prefix in self.search_path: + try: + extant = prefix + target + __import__(extant) + mod = sys.modules[extant] + sys.modules[fullname] = mod + # mysterious hack: + # Remove the reference to the extant package/module + # on later Python versions to cause relative imports + # in the vendor package to resolve the same modules + # as those going through this importer. + if sys.version_info >= (3, ): + del sys.modules[extant] + return mod + except ImportError: + pass + else: + raise ImportError( + "The '{target}' package is required; " + "normally this is bundled with this package so if you get " + "this warning, consult the packager of your " + "distribution.".format(**locals()) + ) + + def install(self): + """ + Install this importer into sys.meta_path if not already present. + """ + if self not in sys.meta_path: + sys.meta_path.append(self) + + +names = 'six', 'packaging', 'pyparsing', +VendorImporter(__name__, names, 'setuptools._vendor').install() diff --git a/setuptools/glibc.py b/setuptools/glibc.py new file mode 100644 index 00000000..a134591c --- /dev/null +++ b/setuptools/glibc.py @@ -0,0 +1,86 @@ +# This file originally from pip: +# https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/utils/glibc.py +from __future__ import absolute_import + +import ctypes +import re +import warnings + + +def glibc_version_string(): + "Returns glibc version string, or None if not using glibc." + + # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen + # manpage says, "If filename is NULL, then the returned handle is for the + # main program". This way we can let the linker do the work to figure out + # which libc our process is actually using. + process_namespace = ctypes.CDLL(None) + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return None + + # Call gnu_get_libc_version, which returns a string like "2.5" + gnu_get_libc_version.restype = ctypes.c_char_p + version_str = gnu_get_libc_version() + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") + + return version_str + + +# Separated out from have_compatible_glibc for easier unit testing +def check_glibc_version(version_str, required_major, minimum_minor): + # Parse string and check against requested version. + # + # We use a regexp instead of str.split because we want to discard any + # random junk that might come after the minor version -- this might happen + # in patched/forked versions of glibc (e.g. Linaro's version of glibc + # uses version strings like "2.20-2014.11"). See gh-3588. + m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str) + if not m: + warnings.warn("Expected glibc version with 2 components major.minor," + " got: %s" % version_str, RuntimeWarning) + return False + return (int(m.group("major")) == required_major and + int(m.group("minor")) >= minimum_minor) + + +def have_compatible_glibc(required_major, minimum_minor): + version_str = glibc_version_string() + if version_str is None: + return False + return check_glibc_version(version_str, required_major, minimum_minor) + + +# platform.libc_ver regularly returns completely nonsensical glibc +# versions. E.g. on my computer, platform says: +# +# ~$ python2.7 -c 'import platform; print(platform.libc_ver())' +# ('glibc', '2.7') +# ~$ python3.5 -c 'import platform; print(platform.libc_ver())' +# ('glibc', '2.9') +# +# But the truth is: +# +# ~$ ldd --version +# ldd (Debian GLIBC 2.22-11) 2.22 +# +# This is unfortunate, because it means that the linehaul data on libc +# versions that was generated by pip 8.1.2 and earlier is useless and +# misleading. Solution: instead of using platform, use our code that actually +# works. +def libc_ver(): + """Try to determine the glibc version + + Returns a tuple of strings (lib, version) which default to empty strings + in case the lookup fails. + """ + glibc_version = glibc_version_string() + if glibc_version is None: + return ("", "") + else: + return ("glibc", glibc_version) diff --git a/setuptools/glob.py b/setuptools/glob.py index 6c781de3..9d7cbc5d 100644 --- a/setuptools/glob.py +++ b/setuptools/glob.py @@ -3,14 +3,12 @@ Filename globbing utility. Mostly a copy of `glob` from Python 3.5. Changes include: * `yield from` and PEP3102 `*` removed. - * `bytes` changed to `six.binary_type`. * Hidden files are not ignored. """ import os import re import fnmatch -from setuptools.extern.six import binary_type __all__ = ["glob", "iglob", "escape"] @@ -92,7 +90,7 @@ def _iglob(pathname, recursive): def glob1(dirname, pattern): if not dirname: - if isinstance(pattern, binary_type): + if isinstance(pattern, bytes): dirname = os.curdir.encode('ASCII') else: dirname = os.curdir @@ -129,8 +127,8 @@ def glob2(dirname, pattern): # Recursively yields relative pathnames inside a literal directory. def _rlistdir(dirname): if not dirname: - if isinstance(dirname, binary_type): - dirname = binary_type(os.curdir, 'ASCII') + if isinstance(dirname, bytes): + dirname = os.curdir.encode('ASCII') else: dirname = os.curdir try: @@ -149,7 +147,7 @@ magic_check_bytes = re.compile(b'([*?[])') def has_magic(s): - if isinstance(s, binary_type): + if isinstance(s, bytes): match = magic_check_bytes.search(s) else: match = magic_check.search(s) @@ -157,7 +155,7 @@ def has_magic(s): def _isrecursive(pattern): - if isinstance(pattern, binary_type): + if isinstance(pattern, bytes): return pattern == b'**' else: return pattern == '**' @@ -169,7 +167,7 @@ def escape(pathname): # Escaping is done by wrapping any of "*?[" between square brackets. # Metacharacters do not work in the drive part and shouldn't be escaped. drive, pathname = os.path.splitdrive(pathname) - if isinstance(pathname, binary_type): + if isinstance(pathname, bytes): pathname = magic_check_bytes.sub(br'[\1]', pathname) else: pathname = magic_check.sub(r'[\1]', pathname) diff --git a/setuptools/monkey.py b/setuptools/monkey.py index 6d3711ec..3c77f8cf 100644 --- a/setuptools/monkey.py +++ b/setuptools/monkey.py @@ -7,9 +7,9 @@ import distutils.filelist import platform import types import functools +from importlib import import_module import inspect -from .py26compat import import_module from setuptools.extern import six import setuptools @@ -75,8 +75,6 @@ def patch_all(): needs_warehouse = ( sys.version_info < (2, 7, 13) or - (3, 0) < sys.version_info < (3, 3, 7) - or (3, 4) < sys.version_info < (3, 4, 6) or (3, 5) < sys.version_info <= (3, 5, 3) @@ -86,8 +84,7 @@ def patch_all(): warehouse = 'https://upload.pypi.org/legacy/' distutils.config.PyPIRCCommand.DEFAULT_REPOSITORY = warehouse - _patch_distribution_metadata_write_pkg_file() - _patch_distribution_metadata_write_pkg_info() + _patch_distribution_metadata() # Install Distribution throughout the distutils for module in distutils.dist, distutils.core, distutils.cmd: @@ -104,26 +101,11 @@ def patch_all(): patch_for_msvc_specialized_compiler() -def _patch_distribution_metadata_write_pkg_file(): - """Patch write_pkg_file to also write Requires-Python/Requires-External""" - distutils.dist.DistributionMetadata.write_pkg_file = ( - setuptools.dist.write_pkg_file - ) - - -def _patch_distribution_metadata_write_pkg_info(): - """ - Workaround issue #197 - Python 3 prior to 3.2.2 uses an environment-local - encoding to save the pkg_info. Monkey-patch its write_pkg_info method to - correct this undesirable behavior. - """ - environment_local = (3,) <= sys.version_info[:3] < (3, 2, 2) - if not environment_local: - return - - distutils.dist.DistributionMetadata.write_pkg_info = ( - setuptools.dist.write_pkg_info - ) +def _patch_distribution_metadata(): + """Patch write_pkg_file and read_pkg_file for higher metadata standards""" + for attr in ('write_pkg_file', 'read_pkg_file', 'get_metadata_version'): + new_val = getattr(setuptools.dist, attr) + setattr(distutils.dist.DistributionMetadata, attr, new_val) def patch_func(replacement, target_mod, func_name): diff --git a/setuptools/msvc.py b/setuptools/msvc.py index 8e3b638f..b9c472f1 100644 --- a/setuptools/msvc.py +++ b/setuptools/msvc.py @@ -22,7 +22,7 @@ import sys import platform import itertools import distutils.errors -from pkg_resources.extern.packaging.version import LegacyVersion +from setuptools.extern.packaging.version import LegacyVersion from setuptools.extern.six.moves import filterfalse @@ -48,7 +48,7 @@ else: _msvc9_suppress_errors = ( # msvc9compiler isn't available on some platforms ImportError, - + # msvc9compiler raises DistutilsPlatformError in some # environments. See #1118. distutils.errors.DistutilsPlatformError, @@ -232,8 +232,7 @@ def _augment_exception(exc, version, arch=''): elif version >= 14.0: # For VC++ 14.0 Redirect user to Visual C++ Build Tools message += (' Get it with "Microsoft Visual C++ Build Tools": ' - r'http://landinghub.visualstudio.com/' - 'visual-cpp-build-tools') + r'https://visualstudio.microsoft.com/downloads/') exc.args = (message, ) diff --git a/setuptools/namespaces.py b/setuptools/namespaces.py index dc16106d..dc16106d 100755..100644 --- a/setuptools/namespaces.py +++ b/setuptools/namespaces.py diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a6363b18..7e9517ce 100755..100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -7,13 +7,9 @@ import socket import base64 import hashlib import itertools +import warnings from functools import wraps -try: - from urllib.parse import splituser -except ImportError: - from urllib2 import splituser - from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client, configparser, map @@ -21,21 +17,23 @@ import setuptools from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, Environment, find_distributions, safe_name, safe_version, - to_filename, Requirement, DEVELOP_DIST, + to_filename, Requirement, DEVELOP_DIST, EGG_DIST, ) from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError from fnmatch import translate -from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers +from setuptools.py33compat import unescape +from setuptools.wheel import Wheel + +__metaclass__ = type EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') -HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) -# this is here to fix emacs' cruddy broken syntax highlighting +HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) PYPI_MD5 = re.compile( - '<a href="([^"#]+)">([^<]+)</a>\n\\s+\\(<a (?:title="MD5 hash"\n\\s+)' - 'href="[^?]+\\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\\)' + r'<a href="([^"#]+)">([^<]+)</a>\n\s+\(<a (?:title="MD5 hash"\n\s+)' + r'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\)' ) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -116,6 +114,17 @@ def distros_for_location(location, basename, metadata=None): if basename.endswith('.egg') and '-' in basename: # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] + if basename.endswith('.whl') and '-' in basename: + wheel = Wheel(basename) + if not wheel.is_compatible(): + return [] + return [Distribution( + location=location, + project_name=wheel.project_name, + version=wheel.version, + # Increase priority over eggs. + precedence=EGG_DIST + 1, + )] if basename.endswith('.exe'): win_base, py_ver, platform = parse_bdist_wininst(basename) if win_base is not None: @@ -141,7 +150,7 @@ def distros_for_filename(filename, metadata=None): def interpret_distro_name( location, basename, metadata, py_version=None, precedence=SOURCE_DIST, platform=None - ): +): """Generate alternative interpretations of a source distro name Note: if `location` is a filesystem filename, you should call @@ -228,7 +237,7 @@ def find_external_links(url, page): yield urllib.parse.urljoin(url, htmldecode(match.group(1))) -class ContentChecker(object): +class ContentChecker: """ A null content checker that defines the interface for checking content """ @@ -290,9 +299,9 @@ class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" def __init__( - self, index_url="https://pypi.python.org/simple", hosts=('*',), + self, index_url="https://pypi.org/simple/", hosts=('*',), ca_bundle=None, verify_ssl=True, *args, **kw - ): + ): Environment.__init__(self, *args, **kw) self.index_url = index_url + "/" [:not index_url.endswith('/')] self.scanned_urls = {} @@ -346,7 +355,8 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. + if not isinstance(page, str): + # In Python 3 and got bytes but want str. if isinstance(f, urllib.error.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' @@ -381,8 +391,9 @@ class PackageIndex(Environment): is_file = s and s.group(1).lower() == 'file' if is_file or self.allows(urllib.parse.urlparse(url)[1]): return True - msg = ("\nNote: Bypassing %s (disallowed host; see " - "http://bit.ly/1dg9ijs for details).\n") + msg = ( + "\nNote: Bypassing %s (disallowed host; see " + "http://bit.ly/2hrImnY for details).\n") if fatal: raise DistutilsError(msg % url) else: @@ -500,15 +511,16 @@ class PackageIndex(Environment): """ checker is a ContentChecker """ - checker.report(self.debug, + checker.report( + self.debug, "Validating %%s checksum for %s" % filename) if not checker.is_valid(): tfp.close() os.unlink(filename) raise DistutilsError( "%s validation failed for %s; " - "possible download problem?" % ( - checker.hash.name, os.path.basename(filename)) + "possible download problem?" + % (checker.hash.name, os.path.basename(filename)) ) def add_find_links(self, urls): @@ -536,7 +548,8 @@ class PackageIndex(Environment): if self[requirement.key]: # we've seen at least one distro meth, msg = self.info, "Couldn't retrieve index page for %r" else: # no distros seen for this name, might be misspelled - meth, msg = (self.warn, + meth, msg = ( + self.warn, "Couldn't find index page for %r (maybe misspelled?)") meth(msg, requirement.unsafe_name) self.scan_all() @@ -577,8 +590,7 @@ class PackageIndex(Environment): def fetch_distribution( self, requirement, tmpdir, force_scan=False, source=False, - develop_ok=False, local_index=None - ): + develop_ok=False, local_index=None): """Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. @@ -609,12 +621,19 @@ class PackageIndex(Environment): if dist.precedence == DEVELOP_DIST and not develop_ok: if dist not in skipped: - self.warn("Skipping development or system egg: %s", dist) + self.warn( + "Skipping development or system egg: %s", dist, + ) skipped[dist] = 1 continue - if dist in req and (dist.precedence <= SOURCE_DIST or not source): - dist.download_location = self.download(dist.location, tmpdir) + test = ( + dist in req + and (dist.precedence <= SOURCE_DIST or not source) + ) + if test: + loc = self.download(dist.location, tmpdir) + dist.download_location = loc if os.path.exists(dist.download_location): return dist @@ -704,10 +723,10 @@ class PackageIndex(Environment): def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file - fp, info = None, None + fp = None try: checker = HashChecker.from_url(url) - fp = self.open_url(strip_fragment(url)) + fp = self.open_url(url) if isinstance(fp, urllib.error.HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code, fp.msg) @@ -830,6 +849,10 @@ class PackageIndex(Environment): raise DistutilsError("Unexpected HTML page found at " + url) def _download_svn(self, url, filename): + warnings.warn("SVN download support is deprecated", UserWarning) + def splituser(host): + user, delim, host = host.rpartition('@') + return user, host url = url.split('#', 1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: @@ -915,27 +938,20 @@ class PackageIndex(Environment): entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub -def uchr(c): - if not isinstance(c, int): - return c - if c > 255: - return six.unichr(c) - return chr(c) - - def decode_entity(match): - what = match.group(1) - if what.startswith('#x'): - what = int(what[2:], 16) - elif what.startswith('#'): - what = int(what[1:]) - else: - what = six.moves.html_entities.name2codepoint.get(what, match.group(0)) - return uchr(what) + what = match.group(0) + return unescape(what) def htmldecode(text): - """Decode HTML entities in the given text.""" + """ + Decode HTML entities in the given text. + + >>> htmldecode( + ... 'https://../package_name-0.1.2.tar.gz' + ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') + 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' + """ return entity_sub(decode_entity, text) @@ -969,15 +985,14 @@ def _encode_auth(auth): auth_s = urllib.parse.unquote(auth) # convert to bytes auth_bytes = auth_s.encode() - # use the legacy interface for Python 2.3 support - encoded_bytes = base64.encodestring(auth_bytes) + encoded_bytes = base64.b64encode(auth_bytes) # convert back to a string encoded = encoded_bytes.decode() # strip the trailing carriage return return encoded.replace('\n', '') -class Credential(object): +class Credential: """ A username/password pair. Use like a namedtuple. """ @@ -1035,15 +1050,16 @@ class PyPIConfig(configparser.RawConfigParser): def open_with_auth(url, opener=urllib.request.urlopen): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url) + parsed = urllib.parse.urlparse(url) + scheme, netloc, path, params, query, frag = parsed # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): raise http_client.InvalidURL("nonnumeric port: ''") - if scheme in ('http', 'https'): - auth, host = splituser(netloc) + if scheme in ('http', 'https') and parsed.username: + auth = ':'.join((parsed.username, parsed.password)) else: auth = None @@ -1056,7 +1072,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): if auth: auth = "Basic " + _encode_auth(auth) - parts = scheme, host, path, params, query, frag + parts = scheme, parsed.hostname, path, params, query, frag new_url = urllib.parse.urlunparse(parts) request = urllib.request.Request(new_url) request.add_header("Authorization", auth) @@ -1070,7 +1086,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) - if s2 == scheme and h2 == host: + if s2 == scheme and h2 == parsed.hostname: parts = s2, netloc, path2, param2, query2, frag2 fp.url = urllib.parse.urlunparse(parts) @@ -1103,7 +1119,8 @@ def local_open(url): f += '/' files.append('<a href="{name}">{name}</a>'.format(name=f)) else: - tmpl = ("<html><head><title>{url}</title>" + tmpl = ( + "<html><head><title>{url}</title>" "</head><body>{files}</body></html>") body = tmpl.format(url=url, files='\n'.join(files)) status, message = 200, "OK" diff --git a/setuptools/pep425tags.py b/setuptools/pep425tags.py new file mode 100644 index 00000000..48745a29 --- /dev/null +++ b/setuptools/pep425tags.py @@ -0,0 +1,319 @@ +# This file originally from pip: +# https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/pep425tags.py +"""Generate and work with PEP 425 Compatibility Tags.""" +from __future__ import absolute_import + +import distutils.util +from distutils import log +import platform +import re +import sys +import sysconfig +import warnings +from collections import OrderedDict + +from .extern import six + +from . import glibc + +_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') + + +def get_config_var(var): + try: + return sysconfig.get_config_var(var) + except IOError as e: # Issue #1074 + warnings.warn("{}".format(e), RuntimeWarning) + return None + + +def get_abbr_impl(): + """Return abbreviated implementation name.""" + if hasattr(sys, 'pypy_version_info'): + pyimpl = 'pp' + elif sys.platform.startswith('java'): + pyimpl = 'jy' + elif sys.platform == 'cli': + pyimpl = 'ip' + else: + pyimpl = 'cp' + return pyimpl + + +def get_impl_ver(): + """Return implementation version.""" + impl_ver = get_config_var("py_version_nodot") + if not impl_ver or get_abbr_impl() == 'pp': + impl_ver = ''.join(map(str, get_impl_version_info())) + return impl_ver + + +def get_impl_version_info(): + """Return sys.version_info-like tuple for use in decrementing the minor + version.""" + if get_abbr_impl() == 'pp': + # as per https://github.com/pypa/pip/issues/2882 + return (sys.version_info[0], sys.pypy_version_info.major, + sys.pypy_version_info.minor) + else: + return sys.version_info[0], sys.version_info[1] + + +def get_impl_tag(): + """ + Returns the Tag for this specific implementation. + """ + return "{}{}".format(get_abbr_impl(), get_impl_ver()) + + +def get_flag(var, fallback, expected=True, warn=True): + """Use a fallback method for determining SOABI flags if the needed config + var is unset or unavailable.""" + val = get_config_var(var) + if val is None: + if warn: + log.debug("Config variable '%s' is unset, Python ABI tag may " + "be incorrect", var) + return fallback() + return val == expected + + +def get_abi_tag(): + """Return the ABI tag based on SOABI (if available) or emulate SOABI + (CPython 2, PyPy).""" + soabi = get_config_var('SOABI') + impl = get_abbr_impl() + if not soabi and impl in {'cp', 'pp'} and hasattr(sys, 'maxunicode'): + d = '' + m = '' + u = '' + if get_flag('Py_DEBUG', + lambda: hasattr(sys, 'gettotalrefcount'), + warn=(impl == 'cp')): + d = 'd' + if get_flag('WITH_PYMALLOC', + lambda: impl == 'cp', + warn=(impl == 'cp')): + m = 'm' + if get_flag('Py_UNICODE_SIZE', + lambda: sys.maxunicode == 0x10ffff, + expected=4, + warn=(impl == 'cp' and + six.PY2)) \ + and six.PY2: + u = 'u' + abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u) + elif soabi and soabi.startswith('cpython-'): + abi = 'cp' + soabi.split('-')[1] + elif soabi: + abi = soabi.replace('.', '_').replace('-', '_') + else: + abi = None + return abi + + +def _is_running_32bit(): + return sys.maxsize == 2147483647 + + +def get_platform(): + """Return our platform name 'win32', 'linux_x86_64'""" + if sys.platform == 'darwin': + # distutils.util.get_platform() returns the release based on the value + # of MACOSX_DEPLOYMENT_TARGET on which Python was built, which may + # be significantly older than the user's current machine. + release, _, machine = platform.mac_ver() + split_ver = release.split('.') + + if machine == "x86_64" and _is_running_32bit(): + machine = "i386" + elif machine == "ppc64" and _is_running_32bit(): + machine = "ppc" + + return 'macosx_{}_{}_{}'.format(split_ver[0], split_ver[1], machine) + + # XXX remove distutils dependency + result = distutils.util.get_platform().replace('.', '_').replace('-', '_') + if result == "linux_x86_64" and _is_running_32bit(): + # 32 bit Python program (running on a 64 bit Linux): pip should only + # install and run 32 bit compiled extensions in that case. + result = "linux_i686" + + return result + + +def is_manylinux1_compatible(): + # Only Linux, and only x86-64 / i686 + if get_platform() not in {"linux_x86_64", "linux_i686"}: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux1_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 5 uses glibc 2.5. + return glibc.have_compatible_glibc(2, 5) + + +def get_darwin_arches(major, minor, machine): + """Return a list of supported arches (including group arches) for + the given major, minor and machine architecture of a macOS machine. + """ + arches = [] + + def _supports_arch(major, minor, arch): + # Looking at the application support for macOS versions in the chart + # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears + # our timeline looks roughly like: + # + # 10.0 - Introduces ppc support. + # 10.4 - Introduces ppc64, i386, and x86_64 support, however the ppc64 + # and x86_64 support is CLI only, and cannot be used for GUI + # applications. + # 10.5 - Extends ppc64 and x86_64 support to cover GUI applications. + # 10.6 - Drops support for ppc64 + # 10.7 - Drops support for ppc + # + # Given that we do not know if we're installing a CLI or a GUI + # application, we must be conservative and assume it might be a GUI + # application and behave as if ppc64 and x86_64 support did not occur + # until 10.5. + # + # Note: The above information is taken from the "Application support" + # column in the chart not the "Processor support" since I believe + # that we care about what instruction sets an application can use + # not which processors the OS supports. + if arch == 'ppc': + return (major, minor) <= (10, 5) + if arch == 'ppc64': + return (major, minor) == (10, 5) + if arch == 'i386': + return (major, minor) >= (10, 4) + if arch == 'x86_64': + return (major, minor) >= (10, 5) + if arch in groups: + for garch in groups[arch]: + if _supports_arch(major, minor, garch): + return True + return False + + groups = OrderedDict([ + ("fat", ("i386", "ppc")), + ("intel", ("x86_64", "i386")), + ("fat64", ("x86_64", "ppc64")), + ("fat32", ("x86_64", "i386", "ppc")), + ]) + + if _supports_arch(major, minor, machine): + arches.append(machine) + + for garch in groups: + if machine in groups[garch] and _supports_arch(major, minor, garch): + arches.append(garch) + + arches.append('universal') + + return arches + + +def get_supported(versions=None, noarch=False, platform=None, + impl=None, abi=None): + """Return a list of supported tags for each version specified in + `versions`. + + :param versions: a list of string versions, of the form ["33", "32"], + or None. The first version will be assumed to support our ABI. + :param platform: specify the exact platform you want valid + tags for, or None. If None, use the local system platform. + :param impl: specify the exact implementation you want valid + tags for, or None. If None, use the local interpreter impl. + :param abi: specify the exact abi you want valid + tags for, or None. If None, use the local interpreter abi. + """ + supported = [] + + # Versions must be given with respect to the preference + if versions is None: + versions = [] + version_info = get_impl_version_info() + major = version_info[:-1] + # Support all previous minor Python versions. + for minor in range(version_info[-1], -1, -1): + versions.append(''.join(map(str, major + (minor,)))) + + impl = impl or get_abbr_impl() + + abis = [] + + abi = abi or get_abi_tag() + if abi: + abis[0:0] = [abi] + + abi3s = set() + import imp + for suffix in imp.get_suffixes(): + if suffix[0].startswith('.abi'): + abi3s.add(suffix[0].split('.', 2)[1]) + + abis.extend(sorted(list(abi3s))) + + abis.append('none') + + if not noarch: + arch = platform or get_platform() + if arch.startswith('macosx'): + # support macosx-10.6-intel on macosx-10.9-x86_64 + match = _osx_arch_pat.match(arch) + if match: + name, major, minor, actual_arch = match.groups() + tpl = '{}_{}_%i_%s'.format(name, major) + arches = [] + for m in reversed(range(int(minor) + 1)): + for a in get_darwin_arches(int(major), m, actual_arch): + arches.append(tpl % (m, a)) + else: + # arch pattern didn't match (?!) + arches = [arch] + elif platform is None and is_manylinux1_compatible(): + arches = [arch.replace('linux', 'manylinux1'), arch] + else: + arches = [arch] + + # Current version, current API (built specifically for our Python): + for abi in abis: + for arch in arches: + supported.append(('%s%s' % (impl, versions[0]), abi, arch)) + + # abi3 modules compatible with older version of Python + for version in versions[1:]: + # abi3 was introduced in Python 3.2 + if version in {'31', '30'}: + break + for abi in abi3s: # empty set if not Python 3 + for arch in arches: + supported.append(("%s%s" % (impl, version), abi, arch)) + + # Has binaries, does not use the Python API: + for arch in arches: + supported.append(('py%s' % (versions[0][0]), 'none', arch)) + + # No abi / arch, but requires our implementation: + supported.append(('%s%s' % (impl, versions[0]), 'none', 'any')) + # Tagged specifically as being cross-version compatible + # (with just the major version specified) + supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any')) + + # No abi / arch, generic Python + for i, version in enumerate(versions): + supported.append(('py%s' % (version,), 'none', 'any')) + if i == 0: + supported.append(('py%s' % (version[0]), 'none', 'any')) + + return supported + + +implementation_tag = get_impl_tag() diff --git a/setuptools/py26compat.py b/setuptools/py26compat.py deleted file mode 100644 index 4d3add8c..00000000 --- a/setuptools/py26compat.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Compatibility Support for Python 2.6 and earlier -""" - -import sys - -try: - from urllib.parse import splittag -except ImportError: - from urllib import splittag - - -def strip_fragment(url): - """ - In `Python 8280 <http://bugs.python.org/issue8280>`_, Python 2.7 and - later was patched to disregard the fragment when making URL requests. - Do the same for Python 2.6 and earlier. - """ - url, fragment = splittag(url) - return url - - -if sys.version_info >= (2, 7): - strip_fragment = lambda x: x - -try: - from importlib import import_module -except ImportError: - - def import_module(module_name): - return __import__(module_name, fromlist=['__name__']) diff --git a/setuptools/py31compat.py b/setuptools/py31compat.py index 44b025d4..1a0705ec 100644 --- a/setuptools/py31compat.py +++ b/setuptools/py31compat.py @@ -1,18 +1,6 @@ -import sys -import unittest +__all__ = [] -__all__ = ['get_config_vars', 'get_path'] - -try: - # Python 2.7 or >=3.2 - from sysconfig import get_config_vars, get_path -except ImportError: - from distutils.sysconfig import get_config_vars, get_python_lib - - def get_path(name): - if name not in ('platlib', 'purelib'): - raise ValueError("Name must be purelib or platlib") - return get_python_lib(name == 'platlib') +__metaclass__ = type try: @@ -22,7 +10,7 @@ except ImportError: import shutil import tempfile - class TemporaryDirectory(object): + class TemporaryDirectory: """ Very simple temporary directory context manager. Will try to delete afterward, but will also ignore OS and similar @@ -42,15 +30,3 @@ except ImportError: except OSError: # removal errors are not the only possible pass self.name = None - - -unittest_main = unittest.main - -_PY31 = (3, 1) <= sys.version_info[:2] < (3, 2) -if _PY31: - # on Python 3.1, translate testRunner==None to TextTestRunner - # for compatibility with Python 2.6, 2.7, and 3.2+ - def unittest_main(*args, **kwargs): - if 'testRunner' in kwargs and kwargs['testRunner'] is None: - kwargs['testRunner'] = unittest.TextTestRunner - return unittest.main(*args, **kwargs) diff --git a/setuptools/py33compat.py b/setuptools/py33compat.py index af64d5d1..87cf5398 100644 --- a/setuptools/py33compat.py +++ b/setuptools/py33compat.py @@ -2,13 +2,20 @@ import dis import array import collections +try: + import html +except ImportError: + html = None + from setuptools.extern import six +from setuptools.extern.six.moves import html_parser +__metaclass__ = type OpArg = collections.namedtuple('OpArg', 'opcode arg') -class Bytecode_compat(object): +class Bytecode_compat: def __init__(self, code): self.code = code @@ -43,3 +50,6 @@ class Bytecode_compat(object): Bytecode = getattr(dis, 'Bytecode', Bytecode_compat) + + +unescape = getattr(html, 'unescape', html_parser.HTMLParser().unescape) diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 1d981f49..685f3f72 100755..100644 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -39,10 +39,6 @@ def _execfile(filename, globals, locals=None): mode = 'rb' with open(filename, mode) as stream: script = stream.read() - # compile() function in Python 2.6 and 3.1 requires LF line endings. - if sys.version_info[:2] < (2, 7) or sys.version_info[:2] >= (3, 0) and sys.version_info[:2] < (3, 2): - script = script.replace(b'\r\n', b'\n') - script = script.replace(b'\r', b'\n') if locals is None: locals = globals code = compile(script, filename, 'exec') diff --git a/setuptools/script (dev).tmpl b/setuptools/script (dev).tmpl index d58b1bb5..39a24b04 100644 --- a/setuptools/script (dev).tmpl +++ b/setuptools/script (dev).tmpl @@ -2,4 +2,5 @@ __requires__ = %(spec)r __import__('pkg_resources').require(%(spec)r) __file__ = %(dev_path)r -exec(compile(open(__file__).read(), __file__, 'exec')) +with open(__file__) as f: + exec(compile(f.read(), __file__, 'exec')) diff --git a/setuptools/site-patch.py b/setuptools/site-patch.py index 0d2d2ff8..40b00de0 100644 --- a/setuptools/site-patch.py +++ b/setuptools/site-patch.py @@ -23,7 +23,7 @@ def __boot(): break else: try: - import imp # Avoid import loop in Python >= 3.3 + import imp # Avoid import loop in Python 3 stream, path, descr = imp.find_module('site', [item]) except ImportError: continue diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py index 72b18ef2..226db694 100644 --- a/setuptools/ssl_support.py +++ b/setuptools/ssl_support.py @@ -59,7 +59,7 @@ if not match_hostname: def _dnsname_match(dn, hostname, max_wildcards=1): """Matching according to RFC 6125, section 6.4.3 - http://tools.ietf.org/html/rfc6125#section-6.4.3 + https://tools.ietf.org/html/rfc6125#section-6.4.3 """ pats = [] if not dn: @@ -186,9 +186,14 @@ class VerifyingHTTPSConn(HTTPSConnection): else: actual_host = self.host - self.sock = ssl.wrap_socket( - sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_bundle - ) + if hasattr(ssl, 'create_default_context'): + ctx = ssl.create_default_context(cafile=self.ca_bundle) + self.sock = ctx.wrap_socket(sock, server_hostname=actual_host) + else: + # This is for python < 2.7.9 and < 3.4? + self.sock = ssl.wrap_socket( + sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_bundle + ) try: match_hostname(self.sock.getpeercert(), actual_host) except CertificateError: diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index 54dd7d2b..5f4a1c29 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -2,5 +2,17 @@ import locale import pytest +from setuptools.extern.six import PY2, PY3 + + +__all__ = [ + 'fail_on_ascii', 'py2_only', 'py3_only' +] + + is_ascii = locale.getpreferredencoding() == 'ANSI_X3.4-1968' fail_on_ascii = pytest.mark.xfail(is_ascii, reason="Test fails in this locale") + + +py2_only = pytest.mark.skipif(not PY2, reason="Test runs on Python 2 only") +py3_only = pytest.mark.skipif(not PY3, reason="Test runs on Python 3 only") diff --git a/setuptools/tests/files.py b/setuptools/tests/files.py index 98de9fc3..465a6b41 100644 --- a/setuptools/tests/files.py +++ b/setuptools/tests/files.py @@ -30,5 +30,9 @@ def build_files(file_defs, prefix=""): pkg_resources.py31compat.makedirs(full_name, exist_ok=True) build_files(contents, prefix=full_name) else: - with open(full_name, 'w') as f: - f.write(contents) + if isinstance(contents, bytes): + with open(full_name, 'wb') as f: + f.write(contents) + else: + with open(full_name, 'w') as f: + f.write(contents) diff --git a/setuptools/tests/py26compat.py b/setuptools/tests/py26compat.py deleted file mode 100644 index 18cece05..00000000 --- a/setuptools/tests/py26compat.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys -import tarfile -import contextlib - - -def _tarfile_open_ex(*args, **kwargs): - """ - Extend result as a context manager. - """ - return contextlib.closing(tarfile.open(*args, **kwargs)) - - -if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 2): - tarfile_open = _tarfile_open_ex -else: - tarfile_open = tarfile.open diff --git a/setuptools/tests/test_bdist_egg.py b/setuptools/tests/test_bdist_egg.py index d24aa366..54742aa6 100644 --- a/setuptools/tests/test_bdist_egg.py +++ b/setuptools/tests/test_bdist_egg.py @@ -2,6 +2,7 @@ """ import os import re +import zipfile import pytest @@ -16,7 +17,7 @@ setup(name='foo', py_modules=['hi']) """ -@pytest.yield_fixture +@pytest.fixture(scope='function') def setup_context(tmpdir): with (tmpdir / 'setup.py').open('w') as f: f.write(SETUP_PY) @@ -32,7 +33,7 @@ class Test: script_name='setup.py', script_args=['bdist_egg'], name='foo', - py_modules=['hi'] + py_modules=['hi'], )) os.makedirs(os.path.join('build', 'src')) with contexts.quiet(): @@ -42,3 +43,24 @@ class Test: # let's see if we got our egg link at the right place [content] = os.listdir('dist') assert re.match(r'foo-0.0.0-py[23].\d.egg$', content) + + @pytest.mark.xfail( + os.environ.get('PYTHONDONTWRITEBYTECODE'), + reason="Byte code disabled", + ) + def test_exclude_source_files(self, setup_context, user_override): + dist = Distribution(dict( + script_name='setup.py', + script_args=['bdist_egg', '--exclude-source-files'], + name='foo', + py_modules=['hi'], + )) + with contexts.quiet(): + dist.parse_command_line() + dist.run_commands() + [dist_name] = os.listdir('dist') + dist_filename = os.path.join('dist', dist_name) + zip = zipfile.ZipFile(dist_filename) + names = list(zi.filename for zi in zip.filelist) + assert 'hi.pyc' in names + assert 'hi.py' not in names diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 69a700c2..82b44c89 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -1,16 +1,24 @@ +from __future__ import unicode_literals + import os +import shutil +import tarfile import pytest +from setuptools.build_meta import build_sdist from .files import build_files from .textwrap import DALS +from . import py2_only +__metaclass__ = type -futures = pytest.importorskip('concurrent.futures') -importlib = pytest.importorskip('importlib') +# Backports on Python 2.7 +import importlib +from concurrent import futures -class BuildBackendBase(object): +class BuildBackendBase: def __init__(self, cwd=None, env={}, backend_name='setuptools.build_meta'): self.cwd = cwd self.env = env @@ -19,12 +27,14 @@ class BuildBackendBase(object): class BuildBackend(BuildBackendBase): """PEP 517 Build Backend""" + def __init__(self, *args, **kwargs): super(BuildBackend, self).__init__(*args, **kwargs) self.pool = futures.ProcessPoolExecutor() def __getattr__(self, name): """Handles aribrary function invocations on the build backend.""" + def method(*args, **kw): root = os.path.abspath(self.cwd) caller = BuildBackendCaller(root, self.env, self.backend_name) @@ -42,12 +52,12 @@ class BuildBackendCaller(BuildBackendBase): return getattr(mod, name)(*args, **kw) -@pytest.fixture -def build_backend(tmpdir): - defn = { +defns = [ + { 'setup.py': DALS(""" __import__('setuptools').setup( name='foo', + version='0.0.0', py_modules=['hello'], setup_requires=['six'], ) @@ -56,15 +66,59 @@ def build_backend(tmpdir): def run(): print('hello') """), - } - build_files(defn, prefix=str(tmpdir)) + }, + { + 'setup.py': DALS(""" + assert __name__ == '__main__' + __import__('setuptools').setup( + name='foo', + version='0.0.0', + py_modules=['hello'], + setup_requires=['six'], + ) + """), + 'hello.py': DALS(""" + def run(): + print('hello') + """), + }, + { + 'setup.py': DALS(""" + variable = True + def function(): + return variable + assert variable + __import__('setuptools').setup( + name='foo', + version='0.0.0', + py_modules=['hello'], + setup_requires=['six'], + ) + """), + 'hello.py': DALS(""" + def run(): + print('hello') + """), + }, +] + + +@pytest.fixture(params=defns) +def build_backend(tmpdir, request): + build_files(request.param, prefix=str(tmpdir)) with tmpdir.as_cwd(): yield BuildBackend(cwd='.') def test_get_requires_for_build_wheel(build_backend): actual = build_backend.get_requires_for_build_wheel() - expected = ['six', 'setuptools', 'wheel'] + expected = ['six', 'wheel'] + assert sorted(actual) == sorted(expected) + + +def test_get_requires_for_build_sdist(build_backend): + actual = build_backend.get_requires_for_build_sdist() + expected = ['six'] assert sorted(actual) == sorted(expected) @@ -91,3 +145,94 @@ def test_prepare_metadata_for_build_wheel(build_backend): dist_info = build_backend.prepare_metadata_for_build_wheel(dist_dir) assert os.path.isfile(os.path.join(dist_dir, dist_info, 'METADATA')) + + +@py2_only +def test_prepare_metadata_for_build_wheel_with_str(build_backend): + dist_dir = os.path.abspath(str('pip-dist-info')) + os.makedirs(dist_dir) + + dist_info = build_backend.prepare_metadata_for_build_wheel(dist_dir) + + assert os.path.isfile(os.path.join(dist_dir, dist_info, 'METADATA')) + + +def test_build_sdist_explicit_dist(build_backend): + # explicitly specifying the dist folder should work + # the folder sdist_directory and the ``--dist-dir`` can be the same + dist_dir = os.path.abspath('dist') + sdist_name = build_backend.build_sdist(dist_dir) + assert os.path.isfile(os.path.join(dist_dir, sdist_name)) + + +def test_build_sdist_version_change(build_backend): + sdist_into_directory = os.path.abspath("out_sdist") + os.makedirs(sdist_into_directory) + + sdist_name = build_backend.build_sdist(sdist_into_directory) + assert os.path.isfile(os.path.join(sdist_into_directory, sdist_name)) + + # if the setup.py changes subsequent call of the build meta + # should still succeed, given the + # sdist_directory the frontend specifies is empty + with open(os.path.abspath("setup.py"), 'rt') as file_handler: + content = file_handler.read() + with open(os.path.abspath("setup.py"), 'wt') as file_handler: + file_handler.write( + content.replace("version='0.0.0'", "version='0.0.1'")) + + shutil.rmtree(sdist_into_directory) + os.makedirs(sdist_into_directory) + + sdist_name = build_backend.build_sdist("out_sdist") + assert os.path.isfile( + os.path.join(os.path.abspath("out_sdist"), sdist_name)) + + +def test_build_sdist_setup_py_exists(tmpdir_cwd): + # If build_sdist is called from a script other than setup.py, + # ensure setup.py is include + build_files(defns[0]) + targz_path = build_sdist("temp") + with tarfile.open(os.path.join("temp", targz_path)) as tar: + assert any('setup.py' in name for name in tar.getnames()) + + +def test_build_sdist_setup_py_manifest_excluded(tmpdir_cwd): + # Ensure that MANIFEST.in can exclude setup.py + files = { + 'setup.py': DALS(""" + __import__('setuptools').setup( + name='foo', + version='0.0.0', + py_modules=['hello'] + )"""), + 'hello.py': '', + 'MANIFEST.in': DALS(""" + exclude setup.py + """) + } + + build_files(files) + targz_path = build_sdist("temp") + with tarfile.open(os.path.join("temp", targz_path)) as tar: + assert not any('setup.py' in name for name in tar.getnames()) + + +def test_build_sdist_builds_targz_even_if_zip_indicated(tmpdir_cwd): + files = { + 'setup.py': DALS(""" + __import__('setuptools').setup( + name='foo', + version='0.0.0', + py_modules=['hello'] + )"""), + 'hello.py': '', + 'setup.cfg': DALS(""" + [sdist] + formats=zip + """) + } + + build_files(files) + build_sdist("temp") diff --git a/setuptools/tests/test_config.py b/setuptools/tests/test_config.py index 89fde257..0b8ae158 100644 --- a/setuptools/tests/test_config.py +++ b/setuptools/tests/test_config.py @@ -3,25 +3,32 @@ from __future__ import unicode_literals import contextlib import pytest + from distutils.errors import DistutilsOptionError, DistutilsFileError -from setuptools.dist import Distribution +from mock import patch +from setuptools.dist import Distribution, _Distribution from setuptools.config import ConfigHandler, read_configuration from setuptools.extern.six.moves.configparser import InterpolationMissingOptionError from setuptools.tests import is_ascii - +from . import py2_only, py3_only +from .textwrap import DALS class ErrConfigHandler(ConfigHandler): """Erroneous handler. Fails to implement required methods.""" -def make_package_dir(name, base_dir): - dir_package = base_dir.mkdir(name) - init_file = dir_package.join('__init__.py') - init_file.write('') +def make_package_dir(name, base_dir, ns=False): + dir_package = base_dir + for dir_name in name.split('/'): + dir_package = dir_package.mkdir(dir_name) + init_file = None + if not ns: + init_file = dir_package.join('__init__.py') + init_file.write('') return dir_package, init_file -def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii'): +def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii', package_path='fake_package'): if setup_py is None: setup_py = ( @@ -33,7 +40,7 @@ def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii'): config = tmpdir.join('setup.cfg') config.write(setup_cfg.encode(encoding), mode='wb') - package_dir, init_file = make_package_dir('fake_package', tmpdir) + package_dir, init_file = make_package_dir(package_path, tmpdir) init_file.write( 'VERSION = (1, 2, 3)\n' @@ -115,6 +122,7 @@ class TestMetadata: '[metadata]\n' 'version = 10.1.1\n' 'description = Some description\n' + 'long_description_content_type = text/something\n' 'long_description = file: README\n' 'name = fake_name\n' 'keywords = one, two\n' @@ -136,6 +144,7 @@ class TestMetadata: assert metadata.version == '10.1.1' assert metadata.description == 'Some description' + assert metadata.long_description_content_type == 'text/something' assert metadata.long_description == 'readme contents\nline2' assert metadata.provides == ['package', 'package.sub'] assert metadata.license == 'BSD 3-Clause License' @@ -144,6 +153,24 @@ class TestMetadata: assert metadata.download_url == 'http://test.test.com/test/' assert metadata.maintainer_email == 'test@test.com' + def test_license_cfg(self, tmpdir): + fake_env( + tmpdir, + DALS(""" + [metadata] + name=foo + version=0.0.1 + license=Apache 2.0 + """) + ) + + with get_dist(tmpdir) as dist: + metadata = dist.metadata + + assert metadata.name == "foo" + assert metadata.version == "0.0.1" + assert metadata.license == "Apache 2.0" + def test_file_mixed(self, tmpdir): fake_env( @@ -220,6 +247,22 @@ class TestMetadata: 'Programming Language :: Python :: 3.5', ] + def test_dict(self, tmpdir): + + fake_env( + tmpdir, + '[metadata]\n' + 'project_urls =\n' + ' Link One = https://example.com/one/\n' + ' Link Two = https://example.com/two/\n' + ) + with get_dist(tmpdir) as dist: + metadata = dist.metadata + assert metadata.project_urls == { + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + } + def test_version(self, tmpdir): _, config = fake_env( @@ -255,6 +298,68 @@ class TestMetadata: with get_dist(tmpdir) as dist: assert dist.metadata.version == '2016.11.26' + def test_version_file(self, tmpdir): + + _, config = fake_env( + tmpdir, + '[metadata]\n' + 'version = file: fake_package/version.txt\n' + ) + tmpdir.join('fake_package', 'version.txt').write('1.2.3\n') + + with get_dist(tmpdir) as dist: + assert dist.metadata.version == '1.2.3' + + tmpdir.join('fake_package', 'version.txt').write('1.2.3\n4.5.6\n') + with pytest.raises(DistutilsOptionError): + with get_dist(tmpdir) as dist: + _ = dist.metadata.version + + def test_version_with_package_dir_simple(self, tmpdir): + + _, config = fake_env( + tmpdir, + '[metadata]\n' + 'version = attr: fake_package_simple.VERSION\n' + '[options]\n' + 'package_dir =\n' + ' = src\n', + package_path='src/fake_package_simple' + ) + + with get_dist(tmpdir) as dist: + assert dist.metadata.version == '1.2.3' + + def test_version_with_package_dir_rename(self, tmpdir): + + _, config = fake_env( + tmpdir, + '[metadata]\n' + 'version = attr: fake_package_rename.VERSION\n' + '[options]\n' + 'package_dir =\n' + ' fake_package_rename = fake_dir\n', + package_path='fake_dir' + ) + + with get_dist(tmpdir) as dist: + assert dist.metadata.version == '1.2.3' + + def test_version_with_package_dir_complex(self, tmpdir): + + _, config = fake_env( + tmpdir, + '[metadata]\n' + 'version = attr: fake_package_complex.VERSION\n' + '[options]\n' + 'package_dir =\n' + ' fake_package_complex = src/fake_dir\n', + package_path='src/fake_dir' + ) + + with get_dist(tmpdir) as dist: + assert dist.metadata.version == '1.2.3' + def test_unknown_meta_item(self, tmpdir): fake_env( @@ -311,6 +416,23 @@ class TestMetadata: with get_dist(tmpdir) as dist: assert set(dist.metadata.classifiers) == expected + def test_deprecated_config_handlers(self, tmpdir): + fake_env( + tmpdir, + '[metadata]\n' + 'version = 10.1.1\n' + 'description = Some description\n' + 'requires = some, requirement\n' + ) + + with pytest.deprecated_call(): + with get_dist(tmpdir) as dist: + metadata = dist.metadata + + assert metadata.version == '10.1.1' + assert metadata.description == 'Some description' + assert metadata.requires == ['some', 'requirement'] + def test_interpolation(self, tmpdir): fake_env( tmpdir, @@ -584,6 +706,60 @@ class TestOptions: assert set(dist.packages) == set( ['fake_package', 'fake_package.sub_two']) + @py2_only + def test_find_namespace_directive_fails_on_py2(self, tmpdir): + dir_package, config = fake_env( + tmpdir, + '[options]\n' + 'packages = find_namespace:\n' + ) + + with pytest.raises(DistutilsOptionError): + with get_dist(tmpdir) as dist: + dist.parse_config_files() + + @py3_only + def test_find_namespace_directive(self, tmpdir): + dir_package, config = fake_env( + tmpdir, + '[options]\n' + 'packages = find_namespace:\n' + ) + + dir_sub_one, _ = make_package_dir('sub_one', dir_package) + dir_sub_two, _ = make_package_dir('sub_two', dir_package, ns=True) + + with get_dist(tmpdir) as dist: + assert set(dist.packages) == { + 'fake_package', 'fake_package.sub_two', 'fake_package.sub_one' + } + + config.write( + '[options]\n' + 'packages = find_namespace:\n' + '\n' + '[options.packages.find]\n' + 'where = .\n' + 'include =\n' + ' fake_package.sub_one\n' + ' two\n' + ) + with get_dist(tmpdir) as dist: + assert dist.packages == ['fake_package.sub_one'] + + config.write( + '[options]\n' + 'packages = find_namespace:\n' + '\n' + '[options.packages.find]\n' + 'exclude =\n' + ' fake_package.sub_one\n' + ) + with get_dist(tmpdir) as dist: + assert set(dist.packages) == { + 'fake_package', 'fake_package.sub_two' + } + def test_extras_require(self, tmpdir): fake_env( tmpdir, @@ -599,6 +775,7 @@ class TestOptions: 'pdf': ['ReportLab>=1.2', 'RXP'], 'rest': ['docutils>=0.3', 'pack==1.1,==1.3'] } + assert dist.metadata.provides_extras == set(['pdf', 'rest']) def test_entry_points(self, tmpdir): _, config = fake_env( @@ -633,3 +810,60 @@ class TestOptions: with get_dist(tmpdir) as dist: assert dist.entry_points == expected + + def test_data_files(self, tmpdir): + fake_env( + tmpdir, + '[options.data_files]\n' + 'cfg =\n' + ' a/b.conf\n' + ' c/d.conf\n' + 'data = e/f.dat, g/h.dat\n' + ) + + with get_dist(tmpdir) as dist: + expected = [ + ('cfg', ['a/b.conf', 'c/d.conf']), + ('data', ['e/f.dat', 'g/h.dat']), + ] + assert sorted(dist.data_files) == sorted(expected) + +saved_dist_init = _Distribution.__init__ +class TestExternalSetters: + # During creation of the setuptools Distribution() object, we call + # the init of the parent distutils Distribution object via + # _Distribution.__init__ (). + # + # It's possible distutils calls out to various keyword + # implementations (i.e. distutils.setup_keywords entry points) + # that may set a range of variables. + # + # This wraps distutil's Distribution.__init__ and simulates + # pbr or something else setting these values. + def _fake_distribution_init(self, dist, attrs): + saved_dist_init(dist, attrs) + # see self._DISTUTUILS_UNSUPPORTED_METADATA + setattr(dist.metadata, 'long_description_content_type', + 'text/something') + # Test overwrite setup() args + setattr(dist.metadata, 'project_urls', { + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + }) + return None + + @patch.object(_Distribution, '__init__', autospec=True) + def test_external_setters(self, mock_parent_init, tmpdir): + mock_parent_init.side_effect = self._fake_distribution_init + + dist = Distribution(attrs={ + 'project_urls': { + 'will_be': 'ignored' + } + }) + + assert dist.metadata.long_description_content_type == 'text/something' + assert dist.metadata.project_urls == { + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + } diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index cb4ff4b4..00d4bd9a 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -8,6 +8,7 @@ import site import sys import io import subprocess +import platform from setuptools.extern import six from setuptools.command import test @@ -61,7 +62,8 @@ class TestDevelop: in_virtualenv = hasattr(sys, 'real_prefix') in_venv = hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix - @pytest.mark.skipif(in_virtualenv or in_venv, + @pytest.mark.skipif( + in_virtualenv or in_venv, reason="Cannot run when invoked in a virtualenv or venv") def test_2to3_user_mode(self, test_env): settings = dict( @@ -101,7 +103,8 @@ class TestDevelop: Test that console scripts are installed and that they reference only the project by name and not the current version. """ - pytest.skip("TODO: needs a fixture to cause 'develop' " + pytest.skip( + "TODO: needs a fixture to cause 'develop' " "to be invoked without mutating environment.") settings = dict( name='foo', @@ -153,8 +156,14 @@ class TestNamespaces: with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(develop_cmd) - @pytest.mark.skipif(bool(os.environ.get("APPVEYOR")), - reason="https://github.com/pypa/setuptools/issues/851") + @pytest.mark.skipif( + bool(os.environ.get("APPVEYOR")), + reason="https://github.com/pypa/setuptools/issues/851", + ) + @pytest.mark.skipif( + platform.python_implementation() == 'PyPy' and six.PY3, + reason="https://github.com/pypa/setuptools/issues/1202", + ) def test_namespace_package_importable(self, tmpdir): """ Installing two packages sharing the same namespace, one installed @@ -169,7 +178,7 @@ class TestNamespaces: install_cmd = [ sys.executable, '-m', - 'pip.__main__', + 'pip', 'install', str(pkg_A), '-t', str(target), diff --git a/setuptools/tests/test_dist.py b/setuptools/tests/test_dist.py index 435ffec0..cf830b43 100644 --- a/setuptools/tests/test_dist.py +++ b/setuptools/tests/test_dist.py @@ -1,10 +1,18 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +import io +from setuptools.dist import DistDeprecationWarning, _get_unpatched from setuptools import Distribution from setuptools.extern.six.moves.urllib.request import pathname2url from setuptools.extern.six.moves.urllib_parse import urljoin +from setuptools.extern import six from .textwrap import DALS from .test_easy_install import make_nspkg_sdist +import pytest def test_dist_fetch_build_egg(tmpdir): """ @@ -12,6 +20,7 @@ def test_dist_fetch_build_egg(tmpdir): """ index = tmpdir.mkdir('index') index_url = urljoin('file://', pathname2url(str(index))) + def sdist_with_index(distname, version): dist_dir = index.mkdir(distname) dist_sdist = '%s-%s.tar.gz' % (distname, version) @@ -39,8 +48,215 @@ def test_dist_fetch_build_egg(tmpdir): '''.split() with tmpdir.as_cwd(): dist = Distribution() + dist.parse_config_files() resolved_dists = [ dist.fetch_build_egg(r) for r in reqs ] assert [dist.key for dist in resolved_dists if dist] == reqs + + +def test_dist__get_unpatched_deprecated(): + pytest.warns(DistDeprecationWarning, _get_unpatched, [""]) + + +def __read_test_cases(): + # Metadata version 1.0 + base_attrs = { + "name": "package", + "version": "0.0.1", + "author": "Foo Bar", + "author_email": "foo@bar.net", + "long_description": "Long\ndescription", + "description": "Short description", + "keywords": ["one", "two"] + } + + def merge_dicts(d1, d2): + d1 = d1.copy() + d1.update(d2) + + return d1 + + test_cases = [ + ('Metadata version 1.0', base_attrs.copy()), + ('Metadata version 1.1: Provides', merge_dicts(base_attrs, { + 'provides': ['package'] + })), + ('Metadata version 1.1: Obsoletes', merge_dicts(base_attrs, { + 'obsoletes': ['foo'] + })), + ('Metadata version 1.1: Classifiers', merge_dicts(base_attrs, { + 'classifiers': [ + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'License :: OSI Approved :: MIT License' + ]})), + ('Metadata version 1.1: Download URL', merge_dicts(base_attrs, { + 'download_url': 'https://example.com' + })), + ('Metadata Version 1.2: Requires-Python', merge_dicts(base_attrs, { + 'python_requires': '>=3.7' + })), + pytest.param('Metadata Version 1.2: Project-Url', + merge_dicts(base_attrs, { + 'project_urls': { + 'Foo': 'https://example.bar' + } + }), marks=pytest.mark.xfail( + reason="Issue #1578: project_urls not read" + )), + ('Metadata Version 2.1: Long Description Content Type', + merge_dicts(base_attrs, { + 'long_description_content_type': 'text/x-rst; charset=UTF-8' + })), + pytest.param('Metadata Version 2.1: Provides Extra', + merge_dicts(base_attrs, { + 'provides_extras': ['foo', 'bar'] + }), marks=pytest.mark.xfail(reason="provides_extras not read")), + ('Missing author, missing author e-mail', + {'name': 'foo', 'version': '1.0.0'}), + ('Missing author', + {'name': 'foo', + 'version': '1.0.0', + 'author_email': 'snorri@sturluson.name'}), + ('Missing author e-mail', + {'name': 'foo', + 'version': '1.0.0', + 'author': 'Snorri Sturluson'}), + ('Missing author', + {'name': 'foo', + 'version': '1.0.0', + 'author': 'Snorri Sturluson'}), + ] + + return test_cases + + +@pytest.mark.parametrize('name,attrs', __read_test_cases()) +def test_read_metadata(name, attrs): + dist = Distribution(attrs) + metadata_out = dist.metadata + dist_class = metadata_out.__class__ + + # Write to PKG_INFO and then load into a new metadata object + if six.PY2: + PKG_INFO = io.BytesIO() + else: + PKG_INFO = io.StringIO() + + metadata_out.write_pkg_file(PKG_INFO) + + PKG_INFO.seek(0) + metadata_in = dist_class() + metadata_in.read_pkg_file(PKG_INFO) + + tested_attrs = [ + ('name', dist_class.get_name), + ('version', dist_class.get_version), + ('author', dist_class.get_contact), + ('author_email', dist_class.get_contact_email), + ('metadata_version', dist_class.get_metadata_version), + ('provides', dist_class.get_provides), + ('description', dist_class.get_description), + ('download_url', dist_class.get_download_url), + ('keywords', dist_class.get_keywords), + ('platforms', dist_class.get_platforms), + ('obsoletes', dist_class.get_obsoletes), + ('requires', dist_class.get_requires), + ('classifiers', dist_class.get_classifiers), + ('project_urls', lambda s: getattr(s, 'project_urls', {})), + ('provides_extras', lambda s: getattr(s, 'provides_extras', set())), + ] + + for attr, getter in tested_attrs: + assert getter(metadata_in) == getter(metadata_out) + + +def __maintainer_test_cases(): + attrs = {"name": "package", + "version": "1.0", + "description": "xxx"} + + def merge_dicts(d1, d2): + d1 = d1.copy() + d1.update(d2) + + return d1 + + test_cases = [ + ('No author, no maintainer', attrs.copy()), + ('Author (no e-mail), no maintainer', merge_dicts( + attrs, + {'author': 'Author Name'})), + ('Author (e-mail), no maintainer', merge_dicts( + attrs, + {'author': 'Author Name', + 'author_email': 'author@name.com'})), + ('No author, maintainer (no e-mail)', merge_dicts( + attrs, + {'maintainer': 'Maintainer Name'})), + ('No author, maintainer (e-mail)', merge_dicts( + attrs, + {'maintainer': 'Maintainer Name', + 'maintainer_email': 'maintainer@name.com'})), + ('Author (no e-mail), Maintainer (no-email)', merge_dicts( + attrs, + {'author': 'Author Name', + 'maintainer': 'Maintainer Name'})), + ('Author (e-mail), Maintainer (e-mail)', merge_dicts( + attrs, + {'author': 'Author Name', + 'author_email': 'author@name.com', + 'maintainer': 'Maintainer Name', + 'maintainer_email': 'maintainer@name.com'})), + ('No author (e-mail), no maintainer (e-mail)', merge_dicts( + attrs, + {'author_email': 'author@name.com', + 'maintainer_email': 'maintainer@name.com'})), + ('Author unicode', merge_dicts( + attrs, + {'author': '鉄沢寛'})), + ('Maintainer unicode', merge_dicts( + attrs, + {'maintainer': 'Jan Łukasiewicz'})), + ] + + return test_cases + + +@pytest.mark.parametrize('name,attrs', __maintainer_test_cases()) +def test_maintainer_author(name, attrs, tmpdir): + tested_keys = { + 'author': 'Author', + 'author_email': 'Author-email', + 'maintainer': 'Maintainer', + 'maintainer_email': 'Maintainer-email', + } + + # Generate a PKG-INFO file + dist = Distribution(attrs) + fn = tmpdir.mkdir('pkg_info') + fn_s = str(fn) + + dist.metadata.write_pkg_info(fn_s) + + with io.open(str(fn.join('PKG-INFO')), 'r', encoding='utf-8') as f: + raw_pkg_lines = f.readlines() + + # Drop blank lines + pkg_lines = list(filter(None, raw_pkg_lines)) + + pkg_lines_set = set(pkg_lines) + + # Duplicate lines should not be generated + assert len(pkg_lines) == len(pkg_lines_set) + + for fkey, dkey in tested_keys.items(): + val = attrs.get(dkey, None) + if val is None: + for line in pkg_lines: + assert not line.startswith(fkey + ':') + else: + line = '%s: %s' % (fkey, val) + assert line in pkg_lines_set diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index c9d396f4..2cf65ae7 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Easy install Tests """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import sys import os @@ -15,8 +15,9 @@ import distutils.errors import io import zipfile import mock - +from setuptools.command.easy_install import EasyInstallDeprecationWarning, ScriptWriter, WindowsScriptWriter import time +from setuptools.extern import six from setuptools.extern.six.moves import urllib import pytest @@ -33,16 +34,17 @@ import setuptools.tests.server from setuptools.tests import fail_on_ascii import pkg_resources -from .py26compat import tarfile_open from . import contexts from .textwrap import DALS +__metaclass__ = type + -class FakeDist(object): +class FakeDist: def get_entry_map(self, group): if group != 'console_scripts': return {} - return {'name': 'ep'} + return {str('name'): 'ep'} def as_requirement(self): return 'spec' @@ -78,7 +80,7 @@ class TestEasyInstallTest: sys.exit( load_entry_point('spec', 'console_scripts', 'name')() ) - """) + """) # noqa: E501 dist = FakeDist() args = next(ei.ScriptWriter.get_args(dist)) @@ -125,7 +127,9 @@ class TestEasyInstallTest: site.getsitepackages. """ path = normalize_path('/setuptools/test/site-packages') - mock_gsp = lambda: [path] + + def mock_gsp(): + return [path] monkeypatch.setattr(site, 'getsitepackages', mock_gsp, raising=False) assert path in ei.get_site_dirs() @@ -153,7 +157,7 @@ class TestEasyInstallTest: "", ), ( - u'mypkg/\u2603.txt', + 'mypkg/☃.txt', "", ), ] @@ -168,7 +172,8 @@ class TestEasyInstallTest: return str(sdist) @fail_on_ascii - def test_unicode_filename_in_sdist(self, sdist_unicode, tmpdir, monkeypatch): + def test_unicode_filename_in_sdist( + self, sdist_unicode, tmpdir, monkeypatch): """ The install command should execute correctly even if the package has unicode filenames. @@ -184,7 +189,122 @@ class TestEasyInstallTest: cmd.ensure_finalized() cmd.easy_install(sdist_unicode) + @pytest.fixture + def sdist_unicode_in_script(self, tmpdir): + files = [ + ( + "setup.py", + DALS(""" + import setuptools + setuptools.setup( + name="setuptools-test-unicode", + version="1.0", + packages=["mypkg"], + include_package_data=True, + scripts=['mypkg/unicode_in_script'], + ) + """), + ), + ("mypkg/__init__.py", ""), + ( + "mypkg/unicode_in_script", + DALS( + """ + #!/bin/sh + # á + + non_python_fn() { + } + """), + ), + ] + sdist_name = "setuptools-test-unicode-script-1.0.zip" + sdist = tmpdir / sdist_name + # can't use make_sdist, because the issue only occurs + # with zip sdists. + sdist_zip = zipfile.ZipFile(str(sdist), "w") + for filename, content in files: + sdist_zip.writestr(filename, content.encode('utf-8')) + sdist_zip.close() + return str(sdist) + + @fail_on_ascii + def test_unicode_content_in_sdist( + self, sdist_unicode_in_script, tmpdir, monkeypatch): + """ + The install command should execute correctly even if + the package has unicode in scripts. + """ + dist = Distribution({"script_args": ["easy_install"]}) + target = (tmpdir / "target").ensure_dir() + cmd = ei.easy_install(dist, install_dir=str(target), args=["x"]) + monkeypatch.setitem(os.environ, "PYTHONPATH", str(target)) + cmd.ensure_finalized() + cmd.easy_install(sdist_unicode_in_script) + + @pytest.fixture + def sdist_script(self, tmpdir): + files = [ + ( + 'setup.py', + DALS(""" + import setuptools + setuptools.setup( + name="setuptools-test-script", + version="1.0", + scripts=["mypkg_script"], + ) + """), + ), + ( + 'mypkg_script', + DALS(""" + #/usr/bin/python + print('mypkg_script') + """), + ), + ] + sdist_name = 'setuptools-test-script-1.0.zip' + sdist = str(tmpdir / sdist_name) + make_sdist(sdist, files) + return sdist + + @pytest.mark.skipif(not sys.platform.startswith('linux'), + reason="Test can only be run on Linux") + def test_script_install(self, sdist_script, tmpdir, monkeypatch): + """ + Check scripts are installed. + """ + dist = Distribution({'script_args': ['easy_install']}) + target = (tmpdir / 'target').ensure_dir() + cmd = ei.easy_install( + dist, + install_dir=str(target), + args=['x'], + ) + monkeypatch.setitem(os.environ, 'PYTHONPATH', str(target)) + cmd.ensure_finalized() + cmd.easy_install(sdist_script) + assert (target / 'mypkg_script').exists() + + + def test_dist_get_script_args_deprecated(self): + with pytest.warns(EasyInstallDeprecationWarning): + ScriptWriter.get_script_args(None, None) + + def test_dist_get_script_header_deprecated(self): + with pytest.warns(EasyInstallDeprecationWarning): + ScriptWriter.get_script_header("") + def test_dist_get_writer_deprecated(self): + with pytest.warns(EasyInstallDeprecationWarning): + ScriptWriter.get_writer(None) + + def test_dist_WindowsScriptWriter_get_writer_deprecated(self): + with pytest.warns(EasyInstallDeprecationWarning): + WindowsScriptWriter.get_writer() + +@pytest.mark.filterwarnings('ignore:Unbuilt egg') class TestPTHFileWriter: def test_add_from_cwd_site_sets_dirty(self): '''a pth file manager should set dirty @@ -352,8 +472,8 @@ class TestSetupRequires: dist_file, ] with sandbox.save_argv(['easy_install']): - # attempt to install the dist. It should fail because - # it doesn't exist. + # attempt to install the dist. It should + # fail because it doesn't exist. with pytest.raises(SystemExit): easy_install_pkg.main(ei_params) # there should have been two or three requests to the server @@ -381,7 +501,15 @@ class TestSetupRequires: """))]) yield dist_path - def test_setup_requires_overrides_version_conflict(self): + use_setup_cfg = ( + (), + ('dependency_links',), + ('setup_requires',), + ('dependency_links', 'setup_requires'), + ) + + @pytest.mark.parametrize('use_setup_cfg', use_setup_cfg) + def test_setup_requires_overrides_version_conflict(self, use_setup_cfg): """ Regression test for distribution issue 323: https://bitbucket.org/tarek/distribute/issues/323 @@ -397,18 +525,20 @@ class TestSetupRequires: with contexts.save_pkg_resources_state(): with contexts.tempdir() as temp_dir: - test_pkg = create_setup_requires_package(temp_dir) + test_pkg = create_setup_requires_package( + temp_dir, use_setup_cfg=use_setup_cfg) test_setup_py = os.path.join(test_pkg, 'setup.py') with contexts.quiet() as (stdout, stderr): # Don't even need to install the package, just # running the setup.py at all is sufficient - run_setup(test_setup_py, ['--name']) + run_setup(test_setup_py, [str('--name')]) lines = stdout.readlines() assert len(lines) > 0 - assert lines[-1].strip(), 'test_pkg' + assert lines[-1].strip() == 'test_pkg' - def test_setup_requires_override_nspkg(self): + @pytest.mark.parametrize('use_setup_cfg', use_setup_cfg) + def test_setup_requires_override_nspkg(self, use_setup_cfg): """ Like ``test_setup_requires_overrides_version_conflict`` but where the ``setup_requires`` package is part of a namespace package that has @@ -423,7 +553,7 @@ class TestSetupRequires: # extracted path to sys.path so foo.bar v0.1 is importable foobar_1_dir = os.path.join(temp_dir, 'foo.bar-0.1') os.mkdir(foobar_1_dir) - with tarfile_open(foobar_1_archive) as tf: + with tarfile.open(foobar_1_archive) as tf: tf.extractall(foobar_1_dir) sys.path.insert(1, foobar_1_dir) @@ -446,7 +576,8 @@ class TestSetupRequires: """) test_pkg = create_setup_requires_package( - temp_dir, 'foo.bar', '0.2', make_nspkg_sdist, template) + temp_dir, 'foo.bar', '0.2', make_nspkg_sdist, template, + use_setup_cfg=use_setup_cfg) test_setup_py = os.path.join(test_pkg, 'setup.py') @@ -454,9 +585,10 @@ class TestSetupRequires: try: # Don't even need to install the package, just # running the setup.py at all is sufficient - run_setup(test_setup_py, ['--name']) + run_setup(test_setup_py, [str('--name')]) except pkg_resources.VersionConflict: - self.fail('Installing setup.py requirements ' + self.fail( + 'Installing setup.py requirements ' 'caused a VersionConflict') assert 'FAIL' not in stdout.getvalue() @@ -464,6 +596,40 @@ class TestSetupRequires: assert len(lines) > 0 assert lines[-1].strip() == 'test_pkg' + @pytest.mark.parametrize('use_setup_cfg', use_setup_cfg) + def test_setup_requires_with_attr_version(self, use_setup_cfg): + def make_dependency_sdist(dist_path, distname, version): + files = [( + 'setup.py', + DALS(""" + import setuptools + setuptools.setup( + name={name!r}, + version={version!r}, + py_modules=[{name!r}], + ) + """.format(name=distname, version=version)), + ), ( + distname + '.py', + DALS(""" + version = 42 + """), + )] + make_sdist(dist_path, files) + with contexts.save_pkg_resources_state(): + with contexts.tempdir() as temp_dir: + test_pkg = create_setup_requires_package( + temp_dir, setup_attrs=dict(version='attr: foobar.version'), + make_package=make_dependency_sdist, + use_setup_cfg=use_setup_cfg+('version',), + ) + test_setup_py = os.path.join(test_pkg, 'setup.py') + with contexts.quiet() as (stdout, stderr): + run_setup(test_setup_py, [str('--version')]) + lines = stdout.readlines() + assert len(lines) > 0 + assert lines[-1].strip() == '42' + def make_trivial_sdist(dist_path, distname, version): """ @@ -521,7 +687,7 @@ def make_sdist(dist_path, files): listed in ``files`` as ``(filename, content)`` tuples. """ - with tarfile_open(dist_path, 'w:gz') as dist: + with tarfile.open(dist_path, 'w:gz') as dist: for filename, content in files: file_bytes = io.BytesIO(content.encode('utf-8')) file_info = tarfile.TarInfo(name=filename) @@ -532,7 +698,8 @@ def make_sdist(dist_path, files): def create_setup_requires_package(path, distname='foobar', version='0.1', make_package=make_trivial_sdist, - setup_py_template=None): + setup_py_template=None, setup_attrs={}, + use_setup_cfg=()): """Creates a source tree under path for a trivial test package that has a single requirement in setup_requires--a tarball for that requirement is also created and added to the dependency_links argument. @@ -547,17 +714,44 @@ def create_setup_requires_package(path, distname='foobar', version='0.1', 'setup_requires': ['%s==%s' % (distname, version)], 'dependency_links': [os.path.abspath(path)] } + test_setup_attrs.update(setup_attrs) test_pkg = os.path.join(path, 'test_pkg') - test_setup_py = os.path.join(test_pkg, 'setup.py') os.mkdir(test_pkg) + if use_setup_cfg: + test_setup_cfg = os.path.join(test_pkg, 'setup.cfg') + options = [] + metadata = [] + for name in use_setup_cfg: + value = test_setup_attrs.pop(name) + if name in 'name version'.split(): + section = metadata + else: + section = options + if isinstance(value, (tuple, list)): + value = ';'.join(value) + section.append('%s: %s' % (name, value)) + with open(test_setup_cfg, 'w') as f: + f.write(DALS( + """ + [metadata] + {metadata} + [options] + {options} + """ + ).format( + options='\n'.join(options), + metadata='\n'.join(metadata), + )) + + test_setup_py = os.path.join(test_pkg, 'setup.py') + if setup_py_template is None: setup_py_template = DALS("""\ import setuptools setuptools.setup(**%r) """) - with open(test_setup_py, 'w') as f: f.write(setup_py_template % test_setup_attrs) @@ -573,27 +767,31 @@ def create_setup_requires_package(path, distname='foobar', version='0.1', ) class TestScriptHeader: non_ascii_exe = '/Users/José/bin/python' - exe_with_spaces = r'C:\Program Files\Python33\python.exe' + if six.PY2: + non_ascii_exe = non_ascii_exe.encode('utf-8') + exe_with_spaces = r'C:\Program Files\Python36\python.exe' def test_get_script_header(self): expected = '#!%s\n' % ei.nt_quote_arg(os.path.normpath(sys.executable)) - actual = ei.ScriptWriter.get_script_header('#!/usr/local/bin/python') + actual = ei.ScriptWriter.get_header('#!/usr/local/bin/python') assert actual == expected def test_get_script_header_args(self): - expected = '#!%s -x\n' % ei.nt_quote_arg(os.path.normpath - (sys.executable)) - actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python -x') + expected = '#!%s -x\n' % ei.nt_quote_arg( + os.path.normpath(sys.executable)) + actual = ei.ScriptWriter.get_header('#!/usr/bin/python -x') assert actual == expected def test_get_script_header_non_ascii_exe(self): - actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', + actual = ei.ScriptWriter.get_header( + '#!/usr/bin/python', executable=self.non_ascii_exe) - expected = '#!%s -x\n' % self.non_ascii_exe + expected = str('#!%s -x\n') % self.non_ascii_exe assert actual == expected def test_get_script_header_exe_with_spaces(self): - actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', + actual = ei.ScriptWriter.get_header( + '#!/usr/bin/python', executable='"' + self.exe_with_spaces + '"') expected = '#!"%s"\n' % self.exe_with_spaces assert actual == expected @@ -636,7 +834,7 @@ class TestCommandSpec: class TestWindowsScriptWriter: def test_header(self): - hdr = ei.WindowsScriptWriter.get_script_header('') + hdr = ei.WindowsScriptWriter.get_header('') assert hdr.startswith('#!') assert hdr.endswith('\n') hdr = hdr.lstrip('#!') diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index 5196f32e..5eb81621 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -1,11 +1,13 @@ +import datetime +import sys import ast import os import glob import re import stat -import sys +import time -from setuptools.command.egg_info import egg_info, manifest_maker +from setuptools.command.egg_info import egg_info, manifest_maker, EggInfoDeprecationWarning, get_pkg_info_revision from setuptools.dist import Distribution from setuptools.extern.six.moves import map @@ -16,12 +18,14 @@ from .files import build_files from .textwrap import DALS from . import contexts +__metaclass__ = type + class Environment(str): pass -class TestEggInfo(object): +class TestEggInfo: setup_script = DALS(""" from setuptools import setup @@ -64,12 +68,6 @@ class TestEggInfo(object): }) yield env - dict_order_fails = pytest.mark.skipif( - sys.version_info < (2, 7), - reason="Intermittent failures on Python 2.6", - ) - - @dict_order_fails def test_egg_info_save_version_info_setup_empty(self, tmpdir_cwd, env): """ When the egg_info section is empty or not present, running @@ -104,7 +102,6 @@ class TestEggInfo(object): flags = re.MULTILINE | re.DOTALL assert re.search(pattern, content, flags) - @dict_order_fails def test_egg_info_save_version_info_setup_defaults(self, tmpdir_cwd, env): """ When running save_version_info on an existing setup.cfg @@ -135,11 +132,11 @@ class TestEggInfo(object): self._validate_content_order(content, expected_order) - def test_egg_base_installed_egg_info(self, tmpdir_cwd, env): + def test_expected_files_produced(self, tmpdir_cwd, env): self._create_project() - self._run_install_command(tmpdir_cwd, env) - actual = self._find_egg_info_files(env.paths['lib']) + self._run_egg_info_command(tmpdir_cwd, env) + actual = os.listdir('foo.egg-info') expected = [ 'PKG-INFO', @@ -151,6 +148,52 @@ class TestEggInfo(object): ] assert sorted(actual) == expected + def test_license_is_a_string(self, tmpdir_cwd, env): + setup_config = DALS(""" + [metadata] + name=foo + version=0.0.1 + license=file:MIT + """) + + setup_script = DALS(""" + from setuptools import setup + + setup() + """) + + build_files({'setup.py': setup_script, + 'setup.cfg': setup_config}) + + # This command should fail with a ValueError, but because it's + # currently configured to use a subprocess, the actual traceback + # object is lost and we need to parse it from stderr + with pytest.raises(AssertionError) as exc: + self._run_egg_info_command(tmpdir_cwd, env) + + # Hopefully this is not too fragile: the only argument to the + # assertion error should be a traceback, ending with: + # ValueError: .... + # + # assert not 1 + tb = exc.value.args[0].split('\n') + assert tb[-3].lstrip().startswith('ValueError') + + def test_rebuilt(self, tmpdir_cwd, env): + """Ensure timestamps are updated when the command is re-run.""" + self._create_project() + + self._run_egg_info_command(tmpdir_cwd, env) + timestamp_a = os.path.getmtime('foo.egg-info') + + # arbitrary sleep just to handle *really* fast systems + time.sleep(.001) + + self._run_egg_info_command(tmpdir_cwd, env) + timestamp_b = os.path.getmtime('foo.egg-info') + + assert timestamp_a != timestamp_b + def test_manifest_template_is_read(self, tmpdir_cwd, env): self._create_project() build_files({ @@ -161,8 +204,8 @@ class TestEggInfo(object): 'usage.rst': "Run 'hi'", } }) - self._run_install_command(tmpdir_cwd, env) - egg_info_dir = self._find_egg_info_files(env.paths['lib']).base + self._run_egg_info_command(tmpdir_cwd, env) + egg_info_dir = os.path.join('.', 'foo.egg-info') sources_txt = os.path.join(egg_info_dir, 'SOURCES.txt') with open(sources_txt) as f: assert 'docs/usage.rst' in f.read().split('\n') @@ -188,7 +231,7 @@ class TestEggInfo(object): ) invalid_marker = "<=>++" - class RequiresTestHelper(object): + class RequiresTestHelper: @staticmethod def parametrize(*test_list, **format_dict): @@ -198,7 +241,8 @@ class TestEggInfo(object): test_params = test.lstrip().split('\n\n', 3) name_kwargs = test_params.pop(0).split('\n') if len(name_kwargs) > 1: - install_cmd_kwargs = ast.literal_eval(name_kwargs[1].strip()) + val = name_kwargs[1].strip() + install_cmd_kwargs = ast.literal_eval(val) else: install_cmd_kwargs = {} name = name_kwargs[0].strip() @@ -218,9 +262,11 @@ class TestEggInfo(object): expected_requires, install_cmd_kwargs, marks=marks)) - return pytest.mark.parametrize('requires,use_setup_cfg,' - 'expected_requires,install_cmd_kwargs', - argvalues, ids=idlist) + return pytest.mark.parametrize( + 'requires,use_setup_cfg,' + 'expected_requires,install_cmd_kwargs', + argvalues, ids=idlist, + ) @RequiresTestHelper.parametrize( # Format of a test: @@ -235,6 +281,32 @@ class TestEggInfo(object): # expected contents of requires.txt ''' + install_requires_deterministic + + install_requires=["wheel>=0.5", "pytest"] + + [options] + install_requires = + wheel>=0.5 + pytest + + wheel>=0.5 + pytest + ''', + + ''' + install_requires_ordered + + install_requires=["pytest>=3.0.2,!=10.9999"] + + [options] + install_requires = + pytest>=3.0.2,!=10.9999 + + pytest!=10.9999,>=3.0.2 + ''', + + ''' install_requires_with_marker install_requires=["barbazquux;{mismatch_marker}"], @@ -368,11 +440,11 @@ class TestEggInfo(object): mismatch_marker=mismatch_marker, mismatch_marker_alternate=mismatch_marker_alternate, ) - def test_requires(self, tmpdir_cwd, env, - requires, use_setup_cfg, - expected_requires, install_cmd_kwargs): + def test_requires( + self, tmpdir_cwd, env, requires, use_setup_cfg, + expected_requires, install_cmd_kwargs): self._setup_script_with_requires(requires, use_setup_cfg) - self._run_install_command(tmpdir_cwd, env, **install_cmd_kwargs) + self._run_egg_info_command(tmpdir_cwd, env, **install_cmd_kwargs) egg_info_dir = os.path.join('.', 'foo.egg-info') requires_txt = os.path.join(egg_info_dir, 'requires.txt') if os.path.exists(requires_txt): @@ -383,12 +455,23 @@ class TestEggInfo(object): assert install_requires.lstrip() == expected_requires assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + def test_install_requires_unordered_disallowed(self, tmpdir_cwd, env): + """ + Packages that pass unordered install_requires sequences + should be rejected as they produce non-deterministic + builds. See #458. + """ + req = 'install_requires={"fake-factory==0.5.2", "pytz"}' + self._setup_script_with_requires(req) + with pytest.raises(AssertionError): + self._run_egg_info_command(tmpdir_cwd, env) + def test_extras_require_with_invalid_marker(self, tmpdir_cwd, env): tmpl = 'extras_require={{":{marker}": ["barbazquux"]}},' req = tmpl.format(marker=self.invalid_marker) self._setup_script_with_requires(req) with pytest.raises(AssertionError): - self._run_install_command(tmpdir_cwd, env) + self._run_egg_info_command(tmpdir_cwd, env) assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] def test_extras_require_with_invalid_marker_in_req(self, tmpdir_cwd, env): @@ -396,9 +479,44 @@ class TestEggInfo(object): req = tmpl.format(marker=self.invalid_marker) self._setup_script_with_requires(req) with pytest.raises(AssertionError): - self._run_install_command(tmpdir_cwd, env) + self._run_egg_info_command(tmpdir_cwd, env) assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == [] + def test_provides_extra(self, tmpdir_cwd, env): + self._setup_script_with_requires( + 'extras_require={"foobar": ["barbazquux"]},') + environ = os.environ.copy().update( + HOME=env.paths['home'], + ) + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + env=environ, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + assert 'Provides-Extra: foobar' in pkg_info_lines + assert 'Metadata-Version: 2.1' in pkg_info_lines + + def test_doesnt_provides_extra(self, tmpdir_cwd, env): + self._setup_script_with_requires( + '''install_requires=["spam ; python_version<'3.6'"]''') + environ = os.environ.copy().update( + HOME=env.paths['home'], + ) + environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + env=environ, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_text = pkginfo_file.read() + assert 'Provides-Extra:' not in pkg_info_text + def test_long_description_content_type(self, tmpdir_cwd, env): # Test that specifying a `long_description_content_type` keyword arg to # the `setup` function results in writing a `Description-Content-Type` @@ -423,6 +541,37 @@ class TestEggInfo(object): pkg_info_lines = pkginfo_file.read().split('\n') expected_line = 'Description-Content-Type: text/markdown' assert expected_line in pkg_info_lines + assert 'Metadata-Version: 2.1' in pkg_info_lines + + def test_project_urls(self, tmpdir_cwd, env): + # Test that specifying a `project_urls` dict to the `setup` + # function results in writing multiple `Project-URL` lines to + # the `PKG-INFO` file in the `<distribution>.egg-info` + # directory. + # `Project-URL` is described at https://packaging.python.org + # /specifications/core-metadata/#project-url-multiple-use + + self._setup_script_with_requires( + """project_urls={ + 'Link One': 'https://example.com/one/', + 'Link Two': 'https://example.com/two/', + },""") + environ = os.environ.copy().update( + HOME=env.paths['home'], + ) + code, data = environment.run_setup_py( + cmd=['egg_info'], + pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]), + data_stream=1, + env=environ, + ) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + expected_line = 'Project-URL: Link One, https://example.com/one/' + assert expected_line in pkg_info_lines + expected_line = 'Project-URL: Link Two, https://example.com/two/' + assert expected_line in pkg_info_lines def test_python_requires_egg_info(self, tmpdir_cwd, env): self._setup_script_with_requires( @@ -442,15 +591,6 @@ class TestEggInfo(object): assert 'Requires-Python: >=2.7.12' in pkg_info_lines assert 'Metadata-Version: 1.2' in pkg_info_lines - def test_python_requires_install(self, tmpdir_cwd, env): - self._setup_script_with_requires( - """python_requires='>=1.2.3',""") - self._run_install_command(tmpdir_cwd, env) - egg_info_dir = self._find_egg_info_files(env.paths['lib']).base - pkginfo = os.path.join(egg_info_dir, 'PKG-INFO') - with open(pkginfo) as f: - assert 'Requires-Python: >=1.2.3' in f.read().split('\n') - def test_manifest_maker_warning_suppression(self): fixtures = [ "standard file not found: should have one of foo.py, bar.py", @@ -460,17 +600,27 @@ class TestEggInfo(object): for msg in fixtures: assert manifest_maker._should_suppress_warning(msg) - def _run_install_command(self, tmpdir_cwd, env, cmd=None, output=None): + def test_egg_info_includes_setup_py(self, tmpdir_cwd): + self._create_project() + dist = Distribution({"name": "foo", "version": "0.0.1"}) + dist.script_name = "non_setup.py" + egg_info_instance = egg_info(dist) + egg_info_instance.finalize_options() + egg_info_instance.run() + + assert 'setup.py' in egg_info_instance.filelist.files + + with open(egg_info_instance.egg_info + "/SOURCES.txt") as f: + sources = f.read().split('\n') + assert 'setup.py' in sources + + def _run_egg_info_command(self, tmpdir_cwd, env, cmd=None, output=None): environ = os.environ.copy().update( HOME=env.paths['home'], ) if cmd is None: cmd = [ - 'install', - '--home', env.paths['home'], - '--install-lib', env.paths['lib'], - '--install-scripts', env.paths['scripts'], - '--install-data', env.paths['data'], + 'egg_info', ] code, data = environment.run_setup_py( cmd=cmd, @@ -478,25 +628,29 @@ class TestEggInfo(object): data_stream=1, env=environ, ) - if code: - raise AssertionError(data) + assert not code, data + if output: assert output in data - def _find_egg_info_files(self, root): - class DirList(list): - def __init__(self, files, base): - super(DirList, self).__init__(files) - self.base = base + def test_egg_info_tag_only_once(self, tmpdir_cwd, env): + self._create_project() + build_files({ + 'setup.cfg': DALS(""" + [egg_info] + tag_build = dev + tag_date = 0 + tag_svn_revision = 0 + """), + }) + self._run_egg_info_command(tmpdir_cwd, env) + egg_info_dir = os.path.join('.', 'foo.egg-info') + with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file: + pkg_info_lines = pkginfo_file.read().split('\n') + assert 'Version: 0.0.0.dev0' in pkg_info_lines - results = ( - DirList(filenames, dirpath) - for dirpath, dirnames, filenames in os.walk(root) - if os.path.basename(dirpath) == 'EGG-INFO' - ) - # expect exactly one result - result, = results - return result + def test_get_pkg_info_revision_deprecated(self): + pytest.warns(EggInfoDeprecationWarning, get_pkg_info_revision) EGG_INFO_TESTS = ( # Check for issue #1136: invalid string type when diff --git a/setuptools/tests/test_find_packages.py b/setuptools/tests/test_find_packages.py index a6023de9..b08f91c7 100644 --- a/setuptools/tests/test_find_packages.py +++ b/setuptools/tests/test_find_packages.py @@ -7,14 +7,15 @@ import platform import pytest -import setuptools -from setuptools import find_packages +from . import py3_only -find_420_packages = setuptools.PEP420PackageFinder.find +from setuptools.extern.six import PY3 +from setuptools import find_packages +if PY3: + from setuptools import find_namespace_packages # modeled after CPython's test.support.can_symlink - def can_symlink(): TESTFN = tempfile.mktemp() symlink_path = TESTFN + "can_symlink" @@ -153,30 +154,35 @@ class TestFindPackages: def _assert_packages(self, actual, expected): assert set(actual) == set(expected) + @py3_only def test_pep420_ns_package(self): - packages = find_420_packages( + packages = find_namespace_packages( self.dist_dir, include=['pkg*'], exclude=['pkg.subpkg.assets']) self._assert_packages(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg']) + @py3_only def test_pep420_ns_package_no_includes(self): - packages = find_420_packages( + packages = find_namespace_packages( self.dist_dir, exclude=['pkg.subpkg.assets']) self._assert_packages(packages, ['docs', 'pkg', 'pkg.nspkg', 'pkg.subpkg']) + @py3_only def test_pep420_ns_package_no_includes_or_excludes(self): - packages = find_420_packages(self.dist_dir) - expected = [ - 'docs', 'pkg', 'pkg.nspkg', 'pkg.subpkg', 'pkg.subpkg.assets'] + packages = find_namespace_packages(self.dist_dir) + expected = ['docs', 'pkg', 'pkg.nspkg', 'pkg.subpkg', 'pkg.subpkg.assets'] self._assert_packages(packages, expected) + @py3_only def test_regular_package_with_nested_pep420_ns_packages(self): self._touch('__init__.py', self.pkg_dir) - packages = find_420_packages( + packages = find_namespace_packages( self.dist_dir, exclude=['docs', 'pkg.subpkg.assets']) self._assert_packages(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg']) + @py3_only def test_pep420_ns_package_no_non_package_dirs(self): shutil.rmtree(self.docs_dir) shutil.rmtree(os.path.join(self.dist_dir, 'pkg/subpkg/assets')) - packages = find_420_packages(self.dist_dir) + packages = find_namespace_packages(self.dist_dir) self._assert_packages(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg']) + diff --git a/setuptools/tests/test_glibc.py b/setuptools/tests/test_glibc.py new file mode 100644 index 00000000..795fdc56 --- /dev/null +++ b/setuptools/tests/test_glibc.py @@ -0,0 +1,52 @@ +import warnings + +import pytest + +from setuptools.glibc import check_glibc_version + +__metaclass__ = type + + +@pytest.fixture(params=[ + "2.20", + # used by "linaro glibc", see gh-3588 + "2.20-2014.11", + # weird possibilities that I just made up + "2.20+dev", + "2.20-custom", + "2.20.1", + ]) +def two_twenty(request): + return request.param + + +@pytest.fixture(params=["asdf", "", "foo.bar"]) +def bad_string(request): + return request.param + + +class TestGlibc: + def test_manylinux1_check_glibc_version(self, two_twenty): + """ + Test that the check_glibc_version function is robust against weird + glibc version strings. + """ + assert check_glibc_version(two_twenty, 2, 15) + assert check_glibc_version(two_twenty, 2, 20) + assert not check_glibc_version(two_twenty, 2, 21) + assert not check_glibc_version(two_twenty, 3, 15) + assert not check_glibc_version(two_twenty, 1, 15) + + def test_bad_versions(self, bad_string): + """ + For unparseable strings, warn and return False + """ + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings("always") + assert not check_glibc_version(bad_string, 2, 5) + for w in ws: + if "Expected glibc version with" in str(w.message): + break + else: + # Didn't find the warning we were expecting + assert False diff --git a/setuptools/tests/test_glob.py b/setuptools/tests/test_glob.py new file mode 100644 index 00000000..a0728c5d --- /dev/null +++ b/setuptools/tests/test_glob.py @@ -0,0 +1,35 @@ +import pytest + +from setuptools.glob import glob + +from .files import build_files + + +@pytest.mark.parametrize('tree, pattern, matches', ( + ('', b'', []), + ('', '', []), + (''' + appveyor.yml + CHANGES.rst + LICENSE + MANIFEST.in + pyproject.toml + README.rst + setup.cfg + setup.py + ''', '*.rst', ('CHANGES.rst', 'README.rst')), + (''' + appveyor.yml + CHANGES.rst + LICENSE + MANIFEST.in + pyproject.toml + README.rst + setup.cfg + setup.py + ''', b'*.rst', (b'CHANGES.rst', b'README.rst')), +)) +def test_glob(monkeypatch, tmpdir, tree, pattern, matches): + monkeypatch.chdir(tmpdir) + build_files({name: '' for name in tree.split()}) + assert list(sorted(glob(pattern))) == list(sorted(matches)) diff --git a/setuptools/tests/test_install_scripts.py b/setuptools/tests/test_install_scripts.py index 7393241f..727ad65b 100644 --- a/setuptools/tests/test_install_scripts.py +++ b/setuptools/tests/test_install_scripts.py @@ -19,7 +19,7 @@ class TestInstallScripts: ) unix_exe = '/usr/dummy-test-path/local/bin/python' unix_spaces_exe = '/usr/bin/env dummy-test-python' - win32_exe = 'C:\\Dummy Test Path\\Program Files\\Python 3.3\\python.exe' + win32_exe = 'C:\\Dummy Test Path\\Program Files\\Python 3.6\\python.exe' def _run_install_scripts(self, install_dir, executable=None): dist = Distribution(self.settings) diff --git a/setuptools/tests/test_manifest.py b/setuptools/tests/test_manifest.py index 65eec7d9..5edfbea0 100644 --- a/setuptools/tests/test_manifest.py +++ b/setuptools/tests/test_manifest.py @@ -15,10 +15,11 @@ from setuptools.command.egg_info import FileList, egg_info, translate_pattern from setuptools.dist import Distribution from setuptools.extern import six from setuptools.tests.textwrap import DALS +from . import py3_only import pytest -py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") +__metaclass__ = type def make_local_path(s): @@ -157,7 +158,7 @@ def test_translated_pattern_mismatch(pattern_mismatch): assert not translate_pattern(pattern).match(target) -class TempDirTestCase(object): +class TempDirTestCase: def setup_method(self, method): self.temp_dir = tempfile.mkdtemp() self.old_cwd = os.getcwd() diff --git a/setuptools/tests/test_namespaces.py b/setuptools/tests/test_namespaces.py index 1ac1b35e..670ccee9 100644 --- a/setuptools/tests/test_namespaces.py +++ b/setuptools/tests/test_namespaces.py @@ -12,10 +12,10 @@ from setuptools.command import test class TestNamespaces: - @pytest.mark.xfail(sys.version_info < (3, 5), - reason="Requires importlib.util.module_from_spec") - @pytest.mark.skipif(bool(os.environ.get("APPVEYOR")), - reason="https://github.com/pypa/setuptools/issues/851") + @pytest.mark.skipif( + sys.version_info < (3, 5), + reason="Requires importlib.util.module_from_spec", + ) def test_mixed_site_and_non_site(self, tmpdir): """ Installing two packages sharing the same namespace, one installed @@ -55,8 +55,6 @@ class TestNamespaces: with test.test.paths_on_pythonpath(map(str, targets)): subprocess.check_call(try_import) - @pytest.mark.skipif(bool(os.environ.get("APPVEYOR")), - reason="https://github.com/pypa/setuptools/issues/851") def test_pkg_resources_import(self, tmpdir): """ Ensure that a namespace package doesn't break on import @@ -81,8 +79,6 @@ class TestNamespaces: with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(try_import) - @pytest.mark.skipif(bool(os.environ.get("APPVEYOR")), - reason="https://github.com/pypa/setuptools/issues/851") def test_namespace_package_installed_and_cwd(self, tmpdir): """ Installing a namespace packages but also having it in the current @@ -109,3 +105,32 @@ class TestNamespaces: ] with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(pkg_resources_imp, cwd=str(pkg_A)) + + def test_packages_in_the_same_namespace_installed_and_cwd(self, tmpdir): + """ + Installing one namespace package and also have another in the same + namespace in the current working directory, both of them must be + importable. + """ + pkg_A = namespaces.build_namespace_package(tmpdir, 'myns.pkgA') + pkg_B = namespaces.build_namespace_package(tmpdir, 'myns.pkgB') + target = tmpdir / 'packages' + # use pip to install to the target directory + install_cmd = [ + sys.executable, + '-m', + 'pip.__main__', + 'install', + str(pkg_A), + '-t', str(target), + ] + subprocess.check_call(install_cmd) + namespaces.make_site_dir(target) + + # ensure that all packages import and pkg_resources imports + pkg_resources_imp = [ + sys.executable, + '-c', 'import pkg_resources; import myns.pkgA; import myns.pkgB', + ] + with test.test.paths_on_pythonpath([str(target)]): + subprocess.check_call(pkg_resources_imp, cwd=str(pkg_B)) diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py index 53e20d44..13cffb7e 100644 --- a/setuptools/tests/test_packageindex.py +++ b/setuptools/tests/test_packageindex.py @@ -6,6 +6,8 @@ import distutils.errors from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client +import mock +import pytest import pkg_resources import setuptools.package_index @@ -223,6 +225,61 @@ class TestPackageIndex: assert dists[0].version == '' assert dists[1].version == vc + def test_download_git_with_rev(self, tmpdir): + url = 'git+https://github.example/group/project@master#egg=foo' + index = setuptools.package_index.PackageIndex() + + with mock.patch("os.system") as os_system_mock: + result = index.download(url, str(tmpdir)) + + os_system_mock.assert_called() + + expected_dir = str(tmpdir / 'project@master') + expected = ( + 'git clone --quiet ' + 'https://github.example/group/project {expected_dir}' + ).format(**locals()) + first_call_args = os_system_mock.call_args_list[0][0] + assert first_call_args == (expected,) + + tmpl = '(cd {expected_dir} && git checkout --quiet master)' + expected = tmpl.format(**locals()) + assert os_system_mock.call_args_list[1][0] == (expected,) + assert result == expected_dir + + def test_download_git_no_rev(self, tmpdir): + url = 'git+https://github.example/group/project#egg=foo' + index = setuptools.package_index.PackageIndex() + + with mock.patch("os.system") as os_system_mock: + result = index.download(url, str(tmpdir)) + + os_system_mock.assert_called() + + expected_dir = str(tmpdir / 'project') + expected = ( + 'git clone --quiet ' + 'https://github.example/group/project {expected_dir}' + ).format(**locals()) + os_system_mock.assert_called_once_with(expected) + + def test_download_svn(self, tmpdir): + url = 'svn+https://svn.example/project#egg=foo' + index = setuptools.package_index.PackageIndex() + + with pytest.warns(UserWarning): + with mock.patch("os.system") as os_system_mock: + result = index.download(url, str(tmpdir)) + + os_system_mock.assert_called() + + expected_dir = str(tmpdir / 'project') + expected = ( + 'svn checkout -q ' + 'svn+https://svn.example/project {expected_dir}' + ).format(**locals()) + os_system_mock.assert_called_once_with(expected) + class TestContentCheckers: def test_md5(self): @@ -265,11 +322,11 @@ class TestPyPIConfig: with pypirc.open('w') as strm: strm.write(DALS(""" [pypi] - repository=https://pypi.python.org + repository=https://pypi.org username=jaraco password=pity% """)) cfg = setuptools.package_index.PyPIConfig() - cred = cfg.creds_by_repository['https://pypi.python.org'] + cred = cfg.creds_by_repository['https://pypi.org'] assert cred.username == 'jaraco' assert cred.password == 'pity%' diff --git a/setuptools/tests/test_pep425tags.py b/setuptools/tests/test_pep425tags.py new file mode 100644 index 00000000..f558a0d8 --- /dev/null +++ b/setuptools/tests/test_pep425tags.py @@ -0,0 +1,168 @@ +import sys + +import pytest +from mock import patch + +from setuptools import pep425tags + +__metaclass__ = type + + +class TestPEP425Tags: + + def mock_get_config_var(self, **kwd): + """ + Patch sysconfig.get_config_var for arbitrary keys. + """ + get_config_var = pep425tags.sysconfig.get_config_var + + def _mock_get_config_var(var): + if var in kwd: + return kwd[var] + return get_config_var(var) + return _mock_get_config_var + + def abi_tag_unicode(self, flags, config_vars): + """ + Used to test ABI tags, verify correct use of the `u` flag + """ + config_vars.update({'SOABI': None}) + base = pep425tags.get_abbr_impl() + pep425tags.get_impl_ver() + + if sys.version_info < (3, 3): + config_vars.update({'Py_UNICODE_SIZE': 2}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('setuptools.pep425tags.sysconfig.get_config_var', mock_gcf): + abi_tag = pep425tags.get_abi_tag() + assert abi_tag == base + flags + + config_vars.update({'Py_UNICODE_SIZE': 4}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('setuptools.pep425tags.sysconfig.get_config_var', + mock_gcf): + abi_tag = pep425tags.get_abi_tag() + assert abi_tag == base + flags + 'u' + + else: + # On Python >= 3.3, UCS-4 is essentially permanently enabled, and + # Py_UNICODE_SIZE is None. SOABI on these builds does not include + # the 'u' so manual SOABI detection should not do so either. + config_vars.update({'Py_UNICODE_SIZE': None}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('setuptools.pep425tags.sysconfig.get_config_var', + mock_gcf): + abi_tag = pep425tags.get_abi_tag() + assert abi_tag == base + flags + + def test_broken_sysconfig(self): + """ + Test that pep425tags still works when sysconfig is broken. + Can be a problem on Python 2.7 + Issue #1074. + """ + def raises_ioerror(var): + raise IOError("I have the wrong path!") + + with patch('setuptools.pep425tags.sysconfig.get_config_var', + raises_ioerror): + with pytest.warns(RuntimeWarning): + assert len(pep425tags.get_supported()) + + def test_no_hyphen_tag(self): + """ + Test that no tag contains a hyphen. + """ + mock_gcf = self.mock_get_config_var(SOABI='cpython-35m-darwin') + + with patch('setuptools.pep425tags.sysconfig.get_config_var', + mock_gcf): + supported = pep425tags.get_supported() + + for (py, abi, plat) in supported: + assert '-' not in py + assert '-' not in abi + assert '-' not in plat + + def test_manual_abi_noflags(self): + """ + Test that no flags are set on a non-PyDebug, non-Pymalloc ABI tag. + """ + self.abi_tag_unicode('', {'Py_DEBUG': False, 'WITH_PYMALLOC': False}) + + def test_manual_abi_d_flag(self): + """ + Test that the `d` flag is set on a PyDebug, non-Pymalloc ABI tag. + """ + self.abi_tag_unicode('d', {'Py_DEBUG': True, 'WITH_PYMALLOC': False}) + + def test_manual_abi_m_flag(self): + """ + Test that the `m` flag is set on a non-PyDebug, Pymalloc ABI tag. + """ + self.abi_tag_unicode('m', {'Py_DEBUG': False, 'WITH_PYMALLOC': True}) + + def test_manual_abi_dm_flags(self): + """ + Test that the `dm` flags are set on a PyDebug, Pymalloc ABI tag. + """ + self.abi_tag_unicode('dm', {'Py_DEBUG': True, 'WITH_PYMALLOC': True}) + + +class TestManylinux1Tags: + + @patch('setuptools.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: True) + def test_manylinux1_compatible_on_linux_x86_64(self): + """ + Test that manylinux1 is enabled on linux_x86_64 + """ + assert pep425tags.is_manylinux1_compatible() + + @patch('setuptools.pep425tags.get_platform', lambda: 'linux_i686') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: True) + def test_manylinux1_compatible_on_linux_i686(self): + """ + Test that manylinux1 is enabled on linux_i686 + """ + assert pep425tags.is_manylinux1_compatible() + + @patch('setuptools.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: False) + def test_manylinux1_2(self): + """ + Test that manylinux1 is disabled with incompatible glibc + """ + assert not pep425tags.is_manylinux1_compatible() + + @patch('setuptools.pep425tags.get_platform', lambda: 'arm6vl') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: True) + def test_manylinux1_3(self): + """ + Test that manylinux1 is disabled on arm6vl + """ + assert not pep425tags.is_manylinux1_compatible() + + @patch('setuptools.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('setuptools.glibc.have_compatible_glibc', + lambda major, minor: True) + @patch('sys.platform', 'linux2') + def test_manylinux1_tag_is_first(self): + """ + Test that the more specific tag manylinux1 comes first. + """ + groups = {} + for pyimpl, abi, arch in pep425tags.get_supported(): + groups.setdefault((pyimpl, abi), []).append(arch) + + for arches in groups.values(): + if arches == ['any']: + continue + # Expect the most specific arch first: + if len(arches) == 3: + assert arches == ['manylinux1_x86_64', 'linux_x86_64', 'any'] + else: + assert arches == ['manylinux1_x86_64', 'linux_x86_64'] diff --git a/setuptools/tests/test_register.py b/setuptools/tests/test_register.py new file mode 100644 index 00000000..96114595 --- /dev/null +++ b/setuptools/tests/test_register.py @@ -0,0 +1,43 @@ +import mock +from distutils import log + +import pytest + +from setuptools.command.register import register +from setuptools.dist import Distribution + + +class TestRegisterTest: + def test_warns_deprecation(self): + dist = Distribution() + + cmd = register(dist) + cmd.run_command = mock.Mock() + cmd.send_metadata = mock.Mock() + cmd.announce = mock.Mock() + + cmd.run() + + cmd.announce.assert_called_with( + "WARNING: Registering is deprecated, use twine to upload instead " + "(https://pypi.org/p/twine/)", + log.WARN + ) + + def test_warns_deprecation_when_raising(self): + dist = Distribution() + + cmd = register(dist) + cmd.run_command = mock.Mock() + cmd.send_metadata = mock.Mock() + cmd.send_metadata.side_effect = Exception + cmd.announce = mock.Mock() + + with pytest.raises(Exception): + cmd.run() + + cmd.announce.assert_called_with( + "WARNING: Registering is deprecated, use twine to upload instead " + "(https://pypi.org/p/twine/)", + log.WARN + ) diff --git a/setuptools/tests/test_sandbox.py b/setuptools/tests/test_sandbox.py index a3f1206d..d8675422 100644 --- a/setuptools/tests/test_sandbox.py +++ b/setuptools/tests/test_sandbox.py @@ -75,6 +75,8 @@ class TestExceptionSaver: def test_unpickleable_exception(self): class CantPickleThis(Exception): "This Exception is unpickleable because it's not in globals" + def __repr__(self): + return 'CantPickleThis%r' % (self.args,) with setuptools.sandbox.ExceptionSaver() as saved_exc: raise CantPickleThis('detail') diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index 02222da5..d2c4e0cf 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -20,8 +20,8 @@ from setuptools.command.egg_info import manifest_maker from setuptools.dist import Distribution from setuptools.tests import fail_on_ascii from .text import Filenames +from . import py3_only -py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") SETUP_ATTRS = { 'name': 'sdist_test', @@ -92,9 +92,8 @@ fail_on_latin1_encoded_filenames = pytest.mark.xfail( class TestSdistTest: def setup_method(self, method): self.temp_dir = tempfile.mkdtemp() - f = open(os.path.join(self.temp_dir, 'setup.py'), 'w') - f.write(SETUP_PY) - f.close() + with open(os.path.join(self.temp_dir, 'setup.py'), 'w') as f: + f.write(SETUP_PY) # Set up the rest of the test package test_pkg = os.path.join(self.temp_dir, 'sdist_test') @@ -135,6 +134,47 @@ class TestSdistTest: assert os.path.join('sdist_test', 'c.rst') not in manifest assert os.path.join('d', 'e.dat') in manifest + def test_setup_py_exists(self): + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'foo.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + with quiet(): + cmd.run() + + manifest = cmd.filelist.files + assert 'setup.py' in manifest + + def test_setup_py_missing(self): + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'foo.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + if os.path.exists("setup.py"): + os.remove("setup.py") + with quiet(): + cmd.run() + + manifest = cmd.filelist.files + assert 'setup.py' not in manifest + + def test_setup_py_excluded(self): + with open("MANIFEST.in", "w") as manifest_file: + manifest_file.write("exclude setup.py") + + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'foo.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + with quiet(): + cmd.run() + + manifest = cmd.filelist.files + assert 'setup.py' not in manifest + def test_defaults_case_sensitivity(self): """ Make sure default files (README.*, etc.) are added in a case-sensitive diff --git a/setuptools/tests/test_setuptools.py b/setuptools/tests/test_setuptools.py index 26e37a6c..7aae3a16 100644 --- a/setuptools/tests/test_setuptools.py +++ b/setuptools/tests/test_setuptools.py @@ -210,6 +210,7 @@ class TestDistro: self.dist.exclude(package_dir=['q']) +@pytest.mark.filterwarnings('ignore:Features are deprecated') class TestFeatures: def setup_method(self, method): self.req = Require('Distutils', '1.0.3', 'distutils') diff --git a/setuptools/tests/test_test.py b/setuptools/tests/test_test.py index 7ea43c57..8d1425e1 100644 --- a/setuptools/tests/test_test.py +++ b/setuptools/tests/test_test.py @@ -2,9 +2,9 @@ from __future__ import unicode_literals +from distutils import log import os -import site -from distutils.errors import DistutilsError +import sys import pytest @@ -66,26 +66,62 @@ def sample_test(tmpdir_cwd): f.write(TEST_PY) -@pytest.mark.skipif('hasattr(sys, "real_prefix")') -@pytest.mark.usefixtures('user_override') -@pytest.mark.usefixtures('sample_test') -class TestTestTest: - def test_test(self): - params = dict( - name='foo', - packages=['name', 'name.space', 'name.space.tests'], - namespace_packages=['name'], - test_suite='name.space.tests.test_suite', - use_2to3=True, - ) - dist = Distribution(params) - dist.script_name = 'setup.py' - cmd = test(dist) - cmd.user = 1 - cmd.ensure_finalized() - cmd.install_dir = site.USER_SITE - cmd.user = 1 - with contexts.quiet(): - # The test runner calls sys.exit - with contexts.suppress_exceptions(SystemExit): - cmd.run() +@pytest.fixture +def quiet_log(): + # Running some of the other tests will automatically + # change the log level to info, messing our output. + log.set_verbosity(0) + + +@pytest.mark.usefixtures('sample_test', 'quiet_log') +def test_test(capfd): + params = dict( + name='foo', + packages=['name', 'name.space', 'name.space.tests'], + namespace_packages=['name'], + test_suite='name.space.tests.test_suite', + use_2to3=True, + ) + dist = Distribution(params) + dist.script_name = 'setup.py' + cmd = test(dist) + cmd.ensure_finalized() + # The test runner calls sys.exit + with contexts.suppress_exceptions(SystemExit): + cmd.run() + out, err = capfd.readouterr() + assert out == 'Foo\n' + + +@pytest.mark.usefixtures('tmpdir_cwd', 'quiet_log') +def test_tests_are_run_once(capfd): + params = dict( + name='foo', + packages=['dummy'], + ) + with open('setup.py', 'wt') as f: + f.write('from setuptools import setup; setup(\n') + for k, v in sorted(params.items()): + f.write(' %s=%r,\n' % (k, v)) + f.write(')\n') + os.makedirs('dummy') + with open('dummy/__init__.py', 'wt'): + pass + with open('dummy/test_dummy.py', 'wt') as f: + f.write(DALS( + """ + from __future__ import print_function + import unittest + class TestTest(unittest.TestCase): + def test_test(self): + print('Foo') + """)) + dist = Distribution(params) + dist.script_name = 'setup.py' + cmd = test(dist) + cmd.ensure_finalized() + # The test runner calls sys.exit + with contexts.suppress_exceptions(SystemExit): + cmd.run() + out, err = capfd.readouterr() + assert out == 'Foo\n' diff --git a/setuptools/tests/test_upload.py b/setuptools/tests/test_upload.py new file mode 100644 index 00000000..320c6959 --- /dev/null +++ b/setuptools/tests/test_upload.py @@ -0,0 +1,213 @@ +import mock +import os +import re + +from distutils import log +from distutils.errors import DistutilsError + +import pytest + +from setuptools.command.upload import upload +from setuptools.dist import Distribution +from setuptools.extern import six + + +def _parse_upload_body(body): + boundary = u'\r\n----------------GHSKFJDLGDS7543FJKLFHRE75642756743254' + entries = [] + name_re = re.compile(u'^Content-Disposition: form-data; name="([^\"]+)"') + + for entry in body.split(boundary): + pair = entry.split(u'\r\n\r\n') + if not len(pair) == 2: + continue + + key, value = map(six.text_type.strip, pair) + m = name_re.match(key) + if m is not None: + key = m.group(1) + + entries.append((key, value)) + + return entries + + +@pytest.fixture +def patched_upload(tmpdir): + class Fix: + def __init__(self, cmd, urlopen): + self.cmd = cmd + self.urlopen = urlopen + + def __iter__(self): + return iter((self.cmd, self.urlopen)) + + def get_uploaded_metadata(self): + request = self.urlopen.call_args_list[0][0][0] + body = request.data.decode('utf-8') + entries = dict(_parse_upload_body(body)) + + return entries + + class ResponseMock(mock.Mock): + def getheader(self, name, default=None): + """Mocked getheader method for response object""" + return { + 'content-type': 'text/plain; charset=utf-8', + }.get(name.lower(), default) + + with mock.patch('setuptools.command.upload.urlopen') as urlopen: + urlopen.return_value = ResponseMock() + urlopen.return_value.getcode.return_value = 200 + urlopen.return_value.read.return_value = b'' + + content = os.path.join(str(tmpdir), "content_data") + + with open(content, 'w') as f: + f.write("Some content") + + dist = Distribution() + dist.dist_files = [('sdist', '3.7.0', content)] + + cmd = upload(dist) + cmd.announce = mock.Mock() + cmd.username = 'user' + cmd.password = 'hunter2' + + yield Fix(cmd, urlopen) + + +class TestUploadTest: + def test_upload_metadata(self, patched_upload): + cmd, patch = patched_upload + + # Set the metadata version to 2.1 + cmd.distribution.metadata.metadata_version = '2.1' + + # Run the command + cmd.ensure_finalized() + cmd.run() + + # Make sure we did the upload + patch.assert_called_once() + + # Make sure the metadata version is correct in the headers + entries = patched_upload.get_uploaded_metadata() + assert entries['metadata_version'] == '2.1' + + def test_warns_deprecation(self): + dist = Distribution() + dist.dist_files = [(mock.Mock(), mock.Mock(), mock.Mock())] + + cmd = upload(dist) + cmd.upload_file = mock.Mock() + cmd.announce = mock.Mock() + + cmd.run() + + cmd.announce.assert_called_once_with( + "WARNING: Uploading via this command is deprecated, use twine to " + "upload instead (https://pypi.org/p/twine/)", + log.WARN + ) + + def test_warns_deprecation_when_raising(self): + dist = Distribution() + dist.dist_files = [(mock.Mock(), mock.Mock(), mock.Mock())] + + cmd = upload(dist) + cmd.upload_file = mock.Mock() + cmd.upload_file.side_effect = Exception + cmd.announce = mock.Mock() + + with pytest.raises(Exception): + cmd.run() + + cmd.announce.assert_called_once_with( + "WARNING: Uploading via this command is deprecated, use twine to " + "upload instead (https://pypi.org/p/twine/)", + log.WARN + ) + + @pytest.mark.parametrize('url', [ + 'https://example.com/a;parameter', # Has parameters + 'https://example.com/a?query', # Has query + 'https://example.com/a#fragment', # Has fragment + 'ftp://example.com', # Invalid scheme + + ]) + def test_upload_file_invalid_url(self, url, patched_upload): + patched_upload.urlopen.side_effect = Exception("Should not be reached") + + cmd = patched_upload.cmd + cmd.repository = url + + cmd.ensure_finalized() + with pytest.raises(AssertionError): + cmd.run() + + def test_upload_file_http_error(self, patched_upload): + patched_upload.urlopen.side_effect = six.moves.urllib.error.HTTPError( + 'https://example.com', + 404, + 'File not found', + None, + None + ) + + cmd = patched_upload.cmd + cmd.ensure_finalized() + + with pytest.raises(DistutilsError): + cmd.run() + + cmd.announce.assert_any_call( + 'Upload failed (404): File not found', + log.ERROR) + + def test_upload_file_os_error(self, patched_upload): + patched_upload.urlopen.side_effect = OSError("Invalid") + + cmd = patched_upload.cmd + cmd.ensure_finalized() + + with pytest.raises(OSError): + cmd.run() + + cmd.announce.assert_any_call('Invalid', log.ERROR) + + @mock.patch('setuptools.command.upload.spawn') + def test_upload_file_gpg(self, spawn, patched_upload): + cmd, urlopen = patched_upload + + cmd.sign = True + cmd.identity = "Alice" + cmd.dry_run = True + content_fname = cmd.distribution.dist_files[0][2] + signed_file = content_fname + '.asc' + + with open(signed_file, 'wb') as f: + f.write("signed-data".encode('utf-8')) + + cmd.ensure_finalized() + cmd.run() + + # Make sure that GPG was called + spawn.assert_called_once_with([ + "gpg", "--detach-sign", "--local-user", "Alice", "-a", + content_fname + ], dry_run=True) + + # Read the 'signed' data that was transmitted + entries = patched_upload.get_uploaded_metadata() + assert entries['gpg_signature'] == 'signed-data' + + def test_show_response_no_error(self, patched_upload): + # This test is just that show_response doesn't throw an error + # It is not really important what the printed response looks like + # in a deprecated command, but we don't want to introduce new + # errors when importing this function from distutils + + patched_upload.cmd.show_response = True + patched_upload.cmd.ensure_finalized() + patched_upload.cmd.run() diff --git a/setuptools/tests/test_virtualenv.py b/setuptools/tests/test_virtualenv.py index 9dbd3c86..7b5fea17 100644 --- a/setuptools/tests/test_virtualenv.py +++ b/setuptools/tests/test_virtualenv.py @@ -1,7 +1,9 @@ +import distutils.command import glob import os import sys +import pytest from pytest import yield_fixture from pytest_fixture_config import yield_requires_config @@ -11,6 +13,20 @@ from .textwrap import DALS from .test_easy_install import make_nspkg_sdist +@pytest.fixture(autouse=True) +def pytest_virtualenv_works(virtualenv): + """ + pytest_virtualenv may not work. if it doesn't, skip these + tests. See #1284. + """ + venv_prefix = virtualenv.run( + 'python -c "import sys; print(sys.prefix)"', + capture=True, + ).strip() + if venv_prefix == sys.prefix: + pytest.skip("virtualenv is broken (see pypa/setuptools#1284)") + + @yield_requires_config(pytest_virtualenv.CONFIG, ['virtualenv_executable']) @yield_fixture(scope='function') def bare_virtualenv(): @@ -26,6 +42,7 @@ def bare_virtualenv(): SOURCE_DIR = os.path.join(os.path.dirname(__file__), '../..') + def test_clean_env_install(bare_virtualenv): """ Check setuptools can be installed in a clean environment. @@ -35,14 +52,12 @@ def test_clean_env_install(bare_virtualenv): 'python setup.py install', )).format(source=SOURCE_DIR)) + def test_pip_upgrade_from_source(virtualenv): """ Check pip can upgrade setuptools from source. """ dist_dir = virtualenv.workspace - if sys.version_info < (2, 7): - # Python 2.6 support was dropped in wheel 0.30.0. - virtualenv.run('pip install -U "wheel<0.30.0"') # Generate source distribution / wheel. virtualenv.run(' && '.join(( 'cd {source}', @@ -56,6 +71,7 @@ def test_pip_upgrade_from_source(virtualenv): # And finally try to upgrade from source. virtualenv.run('pip install --no-cache-dir --upgrade ' + sdist) + def test_test_command_install_requirements(bare_virtualenv, tmpdir): """ Check the test command will install all required dependencies. @@ -64,6 +80,7 @@ def test_test_command_install_requirements(bare_virtualenv, tmpdir): 'cd {source}', 'python setup.py develop', )).format(source=SOURCE_DIR)) + def sdist(distname, version): dist_path = tmpdir.join('%s-%s.tar.gz' % (distname, version)) make_nspkg_sdist(str(dist_path), distname, version) @@ -118,3 +135,14 @@ def test_test_command_install_requirements(bare_virtualenv, tmpdir): 'python setup.py test -s test', )).format(tmpdir=tmpdir)) assert tmpdir.join('success').check() + + +def test_no_missing_dependencies(bare_virtualenv): + """ + Quick and dirty test to ensure all external dependencies are vendored. + """ + for command in ('upload',):#sorted(distutils.command.__all__): + bare_virtualenv.run(' && '.join(( + 'cd {source}', + 'python setup.py {command} -h', + )).format(command=command, source=SOURCE_DIR)) diff --git a/setuptools/tests/test_wheel.py b/setuptools/tests/test_wheel.py new file mode 100644 index 00000000..6db5fa11 --- /dev/null +++ b/setuptools/tests/test_wheel.py @@ -0,0 +1,543 @@ +# -*- coding: utf-8 -*- + +"""wheel tests +""" + +from distutils.sysconfig import get_config_var +from distutils.util import get_platform +import contextlib +import glob +import inspect +import os +import shutil +import subprocess +import sys +import zipfile + +import pytest + +from pkg_resources import Distribution, PathMetadata, PY_MAJOR +from setuptools.extern.packaging.utils import canonicalize_name +from setuptools.wheel import Wheel + +from .contexts import tempdir +from .files import build_files +from .textwrap import DALS + +__metaclass__ = type + + +WHEEL_INFO_TESTS = ( + ('invalid.whl', ValueError), + ('simplewheel-2.0-1-py2.py3-none-any.whl', { + 'project_name': 'simplewheel', + 'version': '2.0', + 'build': '1', + 'py_version': 'py2.py3', + 'abi': 'none', + 'platform': 'any', + }), + ('simple.dist-0.1-py2.py3-none-any.whl', { + 'project_name': 'simple.dist', + 'version': '0.1', + 'build': None, + 'py_version': 'py2.py3', + 'abi': 'none', + 'platform': 'any', + }), + ('example_pkg_a-1-py3-none-any.whl', { + 'project_name': 'example_pkg_a', + 'version': '1', + 'build': None, + 'py_version': 'py3', + 'abi': 'none', + 'platform': 'any', + }), + ('PyQt5-5.9-5.9.1-cp35.cp36.cp37-abi3-manylinux1_x86_64.whl', { + 'project_name': 'PyQt5', + 'version': '5.9', + 'build': '5.9.1', + 'py_version': 'cp35.cp36.cp37', + 'abi': 'abi3', + 'platform': 'manylinux1_x86_64', + }), +) + +@pytest.mark.parametrize( + ('filename', 'info'), WHEEL_INFO_TESTS, + ids=[t[0] for t in WHEEL_INFO_TESTS] +) +def test_wheel_info(filename, info): + if inspect.isclass(info): + with pytest.raises(info): + Wheel(filename) + return + w = Wheel(filename) + assert {k: getattr(w, k) for k in info.keys()} == info + + +@contextlib.contextmanager +def build_wheel(extra_file_defs=None, **kwargs): + file_defs = { + 'setup.py': (DALS( + ''' + # -*- coding: utf-8 -*- + from setuptools import setup + import setuptools + setup(**%r) + ''' + ) % kwargs).encode('utf-8'), + } + if extra_file_defs: + file_defs.update(extra_file_defs) + with tempdir() as source_dir: + build_files(file_defs, source_dir) + subprocess.check_call((sys.executable, 'setup.py', + '-q', 'bdist_wheel'), cwd=source_dir) + yield glob.glob(os.path.join(source_dir, 'dist', '*.whl'))[0] + + +def tree_set(root): + contents = set() + for dirpath, dirnames, filenames in os.walk(root): + for filename in filenames: + contents.add(os.path.join(os.path.relpath(dirpath, root), + filename)) + return contents + + +def flatten_tree(tree): + """Flatten nested dicts and lists into a full list of paths""" + output = set() + for node, contents in tree.items(): + if isinstance(contents, dict): + contents = flatten_tree(contents) + + for elem in contents: + if isinstance(elem, dict): + output |= {os.path.join(node, val) + for val in flatten_tree(elem)} + else: + output.add(os.path.join(node, elem)) + return output + + +def format_install_tree(tree): + return {x.format( + py_version=PY_MAJOR, + platform=get_platform(), + shlib_ext=get_config_var('EXT_SUFFIX') or get_config_var('SO')) + for x in tree} + + +def _check_wheel_install(filename, install_dir, install_tree_includes, + project_name, version, requires_txt): + w = Wheel(filename) + egg_path = os.path.join(install_dir, w.egg_name()) + w.install_as_egg(egg_path) + if install_tree_includes is not None: + install_tree = format_install_tree(install_tree_includes) + exp = tree_set(install_dir) + assert install_tree.issubset(exp), (install_tree - exp) + + metadata = PathMetadata(egg_path, os.path.join(egg_path, 'EGG-INFO')) + dist = Distribution.from_filename(egg_path, metadata=metadata) + assert dist.project_name == project_name + assert dist.version == version + if requires_txt is None: + assert not dist.has_metadata('requires.txt') + else: + assert requires_txt == dist.get_metadata('requires.txt').lstrip() + + +class Record: + + def __init__(self, id, **kwargs): + self._id = id + self._fields = kwargs + + def __repr__(self): + return '%s(**%r)' % (self._id, self._fields) + + +WHEEL_INSTALL_TESTS = ( + + dict( + id='basic', + file_defs={ + 'foo': { + '__init__.py': '' + } + }, + setup_kwargs=dict( + packages=['foo'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt' + ], + 'foo': ['__init__.py'] + } + }), + ), + + dict( + id='utf-8', + setup_kwargs=dict( + description='Description accentuée', + ) + ), + + dict( + id='data', + file_defs={ + 'data.txt': DALS( + ''' + Some data... + ''' + ), + }, + setup_kwargs=dict( + data_files=[('data_dir', ['data.txt'])], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt' + ], + 'data_dir': [ + 'data.txt' + ] + } + }), + ), + + dict( + id='extension', + file_defs={ + 'extension.c': DALS( + ''' + #include "Python.h" + + #if PY_MAJOR_VERSION >= 3 + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "extension", + NULL, + 0, + NULL, + NULL, + NULL, + NULL, + NULL + }; + + #define INITERROR return NULL + + PyMODINIT_FUNC PyInit_extension(void) + + #else + + #define INITERROR return + + void initextension(void) + + #endif + { + #if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); + #else + PyObject *module = Py_InitModule("extension", NULL); + #endif + if (module == NULL) + INITERROR; + #if PY_MAJOR_VERSION >= 3 + return module; + #endif + } + ''' + ), + }, + setup_kwargs=dict( + ext_modules=[ + Record('setuptools.Extension', + name='extension', + sources=['extension.c']) + ], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}-{platform}.egg': [ + 'extension{shlib_ext}', + {'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt', + ]}, + ] + }), + ), + + dict( + id='header', + file_defs={ + 'header.h': DALS( + ''' + ''' + ), + }, + setup_kwargs=dict( + headers=['header.h'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': [ + 'header.h', + {'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt', + ]}, + ] + }), + ), + + dict( + id='script', + file_defs={ + 'script.py': DALS( + ''' + #/usr/bin/python + print('hello world!') + ''' + ), + 'script.sh': DALS( + ''' + #/bin/sh + echo 'hello world!' + ''' + ), + }, + setup_kwargs=dict( + scripts=['script.py', 'script.sh'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt', + {'scripts': [ + 'script.py', + 'script.sh' + ]} + + ] + } + }) + ), + + dict( + id='requires1', + install_requires='foobar==2.0', + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'requires.txt', + 'top_level.txt', + ] + } + }), + requires_txt=DALS( + ''' + foobar==2.0 + ''' + ), + ), + + dict( + id='requires2', + install_requires=''' + bar + foo<=2.0; %r in sys_platform + ''' % sys.platform, + requires_txt=DALS( + ''' + bar + foo<=2.0 + ''' + ), + ), + + dict( + id='requires3', + install_requires=''' + bar; %r != sys_platform + ''' % sys.platform, + ), + + dict( + id='requires4', + install_requires=''' + foo + ''', + extras_require={ + 'extra': 'foobar>3', + }, + requires_txt=DALS( + ''' + foo + + [extra] + foobar>3 + ''' + ), + ), + + dict( + id='requires5', + extras_require={ + 'extra': 'foobar; %r != sys_platform' % sys.platform, + }, + requires_txt=DALS( + ''' + [extra] + ''' + ), + ), + + dict( + id='namespace_package', + file_defs={ + 'foo': { + 'bar': { + '__init__.py': '' + }, + }, + }, + setup_kwargs=dict( + namespace_packages=['foo'], + packages=['foo.bar'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': [ + 'foo-1.0-py{py_version}-nspkg.pth', + {'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'namespace_packages.txt', + 'top_level.txt', + ]}, + {'foo': [ + '__init__.py', + {'bar': ['__init__.py']}, + ]}, + ] + }), + ), + + dict( + id='data_in_package', + file_defs={ + 'foo': { + '__init__.py': '', + 'data_dir': { + 'data.txt': DALS( + ''' + Some data... + ''' + ), + } + } + }, + setup_kwargs=dict( + packages=['foo'], + data_files=[('foo/data_dir', ['foo/data_dir/data.txt'])], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': { + 'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'top_level.txt', + ], + 'foo': [ + '__init__.py', + {'data_dir': [ + 'data.txt', + ]} + ] + } + }), + ), + +) + +@pytest.mark.parametrize( + 'params', WHEEL_INSTALL_TESTS, + ids=list(params['id'] for params in WHEEL_INSTALL_TESTS), +) +def test_wheel_install(params): + project_name = params.get('name', 'foo') + version = params.get('version', '1.0') + install_requires = params.get('install_requires', []) + extras_require = params.get('extras_require', {}) + requires_txt = params.get('requires_txt', None) + install_tree = params.get('install_tree') + file_defs = params.get('file_defs', {}) + setup_kwargs = params.get('setup_kwargs', {}) + with build_wheel( + name=project_name, + version=version, + install_requires=install_requires, + extras_require=extras_require, + extra_file_defs=file_defs, + **setup_kwargs + ) as filename, tempdir() as install_dir: + _check_wheel_install(filename, install_dir, + install_tree, project_name, + version, requires_txt) + + +def test_wheel_install_pep_503(): + project_name = 'Foo_Bar' # PEP 503 canonicalized name is "foo-bar" + version = '1.0' + with build_wheel( + name=project_name, + version=version, + ) as filename, tempdir() as install_dir: + new_filename = filename.replace(project_name, + canonicalize_name(project_name)) + shutil.move(filename, new_filename) + _check_wheel_install(new_filename, install_dir, None, + canonicalize_name(project_name), + version, None) + + +def test_wheel_no_dist_dir(): + project_name = 'nodistinfo' + version = '1.0' + wheel_name = '{0}-{1}-py2.py3-none-any.whl'.format(project_name, version) + with tempdir() as source_dir: + wheel_path = os.path.join(source_dir, wheel_name) + # create an empty zip file + zipfile.ZipFile(wheel_path, 'w').close() + with tempdir() as install_dir: + with pytest.raises(ValueError): + _check_wheel_install(wheel_path, install_dir, None, + project_name, + version, None) diff --git a/setuptools/wheel.py b/setuptools/wheel.py new file mode 100644 index 00000000..e11f0a1d --- /dev/null +++ b/setuptools/wheel.py @@ -0,0 +1,211 @@ +"""Wheels support.""" + +from distutils.util import get_platform +import email +import itertools +import os +import posixpath +import re +import zipfile + +import pkg_resources +import setuptools +from pkg_resources import parse_version +from setuptools.extern.packaging.utils import canonicalize_name +from setuptools.extern.six import PY3 +from setuptools import pep425tags +from setuptools.command.egg_info import write_requirements + + +__metaclass__ = type + + +WHEEL_NAME = re.compile( + r"""^(?P<project_name>.+?)-(?P<version>\d.*?) + ((-(?P<build>\d.*?))?-(?P<py_version>.+?)-(?P<abi>.+?)-(?P<platform>.+?) + )\.whl$""", + re.VERBOSE).match + +NAMESPACE_PACKAGE_INIT = '''\ +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + __path__ = __import__('pkgutil').extend_path(__path__, __name__) +''' + + +def unpack(src_dir, dst_dir): + '''Move everything under `src_dir` to `dst_dir`, and delete the former.''' + for dirpath, dirnames, filenames in os.walk(src_dir): + subdir = os.path.relpath(dirpath, src_dir) + for f in filenames: + src = os.path.join(dirpath, f) + dst = os.path.join(dst_dir, subdir, f) + os.renames(src, dst) + for n, d in reversed(list(enumerate(dirnames))): + src = os.path.join(dirpath, d) + dst = os.path.join(dst_dir, subdir, d) + if not os.path.exists(dst): + # Directory does not exist in destination, + # rename it and prune it from os.walk list. + os.renames(src, dst) + del dirnames[n] + # Cleanup. + for dirpath, dirnames, filenames in os.walk(src_dir, topdown=True): + assert not filenames + os.rmdir(dirpath) + + +class Wheel: + + def __init__(self, filename): + match = WHEEL_NAME(os.path.basename(filename)) + if match is None: + raise ValueError('invalid wheel name: %r' % filename) + self.filename = filename + for k, v in match.groupdict().items(): + setattr(self, k, v) + + def tags(self): + '''List tags (py_version, abi, platform) supported by this wheel.''' + return itertools.product( + self.py_version.split('.'), + self.abi.split('.'), + self.platform.split('.'), + ) + + def is_compatible(self): + '''Is the wheel is compatible with the current platform?''' + supported_tags = pep425tags.get_supported() + return next((True for t in self.tags() if t in supported_tags), False) + + def egg_name(self): + return pkg_resources.Distribution( + project_name=self.project_name, version=self.version, + platform=(None if self.platform == 'any' else get_platform()), + ).egg_name() + '.egg' + + def get_dist_info(self, zf): + # find the correct name of the .dist-info dir in the wheel file + for member in zf.namelist(): + dirname = posixpath.dirname(member) + if (dirname.endswith('.dist-info') and + canonicalize_name(dirname).startswith( + canonicalize_name(self.project_name))): + return dirname + raise ValueError("unsupported wheel format. .dist-info not found") + + def install_as_egg(self, destination_eggdir): + '''Install wheel as an egg directory.''' + with zipfile.ZipFile(self.filename) as zf: + self._install_as_egg(destination_eggdir, zf) + + def _install_as_egg(self, destination_eggdir, zf): + dist_basename = '%s-%s' % (self.project_name, self.version) + dist_info = self.get_dist_info(zf) + dist_data = '%s.data' % dist_basename + egg_info = os.path.join(destination_eggdir, 'EGG-INFO') + + self._convert_metadata(zf, destination_eggdir, dist_info, egg_info) + self._move_data_entries(destination_eggdir, dist_data) + self._fix_namespace_packages(egg_info, destination_eggdir) + + @staticmethod + def _convert_metadata(zf, destination_eggdir, dist_info, egg_info): + def get_metadata(name): + with zf.open(posixpath.join(dist_info, name)) as fp: + value = fp.read().decode('utf-8') if PY3 else fp.read() + return email.parser.Parser().parsestr(value) + + wheel_metadata = get_metadata('WHEEL') + # Check wheel format version is supported. + wheel_version = parse_version(wheel_metadata.get('Wheel-Version')) + wheel_v1 = ( + parse_version('1.0') <= wheel_version < parse_version('2.0dev0') + ) + if not wheel_v1: + raise ValueError( + 'unsupported wheel format version: %s' % wheel_version) + # Extract to target directory. + os.mkdir(destination_eggdir) + zf.extractall(destination_eggdir) + # Convert metadata. + dist_info = os.path.join(destination_eggdir, dist_info) + dist = pkg_resources.Distribution.from_location( + destination_eggdir, dist_info, + metadata=pkg_resources.PathMetadata(destination_eggdir, dist_info), + ) + + # Note: Evaluate and strip markers now, + # as it's difficult to convert back from the syntax: + # foobar; "linux" in sys_platform and extra == 'test' + def raw_req(req): + req.marker = None + return str(req) + install_requires = list(sorted(map(raw_req, dist.requires()))) + extras_require = { + extra: sorted( + req + for req in map(raw_req, dist.requires((extra,))) + if req not in install_requires + ) + for extra in dist.extras + } + os.rename(dist_info, egg_info) + os.rename( + os.path.join(egg_info, 'METADATA'), + os.path.join(egg_info, 'PKG-INFO'), + ) + setup_dist = setuptools.Distribution( + attrs=dict( + install_requires=install_requires, + extras_require=extras_require, + ), + ) + write_requirements( + setup_dist.get_command_obj('egg_info'), + None, + os.path.join(egg_info, 'requires.txt'), + ) + + @staticmethod + def _move_data_entries(destination_eggdir, dist_data): + """Move data entries to their correct location.""" + dist_data = os.path.join(destination_eggdir, dist_data) + dist_data_scripts = os.path.join(dist_data, 'scripts') + if os.path.exists(dist_data_scripts): + egg_info_scripts = os.path.join( + destination_eggdir, 'EGG-INFO', 'scripts') + os.mkdir(egg_info_scripts) + for entry in os.listdir(dist_data_scripts): + # Remove bytecode, as it's not properly handled + # during easy_install scripts install phase. + if entry.endswith('.pyc'): + os.unlink(os.path.join(dist_data_scripts, entry)) + else: + os.rename( + os.path.join(dist_data_scripts, entry), + os.path.join(egg_info_scripts, entry), + ) + os.rmdir(dist_data_scripts) + for subdir in filter(os.path.exists, ( + os.path.join(dist_data, d) + for d in ('data', 'headers', 'purelib', 'platlib') + )): + unpack(subdir, destination_eggdir) + if os.path.exists(dist_data): + os.rmdir(dist_data) + + @staticmethod + def _fix_namespace_packages(egg_info, destination_eggdir): + namespace_packages = os.path.join( + egg_info, 'namespace_packages.txt') + if os.path.exists(namespace_packages): + with open(namespace_packages) as fp: + namespace_packages = fp.read().split() + for mod in namespace_packages: + mod_dir = os.path.join(destination_eggdir, *mod.split('.')) + mod_init = os.path.join(mod_dir, '__init__.py') + if os.path.exists(mod_dir) and not os.path.exists(mod_init): + with open(mod_init, 'w') as fp: + fp.write(NAMESPACE_PACKAGE_INIT) diff --git a/tests/manual_test.py b/tests/manual_test.py index e5aaf179..99db4b01 100644 --- a/tests/manual_test.py +++ b/tests/manual_test.py @@ -8,7 +8,7 @@ import subprocess from distutils.command.install import INSTALL_SCHEMES from string import Template -from six.moves import urllib +from setuptools.extern.six.moves import urllib def _system_call(*args): @@ -89,8 +89,10 @@ def test_full(): assert len(eggs) == 3 assert eggs[1].startswith('setuptools') del eggs[1] - assert eggs == ['extensions-0.3-py2.6.egg', - 'zc.recipe.egg-1.2.2-py2.6.egg'] + assert eggs == [ + 'extensions-0.3-py2.6.egg', + 'zc.recipe.egg-1.2.2-py2.6.egg', + ] if __name__ == '__main__': diff --git a/tests/requirements.txt b/tests/requirements.txt index 4761505f..d301168a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,12 @@ -importlib; python_version<"2.7" mock -pytest-flake8; python_version>="2.7" +pytest-flake8; python_version!="3.4" +pytest-flake8<=1.0.0; python_version=="3.4" virtualenv>=13.0.0 pytest-virtualenv>=1.2.7 -pytest>=3.0.2 +# pytest pinned to <4 due to #1638 +pytest>=3.7,<4 +wheel +coverage>=4.5.1 +pytest-cov>=2.5.1 +paver; python_version>="3.6" +futures; python_version=="2.7" diff --git a/tests/test_pypi.py b/tests/test_pypi.py deleted file mode 100644 index b3425e53..00000000 --- a/tests/test_pypi.py +++ /dev/null @@ -1,82 +0,0 @@ -import os -import subprocess - -import virtualenv -from setuptools.extern.six.moves import http_client -from setuptools.extern.six.moves import xmlrpc_client - -TOP = 200 -PYPI_HOSTNAME = 'pypi.python.org' - - -def rpc_pypi(method, *args): - """Call an XML-RPC method on the Pypi server.""" - conn = http_client.HTTPSConnection(PYPI_HOSTNAME) - headers = {'Content-Type': 'text/xml'} - payload = xmlrpc_client.dumps(args, method) - - conn.request("POST", "/pypi", payload, headers) - response = conn.getresponse() - if response.status == 200: - result = xmlrpc_client.loads(response.read())[0][0] - return result - else: - raise RuntimeError("Unable to download the list of top " - "packages from Pypi.") - - -def get_top_packages(limit): - """Collect the name of the top packages on Pypi.""" - packages = rpc_pypi('top_packages') - return packages[:limit] - - -def _package_install(package_name, tmp_dir=None, local_setuptools=True): - """Try to install a package and return the exit status. - - This function creates a virtual environment, install setuptools using pip - and then install the required package. If local_setuptools is True, it - will install the local version of setuptools. - """ - package_dir = os.path.join(tmp_dir, "test_%s" % package_name) - if not local_setuptools: - package_dir = package_dir + "_baseline" - - virtualenv.create_environment(package_dir) - - pip_path = os.path.join(package_dir, "bin", "pip") - if local_setuptools: - subprocess.check_call([pip_path, "install", "."]) - returncode = subprocess.call([pip_path, "install", package_name]) - return returncode - - -def test_package_install(package_name, tmpdir): - """Test to verify the outcome of installing a package. - - This test compare that the return code when installing a package is the - same as with the current stable version of setuptools. - """ - new_exit_status = _package_install(package_name, tmp_dir=str(tmpdir)) - if new_exit_status: - print("Installation failed, testing against stable setuptools", - package_name) - old_exit_status = _package_install(package_name, tmp_dir=str(tmpdir), - local_setuptools=False) - assert new_exit_status == old_exit_status - - -def pytest_generate_tests(metafunc): - """Generator function for test_package_install. - - This function will generate calls to test_package_install. If a package - list has been specified on the command line, it will be used. Otherwise, - Pypi will be queried to get the current list of top packages. - """ - if "package_name" in metafunc.fixturenames: - if not metafunc.config.option.package_name: - packages = get_top_packages(TOP) - packages = [name for name, downloads in packages] - else: - packages = metafunc.config.option.package_name - metafunc.parametrize("package_name", packages) diff --git a/towncrier_template.rst b/towncrier_template.rst new file mode 100644 index 00000000..fbc5ef03 --- /dev/null +++ b/towncrier_template.rst @@ -0,0 +1,27 @@ +{% for section, _ in sections.items() %} +{% set underline = underlines[0] %}{% if section %}{{section}} +{{ underline * section|length }} +{% endif %} + +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section]%} +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category].items() %} +* {{ values|join(', ') }}: {{ text }} +{% endfor %} +{% else %} +* {{ sections[section][category]['']|join(', ') }} + +{% endif %} +{% if sections[section][category]|length == 0 %} +No significant changes. +{% else %} +{% endif %} +{% endfor %} + +{% else %} +No significant changes. + + +{% endif %} +{% endfor %} @@ -2,10 +2,54 @@ # # To run Tox against all supported Python interpreters, you can set: # -# export TOXENV='py2{6,7},py3{3,4,5,6},pypy' +# export TOXENV='py27,py3{4,5,6},pypy,pypy3' + +[tox] +envlist=python [testenv] deps=-rtests/requirements.txt -passenv=APPDATA USERPROFILE HOMEDRIVE HOMEPATH windir APPVEYOR -commands=py.test {posargs} +# Changed from default (`python -m pip ...`) +# to prevent the current working directory +# from being added to `sys.path`. +install_command={envbindir}/pip install {opts} {packages} +# Same as above. +list_dependencies_command={envbindir}/pip freeze +setenv=COVERAGE_FILE={toxworkdir}/.coverage.{envname} +# TODO: The passed environment variables came from copying other tox.ini files +# These should probably be individually annotated to explain what needs them. +passenv=APPDATA HOMEDRIVE HOMEPATH windir APPVEYOR APPVEYOR_* CI CODECOV_* TRAVIS TRAVIS_* +commands=pytest --cov-config={toxinidir}/tox.ini --cov-report= {posargs} usedevelop=True + + +[testenv:coverage] +description=Combine coverage data and create report +deps=coverage +skip_install=True +changedir={toxworkdir} +setenv=COVERAGE_FILE=.coverage +commands=coverage erase + coverage combine + coverage {posargs:xml} + +[testenv:codecov] +description=[Only run on CI]: Upload coverage data to codecov +deps=codecov +skip_install=True +commands=codecov -X gcov --file {toxworkdir}/coverage.xml + +[testenv:docs] +deps = -r{toxinidir}/docs/requirements.txt +skip_install=True +commands = + python {toxinidir}/bootstrap.py + sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/build/html + sphinx-build -W -b man -d {envtmpdir}/doctrees docs docs/build/man + +[coverage:run] +source= + pkg_resources + setuptools +omit= + */_vendor/* |