diff options
68 files changed, 2973 insertions, 1538 deletions
@@ -182,3 +182,54 @@ fa069bf2411a150c9379d31a04d1c3836e2d3027 9.0.1 651d41db58849d4fc50e466f4dc458d448480c4e 10.2 1f5de53c079d577ead9d80265c9e006503b16457 10.2.1 b4b92805bc0e9802da0b597d00df4fa42b30bc40 11.0 +6cd2b18f4be2a9c188fa505b34505b32f4a4554b 11.1 +feb5971e7827483bbdeb67613126bb79ed09e6d9 11.2 +a1a6a1ac9113b90009052ca7263174a488434099 11.3 +1116e568f534ad8f4f41328a0f5fa183eb739c90 11.3.1 +55666947c9eb7e3ba78081ad6ae004807c84aede 12.0 +747018b2e35a40cb4b1c444f150f013d02197c64 12.0.1 +a177ea34bf81662b904fe3af46f3c8719a947ef1 12.0.2 +bf8c5bcacd49bf0f9648013a40ebfc8f7c727f7b 12.0.3 +73dcfc90e3eecec6baddea19302c6b342e68e2fa 12.0.4 +01fbfc9194a2bc502edd682eebbf4d2f1bc79eee 12.0.5 +7bca8938434839dbb546b8bfccd9aab7a86d851e 12.1 +5ff5c804a8fa580cff499ba0025ff2e6a5474fd0 12.2 +8d50aac3b20793954121edb300b477cc75f3ec96 12.3 +297931cb8cac7d44d970adb927efd6cb36ac3526 12.4 +df34cc18624279faffdbc729c0a11e6ab0f46572 13.0 +ae1a5c5cf78f4f9f98c054f1c8cec6168d1d19b4 13.0.1 +e22a1d613bddf311e125eecd9c1e1cad02ab5063 13.0.2 +a3a105f795f8362f26e84e9acbc237ee2d6bcca4 14.0 +9751a1671a124e30ae344d1510b9c1dbb14f2775 14.1 +07fcc3226782b979cedaaf456c7f1c5b2fdafd2c 14.1.1 +d714fb731de779a1337d2d78cd413931f1f06193 14.2 +e3c635a7d463c7713c647d1aa560f83fd8e27ef0 14.3 +608948cef7e0ab8951691b149f5b6f0184a5635e 14.3.1 +617699fd3e44e54b6f95b80bfcf78164df37f266 15.0b1 +d2c4d84867154243993876d6248aafec1fd12679 15.0 +10fde952613b7a3f650fb1f6b6ed58cbd232fa3c 15.1 +df5dc9c7aa7521f552824dee1ed1315cfe180844 15.2 +e0825f0c7d5963c498266fe3c175220c695ae83b 16.0 +8e56240961015347fed477f00ca6a0783e81d3a2 17.0 +a37bcaaeab367f2364ed8c070659d52a4c0ae38e 17.1 +4a0d01d690ff184904293e7a3244ac24ec060a73 17.1.1 +fac98a49bd984ef5accf7177674d693277bfbaef 18.0b1 +0a49ee524b0a1d67d2a11c8c22f082b57acd7ae1 18.0 +e364795c1b09c70b6abb53770e09763b52bf807d 18.0.1 +c0395f556c35d8311fdfe2bda6846b91149819cd 18.1 +1a981f2e5031f55267dc2a28fa1b42274a1b64b2 18.2 +b59320212c8371d0be9e5e6c5f7eec392124c009 18.3 +7a705b610abb1177ca169311c4ee261f3e4f0957 18.3.1 +1e120f04bcaa2421c4df0eb6678c3019ba4a82f6 18.3.2 +6203335278be7543d31790d9fba55739469a4c6c 18.4 +31dc6d2ac0f5ab766652602fe6ca716fff7180e7 18.5 +dfe190b09908f6b953209d13573063809de451b8 18.6 +804f87045a901f1dc121cf9149143d654228dc13 18.6.1 +67d07805606aead09349d5b91d7d26c68ddad2fc 18.7 +3041e1fc409be90e885968b90faba405420fc161 18.7.1 +c811801ffa1de758cf01fbf6a86e4c04ff0c0935 18.8 +fbf06fa35f93a43f044b1645a7e4ff470edb462c 18.8.1 +cc41477ecf92f221c113736fac2830bf8079d40c 19.0 +834782ce49154e9744e499e00eb392c347f9e034 19.1 +0a2a3d89416e1642cf6f41d22dbc07b3d3c15a4d 19.1.1 +5d24cf9d1ced76c406ab3c4a94c25d1fe79b94bc 19.2 diff --git a/.travis.yml b/.travis.yml index 0e648b38..54d9c395 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,24 @@ language: python python: - 2.6 - 2.7 - - 3.2 - 3.3 - 3.4 + - 3.5 - pypy -# command to run tests +env: + - "" + - LC_ALL=C LC_CTYPE=C script: + # avoid VersionConflict when newer version is required + - pip install -U pytest + + # Output the env, because the travis docs just can't be trusted + - env + # update egg_info based on setup.py in checkout - python bootstrap.py - - python setup.py ptr --addopts='-rs' - - python ez_setup.py --version 10.2.1 + - python setup.py test --addopts='-rs' + + # test the bootstrap script + - python ez_setup.py diff --git a/CHANGES.txt b/CHANGES.txt index 62ab4301..7ed7434b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,459 @@ CHANGES ======= + +---- +19.2 +---- + +* Pull Request #163: Add get_command_list method to Distribution. +* Pull Request #162: Add missing whitespace to multiline string + literals. + +------ +19.1.1 +------ + +* Issue #476: Cast version to string (using default encoding) + to avoid creating Unicode types on Python 2 clients. +* Issue #477: In Powershell downloader, use explicit rendering + of strings, rather than rely on ``repr``, which can be + incorrect (especially on Python 2). + +---- +19.1 +---- + +* Issue #215: The bootstrap script ``ez_setup.py`` now + automatically detects + the latest version of setuptools (using PyPI JSON API) rather + than hard-coding a particular value. +* Issue #475: Fix incorrect usage in _translate_metadata2. + +---- +19.0 +---- + +* Issue #442: Use RawConfigParser for parsing .pypirc file. + Interpolated values are no longer honored in .pypirc files. + +------ +18.8.1 +------ + +* Issue #440: Prevent infinite recursion when a SandboxViolation + or other UnpickleableException occurs in a sandbox context + with setuptools hidden. Fixes regression introduced in Setuptools + 12.0. + +---- +18.8 +---- + +* Deprecated ``egg_info.get_pkg_info_revision``. +* Issue #471: Don't rely on repr for an HTML attribute value in + package_index. +* Issue #419: Avoid errors in FileMetadata when the metadata directory + is broken. +* Issue #472: Remove deprecated use of 'U' in mode parameter + when opening files. + +------ +18.7.1 +------ + +* Issue #469: Refactored logic for Issue #419 fix to re-use metadata + loading from Provider. + +---- +18.7 +---- + +* Update dependency on certify. +* Pull Request #160: Improve detection of gui script in + ``easy_install._adjust_header``. +* Made ``test.test_args`` a non-data property; alternate fix + for the issue reported in Pull Request #155. +* Issue #453: In ``ez_setup`` bootstrap module, unload all + ``pkg_resources`` modules following download. +* Pull Request #158: Honor `PEP-488 + <https://www.python.org/dev/peps/pep-0488/>`_ when excluding + files for namespace packages. +* Issue #419 and Pull Request #144: Add experimental support for + reading the version info from distutils-installed metadata rather + than using the version in the filename. + +------ +18.6.1 +------ + +* Issue #464: Correct regression in invocation of superclass on old-style + class on Python 2. + +---- +18.6 +---- + +* Issue #439: When installing entry_point scripts under development, + omit the version number of the package, allowing any version of the + package to be used. + +---- +18.5 +---- + +* In preparation for dropping support for Python 3.2, a warning is + now logged when pkg_resources is imported on Python 3.2 or earlier + Python 3 versions. +* `Add support for python_platform_implementation environment marker + <https://github.com/jaraco/setuptools/pull/28>`_. +* `Fix dictionary mutation during iteration + <https://github.com/jaraco/setuptools/pull/29>`_. + +---- +18.4 +---- + +* Issue #446: Test command now always invokes unittest, even + if no test suite is supplied. + +------ +18.3.2 +------ + +* Correct another regression in setuptools.findall + where the fix for Python #12885 was lost. + +------ +18.3.1 +------ + +* Issue #425: Correct regression in setuptools.findall. + +---- +18.3 +---- + +* Setuptools now allows disabling of the manipulation of the sys.path + during the processing of the easy-install.pth file. To do so, set + the environment variable ``SETUPTOOLS_SYS_PATH_TECHNIQUE`` to + anything but "rewrite" (consider "raw"). During any install operation + with manipulation disabled, setuptools packages will be appended to + sys.path naturally. + + Future versions may change the default behavior to disable + manipulation. If so, the default behavior can be retained by setting + the variable to "rewrite". + +* Issue #257: ``easy_install --version`` now shows more detail + about the installation location and Python version. + +* Refactor setuptools.findall in preparation for re-submission + back to distutils. + +---- +18.2 +---- + +* Issue #412: More efficient directory search in ``find_packages``. + +---- +18.1 +---- + +* Upgrade to vendored packaging 15.3. + +------ +18.0.1 +------ + +* Issue #401: Fix failure in test suite. + +---- +18.0 +---- + +* Dropped support for builds with Pyrex. Only Cython is supported. +* Issue #288: Detect Cython later in the build process, after + ``setup_requires`` dependencies are resolved. + Projects backed by Cython can now be readily built + with a ``setup_requires`` dependency. For example:: + + ext = setuptools.Extension('mylib', ['src/CythonStuff.pyx', 'src/CStuff.c']) + setuptools.setup( + ... + ext_modules=[ext], + setup_requires=['cython'], + ) + + For compatibility with older versions of setuptools, packagers should + still include ``src/CythonMod.c`` in the source distributions or + require that Cython be present before building source distributions. + However, for systems with this build of setuptools, Cython will be + downloaded on demand. +* Issue #396: Fixed test failure on OS X. +* Pull Request #136: Remove excessive quoting from shebang headers + for Jython. + +------ +17.1.1 +------ + +* Backed out unintended changes to pkg_resources, restoring removal of + deprecated imp module (`ref + <https://bitbucket.org/pypa/setuptools/commits/f572ec9563d647fa8d4ffc534f2af8070ea07a8b#comment-1881283>`_). + +---- +17.1 +---- + +* Issue #380: Add support for range operators on environment + marker evaluation. + +---- +17.0 +---- + +* Issue #378: Do not use internal importlib._bootstrap module. +* Issue #390: Disallow console scripts with path separators in + the name. Removes unintended functionality and brings behavior + into parity with pip. + +---- +16.0 +---- + +* Pull Request #130: Better error messages for errors in + parsed requirements. +* Pull Request #133: Removed ``setuptools.tests`` from the + installed packages. + +---- +15.2 +---- + +* Issue #373: Provisionally expose + ``pkg_resources._initialize_master_working_set``, allowing for + imperative re-initialization of the master working set. + +---- +15.1 +---- + +* Updated to Packaging 15.1 to address Packaging #28. +* Fix ``setuptools.sandbox._execfile()`` with Python 3.1. + +---- +15.0 +---- + +* Pull Request #126: DistributionNotFound message now lists the package or + packages that required it. E.g.:: + + pkg_resources.DistributionNotFound: The 'colorama>=0.3.1' distribution was not found and is required by smlib.log. + + Note that zc.buildout once dependended on the string rendering of this + message to determine the package that was not found. This expectation + has since been changed, but older versions of buildout may experience + problems. See Buildout #242 for details. + +------ +14.3.1 +------ + +* Issue #307: Removed PEP-440 warning during parsing of versions + in ``pkg_resources.Distribution``. +* Issue #364: Replace deprecated usage with recommended usage of + ``EntryPoint.load``. + +---- +14.3 +---- + +* Issue #254: When creating temporary egg cache on Unix, use mode 755 + for creating the directory to avoid the subsequent warning if + the directory is group writable. + +---- +14.2 +---- + +* Issue #137: Update ``Distribution.hashcmp`` so that Distributions with + None for pyversion or platform can be compared against Distributions + defining those attributes. + +------ +14.1.1 +------ + +* Issue #360: Removed undesirable behavior from test runs, preventing + write tests and installation to system site packages. + +---- +14.1 +---- + +* Pull Request #125: Add ``__ne__`` to Requirement class. +* Various refactoring of easy_install. + +---- +14.0 +---- + +* Bootstrap script now accepts ``--to-dir`` to customize save directory or + allow for re-use of existing repository of setuptools versions. See + Pull Request #112 for background. +* Issue #285: ``easy_install`` no longer will default to installing + packages to the "user site packages" directory if it is itself installed + there. Instead, the user must pass ``--user`` in all cases to install + packages to the user site packages. + This behavior now matches that of "pip install". To configure + an environment to always install to the user site packages, consider + using the "install-dir" and "scripts-dir" parameters to easy_install + through an appropriate distutils config file. + +------ +13.0.2 +------ + +* Issue #359: Include pytest.ini in the sdist so invocation of py.test on the + sdist honors the pytest configuration. + +------ +13.0.1 +------ + +Re-release of 13.0. Intermittent connectivity issues caused the release +process to fail and PyPI uploads no longer accept files for 13.0. + +---- +13.0 +---- + +* Issue #356: Back out Pull Request #119 as it requires Setuptools 10 or later + as the source during an upgrade. +* Removed build_py class from setup.py. According to 892f439d216e, this + functionality was added to support upgrades from old Distribute versions, + 0.6.5 and 0.6.6. + +---- +12.4 +---- + +* Pull Request #119: Restore writing of ``setup_requires`` to metadata + (previously added in 8.4 and removed in 9.0). + +---- +12.3 +---- + +* Documentation is now linked using the rst.linker package. +* Fix ``setuptools.command.easy_install.extract_wininst_cfg()`` + with Python 2.6 and 2.7. +* Issue #354. Added documentation on building setuptools + documentation. + +---- +12.2 +---- + +* Issue #345: Unload all modules under pkg_resources during + ``ez_setup.use_setuptools()``. +* Issue #336: Removed deprecation from ``ez_setup.use_setuptools``, + as it is clearly still used by buildout's bootstrap. ``ez_setup`` + remains deprecated for use by individual packages. +* Simplified implementation of ``ez_setup.use_setuptools``. + +---- +12.1 +---- + +* Pull Request #118: Soften warning for non-normalized versions in + Distribution. + +------ +12.0.5 +------ + +* Issue #339: Correct Attribute reference in ``cant_write_to_target``. +* Issue #336: Deprecated ``ez_setup.use_setuptools``. + +------ +12.0.4 +------ + +* Issue #335: Fix script header generation on Windows. + +------ +12.0.3 +------ + +* Fixed incorrect class attribute in ``install_scripts``. Tests would be nice. + +------ +12.0.2 +------ + +* Issue #331: Fixed ``install_scripts`` command on Windows systems corrupting + the header. + +------ +12.0.1 +------ + +* Restore ``setuptools.command.easy_install.sys_executable`` for pbr + compatibility. For the future, tools should construct a CommandSpec + explicitly. + +---- +12.0 +---- + +* Issue #188: Setuptools now support multiple entities in the value for + ``build.executable``, such that an executable of "/usr/bin/env my-python" may + be specified. This means that systems with a specified executable whose name + has spaces in the path must be updated to escape or quote that value. +* Deprecated ``easy_install.ScriptWriter.get_writer``, replaced by ``.best()`` + with slightly different semantics (no force_windows flag). + +------ +11.3.1 +------ + +* Issue #327: Formalize and restore support for any printable character in an + entry point name. + +---- +11.3 +---- + +* Expose ``EntryPoint.resolve`` in place of EntryPoint._load, implementing the + simple, non-requiring load. Deprecated all uses of ``EntryPoint._load`` + except for calling with no parameters, which is just a shortcut for + ``ep.require(); ep.resolve();``. + + Apps currently invoking ``ep.load(require=False)`` should instead do the + following if wanting to avoid the deprecating warning:: + + getattr(ep, "resolve", lambda: ep.load(require=False))() + +---- +11.2 +---- + +* Pip #2326: Report deprecation warning at stacklevel 2 for easier diagnosis. + +---- +11.1 +---- + +* Issue #281: Since Setuptools 6.1 (Issue #268), a ValueError would be raised + in certain cases where VersionConflict was raised with two arguments, which + occurred in ``pkg_resources.WorkingSet.find``. This release adds support + for indicating the dependent packages while maintaining support for + a VersionConflict when no dependent package context is known. New unit tests + now capture the expected interface. + ---- 11.0 ---- @@ -19,7 +472,9 @@ CHANGES 10.2 ---- -* Deprecated use of EntryPoint.load(require=False). +* Deprecated use of EntryPoint.load(require=False). Passing a boolean to a + function to select behavior is an anti-pattern. Instead use + ``Entrypoint._load()``. * Substantial refactoring of all unit tests. Tests are now much leaner and re-use a lot of fixtures and contexts for better clarity of purpose. diff --git a/DEVGUIDE.txt b/DEVGUIDE.txt deleted file mode 100644 index 066a3a6b..00000000 --- a/DEVGUIDE.txt +++ /dev/null @@ -1 +0,0 @@ -The canonical development guide can be found in docs/developer-guide.txt. diff --git a/MANIFEST.in b/MANIFEST.in index 4278f245..dfea2049 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ recursive-include setuptools *.py *.exe *.xml -recursive-include tests *.py *.c *.pyx +recursive-include tests *.py recursive-include setuptools/tests *.html recursive-include docs *.py *.txt *.conf *.css *.css_t Makefile indexsidebar.html recursive-include _markerlib *.py @@ -10,3 +10,4 @@ include *.txt include MANIFEST.in include launcher.c include msvc-build-launcher.cmd +include pytest.ini @@ -2,6 +2,6 @@ empty: exit 1 update-vendored: - rm -rf setuptools/_vendor/packaging - pip install -r setuptools/_vendor/vendored.txt -t setuptools/_vendor/ - rm -rf setuptools/_vendor/*.{egg,dist}-info + rm -rf pkg_resources/_vendor/packaging + pip install -r pkg_resources/_vendor/vendored.txt -t pkg_resources/_vendor/ + rm -rf pkg_resources/_vendor/*.{egg,dist}-info @@ -223,3 +223,14 @@ Credits the Python Packaging Authority (PyPA) and the larger Python community. .. _files: + + +--------------- +Code of Conduct +--------------- + +Everyone interacting in the setuptools project's codebases, issue trackers, +chat rooms, and mailing lists is expected to follow the +`PyPA Code of Conduct`_. + +.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ diff --git a/docs/conf.py b/docs/conf.py index 5ea2e05e..c2a63873 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ import setup as setup_script # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['linkify'] +extensions = ['rst.linker'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -198,3 +198,58 @@ latex_documents = [ # If false, no module index is generated. #latex_use_modindex = True + +link_files = { + 'CHANGES.txt': dict( + using=dict( + BB='https://bitbucket.org', + GH='https://github.com', + ), + replace=[ + dict( + pattern=r"(Issue )?#(?P<issue>\d+)", + url='{BB}/pypa/setuptools/issue/{issue}', + ), + dict( + pattern=r"Pull Request ?#(?P<pull_request>\d+)", + url='{BB}/pypa/setuptools/pull-request/{pull_request}', + ), + dict( + pattern=r"Distribute #(?P<distribute>\d+)", + url='{BB}/tarek/distribute/issue/{distribute}', + ), + dict( + pattern=r"Buildout #(?P<buildout>\d+)", + url='{GH}/buildout/buildout/issues/{buildout}', + ), + dict( + pattern=r"Old Setuptools #(?P<old_setuptools>\d+)", + url='http://bugs.python.org/setuptools/issue{old_setuptools}', + ), + dict( + pattern=r"Jython #(?P<jython>\d+)", + url='http://bugs.jython.org/issue{jython}', + ), + dict( + pattern=r"Python #(?P<python>\d+)", + url='http://bugs.python.org/issue{python}', + ), + dict( + pattern=r"Interop #(?P<interop>\d+)", + url='{GH}/pypa/interoperability-peps/issues/{interop}', + ), + dict( + pattern=r"Pip #(?P<pip>\d+)", + url='{GH}/pypa/pip/issues/{pip}', + ), + dict( + pattern=r"Packaging #(?P<packaging>\d+)", + url='{GH}/pypa/packaging/issues/{packaging}', + ), + dict( + pattern=r"[Pp]ackaging (?P<packaging_ver>\d+(\.\d+)+)", + url='{GH}/pypa/packaging/blob/{packaging_ver}/CHANGELOG.rst', + ), + ], + ), +} diff --git a/docs/developer-guide.txt b/docs/developer-guide.txt index 558d6ee7..ae33649b 100644 --- a/docs/developer-guide.txt +++ b/docs/developer-guide.txt @@ -92,9 +92,10 @@ Testing The primary tests are run using py.test. To run the tests:: - $ python setup.py ptr + $ python setup.py test -Or install py.test into your environment and run ``py.test``. +Or install py.test into your environment and run ``PYTHONPATH=. py.test`` +or ``python -m pytest``. Under continuous integration, additional tests may be run. See the ``.travis.yml`` file for full details on the tests run under Travis-CI. @@ -109,3 +110,20 @@ Setuptools follows ``semver`` with some exceptions: - Omits 'v' prefix for tags. .. explain value of reflecting meaning in versions. + +---------------------- +Building Documentation +---------------------- + +Setuptools relies on the Sphinx system for building documentation and in +particular the ``build_sphinx`` distutils command. To build the +documentation, invoke:: + + python setup.py build_sphinx + +from the root of the repository. Setuptools will download a compatible +build of Sphinx and any requisite plugins and then build the +documentation in the build/sphinx directory. + +Setuptools does not support invoking the doc builder from the docs/ +directory as some tools expect. diff --git a/docs/development.txt b/docs/development.txt index 6fe30f6e..455f038a 100644 --- a/docs/development.txt +++ b/docs/development.txt @@ -33,4 +33,3 @@ setuptools changes. You have been warned. developer-guide formats releases - diff --git a/docs/index.txt b/docs/index.txt index d8eb968b..6ac37252 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -19,9 +19,7 @@ Documentation content: history roadmap python3 - using setuptools easy_install pkg_resources development - merge diff --git a/docs/merge-faq.txt b/docs/merge-faq.txt deleted file mode 100644 index ea45f30c..00000000 --- a/docs/merge-faq.txt +++ /dev/null @@ -1,80 +0,0 @@ -Setuptools/Distribute Merge FAQ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -How do I upgrade from Distribute? -================================= - -Distribute specifically prohibits installation of Setuptools 0.7 from Distribute 0.6. There are then two options for upgrading. - -Note that after upgrading using either technique, the only option to downgrade to either version is to completely uninstall Distribute and Setuptools 0.7 versions before reinstalling an 0.6 release. - -Use Distribute 0.7 ------------------- - -The PYPA has put together a compatibility wrapper, a new release of Distribute version 0.7. This package will install over Distribute 0.6.x installations and will replace Distribute with a simple wrapper that requires Setuptools 0.7 or later. This technique is experimental, but initial results indicate this technique is the easiest upgrade path. - - -Uninstall ---------- - -First, completely uninstall Distribute. Since Distribute does not have an automated installation routine, this process is manual. Follow the instructions in the README for uninstalling. - - -How do I upgrade from Setuptools 0.6? -===================================== - -There are no special instructions for upgrading over older versions of Setuptools. Simply use `easy_install -U` or run the latest `ez_setup.py`. - -Where does the merge occur? -======================================================== - -The merge is occurring between the heads of the default branch of Distribute and the setuptools-0.6 branch of Setuptools. The Setuptools SVN repo has been converted to a Mercurial repo hosted on Bitbucket. The work is still underway, so the exact changesets included may change, although the anticipated merge targets are Setuptools at 0.6c12 and Distribute at 0.6.35. - -What happens to other branches? -======================================================== - -Distribute 0.7 was abandoned long ago and won't be included in the resulting code tree, but may be retained for posterity in the original repo. - -Setuptools default branch (also 0.7 development) may also be abandoned or may be incorporated into the new merged line if desirable (and as resources allow). - -What history is lost/changed? -======================================================== - -As setuptools was not on Mercurial when the fork occurred and as Distribute did not include the full setuptools history (prior to the creation of the setuptools-0.6 branch), the two source trees were not compatible. In order to most effectively communicate the code history, the Distribute code was grafted onto the (originally private) setuptools Mercurial repo. Although this grafting maintained the full code history with names, dates, and changes, it did lose the original hashes of those changes. Therefore, references to changes by hash (including tags) are lost. - -Additionally, any heads that were not actively merged into the Distribute 0.6.35 release were also omitted. As a result, the changesets included in the merge repo are those from the original setuptools repo and all changesets ancestral to the Distribute 0.6.35 release. - -What features will be in the merged code base? -======================================================== - -In general, all "features" added in distribute will be included in setuptools. Where there exist conflicts or undesirable features, we will be explicit about what these limitations are. Changes that are backward-incompatible from setuptools 0.6 to distribute will likely be removed, and these also will be well documented. - -Bootstrapping scripts (ez_setup/distribute_setup) and docs, as with distribute, will be maintained in the repository and built as part of the release process. Documentation and bootstrapping scripts will be hosted at python.org, as they are with distribute now. Documentation at telecommunity will be updated to refer or redirect to the new, merged docs. - -On the whole, the merged setuptools should be largely compatible with the latest releases of both setuptools and distribute and will be an easy transition for users of either library. - -Who is invited to contribute? Who is excluded? -======================================================== - -While we've worked privately to initiate this merge due to the potential sensitivity of the topic, no one is excluded from this effort. We invite all members of the community, especially those most familiar with Python packaging and its challenges to join us in the effort. - -We have lots of ideas for how we'd like to improve the codebase, release process, everything. Like distribute, the post-merge setuptools will have its source hosted on Bitbucket. (So if you're currently a distribute contributor, about the only thing that's going to change is the URL of the repository you follow.) Also like distribute, it'll support Python 3, and hopefully we'll soon merge Vinay Sajip's patches to make it run on Python 3 without needing 2to3 to be run on the code first. - -While we've worked privately to initiate this merge due to the potential sensitivity of the topic, no one is excluded from this effort. We invite all members of the community, especially those most familiar with Python packaging and its challenges to join us in the effort. - -Why Setuptools and not Distribute or another name? -======================================================== - -We do, however, understand that this announcement might be unsettling for some. The setuptools name has been subjected to a lot of deprecation in recent years, so the idea that it will now be the preferred name instead of distribute might be somewhat difficult or disorienting for some. We considered use of another name (Distribute or an entirely new name), but that would serve to only complicate matters further. Instead, our goal is to simplify the packaging landscape but without losing any hard-won advancements. We hope that the people who worked to spread the first message will be equally enthusiastic about spreading the new one, and we especially look forward to seeing the new posters and slogans celebrating setuptools. - -What is the timeframe of release? -======================================================== - -There are no hard timeframes for any of this effort, although progress is underway and a draft merge is underway and being tested privately. As an unfunded volunteer effort, our time to put in on it is limited, and we've both had some recent health and other challenges that have made working on this difficult, which in part explains why we haven't met our original deadline of a completed merge before PyCon. - -The final Setuptools 0.7 was cut on June 1, 2013 and will be released to PyPI shortly thereafter. - -What version number can I expect for the new release? -======================================================== - -The new release will roughly follow the previous trend for setuptools and release the new release as 0.7. This number is somewhat arbitrary, but we wanted something other than 0.6 to distinguish it from its ancestor forks but not 1.0 to avoid putting too much emphasis on the release itself and to focus on merging the functionality. In the future, the project will likely adopt a versioning scheme similar to semver to convey semantic meaning about the release in the version number. diff --git a/docs/merge.txt b/docs/merge.txt deleted file mode 100644 index ba37d6e4..00000000 --- a/docs/merge.txt +++ /dev/null @@ -1,122 +0,0 @@ -Merge with Distribute -~~~~~~~~~~~~~~~~~~~~~ - -In 2013, the fork of Distribute was merged back into Setuptools. This -document describes some of the details of the merge. - -.. toctree:: - :maxdepth: 2 - - merge-faq - -Process -======= - -In order to try to accurately reflect the fork and then re-merge of the -projects, the merge process brought both code trees together into one -repository and grafted the Distribute fork onto the Setuptools development -line (as if it had been created as a branch in the first place). - -The rebase to get distribute onto setuptools went something like this:: - - hg phase -d -f -r 26b4c29b62db - hg rebase -s 26b4c29b62db -d 7a5cf59c78d7 - -The technique required a late version of mercurial (2.5) to work correctly. - -The only code that was included was the code that was ancestral to the public -releases of Distribute 0.6. Additionally, because Setuptools was not hosted -on Mercurial at the time of the fork and because the Distribute fork did not -include a complete conversion of the Setuptools history, the Distribute -changesets had to be re-applied to a new, different conversion of the -Setuptools SVN repository. As a result, all of the hashes have changed. - -Distribute was grafted in a 'distribute' branch and the 'setuptools-0.6' -branch was targeted for the merge. The 'setuptools' branch remains with -unreleased code and may be incorporated in the future. - -Reconciling Differences -======================= - -There were both technical and philosophical differences between Setuptools -and Distribute. To reconcile these differences in a manageable way, the -following technique was undertaken: - -Create a 'Setuptools-Distribute merge' branch, based on a late release of -Distribute (0.6.35). This was done with a00b441856c4. - -In that branch, first remove code that is no longer relevant to -Setuptools (such as the setuptools patching code). - -Next, in the the merge branch, create another base from at the point where the -fork occurred (such that the code is still essentially an older but pristine -setuptools). This base can be found as 955792b069d0. This creates two heads -in the merge branch, each with a basis in the fork. - -Then, repeatedly copy changes for a -single file or small group of files from a late revision of that file in the -'setuptools-0.6' branch (1aae1efe5733 was used) and commit those changes on -the setuptools-only head. That head is then merged with the head with -Distribute changes. It is in this Mercurial -merge operation that the fundamental differences between Distribute and -Setuptools are reconciled, but since only a single file or small set of files -are used, the scope is limited. - -Finally, once all the challenging files have been reconciled and merged, the -remaining changes from the setuptools-0.6 branch are merged, deferring to the -reconciled changes (a1fa855a5a62 and 160ccaa46be0). - -Originally, jaraco attempted all of this using anonymous heads in the -Distribute branch, but later realized this technique made for a somewhat -unclear merge process, so the changes were re-committed as described above -for clarity. In this way, the "distribute" and "setuptools" branches can -continue to track the official Distribute changesets. - -Concessions -=========== - -With the merge of Setuptools and Distribute, the following concessions were -made: - -Differences from setuptools 0.6c12: - -Major Changes -------------- - -* Python 3 support. -* Improved support for GAE. -* Support `PEP-370 <http://www.python.org/dev/peps/pep-0370/>`_ per-user site - packages. -* Sort order of Distributions in pkg_resources now prefers PyPI to external - links (Distribute issue 163). -* Python 2.4 or greater is required (drop support for Python 2.3). - -Minor Changes -------------- - -* Wording of some output has changed to replace contractions with their - canonical form (i.e. prefer "could not" to "couldn't"). -* Manifest files are only written for 32-bit .exe launchers. - -Differences from Distribute 0.6.36: - -Major Changes -------------- - -* The _distribute property of the setuptools module has been removed. -* Distributions are once again installed as zipped eggs by default, per the - rationale given in `the seminal bug report - <http://bugs.python.org/setuptools/issue33>`_ indicates that the feature - should remain and no substantial justification was given in the `Distribute - report <https://bitbucket.org/tarek/distribute/issue/19/>`_. - -Minor Changes -------------- - -* The patch for `#174 <https://bitbucket.org/tarek/distribute/issue/174>`_ - has been rolled-back, as the comment on the ticket indicates that the patch - addressed a symptom and not the fundamental issue. -* ``easy_install`` (the command) once again honors setup.cfg if found in the - current directory. The "mis-behavior" characterized in `#99 - <https://bitbucket.org/tarek/distribute/issue/99>`_ is actually intended - behavior, and no substantial rationale was given for the deviation. diff --git a/docs/pkg_resources.txt b/docs/pkg_resources.txt index 6c6405a8..3d40a1a2 100644 --- a/docs/pkg_resources.txt +++ b/docs/pkg_resources.txt @@ -592,7 +592,7 @@ Requirements Parsing The syntax of a requirement specifier can be defined in EBNF as follows:: - requirement ::= project_name versionspec? extras? + requirement ::= project_name extras? versionspec? versionspec ::= comparison version (',' comparison version)* comparison ::= '<' | '<=' | '!=' | '==' | '>=' | '>' | '~=' | '===' extras ::= '[' extralist? ']' diff --git a/docs/python3.txt b/docs/python3.txt index df173000..d550cb68 100644 --- a/docs/python3.txt +++ b/docs/python3.txt @@ -5,16 +5,22 @@ Supporting both Python 2 and Python 3 with Setuptools Starting with Distribute version 0.6.2 and Setuptools 0.7, the Setuptools project supported Python 3. Installing and using setuptools for Python 3 code works exactly the same as for Python 2 -code, but Setuptools also helps you to support Python 2 and Python 3 from -the same source code by letting you run 2to3 on the code as a part of the -build process, by setting the keyword parameter ``use_2to3`` to True. +code. +Setuptools provides a facility to invoke 2to3 on the code as a part of the +build process, by setting the keyword parameter ``use_2to3`` to True, but +the Setuptools strongly recommends instead developing a unified codebase +using `six <https://pypi.python.org/pypi/six>`_, +`future <https://pypi.python.org/pypi/future>`_, or another compatibility +library. -Setuptools as help during porting -================================= -Setuptools can make the porting process much easier by automatically running -2to3 as a part of the test running. To do this you need to configure the +Using 2to3 +========== + +Setuptools attempts to make the porting process easier by automatically +running +2to3 as a part of running tests. To do so, you need to configure the setup.py so that you can run the unit tests with ``python setup.py test``. See :ref:`test` for more information on this. @@ -37,23 +43,23 @@ to a list of names of packages containing fixers. To exclude fixers, the parameter ``use_2to3_exclude_fixers`` can be set to fixer names to be skipped. -A typical setup.py can look something like this:: +An example setup.py might look something like this:: from setuptools import setup setup( name='your.module', - version = '1.0', + version='1.0', description='This is your awesome module', author='You', author_email='your@email', - package_dir = {'': 'src'}, - packages = ['your', 'you.module'], - test_suite = 'your.module.tests', - use_2to3 = True, - convert_2to3_doctests = ['src/your/module/README.txt'], - use_2to3_fixers = ['your.fixers'], - use_2to3_exclude_fixers = ['lib2to3.fixes.fix_import'], + package_dir={'': 'src'}, + packages=['your', 'you.module'], + test_suite='your.module.tests', + use_2to3=True, + convert_2to3_doctests=['src/your/module/README.txt'], + use_2to3_fixers=['your.fixers'], + use_2to3_exclude_fixers=['lib2to3.fixes.fix_import'], ) Differential conversion @@ -86,39 +92,3 @@ Advanced features If you don't want to run the 2to3 conversion on the doctests in Python files, you can turn that off by setting ``setuptools.use_2to3_on_doctests = False``. - -Note on compatibility with older versions of setuptools -======================================================= - -Setuptools earlier than 0.7 does not know about the new keyword parameters to -support Python 3. -As a result it will warn about the unknown keyword parameters if you use -those versions of setuptools instead of Distribute under Python 2. This output -is not an error, and -install process will continue as normal, but if you want to get rid of that -error this is easy. Simply conditionally add the new parameters into an extra -dict and pass that dict into setup():: - - from setuptools import setup - import sys - - extra = {} - if sys.version_info >= (3,): - extra['use_2to3'] = True - extra['convert_2to3_doctests'] = ['src/your/module/README.txt'] - extra['use_2to3_fixers'] = ['your.fixers'] - - setup( - name='your.module', - version = '1.0', - description='This is your awesome module', - author='You', - author_email='your@email', - package_dir = {'': 'src'}, - packages = ['your', 'you.module'], - test_suite = 'your.module.tests', - **extra - ) - -This way the parameters will only be used under Python 3, where Distribute or -Setuptools 0.7 or later is required. diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 89c08e23..d6a62de8 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -112,10 +112,16 @@ the distutils. Here's a minimal setup script using setuptools:: ) As you can see, it doesn't take much to use setuptools in a project. -Just by doing the above, this project will be able to produce eggs, upload to +Run that script in your project folder, alongside the Python packages +you have developed. + +Invoke that script to produce eggs, upload to PyPI, and automatically include all packages in the directory where the setup.py lives. See the `Command Reference`_ section below to see what -commands you can give to this setup script. +commands you can give to this setup script. For example, +to produce a source distribution, simply invoke:: + + python setup.py sdist Of course, before you release your project to PyPI, you'll want to add a bit more information to your setup script to help people find or learn about your @@ -1467,7 +1473,7 @@ are included in any source distribution you build. This is a big improvement over having to manually write a ``MANIFEST.in`` file and try to keep it in sync with your project. So, if you are using CVS or Subversion, and your source distributions only need to include files that you're tracking in -revision control, don't create a a ``MANIFEST.in`` file for your project. +revision control, don't create a ``MANIFEST.in`` file for your project. (And, if you already have one, you might consider deleting it the next time you would otherwise have to change it.) @@ -2595,8 +2601,8 @@ those methods' docstrings for more details. Adding Support for Other Revision Control Systems ------------------------------------------------- -If you would like to create a plugin for ``setuptools`` to find files in other -source control systems besides CVS and Subversion, you can do so by adding an +If you would like to create a plugin for ``setuptools`` to find files in +source control systems, you can do so by adding an entry point to the ``setuptools.file_finders`` group. The entry point should be a function accepting a single directory name, and should yield all the filenames within that directory (and any subdirectories thereof) that @@ -2652,9 +2658,7 @@ Subclassing ``Command`` ----------------------- Sorry, this section isn't written yet, and neither is a lot of what's below -this point, except for the change log. You might want to `subscribe to changes -in this page <setuptools?action=subscribe>`_ to see when new documentation is -added or updated. +this point. XXX diff --git a/docs/using.txt b/docs/using.txt deleted file mode 100644 index bd80893d..00000000 --- a/docs/using.txt +++ /dev/null @@ -1,13 +0,0 @@ -================================ -Using Setuptools in your project -================================ - -To use Setuptools in your project, the recommended way is to ship -`ez_setup.py` alongside your `setup.py` script and call -it at the very beginning of `setup.py` like this:: - - from ez_setup import use_setuptools - use_setuptools() - -More info on `ez_setup.py` can be found at `the project home page -<https://pypy.python.org/pypi/setuptools>`_. diff --git a/ez_setup.py b/ez_setup.py index 740af722..9715bdc7 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,18 +1,11 @@ #!/usr/bin/env python -"""Bootstrap setuptools installation -To use setuptools in your package's setup.py, include this -file in the same directory and add this to the top of your setup.py:: - - from ez_setup import use_setuptools - use_setuptools() - -To require a specific version of setuptools, set a download -mirror, or use an alternate download directory, simply supply -the appropriate options to ``use_setuptools()``. +""" +Setuptools bootstrapping installer. -This file can also be run as a script to install or upgrade setuptools. +Run this script to install or upgrade setuptools. """ + import os import shutil import sys @@ -23,6 +16,8 @@ import subprocess import platform import textwrap import contextlib +import json +import codecs from distutils import log @@ -36,11 +31,16 @@ try: except ImportError: USER_SITE = None -DEFAULT_VERSION = "11.1" +LATEST = object() +DEFAULT_VERSION = LATEST DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" +DEFAULT_SAVE_DIR = os.curdir + def _python_cmd(*args): """ + Execute a command. + Return True if the command succeeded. """ args = (sys.executable,) + args @@ -48,6 +48,7 @@ def _python_cmd(*args): def _install(archive_filename, install_args=()): + """Install Setuptools.""" with archive_context(archive_filename): # installing log.warn('Installing Setuptools') @@ -59,6 +60,7 @@ def _install(archive_filename, install_args=()): def _build_egg(egg, archive_filename, to_dir): + """Build Setuptools egg.""" with archive_context(archive_filename): # building an egg log.warn('Building a Setuptools egg in %s', to_dir) @@ -70,9 +72,8 @@ def _build_egg(egg, archive_filename, to_dir): class ContextualZipFile(zipfile.ZipFile): - """ - Supplement ZipFile class to support context manager for Python 2.6 - """ + + """Supplement ZipFile class to support context manager for Python 2.6.""" def __enter__(self): return self @@ -81,9 +82,7 @@ class ContextualZipFile(zipfile.ZipFile): self.close() def __new__(cls, *args, **kwargs): - """ - Construct a ZipFile or ContextualZipFile as appropriate - """ + """Construct a ZipFile or ContextualZipFile as appropriate.""" if hasattr(zipfile.ZipFile, '__exit__'): return zipfile.ZipFile(*args, **kwargs) return super(ContextualZipFile, cls).__new__(cls) @@ -91,7 +90,11 @@ class ContextualZipFile(zipfile.ZipFile): @contextlib.contextmanager def archive_context(filename): - # extracting the archive + """ + Unzip filename to a temporary directory, set to the cwd. + + The unzipped target is cleaned up after. + """ tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() @@ -112,6 +115,7 @@ def archive_context(filename): def _do_download(version, download_base, to_dir, download_delay): + """Download Setuptools.""" egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): @@ -123,47 +127,84 @@ def _do_download(version, download_base, to_dir, download_delay): # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: - del sys.modules['pkg_resources'] + _unload_pkg_resources() import setuptools setuptools.bootstrap_install_from = egg -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15): +def use_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, download_delay=15): + """ + Ensure that a setuptools version is installed. + + Return None. Raise SystemExit if the requested version + or later cannot be installed. + """ + version = _resolve_version(version) to_dir = os.path.abspath(to_dir) + + # prior to importing, capture the module state for + # representative modules. rep_modules = 'pkg_resources', 'setuptools' imported = set(sys.modules).intersection(rep_modules) + try: import pkg_resources - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: pkg_resources.require("setuptools>=" + version) + # a suitable version is already installed return + except ImportError: + # pkg_resources not available; setuptools is not installed; download + pass except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, download_delay) + # no version of setuptools was found; allow download + pass except pkg_resources.VersionConflict as VC_err: if imported: - msg = textwrap.dedent(""" - The required version of setuptools (>={version}) is not available, - and can't be installed while this script is running. Please - install a more recent version first, using - 'easy_install -U setuptools'. + _conflict_bail(VC_err, version) - (Currently using {VC_err.args[0]!r}) - """).format(VC_err=VC_err, version=version) - sys.stderr.write(msg) - sys.exit(2) + # otherwise, unload pkg_resources to allow the downloaded version to + # take precedence. + del pkg_resources + _unload_pkg_resources() + + return _do_download(version, download_base, to_dir, download_delay) + + +def _conflict_bail(VC_err, version): + """ + Setuptools was imported prior to invocation, so it is + unsafe to unload it. Bail out. + """ + conflict_tmpl = textwrap.dedent(""" + The required version of setuptools (>={version}) is not available, + and can't be installed while this script is running. Please + install a more recent version first, using + 'easy_install -U setuptools'. + + (Currently using {VC_err.args[0]!r}) + """) + msg = conflict_tmpl.format(**locals()) + sys.stderr.write(msg) + sys.exit(2) + + +def _unload_pkg_resources(): + del_modules = [ + name for name in sys.modules + if name.startswith('pkg_resources') + ] + for mod_name in del_modules: + del sys.modules[mod_name] - # otherwise, reload ok - del pkg_resources, sys.modules['pkg_resources'] - return _do_download(version, download_base, to_dir, download_delay) def _clean_check(cmd, target): """ - Run the command to download target. If the command fails, clean up before - re-raising the error. + Run the command to download target. + + If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) @@ -172,17 +213,20 @@ def _clean_check(cmd, target): os.unlink(target) raise + def download_file_powershell(url, target): """ - Download the file at url to target using Powershell (which will validate - trust). Raise an exception if the command cannot complete. + Download the file at url to target using Powershell. + + Powershell will validate trust. + Raise an exception if the command cannot complete. """ target = os.path.abspath(target) ps_cmd = ( "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " "[System.Net.CredentialCache]::DefaultCredentials; " - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" - % vars() + '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' + % locals() ) cmd = [ 'powershell', @@ -191,7 +235,9 @@ def download_file_powershell(url, target): ] _clean_check(cmd, target) + def has_powershell(): + """Determine if Powershell is available.""" if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] @@ -201,13 +247,14 @@ def has_powershell(): except Exception: return False return True - download_file_powershell.viable = has_powershell + def download_file_curl(url, target): cmd = ['curl', url, '--silent', '--output', target] _clean_check(cmd, target) + def has_curl(): cmd = ['curl', '--version'] with open(os.path.devnull, 'wb') as devnull: @@ -216,13 +263,14 @@ def has_curl(): except Exception: return False return True - download_file_curl.viable = has_curl + def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) + def has_wget(): cmd = ['wget', '--version'] with open(os.path.devnull, 'wb') as devnull: @@ -231,14 +279,11 @@ def has_wget(): except Exception: return False return True - download_file_wget.viable = has_wget + def download_file_insecure(url, target): - """ - Use Python to download the file, even though it cannot authenticate the - connection. - """ + """Use Python to download the file, without connection authentication.""" src = urlopen(url) try: # Read all the data in one block. @@ -249,9 +294,9 @@ def download_file_insecure(url, target): # Write all the data in one block to avoid creating a partial file. with open(target, "wb") as dst: dst.write(data) - download_file_insecure.viable = lambda: True + def get_best_downloader(): downloaders = ( download_file_powershell, @@ -262,10 +307,13 @@ def get_best_downloader(): viable_downloaders = (dl for dl in downloaders if dl.viable()) return next(viable_downloaders, None) -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): + +def download_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, delay=15, + downloader_factory=get_best_downloader): """ - Download setuptools from a specified location and return its filename + Download setuptools from a specified location and return its filename. `version` should be a valid setuptools version number that is available as an sdist for download under the `download_base` URL (which should end @@ -276,6 +324,7 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ + version = _resolve_version(version) # making sure we use the absolute path to_dir = os.path.abspath(to_dir) zip_name = "setuptools-%s.zip" % version @@ -287,16 +336,38 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, downloader(url, saveto) return os.path.realpath(saveto) + +def _resolve_version(version): + """ + Resolve LATEST version + """ + if version is not LATEST: + return version + + resp = urlopen('https://pypi.python.org/pypi/setuptools/json') + with contextlib.closing(resp): + try: + charset = resp.info().get_content_charset() + except Exception: + # Python 2 compat; assume UTF-8 + charset = 'UTF-8' + reader = codecs.getreader(charset) + doc = json.load(reader(resp)) + + return str(doc['info']['version']) + + def _build_install_args(options): """ - Build the arguments to 'python setup.py install' on the setuptools package + Build the arguments to 'python setup.py install' on the setuptools package. + + Returns list of command line arguments. """ return ['--user'] if options.user_install else [] + def _parse_args(): - """ - Parse the command line for options - """ + """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, @@ -314,18 +385,30 @@ def _parse_args(): '--version', help="Specify which version to download", default=DEFAULT_VERSION, ) + parser.add_option( + '--to-dir', + help="Directory to save (and re-use) package", + default=DEFAULT_SAVE_DIR, + ) options, args = parser.parse_args() # positional arguments are ignored return options + +def _download_args(options): + """Return args for download_setuptools function from cmdline args.""" + return dict( + version=options.version, + download_base=options.download_base, + downloader_factory=options.downloader_factory, + to_dir=options.to_dir, + ) + + def main(): - """Install or upgrade setuptools and EasyInstall""" + """Install or upgrade setuptools and EasyInstall.""" options = _parse_args() - archive = download_setuptools( - version=options.version, - download_base=options.download_base, - downloader_factory=options.downloader_factory, - ) + archive = download_setuptools(**_download_args(options)) return _install(archive, _build_install_args(options)) if __name__ == '__main__': diff --git a/linkify.py b/linkify.py deleted file mode 100644 index e7b3ca7b..00000000 --- a/linkify.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Sphinx plugin to add links to the changelog. -""" - -import re -import os - - -link_patterns = [ - r"(Issue )?#(?P<issue>\d+)", - r"Pull Request ?#(?P<pull_request>\d+)", - r"Distribute #(?P<distribute>\d+)", - r"Buildout #(?P<buildout>\d+)", - r"Old Setuptools #(?P<old_setuptools>\d+)", - r"Jython #(?P<jython>\d+)", - r"Python #(?P<python>\d+)", - r"Interop #(?P<interop>\d+)", -] - -issue_urls = dict( - pull_request='https://bitbucket.org' - '/pypa/setuptools/pull-request/{pull_request}', - issue='https://bitbucket.org/pypa/setuptools/issue/{issue}', - distribute='https://bitbucket.org/tarek/distribute/issue/{distribute}', - buildout='https://github.com/buildout/buildout/issues/{buildout}', - old_setuptools='http://bugs.python.org/setuptools/issue{old_setuptools}', - jython='http://bugs.jython.org/issue{jython}', - python='http://bugs.python.org/issue{python}', - interop='https://github.com/pypa/interoperability-peps/issues/{interop}', -) - - -def _linkify(source, dest): - pattern = '|'.join(link_patterns) - with open(source) as source: - out = re.sub(pattern, replacer, source.read()) - with open(dest, 'w') as dest: - dest.write(out) - - -def replacer(match): - text = match.group(0) - match_dict = match.groupdict() - for key in match_dict: - if match_dict[key]: - url = issue_urls[key].format(**match_dict) - return "`{text} <{url}>`_".format(text=text, url=url) - -def setup(app): - _linkify('CHANGES.txt', 'CHANGES (links).txt') - app.connect('build-finished', remove_file) - -def remove_file(app, exception): - os.remove('CHANGES (links).txt') diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index f004315a..b55e4127 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -21,7 +21,7 @@ import os import io import time import re -import imp +import types import zipfile import zipimport import warnings @@ -36,8 +36,16 @@ import collections import plistlib import email.parser import tempfile +import textwrap +import itertools from pkgutil import get_importer +try: + import _imp +except ImportError: + # Python 3.2 compatibility + import imp as _imp + PY3 = sys.version_info > (3,) PY2 = not PY3 @@ -46,6 +54,8 @@ if PY3: if PY2: from urlparse import urlparse, urlunparse + filter = itertools.ifilter + map = itertools.imap if PY3: string_types = str, @@ -68,9 +78,9 @@ from os.path import isdir, split # Avoid try/except due to potential problems with delayed import mechanisms. if sys.version_info >= (3, 3) and sys.implementation.name == "cpython": - import importlib._bootstrap as importlib_bootstrap + import importlib.machinery as importlib_machinery else: - importlib_bootstrap = None + importlib_machinery = None try: import parser @@ -88,6 +98,19 @@ except ImportError: import packaging.specifiers +if (3, 0) < sys.version_info < (3, 3): + msg = ( + "Support for Python 3.0-3.2 has been dropped. Future versions " + "will fail here." + ) + warnings.warn(msg) + +# declare some globals that will be defined later to +# satisfy the linters. +require = None +working_set = None + + class PEP440Warning(RuntimeWarning): """ Used when there is an issue with a version or specifier not complying with @@ -182,8 +205,10 @@ class _SetuptoolsVersionMixin(object): "You have iterated over the result of " "pkg_resources.parse_version. This is a legacy behavior which is " "inconsistent with the new version class introduced in setuptools " - "8.0. That class should be used directly instead of attempting to " - "iterate over the result.", + "8.0. In most cases, conversion to a tuple is unnecessary. For " + "comparison of versions, sort the Version instances directly. If " + "you have another use case requiring the tuple, please file a " + "bug with the setuptools project describing that need.", RuntimeWarning, stacklevel=1, ) @@ -317,12 +342,79 @@ class ResolutionError(Exception): def __repr__(self): return self.__class__.__name__+repr(self.args) + class VersionConflict(ResolutionError): - """An already-installed version conflicts with the requested version""" + """ + An already-installed version conflicts with the requested version. + + Should be initialized with the installed Distribution and the requested + Requirement. + """ + + _template = "{self.dist} is installed but {self.req} is required" + + @property + def dist(self): + return self.args[0] + + @property + def req(self): + return self.args[1] + + def report(self): + return self._template.format(**locals()) + + def with_context(self, required_by): + """ + If required_by is non-empty, return a version of self that is a + ContextualVersionConflict. + """ + if not required_by: + return self + args = self.args + (required_by,) + return ContextualVersionConflict(*args) + + +class ContextualVersionConflict(VersionConflict): + """ + A VersionConflict that accepts a third parameter, the set of the + requirements that required the installed Distribution. + """ + + _template = VersionConflict._template + ' by {self.required_by}' + + @property + def required_by(self): + return self.args[2] + class DistributionNotFound(ResolutionError): """A requested distribution was not found""" + _template = ("The '{self.req}' distribution was not found " + "and is required by {self.requirers_str}") + + @property + def req(self): + return self.args[0] + + @property + def requirers(self): + return self.args[1] + + @property + def requirers_str(self): + if not self.requirers: + return 'the application' + return ', '.join(self.requirers) + + def report(self): + return self._template.format(**locals()) + + def __str__(self): + return self.report() + + class UnknownExtra(ResolutionError): """Distribution doesn't have an "extra feature" of the given name""" _provider_factories = {} @@ -627,8 +719,7 @@ class WorkingSet(object): if dist is not None and dist not in req: # XXX add more info raise VersionConflict(dist, req) - else: - return dist + return dist def iter_entry_points(self, group, name=None): """Yield entry point objects from `group` matching `name` @@ -754,19 +845,13 @@ class WorkingSet(object): ws = WorkingSet([]) dist = best[req.key] = env.best_match(req, ws, installer) if dist is None: - #msg = ("The '%s' distribution was not found on this " - # "system, and is required by this application.") - #raise DistributionNotFound(msg % req) - - # unfortunately, zc.buildout uses a str(err) - # to get the name of the distribution here.. - raise DistributionNotFound(req) + requirers = required_by.get(req, None) + raise DistributionNotFound(req, requirers) to_activate.append(dist) if dist not in req: # Oops, the "best" so far conflicts with a dependency - tmpl = "%s is installed but %s is required by %s" - args = dist, req, list(required_by.get(req, [])) - raise VersionConflict(tmpl % args) + dependent_req = required_by[req] + raise VersionConflict(dist, req).with_context(dependent_req) # push the new requirements onto the stack new_requirements = dist.requires(req.extras)[::-1] @@ -843,8 +928,7 @@ class WorkingSet(object): try: resolvees = shadow_set.resolve(req, env, installer) - except ResolutionError: - v = sys.exc_info()[1] + except ResolutionError as v: # save error info error_info[dist] = v if fallback: @@ -1329,6 +1413,7 @@ class MarkerEvaluation(object): 'python_version': lambda: platform.python_version()[:3], 'platform_version': platform.version, 'platform_machine': platform.machine, + 'platform_python_implementation': platform.python_implementation, 'python_implementation': platform.python_implementation, } @@ -1340,8 +1425,8 @@ class MarkerEvaluation(object): """ try: cls.evaluate_marker(text) - except SyntaxError: - return cls.normalize_exception(sys.exc_info()[1]) + except SyntaxError as e: + return cls.normalize_exception(e) return False @staticmethod @@ -1421,6 +1506,10 @@ class MarkerEvaluation(object): 'in': lambda x, y: x in y, '==': operator.eq, '!=': operator.ne, + '<': operator.lt, + '>': operator.gt, + '<=': operator.le, + '>=': operator.ge, } if hasattr(symbol, 'or_test'): ops[symbol.or_test] = cls.test @@ -1440,6 +1529,17 @@ class MarkerEvaluation(object): """ return cls.interpret(parser.expr(text).totuple(1)[1]) + @staticmethod + def _translate_metadata2(env): + """ + Markerlib implements Metadata 1.2 (PEP 345) environment markers. + Translate the variables to Metadata 2.0 (PEP 426). + """ + return dict( + (key.replace('.', '_'), value) + for key, value in env.items() + ) + @classmethod def _markerlib_evaluate(cls, text): """ @@ -1448,16 +1548,11 @@ class MarkerEvaluation(object): Raise SyntaxError if marker is invalid. """ import _markerlib - # markerlib implements Metadata 1.2 (PEP 345) environment markers. - # Translate the variables to Metadata 2.0 (PEP 426). - env = _markerlib.default_environment() - for key in env.keys(): - new_key = key.replace('.', '_') - env[new_key] = env.pop(key) + + env = cls._translate_metadata2(_markerlib.default_environment()) try: result = _markerlib.interpret(text, env) - except NameError: - e = sys.exc_info()[1] + except NameError as e: raise SyntaxError(e.args[0]) return result @@ -1624,7 +1719,7 @@ class EggProvider(NullProvider): path = self.module_path old = None while path!=old: - if path.lower().endswith('.egg'): + if _is_unpacked_egg(path): self.egg_name = os.path.basename(path) self.egg_info = os.path.join(path, 'EGG-INFO') self.egg_root = path @@ -1653,8 +1748,8 @@ class DefaultProvider(EggProvider): register_loader_type(type(None), DefaultProvider) -if importlib_bootstrap is not None: - register_loader_type(importlib_bootstrap.SourceFileLoader, DefaultProvider) +if importlib_machinery is not None: + register_loader_type(importlib_machinery.SourceFileLoader, DefaultProvider) class EmptyProvider(NullProvider): @@ -1922,11 +2017,11 @@ class FileMetadata(EmptyProvider): self.path = path def has_metadata(self, name): - return name=='PKG-INFO' + return name=='PKG-INFO' and os.path.isfile(self.path) def get_metadata(self, name): if name=='PKG-INFO': - with open(self.path,'rU') as f: + with io.open(self.path, encoding='utf-8') as f: metadata = f.read() return metadata raise KeyError("No metadata except PKG-INFO is available") @@ -2007,7 +2102,7 @@ def find_eggs_in_zip(importer, path_item, only=False): # don't yield nested distros return for subitem in metadata.resource_listdir('/'): - if subitem.endswith('.egg'): + if _is_unpacked_egg(subitem): subpath = os.path.join(path_item, subitem) for dist in find_eggs_in_zip(zipimport.zipimporter(subpath), subpath): yield dist @@ -2023,8 +2118,7 @@ def find_on_path(importer, path_item, only=False): path_item = _normalize_cached(path_item) if os.path.isdir(path_item) and os.access(path_item, os.R_OK): - if path_item.lower().endswith('.egg'): - # unpacked egg + if _is_unpacked_egg(path_item): yield Distribution.from_filename( path_item, metadata=PathMetadata( path_item, os.path.join(path_item,'EGG-INFO') @@ -2044,7 +2138,7 @@ def find_on_path(importer, path_item, only=False): yield Distribution.from_location( path_item, entry, metadata, precedence=DEVELOP_DIST ) - elif not only and lower.endswith('.egg'): + elif not only and _is_unpacked_egg(entry): dists = find_distributions(os.path.join(path_item, entry)) for dist in dists: yield dist @@ -2061,8 +2155,8 @@ def find_on_path(importer, path_item, only=False): break register_finder(pkgutil.ImpImporter, find_on_path) -if importlib_bootstrap is not None: - register_finder(importlib_bootstrap.FileFinder, find_on_path) +if importlib_machinery is not None: + register_finder(importlib_machinery.FileFinder, find_on_path) _declare_state('dict', _namespace_handlers={}) _declare_state('dict', _namespace_packages={}) @@ -2096,7 +2190,7 @@ def _handle_ns(packageName, path_item): return None module = sys.modules.get(packageName) if module is None: - module = sys.modules[packageName] = imp.new_module(packageName) + module = sys.modules[packageName] = types.ModuleType(packageName) module.__path__ = [] _set_parent_ns(packageName) elif not hasattr(module,'__path__'): @@ -2115,7 +2209,7 @@ def _handle_ns(packageName, path_item): def declare_namespace(packageName): """Declare that package 'packageName' is a namespace package""" - imp.acquire_lock() + _imp.acquire_lock() try: if packageName in _namespace_packages: return @@ -2142,18 +2236,18 @@ def declare_namespace(packageName): _handle_ns(packageName, path_item) finally: - imp.release_lock() + _imp.release_lock() def fixup_namespace_packages(path_item, parent=None): """Ensure that previously-declared namespace packages include path_item""" - imp.acquire_lock() + _imp.acquire_lock() try: for package in _namespace_packages.get(parent,()): subpath = _handle_ns(package, path_item) if subpath: fixup_namespace_packages(subpath, package) finally: - imp.release_lock() + _imp.release_lock() def file_ns_handler(importer, path_item, packageName, module): """Compute an ns-package subpath for a filesystem or zipfile importer""" @@ -2170,8 +2264,8 @@ def file_ns_handler(importer, path_item, packageName, module): register_namespace_handler(pkgutil.ImpImporter, file_ns_handler) register_namespace_handler(zipimport.zipimporter, file_ns_handler) -if importlib_bootstrap is not None: - register_namespace_handler(importlib_bootstrap.FileFinder, file_ns_handler) +if importlib_machinery is not None: + register_namespace_handler(importlib_machinery.FileFinder, file_ns_handler) def null_ns_handler(importer, path_item, packageName, module): @@ -2191,6 +2285,14 @@ def _normalize_cached(filename, _cache={}): _cache[filename] = result = normalize_path(filename) return result +def _is_unpacked_egg(path): + """ + Determine if given path appears to be an unpacked egg. + """ + return ( + path.lower().endswith('.egg') + ) + def _set_parent_ns(packageName): parts = packageName.split('.') name = parts.pop() @@ -2226,9 +2328,16 @@ OBRACKET = re.compile(r"\s*\[").match CBRACKET = re.compile(r"\s*\]").match MODULE = re.compile(r"\w+(\.\w+)*$").match EGG_NAME = re.compile( - r"(?P<name>[^-]+)" - r"( -(?P<ver>[^-]+) (-py(?P<pyver>[^-]+) (-(?P<plat>.+))? )? )?", - re.VERBOSE | re.IGNORECASE + r""" + (?P<name>[^-]+) ( + -(?P<ver>[^-]+) ( + -py(?P<pyver>[^-]+) ( + -(?P<plat>.+) + )? + )? + )? + """, + re.VERBOSE | re.IGNORECASE, ).match @@ -2255,18 +2364,25 @@ class EntryPoint(object): def __repr__(self): return "EntryPoint.parse(%r)" % str(self) - def load(self, require=True, env=None, installer=None): - if require: - self.require(env, installer) - else: + def load(self, require=True, *args, **kwargs): + """ + Require packages for this EntryPoint, then resolve it. + """ + if not require or args or kwargs: warnings.warn( - "`require` parameter is deprecated. Use " - "EntryPoint._load instead.", + "Parameters to load are deprecated. Call .resolve and " + ".require separately.", DeprecationWarning, + stacklevel=2, ) - return self._load() + if require: + self.require(*args, **kwargs) + return self.resolve() - def _load(self): + def resolve(self): + """ + Resolve the entry point from its module and attrs. + """ module = __import__(self.module_name, fromlist=['__name__'], level=0) try: return functools.reduce(getattr, self.attrs, module) @@ -2282,7 +2398,7 @@ class EntryPoint(object): pattern = re.compile( r'\s*' - r'(?P<name>[+\w. -]+?)\s*' + r'(?P<name>.+?)\s*' r'=\s*' r'(?P<module>[\w.]+)\s*' r'(:\s*(?P<attr>[\w.]+))?\s*' @@ -2360,6 +2476,18 @@ def _remove_md5_fragment(location): return location +def _version_from_file(lines): + """ + Given an iterable of lines from a Metadata file, return + the value of the Version field, if present, or None otherwise. + """ + is_version_line = lambda line: line.lower().startswith('version:') + version_lines = filter(is_version_line, lines) + line = next(iter(version_lines), '') + _, _, value = line.partition(':') + return safe_version(value.strip()) or None + + class Distribution(object): """Wrap an actual or potential sys.path entry w/metadata""" PKG_INFO = 'PKG-INFO' @@ -2377,21 +2505,24 @@ class Distribution(object): self._provider = metadata or empty_provider @classmethod - def from_location(cls, location, basename, metadata=None,**kw): + def from_location(cls, location, basename, metadata=None, **kw): project_name, version, py_version, platform = [None]*4 basename, ext = os.path.splitext(basename) if ext.lower() in _distributionImpl: - # .dist-info gets much metadata differently + cls = _distributionImpl[ext.lower()] + match = EGG_NAME(basename) if match: project_name, version, py_version, platform = match.group( - 'name','ver','pyver','plat' + 'name', 'ver', 'pyver', 'plat' ) - cls = _distributionImpl[ext.lower()] return cls( location, metadata, project_name=project_name, version=version, py_version=py_version, platform=platform, **kw - ) + )._reload_version() + + def _reload_version(self): + return self @property def hashcmp(self): @@ -2400,8 +2531,8 @@ class Distribution(object): self.precedence, self.key, _remove_md5_fragment(self.location), - self.py_version, - self.platform, + self.py_version or '', + self.platform or '', ) def __hash__(self): @@ -2444,40 +2575,45 @@ class Distribution(object): def parsed_version(self): if not hasattr(self, "_parsed_version"): self._parsed_version = parse_version(self.version) - if isinstance( - self._parsed_version, packaging.version.LegacyVersion): - # While an empty version is techincally a legacy version and - # is not a valid PEP 440 version, it's also unlikely to - # actually come from someone and instead it is more likely that - # it comes from setuptools attempting to parse a filename and - # including it in the list. So for that we'll gate this warning - # on if the version is anything at all or not. - if self.version: - warnings.warn( - "'%s (%s)' is being parsed as a legacy, non PEP 440, " - "version. You may find odd behavior and sort order. " - "In particular it will be sorted as less than 0.0. It " - "is recommend to migrate to PEP 440 compatible " - "versions." % ( - self.project_name, self.version, - ), - PEP440Warning, - ) return self._parsed_version + def _warn_legacy_version(self): + LV = packaging.version.LegacyVersion + is_legacy = isinstance(self._parsed_version, LV) + if not is_legacy: + return + + # While an empty version is technically a legacy version and + # is not a valid PEP 440 version, it's also unlikely to + # actually come from someone and instead it is more likely that + # it comes from setuptools attempting to parse a filename and + # including it in the list. So for that we'll gate this warning + # on if the version is anything at all or not. + if not self.version: + return + + tmpl = textwrap.dedent(""" + '{project_name} ({version})' is being parsed as a legacy, + non PEP 440, + version. You may find odd behavior and sort order. + In particular it will be sorted as less than 0.0. It + is recommended to migrate to PEP 440 compatible + versions. + """).strip().replace('\n', ' ') + + warnings.warn(tmpl.format(**vars(self)), PEP440Warning) + @property def version(self): try: return self._version except AttributeError: - for line in self._get_metadata(self.PKG_INFO): - if line.lower().startswith('version:'): - self._version = safe_version(line.split(':',1)[1].strip()) - return self._version - else: + version = _version_from_file(self._get_metadata(self.PKG_INFO)) + if version is None: tmpl = "Missing 'Version:' header and/or %s file" raise ValueError(tmpl % self.PKG_INFO, self) + return version @property def _dep_map(self): @@ -2682,6 +2818,26 @@ class Distribution(object): return [dep for dep in self._dep_map if dep] +class EggInfoDistribution(Distribution): + + def _reload_version(self): + """ + Packages installed by distutils (e.g. numpy or scipy), + which uses an old safe_version, and so + their version numbers can get mangled when + converted to filenames (e.g., 1.11.0.dev0+2329eae to + 1.11.0.dev0_2329eae). These distributions will not be + parsed properly + downstream by Distribution and safe_version, so + take an extra step and try to get the version number from + the metadata file itself instead of the filename. + """ + md_version = _version_from_file(self._get_metadata(self.PKG_INFO)) + if md_version: + self._version = md_version + return self + + class DistInfoDistribution(Distribution): """Wrap an actual or potential sys.path entry w/metadata, .dist-info style""" PKG_INFO = 'METADATA' @@ -2747,7 +2903,7 @@ class DistInfoDistribution(Distribution): _distributionImpl = { '.egg': Distribution, - '.egg-info': Distribution, + '.egg-info': EggInfoDistribution, '.dist-info': DistInfoDistribution, } @@ -2765,6 +2921,11 @@ def issue_warning(*args,**kw): warnings.warn(stacklevel=level + 1, *args, **kw) +class RequirementParseError(ValueError): + def __str__(self): + return ' '.join(self.args) + + def parse_requirements(strs): """Yield ``Requirement`` objects for each specification in `strs` @@ -2783,14 +2944,13 @@ def parse_requirements(strs): line = next(lines) p = 0 except StopIteration: - raise ValueError( - "\\ must not appear on the last nonblank line" - ) + msg = "\\ must not appear on the last nonblank line" + raise RequirementParseError(msg) match = ITEM(line, p) if not match: msg = "Expected " + item_name + " in" - raise ValueError(msg, line, "at", line[p:]) + raise RequirementParseError(msg, line, "at", line[p:]) items.append(match.group(*groups)) p = match.end() @@ -2801,7 +2961,7 @@ def parse_requirements(strs): p = match.end() elif not TERMINATOR(line, p): msg = "Expected ',' or end-of-list in" - raise ValueError(msg, line, "at", line[p:]) + raise RequirementParseError(msg, line, "at", line[p:]) match = TERMINATOR(line, p) # skip the terminator, if any @@ -2812,7 +2972,7 @@ def parse_requirements(strs): for line in lines: match = DISTRO(line) if not match: - raise ValueError("Missing distribution spec", line) + raise RequirementParseError("Missing distribution spec", line) project_name = match.group(1) p = match.end() extras = [] @@ -2859,6 +3019,9 @@ class Requirement: self.hashCmp == other.hashCmp ) + def __ne__(self, other): + return not self == other + def __contains__(self, item): if isinstance(item, Distribution): if item.key != self.key: @@ -2878,12 +3041,8 @@ class Requirement: @staticmethod def parse(s): - reqs = list(parse_requirements(s)) - if reqs: - if len(reqs) == 1: - return reqs[0] - raise ValueError("Expected only one requirement", s) - raise ValueError("No requirements found", s) + req, = parse_requirements(s) + return req def _get_mro(cls): @@ -2907,14 +3066,14 @@ def ensure_directory(path): os.makedirs(dirname) -def _bypass_ensure_directory(path, mode=0o777): +def _bypass_ensure_directory(path): """Sandbox-bypassing version of ensure_directory()""" if not WRITE_SUPPORT: raise IOError('"os.mkdir" not supported on this platform.') dirname, filename = split(path) if dirname and filename and not isdir(dirname): _bypass_ensure_directory(dirname) - mkdir(dirname, mode) + mkdir(dirname, 0o755) def split_sections(s): @@ -2960,28 +3119,49 @@ def _mkstemp(*args,**kw): warnings.filterwarnings("ignore", category=PEP440Warning, append=True) -# Set up global resource manager (deliberately not state-saved) -_manager = ResourceManager() -def _initialize(g): - for name in dir(_manager): +# from jaraco.functools 1.3 +def _call_aside(f, *args, **kwargs): + f(*args, **kwargs) + return f + + +@_call_aside +def _initialize(g=globals()): + "Set up global resource manager (deliberately not state-saved)" + manager = ResourceManager() + g['_manager'] = manager + for name in dir(manager): if not name.startswith('_'): - g[name] = getattr(_manager, name) -_initialize(globals()) + g[name] = getattr(manager, name) -# Prepare the master working set and make the ``require()`` API available -working_set = WorkingSet._build_master() -_declare_state('object', working_set=working_set) -require = working_set.require -iter_entry_points = working_set.iter_entry_points -add_activation_listener = working_set.subscribe -run_script = working_set.run_script -# backward compatibility -run_main = run_script -# Activate all distributions already on sys.path, and ensure that -# all distributions added to the working set in the future (e.g. by -# calling ``require()``) will get activated as well. -add_activation_listener(lambda dist: dist.activate()) -working_set.entries=[] -# match order -list(map(working_set.add_entry, sys.path)) +@_call_aside +def _initialize_master_working_set(): + """ + Prepare the master working set and make the ``require()`` + API available. + + This function has explicit effects on the global state + of pkg_resources. It is intended to be invoked once at + the initialization of this module. + + Invocation by other packages is unsupported and done + at their own risk. + """ + working_set = WorkingSet._build_master() + _declare_state('object', working_set=working_set) + + require = working_set.require + iter_entry_points = working_set.iter_entry_points + add_activation_listener = working_set.subscribe + run_script = working_set.run_script + # backward compatibility + run_main = run_script + # Activate all distributions already on sys.path, and ensure that + # all distributions added to the working set in the future (e.g. by + # calling ``require()``) will get activated as well. + add_activation_listener(lambda dist: dist.activate()) + working_set.entries=[] + # match order + list(map(working_set.add_entry, sys.path)) + globals().update(locals()) diff --git a/pkg_resources/_vendor/packaging/__about__.py b/pkg_resources/_vendor/packaging/__about__.py index 36f1a35c..eadb794e 100644 --- a/pkg_resources/_vendor/packaging/__about__.py +++ b/pkg_resources/_vendor/packaging/__about__.py @@ -22,7 +22,7 @@ __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "15.0" +__version__ = "15.3" __author__ = "Donald Stufft" __email__ = "donald@stufft.io" diff --git a/pkg_resources/_vendor/packaging/specifiers.py b/pkg_resources/_vendor/packaging/specifiers.py index 9ad0a635..891664f0 100644 --- a/pkg_resources/_vendor/packaging/specifiers.py +++ b/pkg_resources/_vendor/packaging/specifiers.py @@ -152,6 +152,14 @@ class _IndividualSpecifier(BaseSpecifier): return version @property + def operator(self): + return self._spec[0] + + @property + def version(self): + return self._spec[1] + + @property def prereleases(self): return self._prereleases @@ -159,6 +167,9 @@ class _IndividualSpecifier(BaseSpecifier): def prereleases(self, value): self._prereleases = value + def __contains__(self, item): + return self.contains(item) + def contains(self, item, prereleases=None): # Determine if prereleases are to be allowed or not. if prereleases is None: @@ -176,7 +187,7 @@ class _IndividualSpecifier(BaseSpecifier): # Actually do the comparison to determine if this item is contained # within this Specifier or not. - return self._get_operator(self._spec[0])(item, self._spec[1]) + return self._get_operator(self.operator)(item, self.version) def filter(self, iterable, prereleases=None): yielded = False @@ -526,7 +537,7 @@ class Specifier(_IndividualSpecifier): # operators, and if they are if they are including an explicit # prerelease. operator, version = self._spec - if operator in ["==", ">=", "<=", "~="]: + if operator in ["==", ">=", "<=", "~=", "==="]: # The == specifier can include a trailing .*, if it does we # want to remove before parsing. if operator == "==" and version.endswith(".*"): @@ -666,6 +677,12 @@ class SpecifierSet(BaseSpecifier): return self._specs != other._specs + def __len__(self): + return len(self._specs) + + def __iter__(self): + return iter(self._specs) + @property def prereleases(self): # If we have been given an explicit prerelease modifier, then we'll @@ -673,42 +690,43 @@ class SpecifierSet(BaseSpecifier): if self._prereleases is not None: return self._prereleases + # If we don't have any specifiers, and we don't have a forced value, + # then we'll just return None since we don't know if this should have + # pre-releases or not. + if not self._specs: + return None + # Otherwise we'll see if any of the given specifiers accept # prereleases, if any of them do we'll return True, otherwise False. - # Note: The use of any() here means that an empty set of specifiers - # will always return False, this is an explicit design decision. return any(s.prereleases for s in self._specs) @prereleases.setter def prereleases(self, value): self._prereleases = value + def __contains__(self, item): + return self.contains(item) + def contains(self, item, prereleases=None): # Ensure that our item is a Version or LegacyVersion instance. if not isinstance(item, (LegacyVersion, Version)): item = parse(item) + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + # We can determine if we're going to allow pre-releases by looking to # see if any of the underlying items supports them. If none of them do # and this item is a pre-release then we do not allow it and we can # short circuit that here. # Note: This means that 1.0.dev1 would not be contained in something # like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0 - if (not (self.prereleases or prereleases)) and item.is_prerelease: + if not prereleases and item.is_prerelease: return False - # Determine if we're forcing a prerelease or not, we bypass - # self.prereleases here and use self._prereleases because we want to - # only take into consideration actual *forced* values. The underlying - # specifiers will handle the other logic. - # The logic here is: If prereleases is anything but None, we'll just - # go aheand and continue to use that. However if - # prereleases is None, then we'll use whatever the - # value of self._prereleases is as long as it is not - # None itself. - if prereleases is None and self._prereleases is not None: - prereleases = self._prereleases - # We simply dispatch to the underlying specs here to make sure that the # given version is contained within all of them. # Note: This use of all() here means that an empty set of specifiers @@ -719,24 +737,18 @@ class SpecifierSet(BaseSpecifier): ) def filter(self, iterable, prereleases=None): - # Determine if we're forcing a prerelease or not, we bypass - # self.prereleases here and use self._prereleases because we want to - # only take into consideration actual *forced* values. The underlying - # specifiers will handle the other logic. - # The logic here is: If prereleases is anything but None, we'll just - # go aheand and continue to use that. However if - # prereleases is None, then we'll use whatever the - # value of self._prereleases is as long as it is not - # None itself. - if prereleases is None and self._prereleases is not None: - prereleases = self._prereleases + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases # If we have any specifiers, then we want to wrap our iterable in the # filter method for each one, this will act as a logical AND amongst # each specifier. if self._specs: for spec in self._specs: - iterable = spec.filter(iterable, prereleases=prereleases) + iterable = spec.filter(iterable, prereleases=bool(prereleases)) return iterable # If we do not have any specifiers, then we need to have a rough filter # which will filter out any pre-releases, unless there are no final diff --git a/pkg_resources/_vendor/packaging/version.py b/pkg_resources/_vendor/packaging/version.py index cf8afb16..4ba574b9 100644 --- a/pkg_resources/_vendor/packaging/version.py +++ b/pkg_resources/_vendor/packaging/version.py @@ -324,6 +324,8 @@ def _parse_letter_version(letter, number): letter = "b" elif letter in ["c", "pre", "preview"]: letter = "rc" + elif letter in ["rev", "r"]: + letter = "post" return letter, int(number) if not letter and number: diff --git a/pkg_resources/_vendor/vendored.txt b/pkg_resources/_vendor/vendored.txt index 75a31670..4cf7664e 100644 --- a/pkg_resources/_vendor/vendored.txt +++ b/pkg_resources/_vendor/vendored.txt @@ -1 +1 @@ -packaging==15.0 +packaging==15.3 diff --git a/pkg_resources/api_tests.txt b/pkg_resources/api_tests.txt index 50e04b87..d28db0f5 100644 --- a/pkg_resources/api_tests.txt +++ b/pkg_resources/api_tests.txt @@ -210,8 +210,7 @@ working set triggers a ``pkg_resources.VersionConflict`` error: >>> try: ... ws.find(Requirement.parse("Bar==1.0")) - ... except pkg_resources.VersionConflict: - ... exc = sys.exc_info()[1] + ... except pkg_resources.VersionConflict as exc: ... print(str(exc)) ... else: ... raise AssertionError("VersionConflict was not raised") @@ -365,9 +364,6 @@ Environment Markers >>> print(im("'x'=='x' or os.open('foo')=='y'")) # no short-circuit! Language feature not supported in environment markers - >>> print(im("'x' < 'y'")) - '<' operator not allowed in environment markers - >>> print(im("'x' < 'y' < 'z'")) Chained comparison not allowed in environment markers @@ -418,3 +414,12 @@ Environment Markers >>> em("'yx' in 'x'") False + + >>> em("python_version >= '2.6'") + True + + >>> em("python_version > '2.5'") + True + + >>> im("platform_python_implementation=='CPython'") + False diff --git a/pkg_resources/tests/test_markers.py b/pkg_resources/tests/test_markers.py new file mode 100644 index 00000000..d8844e74 --- /dev/null +++ b/pkg_resources/tests/test_markers.py @@ -0,0 +1,16 @@ +try: + import unittest.mock as mock +except ImportError: + import mock + +from pkg_resources import evaluate_marker + + +@mock.patch.dict('pkg_resources.MarkerEvaluation.values', + python_full_version=mock.Mock(return_value='2.7.10')) +def test_lexicographic_ordering(): + """ + Although one might like 2.7.10 to be greater than 2.7.3, + the marker spec only supports lexicographic ordering. + """ + assert evaluate_marker("python_full_version > '2.7.3'") is False diff --git a/pkg_resources/tests/test_pkg_resources.py b/pkg_resources/tests/test_pkg_resources.py index 564d7cec..31eee635 100644 --- a/pkg_resources/tests/test_pkg_resources.py +++ b/pkg_resources/tests/test_pkg_resources.py @@ -1,3 +1,6 @@ +# coding: utf-8 +from __future__ import unicode_literals + import sys import tempfile import os @@ -5,9 +8,15 @@ import zipfile import datetime import time import subprocess +import stat +import distutils.dist +import distutils.command.install_egg_info + +import pytest import pkg_resources + try: unicode except NameError: @@ -109,3 +118,50 @@ class TestIndependence: ) cmd = [sys.executable, '-c', '; '.join(lines)] subprocess.check_call(cmd) + + + +class TestDeepVersionLookupDistutils(object): + + @pytest.fixture + def env(self, tmpdir): + """ + Create a package environment, similar to a virtualenv, + in which packages are installed. + """ + class Environment(str): + pass + + env = Environment(tmpdir) + tmpdir.chmod(stat.S_IRWXU) + subs = 'home', 'lib', 'scripts', 'data', 'egg-base' + env.paths = dict( + (dirname, str(tmpdir / dirname)) + for dirname in subs + ) + list(map(os.mkdir, env.paths.values())) + return env + + def create_foo_pkg(self, env, version): + """ + Create a foo package installed (distutils-style) to env.paths['lib'] + as version. + """ + ld = "This package has unicode metadata! ❄" + attrs = dict(name='foo', version=version, long_description=ld) + dist = distutils.dist.Distribution(attrs) + iei_cmd = distutils.command.install_egg_info.install_egg_info(dist) + iei_cmd.initialize_options() + iei_cmd.install_dir = env.paths['lib'] + iei_cmd.finalize_options() + iei_cmd.run() + + def test_version_resolved_from_egg_info(self, env): + version = '1.11.0.dev0+2329eae' + self.create_foo_pkg(env, version) + + # this requirement parsing will raise a VersionConflict unless the + # .egg-info file is parsed (see #419 on BitBucket) + req = pkg_resources.Requirement.parse('foo>=1.9') + dist = pkg_resources.WorkingSet([env.paths['lib']]).find(req) + assert dist.version == version diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index 4d0c7e9f..92d0e49c 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -2,6 +2,7 @@ import os import sys import tempfile import shutil +import string import pytest @@ -25,21 +26,23 @@ def safe_repr(obj, short=False): return result return result[:pkg_resources._MAX_LENGTH] + ' [truncated]...' + class Metadata(pkg_resources.EmptyProvider): """Mock object to return metadata as if from an on-disk distribution""" - def __init__(self,*pairs): + def __init__(self, *pairs): self.metadata = dict(pairs) - def has_metadata(self,name): + def has_metadata(self, name): return name in self.metadata - def get_metadata(self,name): + def get_metadata(self, name): return self.metadata[name] - def get_metadata_lines(self,name): + def get_metadata_lines(self, name): return pkg_resources.yield_lines(self.get_metadata(name)) + dist_from_fn = pkg_resources.Distribution.from_filename class TestDistro: @@ -174,9 +177,12 @@ class TestDistro: # Activation list now includes resolved dependency assert list(ws.resolve(parse_requirements("Foo[bar]"), ad)) ==[Foo,Baz] # Requests for conflicting versions produce VersionConflict - with pytest.raises(VersionConflict): + with pytest.raises(VersionConflict) as vc: ws.resolve(parse_requirements("Foo==1.2\nFoo!=1.2"), ad) + msg = 'Foo 0.9 is installed but Foo==1.2 is required' + assert vc.value.report() == msg + def testDistroDependsOptions(self): d = self.distRequires(""" Twisted>=1.5 @@ -204,6 +210,49 @@ class TestDistro: d.requires(["foo"]) +class TestWorkingSet: + def test_find_conflicting(self): + ws = WorkingSet([]) + Foo = Distribution.from_filename("/foo_dir/Foo-1.2.egg") + ws.add(Foo) + + # create a requirement that conflicts with Foo 1.2 + req = next(parse_requirements("Foo<1.2")) + + with pytest.raises(VersionConflict) as vc: + ws.find(req) + + msg = 'Foo 1.2 is installed but Foo<1.2 is required' + assert vc.value.report() == msg + + def test_resolve_conflicts_with_prior(self): + """ + A ContextualVersionConflict should be raised when a requirement + conflicts with a prior requirement for a different package. + """ + # Create installation where Foo depends on Baz 1.0 and Bar depends on + # Baz 2.0. + ws = WorkingSet([]) + md = Metadata(('depends.txt', "Baz==1.0")) + Foo = Distribution.from_filename("/foo_dir/Foo-1.0.egg", metadata=md) + ws.add(Foo) + md = Metadata(('depends.txt', "Baz==2.0")) + Bar = Distribution.from_filename("/foo_dir/Bar-1.0.egg", metadata=md) + ws.add(Bar) + Baz = Distribution.from_filename("/foo_dir/Baz-1.0.egg") + ws.add(Baz) + Baz = Distribution.from_filename("/foo_dir/Baz-2.0.egg") + ws.add(Baz) + + with pytest.raises(VersionConflict) as vc: + ws.resolve(parse_requirements("Foo\nBar\n")) + + msg = "Baz 1.0 is installed but Baz==2.0 is required by {'Bar'}" + if pkg_resources.PY2: + msg = msg.replace("{'Bar'}", "set(['Bar'])") + assert vc.value.report() == msg + + class TestEntryPoints: def assertfields(self, ep): @@ -212,10 +261,8 @@ class TestEntryPoints: assert ep.attrs == ("TestEntryPoints",) assert ep.extras == ("x",) assert ep.load() is TestEntryPoints - assert ( - str(ep) == - "foo = pkg_resources.tests.test_resources:TestEntryPoints [x]" - ) + expect = "foo = pkg_resources.tests.test_resources:TestEntryPoints [x]" + assert str(ep) == expect def setup_method(self, method): self.dist = Distribution.from_filename( @@ -250,13 +297,21 @@ class TestEntryPoints: ep = EntryPoint.parse(spec) assert ep.name == 'html+mako' - def testRejects(self): - for ep in [ - "foo", "x=1=2", "x=a:b:c", "q=x/na", "fez=pish:tush-z", "x=f[a]>2", - ]: - try: EntryPoint.parse(ep) - except ValueError: pass - else: raise AssertionError("Should've been bad", ep) + reject_specs = "foo", "x=a:b:c", "q=x/na", "fez=pish:tush-z", "x=f[a]>2" + @pytest.mark.parametrize("reject_spec", reject_specs) + def test_reject_spec(self, reject_spec): + with pytest.raises(ValueError): + EntryPoint.parse(reject_spec) + + def test_printable_name(self): + """ + Allow any printable character in the name. + """ + # Create a name with all printable characters; strip the whitespace. + name = string.printable.strip() + spec = "{name} = module:attr".format(**locals()) + ep = EntryPoint.parse(spec) + assert ep.name == name def checkSubMap(self, m): assert len(m) == len(self.submap_expect) @@ -595,10 +650,8 @@ class TestNamespaces: pkg2_init.close() import pkg1 assert "pkg1" in pkg_resources._namespace_packages - try: - import pkg1.pkg2 - except ImportError: - self.fail("Setuptools tried to import the parent namespace package") + # attempt to import pkg2 from site-pkgs2 + import pkg1.pkg2 # check the _namespace_packages dict assert "pkg1.pkg2" in pkg_resources._namespace_packages assert pkg_resources._namespace_packages["pkg1"] == ["pkg1.pkg2"] diff --git a/pytest.ini b/pytest.ini new file mode 100755 index 00000000..351942f4 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +addopts=--doctest-modules --ignore release.py --ignore setuptools/lib2to3_ex.py --ignore tests/manual_test.py --ignore tests/shlib_test --doctest-glob=pkg_resources/api_tests.txt --ignore scripts/upload-old-releases-as-zip.py +norecursedirs=dist build *.egg @@ -4,24 +4,13 @@ install jaraco.packaging and run 'python -m jaraco.packaging.release' """ import os -import subprocess import pkg_resources pkg_resources.require('jaraco.packaging>=2.0') pkg_resources.require('wheel') - -def before_upload(): - BootstrapBookmark.add() - - -def after_push(): - BootstrapBookmark.push() - -files_with_versions = ( - 'ez_setup.py', 'setuptools/version.py', -) +files_with_versions = 'setuptools/version.py', # bdist_wheel must be included or pip will break dist_commands = 'sdist', 'bdist_wheel' @@ -29,22 +18,3 @@ dist_commands = 'sdist', 'bdist_wheel' test_info = "Travis-CI tests: http://travis-ci.org/#!/jaraco/setuptools" os.environ["SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES"] = "1" - -class BootstrapBookmark: - name = 'bootstrap' - - @classmethod - def add(cls): - cmd = ['hg', 'bookmark', '-i', cls.name, '-f'] - subprocess.Popen(cmd) - - @classmethod - def push(cls): - """ - Push the bootstrap bookmark - """ - push_command = ['hg', 'push', '-B', cls.name] - # don't use check_call here because mercurial will return a non-zero - # code even if it succeeds at pushing the bookmark (because there are - # no changesets to be pushed). !dm mercurial - subprocess.call(push_command) diff --git a/scripts/upload-old-releases-as-zip.py b/scripts/upload-old-releases-as-zip.py new file mode 100644 index 00000000..38cfcd55 --- /dev/null +++ b/scripts/upload-old-releases-as-zip.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# declare and require dependencies +__requires__ = [ + 'twine', +]; __import__('pkg_resources') + +import errno +import glob +import hashlib +import json +import os +import shutil +import tarfile +import codecs +import urllib.request +import urllib.parse +import urllib.error +from distutils.version import LooseVersion + +from twine.commands import upload + + +OK = '\033[92m' +FAIL = '\033[91m' +END = '\033[0m' +DISTRIBUTION = "setuptools" + + +class SetuptoolsOldReleasesWithoutZip: + """docstring for SetuptoolsOldReleases""" + + def __init__(self): + self.dirpath = './dist' + os.makedirs(self.dirpath, exist_ok=True) + print("Downloading %s releases..." % DISTRIBUTION) + print("All releases will be downloaded to %s" % self.dirpath) + self.data_json_setuptools = self.get_json_data(DISTRIBUTION) + self.valid_releases_numbers = sorted([ + release + for release in self.data_json_setuptools['releases'] + # This condition is motivated by 13.0 release, which + # comes as "13.0": [], in the json + if self.data_json_setuptools['releases'][release] + ], key=LooseVersion) + self.total_downloaded_ok = 0 + + def get_json_data(self, package_name): + """ + "releases": { + "0.7.2": [ + { + "has_sig": false, + "upload_time": "2013-06-09T16:10:00", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.2.tar.gz", # NOQA + "md5_digest": "de44cd90f8a1c713d6c2bff67bbca65d", + "downloads": 159014, + "filename": "setuptools-0.7.2.tar.gz", + "packagetype": "sdist", + "size": 633077 + } + ], + "0.7.3": [ + { + "has_sig": false, + "upload_time": "2013-06-18T21:08:56", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.3.tar.gz", # NOQA + "md5_digest": "c854adacbf9067d330a847f06f7a8eba", + "downloads": 30594, + "filename": "setuptools-0.7.3.tar.gz", + "packagetype": "sdist", + "size": 751152 + } + ], + "12.3": [ + { + "has_sig": false, + "upload_time": "2015-02-26T19:15:51", + "comment_text": "", + "python_version": "3.4", + "url": "https://pypi.python.org/packages/3.4/s/setuptools/setuptools-12.3-py2.py3-none-any.whl", # NOQA + "md5_digest": "31f51a38497a70efadf5ce8d4c2211ab", + "downloads": 288451, + "filename": "setuptools-12.3-py2.py3-none-any.whl", + "packagetype": "bdist_wheel", + "size": 501904 + }, + { + "has_sig": false, + "upload_time": "2015-02-26T19:15:43", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.tar.gz", # NOQA + "md5_digest": "67614b6d560fa4f240e99cd553ec7f32", + "downloads": 110109, + "filename": "setuptools-12.3.tar.gz", + "packagetype": "sdist", + "size": 635025 + }, + { + "has_sig": false, + "upload_time": "2015-02-26T19:15:47", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.zip", # NOQA + "md5_digest": "abc799e7db6e7281535bf342bfc41a12", + "downloads": 67539, + "filename": "setuptools-12.3.zip", + "packagetype": "sdist", + "size": 678783 + } + ], + """ + url = "https://pypi.python.org/pypi/%s/json" % (package_name,) + resp = urllib.request.urlopen(urllib.request.Request(url)) + charset = resp.info().get_content_charset() + reader = codecs.getreader(charset)(resp) + data = json.load(reader) + + # Mainly for debug. + json_filename = "%s/%s.json" % (self.dirpath, DISTRIBUTION) + with open(json_filename, 'w') as outfile: + json.dump( + data, + outfile, + sort_keys=True, + indent=4, + separators=(',', ': '), + ) + + return data + + def get_setuptools_releases_without_zip_counterpart(self): + # Get set(all_valid_releases) - set(releases_with_zip), so now we have + # the releases without zip. + return set(self.valid_releases_numbers) - set([ + release + for release in self.valid_releases_numbers + for same_version_release_dict in self.data_json_setuptools['releases'][release] # NOQA + if 'zip' in same_version_release_dict['filename'] + ]) + + def download_setuptools_releases_without_zip_counterpart(self): + try: + releases_without_zip = self.get_setuptools_releases_without_zip_counterpart() # NOQA + failed_md5_releases = [] + # This is a "strange" loop, going through all releases and + # testing only the release I need to download, but I thought it + # would be mouch more readable than trying to iterate through + # releases I need and get into traverse hell values inside dicts + # inside dicts of the json to get the distribution's url to + # download. + for release in self.valid_releases_numbers: + if release in releases_without_zip: + for same_version_release_dict in self.data_json_setuptools['releases'][release]: # NOQA + if 'tar.gz' in same_version_release_dict['filename']: + print("Downloading %s..." % release) + local_file = '%s/%s' % ( + self.dirpath, + same_version_release_dict["filename"] + ) + urllib.request.urlretrieve( + same_version_release_dict["url"], + local_file + ) + targz = open(local_file, 'rb').read() + hexdigest = hashlib.md5(targz).hexdigest() + if (hexdigest != same_version_release_dict['md5_digest']): # NOQA + print(FAIL + "FAIL: md5 for %s didn't match!" % release + END) # NOQA + failed_md5_releases.append(release) + else: + self.total_downloaded_ok += 1 + print('Total releases without zip: %s' % len(releases_without_zip)) + print('Total downloaded: %s' % self.total_downloaded_ok) + if failed_md5_releases: + msg = FAIL + ( + "FAIL: these releases %s failed the md5 check!" % + ','.join(failed_md5_releases) + ) + END + raise Exception(msg) + elif self.total_downloaded_ok != len(releases_without_zip): + msg = FAIL + ( + "FAIL: Unknown error occured. Please check the logs." + ) + END + raise Exception(msg) + else: + print(OK + "All releases downloaded and md5 checked." + END) + + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + def convert_targz_to_zip(self): + print("Converting the tar.gz to zip...") + files = glob.glob('%s/*.tar.gz' % self.dirpath) + total_converted = 0 + for targz in sorted(files, key=LooseVersion): + # Extract and remove tar. + tar = tarfile.open(targz) + tar.extractall(path=self.dirpath) + tar.close() + os.remove(targz) + + # Zip the extracted tar. + setuptools_folder_path = targz.replace('.tar.gz', '') + setuptools_folder_name = setuptools_folder_path.split("/")[-1] + print(setuptools_folder_name) + shutil.make_archive( + setuptools_folder_path, + 'zip', + self.dirpath, + setuptools_folder_name + ) + # Exclude extracted tar folder. + shutil.rmtree(setuptools_folder_path.replace('.zip', '')) + total_converted += 1 + print('Total converted: %s' % total_converted) + if self.total_downloaded_ok != total_converted: + msg = FAIL + ( + "FAIL: Total number of downloaded releases is different" + " from converted ones. Please check the logs." + ) + END + raise Exception(msg) + print("Done with the tar.gz->zip. Check folder %s." % main.dirpath) + + def upload_zips_to_pypi(self): + print('Uploading to pypi...') + zips = sorted(glob.glob('%s/*.zip' % self.dirpath), key=LooseVersion) + print("simulated upload of", zips); return + upload.upload(dists=zips) + + +if __name__ == '__main__': + main = SetuptoolsOldReleasesWithoutZip() + main.download_setuptools_releases_without_zip_counterpart() + main.convert_targz_to_zip() + main.upload_zips_to_pypi() @@ -5,6 +5,7 @@ tag_build = dev release = egg_info -RDb '' source = register sdist binary binary = bdist_egg upload --show-response +test = pytest [build_sphinx] source-dir = docs/ @@ -19,7 +20,3 @@ formats = gztar zip [wheel] universal=1 - -[pytest] -addopts=--doctest-modules --ignore release.py --ignore setuptools/lib2to3_ex.py --ignore tests/manual_test.py --ignore tests/shlib_test --doctest-glob=pkg_resources/api_tests.txt -norecursedirs=dist build *.egg @@ -1,5 +1,8 @@ #!/usr/bin/env python -"""Distutils setup file, used to install or test 'setuptools'""" +""" +Distutils setup file, used to install or test 'setuptools' +""" + import io import os import sys @@ -25,7 +28,6 @@ with open(ver_path) as ver_file: exec(ver_file.read(), main_ns) import setuptools -from setuptools.command.build_py import build_py as _build_py scripts = [] @@ -46,20 +48,6 @@ def _gen_console_scripts(): console_scripts = list(_gen_console_scripts()) - -# specific command that is used to generate windows .exe files -class build_py(_build_py): - def build_package_data(self): - """Copy data files into build directory""" - for package, src_dir, build_dir, filenames in self.data_files: - for filename in filenames: - target = os.path.join(build_dir, filename) - self.mkpath(os.path.dirname(target)) - srcfile = os.path.join(src_dir, filename) - outf, copied = self.copy_file(srcfile, target) - srcfile = os.path.abspath(srcfile) - - readme_file = io.open('README.txt', encoding='utf-8') with readme_file: @@ -75,7 +63,10 @@ if sys.platform == 'win32' or force_windows_specific_files: package_data.setdefault('setuptools', []).extend(['*.exe']) package_data.setdefault('setuptools.command', []).extend(['*.xml']) -pytest_runner = ['pytest-runner'] if 'ptr' in sys.argv else [] +needs_pytest = set(['ptr', 'pytest', 'test']).intersection(sys.argv) +pytest_runner = ['pytest-runner'] if needs_pytest else [] +needs_sphinx = set(['build_sphinx', 'upload_docs']).intersection(sys.argv) +sphinx = ['sphinx', 'rst.linker'] if needs_sphinx else [] setup_params = dict( name="setuptools", @@ -89,7 +80,7 @@ setup_params = dict( keywords="CPAN PyPI distutils eggs package management", url="https://bitbucket.org/pypa/setuptools", src_root=src_root, - packages=setuptools.find_packages(), + packages=setuptools.find_packages(exclude=['*.tests']), package_data=package_data, py_modules=['easy_install'], @@ -149,10 +140,9 @@ setup_params = dict( Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.1 - Programming Language :: Python :: 3.2 Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 Topic :: Software Development :: Libraries :: Python Modules Topic :: System :: Archiving :: Packaging Topic :: System :: Systems Administration @@ -163,20 +153,19 @@ setup_params = dict( ], extras_require={ "ssl:sys_platform=='win32'": "wincertstore==0.2", - "certs": "certifi==1.0.1", + "certs": "certifi==2015.11.20", }, dependency_links=[ - 'https://pypi.python.org/packages/source/c/certifi/certifi-1.0.1.tar.gz#md5=45f5cb94b8af9e1df0f9450a8f61b790', + 'https://pypi.python.org/packages/source/c/certifi/certifi-2015.11.20.tar.gz#md5=25134646672c695c1ff1593c2dd75d08', 'https://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2', ], scripts=[], tests_require=[ 'setuptools[ssl]', - 'pytest', - 'mock', - ], + 'pytest>=2.8', + ] + (['mock'] if sys.version_info[:2] < (3, 3) else []), setup_requires=[ - ] + pytest_runner, + ] + sphinx + pytest_runner, ) if __name__ == '__main__': diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 57236a5b..fffcac76 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -3,6 +3,7 @@ __import__('setuptools.bootstrap').bootstrap.ensure_deps() import os +import functools import distutils.core import distutils.filelist from distutils.core import Command as _Command @@ -76,21 +77,24 @@ class PackageFinder(object): yield pkg @staticmethod - def _all_dirs(base_path): + def _candidate_dirs(base_path): """ - Return all dirs in base_path, relative to base_path + Return all dirs in base_path that might be packages. """ + has_dot = lambda name: '.' in name for root, dirs, files in os.walk(base_path, followlinks=True): + # Exclude directories that contain a period, as they cannot be + # packages. Mutate the list to avoid traversal. + dirs[:] = filterfalse(has_dot, dirs) for dir in dirs: yield os.path.relpath(os.path.join(root, dir), base_path) @classmethod def _find_packages_iter(cls, base_path): - dirs = cls._all_dirs(base_path) - suitable = filterfalse(lambda n: '.' in n, dirs) + candidates = cls._candidate_dirs(base_path) return ( path.replace(os.path.sep, '.') - for path in suitable + for path in candidates if cls._looks_like_package(os.path.join(base_path, path)) ) @@ -123,30 +127,45 @@ class Command(_Command): command_consumes_arguments = False def __init__(self, dist, **kw): - # Add support for keyword arguments - _Command.__init__(self,dist) - for k,v in kw.items(): - setattr(self,k,v) + """ + Construct the command for dist, updating + vars(self) with any keyword parameters. + """ + _Command.__init__(self, dist) + vars(self).update(kw) def reinitialize_command(self, command, reinit_subcommands=0, **kw): cmd = _Command.reinitialize_command(self, command, reinit_subcommands) - for k,v in kw.items(): - setattr(cmd,k,v) # update command with keywords + vars(cmd).update(kw) return cmd -distutils.core.Command = Command # we can't patch distutils.cmd, alas +# we can't patch distutils.cmd, alas +distutils.core.Command = Command + + +def _find_all_simple(path): + """ + Find all files under 'path' + """ + results = ( + os.path.join(base, file) + for base, dirs, files in os.walk(path, followlinks=True) + for file in files + ) + return filter(os.path.isfile, results) + -def findall(dir = os.curdir): - """Find all files under 'dir' and return the list of full filenames - (relative to 'dir'). +def findall(dir=os.curdir): + """ + Find all files under 'dir' and return the list of full filenames. + Unless dir is '.', return full filenames with dir prepended. """ - all_files = [] - for base, dirs, files in os.walk(dir, followlinks=True): - if base==os.curdir or base.startswith(os.curdir+os.sep): - base = base[2:] - if base: - files = [os.path.join(base, f) for f in files] - all_files.extend(filter(os.path.isfile, files)) - return all_files - -distutils.filelist.findall = findall # fix findall bug in distutils. + files = _find_all_simple(dir) + if dir == os.curdir: + make_rel = functools.partial(os.path.relpath, start=dir) + files = map(make_rel, files) + return list(files) + + +# fix findall bug in distutils (http://bugs.python.org/issue12885) +distutils.filelist.findall = findall diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 3d241b99..73f8e3f1 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -2,7 +2,6 @@ Build .egg distributions""" -# This module should be kept compatible with Python 2.3 from distutils.errors import DistutilsSetupError from distutils.dir_util import remove_tree, mkpath from distutils import log @@ -407,10 +406,6 @@ def scan_module(egg_dir, base, name, stubs): if bad in symbols: log.warn("%s: module MAY be using inspect.%s", module, bad) safe = False - if '__name__' in symbols and '__main__' in symbols and '.' not in module: - if sys.version[:3] == "2.4": # -m works w/zipfiles in 2.5 - log.warn("%s: top-level module may be 'python -m' script", module) - safe = False return safe @@ -442,7 +437,7 @@ INSTALL_DIRECTORY_ATTRS = [ ] -def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=None, +def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True, mode='w'): """Create a zip file from all the files under 'base_dir'. The output zip file will be named 'base_dir' + ".zip". Uses either the "zipfile" @@ -464,11 +459,7 @@ def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=None, z.write(path, p) log.debug("adding '%s'" % p) - if compress is None: - # avoid 2.3 zipimport bug when 64 bits - compress = (sys.version >= "2.4") - - compression = [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED][bool(compress)] + compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED if not dry_run: z = zipfile.ZipFile(zip_filename, mode, compression=compression) for dirname, dirs, files in os.walk(base_dir): diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index e4b2c593..92e4a189 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -11,8 +11,8 @@ import itertools from setuptools.extension import Library try: - # Attempt to use Pyrex for building extensions, if available - from Pyrex.Distutils.build_ext import build_ext as _build_ext + # Attempt to use Cython for building extensions, if available + from Cython.Distutils.build_ext import build_ext as _build_ext except ImportError: _build_ext = _du_build_ext @@ -42,7 +42,6 @@ elif os.name != 'nt': if_dl = lambda s: s if have_rtld else '' - class build_ext(_build_ext): def run(self): """Build extensions in build directory, then copy if --inplace""" @@ -74,15 +73,6 @@ class build_ext(_build_ext): if ext._needs_stub: self.write_stub(package_dir or os.curdir, ext, True) - if _build_ext is not _du_build_ext and not hasattr(_build_ext, - 'pyrex_sources'): - # Workaround for problems using some Pyrex versions w/SWIG and/or 2.4 - def swig_sources(self, sources, *otherargs): - # first do any Pyrex processing - sources = _build_ext.swig_sources(self, sources) or sources - # Then do any actual SWIG stuff on the remainder - return _du_build_ext.swig_sources(self, sources, *otherargs) - def get_ext_filename(self, fullname): filename = _build_ext.get_ext_filename(self, fullname) if fullname in self.ext_map: @@ -176,6 +166,7 @@ class build_ext(_build_ext): return _build_ext.get_export_symbols(self, ext) def build_extension(self, ext): + ext._convert_pyx_sources_to_lang() _compiler = self.compiler try: if isinstance(ext, Library): diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 98080694..8a50f032 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -2,9 +2,13 @@ from glob import glob from distutils.util import convert_path import distutils.command.build_py as orig import os -import sys import fnmatch import textwrap +import io +import distutils.errors +import collections +import itertools + try: from setuptools.lib2to3_ex import Mixin2to3 @@ -136,22 +140,7 @@ class build_py(orig.build_py, Mixin2to3): mf.setdefault(src_dirs[d], []).append(path) def get_data_files(self): - pass # kludge 2.4 for lazy computation - - if sys.version < "2.4": # Python 2.4 already has this code - def get_outputs(self, include_bytecode=1): - """Return complete list of files copied to the build directory - - This includes both '.py' files and data files, as well as '.pyc' - and '.pyo' files if 'include_bytecode' is true. (This method is - needed for the 'install_lib' command to do its job properly, and to - generate a correct installation manifest.) - """ - return orig.build_py.get_outputs(self, include_bytecode) + [ - os.path.join(build_dir, filename) - for package, src_dir, build_dir, filenames in self.data_files - for filename in filenames - ] + pass # Lazily compute data files in _get_data_files() function. def check_package(self, package, package_dir): """Check namespace packages' __init__ for declare_namespace""" @@ -172,17 +161,15 @@ class build_py(orig.build_py, Mixin2to3): else: return init_py - f = open(init_py, 'rbU') - if 'declare_namespace'.encode() not in f.read(): - from distutils.errors import DistutilsError - - raise DistutilsError( + with io.open(init_py, 'rb') as f: + contents = f.read() + if b'declare_namespace' not in contents: + raise distutils.errors.DistutilsError( "Namespace package problem: %s is a namespace package, but " "its\n__init__.py does not call declare_namespace()! Please " 'fix it.\n(See the setuptools manual under ' '"Namespace Packages" for details.)\n"' % (package,) ) - f.close() return init_py def initialize_options(self): @@ -197,20 +184,25 @@ class build_py(orig.build_py, Mixin2to3): def exclude_data_files(self, package, src_dir, files): """Filter filenames for package's data files in 'src_dir'""" - globs = (self.exclude_package_data.get('', []) - + self.exclude_package_data.get(package, [])) - bad = [] - for pattern in globs: - bad.extend( - fnmatch.filter( - files, os.path.join(src_dir, convert_path(pattern)) - ) + globs = ( + self.exclude_package_data.get('', []) + + self.exclude_package_data.get(package, []) + ) + bad = set( + item + for pattern in globs + for item in fnmatch.filter( + files, + os.path.join(src_dir, convert_path(pattern)), ) - bad = dict.fromkeys(bad) - seen = {} + ) + seen = collections.defaultdict(itertools.count) return [ - f for f in files if f not in bad - and f not in seen and seen.setdefault(f, 1) # ditch dupes + fn + for fn in files + if fn not in bad + # ditch dupes + and not next(seen[fn]) ] diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index 9f0b6f47..ef9ac22d 100755 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -3,6 +3,7 @@ from distutils import log from distutils.errors import DistutilsError, DistutilsOptionError import os import glob +import io import six @@ -54,8 +55,8 @@ class develop(easy_install): # pick up setup-dir .egg files only: no .egg-info self.package_index.scan(glob.glob('*.egg')) - self.egg_link = os.path.join(self.install_dir, ei.egg_name + - '.egg-link') + egg_link_fn = ei.egg_name + '.egg-link' + self.egg_link = os.path.join(self.install_dir, egg_link_fn) self.egg_base = ei.egg_base if self.egg_path is None: self.egg_path = os.path.abspath(ei.egg_base) @@ -125,9 +126,8 @@ class develop(easy_install): # create an .egg-link in the installation dir, pointing to our egg log.info("Creating %s (link to %s)", self.egg_link, self.egg_base) if not self.dry_run: - f = open(self.egg_link, "w") - f.write(self.egg_path + "\n" + self.setup_path) - f.close() + with open(self.egg_link, "w") as f: + f.write(self.egg_path + "\n" + self.setup_path) # postprocess the installed distro, fixing up .pth, installing scripts, # and handling requirements self.process_distribution(None, self.dist, not self.no_deps) @@ -164,7 +164,33 @@ class develop(easy_install): for script_name in self.distribution.scripts or []: script_path = os.path.abspath(convert_path(script_name)) script_name = os.path.basename(script_path) - f = open(script_path, 'rU') - script_text = f.read() - f.close() + with io.open(script_path) as strm: + script_text = strm.read() self.install_script(dist, script_name, script_text, script_path) + + def install_wrapper_scripts(self, dist): + dist = VersionlessRequirement(dist) + return easy_install.install_wrapper_scripts(self, dist) + + +class VersionlessRequirement(object): + """ + Adapt a pkg_resources.Distribution to simply return the project + name as the 'requirement' so that scripts will work across + multiple versions. + + >>> dist = Distribution(project_name='foo', version='1.0') + >>> str(dist.as_requirement()) + 'foo==1.0' + >>> adapted_dist = VersionlessRequirement(dist) + >>> str(adapted_dist.as_requirement()) + 'foo' + """ + def __init__(self, dist): + self.__dist = dist + + def __getattr__(self, name): + return getattr(self.__dist, name) + + def as_requirement(self): + return self.project_name diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index a24e3b59..6aab38c8 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -20,6 +20,7 @@ from distutils.errors import DistutilsArgError, DistutilsOptionError, \ from distutils.command.install import INSTALL_SCHEMES, SCHEME_KEYS from distutils import log, dir_util from distutils.command.build_scripts import first_line_re +from distutils.spawn import find_executable import sys import os import zipimport @@ -35,6 +36,9 @@ import warnings import site import struct import contextlib +import subprocess +import shlex +import io import six from six.moves import configparser @@ -55,15 +59,10 @@ from pkg_resources import ( ) import pkg_resources - # Turn on PEP440Warnings warnings.filterwarnings("default", category=pkg_resources.PEP440Warning) -sys_executable = os.environ.get('__PYVENV_LAUNCHER__', - os.path.normpath(sys.executable)) - - __all__ = [ 'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg', 'main', 'get_exe_prefixes', @@ -155,12 +154,9 @@ class easy_install(Command): create_index = PackageIndex def initialize_options(self): - if site.ENABLE_USER_SITE: - whereami = os.path.abspath(__file__) - self.user = whereami.startswith(site.USER_SITE) - else: - self.user = 0 - + # the --user option seems to be an opt-in one, + # so the default should be False. + self.user = 0 self.zip_ok = self.local_snapshots_ok = None self.install_dir = self.script_dir = self.exclude_scripts = None self.index_url = None @@ -206,20 +202,34 @@ class easy_install(Command): ) def delete_blockers(self, blockers): - for filename in blockers: - if os.path.exists(filename) or os.path.islink(filename): - log.info("Deleting %s", filename) - if not self.dry_run: - if (os.path.isdir(filename) and - not os.path.islink(filename)): - rmtree(filename) - else: - os.unlink(filename) + extant_blockers = ( + filename for filename in blockers + if os.path.exists(filename) or os.path.islink(filename) + ) + list(map(self._delete_path, extant_blockers)) + + def _delete_path(self, path): + log.info("Deleting %s", path) + if self.dry_run: + return + + is_tree = os.path.isdir(path) and not os.path.islink(path) + remover = rmtree if is_tree else os.unlink + remover(path) + + @staticmethod + def _render_version(): + """ + Render the Setuptools version and installation details, then exit. + """ + ver = sys.version[:3] + dist = get_distribution('setuptools') + tmpl = 'setuptools {dist.version} from {dist.location} (Python {ver})' + print(tmpl.format(**locals())) + raise SystemExit() def finalize_options(self): - if self.version: - print('setuptools %s' % get_distribution('setuptools').version) - sys.exit() + self.version and self._render_version() py_version = sys.version.split()[0] prefix, exec_prefix = get_config_vars('prefix', 'exec_prefix') @@ -243,18 +253,7 @@ class easy_install(Command): self.config_vars['userbase'] = self.install_userbase self.config_vars['usersite'] = self.install_usersite - # fix the install_dir if "--user" was used - # XXX: duplicate of the code in the setup command - if self.user and site.ENABLE_USER_SITE: - self.create_home_path() - if self.install_userbase is None: - raise DistutilsPlatformError( - "User base directory is not specified") - self.install_base = self.install_platbase = self.install_userbase - if os.name == 'posix': - self.select_scheme("unix_user") - else: - self.select_scheme(os.name + "_user") + self._fix_install_dir_for_user_site() self.expand_basedirs() self.expand_dirs() @@ -349,6 +348,21 @@ class easy_install(Command): self.outputs = [] + def _fix_install_dir_for_user_site(self): + """ + Fix the install_dir if "--user" was used. + """ + if not self.user or not site.ENABLE_USER_SITE: + return + + self.create_home_path() + if self.install_userbase is None: + msg = "User base directory is not specified" + raise DistutilsPlatformError(msg) + self.install_base = self.install_platbase = self.install_userbase + scheme_name = os.name.replace('posix', 'unix') + '_user' + self.select_scheme(scheme_name) + def _expand_attrs(self, attrs): for attr in attrs: val = getattr(self, attr) @@ -441,7 +455,7 @@ class easy_install(Command): self.pth_file = None PYTHONPATH = os.environ.get('PYTHONPATH', '').split(os.pathsep) - if instdir not in map(normalize_path, [_f for _f in PYTHONPATH if _f]): + if instdir not in map(normalize_path, filter(None, PYTHONPATH)): # only PYTHONPATH dirs need a site.py, so pretend it's there self.sitepy_installed = True elif self.multi_version and not os.path.exists(pth_file): @@ -449,43 +463,49 @@ class easy_install(Command): self.pth_file = None # and don't create a .pth file self.install_dir = instdir - def cant_write_to_target(self): - template = """can't create or remove files in install directory + __cant_write_msg = textwrap.dedent(""" + can't create or remove files in install directory -The following error occurred while trying to add or remove files in the -installation directory: + The following error occurred while trying to add or remove files in the + installation directory: - %s + %s -The installation directory you specified (via --install-dir, --prefix, or -the distutils default setting) was: + The installation directory you specified (via --install-dir, --prefix, or + the distutils default setting) was: - %s -""" - msg = template % (sys.exc_info()[1], self.install_dir,) + %s + """).lstrip() - if not os.path.exists(self.install_dir): - msg += """ -This directory does not currently exist. Please create it and try again, or -choose a different installation directory (using the -d or --install-dir -option). -""" - else: - msg += """ -Perhaps your account does not have write access to this directory? If the -installation directory is a system-owned directory, you may need to sign in -as the administrator or "root" account. If you do not have administrative -access to this machine, you may wish to choose a different installation -directory, preferably one that is listed in your PYTHONPATH environment -variable. + __not_exists_id = textwrap.dedent(""" + This directory does not currently exist. Please create it and try again, or + choose a different installation directory (using the -d or --install-dir + option). + """).lstrip() -For information on other options, you may wish to consult the -documentation at: + __access_msg = textwrap.dedent(""" + Perhaps your account does not have write access to this directory? If the + installation directory is a system-owned directory, you may need to sign in + as the administrator or "root" account. If you do not have administrative + access to this machine, you may wish to choose a different installation + directory, preferably one that is listed in your PYTHONPATH environment + variable. - https://pythonhosted.org/setuptools/easy_install.html + For information on other options, you may wish to consult the + documentation at: -Please make the appropriate changes for your system and try again. -""" + https://pythonhosted.org/setuptools/easy_install.html + + Please make the appropriate changes for your system and try again. + """).lstrip() + + def cant_write_to_target(self): + msg = self.__cant_write_msg % (sys.exc_info()[1], self.install_dir,) + + if not os.path.exists(self.install_dir): + msg += '\n' + self.__not_exists_id + else: + msg += '\n' + self.__access_msg raise DistutilsError(msg) def check_pth_processing(self): @@ -699,17 +719,10 @@ Please make the appropriate changes for your system and try again. distros = WorkingSet([]).resolve( [requirement], self.local_index, self.easy_install ) - except DistributionNotFound: - e = sys.exc_info()[1] - raise DistutilsError( - "Could not find required distribution %s" % e.args - ) - except VersionConflict: - e = sys.exc_info()[1] - raise DistutilsError( - "Installed distribution %s conflicts with requirement %s" - % e.args - ) + except DistributionNotFound as e: + raise DistutilsError(str(e)) + except VersionConflict as e: + raise DistutilsError(e.report()) if self.always_copy or self.always_copy_from: # Force all the relevant distros to be copied or activated for dist in distros: @@ -749,9 +762,10 @@ Please make the appropriate changes for your system and try again. return dst def install_wrapper_scripts(self, dist): - if not self.exclude_scripts: - for args in get_script_args(dist): - self.write_script(*args) + if self.exclude_scripts: + return + for args in ScriptWriter.best().get_args(dist): + self.write_script(*args) def install_script(self, dist, script_name, script_text, dev_path=None): """Generate a legacy script wrapper and install it""" @@ -759,8 +773,8 @@ Please make the appropriate changes for your system and try again. is_script = is_python_script(script_text, script_name) if is_script: - script_text = (get_script_header(script_text) + - self._load_template(dev_path) % locals()) + body = self._load_template(dev_path) % locals() + script_text = ScriptWriter.get_header(script_text) + body self.write_script(script_name, _to_ascii(script_text), 'b') @staticmethod @@ -792,9 +806,8 @@ Please make the appropriate changes for your system and try again. ensure_directory(target) if os.path.exists(target): os.unlink(target) - f = open(target, "w" + mode) - f.write(contents) - f.close() + with open(target, "w" + mode) as f: + f.write(contents) chmod(target, 0o777 - mask) def install_eggs(self, spec, dist_filename, tmpdir): @@ -923,9 +936,10 @@ Please make the appropriate changes for your system and try again. f.write('%s: %s\n' % (k.replace('_', '-').title(), v)) f.close() script_dir = os.path.join(_egg_info, 'scripts') - self.delete_blockers( # delete entry-point scripts to avoid duping + # delete entry-point scripts to avoid duping + self.delete_blockers( [os.path.join(script_dir, args[0]) for args in - get_script_args(dist)] + ScriptWriter.get_args(dist)] ) # Build .egg file from tmpdir bdist_egg.make_zipfile( @@ -987,46 +1001,52 @@ Please make the appropriate changes for your system and try again. f.write('\n'.join(locals()[name]) + '\n') f.close() + __mv_warning = textwrap.dedent(""" + Because this distribution was installed --multi-version, before you can + import modules from this package in an application, you will need to + 'import pkg_resources' and then use a 'require()' call similar to one of + these examples, in order to select the desired version: + + pkg_resources.require("%(name)s") # latest installed version + pkg_resources.require("%(name)s==%(version)s") # this exact version + pkg_resources.require("%(name)s>=%(version)s") # this version or higher + """).lstrip() + + __id_warning = textwrap.dedent(""" + Note also that the installation directory must be on sys.path at runtime for + this to work. (e.g. by being the application's script directory, by being on + PYTHONPATH, or by being added to sys.path by your code.) + """) + def installation_report(self, req, dist, what="Installed"): """Helpful installation message for display to package users""" msg = "\n%(what)s %(eggloc)s%(extras)s" if self.multi_version and not self.no_report: - msg += """ - -Because this distribution was installed --multi-version, before you can -import modules from this package in an application, you will need to -'import pkg_resources' and then use a 'require()' call similar to one of -these examples, in order to select the desired version: - - pkg_resources.require("%(name)s") # latest installed version - pkg_resources.require("%(name)s==%(version)s") # this exact version - pkg_resources.require("%(name)s>=%(version)s") # this version or higher -""" + msg += '\n' + self.__mv_warning if self.install_dir not in map(normalize_path, sys.path): - msg += """ + msg += '\n' + self.__id_warning -Note also that the installation directory must be on sys.path at runtime for -this to work. (e.g. by being the application's script directory, by being on -PYTHONPATH, or by being added to sys.path by your code.) -""" eggloc = dist.location name = dist.project_name version = dist.version extras = '' # TODO: self.report_extras(req, dist) return msg % locals() - def report_editable(self, spec, setup_script): - dirname = os.path.dirname(setup_script) - python = sys.executable - return """\nExtracted editable version of %(spec)s to %(dirname)s + __editable_msg = textwrap.dedent(""" + Extracted editable version of %(spec)s to %(dirname)s -If it uses setuptools in its setup script, you can activate it in -"development" mode by going to that directory and running:: + If it uses setuptools in its setup script, you can activate it in + "development" mode by going to that directory and running:: - %(python)s setup.py develop + %(python)s setup.py develop -See the setuptools documentation for the "develop" command for more info. -""" % locals() + See the setuptools documentation for the "develop" command for more info. + """).lstrip() + + def report_editable(self, spec, setup_script): + dirname = os.path.dirname(setup_script) + python = sys.executable + return '\n' + self.__editable_msg % locals() def run_setup(self, setup_script, setup_base, args): sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg) @@ -1045,8 +1065,7 @@ See the setuptools documentation for the "develop" command for more info. ) try: run_setup(setup_script, args) - except SystemExit: - v = sys.exc_info()[1] + except SystemExit as v: raise DistutilsError("Setup script exited with %s" % (v.args[0],)) def build_and_install(self, setup_script, setup_base): @@ -1178,35 +1197,38 @@ See the setuptools documentation for the "develop" command for more info. finally: log.set_verbosity(self.verbose) # restore original verbosity - def no_default_version_msg(self): - template = """bad install directory or PYTHONPATH + __no_default_msg = textwrap.dedent(""" + bad install directory or PYTHONPATH + + You are attempting to install a package to a directory that is not + on PYTHONPATH and which Python does not read ".pth" files from. The + installation directory you specified (via --install-dir, --prefix, or + the distutils default setting) was: -You are attempting to install a package to a directory that is not -on PYTHONPATH and which Python does not read ".pth" files from. The -installation directory you specified (via --install-dir, --prefix, or -the distutils default setting) was: + %s - %s + and your PYTHONPATH environment variable currently contains: -and your PYTHONPATH environment variable currently contains: + %r - %r + Here are some of your options for correcting the problem: -Here are some of your options for correcting the problem: + * You can choose a different installation directory, i.e., one that is + on PYTHONPATH or supports .pth files -* You can choose a different installation directory, i.e., one that is - on PYTHONPATH or supports .pth files + * You can add the installation directory to the PYTHONPATH environment + variable. (It must then also be on PYTHONPATH whenever you run + Python and want to use the package(s) you are installing.) -* You can add the installation directory to the PYTHONPATH environment - variable. (It must then also be on PYTHONPATH whenever you run - Python and want to use the package(s) you are installing.) + * You can set up the installation directory to support ".pth" files by + using one of the approaches described here: -* You can set up the installation directory to support ".pth" files by - using one of the approaches described here: + https://pythonhosted.org/setuptools/easy_install.html#custom-installation-locations - https://pythonhosted.org/setuptools/easy_install.html#custom-installation-locations + Please make the appropriate changes for your system and try again.""").lstrip() -Please make the appropriate changes for your system and try again.""" + def no_default_version_msg(self): + template = self.__no_default_msg return template % (self.install_dir, os.environ.get('PYTHONPATH', '')) def install_site_py(self): @@ -1403,13 +1425,8 @@ def extract_wininst_cfg(dist_filename): {'version': '', 'target_version': ''}) try: part = f.read(cfglen) - # part is in bytes, but we need to read up to the first null - # byte. - if sys.version_info >= (2, 6): - null_byte = bytes([0]) - else: - null_byte = chr(0) - config = part.split(null_byte, 1)[0] + # Read up to the first null byte. + config = part.split(b'\0', 1)[0] # Now the config is in bytes, but for RawConfigParser, it should # be text, so decode it. config = config.decode(sys.getfilesystemencoding()) @@ -1521,23 +1538,16 @@ class PthDistributions(Environment): if not self.dirty: return - data = '\n'.join(map(self.make_relative, self.paths)) - if data: + rel_paths = list(map(self.make_relative, self.paths)) + if rel_paths: log.debug("Saving %s", self.filename) - data = ( - "import sys; sys.__plen = len(sys.path)\n" - "%s\n" - "import sys; new=sys.path[sys.__plen:];" - " del sys.path[sys.__plen:];" - " p=getattr(sys,'__egginsert',0); sys.path[p:p]=new;" - " sys.__egginsert = p+len(new)\n" - ) % data + lines = self._wrap_lines(rel_paths) + data = '\n'.join(lines) + '\n' if os.path.islink(self.filename): os.unlink(self.filename) - f = open(self.filename, 'wt') - f.write(data) - f.close() + with open(self.filename, 'wt') as f: + f.write(data) elif os.path.exists(self.filename): log.debug("Deleting empty %s", self.filename) @@ -1545,6 +1555,10 @@ class PthDistributions(Environment): self.dirty = False + @staticmethod + def _wrap_lines(lines): + return lines + def add(self, dist): """Add `dist` to the distribution map""" new_path = ( @@ -1582,6 +1596,34 @@ class PthDistributions(Environment): return path +class RewritePthDistributions(PthDistributions): + + @classmethod + def _wrap_lines(cls, lines): + yield cls.prelude + for line in lines: + yield line + yield cls.postlude + + _inline = lambda text: textwrap.dedent(text).strip().replace('\n', '; ') + prelude = _inline(""" + import sys + sys.__plen = len(sys.path) + """) + postlude = _inline(""" + import sys + new = sys.path[sys.__plen:] + del sys.path[sys.__plen:] + p = getattr(sys, '__egginsert', 0) + sys.path[p:p] = new + sys.__egginsert = p + len(new) + """) + + +if os.environ.get('SETUPTOOLS_SYS_PATH_TECHNIQUE', 'rewrite') == 'rewrite': + PthDistributions = RewritePthDistributions + + def _first_line_re(): """ Return a regular expression based on first_line_re suitable for matching @@ -1594,33 +1636,6 @@ def _first_line_re(): return re.compile(first_line_re.pattern.decode()) -def get_script_header(script_text, executable=sys_executable, wininst=False): - """Create a #! line, getting options (if any) from script_text""" - first = (script_text + '\n').splitlines()[0] - match = _first_line_re().match(first) - options = '' - if match: - options = match.group(1) or '' - if options: - options = ' ' + options - if wininst: - executable = "python.exe" - else: - executable = nt_quote_arg(executable) - hdr = "#!%(executable)s%(options)s\n" % locals() - if not isascii(hdr): - # Non-ascii path to sys.executable, use -x to prevent warnings - if options: - if options.strip().startswith('-'): - options = ' -x' + options.strip()[1:] - # else: punt, we can't do it, let the warning happen anyway - else: - options = ' -x' - executable = fix_jython_executable(executable, options) - hdr = "#!%(executable)s%(options)s\n" % locals() - return hdr - - def auto_chmod(func, arg, exc): if func is os.remove and os.name == 'nt': chmod(arg, stat.S_IWRITE) @@ -1819,9 +1834,8 @@ def is_python(text, filename='<string>'): def is_sh(executable): """Determine if the specified executable is a .sh (contains a #! line)""" try: - fp = open(executable) - magic = fp.read(2) - fp.close() + with io.open(executable, encoding='latin-1') as fp: + magic = fp.read(2) except (OSError, IOError): return executable return magic == '#!' @@ -1829,36 +1843,7 @@ def is_sh(executable): def nt_quote_arg(arg): """Quote a command line argument according to Windows parsing rules""" - - result = [] - needquote = False - nb = 0 - - needquote = (" " in arg) or ("\t" in arg) - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - nb += 1 - elif c == '"': - # double preceding backslashes, then add a \" - result.append('\\' * (nb * 2) + '\\"') - nb = 0 - else: - if nb: - result.append('\\' * nb) - nb = 0 - result.append(c) - - if nb: - result.append('\\' * nb) - - if needquote: - result.append('\\' * nb) # double the trailing backslashes - result.append('"') - - return ''.join(result) + return subprocess.list2cmdline([arg]) def is_python_script(script_text, filename): @@ -1887,31 +1872,130 @@ def chmod(path, mode): log.debug("changing mode of %s to %o", path, mode) try: _chmod(path, mode) - except os.error: - e = sys.exc_info()[1] + except os.error as e: log.debug("chmod failed: %s", e) def fix_jython_executable(executable, options): - if sys.platform.startswith('java') and is_sh(executable): - # Workaround for Jython is not needed on Linux systems. - import java + warnings.warn("Use JythonCommandSpec", DeprecationWarning, stacklevel=2) + + if not JythonCommandSpec.relevant(): + return executable + + cmd = CommandSpec.best().from_param(executable) + cmd.install_options(options) + return cmd.as_header().lstrip('#!').rstrip('\n') + + +class CommandSpec(list): + """ + A command spec for a #! header, specified as a list of arguments akin to + those passed to Popen. + """ + + options = [] + split_args = dict() + + @classmethod + def best(cls): + """ + Choose the best CommandSpec class based on environmental conditions. + """ + return cls if not JythonCommandSpec.relevant() else JythonCommandSpec - if java.lang.System.getProperty("os.name") == "Linux": - return executable + @classmethod + def _sys_executable(cls): + _default = os.path.normpath(sys.executable) + return os.environ.get('__PYVENV_LAUNCHER__', _default) + + @classmethod + def from_param(cls, param): + """ + Construct a CommandSpec from a parameter to build_scripts, which may + be None. + """ + if isinstance(param, cls): + return param + if isinstance(param, list): + return cls(param) + if param is None: + return cls.from_environment() + # otherwise, assume it's a string. + return cls.from_string(param) + + @classmethod + def from_environment(cls): + return cls([cls._sys_executable()]) + + @classmethod + def from_string(cls, string): + """ + Construct a command spec from a simple string representing a command + line parseable by shlex.split. + """ + items = shlex.split(string, **cls.split_args) + return cls(items) + + def install_options(self, script_text): + self.options = shlex.split(self._extract_options(script_text)) + cmdline = subprocess.list2cmdline(self) + if not isascii(cmdline): + self.options[:0] = ['-x'] + + @staticmethod + def _extract_options(orig_script): + """ + Extract any options from the first line of the script. + """ + first = (orig_script + '\n').splitlines()[0] + match = _first_line_re().match(first) + options = match.group(1) or '' if match else '' + return options.strip() + + def as_header(self): + return self._render(self + list(self.options)) + + @staticmethod + def _render(items): + cmdline = subprocess.list2cmdline(items) + return '#!' + cmdline + '\n' - # Workaround Jython's sys.executable being a .sh (an invalid - # shebang line interpreter) - if options: +# For pbr compat; will be removed in a future version. +sys_executable = CommandSpec._sys_executable() + + +class WindowsCommandSpec(CommandSpec): + split_args = dict(posix=False) + + +class JythonCommandSpec(CommandSpec): + @classmethod + def relevant(cls): + return ( + sys.platform.startswith('java') + and + __import__('java').lang.System.getProperty('os.name') != 'Linux' + ) + + def as_header(self): + """ + Workaround Jython's sys.executable being a .sh (an invalid + shebang line interpreter) + """ + if not is_sh(self[0]): + return super(JythonCommandSpec, self).as_header() + + if self.options: # Can't apply the workaround, leave it broken log.warn( "WARNING: Unable to adapt shebang line for Jython," " the following script is NOT executable\n" " see http://bugs.jython.org/issue1112 for" " more information.") - else: - return '/usr/bin/env %s' % executable - return executable + return super(JythonCommandSpec, self).as_header() + + items = ['/usr/bin/env'] + self + list(self.options) + return self._render(items) class ScriptWriter(object): @@ -1932,39 +2016,92 @@ class ScriptWriter(object): ) """).lstrip() + command_spec_class = CommandSpec + + @classmethod + def get_script_args(cls, dist, executable=None, wininst=False): + # for backward compatibility + warnings.warn("Use get_args", DeprecationWarning) + writer = (WindowsScriptWriter if wininst else ScriptWriter).best() + header = cls.get_script_header("", executable, wininst) + return writer.get_args(dist, header) + @classmethod - def get_script_args(cls, dist, executable=sys_executable, wininst=False): + def get_script_header(cls, script_text, executable=None, wininst=False): + # for backward compatibility + warnings.warn("Use get_header", DeprecationWarning) + if wininst: + executable = "python.exe" + cmd = cls.command_spec_class.best().from_param(executable) + cmd.install_options(script_text) + return cmd.as_header() + + @classmethod + def get_args(cls, dist, header=None): """ - Yield write_script() argument tuples for a distribution's entrypoints + Yield write_script() argument tuples for a distribution's + console_scripts and gui_scripts entry points. """ - gen_class = cls.get_writer(wininst) + if header is None: + header = cls.get_header() spec = str(dist.as_requirement()) - header = get_script_header("", executable, wininst) for type_ in 'console', 'gui': group = type_ + '_scripts' for name, ep in dist.get_entry_map(group).items(): - script_text = gen_class.template % locals() - for res in gen_class._get_script_args(type_, name, header, - script_text): + cls._ensure_safe_name(name) + script_text = cls.template % locals() + args = cls._get_script_args(type_, name, header, script_text) + for res in args: yield res + @staticmethod + def _ensure_safe_name(name): + """ + Prevent paths in *_scripts entry point names. + """ + has_path_sep = re.search(r'[\\/]', name) + if has_path_sep: + raise ValueError("Path separators not allowed in script names") + @classmethod def get_writer(cls, force_windows): - if force_windows or sys.platform == 'win32': - return WindowsScriptWriter.get_writer() - return cls + # for backward compatibility + warnings.warn("Use best", DeprecationWarning) + return WindowsScriptWriter.best() if force_windows else cls.best() + + @classmethod + def best(cls): + """ + Select the best ScriptWriter for this environment. + """ + return WindowsScriptWriter.best() if sys.platform == 'win32' else cls @classmethod def _get_script_args(cls, type_, name, header, script_text): # Simply write the stub with no extension. yield (name, header + script_text) + @classmethod + def get_header(cls, script_text="", executable=None): + """Create a #! line, getting options (if any) from script_text""" + cmd = cls.command_spec_class.best().from_param(executable) + cmd.install_options(script_text) + return cmd.as_header() + class WindowsScriptWriter(ScriptWriter): + command_spec_class = WindowsCommandSpec + @classmethod def get_writer(cls): + # for backward compatibility + warnings.warn("Use best", DeprecationWarning) + return cls.best() + + @classmethod + def best(cls): """ - Get a script writer suitable for Windows + Select the best ScriptWriter suitable for Windows """ writer_lookup = dict( executable=WindowsExecutableLauncherWriter, @@ -1987,8 +2124,8 @@ class WindowsScriptWriter(ScriptWriter): blockers = [name + x for x in old] yield name + ext, header + script_text, 't', blockers - @staticmethod - def _adjust_header(type_, orig_header): + @classmethod + def _adjust_header(cls, type_, orig_header): """ Make sure 'pythonw' is used for gui and and 'python' is used for console (regardless of what sys.executable is). @@ -1999,11 +2136,19 @@ class WindowsScriptWriter(ScriptWriter): pattern, repl = repl, pattern pattern_ob = re.compile(re.escape(pattern), re.IGNORECASE) new_header = pattern_ob.sub(string=orig_header, repl=repl) + return new_header if cls._use_header(new_header) else orig_header + + @staticmethod + def _use_header(new_header): + """ + Should _adjust_header use the replaced header? + + On non-windows systems, always use. On + Windows systems, only use the replaced header if it resolves + to an executable on the system. + """ clean_header = new_header[2:-1].strip('"') - if sys.platform == 'win32' and not os.path.exists(clean_header): - # the adjusted version doesn't exist, so return the original - return orig_header - return new_header + return sys.platform != 'win32' or find_executable(clean_header) class WindowsExecutableLauncherWriter(WindowsScriptWriter): @@ -2039,6 +2184,7 @@ class WindowsExecutableLauncherWriter(WindowsScriptWriter): # for backward-compatibility get_script_args = ScriptWriter.get_script_args +get_script_header = ScriptWriter.get_script_header def get_win_launcher(type): @@ -2160,4 +2306,3 @@ def _patch_usage(): yield finally: distutils.core.gen_usage = saved - diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 3f1db996..19849e66 100755 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -10,17 +10,17 @@ import distutils.filelist import os import re import sys +import io +import warnings +import time import six -try: - from setuptools_svn import svn_utils -except ImportError: - pass - from setuptools import Command from setuptools.command.sdist import sdist from setuptools.command.sdist import walk_revctrl +from setuptools.command.setopt import edit_config +from setuptools.command import bdist_egg from pkg_resources import ( parse_requirements, safe_name, parse_version, safe_version, yield_lines, EntryPoint, iter_entry_points, to_filename) @@ -28,6 +28,12 @@ import setuptools.unicode_utils as unicode_utils from pkg_resources import packaging +try: + from setuptools_svn import svn_utils +except ImportError: + pass + + class egg_info(Command): description = "create a distribution's .egg-info directory" @@ -59,8 +65,6 @@ class egg_info(Command): self.vtags = None def save_version_info(self, filename): - from setuptools.command.setopt import edit_config - values = dict( egg_info=dict( tag_svn_revision=0, @@ -169,7 +173,8 @@ class egg_info(Command): self.mkpath(self.egg_info) installer = self.distribution.fetch_build_egg for ep in iter_entry_points('egg_info.writers'): - writer = ep.load(installer=installer) + ep.require(installer=installer) + writer = ep.resolve() writer(self, ep.name, os.path.join(self.egg_info, ep.name)) # Get rid of native_libs.txt if it was put there by older bdist_egg @@ -184,12 +189,8 @@ class egg_info(Command): if self.tag_build: version += self.tag_build if self.tag_svn_revision: - rev = self.get_svn_revision() - if rev: # is 0 if it's not an svn working copy - version += '-r%s' % rev + version += '-r%s' % self.get_svn_revision() if self.tag_date: - import time - version += time.strftime("-%Y%m%d") return version @@ -390,7 +391,6 @@ def write_pkg_info(cmd, basename, filename): metadata.name, metadata.version = oldname, oldver safe = getattr(cmd.distribution, 'zip_safe', None) - from setuptools.command import bdist_egg bdist_egg.write_safety_flag(cmd.egg_info, safe) @@ -467,14 +467,15 @@ def write_entries(cmd, basename, filename): def get_pkg_info_revision(): - # See if we can get a -r### off of PKG-INFO, in case this is an sdist of - # a subversion revision - # + """ + Get a -r### off of PKG-INFO Version in case this is an sdist of + a subversion revision. + """ + warnings.warn("get_pkg_info_revision is deprecated.", DeprecationWarning) if os.path.exists('PKG-INFO'): - f = open('PKG-INFO', 'rU') - for line in f: - match = re.match(r"Version:.*-r(\d+)\s*$", line) - if match: - return int(match.group(1)) - f.close() + with io.open('PKG-INFO') as f: + for line in f: + match = re.match(r"Version:.*-r(\d+)\s*$", line) + if match: + return int(match.group(1)) return 0 diff --git a/setuptools/command/install_lib.py b/setuptools/command/install_lib.py index 9b772227..78fe6891 100644 --- a/setuptools/command/install_lib.py +++ b/setuptools/command/install_lib.py @@ -79,6 +79,8 @@ class install_lib(orig.install_lib): base = os.path.join('__pycache__', '__init__.' + imp.get_tag()) yield base + '.pyc' yield base + '.pyo' + yield base + '.opt-1.pyc' + yield base + '.opt-2.pyc' def copy_tree( self, infile, outfile, diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py index eb79fa3c..be66cb22 100755 --- a/setuptools/command/install_scripts.py +++ b/setuptools/command/install_scripts.py @@ -13,8 +13,7 @@ class install_scripts(orig.install_scripts): self.no_ep = False def run(self): - from setuptools.command.easy_install import get_script_args - from setuptools.command.easy_install import sys_executable + import setuptools.command.easy_install as ei self.run_command("egg_info") if self.distribution.scripts: @@ -31,11 +30,17 @@ class install_scripts(orig.install_scripts): ei_cmd.egg_name, ei_cmd.egg_version, ) bs_cmd = self.get_finalized_command('build_scripts') - executable = getattr(bs_cmd, 'executable', sys_executable) - is_wininst = getattr( - self.get_finalized_command("bdist_wininst"), '_is_running', False - ) - for args in get_script_args(dist, executable, is_wininst): + exec_param = getattr(bs_cmd, 'executable', None) + bw_cmd = self.get_finalized_command("bdist_wininst") + is_wininst = getattr(bw_cmd, '_is_running', False) + writer = ei.ScriptWriter + if is_wininst: + exec_param = "python.exe" + writer = ei.WindowsScriptWriter + # resolve the writer to the environment + writer = writer.best() + cmd = writer.command_spec_class.best().from_param(exec_param) + for args in writer.get_args(dist, cmd.as_header()): self.write_script(*args) def write_script(self, script_name, contents, mode="t", *ignored): diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 4ec7ec91..3b9f7dd5 100755 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -3,6 +3,7 @@ from distutils import log import distutils.command.sdist as orig import os import sys +import io import six @@ -71,7 +72,8 @@ class sdist(orig.sdist): try: orig.sdist.read_template(self) except: - sys.exc_info()[2].tb_next.tb_frame.f_locals['template'].close() + _, _, tb = sys.exc_info() + tb.tb_next.tb_frame.f_locals['template'].close() raise # Beginning with Python 2.7.2, 3.1.4, and 3.2.1, this leaky file handle @@ -166,11 +168,8 @@ class sdist(orig.sdist): if not os.path.isfile(self.manifest): return False - fp = open(self.manifest, 'rbU') - try: + with io.open(self.manifest, 'rb') as fp: first_line = fp.readline() - finally: - fp.close() return (first_line != '# file GENERATED by distutils, do NOT edit\n'.encode()) diff --git a/setuptools/command/test.py b/setuptools/command/test.py index c5644530..5f2e2299 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -1,6 +1,5 @@ from distutils.errors import DistutilsOptionError from unittest import TestLoader -import unittest import sys import six @@ -13,7 +12,7 @@ from setuptools.py31compat import unittest_main class ScanningLoader(TestLoader): - def loadTestsFromModule(self, module): + def loadTestsFromModule(self, module, pattern=None): """Return a suite of all tests cases contained in the given module If the module is a package, load tests from all the modules in it. @@ -43,6 +42,17 @@ class ScanningLoader(TestLoader): return tests[0] # don't create a nested suite for only one return +# adapted from jaraco.classes.properties:NonDataProperty +class NonDataProperty(object): + def __init__(self, fget): + self.fget = fget + + def __get__(self, obj, objtype=None): + if obj is None: + return self + return self.fget(obj) + + class test(Command): """Command to run unit tests after in-place build""" @@ -63,20 +73,16 @@ class test(Command): def finalize_options(self): + if self.test_suite and self.test_module: + msg = "You may specify a module or a suite, but not both" + raise DistutilsOptionError(msg) + if self.test_suite is None: if self.test_module is None: self.test_suite = self.distribution.test_suite else: self.test_suite = self.test_module + ".test_suite" - elif self.test_module: - raise DistutilsOptionError( - "You may specify a module or a suite, but not both" - ) - self.test_args = [self.test_suite] - - if self.verbose: - self.test_args.insert(0, '--verbose') if self.test_loader is None: self.test_loader = getattr(self.distribution, 'test_loader', None) if self.test_loader is None: @@ -84,6 +90,16 @@ class test(Command): if self.test_runner is None: self.test_runner = getattr(self.distribution, 'test_runner', None) + @NonDataProperty + def test_args(self): + return list(self._test_args()) + + def _test_args(self): + if self.verbose: + yield '--verbose' + if self.test_suite: + yield self.test_suite + def with_project_on_sys_path(self, func): with_2to3 = six.PY3 and getattr(self.distribution, 'use_2to3', False) @@ -134,20 +150,19 @@ class test(Command): if self.distribution.tests_require: self.distribution.fetch_build_eggs(self.distribution.tests_require) - if self.test_suite: - cmd = ' '.join(self.test_args) - if self.dry_run: - self.announce('skipping "unittest %s" (dry run)' % cmd) - else: - self.announce('running "unittest %s"' % cmd) - self.with_project_on_sys_path(self.run_tests) + cmd = ' '.join(self._argv) + if self.dry_run: + self.announce('skipping "%s" (dry run)' % cmd) + else: + self.announce('running "%s"' % cmd) + self.with_project_on_sys_path(self.run_tests) def run_tests(self): # Purge modules under test from sys.modules. The test loader will # re-import them from the build location. Required when 2to3 is used # with namespace packages. if six.PY3 and getattr(self.distribution, 'use_2to3', False): - module = self.test_args[-1].split('.')[0] + module = self.test_suite.split('.')[0] if module in _namespace_packages: del_modules = [] if module in sys.modules: @@ -159,11 +174,15 @@ class test(Command): list(map(sys.modules.__delitem__, del_modules)) unittest_main( - None, None, [unittest.__file__] + self.test_args, + None, None, self._argv, testLoader=self._resolve_as_ep(self.test_loader), testRunner=self._resolve_as_ep(self.test_runner), ) + @property + def _argv(self): + return ['unittest'] + self.test_args + @staticmethod def _resolve_as_ep(val): """ @@ -173,4 +192,4 @@ class test(Command): if val is None: return parsed = EntryPoint.parse("x=" + val) - return parsed._load()() + return parsed.resolve()() diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 360c10e8..43b5d76a 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -171,8 +171,7 @@ class upload_docs(upload): conn.putheader('Authorization', auth) conn.endheaders() conn.send(body) - except socket.error: - e = sys.exc_info()[1] + except socket.error as e: self.announce(str(e), log.ERROR) return diff --git a/setuptools/dist.py b/setuptools/dist.py index cdc15e46..7335c967 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -116,24 +116,26 @@ def check_extras(dist, attr, value): def assert_bool(dist, attr, value): """Verify that value is True, False, 0, or 1""" if bool(value) != value: - raise DistutilsSetupError( - "%r must be a boolean value (got %r)" % (attr,value) - ) + tmpl = "{attr!r} must be a boolean value (got {value!r})" + raise DistutilsSetupError(tmpl.format(attr=attr, value=value)) + + def check_requirements(dist, attr, value): """Verify that install_requires is a valid requirements list""" try: list(pkg_resources.parse_requirements(value)) - except (TypeError,ValueError): - raise DistutilsSetupError( - "%r must be a string or list of strings " - "containing valid project/version requirement specifiers" % (attr,) + except (TypeError, ValueError) as error: + tmpl = ( + "{attr!r} must be a string or list of strings " + "containing valid project/version requirement specifiers; {error}" ) + raise DistutilsSetupError(tmpl.format(attr=attr, error=error)) + def check_entry_points(dist, attr, value): """Verify that entry_points map is parseable""" try: pkg_resources.EntryPoint.parse_map(value) - except ValueError: - e = sys.exc_info()[1] + except ValueError as e: raise DistutilsSetupError(e) def check_test_suite(dist, attr, value): @@ -159,7 +161,7 @@ def check_packages(dist, attr, value): for pkgname in value: if not re.match(r'\w+(\.\w+)*', pkgname): distutils.log.warn( - "WARNING: %r not a valid package name; please use only" + "WARNING: %r not a valid package name; please use only " ".-separated package names in setup.py", pkgname ) @@ -266,8 +268,7 @@ class Distribution(_Distribution): if attrs and 'setup_requires' in attrs: self.fetch_build_eggs(attrs['setup_requires']) for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): - if not hasattr(self,ep.name): - setattr(self,ep.name,None) + vars(self).setdefault(ep.name, None) _Distribution.__init__(self,attrs) if isinstance(self.metadata.version, numbers.Number): # Some people apparently take "version number" too literally :) @@ -279,10 +280,9 @@ class Distribution(_Distribution): normalized_version = str(ver) if self.metadata.version != normalized_version: warnings.warn( - "The version specified requires normalization, " - "consider using '%s' instead of '%s'." % ( - normalized_version, + "Normalizing '%s' to '%s'" % ( self.metadata.version, + normalized_version, ) ) self.metadata.version = normalized_version @@ -436,10 +436,18 @@ class Distribution(_Distribution): for ep in pkg_resources.iter_entry_points('distutils.commands'): if ep.name not in self.cmdclass: # don't require extras as the commands won't be invoked - cmdclass = ep._load() + cmdclass = ep.resolve() self.cmdclass[ep.name] = cmdclass return _Distribution.print_commands(self) + def get_command_list(self): + for ep in pkg_resources.iter_entry_points('distutils.commands'): + if ep.name not in self.cmdclass: + # don't require extras as the commands won't be invoked + cmdclass = ep.resolve() + self.cmdclass[ep.name] = cmdclass + return _Distribution.get_command_list(self) + def _set_feature(self,name,status): """Set feature's inclusion status""" setattr(self,self._feature_attrname(name),status) @@ -818,7 +826,7 @@ class Feature: if not self.available: raise DistutilsPlatformError( - self.description+" is required," + self.description+" is required, " "but is not available on this platform" ) diff --git a/setuptools/extension.py b/setuptools/extension.py index 8178ed33..35eb7c7c 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -12,35 +12,33 @@ _Extension = _get_unpatched(distutils.core.Extension) msvc9_support.patch_for_specialized_compiler() -def have_pyrex(): +def _have_cython(): """ - Return True if Cython or Pyrex can be imported. + Return True if Cython can be imported. """ - pyrex_impls = 'Cython.Distutils.build_ext', 'Pyrex.Distutils.build_ext' - for pyrex_impl in pyrex_impls: - try: - # from (pyrex_impl) import build_ext - __import__(pyrex_impl, fromlist=['build_ext']).build_ext - return True - except Exception: - pass + cython_impl = 'Cython.Distutils.build_ext', + try: + # from (cython_impl) import build_ext + __import__(cython_impl, fromlist=['build_ext']).build_ext + return True + except Exception: + pass return False +# for compatibility +have_pyrex = _have_cython + class Extension(_Extension): """Extension that uses '.c' files in place of '.pyx' files""" - def __init__(self, *args, **kw): - _Extension.__init__(self, *args, **kw) - self._convert_pyx_sources_to_lang() - def _convert_pyx_sources_to_lang(self): """ Replace sources with .pyx extensions to sources with the target language extension. This mechanism allows language authors to supply pre-converted sources but to prefer the .pyx sources. """ - if have_pyrex(): + if _have_cython(): # the build has Cython, so allow it to compile the .pyx files return lang = self.language or '' diff --git a/setuptools/msvc9_support.py b/setuptools/msvc9_support.py index d0be70e2..a69c7474 100644 --- a/setuptools/msvc9_support.py +++ b/setuptools/msvc9_support.py @@ -1,5 +1,3 @@ -import sys - try: import distutils.msvc9compiler except ImportError: @@ -29,13 +27,15 @@ def patch_for_specialized_compiler(): def find_vcvarsall(version): Reg = distutils.msvc9compiler.Reg VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' + key = VC_BASE % ('', version) try: # Per-user installs register the compiler path here - productdir = Reg.get_value(VC_BASE % ('', version), "installdir") + productdir = Reg.get_value(key, "installdir") except KeyError: try: # All-user installs on a 64-bit system register here - productdir = Reg.get_value(VC_BASE % ('Wow6432Node\\', version), "installdir") + key = VC_BASE % ('Wow6432Node\\', version) + productdir = Reg.get_value(key, "installdir") except KeyError: productdir = None @@ -50,8 +50,7 @@ def find_vcvarsall(version): def query_vcvarsall(version, *args, **kwargs): try: return unpatched['query_vcvarsall'](version, *args, **kwargs) - except distutils.errors.DistutilsPlatformError: - exc = sys.exc_info()[1] + except distutils.errors.DistutilsPlatformError as exc: if exc and "vcvarsall.bat" in exc.args[0]: message = 'Microsoft Visual C++ %0.1f is required (%s).' % (version, exc.args[0]) if int(version) == 9: diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a14c8ac6..657b467f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -6,6 +6,7 @@ import shutil import socket import base64 import hashlib +import itertools from functools import wraps try: @@ -14,7 +15,7 @@ except ImportError: from urllib2 import splituser import six -from six.moves import urllib, http_client +from six.moves import urllib, http_client, configparser from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, @@ -141,10 +142,9 @@ def interpret_distro_name( # versions in distribution archive names (sdist and bdist). parts = basename.split('-') - if not py_version: - for i,p in enumerate(parts[2:]): - if len(p)==5 and p.startswith('py2.'): - return # It's a bdist_dumb, not an sdist -- bail out + if not py_version and any(re.match('py\d\.\d$', p) for p in parts[2:]): + # it is a bdist_dumb, not an sdist -- bail out + return for p in range(1,len(parts)+1): yield Distribution( @@ -356,20 +356,30 @@ class PackageIndex(Environment): self.warn(msg, url) def scan_egg_links(self, search_path): - for item in search_path: - if os.path.isdir(item): - for entry in os.listdir(item): - if entry.endswith('.egg-link'): - self.scan_egg_link(item, entry) + dirs = filter(os.path.isdir, search_path) + egg_links = ( + (path, entry) + for path in dirs + for entry in os.listdir(path) + if entry.endswith('.egg-link') + ) + list(itertools.starmap(self.scan_egg_link, egg_links)) def scan_egg_link(self, path, entry): - lines = [_f for _f in map(str.strip, - open(os.path.join(path, entry))) if _f] - if len(lines)==2: - for dist in find_distributions(os.path.join(path, lines[0])): - dist.location = os.path.join(path, *lines) - dist.precedence = SOURCE_DIST - self.add(dist) + with open(os.path.join(path, entry)) as raw_lines: + # filter non-empty lines + lines = list(filter(None, map(str.strip, raw_lines))) + + if len(lines) != 2: + # format is not recognized; punt + return + + egg_path, setup_path = lines + + for dist in find_distributions(os.path.join(path, egg_path)): + dist.location = os.path.join(path, *lines) + dist.precedence = SOURCE_DIST + self.add(dist) def process_index(self,url,page): """Process the contents of a PyPI page""" @@ -702,25 +712,21 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url, self.opener) - except (ValueError, http_client.InvalidURL): - v = sys.exc_info()[1] + except (ValueError, http_client.InvalidURL) as v: msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib.error.HTTPError: - v = sys.exc_info()[1] + except urllib.error.HTTPError as v: return v - except urllib.error.URLError: - v = sys.exc_info()[1] + except urllib.error.URLError as v: if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except http_client.BadStatusLine: - v = sys.exc_info()[1] + except http_client.BadStatusLine as v: if warning: self.warn(warning, v.line) else: @@ -729,8 +735,7 @@ class PackageIndex(Environment): 'down, %s' % (url, v.line) ) - except http_client.HTTPException: - v = sys.exc_info()[1] + except http_client.HTTPException as v: if warning: self.warn(warning, v) else: @@ -944,14 +949,14 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) -class PyPIConfig(six.moves.configparser.ConfigParser): +class PyPIConfig(configparser.RawConfigParser): def __init__(self): """ Load from ~/.pypirc """ defaults = dict.fromkeys(['username', 'password', 'repository'], '') - six.moves.configparser.ConfigParser.__init__(self, defaults) + configparser.RawConfigParser.__init__(self, defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): @@ -1043,16 +1048,18 @@ def local_open(url): elif path.endswith('/') and os.path.isdir(filename): files = [] for f in os.listdir(filename): - if f=='index.html': - with open(os.path.join(filename,f),'r') as fp: + filepath = os.path.join(filename, f) + if f == 'index.html': + with open(filepath, 'r') as fp: body = fp.read() break - elif os.path.isdir(os.path.join(filename,f)): - f+='/' - files.append("<a href=%r>%s</a>" % (f,f)) + elif os.path.isdir(filepath): + f += '/' + files.append('<a href="{name}">{name}</a>'.format(name=f)) else: - body = ("<html><head><title>%s</title>" % url) + \ - "</head><body>%s</body></html>" % '\n'.join(files) + tmpl = ("<html><head><title>{url}</title>" + "</head><body>{files}</body></html>") + body = tmpl.format(url=url, files='\n'.join(files)) status, message = 200, "OK" else: status, message, body = 404, "Path not found", "Not found" diff --git a/setuptools/py31compat.py b/setuptools/py31compat.py index c487ac04..8fe6dd9d 100644 --- a/setuptools/py31compat.py +++ b/setuptools/py31compat.py @@ -20,7 +20,7 @@ except ImportError: import shutil import tempfile class TemporaryDirectory(object): - """" + """ Very simple temporary directory context manager. Will try to delete afterward, but will also ignore OS and similar errors on deletion. diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index f99532f6..43b84791 100755 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -13,7 +13,7 @@ from six.moves import builtins import pkg_resources -if os.name == "java": +if sys.platform.startswith('java'): import org.python.modules.posix.PosixModule as _os else: _os = sys.modules[os.name] @@ -34,12 +34,12 @@ def _execfile(filename, globals, locals=None): Python 3 implementation of execfile. """ mode = 'rb' - # Python 2.6 compile requires LF for newlines, so use deprecated - # Universal newlines support. - if sys.version_info < (2, 7): - mode += 'U' with open(filename, mode) as stream: script = stream.read() + # compile() function in Python 2.6 and 3.1 requires LF line endings. + if sys.version_info[:2] < (2, 7) or sys.version_info[:2] >= (3, 0) and sys.version_info[:2] < (3, 2): + script = script.replace(b'\r\n', b'\n') + script = script.replace(b'\r', b'\n') if locals is None: locals = globals code = compile(script, filename, 'exec') @@ -47,8 +47,10 @@ def _execfile(filename, globals, locals=None): @contextlib.contextmanager -def save_argv(): +def save_argv(repl=None): saved = sys.argv[:] + if repl is not None: + sys.argv[:] = repl try: yield saved finally: @@ -92,6 +94,53 @@ def pushd(target): os.chdir(saved) +class UnpickleableException(Exception): + """ + An exception representing another Exception that could not be pickled. + """ + @staticmethod + def dump(type, exc): + """ + Always return a dumped (pickled) type and exc. If exc can't be pickled, + wrap it in UnpickleableException first. + """ + try: + return pickle.dumps(type), pickle.dumps(exc) + except Exception: + # get UnpickleableException inside the sandbox + from setuptools.sandbox import UnpickleableException as cls + return cls.dump(cls, cls(repr(exc))) + + +class ExceptionSaver: + """ + A Context Manager that will save an exception, serialized, and restore it + later. + """ + def __enter__(self): + return self + + def __exit__(self, type, exc, tb): + if not exc: + return + + # dump the exception + self._saved = UnpickleableException.dump(type, exc) + self._tb = tb + + # suppress the exception + return True + + def resume(self): + "restore and re-raise any exception" + + if '_saved' not in vars(self): + return + + type, exc = map(pickle.loads, self._saved) + six.reraise(type, exc, self._tb) + + @contextlib.contextmanager def save_modules(): """ @@ -101,31 +150,20 @@ def save_modules(): outside the context. """ saved = sys.modules.copy() - try: - try: - yield saved - except: - # dump any exception - class_, exc, tb = sys.exc_info() - saved_cls = pickle.dumps(class_) - saved_exc = pickle.dumps(exc) - raise - finally: - sys.modules.update(saved) - # remove any modules imported since - del_modules = ( - mod_name for mod_name in sys.modules - if mod_name not in saved - # exclude any encodings modules. See #285 - and not mod_name.startswith('encodings.') - ) - _clear_modules(del_modules) - except: - # reload and re-raise any exception, using restored modules - class_, exc, tb = sys.exc_info() - new_cls = pickle.loads(saved_cls) - new_exc = pickle.loads(saved_exc) - six.reraise(new_cls, new_exc, tb) + with ExceptionSaver() as saved_exc: + yield saved + + sys.modules.update(saved) + # remove any modules imported since + del_modules = ( + mod_name for mod_name in sys.modules + if mod_name not in saved + # exclude any encodings modules. See #285 + and not mod_name.startswith('encodings.') + ) + _clear_modules(del_modules) + + saved_exc.resume() def _clear_modules(module_names): @@ -199,8 +237,7 @@ def run_setup(setup_script, args): ns = dict(__file__=setup_script, __name__='__main__') _execfile(setup_script, ns) DirectorySandbox(setup_dir).run(runner) - except SystemExit: - v = sys.exc_info()[1] + except SystemExit as v: if v.args and v.args[0]: raise # Normal exit, just return @@ -347,6 +384,7 @@ class DirectorySandbox(AbstractSandbox): AbstractSandbox.__init__(self) def _violation(self, operation, *args, **kw): + from setuptools.sandbox import SandboxViolation raise SandboxViolation(operation, args, kw) if _file: diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py index c618ea7c..8fd7836b 100644 --- a/setuptools/ssl_support.py +++ b/setuptools/ssl_support.py @@ -218,6 +218,12 @@ def get_win_certfile(): self.addcerts(certs) atexit.register(self.close) + def close(self): + try: + super(MyCertFile, self).close() + except OSError: + pass + _wincerts = MyCertFile(stores=['CA', 'ROOT']) return _wincerts.name diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index 8cde6f60..b2c6894f 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -16,6 +16,11 @@ import setuptools.depends as dep from setuptools import Feature from setuptools.depends import Require +c_type = os.environ.get("LC_CTYPE", os.environ.get("LC_ALL")) +is_ascii = c_type in ("C", "POSIX") +fail_on_ascii = pytest.mark.xfail(is_ascii, reason="Test fails in this locale") + + def makeSetup(**args): """Return distribution from 'setup(**args)', without executing commands""" diff --git a/setuptools/tests/contexts.py b/setuptools/tests/contexts.py index fabab071..d9dcad84 100644 --- a/setuptools/tests/contexts.py +++ b/setuptools/tests/contexts.py @@ -27,7 +27,7 @@ def environment(**replacements): to clear the values. """ saved = dict( - (key, os.environ['key']) + (key, os.environ[key]) for key in replacements if key in os.environ ) @@ -49,14 +49,6 @@ def environment(**replacements): @contextlib.contextmanager -def argv(repl): - old_argv = sys.argv[:] - sys.argv[:] = repl - yield - sys.argv[:] = old_argv - - -@contextlib.contextmanager def quiet(): """ Redirect stdout/stderr to StringIO objects to prevent console output from diff --git a/setuptools/tests/files.py b/setuptools/tests/files.py new file mode 100644 index 00000000..4364241b --- /dev/null +++ b/setuptools/tests/files.py @@ -0,0 +1,32 @@ +import os + + +def build_files(file_defs, prefix=""): + """ + Build a set of files/directories, as described by the file_defs dictionary. + + Each key/value pair in the dictionary is interpreted as a filename/contents + pair. If the contents value is a dictionary, a directory is created, and the + dictionary interpreted as the files within it, recursively. + + For example: + + {"README.txt": "A README file", + "foo": { + "__init__.py": "", + "bar": { + "__init__.py": "", + }, + "baz.py": "# Some code", + } + } + """ + for name, contents in file_defs.items(): + full_name = os.path.join(prefix, name) + if isinstance(contents, dict): + if not os.path.exists(full_name): + os.makedirs(full_name) + build_files(contents, prefix=full_name) + else: + with open(full_name, 'w') as f: + f.write(contents) diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index 0b1eaf5f..c70c38cb 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -1,4 +1,7 @@ -import mock +try: + from unittest import mock +except ImportError: + import mock import pytest from . import contexts diff --git a/setuptools/tests/py26compat.py b/setuptools/tests/py26compat.py index c53b4809..c5680881 100644 --- a/setuptools/tests/py26compat.py +++ b/setuptools/tests/py26compat.py @@ -8,4 +8,7 @@ def _tarfile_open_ex(*args, **kwargs): """ return contextlib.closing(tarfile.open(*args, **kwargs)) -tarfile_open = _tarfile_open_ex if sys.version_info < (2,7) else tarfile.open +if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 2): + tarfile_open = _tarfile_open_ex +else: + tarfile_open = tarfile.open diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index ed1b194a..236b3aa6 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -1,13 +1,18 @@ """develop tests """ import os -import shutil import site import sys -import tempfile +import io + +import six + +import pytest from setuptools.command.develop import develop from setuptools.dist import Distribution +from . import contexts + SETUP_PY = """\ from setuptools import setup @@ -21,65 +26,52 @@ setup(name='foo', INIT_PY = """print "foo" """ -class TestDevelopTest: +@pytest.yield_fixture +def temp_user(monkeypatch): + with contexts.tempdir() as user_base: + with contexts.tempdir() as user_site: + monkeypatch.setattr('site.USER_BASE', user_base) + monkeypatch.setattr('site.USER_SITE', user_site) + yield - def setup_method(self, method): - if hasattr(sys, 'real_prefix'): - return - # Directory structure - self.dir = tempfile.mkdtemp() - os.mkdir(os.path.join(self.dir, 'foo')) - # setup.py - setup = os.path.join(self.dir, 'setup.py') - f = open(setup, 'w') +@pytest.yield_fixture +def test_env(tmpdir, temp_user): + target = tmpdir + foo = target.mkdir('foo') + setup = target / 'setup.py' + if setup.isfile(): + raise ValueError(dir(target)) + with setup.open('w') as f: f.write(SETUP_PY) - f.close() - self.old_cwd = os.getcwd() - # foo/__init__.py - init = os.path.join(self.dir, 'foo', '__init__.py') - f = open(init, 'w') + init = foo / '__init__.py' + with init.open('w') as f: f.write(INIT_PY) - f.close() - - os.chdir(self.dir) - self.old_base = site.USER_BASE - site.USER_BASE = tempfile.mkdtemp() - self.old_site = site.USER_SITE - site.USER_SITE = tempfile.mkdtemp() - - def teardown_method(self, method): - if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix): - return - - os.chdir(self.old_cwd) - shutil.rmtree(self.dir) - shutil.rmtree(site.USER_BASE) - shutil.rmtree(site.USER_SITE) - site.USER_BASE = self.old_base - site.USER_SITE = self.old_site - - def test_develop(self): - if hasattr(sys, 'real_prefix'): - return - dist = Distribution( - dict(name='foo', - packages=['foo'], - use_2to3=True, - version='0.0', - )) + with target.as_cwd(): + yield target + + +class TestDevelop: + in_virtualenv = hasattr(sys, 'real_prefix') + in_venv = hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix + @pytest.mark.skipif(in_virtualenv or in_venv, + reason="Cannot run when invoked in a virtualenv or venv") + def test_2to3_user_mode(self, test_env): + settings = dict( + name='foo', + packages=['foo'], + use_2to3=True, + version='0.0', + ) + dist = Distribution(settings) dist.script_name = 'setup.py' cmd = develop(dist) cmd.user = 1 cmd.ensure_finalized() cmd.install_dir = site.USER_SITE cmd.user = 1 - old_stdout = sys.stdout - #sys.stdout = StringIO() - try: + with contexts.quiet(): cmd.run() - finally: - sys.stdout = old_stdout # let's see if we got our egg link at the right place content = os.listdir(site.USER_SITE) @@ -87,17 +79,37 @@ class TestDevelopTest: assert content == ['easy-install.pth', 'foo.egg-link'] # Check that we are using the right code. - egg_link_file = open(os.path.join(site.USER_SITE, 'foo.egg-link'), 'rt') - try: + fn = os.path.join(site.USER_SITE, 'foo.egg-link') + with io.open(fn) as egg_link_file: path = egg_link_file.read().split()[0].strip() - finally: - egg_link_file.close() - init_file = open(os.path.join(path, 'foo', '__init__.py'), 'rt') - try: + fn = os.path.join(path, 'foo', '__init__.py') + with io.open(fn) as init_file: init = init_file.read().strip() - finally: - init_file.close() - if sys.version < "3": - assert init == 'print "foo"' - else: - assert init == 'print("foo")' + + expected = 'print("foo")' if six.PY3 else 'print "foo"' + assert init == expected + + def test_console_scripts(self, tmpdir): + """ + Test that console scripts are installed and that they reference + only the project by name and not the current version. + """ + pytest.skip("TODO: needs a fixture to cause 'develop' " + "to be invoked without mutating environment.") + settings = dict( + name='foo', + packages=['foo'], + version='0.0', + entry_points={ + 'console_scripts': [ + 'foocmd = foo:foo', + ], + }, + ) + dist = Distribution(settings) + dist.script_name = 'setup.py' + cmd = develop(dist) + cmd.ensure_finalized() + cmd.install_dir = tmpdir + cmd.run() + #assert '0.0' not in foocmd_text diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index 5d5ec16d..30220b7f 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -1,4 +1,4 @@ -#! -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """Easy install Tests """ @@ -13,29 +13,30 @@ import contextlib import tarfile import logging import itertools +import distutils.errors import io import six from six.moves import urllib import pytest -import mock +try: + from unittest import mock +except ImportError: + import mock from setuptools import sandbox -from setuptools.sandbox import run_setup, SandboxViolation -from setuptools.command.easy_install import ( - easy_install, fix_jython_executable, get_script_args, nt_quote_arg, - get_script_header, is_sh, -) +from setuptools.sandbox import run_setup +import setuptools.command.easy_install as ei from setuptools.command.easy_install import PthDistributions from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution -from pkg_resources import working_set, VersionConflict +from pkg_resources import working_set from pkg_resources import Distribution as PRDistribution import setuptools.tests.server import pkg_resources from .py26compat import tarfile_open -from . import contexts +from . import contexts, is_ascii from .textwrap import DALS @@ -48,19 +49,6 @@ class FakeDist(object): def as_requirement(self): return 'spec' -WANTED = DALS(""" - #!%s - # EASY-INSTALL-ENTRY-SCRIPT: 'spec','console_scripts','name' - __requires__ = 'spec' - import sys - from pkg_resources import load_entry_point - - if __name__ == '__main__': - sys.exit( - load_entry_point('spec', 'console_scripts', 'name')() - ) - """) % nt_quote_arg(fix_jython_executable(sys.executable, "")) - SETUP_PY = DALS(""" from setuptools import setup @@ -71,7 +59,7 @@ class TestEasyInstallTest: def test_install_site_py(self): dist = Distribution() - cmd = easy_install(dist) + cmd = ei.easy_install(dist) cmd.sitepy_installed = False cmd.install_dir = tempfile.mkdtemp() try: @@ -82,18 +70,30 @@ class TestEasyInstallTest: shutil.rmtree(cmd.install_dir) def test_get_script_args(self): + header = ei.CommandSpec.best().from_environment().as_header() + expected = header + DALS(""" + # EASY-INSTALL-ENTRY-SCRIPT: 'spec','console_scripts','name' + __requires__ = 'spec' + import sys + from pkg_resources import load_entry_point + + if __name__ == '__main__': + sys.exit( + load_entry_point('spec', 'console_scripts', 'name')() + ) + """) dist = FakeDist() - args = next(get_script_args(dist)) + args = next(ei.ScriptWriter.get_args(dist)) name, script = itertools.islice(args, 2) - assert script == WANTED + assert script == expected def test_no_find_links(self): # new option '--no-find-links', that blocks find-links added at # the project level dist = Distribution() - cmd = easy_install(dist) + cmd = ei.easy_install(dist) cmd.check_pth_processing = lambda: True cmd.no_find_links = True cmd.find_links = ['link1', 'link2'] @@ -103,7 +103,7 @@ class TestEasyInstallTest: assert cmd.package_index.scanned_urls == {} # let's try without it (default behavior) - cmd = easy_install(dist) + cmd = ei.easy_install(dist) cmd.check_pth_processing = lambda: True cmd.find_links = ['link1', 'link2'] cmd.install_dir = os.path.join(tempfile.mkdtemp(), 'ok') @@ -112,6 +112,16 @@ class TestEasyInstallTest: keys = sorted(cmd.package_index.scanned_urls.keys()) assert keys == ['link1', 'link2'] + def test_write_exception(self): + """ + Test that `cant_write_to_target` is rendered as a DistutilsError. + """ + dist = Distribution() + cmd = ei.easy_install(dist) + cmd.install_dir = os.getcwd() + with pytest.raises(distutils.errors.DistutilsError): + cmd.cant_write_to_target() + class TestPTHFileWriter: def test_add_from_cwd_site_sets_dirty(self): @@ -145,77 +155,74 @@ def setup_context(tmpdir): @pytest.mark.usefixtures("setup_context") class TestUserInstallTest: - @mock.patch('setuptools.command.easy_install.__file__', None) - def test_user_install_implied(self): - easy_install_pkg.__file__ = site.USER_SITE - site.ENABLE_USER_SITE = True # disabled sometimes - #XXX: replace with something meaningfull + # prevent check that site-packages is writable. easy_install + # shouldn't be writing to system site-packages during finalize + # options, but while it does, bypass the behavior. + prev_sp_write = mock.patch( + 'setuptools.command.easy_install.easy_install.check_site_dir', + mock.Mock(), + ) + + # simulate setuptools installed in user site packages + @mock.patch('setuptools.command.easy_install.__file__', site.USER_SITE) + @mock.patch('site.ENABLE_USER_SITE', True) + @prev_sp_write + def test_user_install_not_implied_user_site_enabled(self): + self.assert_not_user_site() + + @mock.patch('site.ENABLE_USER_SITE', False) + @prev_sp_write + def test_user_install_not_implied_user_site_disabled(self): + self.assert_not_user_site() + + @staticmethod + def assert_not_user_site(): + # create a finalized easy_install command dist = Distribution() dist.script_name = 'setup.py' - cmd = easy_install(dist) + cmd = ei.easy_install(dist) cmd.args = ['py'] cmd.ensure_finalized() - assert cmd.user, 'user should be implied' + assert not cmd.user, 'user should not be implied' def test_multiproc_atexit(self): - try: - __import__('multiprocessing') - except ImportError: - # skip the test if multiprocessing is not available - return + pytest.importorskip('multiprocessing') log = logging.getLogger('test_easy_install') logging.basicConfig(level=logging.INFO, stream=sys.stderr) log.info('this should not break') - def test_user_install_not_implied_without_usersite_enabled(self): - site.ENABLE_USER_SITE = False # usually enabled - #XXX: replace with something meaningfull - dist = Distribution() - dist.script_name = 'setup.py' - cmd = easy_install(dist) - cmd.args = ['py'] - cmd.initialize_options() - assert not cmd.user, 'NOT user should be implied' - - def test_local_index(self): - # make sure the local index is used - # when easy_install looks for installed - # packages - new_location = tempfile.mkdtemp() - target = tempfile.mkdtemp() - egg_file = os.path.join(new_location, 'foo-1.0.egg-info') - with open(egg_file, 'w') as f: + @pytest.fixture() + def foo_package(self, tmpdir): + egg_file = tmpdir / 'foo-1.0.egg-info' + with egg_file.open('w') as f: f.write('Name: foo\n') + return str(tmpdir) - sys.path.append(target) - old_ppath = os.environ.get('PYTHONPATH') - os.environ['PYTHONPATH'] = os.path.pathsep.join(sys.path) - try: - dist = Distribution() - dist.script_name = 'setup.py' - cmd = easy_install(dist) - cmd.install_dir = target - cmd.args = ['foo'] - cmd.ensure_finalized() - cmd.local_index.scan([new_location]) - res = cmd.easy_install('foo') - actual = os.path.normcase(os.path.realpath(res.location)) - expected = os.path.normcase(os.path.realpath(new_location)) - assert actual == expected - finally: - sys.path.remove(target) - for basedir in [new_location, target, ]: - if not os.path.exists(basedir) or not os.path.isdir(basedir): - continue - try: - shutil.rmtree(basedir) - except: - pass - if old_ppath is not None: - os.environ['PYTHONPATH'] = old_ppath - else: - del os.environ['PYTHONPATH'] + @pytest.yield_fixture() + def install_target(self, tmpdir): + target = str(tmpdir) + with mock.patch('sys.path', sys.path + [target]): + python_path = os.path.pathsep.join(sys.path) + with mock.patch.dict(os.environ, PYTHONPATH=python_path): + yield target + + def test_local_index(self, foo_package, install_target): + """ + The local index must be used when easy_install locates installed + packages. + """ + dist = Distribution() + dist.script_name = 'setup.py' + cmd = ei.easy_install(dist) + cmd.install_dir = install_target + cmd.args = ['foo'] + cmd.ensure_finalized() + cmd.local_index.scan([foo_package]) + res = cmd.easy_install('foo') + actual = os.path.normcase(os.path.realpath(res.location)) + expected = os.path.normcase(os.path.realpath(foo_package)) + assert actual == expected @contextlib.contextmanager def user_install_setup_context(self, *args, **kwargs): @@ -236,28 +243,6 @@ class TestUserInstallTest: self.user_install_setup_context, ) - def test_setup_requires(self): - """Regression test for Distribute issue #318 - - Ensure that a package with setup_requires can be installed when - setuptools is installed in the user site-packages without causing a - SandboxViolation. - """ - - test_pkg = create_setup_requires_package(os.getcwd()) - test_setup_py = os.path.join(test_pkg, 'setup.py') - - try: - with contexts.quiet(): - with self.patched_setup_context(): - run_setup(test_setup_py, ['install']) - except SandboxViolation: - self.fail('Installation caused SandboxViolation') - except IndexError: - # Test fails in some cases due to bugs in Python - # See https://bitbucket.org/pypa/setuptools/issue/201 - pass - @pytest.yield_fixture def distutils_package(): @@ -305,7 +290,7 @@ class TestSetupRequires: '--install-dir', temp_install_dir, dist_file, ] - with contexts.argv(['easy_install']): + with sandbox.save_argv(['easy_install']): # attempt to install the dist. It should fail because # it doesn't exist. with pytest.raises(SystemExit): @@ -354,13 +339,9 @@ class TestSetupRequires: test_pkg = create_setup_requires_package(temp_dir) test_setup_py = os.path.join(test_pkg, 'setup.py') with contexts.quiet() as (stdout, stderr): - try: - # Don't even need to install the package, just - # running the setup.py at all is sufficient - run_setup(test_setup_py, ['--name']) - except VersionConflict: - self.fail('Installing setup.py requirements ' - 'caused a VersionConflict') + # Don't even need to install the package, just + # running the setup.py at all is sufficient + run_setup(test_setup_py, ['--name']) lines = stdout.readlines() assert len(lines) > 0 @@ -422,23 +403,31 @@ class TestScriptHeader: exe_with_spaces = r'C:\Program Files\Python33\python.exe' @pytest.mark.skipif( - sys.platform.startswith('java') and is_sh(sys.executable), + sys.platform.startswith('java') and ei.is_sh(sys.executable), reason="Test cannot run under java when executable is sh" ) def test_get_script_header(self): - expected = '#!%s\n' % nt_quote_arg(os.path.normpath(sys.executable)) - assert get_script_header('#!/usr/local/bin/python') == expected - expected = '#!%s -x\n' % nt_quote_arg(os.path.normpath(sys.executable)) - assert get_script_header('#!/usr/bin/python -x') == expected - candidate = get_script_header('#!/usr/bin/python', + expected = '#!%s\n' % ei.nt_quote_arg(os.path.normpath(sys.executable)) + actual = ei.ScriptWriter.get_script_header('#!/usr/local/bin/python') + assert actual == expected + + expected = '#!%s -x\n' % ei.nt_quote_arg(os.path.normpath + (sys.executable)) + actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python -x') + assert actual == expected + + actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', executable=self.non_ascii_exe) - assert candidate == '#!%s -x\n' % self.non_ascii_exe - candidate = get_script_header('#!/usr/bin/python', - executable=self.exe_with_spaces) - assert candidate == '#!"%s"\n' % self.exe_with_spaces + expected = '#!%s -x\n' % self.non_ascii_exe + assert actual == expected + + actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', + executable='"'+self.exe_with_spaces+'"') + expected = '#!"%s"\n' % self.exe_with_spaces + assert actual == expected @pytest.mark.xfail( - six.PY3 and os.environ.get("LC_CTYPE") in ("C", "POSIX"), + six.PY3 and is_ascii, reason="Test fails in this locale on Python 3" ) @mock.patch.dict(sys.modules, java=mock.Mock(lang=mock.Mock(System= @@ -453,9 +442,15 @@ class TestScriptHeader: exe = tmpdir / 'exe.py' with exe.open('w') as f: f.write(header) - exe = str(exe) - header = get_script_header('#!/usr/local/bin/python', executable=exe) + exe = ei.nt_quote_arg(os.path.normpath(str(exe))) + + # Make sure Windows paths are quoted properly before they're sent + # through shlex.split by get_script_header + executable = '"%s"' % exe if os.path.splitdrive(exe)[0] else exe + + header = ei.ScriptWriter.get_script_header('#!/usr/local/bin/python', + executable=executable) assert header == '#!/usr/bin/env %s\n' % exe expect_out = 'stdout' if sys.version_info < (2,7) else 'stderr' @@ -463,15 +458,70 @@ class TestScriptHeader: with contexts.quiet() as (stdout, stderr): # When options are included, generate a broken shebang line # with a warning emitted - candidate = get_script_header('#!/usr/bin/python -x', - executable=exe) - assert candidate == '#!%s -x\n' % exe + candidate = ei.ScriptWriter.get_script_header('#!/usr/bin/python -x', + executable=executable) + assert candidate == '#!%s -x\n' % exe output = locals()[expect_out] assert 'Unable to adapt shebang line' in output.getvalue() with contexts.quiet() as (stdout, stderr): - candidate = get_script_header('#!/usr/bin/python', + candidate = ei.ScriptWriter.get_script_header('#!/usr/bin/python', executable=self.non_ascii_exe) assert candidate == '#!%s -x\n' % self.non_ascii_exe output = locals()[expect_out] assert 'Unable to adapt shebang line' in output.getvalue() + + +class TestCommandSpec: + def test_custom_launch_command(self): + """ + Show how a custom CommandSpec could be used to specify a #! executable + which takes parameters. + """ + cmd = ei.CommandSpec(['/usr/bin/env', 'python3']) + assert cmd.as_header() == '#!/usr/bin/env python3\n' + + def test_from_param_for_CommandSpec_is_passthrough(self): + """ + from_param should return an instance of a CommandSpec + """ + cmd = ei.CommandSpec(['python']) + cmd_new = ei.CommandSpec.from_param(cmd) + assert cmd is cmd_new + + @mock.patch('sys.executable', TestScriptHeader.exe_with_spaces) + @mock.patch.dict(os.environ) + def test_from_environment_with_spaces_in_executable(self): + os.environ.pop('__PYVENV_LAUNCHER__', None) + cmd = ei.CommandSpec.from_environment() + assert len(cmd) == 1 + assert cmd.as_header().startswith('#!"') + + def test_from_simple_string_uses_shlex(self): + """ + In order to support `executable = /usr/bin/env my-python`, make sure + from_param invokes shlex on that input. + """ + cmd = ei.CommandSpec.from_param('/usr/bin/env my-python') + assert len(cmd) == 2 + assert '"' not in cmd.as_header() + + def test_sys_executable(self): + """ + CommandSpec.from_string(sys.executable) should contain just that param. + """ + writer = ei.ScriptWriter.best() + cmd = writer.command_spec_class.from_string(sys.executable) + assert len(cmd) == 1 + assert cmd[0] == sys.executable + + +class TestWindowsScriptWriter: + def test_header(self): + hdr = ei.WindowsScriptWriter.get_script_header('') + assert hdr.startswith('#!') + assert hdr.endswith('\n') + hdr = hdr.lstrip('#!') + hdr = hdr.rstrip('\n') + # header should not start with an escaped quote + assert not hdr.startswith('\\"') diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index a1caf9fd..333d11d6 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -4,11 +4,16 @@ import stat import pytest from . import environment +from .files import build_files from .textwrap import DALS from . import contexts -class TestEggInfo: +class Environment(str): + pass + + +class TestEggInfo(object): setup_script = DALS(""" from setuptools import setup @@ -22,19 +27,16 @@ class TestEggInfo: """) def _create_project(self): - with open('setup.py', 'w') as f: - f.write(self.setup_script) - - with open('hello.py', 'w') as f: - f.write(DALS(""" + build_files({ + 'setup.py': self.setup_script, + 'hello.py': DALS(""" def run(): print('hello') - """)) + """) + }) @pytest.yield_fixture def env(self): - class Environment(str): pass - with contexts.tempdir(prefix='setuptools-test.') as env_dir: env = Environment(env_dir) os.chmod(env_dir, stat.S_IRWXU) @@ -44,18 +46,48 @@ class TestEggInfo: for dirname in subs ) list(map(os.mkdir, env.paths.values())) - config = os.path.join(env.paths['home'], '.pydistutils.cfg') - with open(config, 'w') as f: - f.write(DALS(""" + build_files({ + env.paths['home']: { + '.pydistutils.cfg': DALS(""" [egg_info] egg-base = %(egg-base)s - """ % env.paths - )) + """ % env.paths) + } + }) yield env def test_egg_base_installed_egg_info(self, tmpdir_cwd, env): self._create_project() + self._run_install_command(tmpdir_cwd, env) + actual = self._find_egg_info_files(env.paths['lib']) + + expected = [ + 'PKG-INFO', + 'SOURCES.txt', + 'dependency_links.txt', + 'entry_points.txt', + 'not-zip-safe', + 'top_level.txt', + ] + assert sorted(actual) == expected + + def test_manifest_template_is_read(self, tmpdir_cwd, env): + self._create_project() + build_files({ + 'MANIFEST.in': DALS(""" + recursive-include docs *.rst + """), + 'docs': { + 'usage.rst': "Run 'hi'", + } + }) + self._run_install_command(tmpdir_cwd, env) + egg_info_dir = self._find_egg_info_files(env.paths['lib']).base + sources_txt = os.path.join(egg_info_dir, 'SOURCES.txt') + assert 'docs/usage.rst' in open(sources_txt).read().split('\n') + + def _run_install_command(self, tmpdir_cwd, env): environ = os.environ.copy().update( HOME=env.paths['home'], ) @@ -75,21 +107,14 @@ class TestEggInfo: if code: raise AssertionError(data) - actual = self._find_egg_info_files(env.paths['lib']) - - expected = [ - 'PKG-INFO', - 'SOURCES.txt', - 'dependency_links.txt', - 'entry_points.txt', - 'not-zip-safe', - 'top_level.txt', - ] - assert sorted(actual) == expected - def _find_egg_info_files(self, root): + class DirList(list): + def __init__(self, files, base): + super(DirList, self).__init__(files) + self.base = base + results = ( - filenames + DirList(filenames, dirpath) for dirpath, dirnames, filenames in os.walk(root) if os.path.basename(dirpath) == 'EGG-INFO' ) diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 3a6abeaa..11a6ff5a 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -7,6 +7,8 @@ import glob import os import sys +from six.moves import urllib + import pytest from setuptools.command.easy_install import easy_install @@ -14,6 +16,22 @@ from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution +def setup_module(module): + packages = 'stevedore', 'virtualenvwrapper', 'pbr', 'novaclient' + for pkg in packages: + try: + __import__(pkg) + tmpl = "Integration tests cannot run when {pkg} is installed" + pytest.skip(tmpl.format(**locals())) + except ImportError: + pass + + try: + urllib.request.urlopen('https://pypi.python.org/pypi') + except Exception as exc: + pytest.skip(str(exc)) + + @pytest.fixture def install_context(request, tmpdir, monkeypatch): """Fixture to set up temporary installation directory. diff --git a/setuptools/tests/test_msvc9compiler.py b/setuptools/tests/test_msvc9compiler.py index a0820fff..09e0460c 100644 --- a/setuptools/tests/test_msvc9compiler.py +++ b/setuptools/tests/test_msvc9compiler.py @@ -7,7 +7,10 @@ import contextlib import distutils.errors import pytest -import mock +try: + from unittest import mock +except ImportError: + import mock from . import contexts @@ -110,7 +113,8 @@ class TestModulePatch: Ensure user's settings are preferred. """ result = distutils.msvc9compiler.find_vcvarsall(9.0) - assert user_preferred_setting == result + expected = os.path.join(user_preferred_setting, 'vcvarsall.bat') + assert expected == result @pytest.yield_fixture def local_machine_setting(self): @@ -131,13 +135,14 @@ class TestModulePatch: Ensure machine setting is honored if user settings are not present. """ result = distutils.msvc9compiler.find_vcvarsall(9.0) - assert local_machine_setting == result + expected = os.path.join(local_machine_setting, 'vcvarsall.bat') + assert expected == result @pytest.yield_fixture def x64_preferred_setting(self): """ Set up environment with 64-bit and 32-bit system settings configured - and yield the 64-bit location. + and yield the canonical location. """ with self.mock_install_dir() as x32_dir: with self.mock_install_dir() as x64_dir: @@ -150,14 +155,15 @@ class TestModulePatch: }, ) with reg: - yield x64_dir + yield x32_dir def test_ensure_64_bit_preferred(self, x64_preferred_setting): """ Ensure 64-bit system key is preferred. """ result = distutils.msvc9compiler.find_vcvarsall(9.0) - assert x64_preferred_setting == result + expected = os.path.join(x64_preferred_setting, 'vcvarsall.bat') + assert expected == result @staticmethod @contextlib.contextmanager @@ -170,4 +176,4 @@ class TestModulePatch: vcvarsall = os.path.join(result, 'vcvarsall.bat') with open(vcvarsall, 'w'): pass - yield + yield result diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py index 4eb98bb1..dca4c2aa 100644 --- a/setuptools/tests/test_packageindex.py +++ b/setuptools/tests/test_packageindex.py @@ -1,9 +1,13 @@ +from __future__ import absolute_import + import sys +import os import distutils.errors import six from six.moves import urllib, http_client +from .textwrap import DALS import pkg_resources import setuptools.package_index from setuptools.tests.server import IndexServer @@ -16,8 +20,7 @@ class TestPackageIndex: url = 'http://127.0.0.1:0/nonesuch/test_package_index' try: v = index.open_url(url) - except Exception: - v = sys.exc_info()[1] + except Exception as v: assert url in str(v) else: assert isinstance(v, urllib.error.HTTPError) @@ -33,8 +36,7 @@ class TestPackageIndex: url = 'url:%20https://svn.plone.org/svn/collective/inquant.contentmirror.plone/trunk' try: v = index.open_url(url) - except Exception: - v = sys.exc_info()[1] + except Exception as v: assert url in str(v) else: assert isinstance(v, urllib.error.HTTPError) @@ -51,8 +53,7 @@ class TestPackageIndex: url = 'http://example.com' try: v = index.open_url(url) - except Exception: - v = sys.exc_info()[1] + except Exception as v: assert 'line' in str(v) else: raise AssertionError('Should have raise here!') @@ -69,8 +70,7 @@ class TestPackageIndex: url = 'http://http://svn.pythonpaste.org/Paste/wphp/trunk' try: index.open_url(url) - except distutils.errors.DistutilsError: - error = sys.exc_info()[1] + except distutils.errors.DistutilsError as error: msg = six.text_type(error) assert 'nonnumeric port' in msg or 'getaddrinfo failed' in msg or 'Name or service not known' in msg return @@ -206,3 +206,20 @@ class TestContentCheckers: 'http://foo/bar#md5=f12895fdffbd45007040d2e44df98478') rep = checker.report(lambda x: x, 'My message about %s') assert rep == 'My message about md5' + + +class TestPyPIConfig: + def test_percent_in_password(self, tmpdir, monkeypatch): + monkeypatch.setitem(os.environ, 'HOME', str(tmpdir)) + pypirc = tmpdir / '.pypirc' + with pypirc.open('w') as strm: + strm.write(DALS(""" + [pypi] + repository=https://pypi.python.org + username=jaraco + password=pity% + """)) + cfg = setuptools.package_index.PyPIConfig() + cred = cfg.creds_by_repository['https://pypi.python.org'] + assert cred.username == 'jaraco' + assert cred.password == 'pity%' diff --git a/setuptools/tests/test_sandbox.py b/setuptools/tests/test_sandbox.py index 6e5ce04a..fefd46f7 100644 --- a/setuptools/tests/test_sandbox.py +++ b/setuptools/tests/test_sandbox.py @@ -7,7 +7,7 @@ import pytest import pkg_resources import setuptools.sandbox -from setuptools.sandbox import DirectorySandbox, SandboxViolation +from setuptools.sandbox import DirectorySandbox class TestSandbox: @@ -33,10 +33,8 @@ class TestSandbox: target = os.path.join(gen_py, 'test_write') sandbox = DirectorySandbox(str(tmpdir)) try: - try: - sandbox.run(self._file_writer(target)) - except SandboxViolation: - self.fail("Could not create gen_py file due to SandboxViolation") + # attempt to create gen_py file + sandbox.run(self._file_writer(target)) finally: if os.path.exists(target): os.remove(target) @@ -56,3 +54,88 @@ class TestSandbox: with setup_py.open('wb') as stream: stream.write(b'"degenerate script"\r\n') setuptools.sandbox._execfile(str(setup_py), globals()) + + +class TestExceptionSaver: + def test_exception_trapped(self): + with setuptools.sandbox.ExceptionSaver(): + raise ValueError("details") + + def test_exception_resumed(self): + with setuptools.sandbox.ExceptionSaver() as saved_exc: + raise ValueError("details") + + with pytest.raises(ValueError) as caught: + saved_exc.resume() + + assert isinstance(caught.value, ValueError) + assert str(caught.value) == 'details' + + def test_exception_reconstructed(self): + orig_exc = ValueError("details") + + with setuptools.sandbox.ExceptionSaver() as saved_exc: + raise orig_exc + + with pytest.raises(ValueError) as caught: + saved_exc.resume() + + assert isinstance(caught.value, ValueError) + assert caught.value is not orig_exc + + def test_no_exception_passes_quietly(self): + with setuptools.sandbox.ExceptionSaver() as saved_exc: + pass + + saved_exc.resume() + + def test_unpickleable_exception(self): + class CantPickleThis(Exception): + "This Exception is unpickleable because it's not in globals" + + with setuptools.sandbox.ExceptionSaver() as saved_exc: + raise CantPickleThis('detail') + + with pytest.raises(setuptools.sandbox.UnpickleableException) as caught: + saved_exc.resume() + + assert str(caught.value) == "CantPickleThis('detail',)" + + def test_unpickleable_exception_when_hiding_setuptools(self): + """ + As revealed in #440, an infinite recursion can occur if an unpickleable + exception while setuptools is hidden. Ensure this doesn't happen. + """ + class ExceptionUnderTest(Exception): + """ + An unpickleable exception (not in globals). + """ + + with pytest.raises(setuptools.sandbox.UnpickleableException) as caught: + with setuptools.sandbox.save_modules(): + setuptools.sandbox.hide_setuptools() + raise ExceptionUnderTest() + + msg, = caught.value.args + assert msg == 'ExceptionUnderTest()' + + def test_sandbox_violation_raised_hiding_setuptools(self, tmpdir): + """ + When in a sandbox with setuptools hidden, a SandboxViolation + should reflect a proper exception and not be wrapped in + an UnpickleableException. + """ + def write_file(): + "Trigger a SandboxViolation by writing outside the sandbox" + with open('/etc/foo', 'w'): + pass + sandbox = DirectorySandbox(str(tmpdir)) + with pytest.raises(setuptools.sandbox.SandboxViolation) as caught: + with setuptools.sandbox.save_modules(): + setuptools.sandbox.hide_setuptools() + sandbox.run(write_file) + + cmd, args, kwargs = caught.value.args + assert cmd == 'open' + assert args == ('/etc/foo', 'w') + assert kwargs == {} diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index d30e21ac..c173d713 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- """sdist tests""" -import locale import os import shutil import sys import tempfile import unicodedata import contextlib +import io import six import pytest @@ -16,6 +16,11 @@ import pkg_resources from setuptools.command.sdist import sdist from setuptools.command.egg_info import manifest_maker from setuptools.dist import Distribution +from setuptools.tests import fail_on_ascii + + +py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") + SETUP_ATTRS = { 'name': 'sdist_test', @@ -77,6 +82,11 @@ def decompose(path): return path +def read_all_bytes(filename): + with io.open(filename, 'rb') as fp: + return fp.read() + + class TestSdistTest: def setup_method(self, method): @@ -147,6 +157,7 @@ class TestSdistTest: assert 'setup.py' not in manifest, manifest assert 'setup.cfg' not in manifest, manifest + @fail_on_ascii def test_manifest_is_written_with_utf8_encoding(self): # Test for #303. dist = Distribution(SETUP_ATTRS) @@ -167,16 +178,10 @@ class TestSdistTest: mm.filelist.append(filename) mm.write_manifest() - manifest = open(mm.manifest, 'rbU') - contents = manifest.read() - manifest.close() + contents = read_all_bytes(mm.manifest) # The manifest should be UTF-8 encoded - try: - u_contents = contents.decode('UTF-8') - except UnicodeDecodeError: - e = sys.exc_info()[1] - self.fail(e) + u_contents = contents.decode('UTF-8') # The manifest should contain the UTF-8 filename if six.PY2: @@ -185,89 +190,78 @@ class TestSdistTest: assert posix(filename) in u_contents - # Python 3 only - if six.PY3: + @py3_only + @fail_on_ascii + def test_write_manifest_allows_utf8_filenames(self): + # Test for #303. + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'setup.py' + mm = manifest_maker(dist) + mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') + os.mkdir('sdist_test.egg-info') - def test_write_manifest_allows_utf8_filenames(self): - # Test for #303. - dist = Distribution(SETUP_ATTRS) - dist.script_name = 'setup.py' - mm = manifest_maker(dist) - mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') - os.mkdir('sdist_test.egg-info') - - # UTF-8 filename - filename = os.path.join(b('sdist_test'), b('smörbröd.py')) - - # Must touch the file or risk removal - open(filename, "w").close() - - # Add filename and write manifest - with quiet(): - mm.run() - u_filename = filename.decode('utf-8') - mm.filelist.files.append(u_filename) - # Re-write manifest - mm.write_manifest() - - manifest = open(mm.manifest, 'rbU') - contents = manifest.read() - manifest.close() - - # The manifest should be UTF-8 encoded - try: - contents.decode('UTF-8') - except UnicodeDecodeError: - e = sys.exc_info()[1] - self.fail(e) - - # The manifest should contain the UTF-8 filename - assert posix(filename) in contents - - # The filelist should have been updated as well - assert u_filename in mm.filelist.files - - def test_write_manifest_skips_non_utf8_filenames(self): - """ - Files that cannot be encoded to UTF-8 (specifically, those that - weren't originally successfully decoded and have surrogate - escapes) should be omitted from the manifest. - See https://bitbucket.org/tarek/distribute/issue/303 for history. - """ - dist = Distribution(SETUP_ATTRS) - dist.script_name = 'setup.py' - mm = manifest_maker(dist) - mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') - os.mkdir('sdist_test.egg-info') - - # Latin-1 filename - filename = os.path.join(b('sdist_test'), LATIN1_FILENAME) - - # Add filename with surrogates and write manifest - with quiet(): - mm.run() - u_filename = filename.decode('utf-8', 'surrogateescape') - mm.filelist.append(u_filename) - # Re-write manifest - mm.write_manifest() - - manifest = open(mm.manifest, 'rbU') - contents = manifest.read() - manifest.close() - - # The manifest should be UTF-8 encoded - try: - contents.decode('UTF-8') - except UnicodeDecodeError: - e = sys.exc_info()[1] - self.fail(e) + # UTF-8 filename + filename = os.path.join(b('sdist_test'), b('smörbröd.py')) - # The Latin-1 filename should have been skipped - assert posix(filename) not in contents + # Must touch the file or risk removal + open(filename, "w").close() - # The filelist should have been updated as well - assert u_filename not in mm.filelist.files + # Add filename and write manifest + with quiet(): + mm.run() + u_filename = filename.decode('utf-8') + mm.filelist.files.append(u_filename) + # Re-write manifest + mm.write_manifest() + + contents = read_all_bytes(mm.manifest) + + # The manifest should be UTF-8 encoded + contents.decode('UTF-8') + + # The manifest should contain the UTF-8 filename + assert posix(filename) in contents + + # The filelist should have been updated as well + assert u_filename in mm.filelist.files + @py3_only + def test_write_manifest_skips_non_utf8_filenames(self): + """ + Files that cannot be encoded to UTF-8 (specifically, those that + weren't originally successfully decoded and have surrogate + escapes) should be omitted from the manifest. + See https://bitbucket.org/tarek/distribute/issue/303 for history. + """ + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'setup.py' + mm = manifest_maker(dist) + mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') + os.mkdir('sdist_test.egg-info') + + # Latin-1 filename + filename = os.path.join(b('sdist_test'), LATIN1_FILENAME) + + # Add filename with surrogates and write manifest + with quiet(): + mm.run() + u_filename = filename.decode('utf-8', 'surrogateescape') + mm.filelist.append(u_filename) + # Re-write manifest + mm.write_manifest() + + contents = read_all_bytes(mm.manifest) + + # The manifest should be UTF-8 encoded + contents.decode('UTF-8') + + # The Latin-1 filename should have been skipped + assert posix(filename) not in contents + + # The filelist should have been updated as well + assert u_filename not in mm.filelist.files + + @fail_on_ascii def test_manifest_is_read_with_utf8_encoding(self): # Test for #303. dist = Distribution(SETUP_ATTRS) @@ -299,46 +293,38 @@ class TestSdistTest: filename = filename.decode('utf-8') assert filename in cmd.filelist.files - # Python 3 only - if six.PY3: + @py3_only + def test_read_manifest_skips_non_utf8_filenames(self): + # Test for #303. + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'setup.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + # Create manifest + with quiet(): + cmd.run() + + # Add Latin-1 filename to manifest + filename = os.path.join(b('sdist_test'), LATIN1_FILENAME) + cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') + manifest = open(cmd.manifest, 'ab') + manifest.write(b('\n') + filename) + manifest.close() + + # The file must exist to be included in the filelist + open(filename, 'w').close() + + # Re-read manifest + cmd.filelist.files = [] + with quiet(): + cmd.read_manifest() + + # The Latin-1 filename should have been skipped + filename = filename.decode('latin-1') + assert filename not in cmd.filelist.files - def test_read_manifest_skips_non_utf8_filenames(self): - # Test for #303. - dist = Distribution(SETUP_ATTRS) - dist.script_name = 'setup.py' - cmd = sdist(dist) - cmd.ensure_finalized() - - # Create manifest - with quiet(): - cmd.run() - - # Add Latin-1 filename to manifest - filename = os.path.join(b('sdist_test'), LATIN1_FILENAME) - cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') - manifest = open(cmd.manifest, 'ab') - manifest.write(b('\n') + filename) - manifest.close() - - # The file must exist to be included in the filelist - open(filename, 'w').close() - - # Re-read manifest - cmd.filelist.files = [] - with quiet(): - try: - cmd.read_manifest() - except UnicodeDecodeError: - e = sys.exc_info()[1] - self.fail(e) - - # The Latin-1 filename should have been skipped - filename = filename.decode('latin-1') - assert filename not in cmd.filelist.files - - @pytest.mark.skipif(six.PY3 and locale.getpreferredencoding() != 'UTF-8', - reason='Unittest fails if locale is not utf-8 but the manifests is ' - 'recorded correctly') + @fail_on_ascii def test_sdist_with_utf8_encoded_filename(self): # Test for #303. dist = Distribution(SETUP_ATTRS) @@ -431,5 +417,5 @@ def test_default_revctrl(): """ ep_def = 'svn_cvs = setuptools.command.sdist:_default_revctrl' ep = pkg_resources.EntryPoint.parse(ep_def) - res = ep._load() + res = ep.resolve() assert hasattr(res, '__iter__') diff --git a/setuptools/tests/test_setuptools.py b/setuptools/tests/test_setuptools.py new file mode 100644 index 00000000..e59800d2 --- /dev/null +++ b/setuptools/tests/test_setuptools.py @@ -0,0 +1,48 @@ +import os + +import pytest + +import setuptools + + +@pytest.fixture +def example_source(tmpdir): + tmpdir.mkdir('foo') + (tmpdir / 'foo/bar.py').write('') + (tmpdir / 'readme.txt').write('') + return tmpdir + + +def test_findall(example_source): + found = list(setuptools.findall(str(example_source))) + expected = ['readme.txt', 'foo/bar.py'] + expected = [example_source.join(fn) for fn in expected] + assert found == expected + + +def test_findall_curdir(example_source): + with example_source.as_cwd(): + found = list(setuptools.findall()) + expected = ['readme.txt', os.path.join('foo', 'bar.py')] + assert found == expected + + +@pytest.fixture +def can_symlink(tmpdir): + """ + Skip if cannot create a symbolic link + """ + link_fn = 'link' + target_fn = 'target' + try: + os.symlink(target_fn, link_fn) + except (OSError, NotImplementedError, AttributeError): + pytest.skip("Cannot create symbolic links") + os.remove(link_fn) + + +def test_findall_missing_symlink(tmpdir, can_symlink): + with tmpdir.as_cwd(): + os.symlink('foo', 'bar') + found = list(setuptools.findall()) + assert found == [] diff --git a/setuptools/version.py b/setuptools/version.py index 1b1703fd..09bbb730 100644 --- a/setuptools/version.py +++ b/setuptools/version.py @@ -1 +1 @@ -__version__ = '11.1' +__version__ = '19.3' @@ -1,5 +1,5 @@ [tox] envlist = py26,py27,py31,py32,py33,py34 + [testenv] -deps=pytest -commands=py.test {posargs} +commands=python setup.py test |