From f6cb3d29d919ff6b9313b8a3281c66cfb368c29b Mon Sep 17 00:00:00 2001 From: Erik Bray Date: Mon, 13 Feb 2012 16:22:33 -0500 Subject: This allows the sdist command to ensure that any files listed in package_data are included in the dist, regardless of whether it's under version control, as is the case with distutil's sdist. Setting include_package_data=True disables this functionality. --HG-- branch : distribute extra : rebase_source : 2cae1675c638dc12fd556368074c6b5c691c6f58 --- setuptools/command/sdist.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index c49839cd..484f8276 100755 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -186,6 +186,14 @@ class sdist(_sdist): if self.distribution.has_pure_modules(): build_py = self.get_finalized_command('build_py') self.filelist.extend(build_py.get_source_files()) + # This functionality is incompatible with include_package_data, and + # will in fact create an infinite recursion if include_package_data + # is True. Use of include_package_data will imply that + # distutils-style automatic handling of package_data is disabled + if not self.distribution.include_package_data: + for _, src_dir, _, filenames in build_py.data_files: + self.filelist.extend([os.path.join(src_dir, filename) + for filename in filenames]) if self.distribution.has_ext_modules(): build_ext = self.get_finalized_command('build_ext') -- cgit v1.2.3 From dbc8208b379835ee43224e83829b41790347e1e2 Mon Sep 17 00:00:00 2001 From: Pedro Romano Date: Thu, 30 Aug 2012 11:58:22 +0100 Subject: Use 'optparse' module to parse command line arguments. --HG-- branch : distribute extra : rebase_source : 09f901daf15d650d2b2c5f620ef3660d78e5e4b7 --- distribute_setup.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/distribute_setup.py b/distribute_setup.py index f3199d12..5b10f1ef 100644 --- a/distribute_setup.py +++ b/distribute_setup.py @@ -19,6 +19,7 @@ import time import fnmatch import tempfile import tarfile +import optparse from distutils import log try: @@ -495,22 +496,27 @@ def _extractall(self, path=".", members=None): self._dbg(1, "tarfile: %s" % e) -def _build_install_args(argv): +def _build_install_args(user_install): install_args = [] - user_install = '--user' in argv - if user_install and sys.version_info < (2, 6): - log.warn("--user requires Python 2.6 or later") - raise SystemExit(1) if user_install: - install_args.append('--user') + if sys.version_info < (2, 6): + log.warn("--user requires Python 2.6 or later") + raise SystemExit(1) + else: + install_args.append('--user') return install_args -def main(argv, version=DEFAULT_VERSION): +def main(version=DEFAULT_VERSION): """Install or upgrade setuptools and EasyInstall""" + parser = optparse.OptionParser() + parser.add_option( + '--user', dest='user_install', action='store_true', default=False, + help='install in user site package (requires Python 2.6 or later)') + options, args = parser.parse_args() tarball = download_setuptools() - _install(tarball, _build_install_args(argv)) + _install(tarball, _build_install_args(options.user_install)) if __name__ == '__main__': - main(sys.argv[1:]) + main() -- cgit v1.2.3 From 9c822a826b6d0ed1ecd746c6b3f8039054d2c84b Mon Sep 17 00:00:00 2001 From: Pedro Romano Date: Thu, 30 Aug 2012 12:18:15 +0100 Subject: New command line option to enable downloading the distribute package from an alternative location. --HG-- branch : distribute extra : rebase_source : 114d0bb6c9c065ae369b8d0e603eb6d1d3a0b6fc --- distribute_setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/distribute_setup.py b/distribute_setup.py index 5b10f1ef..eca355ce 100644 --- a/distribute_setup.py +++ b/distribute_setup.py @@ -513,8 +513,12 @@ def main(version=DEFAULT_VERSION): parser.add_option( '--user', dest='user_install', action='store_true', default=False, help='install in user site package (requires Python 2.6 or later)') + parser.add_option( + '--download-base', dest='download_base', metavar="URL", + default=DEFAULT_URL, + help='alternative URL from where to download the distribute package') options, args = parser.parse_args() - tarball = download_setuptools() + tarball = download_setuptools(download_base=options.download_base) _install(tarball, _build_install_args(options.user_install)) -- cgit v1.2.3 From bf06ab669aacba63dfe4ee8adb5f50b72387c0bd Mon Sep 17 00:00:00 2001 From: Erik Bray Date: Wed, 5 Sep 2012 19:01:58 -0400 Subject: adds a test for pr #4 --HG-- branch : distribute extra : rebase_source : 347b39fb279b7168490a2c62562b4223b6c419e2 --- setuptools/tests/test_sdist.py | 79 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 setuptools/tests/test_sdist.py diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py new file mode 100644 index 00000000..8d9ed922 --- /dev/null +++ b/setuptools/tests/test_sdist.py @@ -0,0 +1,79 @@ +"""sdist tests""" + + +import os +import shutil +import sys +import tempfile +import unittest +from StringIO import StringIO + + +from setuptools.command.sdist import sdist +from setuptools.dist import Distribution + + +SETUP_ATTRS = { + 'name': 'sdist_test', + 'version': '0.0', + 'packages': ['sdist_test'], + 'package_data': {'sdist_test': ['*.txt']} +} + + +SETUP_PY = """\ +from setuptools import setup + +setup(**%r) +""" % SETUP_ATTRS + + +class TestSdistTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + f = open(os.path.join(self.temp_dir, 'setup.py'), 'w') + f.write(SETUP_PY) + f.close() + # Set up the rest of the test package + test_pkg = os.path.join(self.temp_dir, 'sdist_test') + os.mkdir(test_pkg) + # *.rst was not included in package_data, so c.rst should not be + # automatically added to the manifest when not under version control + for fname in ['__init__.py', 'a.txt', 'b.txt', 'c.rst']: + # Just touch the files; their contents are irrelevant + open(os.path.join(test_pkg, fname), 'w').close() + + self.old_cwd = os.getcwd() + os.chdir(self.temp_dir) + + def tearDown(self): + os.chdir(self.old_cwd) + shutil.rmtree(self.temp_dir) + + def test_package_data_in_sdist(self): + """Regression test for pull request #4: ensures that files listed in + package_data are included in the manifest even if they're not added to + version control. + """ + + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'setup.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + # squelch output + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = StringIO() + sys.stderr = StringIO() + try: + cmd.run() + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + + manifest = cmd.filelist.files + + self.assert_(os.path.join('sdist_test', 'a.txt') in manifest) + self.assert_(os.path.join('sdist_test', 'b.txt') in manifest) + self.assert_(os.path.join('sdist_test', 'c.rst') not in manifest) -- cgit v1.2.3 From 6eece1023d4a0d5e7229d33dc53600e8e31d7942 Mon Sep 17 00:00:00 2001 From: Erik Bray Date: Wed, 5 Sep 2012 19:12:10 -0400 Subject: Add a note about how this fix relates to include_package_data. Although still somewhat buried, this should clarify the issue raised in #218, and points out that include_package_data should really only be used if all the relevant files are in a supported version control system. --HG-- branch : distribute extra : rebase_source : e29284c59469ba9543b57afa64a0f52ee07eebf3 --- docs/setuptools.txt | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 4105dc2e..34169c7e 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -756,17 +756,18 @@ e.g.:: include_package_data = True ) -This tells setuptools to install any data files it finds in your packages. The -data files must be under CVS or Subversion control, or else they must be +This tells setuptools to install any data files it finds in your packages. +The data files must be under CVS or Subversion control, or else they must be specified via the distutils' ``MANIFEST.in`` file. (They can also be tracked by another revision control system, using an appropriate plugin. See the section below on `Adding Support for Other Revision Control Systems`_ for information on how to write such plugins.) -If you want finer-grained control over what files are included (for example, if -you have documentation files in your package directories and want to exclude -them from installation), then you can also use the ``package_data`` keyword, -e.g.:: +If the data files are not under version control, or are not in a supported +version control system, or if you want finer-grained control over what files +are included (for example, if you have documentation files in your package +directories and want to exclude them from installation), then you can also use +the ``package_data`` keyword, e.g.:: from setuptools import setup, find_packages setup( @@ -822,7 +823,10 @@ converts slashes to appropriate platform-specific separators at build time. (Note: although the ``package_data`` argument was previously only available in ``setuptools``, it was also added to the Python ``distutils`` package as of Python 2.4; there is `some documentation for the feature`__ available on the -python.org website.) +python.org website. If using the setuptools-specific ``include_package_data`` +argument, files specified by ``package_data`` will *not* be automatically +added to the manifest unless they are tracked by a supported version control +system, or are listed in the MANIFEST.in file.) __ http://docs.python.org/dist/node11.html -- cgit v1.2.3 From 9fbb4e4ac9f4ba7c9e4de2168844bf3e6c64416e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 5 Sep 2012 20:45:21 -0400 Subject: Updated changelog --HG-- branch : distribute extra : rebase_source : 96d2de7069a6d567ad125e408df82241f515bd07 --- CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 3eea84eb..82ae6e58 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -16,6 +16,9 @@ CHANGES * Issue #313: Support for sdist subcommands (Python 2.7) * Issue #314: test_local_index() would fail an OS X. * Issue #310: Non-ascii characters in a namespace __init__.py causes errors. +* Issue #218: Improved documentation on behavior of `package_data` and + `include_package_data`. Files indicated by `package_data` are now included + in the manifest. ------ 0.6.28 -- cgit v1.2.3 From 932ea7cc2b1ef4eba5d8c1ee46c64cce529e3207 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 5 Sep 2012 21:10:29 -0400 Subject: Extract argument parsing as a separate function --HG-- branch : distribute extra : rebase_source : d2143d4dd1332558689b6b90770c013bce3a7e37 --- distribute_setup.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/distribute_setup.py b/distribute_setup.py index eca355ce..95ba23c5 100644 --- a/distribute_setup.py +++ b/distribute_setup.py @@ -20,6 +20,7 @@ import fnmatch import tempfile import tarfile import optparse + from distutils import log try: @@ -496,19 +497,22 @@ def _extractall(self, path=".", members=None): self._dbg(1, "tarfile: %s" % e) -def _build_install_args(user_install): +def _build_install_args(options): + """ + Build the arguments to 'python setup.py install' on the distribute package + """ install_args = [] - if user_install: + if options.user_install: if sys.version_info < (2, 6): log.warn("--user requires Python 2.6 or later") raise SystemExit(1) - else: - install_args.append('--user') + install_args.append('--user') return install_args - -def main(version=DEFAULT_VERSION): - """Install or upgrade setuptools and EasyInstall""" +def _parse_args(): + """ + Parse the command line for options + """ parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, @@ -518,9 +522,14 @@ def main(version=DEFAULT_VERSION): default=DEFAULT_URL, help='alternative URL from where to download the distribute package') options, args = parser.parse_args() - tarball = download_setuptools(download_base=options.download_base) - _install(tarball, _build_install_args(options.user_install)) + # positional arguments are ignored + return options +def main(version=DEFAULT_VERSION): + """Install or upgrade setuptools and EasyInstall""" + options = _parse_args() + tarball = download_setuptools(download_base=options.download_base) + _install(tarball, _build_install_args(options)) if __name__ == '__main__': main() -- cgit v1.2.3 From 06959116eed7effdbf3af575946f619cdb7fcd71 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 5 Sep 2012 21:11:19 -0400 Subject: Updated changelog --HG-- branch : distribute extra : rebase_source : f92b138358cd2699cea63bf39a552e10bf62c02d --- CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 82ae6e58..b59a0b00 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -19,6 +19,8 @@ CHANGES * Issue #218: Improved documentation on behavior of `package_data` and `include_package_data`. Files indicated by `package_data` are now included in the manifest. +* `distribute_setup.py` now allows a `--download-base` argument for retrieving + distribute from a specified location. ------ 0.6.28 -- cgit v1.2.3 From d847dbfeee2b2c85d7bc0a66813c33836a9850fa Mon Sep 17 00:00:00 2001 From: Erik Bray Date: Thu, 6 Sep 2012 15:51:09 -0400 Subject: Adds a fix for issue #318, including a regression test. This also fixes another test that was passing trivially due to *bug* in yet another test, ugh --HG-- branch : distribute extra : rebase_source : 476550766b7b756dced040ad4356b7685d6f062a --- setuptools/dist.py | 3 +- setuptools/sandbox.py | 10 ++-- setuptools/tests/__init__.py | 2 +- setuptools/tests/test_easy_install.py | 102 ++++++++++++++++++++++++++-------- 4 files changed, 88 insertions(+), 29 deletions(-) diff --git a/setuptools/dist.py b/setuptools/dist.py index 0ad18122..6607cf7b 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -264,6 +264,7 @@ class Distribution(_Distribution): def fetch_build_egg(self, req): """Fetch an egg needed for building""" + try: cmd = self._egg_fetcher cmd.package_index.to_scan = [] @@ -287,7 +288,7 @@ class Distribution(_Distribution): cmd = easy_install( dist, args=["x"], install_dir=os.curdir, exclude_scripts=True, always_copy=False, build_directory=None, editable=False, - upgrade=False, multi_version=True, no_report = True + upgrade=False, multi_version=True, no_report=True, user=False ) cmd.ensure_finalized() self._egg_fetcher = cmd diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index ab2543d9..64f725e7 100755 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -166,12 +166,12 @@ else: _EXCEPTIONS = [] try: - from win32com.client.gencache import GetGeneratePath - _EXCEPTIONS.append(GetGeneratePath()) - del GetGeneratePath + from win32com.client.gencache import GetGeneratePath + _EXCEPTIONS.append(GetGeneratePath()) + del GetGeneratePath except ImportError: - # it appears pywin32 is not installed, so no need to exclude. - pass + # it appears pywin32 is not installed, so no need to exclude. + pass class DirectorySandbox(AbstractSandbox): """Restrict operations to a single subdirectory - pseudo-chroot""" diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index eec76efd..b6988a08 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -38,7 +38,7 @@ def makeSetup(**args): try: return setuptools.setup(**args) finally: - distutils.core_setup_stop_after = None + distutils.core._setup_stop_after = None class DependsTests(unittest.TestCase): diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index 13f668d0..ce8b611b 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -12,6 +12,7 @@ import urlparse import StringIO import distutils.core +from setuptools.sandbox import run_setup, SandboxViolation from setuptools.command.easy_install import easy_install, get_script_args, main from setuptools.command.easy_install import PthDistributions from setuptools.command import easy_install as easy_install_pkg @@ -110,7 +111,9 @@ class TestEasyInstallTest(unittest.TestCase): old_wd = os.getcwd() try: os.chdir(dir) - main([]) + reset_setup_stop_context( + lambda: self.assertRaises(SystemExit, main, []) + ) finally: os.chdir(old_wd) shutil.rmtree(dir) @@ -247,7 +250,7 @@ class TestUserInstallTest(unittest.TestCase): cmd.local_index.scan([new_location]) res = cmd.easy_install('foo') self.assertEqual(os.path.realpath(res.location), - os.path.realpath(new_location)) + os.path.realpath(new_location)) finally: sys.path.remove(target) for basedir in [new_location, target, ]: @@ -262,6 +265,50 @@ class TestUserInstallTest(unittest.TestCase): else: del os.environ['PYTHONPATH'] + def test_setup_requires(self): + """Regression test for issue #318 + + Ensures that a package with setup_requires can be installed when + distribute is installed in the user site-packages without causing a + SandboxViolation. + """ + + test_setup_attrs = { + 'name': 'test_pkg', 'version': '0.0', + 'setup_requires': ['foobar'], + 'dependency_links': [os.path.abspath(self.dir)] + } + + test_pkg = os.path.join(self.dir, 'test_pkg') + test_setup_py = os.path.join(test_pkg, 'setup.py') + test_setup_cfg = os.path.join(test_pkg, 'setup.cfg') + os.mkdir(test_pkg) + + f = open(test_setup_py, 'w') + f.write(textwrap.dedent("""\ + import setuptools + setuptools.setup(**%r) + """ % test_setup_attrs)) + f.close() + + foobar_path = os.path.join(self.dir, 'foobar-0.1.tar.gz') + make_trivial_sdist( + foobar_path, + textwrap.dedent("""\ + import setuptools + setuptools.setup( + name='foobar', + version='0.1' + ) + """)) + + try: + reset_setup_stop_context( + lambda: run_setup(test_setup_py, ['install']) + ) + except SandboxViolation: + self.fail('Installation caused SandboxViolation') + class TestSetupRequires(unittest.TestCase): @@ -319,30 +366,41 @@ class TestSetupRequires(unittest.TestCase): doesn't exist) and invoke installer on it. """ def build_sdist(dir): - setup_py = tarfile.TarInfo(name="setup.py") - try: - # Python 3 (StringIO gets converted to io module) - MemFile = StringIO.BytesIO - except AttributeError: - MemFile = StringIO.StringIO - setup_py_bytes = MemFile(textwrap.dedent(""" - import setuptools - setuptools.setup( - name="distribute-test-fetcher", - version="1.0", - setup_requires = ['does-not-exist'], - ) - """).lstrip().encode('utf-8')) - setup_py.size = len(setup_py_bytes.getvalue()) dist_path = os.path.join(dir, 'distribute-test-fetcher-1.0.tar.gz') - dist = tarfile.open(dist_path, 'w:gz') - try: - dist.addfile(setup_py, fileobj=setup_py_bytes) - finally: - dist.close() + make_trivial_sdist( + dist_path, + textwrap.dedent(""" + import setuptools + setuptools.setup( + name="distribute-test-fetcher", + version="1.0", + setup_requires = ['does-not-exist'], + ) + """).lstrip()) installer(dist_path) tempdir_context(build_sdist) + +def make_trivial_sdist(dist_path, setup_py): + """Create a simple sdist tarball at dist_path, containing just a + setup.py, the contents of which are provided by the setup_py string. + """ + + setup_py_file = tarfile.TarInfo(name='setup.py') + try: + # Python 3 (StringIO gets converted to io module) + MemFile = StringIO.BytesIO + except AttributeError: + MemFile = StringIO.StringIO + setup_py_bytes = MemFile(setup_py.encode('utf-8')) + setup_py_file.size = len(setup_py_bytes.getvalue()) + dist = tarfile.open(dist_path, 'w:gz') + try: + dist.addfile(setup_py_file, fileobj=setup_py_bytes) + finally: + dist.close() + + def tempdir_context(f, cd=lambda dir:None): """ Invoke f in the context -- cgit v1.2.3 From 8866de1785cc6961d2111f1e0f55b781a7de660d Mon Sep 17 00:00:00 2001 From: Erik Bray Date: Tue, 11 Sep 2012 08:00:12 -0400 Subject: Remove missing import (since b62968cd2666) --HG-- branch : distribute extra : rebase_source : d1190f895d794dfcb838f7eb40a60ab07b8b309e --- _markerlib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_markerlib/__init__.py b/_markerlib/__init__.py index a7b26037..9c37bb9a 100644 --- a/_markerlib/__init__.py +++ b/_markerlib/__init__.py @@ -1,2 +1,2 @@ """Used by pkg_resources to interpret PEP 345 environment markers.""" -from _markerlib.markers import default_environment, compile, interpret, as_function +from _markerlib.markers import default_environment, compile, interpret -- cgit v1.2.3 From cada83b25777a9b089b85bc4417baa7016a9c652 Mon Sep 17 00:00:00 2001 From: Erik Bray Date: Tue, 11 Sep 2012 08:12:04 -0400 Subject: Make this test less chatty --HG-- branch : distribute extra : rebase_source : da18973713f46ff00931ea79f0fcd39a13fb8349 --- setuptools/tests/test_easy_install.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index ce8b611b..e49c6f49 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -302,12 +302,19 @@ class TestUserInstallTest(unittest.TestCase): ) """)) + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = StringIO.StringIO() + sys.stderr = StringIO.StringIO() try: reset_setup_stop_context( lambda: run_setup(test_setup_py, ['install']) ) except SandboxViolation: self.fail('Installation caused SandboxViolation') + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr class TestSetupRequires(unittest.TestCase): -- cgit v1.2.3