diff options
Diffstat (limited to 'setuptools')
34 files changed, 1405 insertions, 880 deletions
diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 57236a5b..fffcac76 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -3,6 +3,7 @@ __import__('setuptools.bootstrap').bootstrap.ensure_deps() import os +import functools import distutils.core import distutils.filelist from distutils.core import Command as _Command @@ -76,21 +77,24 @@ class PackageFinder(object): yield pkg @staticmethod - def _all_dirs(base_path): + def _candidate_dirs(base_path): """ - Return all dirs in base_path, relative to base_path + Return all dirs in base_path that might be packages. """ + has_dot = lambda name: '.' in name for root, dirs, files in os.walk(base_path, followlinks=True): + # Exclude directories that contain a period, as they cannot be + # packages. Mutate the list to avoid traversal. + dirs[:] = filterfalse(has_dot, dirs) for dir in dirs: yield os.path.relpath(os.path.join(root, dir), base_path) @classmethod def _find_packages_iter(cls, base_path): - dirs = cls._all_dirs(base_path) - suitable = filterfalse(lambda n: '.' in n, dirs) + candidates = cls._candidate_dirs(base_path) return ( path.replace(os.path.sep, '.') - for path in suitable + for path in candidates if cls._looks_like_package(os.path.join(base_path, path)) ) @@ -123,30 +127,45 @@ class Command(_Command): command_consumes_arguments = False def __init__(self, dist, **kw): - # Add support for keyword arguments - _Command.__init__(self,dist) - for k,v in kw.items(): - setattr(self,k,v) + """ + Construct the command for dist, updating + vars(self) with any keyword parameters. + """ + _Command.__init__(self, dist) + vars(self).update(kw) def reinitialize_command(self, command, reinit_subcommands=0, **kw): cmd = _Command.reinitialize_command(self, command, reinit_subcommands) - for k,v in kw.items(): - setattr(cmd,k,v) # update command with keywords + vars(cmd).update(kw) return cmd -distutils.core.Command = Command # we can't patch distutils.cmd, alas +# we can't patch distutils.cmd, alas +distutils.core.Command = Command + + +def _find_all_simple(path): + """ + Find all files under 'path' + """ + results = ( + os.path.join(base, file) + for base, dirs, files in os.walk(path, followlinks=True) + for file in files + ) + return filter(os.path.isfile, results) + -def findall(dir = os.curdir): - """Find all files under 'dir' and return the list of full filenames - (relative to 'dir'). +def findall(dir=os.curdir): + """ + Find all files under 'dir' and return the list of full filenames. + Unless dir is '.', return full filenames with dir prepended. """ - all_files = [] - for base, dirs, files in os.walk(dir, followlinks=True): - if base==os.curdir or base.startswith(os.curdir+os.sep): - base = base[2:] - if base: - files = [os.path.join(base, f) for f in files] - all_files.extend(filter(os.path.isfile, files)) - return all_files - -distutils.filelist.findall = findall # fix findall bug in distutils. + files = _find_all_simple(dir) + if dir == os.curdir: + make_rel = functools.partial(os.path.relpath, start=dir) + files = map(make_rel, files) + return list(files) + + +# fix findall bug in distutils (http://bugs.python.org/issue12885) +distutils.filelist.findall = findall diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 3d241b99..73f8e3f1 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -2,7 +2,6 @@ Build .egg distributions""" -# This module should be kept compatible with Python 2.3 from distutils.errors import DistutilsSetupError from distutils.dir_util import remove_tree, mkpath from distutils import log @@ -407,10 +406,6 @@ def scan_module(egg_dir, base, name, stubs): if bad in symbols: log.warn("%s: module MAY be using inspect.%s", module, bad) safe = False - if '__name__' in symbols and '__main__' in symbols and '.' not in module: - if sys.version[:3] == "2.4": # -m works w/zipfiles in 2.5 - log.warn("%s: top-level module may be 'python -m' script", module) - safe = False return safe @@ -442,7 +437,7 @@ INSTALL_DIRECTORY_ATTRS = [ ] -def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=None, +def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True, mode='w'): """Create a zip file from all the files under 'base_dir'. The output zip file will be named 'base_dir' + ".zip". Uses either the "zipfile" @@ -464,11 +459,7 @@ def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=None, z.write(path, p) log.debug("adding '%s'" % p) - if compress is None: - # avoid 2.3 zipimport bug when 64 bits - compress = (sys.version >= "2.4") - - compression = [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED][bool(compress)] + compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED if not dry_run: z = zipfile.ZipFile(zip_filename, mode, compression=compression) for dirname, dirs, files in os.walk(base_dir): diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index e4b2c593..92e4a189 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -11,8 +11,8 @@ import itertools from setuptools.extension import Library try: - # Attempt to use Pyrex for building extensions, if available - from Pyrex.Distutils.build_ext import build_ext as _build_ext + # Attempt to use Cython for building extensions, if available + from Cython.Distutils.build_ext import build_ext as _build_ext except ImportError: _build_ext = _du_build_ext @@ -42,7 +42,6 @@ elif os.name != 'nt': if_dl = lambda s: s if have_rtld else '' - class build_ext(_build_ext): def run(self): """Build extensions in build directory, then copy if --inplace""" @@ -74,15 +73,6 @@ class build_ext(_build_ext): if ext._needs_stub: self.write_stub(package_dir or os.curdir, ext, True) - if _build_ext is not _du_build_ext and not hasattr(_build_ext, - 'pyrex_sources'): - # Workaround for problems using some Pyrex versions w/SWIG and/or 2.4 - def swig_sources(self, sources, *otherargs): - # first do any Pyrex processing - sources = _build_ext.swig_sources(self, sources) or sources - # Then do any actual SWIG stuff on the remainder - return _du_build_ext.swig_sources(self, sources, *otherargs) - def get_ext_filename(self, fullname): filename = _build_ext.get_ext_filename(self, fullname) if fullname in self.ext_map: @@ -176,6 +166,7 @@ class build_ext(_build_ext): return _build_ext.get_export_symbols(self, ext) def build_extension(self, ext): + ext._convert_pyx_sources_to_lang() _compiler = self.compiler try: if isinstance(ext, Library): diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 98080694..8a50f032 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -2,9 +2,13 @@ from glob import glob from distutils.util import convert_path import distutils.command.build_py as orig import os -import sys import fnmatch import textwrap +import io +import distutils.errors +import collections +import itertools + try: from setuptools.lib2to3_ex import Mixin2to3 @@ -136,22 +140,7 @@ class build_py(orig.build_py, Mixin2to3): mf.setdefault(src_dirs[d], []).append(path) def get_data_files(self): - pass # kludge 2.4 for lazy computation - - if sys.version < "2.4": # Python 2.4 already has this code - def get_outputs(self, include_bytecode=1): - """Return complete list of files copied to the build directory - - This includes both '.py' files and data files, as well as '.pyc' - and '.pyo' files if 'include_bytecode' is true. (This method is - needed for the 'install_lib' command to do its job properly, and to - generate a correct installation manifest.) - """ - return orig.build_py.get_outputs(self, include_bytecode) + [ - os.path.join(build_dir, filename) - for package, src_dir, build_dir, filenames in self.data_files - for filename in filenames - ] + pass # Lazily compute data files in _get_data_files() function. def check_package(self, package, package_dir): """Check namespace packages' __init__ for declare_namespace""" @@ -172,17 +161,15 @@ class build_py(orig.build_py, Mixin2to3): else: return init_py - f = open(init_py, 'rbU') - if 'declare_namespace'.encode() not in f.read(): - from distutils.errors import DistutilsError - - raise DistutilsError( + with io.open(init_py, 'rb') as f: + contents = f.read() + if b'declare_namespace' not in contents: + raise distutils.errors.DistutilsError( "Namespace package problem: %s is a namespace package, but " "its\n__init__.py does not call declare_namespace()! Please " 'fix it.\n(See the setuptools manual under ' '"Namespace Packages" for details.)\n"' % (package,) ) - f.close() return init_py def initialize_options(self): @@ -197,20 +184,25 @@ class build_py(orig.build_py, Mixin2to3): def exclude_data_files(self, package, src_dir, files): """Filter filenames for package's data files in 'src_dir'""" - globs = (self.exclude_package_data.get('', []) - + self.exclude_package_data.get(package, [])) - bad = [] - for pattern in globs: - bad.extend( - fnmatch.filter( - files, os.path.join(src_dir, convert_path(pattern)) - ) + globs = ( + self.exclude_package_data.get('', []) + + self.exclude_package_data.get(package, []) + ) + bad = set( + item + for pattern in globs + for item in fnmatch.filter( + files, + os.path.join(src_dir, convert_path(pattern)), ) - bad = dict.fromkeys(bad) - seen = {} + ) + seen = collections.defaultdict(itertools.count) return [ - f for f in files if f not in bad - and f not in seen and seen.setdefault(f, 1) # ditch dupes + fn + for fn in files + if fn not in bad + # ditch dupes + and not next(seen[fn]) ] diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index 9f0b6f47..ef9ac22d 100755 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -3,6 +3,7 @@ from distutils import log from distutils.errors import DistutilsError, DistutilsOptionError import os import glob +import io import six @@ -54,8 +55,8 @@ class develop(easy_install): # pick up setup-dir .egg files only: no .egg-info self.package_index.scan(glob.glob('*.egg')) - self.egg_link = os.path.join(self.install_dir, ei.egg_name + - '.egg-link') + egg_link_fn = ei.egg_name + '.egg-link' + self.egg_link = os.path.join(self.install_dir, egg_link_fn) self.egg_base = ei.egg_base if self.egg_path is None: self.egg_path = os.path.abspath(ei.egg_base) @@ -125,9 +126,8 @@ class develop(easy_install): # create an .egg-link in the installation dir, pointing to our egg log.info("Creating %s (link to %s)", self.egg_link, self.egg_base) if not self.dry_run: - f = open(self.egg_link, "w") - f.write(self.egg_path + "\n" + self.setup_path) - f.close() + with open(self.egg_link, "w") as f: + f.write(self.egg_path + "\n" + self.setup_path) # postprocess the installed distro, fixing up .pth, installing scripts, # and handling requirements self.process_distribution(None, self.dist, not self.no_deps) @@ -164,7 +164,33 @@ class develop(easy_install): for script_name in self.distribution.scripts or []: script_path = os.path.abspath(convert_path(script_name)) script_name = os.path.basename(script_path) - f = open(script_path, 'rU') - script_text = f.read() - f.close() + with io.open(script_path) as strm: + script_text = strm.read() self.install_script(dist, script_name, script_text, script_path) + + def install_wrapper_scripts(self, dist): + dist = VersionlessRequirement(dist) + return easy_install.install_wrapper_scripts(self, dist) + + +class VersionlessRequirement(object): + """ + Adapt a pkg_resources.Distribution to simply return the project + name as the 'requirement' so that scripts will work across + multiple versions. + + >>> dist = Distribution(project_name='foo', version='1.0') + >>> str(dist.as_requirement()) + 'foo==1.0' + >>> adapted_dist = VersionlessRequirement(dist) + >>> str(adapted_dist.as_requirement()) + 'foo' + """ + def __init__(self, dist): + self.__dist = dist + + def __getattr__(self, name): + return getattr(self.__dist, name) + + def as_requirement(self): + return self.project_name diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index a24e3b59..6aab38c8 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -20,6 +20,7 @@ from distutils.errors import DistutilsArgError, DistutilsOptionError, \ from distutils.command.install import INSTALL_SCHEMES, SCHEME_KEYS from distutils import log, dir_util from distutils.command.build_scripts import first_line_re +from distutils.spawn import find_executable import sys import os import zipimport @@ -35,6 +36,9 @@ import warnings import site import struct import contextlib +import subprocess +import shlex +import io import six from six.moves import configparser @@ -55,15 +59,10 @@ from pkg_resources import ( ) import pkg_resources - # Turn on PEP440Warnings warnings.filterwarnings("default", category=pkg_resources.PEP440Warning) -sys_executable = os.environ.get('__PYVENV_LAUNCHER__', - os.path.normpath(sys.executable)) - - __all__ = [ 'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg', 'main', 'get_exe_prefixes', @@ -155,12 +154,9 @@ class easy_install(Command): create_index = PackageIndex def initialize_options(self): - if site.ENABLE_USER_SITE: - whereami = os.path.abspath(__file__) - self.user = whereami.startswith(site.USER_SITE) - else: - self.user = 0 - + # the --user option seems to be an opt-in one, + # so the default should be False. + self.user = 0 self.zip_ok = self.local_snapshots_ok = None self.install_dir = self.script_dir = self.exclude_scripts = None self.index_url = None @@ -206,20 +202,34 @@ class easy_install(Command): ) def delete_blockers(self, blockers): - for filename in blockers: - if os.path.exists(filename) or os.path.islink(filename): - log.info("Deleting %s", filename) - if not self.dry_run: - if (os.path.isdir(filename) and - not os.path.islink(filename)): - rmtree(filename) - else: - os.unlink(filename) + extant_blockers = ( + filename for filename in blockers + if os.path.exists(filename) or os.path.islink(filename) + ) + list(map(self._delete_path, extant_blockers)) + + def _delete_path(self, path): + log.info("Deleting %s", path) + if self.dry_run: + return + + is_tree = os.path.isdir(path) and not os.path.islink(path) + remover = rmtree if is_tree else os.unlink + remover(path) + + @staticmethod + def _render_version(): + """ + Render the Setuptools version and installation details, then exit. + """ + ver = sys.version[:3] + dist = get_distribution('setuptools') + tmpl = 'setuptools {dist.version} from {dist.location} (Python {ver})' + print(tmpl.format(**locals())) + raise SystemExit() def finalize_options(self): - if self.version: - print('setuptools %s' % get_distribution('setuptools').version) - sys.exit() + self.version and self._render_version() py_version = sys.version.split()[0] prefix, exec_prefix = get_config_vars('prefix', 'exec_prefix') @@ -243,18 +253,7 @@ class easy_install(Command): self.config_vars['userbase'] = self.install_userbase self.config_vars['usersite'] = self.install_usersite - # fix the install_dir if "--user" was used - # XXX: duplicate of the code in the setup command - if self.user and site.ENABLE_USER_SITE: - self.create_home_path() - if self.install_userbase is None: - raise DistutilsPlatformError( - "User base directory is not specified") - self.install_base = self.install_platbase = self.install_userbase - if os.name == 'posix': - self.select_scheme("unix_user") - else: - self.select_scheme(os.name + "_user") + self._fix_install_dir_for_user_site() self.expand_basedirs() self.expand_dirs() @@ -349,6 +348,21 @@ class easy_install(Command): self.outputs = [] + def _fix_install_dir_for_user_site(self): + """ + Fix the install_dir if "--user" was used. + """ + if not self.user or not site.ENABLE_USER_SITE: + return + + self.create_home_path() + if self.install_userbase is None: + msg = "User base directory is not specified" + raise DistutilsPlatformError(msg) + self.install_base = self.install_platbase = self.install_userbase + scheme_name = os.name.replace('posix', 'unix') + '_user' + self.select_scheme(scheme_name) + def _expand_attrs(self, attrs): for attr in attrs: val = getattr(self, attr) @@ -441,7 +455,7 @@ class easy_install(Command): self.pth_file = None PYTHONPATH = os.environ.get('PYTHONPATH', '').split(os.pathsep) - if instdir not in map(normalize_path, [_f for _f in PYTHONPATH if _f]): + if instdir not in map(normalize_path, filter(None, PYTHONPATH)): # only PYTHONPATH dirs need a site.py, so pretend it's there self.sitepy_installed = True elif self.multi_version and not os.path.exists(pth_file): @@ -449,43 +463,49 @@ class easy_install(Command): self.pth_file = None # and don't create a .pth file self.install_dir = instdir - def cant_write_to_target(self): - template = """can't create or remove files in install directory + __cant_write_msg = textwrap.dedent(""" + can't create or remove files in install directory -The following error occurred while trying to add or remove files in the -installation directory: + The following error occurred while trying to add or remove files in the + installation directory: - %s + %s -The installation directory you specified (via --install-dir, --prefix, or -the distutils default setting) was: + The installation directory you specified (via --install-dir, --prefix, or + the distutils default setting) was: - %s -""" - msg = template % (sys.exc_info()[1], self.install_dir,) + %s + """).lstrip() - if not os.path.exists(self.install_dir): - msg += """ -This directory does not currently exist. Please create it and try again, or -choose a different installation directory (using the -d or --install-dir -option). -""" - else: - msg += """ -Perhaps your account does not have write access to this directory? If the -installation directory is a system-owned directory, you may need to sign in -as the administrator or "root" account. If you do not have administrative -access to this machine, you may wish to choose a different installation -directory, preferably one that is listed in your PYTHONPATH environment -variable. + __not_exists_id = textwrap.dedent(""" + This directory does not currently exist. Please create it and try again, or + choose a different installation directory (using the -d or --install-dir + option). + """).lstrip() -For information on other options, you may wish to consult the -documentation at: + __access_msg = textwrap.dedent(""" + Perhaps your account does not have write access to this directory? If the + installation directory is a system-owned directory, you may need to sign in + as the administrator or "root" account. If you do not have administrative + access to this machine, you may wish to choose a different installation + directory, preferably one that is listed in your PYTHONPATH environment + variable. - https://pythonhosted.org/setuptools/easy_install.html + For information on other options, you may wish to consult the + documentation at: -Please make the appropriate changes for your system and try again. -""" + https://pythonhosted.org/setuptools/easy_install.html + + Please make the appropriate changes for your system and try again. + """).lstrip() + + def cant_write_to_target(self): + msg = self.__cant_write_msg % (sys.exc_info()[1], self.install_dir,) + + if not os.path.exists(self.install_dir): + msg += '\n' + self.__not_exists_id + else: + msg += '\n' + self.__access_msg raise DistutilsError(msg) def check_pth_processing(self): @@ -699,17 +719,10 @@ Please make the appropriate changes for your system and try again. distros = WorkingSet([]).resolve( [requirement], self.local_index, self.easy_install ) - except DistributionNotFound: - e = sys.exc_info()[1] - raise DistutilsError( - "Could not find required distribution %s" % e.args - ) - except VersionConflict: - e = sys.exc_info()[1] - raise DistutilsError( - "Installed distribution %s conflicts with requirement %s" - % e.args - ) + except DistributionNotFound as e: + raise DistutilsError(str(e)) + except VersionConflict as e: + raise DistutilsError(e.report()) if self.always_copy or self.always_copy_from: # Force all the relevant distros to be copied or activated for dist in distros: @@ -749,9 +762,10 @@ Please make the appropriate changes for your system and try again. return dst def install_wrapper_scripts(self, dist): - if not self.exclude_scripts: - for args in get_script_args(dist): - self.write_script(*args) + if self.exclude_scripts: + return + for args in ScriptWriter.best().get_args(dist): + self.write_script(*args) def install_script(self, dist, script_name, script_text, dev_path=None): """Generate a legacy script wrapper and install it""" @@ -759,8 +773,8 @@ Please make the appropriate changes for your system and try again. is_script = is_python_script(script_text, script_name) if is_script: - script_text = (get_script_header(script_text) + - self._load_template(dev_path) % locals()) + body = self._load_template(dev_path) % locals() + script_text = ScriptWriter.get_header(script_text) + body self.write_script(script_name, _to_ascii(script_text), 'b') @staticmethod @@ -792,9 +806,8 @@ Please make the appropriate changes for your system and try again. ensure_directory(target) if os.path.exists(target): os.unlink(target) - f = open(target, "w" + mode) - f.write(contents) - f.close() + with open(target, "w" + mode) as f: + f.write(contents) chmod(target, 0o777 - mask) def install_eggs(self, spec, dist_filename, tmpdir): @@ -923,9 +936,10 @@ Please make the appropriate changes for your system and try again. f.write('%s: %s\n' % (k.replace('_', '-').title(), v)) f.close() script_dir = os.path.join(_egg_info, 'scripts') - self.delete_blockers( # delete entry-point scripts to avoid duping + # delete entry-point scripts to avoid duping + self.delete_blockers( [os.path.join(script_dir, args[0]) for args in - get_script_args(dist)] + ScriptWriter.get_args(dist)] ) # Build .egg file from tmpdir bdist_egg.make_zipfile( @@ -987,46 +1001,52 @@ Please make the appropriate changes for your system and try again. f.write('\n'.join(locals()[name]) + '\n') f.close() + __mv_warning = textwrap.dedent(""" + Because this distribution was installed --multi-version, before you can + import modules from this package in an application, you will need to + 'import pkg_resources' and then use a 'require()' call similar to one of + these examples, in order to select the desired version: + + pkg_resources.require("%(name)s") # latest installed version + pkg_resources.require("%(name)s==%(version)s") # this exact version + pkg_resources.require("%(name)s>=%(version)s") # this version or higher + """).lstrip() + + __id_warning = textwrap.dedent(""" + Note also that the installation directory must be on sys.path at runtime for + this to work. (e.g. by being the application's script directory, by being on + PYTHONPATH, or by being added to sys.path by your code.) + """) + def installation_report(self, req, dist, what="Installed"): """Helpful installation message for display to package users""" msg = "\n%(what)s %(eggloc)s%(extras)s" if self.multi_version and not self.no_report: - msg += """ - -Because this distribution was installed --multi-version, before you can -import modules from this package in an application, you will need to -'import pkg_resources' and then use a 'require()' call similar to one of -these examples, in order to select the desired version: - - pkg_resources.require("%(name)s") # latest installed version - pkg_resources.require("%(name)s==%(version)s") # this exact version - pkg_resources.require("%(name)s>=%(version)s") # this version or higher -""" + msg += '\n' + self.__mv_warning if self.install_dir not in map(normalize_path, sys.path): - msg += """ + msg += '\n' + self.__id_warning -Note also that the installation directory must be on sys.path at runtime for -this to work. (e.g. by being the application's script directory, by being on -PYTHONPATH, or by being added to sys.path by your code.) -""" eggloc = dist.location name = dist.project_name version = dist.version extras = '' # TODO: self.report_extras(req, dist) return msg % locals() - def report_editable(self, spec, setup_script): - dirname = os.path.dirname(setup_script) - python = sys.executable - return """\nExtracted editable version of %(spec)s to %(dirname)s + __editable_msg = textwrap.dedent(""" + Extracted editable version of %(spec)s to %(dirname)s -If it uses setuptools in its setup script, you can activate it in -"development" mode by going to that directory and running:: + If it uses setuptools in its setup script, you can activate it in + "development" mode by going to that directory and running:: - %(python)s setup.py develop + %(python)s setup.py develop -See the setuptools documentation for the "develop" command for more info. -""" % locals() + See the setuptools documentation for the "develop" command for more info. + """).lstrip() + + def report_editable(self, spec, setup_script): + dirname = os.path.dirname(setup_script) + python = sys.executable + return '\n' + self.__editable_msg % locals() def run_setup(self, setup_script, setup_base, args): sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg) @@ -1045,8 +1065,7 @@ See the setuptools documentation for the "develop" command for more info. ) try: run_setup(setup_script, args) - except SystemExit: - v = sys.exc_info()[1] + except SystemExit as v: raise DistutilsError("Setup script exited with %s" % (v.args[0],)) def build_and_install(self, setup_script, setup_base): @@ -1178,35 +1197,38 @@ See the setuptools documentation for the "develop" command for more info. finally: log.set_verbosity(self.verbose) # restore original verbosity - def no_default_version_msg(self): - template = """bad install directory or PYTHONPATH + __no_default_msg = textwrap.dedent(""" + bad install directory or PYTHONPATH + + You are attempting to install a package to a directory that is not + on PYTHONPATH and which Python does not read ".pth" files from. The + installation directory you specified (via --install-dir, --prefix, or + the distutils default setting) was: -You are attempting to install a package to a directory that is not -on PYTHONPATH and which Python does not read ".pth" files from. The -installation directory you specified (via --install-dir, --prefix, or -the distutils default setting) was: + %s - %s + and your PYTHONPATH environment variable currently contains: -and your PYTHONPATH environment variable currently contains: + %r - %r + Here are some of your options for correcting the problem: -Here are some of your options for correcting the problem: + * You can choose a different installation directory, i.e., one that is + on PYTHONPATH or supports .pth files -* You can choose a different installation directory, i.e., one that is - on PYTHONPATH or supports .pth files + * You can add the installation directory to the PYTHONPATH environment + variable. (It must then also be on PYTHONPATH whenever you run + Python and want to use the package(s) you are installing.) -* You can add the installation directory to the PYTHONPATH environment - variable. (It must then also be on PYTHONPATH whenever you run - Python and want to use the package(s) you are installing.) + * You can set up the installation directory to support ".pth" files by + using one of the approaches described here: -* You can set up the installation directory to support ".pth" files by - using one of the approaches described here: + https://pythonhosted.org/setuptools/easy_install.html#custom-installation-locations - https://pythonhosted.org/setuptools/easy_install.html#custom-installation-locations + Please make the appropriate changes for your system and try again.""").lstrip() -Please make the appropriate changes for your system and try again.""" + def no_default_version_msg(self): + template = self.__no_default_msg return template % (self.install_dir, os.environ.get('PYTHONPATH', '')) def install_site_py(self): @@ -1403,13 +1425,8 @@ def extract_wininst_cfg(dist_filename): {'version': '', 'target_version': ''}) try: part = f.read(cfglen) - # part is in bytes, but we need to read up to the first null - # byte. - if sys.version_info >= (2, 6): - null_byte = bytes([0]) - else: - null_byte = chr(0) - config = part.split(null_byte, 1)[0] + # Read up to the first null byte. + config = part.split(b'\0', 1)[0] # Now the config is in bytes, but for RawConfigParser, it should # be text, so decode it. config = config.decode(sys.getfilesystemencoding()) @@ -1521,23 +1538,16 @@ class PthDistributions(Environment): if not self.dirty: return - data = '\n'.join(map(self.make_relative, self.paths)) - if data: + rel_paths = list(map(self.make_relative, self.paths)) + if rel_paths: log.debug("Saving %s", self.filename) - data = ( - "import sys; sys.__plen = len(sys.path)\n" - "%s\n" - "import sys; new=sys.path[sys.__plen:];" - " del sys.path[sys.__plen:];" - " p=getattr(sys,'__egginsert',0); sys.path[p:p]=new;" - " sys.__egginsert = p+len(new)\n" - ) % data + lines = self._wrap_lines(rel_paths) + data = '\n'.join(lines) + '\n' if os.path.islink(self.filename): os.unlink(self.filename) - f = open(self.filename, 'wt') - f.write(data) - f.close() + with open(self.filename, 'wt') as f: + f.write(data) elif os.path.exists(self.filename): log.debug("Deleting empty %s", self.filename) @@ -1545,6 +1555,10 @@ class PthDistributions(Environment): self.dirty = False + @staticmethod + def _wrap_lines(lines): + return lines + def add(self, dist): """Add `dist` to the distribution map""" new_path = ( @@ -1582,6 +1596,34 @@ class PthDistributions(Environment): return path +class RewritePthDistributions(PthDistributions): + + @classmethod + def _wrap_lines(cls, lines): + yield cls.prelude + for line in lines: + yield line + yield cls.postlude + + _inline = lambda text: textwrap.dedent(text).strip().replace('\n', '; ') + prelude = _inline(""" + import sys + sys.__plen = len(sys.path) + """) + postlude = _inline(""" + import sys + new = sys.path[sys.__plen:] + del sys.path[sys.__plen:] + p = getattr(sys, '__egginsert', 0) + sys.path[p:p] = new + sys.__egginsert = p + len(new) + """) + + +if os.environ.get('SETUPTOOLS_SYS_PATH_TECHNIQUE', 'rewrite') == 'rewrite': + PthDistributions = RewritePthDistributions + + def _first_line_re(): """ Return a regular expression based on first_line_re suitable for matching @@ -1594,33 +1636,6 @@ def _first_line_re(): return re.compile(first_line_re.pattern.decode()) -def get_script_header(script_text, executable=sys_executable, wininst=False): - """Create a #! line, getting options (if any) from script_text""" - first = (script_text + '\n').splitlines()[0] - match = _first_line_re().match(first) - options = '' - if match: - options = match.group(1) or '' - if options: - options = ' ' + options - if wininst: - executable = "python.exe" - else: - executable = nt_quote_arg(executable) - hdr = "#!%(executable)s%(options)s\n" % locals() - if not isascii(hdr): - # Non-ascii path to sys.executable, use -x to prevent warnings - if options: - if options.strip().startswith('-'): - options = ' -x' + options.strip()[1:] - # else: punt, we can't do it, let the warning happen anyway - else: - options = ' -x' - executable = fix_jython_executable(executable, options) - hdr = "#!%(executable)s%(options)s\n" % locals() - return hdr - - def auto_chmod(func, arg, exc): if func is os.remove and os.name == 'nt': chmod(arg, stat.S_IWRITE) @@ -1819,9 +1834,8 @@ def is_python(text, filename='<string>'): def is_sh(executable): """Determine if the specified executable is a .sh (contains a #! line)""" try: - fp = open(executable) - magic = fp.read(2) - fp.close() + with io.open(executable, encoding='latin-1') as fp: + magic = fp.read(2) except (OSError, IOError): return executable return magic == '#!' @@ -1829,36 +1843,7 @@ def is_sh(executable): def nt_quote_arg(arg): """Quote a command line argument according to Windows parsing rules""" - - result = [] - needquote = False - nb = 0 - - needquote = (" " in arg) or ("\t" in arg) - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - nb += 1 - elif c == '"': - # double preceding backslashes, then add a \" - result.append('\\' * (nb * 2) + '\\"') - nb = 0 - else: - if nb: - result.append('\\' * nb) - nb = 0 - result.append(c) - - if nb: - result.append('\\' * nb) - - if needquote: - result.append('\\' * nb) # double the trailing backslashes - result.append('"') - - return ''.join(result) + return subprocess.list2cmdline([arg]) def is_python_script(script_text, filename): @@ -1887,31 +1872,130 @@ def chmod(path, mode): log.debug("changing mode of %s to %o", path, mode) try: _chmod(path, mode) - except os.error: - e = sys.exc_info()[1] + except os.error as e: log.debug("chmod failed: %s", e) def fix_jython_executable(executable, options): - if sys.platform.startswith('java') and is_sh(executable): - # Workaround for Jython is not needed on Linux systems. - import java + warnings.warn("Use JythonCommandSpec", DeprecationWarning, stacklevel=2) + + if not JythonCommandSpec.relevant(): + return executable + + cmd = CommandSpec.best().from_param(executable) + cmd.install_options(options) + return cmd.as_header().lstrip('#!').rstrip('\n') + + +class CommandSpec(list): + """ + A command spec for a #! header, specified as a list of arguments akin to + those passed to Popen. + """ + + options = [] + split_args = dict() + + @classmethod + def best(cls): + """ + Choose the best CommandSpec class based on environmental conditions. + """ + return cls if not JythonCommandSpec.relevant() else JythonCommandSpec - if java.lang.System.getProperty("os.name") == "Linux": - return executable + @classmethod + def _sys_executable(cls): + _default = os.path.normpath(sys.executable) + return os.environ.get('__PYVENV_LAUNCHER__', _default) + + @classmethod + def from_param(cls, param): + """ + Construct a CommandSpec from a parameter to build_scripts, which may + be None. + """ + if isinstance(param, cls): + return param + if isinstance(param, list): + return cls(param) + if param is None: + return cls.from_environment() + # otherwise, assume it's a string. + return cls.from_string(param) + + @classmethod + def from_environment(cls): + return cls([cls._sys_executable()]) + + @classmethod + def from_string(cls, string): + """ + Construct a command spec from a simple string representing a command + line parseable by shlex.split. + """ + items = shlex.split(string, **cls.split_args) + return cls(items) + + def install_options(self, script_text): + self.options = shlex.split(self._extract_options(script_text)) + cmdline = subprocess.list2cmdline(self) + if not isascii(cmdline): + self.options[:0] = ['-x'] + + @staticmethod + def _extract_options(orig_script): + """ + Extract any options from the first line of the script. + """ + first = (orig_script + '\n').splitlines()[0] + match = _first_line_re().match(first) + options = match.group(1) or '' if match else '' + return options.strip() + + def as_header(self): + return self._render(self + list(self.options)) + + @staticmethod + def _render(items): + cmdline = subprocess.list2cmdline(items) + return '#!' + cmdline + '\n' - # Workaround Jython's sys.executable being a .sh (an invalid - # shebang line interpreter) - if options: +# For pbr compat; will be removed in a future version. +sys_executable = CommandSpec._sys_executable() + + +class WindowsCommandSpec(CommandSpec): + split_args = dict(posix=False) + + +class JythonCommandSpec(CommandSpec): + @classmethod + def relevant(cls): + return ( + sys.platform.startswith('java') + and + __import__('java').lang.System.getProperty('os.name') != 'Linux' + ) + + def as_header(self): + """ + Workaround Jython's sys.executable being a .sh (an invalid + shebang line interpreter) + """ + if not is_sh(self[0]): + return super(JythonCommandSpec, self).as_header() + + if self.options: # Can't apply the workaround, leave it broken log.warn( "WARNING: Unable to adapt shebang line for Jython," " the following script is NOT executable\n" " see http://bugs.jython.org/issue1112 for" " more information.") - else: - return '/usr/bin/env %s' % executable - return executable + return super(JythonCommandSpec, self).as_header() + + items = ['/usr/bin/env'] + self + list(self.options) + return self._render(items) class ScriptWriter(object): @@ -1932,39 +2016,92 @@ class ScriptWriter(object): ) """).lstrip() + command_spec_class = CommandSpec + + @classmethod + def get_script_args(cls, dist, executable=None, wininst=False): + # for backward compatibility + warnings.warn("Use get_args", DeprecationWarning) + writer = (WindowsScriptWriter if wininst else ScriptWriter).best() + header = cls.get_script_header("", executable, wininst) + return writer.get_args(dist, header) + @classmethod - def get_script_args(cls, dist, executable=sys_executable, wininst=False): + def get_script_header(cls, script_text, executable=None, wininst=False): + # for backward compatibility + warnings.warn("Use get_header", DeprecationWarning) + if wininst: + executable = "python.exe" + cmd = cls.command_spec_class.best().from_param(executable) + cmd.install_options(script_text) + return cmd.as_header() + + @classmethod + def get_args(cls, dist, header=None): """ - Yield write_script() argument tuples for a distribution's entrypoints + Yield write_script() argument tuples for a distribution's + console_scripts and gui_scripts entry points. """ - gen_class = cls.get_writer(wininst) + if header is None: + header = cls.get_header() spec = str(dist.as_requirement()) - header = get_script_header("", executable, wininst) for type_ in 'console', 'gui': group = type_ + '_scripts' for name, ep in dist.get_entry_map(group).items(): - script_text = gen_class.template % locals() - for res in gen_class._get_script_args(type_, name, header, - script_text): + cls._ensure_safe_name(name) + script_text = cls.template % locals() + args = cls._get_script_args(type_, name, header, script_text) + for res in args: yield res + @staticmethod + def _ensure_safe_name(name): + """ + Prevent paths in *_scripts entry point names. + """ + has_path_sep = re.search(r'[\\/]', name) + if has_path_sep: + raise ValueError("Path separators not allowed in script names") + @classmethod def get_writer(cls, force_windows): - if force_windows or sys.platform == 'win32': - return WindowsScriptWriter.get_writer() - return cls + # for backward compatibility + warnings.warn("Use best", DeprecationWarning) + return WindowsScriptWriter.best() if force_windows else cls.best() + + @classmethod + def best(cls): + """ + Select the best ScriptWriter for this environment. + """ + return WindowsScriptWriter.best() if sys.platform == 'win32' else cls @classmethod def _get_script_args(cls, type_, name, header, script_text): # Simply write the stub with no extension. yield (name, header + script_text) + @classmethod + def get_header(cls, script_text="", executable=None): + """Create a #! line, getting options (if any) from script_text""" + cmd = cls.command_spec_class.best().from_param(executable) + cmd.install_options(script_text) + return cmd.as_header() + class WindowsScriptWriter(ScriptWriter): + command_spec_class = WindowsCommandSpec + @classmethod def get_writer(cls): + # for backward compatibility + warnings.warn("Use best", DeprecationWarning) + return cls.best() + + @classmethod + def best(cls): """ - Get a script writer suitable for Windows + Select the best ScriptWriter suitable for Windows """ writer_lookup = dict( executable=WindowsExecutableLauncherWriter, @@ -1987,8 +2124,8 @@ class WindowsScriptWriter(ScriptWriter): blockers = [name + x for x in old] yield name + ext, header + script_text, 't', blockers - @staticmethod - def _adjust_header(type_, orig_header): + @classmethod + def _adjust_header(cls, type_, orig_header): """ Make sure 'pythonw' is used for gui and and 'python' is used for console (regardless of what sys.executable is). @@ -1999,11 +2136,19 @@ class WindowsScriptWriter(ScriptWriter): pattern, repl = repl, pattern pattern_ob = re.compile(re.escape(pattern), re.IGNORECASE) new_header = pattern_ob.sub(string=orig_header, repl=repl) + return new_header if cls._use_header(new_header) else orig_header + + @staticmethod + def _use_header(new_header): + """ + Should _adjust_header use the replaced header? + + On non-windows systems, always use. On + Windows systems, only use the replaced header if it resolves + to an executable on the system. + """ clean_header = new_header[2:-1].strip('"') - if sys.platform == 'win32' and not os.path.exists(clean_header): - # the adjusted version doesn't exist, so return the original - return orig_header - return new_header + return sys.platform != 'win32' or find_executable(clean_header) class WindowsExecutableLauncherWriter(WindowsScriptWriter): @@ -2039,6 +2184,7 @@ class WindowsExecutableLauncherWriter(WindowsScriptWriter): # for backward-compatibility get_script_args = ScriptWriter.get_script_args +get_script_header = ScriptWriter.get_script_header def get_win_launcher(type): @@ -2160,4 +2306,3 @@ def _patch_usage(): yield finally: distutils.core.gen_usage = saved - diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 3f1db996..19849e66 100755 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -10,17 +10,17 @@ import distutils.filelist import os import re import sys +import io +import warnings +import time import six -try: - from setuptools_svn import svn_utils -except ImportError: - pass - from setuptools import Command from setuptools.command.sdist import sdist from setuptools.command.sdist import walk_revctrl +from setuptools.command.setopt import edit_config +from setuptools.command import bdist_egg from pkg_resources import ( parse_requirements, safe_name, parse_version, safe_version, yield_lines, EntryPoint, iter_entry_points, to_filename) @@ -28,6 +28,12 @@ import setuptools.unicode_utils as unicode_utils from pkg_resources import packaging +try: + from setuptools_svn import svn_utils +except ImportError: + pass + + class egg_info(Command): description = "create a distribution's .egg-info directory" @@ -59,8 +65,6 @@ class egg_info(Command): self.vtags = None def save_version_info(self, filename): - from setuptools.command.setopt import edit_config - values = dict( egg_info=dict( tag_svn_revision=0, @@ -169,7 +173,8 @@ class egg_info(Command): self.mkpath(self.egg_info) installer = self.distribution.fetch_build_egg for ep in iter_entry_points('egg_info.writers'): - writer = ep.load(installer=installer) + ep.require(installer=installer) + writer = ep.resolve() writer(self, ep.name, os.path.join(self.egg_info, ep.name)) # Get rid of native_libs.txt if it was put there by older bdist_egg @@ -184,12 +189,8 @@ class egg_info(Command): if self.tag_build: version += self.tag_build if self.tag_svn_revision: - rev = self.get_svn_revision() - if rev: # is 0 if it's not an svn working copy - version += '-r%s' % rev + version += '-r%s' % self.get_svn_revision() if self.tag_date: - import time - version += time.strftime("-%Y%m%d") return version @@ -390,7 +391,6 @@ def write_pkg_info(cmd, basename, filename): metadata.name, metadata.version = oldname, oldver safe = getattr(cmd.distribution, 'zip_safe', None) - from setuptools.command import bdist_egg bdist_egg.write_safety_flag(cmd.egg_info, safe) @@ -467,14 +467,15 @@ def write_entries(cmd, basename, filename): def get_pkg_info_revision(): - # See if we can get a -r### off of PKG-INFO, in case this is an sdist of - # a subversion revision - # + """ + Get a -r### off of PKG-INFO Version in case this is an sdist of + a subversion revision. + """ + warnings.warn("get_pkg_info_revision is deprecated.", DeprecationWarning) if os.path.exists('PKG-INFO'): - f = open('PKG-INFO', 'rU') - for line in f: - match = re.match(r"Version:.*-r(\d+)\s*$", line) - if match: - return int(match.group(1)) - f.close() + with io.open('PKG-INFO') as f: + for line in f: + match = re.match(r"Version:.*-r(\d+)\s*$", line) + if match: + return int(match.group(1)) return 0 diff --git a/setuptools/command/install_lib.py b/setuptools/command/install_lib.py index 9b772227..78fe6891 100644 --- a/setuptools/command/install_lib.py +++ b/setuptools/command/install_lib.py @@ -79,6 +79,8 @@ class install_lib(orig.install_lib): base = os.path.join('__pycache__', '__init__.' + imp.get_tag()) yield base + '.pyc' yield base + '.pyo' + yield base + '.opt-1.pyc' + yield base + '.opt-2.pyc' def copy_tree( self, infile, outfile, diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py index eb79fa3c..be66cb22 100755 --- a/setuptools/command/install_scripts.py +++ b/setuptools/command/install_scripts.py @@ -13,8 +13,7 @@ class install_scripts(orig.install_scripts): self.no_ep = False def run(self): - from setuptools.command.easy_install import get_script_args - from setuptools.command.easy_install import sys_executable + import setuptools.command.easy_install as ei self.run_command("egg_info") if self.distribution.scripts: @@ -31,11 +30,17 @@ class install_scripts(orig.install_scripts): ei_cmd.egg_name, ei_cmd.egg_version, ) bs_cmd = self.get_finalized_command('build_scripts') - executable = getattr(bs_cmd, 'executable', sys_executable) - is_wininst = getattr( - self.get_finalized_command("bdist_wininst"), '_is_running', False - ) - for args in get_script_args(dist, executable, is_wininst): + exec_param = getattr(bs_cmd, 'executable', None) + bw_cmd = self.get_finalized_command("bdist_wininst") + is_wininst = getattr(bw_cmd, '_is_running', False) + writer = ei.ScriptWriter + if is_wininst: + exec_param = "python.exe" + writer = ei.WindowsScriptWriter + # resolve the writer to the environment + writer = writer.best() + cmd = writer.command_spec_class.best().from_param(exec_param) + for args in writer.get_args(dist, cmd.as_header()): self.write_script(*args) def write_script(self, script_name, contents, mode="t", *ignored): diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 4ec7ec91..3b9f7dd5 100755 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -3,6 +3,7 @@ from distutils import log import distutils.command.sdist as orig import os import sys +import io import six @@ -71,7 +72,8 @@ class sdist(orig.sdist): try: orig.sdist.read_template(self) except: - sys.exc_info()[2].tb_next.tb_frame.f_locals['template'].close() + _, _, tb = sys.exc_info() + tb.tb_next.tb_frame.f_locals['template'].close() raise # Beginning with Python 2.7.2, 3.1.4, and 3.2.1, this leaky file handle @@ -166,11 +168,8 @@ class sdist(orig.sdist): if not os.path.isfile(self.manifest): return False - fp = open(self.manifest, 'rbU') - try: + with io.open(self.manifest, 'rb') as fp: first_line = fp.readline() - finally: - fp.close() return (first_line != '# file GENERATED by distutils, do NOT edit\n'.encode()) diff --git a/setuptools/command/test.py b/setuptools/command/test.py index c5644530..5f2e2299 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -1,6 +1,5 @@ from distutils.errors import DistutilsOptionError from unittest import TestLoader -import unittest import sys import six @@ -13,7 +12,7 @@ from setuptools.py31compat import unittest_main class ScanningLoader(TestLoader): - def loadTestsFromModule(self, module): + def loadTestsFromModule(self, module, pattern=None): """Return a suite of all tests cases contained in the given module If the module is a package, load tests from all the modules in it. @@ -43,6 +42,17 @@ class ScanningLoader(TestLoader): return tests[0] # don't create a nested suite for only one return +# adapted from jaraco.classes.properties:NonDataProperty +class NonDataProperty(object): + def __init__(self, fget): + self.fget = fget + + def __get__(self, obj, objtype=None): + if obj is None: + return self + return self.fget(obj) + + class test(Command): """Command to run unit tests after in-place build""" @@ -63,20 +73,16 @@ class test(Command): def finalize_options(self): + if self.test_suite and self.test_module: + msg = "You may specify a module or a suite, but not both" + raise DistutilsOptionError(msg) + if self.test_suite is None: if self.test_module is None: self.test_suite = self.distribution.test_suite else: self.test_suite = self.test_module + ".test_suite" - elif self.test_module: - raise DistutilsOptionError( - "You may specify a module or a suite, but not both" - ) - self.test_args = [self.test_suite] - - if self.verbose: - self.test_args.insert(0, '--verbose') if self.test_loader is None: self.test_loader = getattr(self.distribution, 'test_loader', None) if self.test_loader is None: @@ -84,6 +90,16 @@ class test(Command): if self.test_runner is None: self.test_runner = getattr(self.distribution, 'test_runner', None) + @NonDataProperty + def test_args(self): + return list(self._test_args()) + + def _test_args(self): + if self.verbose: + yield '--verbose' + if self.test_suite: + yield self.test_suite + def with_project_on_sys_path(self, func): with_2to3 = six.PY3 and getattr(self.distribution, 'use_2to3', False) @@ -134,20 +150,19 @@ class test(Command): if self.distribution.tests_require: self.distribution.fetch_build_eggs(self.distribution.tests_require) - if self.test_suite: - cmd = ' '.join(self.test_args) - if self.dry_run: - self.announce('skipping "unittest %s" (dry run)' % cmd) - else: - self.announce('running "unittest %s"' % cmd) - self.with_project_on_sys_path(self.run_tests) + cmd = ' '.join(self._argv) + if self.dry_run: + self.announce('skipping "%s" (dry run)' % cmd) + else: + self.announce('running "%s"' % cmd) + self.with_project_on_sys_path(self.run_tests) def run_tests(self): # Purge modules under test from sys.modules. The test loader will # re-import them from the build location. Required when 2to3 is used # with namespace packages. if six.PY3 and getattr(self.distribution, 'use_2to3', False): - module = self.test_args[-1].split('.')[0] + module = self.test_suite.split('.')[0] if module in _namespace_packages: del_modules = [] if module in sys.modules: @@ -159,11 +174,15 @@ class test(Command): list(map(sys.modules.__delitem__, del_modules)) unittest_main( - None, None, [unittest.__file__] + self.test_args, + None, None, self._argv, testLoader=self._resolve_as_ep(self.test_loader), testRunner=self._resolve_as_ep(self.test_runner), ) + @property + def _argv(self): + return ['unittest'] + self.test_args + @staticmethod def _resolve_as_ep(val): """ @@ -173,4 +192,4 @@ class test(Command): if val is None: return parsed = EntryPoint.parse("x=" + val) - return parsed._load()() + return parsed.resolve()() diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 360c10e8..43b5d76a 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -171,8 +171,7 @@ class upload_docs(upload): conn.putheader('Authorization', auth) conn.endheaders() conn.send(body) - except socket.error: - e = sys.exc_info()[1] + except socket.error as e: self.announce(str(e), log.ERROR) return diff --git a/setuptools/dist.py b/setuptools/dist.py index cdc15e46..7335c967 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -116,24 +116,26 @@ def check_extras(dist, attr, value): def assert_bool(dist, attr, value): """Verify that value is True, False, 0, or 1""" if bool(value) != value: - raise DistutilsSetupError( - "%r must be a boolean value (got %r)" % (attr,value) - ) + tmpl = "{attr!r} must be a boolean value (got {value!r})" + raise DistutilsSetupError(tmpl.format(attr=attr, value=value)) + + def check_requirements(dist, attr, value): """Verify that install_requires is a valid requirements list""" try: list(pkg_resources.parse_requirements(value)) - except (TypeError,ValueError): - raise DistutilsSetupError( - "%r must be a string or list of strings " - "containing valid project/version requirement specifiers" % (attr,) + except (TypeError, ValueError) as error: + tmpl = ( + "{attr!r} must be a string or list of strings " + "containing valid project/version requirement specifiers; {error}" ) + raise DistutilsSetupError(tmpl.format(attr=attr, error=error)) + def check_entry_points(dist, attr, value): """Verify that entry_points map is parseable""" try: pkg_resources.EntryPoint.parse_map(value) - except ValueError: - e = sys.exc_info()[1] + except ValueError as e: raise DistutilsSetupError(e) def check_test_suite(dist, attr, value): @@ -159,7 +161,7 @@ def check_packages(dist, attr, value): for pkgname in value: if not re.match(r'\w+(\.\w+)*', pkgname): distutils.log.warn( - "WARNING: %r not a valid package name; please use only" + "WARNING: %r not a valid package name; please use only " ".-separated package names in setup.py", pkgname ) @@ -266,8 +268,7 @@ class Distribution(_Distribution): if attrs and 'setup_requires' in attrs: self.fetch_build_eggs(attrs['setup_requires']) for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): - if not hasattr(self,ep.name): - setattr(self,ep.name,None) + vars(self).setdefault(ep.name, None) _Distribution.__init__(self,attrs) if isinstance(self.metadata.version, numbers.Number): # Some people apparently take "version number" too literally :) @@ -279,10 +280,9 @@ class Distribution(_Distribution): normalized_version = str(ver) if self.metadata.version != normalized_version: warnings.warn( - "The version specified requires normalization, " - "consider using '%s' instead of '%s'." % ( - normalized_version, + "Normalizing '%s' to '%s'" % ( self.metadata.version, + normalized_version, ) ) self.metadata.version = normalized_version @@ -436,10 +436,18 @@ class Distribution(_Distribution): for ep in pkg_resources.iter_entry_points('distutils.commands'): if ep.name not in self.cmdclass: # don't require extras as the commands won't be invoked - cmdclass = ep._load() + cmdclass = ep.resolve() self.cmdclass[ep.name] = cmdclass return _Distribution.print_commands(self) + def get_command_list(self): + for ep in pkg_resources.iter_entry_points('distutils.commands'): + if ep.name not in self.cmdclass: + # don't require extras as the commands won't be invoked + cmdclass = ep.resolve() + self.cmdclass[ep.name] = cmdclass + return _Distribution.get_command_list(self) + def _set_feature(self,name,status): """Set feature's inclusion status""" setattr(self,self._feature_attrname(name),status) @@ -818,7 +826,7 @@ class Feature: if not self.available: raise DistutilsPlatformError( - self.description+" is required," + self.description+" is required, " "but is not available on this platform" ) diff --git a/setuptools/extension.py b/setuptools/extension.py index 8178ed33..35eb7c7c 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -12,35 +12,33 @@ _Extension = _get_unpatched(distutils.core.Extension) msvc9_support.patch_for_specialized_compiler() -def have_pyrex(): +def _have_cython(): """ - Return True if Cython or Pyrex can be imported. + Return True if Cython can be imported. """ - pyrex_impls = 'Cython.Distutils.build_ext', 'Pyrex.Distutils.build_ext' - for pyrex_impl in pyrex_impls: - try: - # from (pyrex_impl) import build_ext - __import__(pyrex_impl, fromlist=['build_ext']).build_ext - return True - except Exception: - pass + cython_impl = 'Cython.Distutils.build_ext', + try: + # from (cython_impl) import build_ext + __import__(cython_impl, fromlist=['build_ext']).build_ext + return True + except Exception: + pass return False +# for compatibility +have_pyrex = _have_cython + class Extension(_Extension): """Extension that uses '.c' files in place of '.pyx' files""" - def __init__(self, *args, **kw): - _Extension.__init__(self, *args, **kw) - self._convert_pyx_sources_to_lang() - def _convert_pyx_sources_to_lang(self): """ Replace sources with .pyx extensions to sources with the target language extension. This mechanism allows language authors to supply pre-converted sources but to prefer the .pyx sources. """ - if have_pyrex(): + if _have_cython(): # the build has Cython, so allow it to compile the .pyx files return lang = self.language or '' diff --git a/setuptools/msvc9_support.py b/setuptools/msvc9_support.py index d0be70e2..a69c7474 100644 --- a/setuptools/msvc9_support.py +++ b/setuptools/msvc9_support.py @@ -1,5 +1,3 @@ -import sys - try: import distutils.msvc9compiler except ImportError: @@ -29,13 +27,15 @@ def patch_for_specialized_compiler(): def find_vcvarsall(version): Reg = distutils.msvc9compiler.Reg VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' + key = VC_BASE % ('', version) try: # Per-user installs register the compiler path here - productdir = Reg.get_value(VC_BASE % ('', version), "installdir") + productdir = Reg.get_value(key, "installdir") except KeyError: try: # All-user installs on a 64-bit system register here - productdir = Reg.get_value(VC_BASE % ('Wow6432Node\\', version), "installdir") + key = VC_BASE % ('Wow6432Node\\', version) + productdir = Reg.get_value(key, "installdir") except KeyError: productdir = None @@ -50,8 +50,7 @@ def find_vcvarsall(version): def query_vcvarsall(version, *args, **kwargs): try: return unpatched['query_vcvarsall'](version, *args, **kwargs) - except distutils.errors.DistutilsPlatformError: - exc = sys.exc_info()[1] + except distutils.errors.DistutilsPlatformError as exc: if exc and "vcvarsall.bat" in exc.args[0]: message = 'Microsoft Visual C++ %0.1f is required (%s).' % (version, exc.args[0]) if int(version) == 9: diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a14c8ac6..657b467f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -6,6 +6,7 @@ import shutil import socket import base64 import hashlib +import itertools from functools import wraps try: @@ -14,7 +15,7 @@ except ImportError: from urllib2 import splituser import six -from six.moves import urllib, http_client +from six.moves import urllib, http_client, configparser from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, @@ -141,10 +142,9 @@ def interpret_distro_name( # versions in distribution archive names (sdist and bdist). parts = basename.split('-') - if not py_version: - for i,p in enumerate(parts[2:]): - if len(p)==5 and p.startswith('py2.'): - return # It's a bdist_dumb, not an sdist -- bail out + if not py_version and any(re.match('py\d\.\d$', p) for p in parts[2:]): + # it is a bdist_dumb, not an sdist -- bail out + return for p in range(1,len(parts)+1): yield Distribution( @@ -356,20 +356,30 @@ class PackageIndex(Environment): self.warn(msg, url) def scan_egg_links(self, search_path): - for item in search_path: - if os.path.isdir(item): - for entry in os.listdir(item): - if entry.endswith('.egg-link'): - self.scan_egg_link(item, entry) + dirs = filter(os.path.isdir, search_path) + egg_links = ( + (path, entry) + for path in dirs + for entry in os.listdir(path) + if entry.endswith('.egg-link') + ) + list(itertools.starmap(self.scan_egg_link, egg_links)) def scan_egg_link(self, path, entry): - lines = [_f for _f in map(str.strip, - open(os.path.join(path, entry))) if _f] - if len(lines)==2: - for dist in find_distributions(os.path.join(path, lines[0])): - dist.location = os.path.join(path, *lines) - dist.precedence = SOURCE_DIST - self.add(dist) + with open(os.path.join(path, entry)) as raw_lines: + # filter non-empty lines + lines = list(filter(None, map(str.strip, raw_lines))) + + if len(lines) != 2: + # format is not recognized; punt + return + + egg_path, setup_path = lines + + for dist in find_distributions(os.path.join(path, egg_path)): + dist.location = os.path.join(path, *lines) + dist.precedence = SOURCE_DIST + self.add(dist) def process_index(self,url,page): """Process the contents of a PyPI page""" @@ -702,25 +712,21 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url, self.opener) - except (ValueError, http_client.InvalidURL): - v = sys.exc_info()[1] + except (ValueError, http_client.InvalidURL) as v: msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib.error.HTTPError: - v = sys.exc_info()[1] + except urllib.error.HTTPError as v: return v - except urllib.error.URLError: - v = sys.exc_info()[1] + except urllib.error.URLError as v: if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except http_client.BadStatusLine: - v = sys.exc_info()[1] + except http_client.BadStatusLine as v: if warning: self.warn(warning, v.line) else: @@ -729,8 +735,7 @@ class PackageIndex(Environment): 'down, %s' % (url, v.line) ) - except http_client.HTTPException: - v = sys.exc_info()[1] + except http_client.HTTPException as v: if warning: self.warn(warning, v) else: @@ -944,14 +949,14 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) -class PyPIConfig(six.moves.configparser.ConfigParser): +class PyPIConfig(configparser.RawConfigParser): def __init__(self): """ Load from ~/.pypirc """ defaults = dict.fromkeys(['username', 'password', 'repository'], '') - six.moves.configparser.ConfigParser.__init__(self, defaults) + configparser.RawConfigParser.__init__(self, defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): @@ -1043,16 +1048,18 @@ def local_open(url): elif path.endswith('/') and os.path.isdir(filename): files = [] for f in os.listdir(filename): - if f=='index.html': - with open(os.path.join(filename,f),'r') as fp: + filepath = os.path.join(filename, f) + if f == 'index.html': + with open(filepath, 'r') as fp: body = fp.read() break - elif os.path.isdir(os.path.join(filename,f)): - f+='/' - files.append("<a href=%r>%s</a>" % (f,f)) + elif os.path.isdir(filepath): + f += '/' + files.append('<a href="{name}">{name}</a>'.format(name=f)) else: - body = ("<html><head><title>%s</title>" % url) + \ - "</head><body>%s</body></html>" % '\n'.join(files) + tmpl = ("<html><head><title>{url}</title>" + "</head><body>{files}</body></html>") + body = tmpl.format(url=url, files='\n'.join(files)) status, message = 200, "OK" else: status, message, body = 404, "Path not found", "Not found" diff --git a/setuptools/py31compat.py b/setuptools/py31compat.py index c487ac04..8fe6dd9d 100644 --- a/setuptools/py31compat.py +++ b/setuptools/py31compat.py @@ -20,7 +20,7 @@ except ImportError: import shutil import tempfile class TemporaryDirectory(object): - """" + """ Very simple temporary directory context manager. Will try to delete afterward, but will also ignore OS and similar errors on deletion. diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index f99532f6..43b84791 100755 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -13,7 +13,7 @@ from six.moves import builtins import pkg_resources -if os.name == "java": +if sys.platform.startswith('java'): import org.python.modules.posix.PosixModule as _os else: _os = sys.modules[os.name] @@ -34,12 +34,12 @@ def _execfile(filename, globals, locals=None): Python 3 implementation of execfile. """ mode = 'rb' - # Python 2.6 compile requires LF for newlines, so use deprecated - # Universal newlines support. - if sys.version_info < (2, 7): - mode += 'U' with open(filename, mode) as stream: script = stream.read() + # compile() function in Python 2.6 and 3.1 requires LF line endings. + if sys.version_info[:2] < (2, 7) or sys.version_info[:2] >= (3, 0) and sys.version_info[:2] < (3, 2): + script = script.replace(b'\r\n', b'\n') + script = script.replace(b'\r', b'\n') if locals is None: locals = globals code = compile(script, filename, 'exec') @@ -47,8 +47,10 @@ def _execfile(filename, globals, locals=None): @contextlib.contextmanager -def save_argv(): +def save_argv(repl=None): saved = sys.argv[:] + if repl is not None: + sys.argv[:] = repl try: yield saved finally: @@ -92,6 +94,53 @@ def pushd(target): os.chdir(saved) +class UnpickleableException(Exception): + """ + An exception representing another Exception that could not be pickled. + """ + @staticmethod + def dump(type, exc): + """ + Always return a dumped (pickled) type and exc. If exc can't be pickled, + wrap it in UnpickleableException first. + """ + try: + return pickle.dumps(type), pickle.dumps(exc) + except Exception: + # get UnpickleableException inside the sandbox + from setuptools.sandbox import UnpickleableException as cls + return cls.dump(cls, cls(repr(exc))) + + +class ExceptionSaver: + """ + A Context Manager that will save an exception, serialized, and restore it + later. + """ + def __enter__(self): + return self + + def __exit__(self, type, exc, tb): + if not exc: + return + + # dump the exception + self._saved = UnpickleableException.dump(type, exc) + self._tb = tb + + # suppress the exception + return True + + def resume(self): + "restore and re-raise any exception" + + if '_saved' not in vars(self): + return + + type, exc = map(pickle.loads, self._saved) + six.reraise(type, exc, self._tb) + + @contextlib.contextmanager def save_modules(): """ @@ -101,31 +150,20 @@ def save_modules(): outside the context. """ saved = sys.modules.copy() - try: - try: - yield saved - except: - # dump any exception - class_, exc, tb = sys.exc_info() - saved_cls = pickle.dumps(class_) - saved_exc = pickle.dumps(exc) - raise - finally: - sys.modules.update(saved) - # remove any modules imported since - del_modules = ( - mod_name for mod_name in sys.modules - if mod_name not in saved - # exclude any encodings modules. See #285 - and not mod_name.startswith('encodings.') - ) - _clear_modules(del_modules) - except: - # reload and re-raise any exception, using restored modules - class_, exc, tb = sys.exc_info() - new_cls = pickle.loads(saved_cls) - new_exc = pickle.loads(saved_exc) - six.reraise(new_cls, new_exc, tb) + with ExceptionSaver() as saved_exc: + yield saved + + sys.modules.update(saved) + # remove any modules imported since + del_modules = ( + mod_name for mod_name in sys.modules + if mod_name not in saved + # exclude any encodings modules. See #285 + and not mod_name.startswith('encodings.') + ) + _clear_modules(del_modules) + + saved_exc.resume() def _clear_modules(module_names): @@ -199,8 +237,7 @@ def run_setup(setup_script, args): ns = dict(__file__=setup_script, __name__='__main__') _execfile(setup_script, ns) DirectorySandbox(setup_dir).run(runner) - except SystemExit: - v = sys.exc_info()[1] + except SystemExit as v: if v.args and v.args[0]: raise # Normal exit, just return @@ -347,6 +384,7 @@ class DirectorySandbox(AbstractSandbox): AbstractSandbox.__init__(self) def _violation(self, operation, *args, **kw): + from setuptools.sandbox import SandboxViolation raise SandboxViolation(operation, args, kw) if _file: diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py index c618ea7c..8fd7836b 100644 --- a/setuptools/ssl_support.py +++ b/setuptools/ssl_support.py @@ -218,6 +218,12 @@ def get_win_certfile(): self.addcerts(certs) atexit.register(self.close) + def close(self): + try: + super(MyCertFile, self).close() + except OSError: + pass + _wincerts = MyCertFile(stores=['CA', 'ROOT']) return _wincerts.name diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index 8cde6f60..b2c6894f 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -16,6 +16,11 @@ import setuptools.depends as dep from setuptools import Feature from setuptools.depends import Require +c_type = os.environ.get("LC_CTYPE", os.environ.get("LC_ALL")) +is_ascii = c_type in ("C", "POSIX") +fail_on_ascii = pytest.mark.xfail(is_ascii, reason="Test fails in this locale") + + def makeSetup(**args): """Return distribution from 'setup(**args)', without executing commands""" diff --git a/setuptools/tests/contexts.py b/setuptools/tests/contexts.py index fabab071..d9dcad84 100644 --- a/setuptools/tests/contexts.py +++ b/setuptools/tests/contexts.py @@ -27,7 +27,7 @@ def environment(**replacements): to clear the values. """ saved = dict( - (key, os.environ['key']) + (key, os.environ[key]) for key in replacements if key in os.environ ) @@ -49,14 +49,6 @@ def environment(**replacements): @contextlib.contextmanager -def argv(repl): - old_argv = sys.argv[:] - sys.argv[:] = repl - yield - sys.argv[:] = old_argv - - -@contextlib.contextmanager def quiet(): """ Redirect stdout/stderr to StringIO objects to prevent console output from diff --git a/setuptools/tests/files.py b/setuptools/tests/files.py new file mode 100644 index 00000000..4364241b --- /dev/null +++ b/setuptools/tests/files.py @@ -0,0 +1,32 @@ +import os + + +def build_files(file_defs, prefix=""): + """ + Build a set of files/directories, as described by the file_defs dictionary. + + Each key/value pair in the dictionary is interpreted as a filename/contents + pair. If the contents value is a dictionary, a directory is created, and the + dictionary interpreted as the files within it, recursively. + + For example: + + {"README.txt": "A README file", + "foo": { + "__init__.py": "", + "bar": { + "__init__.py": "", + }, + "baz.py": "# Some code", + } + } + """ + for name, contents in file_defs.items(): + full_name = os.path.join(prefix, name) + if isinstance(contents, dict): + if not os.path.exists(full_name): + os.makedirs(full_name) + build_files(contents, prefix=full_name) + else: + with open(full_name, 'w') as f: + f.write(contents) diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index 0b1eaf5f..c70c38cb 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -1,4 +1,7 @@ -import mock +try: + from unittest import mock +except ImportError: + import mock import pytest from . import contexts diff --git a/setuptools/tests/py26compat.py b/setuptools/tests/py26compat.py index c53b4809..c5680881 100644 --- a/setuptools/tests/py26compat.py +++ b/setuptools/tests/py26compat.py @@ -8,4 +8,7 @@ def _tarfile_open_ex(*args, **kwargs): """ return contextlib.closing(tarfile.open(*args, **kwargs)) -tarfile_open = _tarfile_open_ex if sys.version_info < (2,7) else tarfile.open +if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 2): + tarfile_open = _tarfile_open_ex +else: + tarfile_open = tarfile.open diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index ed1b194a..236b3aa6 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -1,13 +1,18 @@ """develop tests """ import os -import shutil import site import sys -import tempfile +import io + +import six + +import pytest from setuptools.command.develop import develop from setuptools.dist import Distribution +from . import contexts + SETUP_PY = """\ from setuptools import setup @@ -21,65 +26,52 @@ setup(name='foo', INIT_PY = """print "foo" """ -class TestDevelopTest: +@pytest.yield_fixture +def temp_user(monkeypatch): + with contexts.tempdir() as user_base: + with contexts.tempdir() as user_site: + monkeypatch.setattr('site.USER_BASE', user_base) + monkeypatch.setattr('site.USER_SITE', user_site) + yield - def setup_method(self, method): - if hasattr(sys, 'real_prefix'): - return - # Directory structure - self.dir = tempfile.mkdtemp() - os.mkdir(os.path.join(self.dir, 'foo')) - # setup.py - setup = os.path.join(self.dir, 'setup.py') - f = open(setup, 'w') +@pytest.yield_fixture +def test_env(tmpdir, temp_user): + target = tmpdir + foo = target.mkdir('foo') + setup = target / 'setup.py' + if setup.isfile(): + raise ValueError(dir(target)) + with setup.open('w') as f: f.write(SETUP_PY) - f.close() - self.old_cwd = os.getcwd() - # foo/__init__.py - init = os.path.join(self.dir, 'foo', '__init__.py') - f = open(init, 'w') + init = foo / '__init__.py' + with init.open('w') as f: f.write(INIT_PY) - f.close() - - os.chdir(self.dir) - self.old_base = site.USER_BASE - site.USER_BASE = tempfile.mkdtemp() - self.old_site = site.USER_SITE - site.USER_SITE = tempfile.mkdtemp() - - def teardown_method(self, method): - if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix): - return - - os.chdir(self.old_cwd) - shutil.rmtree(self.dir) - shutil.rmtree(site.USER_BASE) - shutil.rmtree(site.USER_SITE) - site.USER_BASE = self.old_base - site.USER_SITE = self.old_site - - def test_develop(self): - if hasattr(sys, 'real_prefix'): - return - dist = Distribution( - dict(name='foo', - packages=['foo'], - use_2to3=True, - version='0.0', - )) + with target.as_cwd(): + yield target + + +class TestDevelop: + in_virtualenv = hasattr(sys, 'real_prefix') + in_venv = hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix + @pytest.mark.skipif(in_virtualenv or in_venv, + reason="Cannot run when invoked in a virtualenv or venv") + def test_2to3_user_mode(self, test_env): + settings = dict( + name='foo', + packages=['foo'], + use_2to3=True, + version='0.0', + ) + dist = Distribution(settings) dist.script_name = 'setup.py' cmd = develop(dist) cmd.user = 1 cmd.ensure_finalized() cmd.install_dir = site.USER_SITE cmd.user = 1 - old_stdout = sys.stdout - #sys.stdout = StringIO() - try: + with contexts.quiet(): cmd.run() - finally: - sys.stdout = old_stdout # let's see if we got our egg link at the right place content = os.listdir(site.USER_SITE) @@ -87,17 +79,37 @@ class TestDevelopTest: assert content == ['easy-install.pth', 'foo.egg-link'] # Check that we are using the right code. - egg_link_file = open(os.path.join(site.USER_SITE, 'foo.egg-link'), 'rt') - try: + fn = os.path.join(site.USER_SITE, 'foo.egg-link') + with io.open(fn) as egg_link_file: path = egg_link_file.read().split()[0].strip() - finally: - egg_link_file.close() - init_file = open(os.path.join(path, 'foo', '__init__.py'), 'rt') - try: + fn = os.path.join(path, 'foo', '__init__.py') + with io.open(fn) as init_file: init = init_file.read().strip() - finally: - init_file.close() - if sys.version < "3": - assert init == 'print "foo"' - else: - assert init == 'print("foo")' + + expected = 'print("foo")' if six.PY3 else 'print "foo"' + assert init == expected + + def test_console_scripts(self, tmpdir): + """ + Test that console scripts are installed and that they reference + only the project by name and not the current version. + """ + pytest.skip("TODO: needs a fixture to cause 'develop' " + "to be invoked without mutating environment.") + settings = dict( + name='foo', + packages=['foo'], + version='0.0', + entry_points={ + 'console_scripts': [ + 'foocmd = foo:foo', + ], + }, + ) + dist = Distribution(settings) + dist.script_name = 'setup.py' + cmd = develop(dist) + cmd.ensure_finalized() + cmd.install_dir = tmpdir + cmd.run() + #assert '0.0' not in foocmd_text diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index 5d5ec16d..30220b7f 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -1,4 +1,4 @@ -#! -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """Easy install Tests """ @@ -13,29 +13,30 @@ import contextlib import tarfile import logging import itertools +import distutils.errors import io import six from six.moves import urllib import pytest -import mock +try: + from unittest import mock +except ImportError: + import mock from setuptools import sandbox -from setuptools.sandbox import run_setup, SandboxViolation -from setuptools.command.easy_install import ( - easy_install, fix_jython_executable, get_script_args, nt_quote_arg, - get_script_header, is_sh, -) +from setuptools.sandbox import run_setup +import setuptools.command.easy_install as ei from setuptools.command.easy_install import PthDistributions from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution -from pkg_resources import working_set, VersionConflict +from pkg_resources import working_set from pkg_resources import Distribution as PRDistribution import setuptools.tests.server import pkg_resources from .py26compat import tarfile_open -from . import contexts +from . import contexts, is_ascii from .textwrap import DALS @@ -48,19 +49,6 @@ class FakeDist(object): def as_requirement(self): return 'spec' -WANTED = DALS(""" - #!%s - # EASY-INSTALL-ENTRY-SCRIPT: 'spec','console_scripts','name' - __requires__ = 'spec' - import sys - from pkg_resources import load_entry_point - - if __name__ == '__main__': - sys.exit( - load_entry_point('spec', 'console_scripts', 'name')() - ) - """) % nt_quote_arg(fix_jython_executable(sys.executable, "")) - SETUP_PY = DALS(""" from setuptools import setup @@ -71,7 +59,7 @@ class TestEasyInstallTest: def test_install_site_py(self): dist = Distribution() - cmd = easy_install(dist) + cmd = ei.easy_install(dist) cmd.sitepy_installed = False cmd.install_dir = tempfile.mkdtemp() try: @@ -82,18 +70,30 @@ class TestEasyInstallTest: shutil.rmtree(cmd.install_dir) def test_get_script_args(self): + header = ei.CommandSpec.best().from_environment().as_header() + expected = header + DALS(""" + # EASY-INSTALL-ENTRY-SCRIPT: 'spec','console_scripts','name' + __requires__ = 'spec' + import sys + from pkg_resources import load_entry_point + + if __name__ == '__main__': + sys.exit( + load_entry_point('spec', 'console_scripts', 'name')() + ) + """) dist = FakeDist() - args = next(get_script_args(dist)) + args = next(ei.ScriptWriter.get_args(dist)) name, script = itertools.islice(args, 2) - assert script == WANTED + assert script == expected def test_no_find_links(self): # new option '--no-find-links', that blocks find-links added at # the project level dist = Distribution() - cmd = easy_install(dist) + cmd = ei.easy_install(dist) cmd.check_pth_processing = lambda: True cmd.no_find_links = True cmd.find_links = ['link1', 'link2'] @@ -103,7 +103,7 @@ class TestEasyInstallTest: assert cmd.package_index.scanned_urls == {} # let's try without it (default behavior) - cmd = easy_install(dist) + cmd = ei.easy_install(dist) cmd.check_pth_processing = lambda: True cmd.find_links = ['link1', 'link2'] cmd.install_dir = os.path.join(tempfile.mkdtemp(), 'ok') @@ -112,6 +112,16 @@ class TestEasyInstallTest: keys = sorted(cmd.package_index.scanned_urls.keys()) assert keys == ['link1', 'link2'] + def test_write_exception(self): + """ + Test that `cant_write_to_target` is rendered as a DistutilsError. + """ + dist = Distribution() + cmd = ei.easy_install(dist) + cmd.install_dir = os.getcwd() + with pytest.raises(distutils.errors.DistutilsError): + cmd.cant_write_to_target() + class TestPTHFileWriter: def test_add_from_cwd_site_sets_dirty(self): @@ -145,77 +155,74 @@ def setup_context(tmpdir): @pytest.mark.usefixtures("setup_context") class TestUserInstallTest: - @mock.patch('setuptools.command.easy_install.__file__', None) - def test_user_install_implied(self): - easy_install_pkg.__file__ = site.USER_SITE - site.ENABLE_USER_SITE = True # disabled sometimes - #XXX: replace with something meaningfull + # prevent check that site-packages is writable. easy_install + # shouldn't be writing to system site-packages during finalize + # options, but while it does, bypass the behavior. + prev_sp_write = mock.patch( + 'setuptools.command.easy_install.easy_install.check_site_dir', + mock.Mock(), + ) + + # simulate setuptools installed in user site packages + @mock.patch('setuptools.command.easy_install.__file__', site.USER_SITE) + @mock.patch('site.ENABLE_USER_SITE', True) + @prev_sp_write + def test_user_install_not_implied_user_site_enabled(self): + self.assert_not_user_site() + + @mock.patch('site.ENABLE_USER_SITE', False) + @prev_sp_write + def test_user_install_not_implied_user_site_disabled(self): + self.assert_not_user_site() + + @staticmethod + def assert_not_user_site(): + # create a finalized easy_install command dist = Distribution() dist.script_name = 'setup.py' - cmd = easy_install(dist) + cmd = ei.easy_install(dist) cmd.args = ['py'] cmd.ensure_finalized() - assert cmd.user, 'user should be implied' + assert not cmd.user, 'user should not be implied' def test_multiproc_atexit(self): - try: - __import__('multiprocessing') - except ImportError: - # skip the test if multiprocessing is not available - return + pytest.importorskip('multiprocessing') log = logging.getLogger('test_easy_install') logging.basicConfig(level=logging.INFO, stream=sys.stderr) log.info('this should not break') - def test_user_install_not_implied_without_usersite_enabled(self): - site.ENABLE_USER_SITE = False # usually enabled - #XXX: replace with something meaningfull - dist = Distribution() - dist.script_name = 'setup.py' - cmd = easy_install(dist) - cmd.args = ['py'] - cmd.initialize_options() - assert not cmd.user, 'NOT user should be implied' - - def test_local_index(self): - # make sure the local index is used - # when easy_install looks for installed - # packages - new_location = tempfile.mkdtemp() - target = tempfile.mkdtemp() - egg_file = os.path.join(new_location, 'foo-1.0.egg-info') - with open(egg_file, 'w') as f: + @pytest.fixture() + def foo_package(self, tmpdir): + egg_file = tmpdir / 'foo-1.0.egg-info' + with egg_file.open('w') as f: f.write('Name: foo\n') + return str(tmpdir) - sys.path.append(target) - old_ppath = os.environ.get('PYTHONPATH') - os.environ['PYTHONPATH'] = os.path.pathsep.join(sys.path) - try: - dist = Distribution() - dist.script_name = 'setup.py' - cmd = easy_install(dist) - cmd.install_dir = target - cmd.args = ['foo'] - cmd.ensure_finalized() - cmd.local_index.scan([new_location]) - res = cmd.easy_install('foo') - actual = os.path.normcase(os.path.realpath(res.location)) - expected = os.path.normcase(os.path.realpath(new_location)) - assert actual == expected - finally: - sys.path.remove(target) - for basedir in [new_location, target, ]: - if not os.path.exists(basedir) or not os.path.isdir(basedir): - continue - try: - shutil.rmtree(basedir) - except: - pass - if old_ppath is not None: - os.environ['PYTHONPATH'] = old_ppath - else: - del os.environ['PYTHONPATH'] + @pytest.yield_fixture() + def install_target(self, tmpdir): + target = str(tmpdir) + with mock.patch('sys.path', sys.path + [target]): + python_path = os.path.pathsep.join(sys.path) + with mock.patch.dict(os.environ, PYTHONPATH=python_path): + yield target + + def test_local_index(self, foo_package, install_target): + """ + The local index must be used when easy_install locates installed + packages. + """ + dist = Distribution() + dist.script_name = 'setup.py' + cmd = ei.easy_install(dist) + cmd.install_dir = install_target + cmd.args = ['foo'] + cmd.ensure_finalized() + cmd.local_index.scan([foo_package]) + res = cmd.easy_install('foo') + actual = os.path.normcase(os.path.realpath(res.location)) + expected = os.path.normcase(os.path.realpath(foo_package)) + assert actual == expected @contextlib.contextmanager def user_install_setup_context(self, *args, **kwargs): @@ -236,28 +243,6 @@ class TestUserInstallTest: self.user_install_setup_context, ) - def test_setup_requires(self): - """Regression test for Distribute issue #318 - - Ensure that a package with setup_requires can be installed when - setuptools is installed in the user site-packages without causing a - SandboxViolation. - """ - - test_pkg = create_setup_requires_package(os.getcwd()) - test_setup_py = os.path.join(test_pkg, 'setup.py') - - try: - with contexts.quiet(): - with self.patched_setup_context(): - run_setup(test_setup_py, ['install']) - except SandboxViolation: - self.fail('Installation caused SandboxViolation') - except IndexError: - # Test fails in some cases due to bugs in Python - # See https://bitbucket.org/pypa/setuptools/issue/201 - pass - @pytest.yield_fixture def distutils_package(): @@ -305,7 +290,7 @@ class TestSetupRequires: '--install-dir', temp_install_dir, dist_file, ] - with contexts.argv(['easy_install']): + with sandbox.save_argv(['easy_install']): # attempt to install the dist. It should fail because # it doesn't exist. with pytest.raises(SystemExit): @@ -354,13 +339,9 @@ class TestSetupRequires: test_pkg = create_setup_requires_package(temp_dir) test_setup_py = os.path.join(test_pkg, 'setup.py') with contexts.quiet() as (stdout, stderr): - try: - # Don't even need to install the package, just - # running the setup.py at all is sufficient - run_setup(test_setup_py, ['--name']) - except VersionConflict: - self.fail('Installing setup.py requirements ' - 'caused a VersionConflict') + # Don't even need to install the package, just + # running the setup.py at all is sufficient + run_setup(test_setup_py, ['--name']) lines = stdout.readlines() assert len(lines) > 0 @@ -422,23 +403,31 @@ class TestScriptHeader: exe_with_spaces = r'C:\Program Files\Python33\python.exe' @pytest.mark.skipif( - sys.platform.startswith('java') and is_sh(sys.executable), + sys.platform.startswith('java') and ei.is_sh(sys.executable), reason="Test cannot run under java when executable is sh" ) def test_get_script_header(self): - expected = '#!%s\n' % nt_quote_arg(os.path.normpath(sys.executable)) - assert get_script_header('#!/usr/local/bin/python') == expected - expected = '#!%s -x\n' % nt_quote_arg(os.path.normpath(sys.executable)) - assert get_script_header('#!/usr/bin/python -x') == expected - candidate = get_script_header('#!/usr/bin/python', + expected = '#!%s\n' % ei.nt_quote_arg(os.path.normpath(sys.executable)) + actual = ei.ScriptWriter.get_script_header('#!/usr/local/bin/python') + assert actual == expected + + expected = '#!%s -x\n' % ei.nt_quote_arg(os.path.normpath + (sys.executable)) + actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python -x') + assert actual == expected + + actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', executable=self.non_ascii_exe) - assert candidate == '#!%s -x\n' % self.non_ascii_exe - candidate = get_script_header('#!/usr/bin/python', - executable=self.exe_with_spaces) - assert candidate == '#!"%s"\n' % self.exe_with_spaces + expected = '#!%s -x\n' % self.non_ascii_exe + assert actual == expected + + actual = ei.ScriptWriter.get_script_header('#!/usr/bin/python', + executable='"'+self.exe_with_spaces+'"') + expected = '#!"%s"\n' % self.exe_with_spaces + assert actual == expected @pytest.mark.xfail( - six.PY3 and os.environ.get("LC_CTYPE") in ("C", "POSIX"), + six.PY3 and is_ascii, reason="Test fails in this locale on Python 3" ) @mock.patch.dict(sys.modules, java=mock.Mock(lang=mock.Mock(System= @@ -453,9 +442,15 @@ class TestScriptHeader: exe = tmpdir / 'exe.py' with exe.open('w') as f: f.write(header) - exe = str(exe) - header = get_script_header('#!/usr/local/bin/python', executable=exe) + exe = ei.nt_quote_arg(os.path.normpath(str(exe))) + + # Make sure Windows paths are quoted properly before they're sent + # through shlex.split by get_script_header + executable = '"%s"' % exe if os.path.splitdrive(exe)[0] else exe + + header = ei.ScriptWriter.get_script_header('#!/usr/local/bin/python', + executable=executable) assert header == '#!/usr/bin/env %s\n' % exe expect_out = 'stdout' if sys.version_info < (2,7) else 'stderr' @@ -463,15 +458,70 @@ class TestScriptHeader: with contexts.quiet() as (stdout, stderr): # When options are included, generate a broken shebang line # with a warning emitted - candidate = get_script_header('#!/usr/bin/python -x', - executable=exe) - assert candidate == '#!%s -x\n' % exe + candidate = ei.ScriptWriter.get_script_header('#!/usr/bin/python -x', + executable=executable) + assert candidate == '#!%s -x\n' % exe output = locals()[expect_out] assert 'Unable to adapt shebang line' in output.getvalue() with contexts.quiet() as (stdout, stderr): - candidate = get_script_header('#!/usr/bin/python', + candidate = ei.ScriptWriter.get_script_header('#!/usr/bin/python', executable=self.non_ascii_exe) assert candidate == '#!%s -x\n' % self.non_ascii_exe output = locals()[expect_out] assert 'Unable to adapt shebang line' in output.getvalue() + + +class TestCommandSpec: + def test_custom_launch_command(self): + """ + Show how a custom CommandSpec could be used to specify a #! executable + which takes parameters. + """ + cmd = ei.CommandSpec(['/usr/bin/env', 'python3']) + assert cmd.as_header() == '#!/usr/bin/env python3\n' + + def test_from_param_for_CommandSpec_is_passthrough(self): + """ + from_param should return an instance of a CommandSpec + """ + cmd = ei.CommandSpec(['python']) + cmd_new = ei.CommandSpec.from_param(cmd) + assert cmd is cmd_new + + @mock.patch('sys.executable', TestScriptHeader.exe_with_spaces) + @mock.patch.dict(os.environ) + def test_from_environment_with_spaces_in_executable(self): + os.environ.pop('__PYVENV_LAUNCHER__', None) + cmd = ei.CommandSpec.from_environment() + assert len(cmd) == 1 + assert cmd.as_header().startswith('#!"') + + def test_from_simple_string_uses_shlex(self): + """ + In order to support `executable = /usr/bin/env my-python`, make sure + from_param invokes shlex on that input. + """ + cmd = ei.CommandSpec.from_param('/usr/bin/env my-python') + assert len(cmd) == 2 + assert '"' not in cmd.as_header() + + def test_sys_executable(self): + """ + CommandSpec.from_string(sys.executable) should contain just that param. + """ + writer = ei.ScriptWriter.best() + cmd = writer.command_spec_class.from_string(sys.executable) + assert len(cmd) == 1 + assert cmd[0] == sys.executable + + +class TestWindowsScriptWriter: + def test_header(self): + hdr = ei.WindowsScriptWriter.get_script_header('') + assert hdr.startswith('#!') + assert hdr.endswith('\n') + hdr = hdr.lstrip('#!') + hdr = hdr.rstrip('\n') + # header should not start with an escaped quote + assert not hdr.startswith('\\"') diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index a1caf9fd..333d11d6 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -4,11 +4,16 @@ import stat import pytest from . import environment +from .files import build_files from .textwrap import DALS from . import contexts -class TestEggInfo: +class Environment(str): + pass + + +class TestEggInfo(object): setup_script = DALS(""" from setuptools import setup @@ -22,19 +27,16 @@ class TestEggInfo: """) def _create_project(self): - with open('setup.py', 'w') as f: - f.write(self.setup_script) - - with open('hello.py', 'w') as f: - f.write(DALS(""" + build_files({ + 'setup.py': self.setup_script, + 'hello.py': DALS(""" def run(): print('hello') - """)) + """) + }) @pytest.yield_fixture def env(self): - class Environment(str): pass - with contexts.tempdir(prefix='setuptools-test.') as env_dir: env = Environment(env_dir) os.chmod(env_dir, stat.S_IRWXU) @@ -44,18 +46,48 @@ class TestEggInfo: for dirname in subs ) list(map(os.mkdir, env.paths.values())) - config = os.path.join(env.paths['home'], '.pydistutils.cfg') - with open(config, 'w') as f: - f.write(DALS(""" + build_files({ + env.paths['home']: { + '.pydistutils.cfg': DALS(""" [egg_info] egg-base = %(egg-base)s - """ % env.paths - )) + """ % env.paths) + } + }) yield env def test_egg_base_installed_egg_info(self, tmpdir_cwd, env): self._create_project() + self._run_install_command(tmpdir_cwd, env) + actual = self._find_egg_info_files(env.paths['lib']) + + expected = [ + 'PKG-INFO', + 'SOURCES.txt', + 'dependency_links.txt', + 'entry_points.txt', + 'not-zip-safe', + 'top_level.txt', + ] + assert sorted(actual) == expected + + def test_manifest_template_is_read(self, tmpdir_cwd, env): + self._create_project() + build_files({ + 'MANIFEST.in': DALS(""" + recursive-include docs *.rst + """), + 'docs': { + 'usage.rst': "Run 'hi'", + } + }) + self._run_install_command(tmpdir_cwd, env) + egg_info_dir = self._find_egg_info_files(env.paths['lib']).base + sources_txt = os.path.join(egg_info_dir, 'SOURCES.txt') + assert 'docs/usage.rst' in open(sources_txt).read().split('\n') + + def _run_install_command(self, tmpdir_cwd, env): environ = os.environ.copy().update( HOME=env.paths['home'], ) @@ -75,21 +107,14 @@ class TestEggInfo: if code: raise AssertionError(data) - actual = self._find_egg_info_files(env.paths['lib']) - - expected = [ - 'PKG-INFO', - 'SOURCES.txt', - 'dependency_links.txt', - 'entry_points.txt', - 'not-zip-safe', - 'top_level.txt', - ] - assert sorted(actual) == expected - def _find_egg_info_files(self, root): + class DirList(list): + def __init__(self, files, base): + super(DirList, self).__init__(files) + self.base = base + results = ( - filenames + DirList(filenames, dirpath) for dirpath, dirnames, filenames in os.walk(root) if os.path.basename(dirpath) == 'EGG-INFO' ) diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 3a6abeaa..11a6ff5a 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -7,6 +7,8 @@ import glob import os import sys +from six.moves import urllib + import pytest from setuptools.command.easy_install import easy_install @@ -14,6 +16,22 @@ from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution +def setup_module(module): + packages = 'stevedore', 'virtualenvwrapper', 'pbr', 'novaclient' + for pkg in packages: + try: + __import__(pkg) + tmpl = "Integration tests cannot run when {pkg} is installed" + pytest.skip(tmpl.format(**locals())) + except ImportError: + pass + + try: + urllib.request.urlopen('https://pypi.python.org/pypi') + except Exception as exc: + pytest.skip(str(exc)) + + @pytest.fixture def install_context(request, tmpdir, monkeypatch): """Fixture to set up temporary installation directory. diff --git a/setuptools/tests/test_msvc9compiler.py b/setuptools/tests/test_msvc9compiler.py index a0820fff..09e0460c 100644 --- a/setuptools/tests/test_msvc9compiler.py +++ b/setuptools/tests/test_msvc9compiler.py @@ -7,7 +7,10 @@ import contextlib import distutils.errors import pytest -import mock +try: + from unittest import mock +except ImportError: + import mock from . import contexts @@ -110,7 +113,8 @@ class TestModulePatch: Ensure user's settings are preferred. """ result = distutils.msvc9compiler.find_vcvarsall(9.0) - assert user_preferred_setting == result + expected = os.path.join(user_preferred_setting, 'vcvarsall.bat') + assert expected == result @pytest.yield_fixture def local_machine_setting(self): @@ -131,13 +135,14 @@ class TestModulePatch: Ensure machine setting is honored if user settings are not present. """ result = distutils.msvc9compiler.find_vcvarsall(9.0) - assert local_machine_setting == result + expected = os.path.join(local_machine_setting, 'vcvarsall.bat') + assert expected == result @pytest.yield_fixture def x64_preferred_setting(self): """ Set up environment with 64-bit and 32-bit system settings configured - and yield the 64-bit location. + and yield the canonical location. """ with self.mock_install_dir() as x32_dir: with self.mock_install_dir() as x64_dir: @@ -150,14 +155,15 @@ class TestModulePatch: }, ) with reg: - yield x64_dir + yield x32_dir def test_ensure_64_bit_preferred(self, x64_preferred_setting): """ Ensure 64-bit system key is preferred. """ result = distutils.msvc9compiler.find_vcvarsall(9.0) - assert x64_preferred_setting == result + expected = os.path.join(x64_preferred_setting, 'vcvarsall.bat') + assert expected == result @staticmethod @contextlib.contextmanager @@ -170,4 +176,4 @@ class TestModulePatch: vcvarsall = os.path.join(result, 'vcvarsall.bat') with open(vcvarsall, 'w'): pass - yield + yield result diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py index 4eb98bb1..dca4c2aa 100644 --- a/setuptools/tests/test_packageindex.py +++ b/setuptools/tests/test_packageindex.py @@ -1,9 +1,13 @@ +from __future__ import absolute_import + import sys +import os import distutils.errors import six from six.moves import urllib, http_client +from .textwrap import DALS import pkg_resources import setuptools.package_index from setuptools.tests.server import IndexServer @@ -16,8 +20,7 @@ class TestPackageIndex: url = 'http://127.0.0.1:0/nonesuch/test_package_index' try: v = index.open_url(url) - except Exception: - v = sys.exc_info()[1] + except Exception as v: assert url in str(v) else: assert isinstance(v, urllib.error.HTTPError) @@ -33,8 +36,7 @@ class TestPackageIndex: url = 'url:%20https://svn.plone.org/svn/collective/inquant.contentmirror.plone/trunk' try: v = index.open_url(url) - except Exception: - v = sys.exc_info()[1] + except Exception as v: assert url in str(v) else: assert isinstance(v, urllib.error.HTTPError) @@ -51,8 +53,7 @@ class TestPackageIndex: url = 'http://example.com' try: v = index.open_url(url) - except Exception: - v = sys.exc_info()[1] + except Exception as v: assert 'line' in str(v) else: raise AssertionError('Should have raise here!') @@ -69,8 +70,7 @@ class TestPackageIndex: url = 'http://http://svn.pythonpaste.org/Paste/wphp/trunk' try: index.open_url(url) - except distutils.errors.DistutilsError: - error = sys.exc_info()[1] + except distutils.errors.DistutilsError as error: msg = six.text_type(error) assert 'nonnumeric port' in msg or 'getaddrinfo failed' in msg or 'Name or service not known' in msg return @@ -206,3 +206,20 @@ class TestContentCheckers: 'http://foo/bar#md5=f12895fdffbd45007040d2e44df98478') rep = checker.report(lambda x: x, 'My message about %s') assert rep == 'My message about md5' + + +class TestPyPIConfig: + def test_percent_in_password(self, tmpdir, monkeypatch): + monkeypatch.setitem(os.environ, 'HOME', str(tmpdir)) + pypirc = tmpdir / '.pypirc' + with pypirc.open('w') as strm: + strm.write(DALS(""" + [pypi] + repository=https://pypi.python.org + username=jaraco + password=pity% + """)) + cfg = setuptools.package_index.PyPIConfig() + cred = cfg.creds_by_repository['https://pypi.python.org'] + assert cred.username == 'jaraco' + assert cred.password == 'pity%' diff --git a/setuptools/tests/test_sandbox.py b/setuptools/tests/test_sandbox.py index 6e5ce04a..fefd46f7 100644 --- a/setuptools/tests/test_sandbox.py +++ b/setuptools/tests/test_sandbox.py @@ -7,7 +7,7 @@ import pytest import pkg_resources import setuptools.sandbox -from setuptools.sandbox import DirectorySandbox, SandboxViolation +from setuptools.sandbox import DirectorySandbox class TestSandbox: @@ -33,10 +33,8 @@ class TestSandbox: target = os.path.join(gen_py, 'test_write') sandbox = DirectorySandbox(str(tmpdir)) try: - try: - sandbox.run(self._file_writer(target)) - except SandboxViolation: - self.fail("Could not create gen_py file due to SandboxViolation") + # attempt to create gen_py file + sandbox.run(self._file_writer(target)) finally: if os.path.exists(target): os.remove(target) @@ -56,3 +54,88 @@ class TestSandbox: with setup_py.open('wb') as stream: stream.write(b'"degenerate script"\r\n') setuptools.sandbox._execfile(str(setup_py), globals()) + + +class TestExceptionSaver: + def test_exception_trapped(self): + with setuptools.sandbox.ExceptionSaver(): + raise ValueError("details") + + def test_exception_resumed(self): + with setuptools.sandbox.ExceptionSaver() as saved_exc: + raise ValueError("details") + + with pytest.raises(ValueError) as caught: + saved_exc.resume() + + assert isinstance(caught.value, ValueError) + assert str(caught.value) == 'details' + + def test_exception_reconstructed(self): + orig_exc = ValueError("details") + + with setuptools.sandbox.ExceptionSaver() as saved_exc: + raise orig_exc + + with pytest.raises(ValueError) as caught: + saved_exc.resume() + + assert isinstance(caught.value, ValueError) + assert caught.value is not orig_exc + + def test_no_exception_passes_quietly(self): + with setuptools.sandbox.ExceptionSaver() as saved_exc: + pass + + saved_exc.resume() + + def test_unpickleable_exception(self): + class CantPickleThis(Exception): + "This Exception is unpickleable because it's not in globals" + + with setuptools.sandbox.ExceptionSaver() as saved_exc: + raise CantPickleThis('detail') + + with pytest.raises(setuptools.sandbox.UnpickleableException) as caught: + saved_exc.resume() + + assert str(caught.value) == "CantPickleThis('detail',)" + + def test_unpickleable_exception_when_hiding_setuptools(self): + """ + As revealed in #440, an infinite recursion can occur if an unpickleable + exception while setuptools is hidden. Ensure this doesn't happen. + """ + class ExceptionUnderTest(Exception): + """ + An unpickleable exception (not in globals). + """ + + with pytest.raises(setuptools.sandbox.UnpickleableException) as caught: + with setuptools.sandbox.save_modules(): + setuptools.sandbox.hide_setuptools() + raise ExceptionUnderTest() + + msg, = caught.value.args + assert msg == 'ExceptionUnderTest()' + + def test_sandbox_violation_raised_hiding_setuptools(self, tmpdir): + """ + When in a sandbox with setuptools hidden, a SandboxViolation + should reflect a proper exception and not be wrapped in + an UnpickleableException. + """ + def write_file(): + "Trigger a SandboxViolation by writing outside the sandbox" + with open('/etc/foo', 'w'): + pass + sandbox = DirectorySandbox(str(tmpdir)) + with pytest.raises(setuptools.sandbox.SandboxViolation) as caught: + with setuptools.sandbox.save_modules(): + setuptools.sandbox.hide_setuptools() + sandbox.run(write_file) + + cmd, args, kwargs = caught.value.args + assert cmd == 'open' + assert args == ('/etc/foo', 'w') + assert kwargs == {} diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index d30e21ac..c173d713 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- """sdist tests""" -import locale import os import shutil import sys import tempfile import unicodedata import contextlib +import io import six import pytest @@ -16,6 +16,11 @@ import pkg_resources from setuptools.command.sdist import sdist from setuptools.command.egg_info import manifest_maker from setuptools.dist import Distribution +from setuptools.tests import fail_on_ascii + + +py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") + SETUP_ATTRS = { 'name': 'sdist_test', @@ -77,6 +82,11 @@ def decompose(path): return path +def read_all_bytes(filename): + with io.open(filename, 'rb') as fp: + return fp.read() + + class TestSdistTest: def setup_method(self, method): @@ -147,6 +157,7 @@ class TestSdistTest: assert 'setup.py' not in manifest, manifest assert 'setup.cfg' not in manifest, manifest + @fail_on_ascii def test_manifest_is_written_with_utf8_encoding(self): # Test for #303. dist = Distribution(SETUP_ATTRS) @@ -167,16 +178,10 @@ class TestSdistTest: mm.filelist.append(filename) mm.write_manifest() - manifest = open(mm.manifest, 'rbU') - contents = manifest.read() - manifest.close() + contents = read_all_bytes(mm.manifest) # The manifest should be UTF-8 encoded - try: - u_contents = contents.decode('UTF-8') - except UnicodeDecodeError: - e = sys.exc_info()[1] - self.fail(e) + u_contents = contents.decode('UTF-8') # The manifest should contain the UTF-8 filename if six.PY2: @@ -185,89 +190,78 @@ class TestSdistTest: assert posix(filename) in u_contents - # Python 3 only - if six.PY3: + @py3_only + @fail_on_ascii + def test_write_manifest_allows_utf8_filenames(self): + # Test for #303. + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'setup.py' + mm = manifest_maker(dist) + mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') + os.mkdir('sdist_test.egg-info') - def test_write_manifest_allows_utf8_filenames(self): - # Test for #303. - dist = Distribution(SETUP_ATTRS) - dist.script_name = 'setup.py' - mm = manifest_maker(dist) - mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') - os.mkdir('sdist_test.egg-info') - - # UTF-8 filename - filename = os.path.join(b('sdist_test'), b('smörbröd.py')) - - # Must touch the file or risk removal - open(filename, "w").close() - - # Add filename and write manifest - with quiet(): - mm.run() - u_filename = filename.decode('utf-8') - mm.filelist.files.append(u_filename) - # Re-write manifest - mm.write_manifest() - - manifest = open(mm.manifest, 'rbU') - contents = manifest.read() - manifest.close() - - # The manifest should be UTF-8 encoded - try: - contents.decode('UTF-8') - except UnicodeDecodeError: - e = sys.exc_info()[1] - self.fail(e) - - # The manifest should contain the UTF-8 filename - assert posix(filename) in contents - - # The filelist should have been updated as well - assert u_filename in mm.filelist.files - - def test_write_manifest_skips_non_utf8_filenames(self): - """ - Files that cannot be encoded to UTF-8 (specifically, those that - weren't originally successfully decoded and have surrogate - escapes) should be omitted from the manifest. - See https://bitbucket.org/tarek/distribute/issue/303 for history. - """ - dist = Distribution(SETUP_ATTRS) - dist.script_name = 'setup.py' - mm = manifest_maker(dist) - mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') - os.mkdir('sdist_test.egg-info') - - # Latin-1 filename - filename = os.path.join(b('sdist_test'), LATIN1_FILENAME) - - # Add filename with surrogates and write manifest - with quiet(): - mm.run() - u_filename = filename.decode('utf-8', 'surrogateescape') - mm.filelist.append(u_filename) - # Re-write manifest - mm.write_manifest() - - manifest = open(mm.manifest, 'rbU') - contents = manifest.read() - manifest.close() - - # The manifest should be UTF-8 encoded - try: - contents.decode('UTF-8') - except UnicodeDecodeError: - e = sys.exc_info()[1] - self.fail(e) + # UTF-8 filename + filename = os.path.join(b('sdist_test'), b('smörbröd.py')) - # The Latin-1 filename should have been skipped - assert posix(filename) not in contents + # Must touch the file or risk removal + open(filename, "w").close() - # The filelist should have been updated as well - assert u_filename not in mm.filelist.files + # Add filename and write manifest + with quiet(): + mm.run() + u_filename = filename.decode('utf-8') + mm.filelist.files.append(u_filename) + # Re-write manifest + mm.write_manifest() + + contents = read_all_bytes(mm.manifest) + + # The manifest should be UTF-8 encoded + contents.decode('UTF-8') + + # The manifest should contain the UTF-8 filename + assert posix(filename) in contents + + # The filelist should have been updated as well + assert u_filename in mm.filelist.files + @py3_only + def test_write_manifest_skips_non_utf8_filenames(self): + """ + Files that cannot be encoded to UTF-8 (specifically, those that + weren't originally successfully decoded and have surrogate + escapes) should be omitted from the manifest. + See https://bitbucket.org/tarek/distribute/issue/303 for history. + """ + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'setup.py' + mm = manifest_maker(dist) + mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') + os.mkdir('sdist_test.egg-info') + + # Latin-1 filename + filename = os.path.join(b('sdist_test'), LATIN1_FILENAME) + + # Add filename with surrogates and write manifest + with quiet(): + mm.run() + u_filename = filename.decode('utf-8', 'surrogateescape') + mm.filelist.append(u_filename) + # Re-write manifest + mm.write_manifest() + + contents = read_all_bytes(mm.manifest) + + # The manifest should be UTF-8 encoded + contents.decode('UTF-8') + + # The Latin-1 filename should have been skipped + assert posix(filename) not in contents + + # The filelist should have been updated as well + assert u_filename not in mm.filelist.files + + @fail_on_ascii def test_manifest_is_read_with_utf8_encoding(self): # Test for #303. dist = Distribution(SETUP_ATTRS) @@ -299,46 +293,38 @@ class TestSdistTest: filename = filename.decode('utf-8') assert filename in cmd.filelist.files - # Python 3 only - if six.PY3: + @py3_only + def test_read_manifest_skips_non_utf8_filenames(self): + # Test for #303. + dist = Distribution(SETUP_ATTRS) + dist.script_name = 'setup.py' + cmd = sdist(dist) + cmd.ensure_finalized() + + # Create manifest + with quiet(): + cmd.run() + + # Add Latin-1 filename to manifest + filename = os.path.join(b('sdist_test'), LATIN1_FILENAME) + cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') + manifest = open(cmd.manifest, 'ab') + manifest.write(b('\n') + filename) + manifest.close() + + # The file must exist to be included in the filelist + open(filename, 'w').close() + + # Re-read manifest + cmd.filelist.files = [] + with quiet(): + cmd.read_manifest() + + # The Latin-1 filename should have been skipped + filename = filename.decode('latin-1') + assert filename not in cmd.filelist.files - def test_read_manifest_skips_non_utf8_filenames(self): - # Test for #303. - dist = Distribution(SETUP_ATTRS) - dist.script_name = 'setup.py' - cmd = sdist(dist) - cmd.ensure_finalized() - - # Create manifest - with quiet(): - cmd.run() - - # Add Latin-1 filename to manifest - filename = os.path.join(b('sdist_test'), LATIN1_FILENAME) - cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt') - manifest = open(cmd.manifest, 'ab') - manifest.write(b('\n') + filename) - manifest.close() - - # The file must exist to be included in the filelist - open(filename, 'w').close() - - # Re-read manifest - cmd.filelist.files = [] - with quiet(): - try: - cmd.read_manifest() - except UnicodeDecodeError: - e = sys.exc_info()[1] - self.fail(e) - - # The Latin-1 filename should have been skipped - filename = filename.decode('latin-1') - assert filename not in cmd.filelist.files - - @pytest.mark.skipif(six.PY3 and locale.getpreferredencoding() != 'UTF-8', - reason='Unittest fails if locale is not utf-8 but the manifests is ' - 'recorded correctly') + @fail_on_ascii def test_sdist_with_utf8_encoded_filename(self): # Test for #303. dist = Distribution(SETUP_ATTRS) @@ -431,5 +417,5 @@ def test_default_revctrl(): """ ep_def = 'svn_cvs = setuptools.command.sdist:_default_revctrl' ep = pkg_resources.EntryPoint.parse(ep_def) - res = ep._load() + res = ep.resolve() assert hasattr(res, '__iter__') diff --git a/setuptools/tests/test_setuptools.py b/setuptools/tests/test_setuptools.py new file mode 100644 index 00000000..e59800d2 --- /dev/null +++ b/setuptools/tests/test_setuptools.py @@ -0,0 +1,48 @@ +import os + +import pytest + +import setuptools + + +@pytest.fixture +def example_source(tmpdir): + tmpdir.mkdir('foo') + (tmpdir / 'foo/bar.py').write('') + (tmpdir / 'readme.txt').write('') + return tmpdir + + +def test_findall(example_source): + found = list(setuptools.findall(str(example_source))) + expected = ['readme.txt', 'foo/bar.py'] + expected = [example_source.join(fn) for fn in expected] + assert found == expected + + +def test_findall_curdir(example_source): + with example_source.as_cwd(): + found = list(setuptools.findall()) + expected = ['readme.txt', os.path.join('foo', 'bar.py')] + assert found == expected + + +@pytest.fixture +def can_symlink(tmpdir): + """ + Skip if cannot create a symbolic link + """ + link_fn = 'link' + target_fn = 'target' + try: + os.symlink(target_fn, link_fn) + except (OSError, NotImplementedError, AttributeError): + pytest.skip("Cannot create symbolic links") + os.remove(link_fn) + + +def test_findall_missing_symlink(tmpdir, can_symlink): + with tmpdir.as_cwd(): + os.symlink('foo', 'bar') + found = list(setuptools.findall()) + assert found == [] diff --git a/setuptools/version.py b/setuptools/version.py index 1b1703fd..09bbb730 100644 --- a/setuptools/version.py +++ b/setuptools/version.py @@ -1 +1 @@ -__version__ = '11.1' +__version__ = '19.3' |