Build Update for wikimedia/pywikibot-core
-------------------------------------
Build: #3431
Status: Errored
Duration: 1 hour, 21 minutes, and 41 seconds
Commit: 65cabc8 (master)
Author: AbdealiJK
Message: pywikibot: Store ImportError in imported variable
If a lazy ImportError is to be handled, the ImportError
needs to be stored in a variable and then used when required.
Earlier, the imported variable was set to `None` and the
import error which was caught with `except` was used. But
this import error may not exist at a later time as python
guarantees it's existence only inside the `except` statement.
Hence, this is changed so that the ImportError is saved in the
variable that was being imported and checked later using
`isinstance(<variable>, ImportError)`.
Bug: T134336
Change-Id: I80f82e7f0674bfca70688f28640620dacac1d80b
View the changeset: https://github.com/wikimedia/pywikibot-core/compare/4753f89eeb38...65cabc82…
View the full build log and details: https://travis-ci.org/wikimedia/pywikibot-core/builds/127715427
--
You can configure recipients for build notifications in your .travis.yml file. See https://docs.travis-ci.com/user/notifications
jenkins-bot has submitted this change and it was merged.
Change subject: pywikibot: Store ImportError in imported variable
......................................................................
pywikibot: Store ImportError in imported variable
If a lazy ImportError is to be handled, the ImportError
needs to be stored in a variable and then used when required.
Earlier, the imported variable was set to `None` and the
import error which was caught with `except` was used. But
this import error may not exist at a later time as python
guarantees it's existence only inside the `except` statement.
Hence, this is changed so that the ImportError is saved in the
variable that was being imported and checked later using
`isinstance(<variable>, ImportError)`.
Bug: T134336
Change-Id: I80f82e7f0674bfca70688f28640620dacac1d80b
---
M pywikibot/botirc.py
M pywikibot/diff.py
M scripts/flickrripper.py
3 files changed, 7 insertions(+), 12 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/botirc.py b/pywikibot/botirc.py
index b509221..1d9f13a 100644
--- a/pywikibot/botirc.py
+++ b/pywikibot/botirc.py
@@ -25,13 +25,15 @@
try:
from ircbot import SingleServerIRCBot
except ImportError as e:
+ ircbot_import_error = e
+
class SingleServerIRCBot(object):
"""Fake SingleServerIRCBot."""
def __init__(*args, **kwargs):
"""Report import exception."""
- raise e
+ raise ircbot_import_error
_logger = "botirc"
diff --git a/pywikibot/diff.py b/pywikibot/diff.py
index f83033f..cc55fd9 100644
--- a/pywikibot/diff.py
+++ b/pywikibot/diff.py
@@ -20,11 +20,6 @@
else:
from itertools import izip_longest as zip_longest
-try:
- from bs4 import BeautifulSoup
-except ImportError as bserror:
- BeautifulSoup = False
-
import pywikibot
from pywikibot.tools import chars
@@ -579,9 +574,7 @@
@return: deleted and added list of contexts
@rtype: dict
"""
- # check if BeautifulSoup imported
- if not BeautifulSoup:
- raise bserror # should have been raised and stored earlier.
+ from bs4 import BeautifulSoup
comparands = {'deleted-context': [], 'added-context': []}
soup = BeautifulSoup(compare_string)
diff --git a/scripts/flickrripper.py b/scripts/flickrripper.py
index 7d549aa..4dfa9b9 100755
--- a/scripts/flickrripper.py
+++ b/scripts/flickrripper.py
@@ -64,7 +64,7 @@
try:
from pywikibot.userinterfaces.gui import Tkdialog
except ImportError as _tk_error:
- Tkdialog = None
+ Tkdialog = _tk_error
flickr_allowed_license = {
@@ -304,7 +304,7 @@
override, addCategory,
removeCategories)
# pywikibot.output(photoDescription)
- if Tkdialog is not None and not autonomous:
+ if not isinstance(Tkdialog, ImportError) and not autonomous:
try:
(newPhotoDescription, newFilename, skip) = Tkdialog(
photoDescription, photo, filename).show_dialog()
@@ -315,7 +315,7 @@
elif not autonomous:
pywikibot.warning('Switching to autonomous mode because GUI '
'interface cannot be used')
- pywikibot.warning(_tk_error)
+ pywikibot.warning(Tkdialog)
autonomous = True
if autonomous:
newPhotoDescription = photoDescription
--
To view, visit https://gerrit.wikimedia.org/r/286787
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I80f82e7f0674bfca70688f28640620dacac1d80b
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: AbdealiJK <abdealikothari(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Update ez_setup.py to v19.1.1
......................................................................
Update ez_setup.py to v19.1.1
Supports Python 2.6+.
Does not use `cmp`, which is removed in Python 3
causing pyflakes on Python 3 to report it as an error.
Bug: T134063
Change-Id: I08f02ba7f4c6d7ea6c32e3845024a5bd00f58b35
---
M ez_setup.py
1 file changed, 251 insertions(+), 206 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/ez_setup.py b/ez_setup.py
index b02f3f1..9715bdc 100644
--- a/ez_setup.py
+++ b/ez_setup.py
@@ -1,66 +1,55 @@
-#!python
-"""Bootstrap setuptools installation
+#!/usr/bin/env python
-If you want to use setuptools in your package's setup.py, just include this
-file in the same directory with it, and add this to the top of your setup.py::
-
- from ez_setup import use_setuptools
- use_setuptools()
-
-If you want to require a specific version of setuptools, set a download
-mirror, or use an alternate download directory, you can do so by supplying
-the appropriate options to ``use_setuptools()``.
-
-This file can also be run as a script to install or upgrade setuptools.
"""
+Setuptools bootstrapping installer.
+
+Run this script to install or upgrade setuptools.
+"""
+
import os
import shutil
import sys
import tempfile
-import tarfile
+import zipfile
import optparse
import subprocess
import platform
+import textwrap
+import contextlib
+import json
+import codecs
from distutils import log
+
+try:
+ from urllib.request import urlopen
+except ImportError:
+ from urllib2 import urlopen
try:
from site import USER_SITE
except ImportError:
USER_SITE = None
-DEFAULT_VERSION = "1.1.6"
+LATEST = object()
+DEFAULT_VERSION = LATEST
DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/"
+DEFAULT_SAVE_DIR = os.curdir
+
def _python_cmd(*args):
+ """
+ Execute a command.
+
+ Return True if the command succeeded.
+ """
args = (sys.executable,) + args
return subprocess.call(args) == 0
-def _check_call_py24(cmd, *args, **kwargs):
- res = subprocess.call(cmd, *args, **kwargs)
- class CalledProcessError(Exception):
- pass
- if not res == 0:
- msg = "Command '%s' return non-zero exit status %d" % (cmd, res)
- raise CalledProcessError(msg)
-vars(subprocess).setdefault('check_call', _check_call_py24)
-def _install(tarball, install_args=()):
- # extracting the tarball
- tmpdir = tempfile.mkdtemp()
- log.warn('Extracting in %s', tmpdir)
- old_wd = os.getcwd()
- try:
- os.chdir(tmpdir)
- tar = tarfile.open(tarball)
- _extractall(tar)
- tar.close()
-
- # going in the directory
- subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
- os.chdir(subdir)
- log.warn('Now working in %s', subdir)
-
+def _install(archive_filename, install_args=()):
+ """Install Setuptools."""
+ with archive_context(archive_filename):
# installing
log.warn('Installing Setuptools')
if not _python_cmd('setup.py', 'install', *install_args):
@@ -68,198 +57,266 @@
log.warn('See the error message above.')
# exitcode will be 2
return 2
- finally:
- os.chdir(old_wd)
- shutil.rmtree(tmpdir)
-def _build_egg(egg, tarball, to_dir):
- # extracting the tarball
- tmpdir = tempfile.mkdtemp()
- log.warn('Extracting in %s', tmpdir)
- old_wd = os.getcwd()
- try:
- os.chdir(tmpdir)
- tar = tarfile.open(tarball)
- _extractall(tar)
- tar.close()
-
- # going in the directory
- subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
- os.chdir(subdir)
- log.warn('Now working in %s', subdir)
-
+def _build_egg(egg, archive_filename, to_dir):
+ """Build Setuptools egg."""
+ with archive_context(archive_filename):
# building an egg
log.warn('Building a Setuptools egg in %s', to_dir)
_python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
-
- finally:
- os.chdir(old_wd)
- shutil.rmtree(tmpdir)
# returning the result
log.warn(egg)
if not os.path.exists(egg):
raise IOError('Could not build the egg.')
+class ContextualZipFile(zipfile.ZipFile):
+
+ """Supplement ZipFile class to support context manager for Python 2.6."""
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.close()
+
+ def __new__(cls, *args, **kwargs):
+ """Construct a ZipFile or ContextualZipFile as appropriate."""
+ if hasattr(zipfile.ZipFile, '__exit__'):
+ return zipfile.ZipFile(*args, **kwargs)
+ return super(ContextualZipFile, cls).__new__(cls)
+
+
+(a)contextlib.contextmanager
+def archive_context(filename):
+ """
+ Unzip filename to a temporary directory, set to the cwd.
+
+ The unzipped target is cleaned up after.
+ """
+ tmpdir = tempfile.mkdtemp()
+ log.warn('Extracting in %s', tmpdir)
+ old_wd = os.getcwd()
+ try:
+ os.chdir(tmpdir)
+ with ContextualZipFile(filename) as archive:
+ archive.extractall()
+
+ # going in the directory
+ subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+ os.chdir(subdir)
+ log.warn('Now working in %s', subdir)
+ yield
+
+ finally:
+ os.chdir(old_wd)
+ shutil.rmtree(tmpdir)
+
+
def _do_download(version, download_base, to_dir, download_delay):
+ """Download Setuptools."""
egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg'
% (version, sys.version_info[0], sys.version_info[1]))
if not os.path.exists(egg):
- tarball = download_setuptools(version, download_base,
+ archive = download_setuptools(version, download_base,
to_dir, download_delay)
- _build_egg(egg, tarball, to_dir)
+ _build_egg(egg, archive, to_dir)
sys.path.insert(0, egg)
# Remove previously-imported pkg_resources if present (see
# https://bitbucket.org/pypa/setuptools/pull-request/7/ for details).
if 'pkg_resources' in sys.modules:
- del sys.modules['pkg_resources']
+ _unload_pkg_resources()
import setuptools
setuptools.bootstrap_install_from = egg
-def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
- to_dir=os.curdir, download_delay=15):
- # making sure we use the absolute path
+def use_setuptools(
+ version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+ to_dir=DEFAULT_SAVE_DIR, download_delay=15):
+ """
+ Ensure that a setuptools version is installed.
+
+ Return None. Raise SystemExit if the requested version
+ or later cannot be installed.
+ """
+ version = _resolve_version(version)
to_dir = os.path.abspath(to_dir)
- was_imported = 'pkg_resources' in sys.modules or \
- 'setuptools' in sys.modules
+
+ # prior to importing, capture the module state for
+ # representative modules.
+ rep_modules = 'pkg_resources', 'setuptools'
+ imported = set(sys.modules).intersection(rep_modules)
+
try:
import pkg_resources
- except ImportError:
- return _do_download(version, download_base, to_dir, download_delay)
- try:
pkg_resources.require("setuptools>=" + version)
+ # a suitable version is already installed
return
- except pkg_resources.VersionConflict:
- e = sys.exc_info()[1]
- if was_imported:
- sys.stderr.write(
- "The required version of setuptools (>=%s) is not available,\n"
- "and can't be installed while this script is running. Please\n"
- "install a more recent version first, using\n"
- "'easy_install -U setuptools'."
- "\n\n(Currently using %r)\n" % (version, e.args[0]))
- sys.exit(2)
- else:
- del pkg_resources, sys.modules['pkg_resources'] # reload ok
- return _do_download(version, download_base, to_dir,
- download_delay)
+ except ImportError:
+ # pkg_resources not available; setuptools is not installed; download
+ pass
except pkg_resources.DistributionNotFound:
- return _do_download(version, download_base, to_dir,
- download_delay)
+ # no version of setuptools was found; allow download
+ pass
+ except pkg_resources.VersionConflict as VC_err:
+ if imported:
+ _conflict_bail(VC_err, version)
+
+ # otherwise, unload pkg_resources to allow the downloaded version to
+ # take precedence.
+ del pkg_resources
+ _unload_pkg_resources()
+
+ return _do_download(version, download_base, to_dir, download_delay)
+
+
+def _conflict_bail(VC_err, version):
+ """
+ Setuptools was imported prior to invocation, so it is
+ unsafe to unload it. Bail out.
+ """
+ conflict_tmpl = textwrap.dedent("""
+ The required version of setuptools (>={version}) is not available,
+ and can't be installed while this script is running. Please
+ install a more recent version first, using
+ 'easy_install -U setuptools'.
+
+ (Currently using {VC_err.args[0]!r})
+ """)
+ msg = conflict_tmpl.format(**locals())
+ sys.stderr.write(msg)
+ sys.exit(2)
+
+
+def _unload_pkg_resources():
+ del_modules = [
+ name for name in sys.modules
+ if name.startswith('pkg_resources')
+ ]
+ for mod_name in del_modules:
+ del sys.modules[mod_name]
+
+
+def _clean_check(cmd, target):
+ """
+ Run the command to download target.
+
+ If the command fails, clean up before re-raising the error.
+ """
+ try:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError:
+ if os.access(target, os.F_OK):
+ os.unlink(target)
+ raise
+
def download_file_powershell(url, target):
"""
- Download the file at url to target using Powershell (which will validate
- trust). Raise an exception if the command cannot complete.
+ Download the file at url to target using Powershell.
+
+ Powershell will validate trust.
+ Raise an exception if the command cannot complete.
"""
target = os.path.abspath(target)
+ ps_cmd = (
+ "[System.Net.WebRequest]::DefaultWebProxy.Credentials = "
+ "[System.Net.CredentialCache]::DefaultCredentials; "
+ '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")'
+ % locals()
+ )
cmd = [
'powershell',
'-Command',
- "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(),
+ ps_cmd,
]
- subprocess.check_call(cmd)
+ _clean_check(cmd, target)
+
def has_powershell():
+ """Determine if Powershell is available."""
if platform.system() != 'Windows':
return False
cmd = ['powershell', '-Command', 'echo test']
- devnull = open(os.path.devnull, 'wb')
- try:
+ with open(os.path.devnull, 'wb') as devnull:
try:
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
- except:
+ except Exception:
return False
- finally:
- devnull.close()
return True
-
download_file_powershell.viable = has_powershell
+
def download_file_curl(url, target):
cmd = ['curl', url, '--silent', '--output', target]
- subprocess.check_call(cmd)
+ _clean_check(cmd, target)
+
def has_curl():
cmd = ['curl', '--version']
- devnull = open(os.path.devnull, 'wb')
- try:
+ with open(os.path.devnull, 'wb') as devnull:
try:
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
- except:
+ except Exception:
return False
- finally:
- devnull.close()
return True
-
download_file_curl.viable = has_curl
+
def download_file_wget(url, target):
cmd = ['wget', url, '--quiet', '--output-document', target]
- subprocess.check_call(cmd)
+ _clean_check(cmd, target)
+
def has_wget():
cmd = ['wget', '--version']
- devnull = open(os.path.devnull, 'wb')
- try:
+ with open(os.path.devnull, 'wb') as devnull:
try:
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
- except:
+ except Exception:
return False
- finally:
- devnull.close()
return True
-
download_file_wget.viable = has_wget
-def download_file_insecure(url, target):
- """
- Use Python to download the file, even though it cannot authenticate the
- connection.
- """
- try:
- from urllib.request import urlopen
- except ImportError:
- from urllib2 import urlopen
- src = dst = None
- try:
- src = urlopen(url)
- # Read/write all in one block, so we don't create a corrupt file
- # if the download is interrupted.
- data = src.read()
- dst = open(target, "wb")
- dst.write(data)
- finally:
- if src:
- src.close()
- if dst:
- dst.close()
+def download_file_insecure(url, target):
+ """Use Python to download the file, without connection authentication."""
+ src = urlopen(url)
+ try:
+ # Read all the data in one block.
+ data = src.read()
+ finally:
+ src.close()
+
+ # Write all the data in one block to avoid creating a partial file.
+ with open(target, "wb") as dst:
+ dst.write(data)
download_file_insecure.viable = lambda: True
+
def get_best_downloader():
- downloaders = [
+ downloaders = (
download_file_powershell,
download_file_curl,
download_file_wget,
download_file_insecure,
- ]
+ )
+ viable_downloaders = (dl for dl in downloaders if dl.viable())
+ return next(viable_downloaders, None)
- for dl in downloaders:
- if dl.viable():
- return dl
-def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
- to_dir=os.curdir, delay=15,
- downloader_factory=get_best_downloader):
- """Download setuptools from a specified location and return its filename
+def download_setuptools(
+ version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+ to_dir=DEFAULT_SAVE_DIR, delay=15,
+ downloader_factory=get_best_downloader):
+ """
+ Download setuptools from a specified location and return its filename.
`version` should be a valid setuptools version number that is available
- as an egg for download under the `download_base` URL (which should end
+ as an sdist for download under the `download_base` URL (which should end
with a '/'). `to_dir` is the directory where the egg will be downloaded.
`delay` is the number of seconds to pause before an actual download
attempt.
@@ -267,11 +324,12 @@
``downloader_factory`` should be a function taking no arguments and
returning a function for downloading a URL to a target.
"""
+ version = _resolve_version(version)
# making sure we use the absolute path
to_dir = os.path.abspath(to_dir)
- tgz_name = "setuptools-%s.tar.gz" % version
- url = download_base + tgz_name
- saveto = os.path.join(to_dir, tgz_name)
+ zip_name = "setuptools-%s.zip" % version
+ url = download_base + zip_name
+ saveto = os.path.join(to_dir, zip_name)
if not os.path.exists(saveto): # Avoid repeated downloads
log.warn("Downloading %s", url)
downloader = downloader_factory()
@@ -279,69 +337,37 @@
return os.path.realpath(saveto)
-def _extractall(self, path=".", members=None):
- """Extract all members from the archive to the current working
- directory and set owner, modification time and permissions on
- directories afterwards. `path' specifies a different directory
- to extract to. `members' is optional and must be a subset of the
- list returned by getmembers().
+def _resolve_version(version):
"""
- import copy
- import operator
- from tarfile import ExtractError
- directories = []
+ Resolve LATEST version
+ """
+ if version is not LATEST:
+ return version
- if members is None:
- members = self
-
- for tarinfo in members:
- if tarinfo.isdir():
- # Extract directories with a safe mode.
- directories.append(tarinfo)
- tarinfo = copy.copy(tarinfo)
- tarinfo.mode = 448 # decimal for oct 0700
- self.extract(tarinfo, path)
-
- # Reverse sort directories.
- if sys.version_info < (2, 4):
- def sorter(dir1, dir2):
- return cmp(dir1.name, dir2.name)
- directories.sort(sorter)
- directories.reverse()
- else:
- directories.sort(key=operator.attrgetter('name'), reverse=True)
-
- # Set correct owner, mtime and filemode on directories.
- for tarinfo in directories:
- dirpath = os.path.join(path, tarinfo.name)
+ resp = urlopen('https://pypi.python.org/pypi/setuptools/json')
+ with contextlib.closing(resp):
try:
- self.chown(tarinfo, dirpath)
- self.utime(tarinfo, dirpath)
- self.chmod(tarinfo, dirpath)
- except ExtractError:
- e = sys.exc_info()[1]
- if self.errorlevel > 1:
- raise
- else:
- self._dbg(1, "tarfile: %s" % e)
+ charset = resp.info().get_content_charset()
+ except Exception:
+ # Python 2 compat; assume UTF-8
+ charset = 'UTF-8'
+ reader = codecs.getreader(charset)
+ doc = json.load(reader(resp))
+
+ return str(doc['info']['version'])
def _build_install_args(options):
"""
- Build the arguments to 'python setup.py install' on the setuptools package
+ Build the arguments to 'python setup.py install' on the setuptools package.
+
+ Returns list of command line arguments.
"""
- install_args = []
- if options.user_install:
- if sys.version_info < (2, 6):
- log.warn("--user requires Python 2.6 or later")
- raise SystemExit(1)
- install_args.append('--user')
- return install_args
+ return ['--user'] if options.user_install else []
+
def _parse_args():
- """
- Parse the command line for options
- """
+ """Parse the command line for options."""
parser = optparse.OptionParser()
parser.add_option(
'--user', dest='user_install', action='store_true', default=False,
@@ -355,16 +381,35 @@
const=lambda: download_file_insecure, default=get_best_downloader,
help='Use internal, non-validating downloader'
)
+ parser.add_option(
+ '--version', help="Specify which version to download",
+ default=DEFAULT_VERSION,
+ )
+ parser.add_option(
+ '--to-dir',
+ help="Directory to save (and re-use) package",
+ default=DEFAULT_SAVE_DIR,
+ )
options, args = parser.parse_args()
# positional arguments are ignored
return options
-def main(version=DEFAULT_VERSION):
- """Install or upgrade setuptools and EasyInstall"""
+
+def _download_args(options):
+ """Return args for download_setuptools function from cmdline args."""
+ return dict(
+ version=options.version,
+ download_base=options.download_base,
+ downloader_factory=options.downloader_factory,
+ to_dir=options.to_dir,
+ )
+
+
+def main():
+ """Install or upgrade setuptools and EasyInstall."""
options = _parse_args()
- tarball = download_setuptools(download_base=options.download_base,
- downloader_factory=options.downloader_factory)
- return _install(tarball, _build_install_args(options))
+ archive = download_setuptools(**_download_args(options))
+ return _install(archive, _build_install_args(options))
if __name__ == '__main__':
sys.exit(main())
--
To view, visit https://gerrit.wikimedia.org/r/260908
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I08f02ba7f4c6d7ea6c32e3845024a5bd00f58b35
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>