jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] Modify the code to be Python 3 compatible
......................................................................
[FEAT] Modify the code to be Python 3 compatible
- Use UnicodeMixin instead of a separate __str__ method
- Replace __cmp__ with _cmpkey and ComparableMixin
- Use bytes.decode instead of unicode(,)
- Use BytesIO instead of StringIO if it's binary data
- Open binary files in binary mode
- Only decode/encode std* in Py2
- Always have no 'space' before \n in json strings (< Py3.4)
- Map basestring to (str,) in scripts (in Py3)
- Map xrange to range in casechecker (in Py3)
- Use future print_function if necessary
- Decode password file as UTF8
Change-Id: Ia7d416a578cc0fb52eaf1e2426244a693395738a
---
M pywikibot/__init__.py
M pywikibot/bot.py
M pywikibot/botirc.py
M pywikibot/data/wikidataquery.py
M pywikibot/login.py
M pywikibot/page.py
M pywikibot/site.py
M pywikibot/userinterfaces/terminal_interface_base.py
M scripts/casechecker.py
M scripts/category.py
M scripts/checkimages.py
M scripts/data_ingestion.py
M scripts/editarticle.py
M scripts/replace.py
M scripts/weblinkchecker.py
M tests/archivebot_tests.py
M tests/data_ingestion_tests.py
M tests/namespace_tests.py
M tests/script_tests.py
M tests/site_tests.py
M tests/wikibase_tests.py
21 files changed, 126 insertions(+), 86 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py
index a5b964d..5257f65 100644
--- a/pywikibot/__init__.py
+++ b/pywikibot/__init__.py
@@ -378,7 +378,8 @@
ts[u'calendarmodel'])
def __str__(self):
- return json.dumps(self.toWikibase(), indent=4, sort_keys=True)
+ return json.dumps(self.toWikibase(), indent=4, sort_keys=True,
+ separators=(',', ': '))
def __eq__(self, other):
return self.__dict__ == other.__dict__
@@ -446,7 +447,8 @@
return WbQuantity(amount, wb['unit'], error)
def __str__(self):
- return json.dumps(self.toWikibase(), indent=4, sort_keys=True)
+ return json.dumps(self.toWikibase(), indent=4, sort_keys=True,
+ separators=(',', ': '))
def __eq__(self, other):
return self.__dict__ == other.__dict__
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index 357bc40..2be8a2f 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -132,7 +132,7 @@
"""
strExc = logging.Formatter.formatException(self, ei)
- if isinstance(strExc, str):
+ if sys.version_info[0] < 3 and isinstance(strExc, str):
return strExc.decode(config.console_encoding) + '\n'
else:
return strExc + '\n'
@@ -757,7 +757,9 @@
''' % module_name
try:
module = __import__('%s' % module_name)
- helpText = module.__doc__.decode('utf-8')
+ helpText = module.__doc__
+ if sys.version_info[0] < 3:
+ helpText = helpText.decode('utf-8')
if hasattr(module, 'docuReplacements'):
for key, value in module.docuReplacements.items():
helpText = helpText.replace(key, value.strip('\n\r'))
diff --git a/pywikibot/botirc.py b/pywikibot/botirc.py
index 11a27a1..7ec9b99 100644
--- a/pywikibot/botirc.py
+++ b/pywikibot/botirc.py
@@ -74,7 +74,7 @@
if not ('N' in match.group('flags')):
return
try:
- msg = unicode(e.arguments()[0], 'utf-8')
+ msg = e.arguments()[0].decode('utf-8')
except UnicodeDecodeError:
return
if self.other_ns.match(msg):
diff --git a/pywikibot/data/wikidataquery.py b/pywikibot/data/wikidataquery.py
index 2794892..33f6f7d 100644
--- a/pywikibot/data/wikidataquery.py
+++ b/pywikibot/data/wikidataquery.py
@@ -464,7 +464,7 @@
"""
Encode a query into a unique and universally safe format.
"""
- encQuery = hashlib.sha1(queryStr).hexdigest() + ".wdq_cache"
+ encQuery = hashlib.sha1(queryStr.encode('utf8')).hexdigest() +
".wdq_cache"
return os.path.join(self.cacheDir, encQuery)
def readFromCache(self, queryStr):
diff --git a/pywikibot/login.py b/pywikibot/login.py
index feece17..2c864e6 100644
--- a/pywikibot/login.py
+++ b/pywikibot/login.py
@@ -9,6 +9,7 @@
#
__version__ = '$Id$'
#
+import codecs
import pywikibot
from pywikibot import config
@@ -146,11 +147,11 @@
(u"wikipedia", u"my_wikipedia_user",
u"my_wikipedia_pass")
(u"en", u"wikipedia", u"my_en_wikipedia_user",
u"my_en_wikipedia_pass")
"""
- password_f = open(config.password_file)
+ password_f = codecs.open(config.password_file, encoding='utf-8')
for line in password_f:
if not line.strip():
continue
- entry = eval(line.decode('utf-8'))
+ entry = eval(line)
if len(entry) == 4: # for userinfo included code and family
if entry[0] == self.site.code and \
entry[1] == self.site.family.name and \
diff --git a/pywikibot/page.py b/pywikibot/page.py
index 1248b72..39cb3ad 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -32,6 +32,7 @@
from urllib import urlopen
else:
unicode = basestring = str
+ long = int
from html import entities as htmlentitydefs
from urllib.parse import quote_from_bytes, unquote_to_bytes
from urllib.request import urlopen
@@ -2863,7 +2864,7 @@
if diffto and 'aliases' in diffto:
for lang in set(diffto['aliases'].keys()) - set(aliases.keys()):
aliases[lang] = []
- for lang, strings in aliases.items():
+ for lang, strings in list(aliases.items()):
if diffto and 'aliases' in diffto and lang in
diffto['aliases']:
empty = len(diffto['aliases'][lang]) - len(strings)
if empty > 0:
@@ -3622,7 +3623,7 @@
if len(self.qualifiers) > 0:
data['qualifiers'] = {}
data['qualifiers-order'] = list(self.qualifiers.keys())
- for prop, qualifiers in self.qualifiers.iteritems():
+ for prop, qualifiers in self.qualifiers.items():
for qualifier in qualifiers:
qualifier.isQualifier = True
data['qualifiers'][prop] = [qualifier.toJSON() for qualifier
in qualifiers]
@@ -3630,7 +3631,7 @@
data['references'] = []
for collection in self.sources:
reference = {'snaks': {}, 'snaks-order':
list(collection.keys())}
- for prop, val in collection.iteritems():
+ for prop, val in collection.items():
reference['snaks'][prop] = []
for source in val:
source.isReference = True
@@ -4237,9 +4238,14 @@
self.site.code,
title)
- def __str__(self):
- """Return a string representation."""
- return self.astext().encode("ascii", "backslashreplace")
+ if sys.version_info[0] > 2:
+ def __str__(self):
+ """Return a string representation."""
+ return self.__unicode__()
+ else:
+ def __str__(self):
+ """Return a string representation."""
+ return self.astext().encode("ascii", "backslashreplace")
def _cmpkey(self):
"""
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 20c1f03..4967aa0 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -29,7 +29,9 @@
import pywikibot
from pywikibot import config
from pywikibot.family import AutoFamily
-from pywikibot.tools import itergroup, deprecated, deprecate_arg
+from pywikibot.tools import (
+ itergroup, deprecated, deprecate_arg, UnicodeMixin, ComparableMixin,
+)
from pywikibot.throttle import Throttle
from pywikibot.data import api
from pywikibot.exceptions import (
@@ -146,7 +148,7 @@
return _families[fam]
-class Namespace(Iterable):
+class Namespace(Iterable, ComparableMixin, UnicodeMixin):
""" Namespace site data object.
@@ -306,26 +308,31 @@
else:
return self.aliases[index - 1]
- def __str__(self):
- """Return a string representation."""
- if sys.version_info[0] > 2:
- return self.__unicode__()
-
- if self.id == 0:
- return ':'
- elif self.id in (6, 14):
- return ':' + self.canonical_name + ':'
+ @staticmethod
+ def _colons(id, name):
+ """Return the name with required colons, depending on the
ID."""
+ if id == 0:
+ return u':'
+ elif id in (6, 14):
+ return u':' + name + u':'
else:
- return self.canonical_name + ':'
+ return u'' + name + u':'
+
+ def __str__(self):
+ """Return a the canonical string
representation."""
+ return self.canonical_prefix()
def __unicode__(self):
- """Return a unicode string representation."""
- if self.id == 0:
- return u':'
- elif self.id in (6, 14):
- return u':' + self.custom_name + u':'
- else:
- return u'' + self.custom_name + u':'
+ """Return a the custom string representation."""
+ return self.custom_prefix()
+
+ def canonical_prefix(self):
+ """Return the canonical name with required
colons."""
+ return Namespace._colons(self.id, self.canonical_name)
+
+ def custom_prefix(self):
+ """Return the custom name with required colons."""
+ return Namespace._colons(self.id, self.custom_name)
def __index__(self):
return self.id
@@ -348,14 +355,9 @@
else:
return True
- def __cmp__(self, other):
- """Compare two namespace ids."""
- if self.id == other.id:
- return 0
- elif self.id > other.id:
- return 1
- else:
- return -1
+ def _cmpkey(self):
+ """Return the ID as a comparison key."""
+ return self.id
def __repr__(self):
"""Return a reconstructable representation."""
@@ -425,7 +427,7 @@
return None
-class BaseSite(object):
+class BaseSite(ComparableMixin):
"""Site methods that are independent of the communication
interface."""
@@ -519,13 +521,9 @@
"""
return self.__code
- def __cmp__(self, other):
+ def _cmpkey(self):
"""Perform equality and inequality tests on Site
objects."""
- if not isinstance(other, BaseSite):
- return 1
- if self.family == other.family:
- return cmp(self.code, other.code)
- return cmp(self.family.name, other.family.name)
+ return (self.family.name, self.code)
def __getstate__(self):
""" Remove Lock based classes before pickling. """
diff --git a/pywikibot/userinterfaces/terminal_interface_base.py
b/pywikibot/userinterfaces/terminal_interface_base.py
index 03b51f8..89214a5 100755
--- a/pywikibot/userinterfaces/terminal_interface_base.py
+++ b/pywikibot/userinterfaces/terminal_interface_base.py
@@ -102,7 +102,9 @@
line, count = colorTagR.subn('', line)
if count > 0:
line += ' ***'
- targetStream.write(line.encode(self.encoding, 'replace'))
+ if sys.version_info[0] < 3:
+ line = line.encode(self.encoding, 'replace')
+ targetStream.write(line)
printColorized = printNonColorized
@@ -198,7 +200,8 @@
text = self._raw_input()
except KeyboardInterrupt:
raise pywikibot.QuitKeyboardInterrupt()
- text = unicode(text, self.encoding)
+ if sys.version_info[0] < 3:
+ text = text.decode(self.encoding)
return text
def inputChoice(self, question, options, hotkeys, default=None):
diff --git a/scripts/casechecker.py b/scripts/casechecker.py
index 75e55c6..7d0c4da 100644
--- a/scripts/casechecker.py
+++ b/scripts/casechecker.py
@@ -8,6 +8,7 @@
#
# Distributed under the terms of the MIT license.
#
+from __future__ import print_function
__version__ = '$Id$'
import os
@@ -17,6 +18,9 @@
import pywikibot
from pywikibot import i18n
from pywikibot.data import api
+
+if sys.version_info[0] > 2:
+ xrange = range
#
@@ -69,11 +73,11 @@
pass
if color == FOREGROUND_BLUE:
- print('(b:'),
+ print('(b:', end=' ')
if color == FOREGROUND_GREEN:
- print('(g:'),
+ print('(g:', end=' ')
if color == FOREGROUND_RED:
- print('(r:'),
+ print('(r:', end=' ')
# end of console code
diff --git a/scripts/category.py b/scripts/category.py
index 246d637..d5044a8 100755
--- a/scripts/category.py
+++ b/scripts/category.py
@@ -103,11 +103,16 @@
import re
import pickle
import bz2
+import sys
+
import pywikibot
from pywikibot import config, pagegenerators
from pywikibot import i18n, textlib
from pywikibot.tools import deprecate_arg, deprecated
+if sys.version_info[0] > 2:
+ basestring = (str, )
+
# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {
diff --git a/scripts/checkimages.py b/scripts/checkimages.py
index 8004226..c851382 100644
--- a/scripts/checkimages.py
+++ b/scripts/checkimages.py
@@ -95,10 +95,15 @@
import time
import datetime
import locale
+import sys
+
import pywikibot
from pywikibot import pagegenerators as pg
from pywikibot import config, i18n
+if sys.version_info[0] > 2:
+ basestring = (str, )
+
locale.setlocale(locale.LC_ALL, '')
###############################################################################
diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py
index e62ce32..70856eb 100755
--- a/scripts/data_ingestion.py
+++ b/scripts/data_ingestion.py
@@ -13,6 +13,7 @@
import hashlib
import base64
import sys
+import io
import pywikibot
# TODO: nosetests3 fails on 'import <other_script>', which is used by many
@@ -23,11 +24,9 @@
if sys.version_info[0] > 2:
from urllib.parse import urlparse
from urllib.request import urlopen
- import io as StringIO
else:
from urlparse import urlparse
from urllib import urlopen
- import StringIO
class Photo(object):
@@ -55,13 +54,13 @@
def downloadPhoto(self):
"""
- Download the photo and store it in a StringIO.StringIO object.
+ Download the photo and store it in a io.BytesIO object.
TODO: Add exception handling
"""
if not self.contents:
imageFile = urlopen(self.URL).read()
- self.contents = StringIO.StringIO(imageFile)
+ self.contents = io.BytesIO(imageFile)
return self.contents
def findDuplicateImages(self,
@@ -189,12 +188,12 @@
def downloadPhoto(self, photoUrl=''):
"""
- Download the photo and store it in a StrinIO.StringIO object.
+ Download the photo and store it in a io.BytesIO object.
TODO: Add exception handling
"""
imageFile = urlopen(photoUrl).read()
- return StringIO.StringIO(imageFile)
+ return io.BytesIO(imageFile)
def findDuplicateImages(self, photo=None, site=pywikibot.Site(u'commons',
u'commons')):
"""
diff --git a/scripts/editarticle.py b/scripts/editarticle.py
index 15b6eb2..74af631 100755
--- a/scripts/editarticle.py
+++ b/scripts/editarticle.py
@@ -19,7 +19,6 @@
#
import os
-import string
import optparse
import tempfile
@@ -30,7 +29,8 @@
class ArticleEditor(object):
# join lines if line starts with this ones
- joinchars = string.letters + '[]' + string.digits
+ # TODO: No apparent usage
+ # joinchars = string.letters + '[]' + string.digits
def __init__(self, *args):
self.set_options(*args)
diff --git a/scripts/replace.py b/scripts/replace.py
index 9d92a1e..5c9e750 100755
--- a/scripts/replace.py
+++ b/scripts/replace.py
@@ -125,14 +125,19 @@
import re
import time
+import webbrowser
+import sys
+
import pywikibot
from pywikibot import i18n, textlib, pagegenerators, Bot
from pywikibot import editor as editarticle
-import webbrowser
# Imports predefined replacements tasks from fixes.py
from pywikibot import fixes
+if sys.version_info[0] > 2:
+ basestring = (str, )
+
# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {
diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py
index 5d1158e..2a4d991 100644
--- a/scripts/weblinkchecker.py
+++ b/scripts/weblinkchecker.py
@@ -112,6 +112,7 @@
import urllib.parse as urlparse
import urllib.request as urllib
import http.client as httplib
+ basestring = (str, )
else:
import urlparse
import urllib
diff --git a/tests/archivebot_tests.py b/tests/archivebot_tests.py
index ba1a8ca..abe42aa 100644
--- a/tests/archivebot_tests.py
+++ b/tests/archivebot_tests.py
@@ -8,12 +8,16 @@
__version__ = '$Id$'
from datetime import datetime
+import sys
import pywikibot
import pywikibot.page
from pywikibot.textlib import TimeStripper
from scripts import archivebot
from tests.aspects import unittest, TestCase
+if sys.version_info[0] > 2:
+ basestring = (str,)
+
THREADS = {
'als': 4, 'ar': 1, 'bar': 0, 'bg': 0, 'bjn':
1, 'bs': 0, 'ca': 5, 'ckb': 2,
'cs': 0, 'de': 7, 'en': 25, 'eo': 1, 'es':
13, 'fa': 2, 'fr': 25, 'frr': 2,
diff --git a/tests/data_ingestion_tests.py b/tests/data_ingestion_tests.py
index 741ccef..f7f6711 100644
--- a/tests/data_ingestion_tests.py
+++ b/tests/data_ingestion_tests.py
@@ -25,8 +25,8 @@
)
def test_downloadPhoto(self):
- f = open(os.path.join(os.path.split(__file__)[0], 'data',
'MP_sounds.png'))
- self.assertEqual(f.read(), self.obj.downloadPhoto().read())
+ with open(os.path.join(os.path.split(__file__)[0], 'data',
'MP_sounds.png'), 'rb') as f:
+ self.assertEqual(f.read(), self.obj.downloadPhoto().read())
def test_findDuplicateImages(self):
duplicates = self.obj.findDuplicateImages()
diff --git a/tests/namespace_tests.py b/tests/namespace_tests.py
index 73930e3..5dbd47d 100644
--- a/tests/namespace_tests.py
+++ b/tests/namespace_tests.py
@@ -129,11 +129,11 @@
y = Namespace(id=6, custom_name=u'ملف', canonical_name=u'File',
aliases=[u'Image', u'Immagine'], **kwargs)
- if sys.version_info[0] == 2:
- self.assertEqual(str(y), ':File:')
+ self.assertEqual(str(y), ':File:')
+ if sys.version_info[0] <= 2:
self.assertEqual(unicode(y), u':ملف:')
- else:
- self.assertEqual(str(y), u':ملف:')
+ self.assertEqual(y.canonical_prefix(), ':File:')
+ self.assertEqual(y.custom_prefix(), u':ملف:')
def testNamespaceCompare(self):
a = Namespace(id=0, canonical_name=u'')
@@ -166,10 +166,6 @@
self.assertEqual(x, u'Image')
self.assertEqual(y, u'ملف')
-
- # FIXME: Namespace is missing operators required for py3
- if sys.version_info[0] > 2:
- return
self.assertLess(a, x)
self.assertGreater(x, a)
diff --git a/tests/script_tests.py b/tests/script_tests.py
index e2324c5..7b26ed2 100644
--- a/tests/script_tests.py
+++ b/tests/script_tests.py
@@ -127,7 +127,6 @@
'login': 'Logged in on ',
'pagefromfile': 'Please enter the file name',
'replace': 'Press Enter to use this default message',
- 'replicate_wiki': 'error: too few arguments',
'script_wui': 'Pre-loading all relevant page contents',
'shell': 'Welcome to the',
'spamremove': 'No spam site specified',
@@ -141,6 +140,11 @@
'revertbot': 'Fetching new batch of contributions',
'upload': 'ERROR: Upload error',
}
+
+if sys.version_info[0] > 2:
+ no_args_expected_results['replicate_wiki'] = 'error: the following
arguments are required: destination'
+else:
+ no_args_expected_results['replicate_wiki'] = 'error: too few
arguments'
def collector(loader=unittest.loader.defaultTestLoader):
@@ -184,6 +188,11 @@
def execute(command, data_in=None, timeout=0):
"""Execute a command and capture outputs."""
+ def decode(stream):
+ if sys.version_info[0] > 2:
+ return stream.decode(config.console_encoding)
+ else:
+ return stream
options = {
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE
@@ -193,7 +202,10 @@
p = subprocess.Popen(command, **options)
if data_in is not None:
+ if sys.version_info[0] > 2:
+ data_in = data_in.encode(config.console_encoding)
p.stdin.write(data_in)
+ p.stdin.flush() # _communicate() otherwise has a broken pipe
waited = 0
while waited < timeout and p.poll() is None:
time.sleep(1)
@@ -202,8 +214,8 @@
p.kill()
data_out = p.communicate()
return {'exit_code': p.returncode,
- 'stdout': data_out[0],
- 'stderr': data_out[1]}
+ 'stdout': decode(data_out[0]),
+ 'stderr': decode(data_out[1])}
class TestScriptMeta(MetaTestCaseClass):
diff --git a/tests/site_tests.py b/tests/site_tests.py
index b2fb360..3954141 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -951,10 +951,6 @@
self.assertLessEqual(len(dr['revisions']), 10)
self.assertTrue(all(isinstance(rev, dict)
for rev in dr['revisions']))
- dr2 = list(mysite.deletedrevs(page=mainpage, total=10))[0]
- self.assertLessEqual(len(dr2['revisions']), 10)
- self.assertTrue(all(isinstance(rev, dict)
- for rev in dr2['revisions']))
for item in mysite.deletedrevs(start="2008-10-11T01:02:03Z",
page=mainpage, total=5):
for rev in item['revisions']:
@@ -1303,7 +1299,8 @@
self.assertEqual(image_namespace.custom_name, 'Fil')
self.assertEqual(image_namespace.canonical_name, 'File')
self.assertEqual(str(image_namespace), ':File:')
- self.assertEqual(unicode(image_namespace), ':Fil:')
+ self.assertEqual(image_namespace.custom_prefix(), ':Fil:')
+ self.assertEqual(image_namespace.canonical_prefix(), ':File:')
self.assertEqual(image_namespace.aliases, ['Image'])
self.assertEqual(len(image_namespace), 3)
diff --git a/tests/wikibase_tests.py b/tests/wikibase_tests.py
index a4d7b6b..b288484 100644
--- a/tests/wikibase_tests.py
+++ b/tests/wikibase_tests.py
@@ -79,15 +79,15 @@
{'amount': 5, 'lowerBound': 2,
'upperBound': 7,
'unit': '1', })
q = pywikibot.WbQuantity(amount=0.044405586)
- self.assertEqual(q.toWikibase(),
- {'amount': 0.044405586, 'lowerBound':
0.044405586,
- 'upperBound': 0.044405586, 'unit': '1',
})
+ q_dict = {'amount': 0.044405586, 'lowerBound': 0.044405586,
+ 'upperBound': 0.044405586, 'unit': '1', }
+ self.assertEqual(q.toWikibase(), q_dict)
# test other WbQuantity methods
self.assertEqual("%s" % q,
'{\n'
- ' "amount": %(val)r, \n'
- ' "lowerBound": %(val)r, \n'
- ' "unit": "1", \n'
+ ' "amount": %(val)r,\n'
+ ' "lowerBound": %(val)r,\n'
+ ' "unit": "1",\n'
' "upperBound": %(val)r\n'
'}' % {'val': 0.044405586})
self.assertEqual("%r" % q,
--
To view, visit
https://gerrit.wikimedia.org/r/160273
To unsubscribe, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia7d416a578cc0fb52eaf1e2426244a693395738a
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>