Pywikibot-commits September 2015

pywikibot-commits@lists.wikimedia.org

2 participants
243 discussions

[Gerrit] [FIX] script_tests: Remove weblinkchecker dependency - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [FIX] script_tests: Remove weblinkchecker dependency ...................................................................... [FIX] script_tests: Remove weblinkchecker dependency When the `memento_client` dependency for `weblinkchecker` was introduced in 3bf9bfac it was added to the script tests. But the script runs fine without it when it's not installed and causes an unexpected success. But this didn't cause a problem because the script included the file name extension so it wasn't expecting a failure as it was expecting it from a script with an additional `.py` extension. Change-Id: I8510cfa2553d5d5ad7e04c633b60b0a7ed9687be --- M tests/script_tests.py 1 file changed, 0 insertions(+), 1 deletion(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/script_tests.py b/tests/script_tests.py index 3756ddd..10cd8d1 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -45,7 +45,6 @@ 'panoramiopicker': ['BeautifulSoup'], 'states_redirect': ['pycountry'], 'patrol': ['mwlib'], - 'weblinkchecker.py': ['memento_client'], } if PYTHON_VERSION < (2, 7): -- To view, visit https://gerrit.wikimedia.org/r/241303 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I8510cfa2553d5d5ad7e04c633b60b0a7ed9687be Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] [FEAT] patrol: Basic test of PatrolBot class - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [FEAT] patrol: Basic test of PatrolBot class ...................................................................... [FEAT] patrol: Basic test of PatrolBot class This makes the `PatrolBot` class a `SingleSiteBot` class and allows to set a site via the constructor. It also adds basic tests for `PatrolBot.parse_page_tuples` and `PatrolBot.in_list`. Change-Id: I2cfb34a10e40e84634cd1ee15c5879741abe26d9 --- M scripts/patrol.py A tests/patrolbot_tests.py 2 files changed, 86 insertions(+), 6 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/patrol.py b/scripts/patrol.py index 5cc61e0..e37907d 100755 --- a/scripts/patrol.py +++ b/scripts/patrol.py @@ -25,11 +25,14 @@ from __future__ import absolute_import, unicode_literals __version__ = '$Id$' -import pywikibot -from pywikibot import pagegenerators, Bot import mwlib.uparser # used to parse the whitelist import mwlib.parser # used to parse the whitelist import time + +import pywikibot + +from pywikibot import pagegenerators +from pywikibot.bot import SingleSiteBot _logger = 'patrol' @@ -40,7 +43,7 @@ } -class PatrolBot(Bot): +class PatrolBot(SingleSiteBot): """Bot marks the edits as patrolled based on info obtained by whitelist.""" @@ -49,7 +52,7 @@ 'en': u'patrol_whitelist', } - def __init__(self, **kwargs): + def __init__(self, site=True, **kwargs): """ Constructor. @@ -67,10 +70,9 @@ 'versionchecktime': 300, 'autopatroluserns': False }) - super(PatrolBot, self).__init__(**kwargs) + super(PatrolBot, self).__init__(site, **kwargs) self.recent_gen = True self.user = None - self.site = pywikibot.Site() if self.getOption('whitelist'): self.whitelist_pagename = self.getOption('whitelist') else: diff --git a/tests/patrolbot_tests.py b/tests/patrolbot_tests.py new file mode 100644 index 0000000..961d637 --- /dev/null +++ b/tests/patrolbot_tests.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +"""Tests for the patrol script.""" +# +# (C) Pywikibot team, 2015 +# +# Distributed under the terms of the MIT license. +# +from __future__ import absolute_import, unicode_literals + +__version__ = '$Id$' +# +try: + from scripts import patrol +except ImportError: + patrol = None # if mwlib is not installed + +from tests.aspects import require_modules, unittest, DefaultDrySiteTestCase + +DUMMY_PAGE_TUPLES = """ +This is some text above the entries: + +== Header == +* [[User:Test 1]]: [[Page 1]], [[Page 2]] +* [[User:Test 2]]: [[Page 2]], [[Page 4]], [[Page 6]] + +== Others == +* [[User:Prefixed]]: [[Special:PrefixIndex/Page 1]], [[Special:PREFIXINDEX/Page 2]] + +== More test 1 == +* [[User:Test 1]]: [[Page 3]] +""" + + +@require_modules('mwlib') +class TestPatrolBot(DefaultDrySiteTestCase): + + """Test the PatrolBot class.""" + + def setUp(self): + """Create a bot dummy instance.""" + super(TestPatrolBot, self).setUp() + self.bot = patrol.PatrolBot(self.site) + + def test_parse_page_tuples(self): + """Test parsing the page tuples from a dummy text.""" + tuples = self.bot.parse_page_tuples(DUMMY_PAGE_TUPLES) + for gen_user in (1, 2): + user = 'Test {0}'.format(gen_user) + self.assertIn(user, tuples) + self.assertEqual(tuples[user], ['Page {0}'.format(i * gen_user) + for i in range(1, 4)]) + self.assertIn('Prefixed', tuples) + self.assertEqual(tuples['Prefixed'], ['Page 1', 'Page 2']) + self.assertRaises(Exception, self.bot.parse_page_tuples, '[[link]]') + + def test_in_list(self): + """Test the method which returns whether a page is in the list.""" + # Return the title if there is an exact match + self.assertEqual(self.bot.in_list(['Foo', 'Foobar'], 'Foo'), 'Foo') + self.assertEqual(self.bot.in_list(['Foo', 'Foobar'], 'Foobar'), 'Foobar') + + # Return the first entry which starts with the title if there is no + # exact match + self.assertEqual(self.bot.in_list(['Foo', 'Foobar'], 'Foob'), 'Foo') + self.assertEqual(self.bot.in_list(['Foo', 'Foobar'], 'Foobarz'), 'Foo') + self.assertEqual(self.bot.in_list(['Foo', 'Foobar', 'Bar'], 'Barz'), 'Bar') + + # '' returns .* if there is no exact match + self.assertEqual(self.bot.in_list([''], 'Foo'), '.*') + self.assertEqual(self.bot.in_list(['', 'Foobar'], 'Foo'), '.*') + self.assertEqual(self.bot.in_list(['', 'Foo'], 'Foo'), 'Foo') + + +if __name__ == '__main__': + try: + unittest.main() + except SystemExit: + pass -- To view, visit https://gerrit.wikimedia.org/r/241022 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I2cfb34a10e40e84634cd1ee15c5879741abe26d9 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] Make consistent use of "Bot" - change (pywikibot/i18n)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Make consistent use of "Bot" ...................................................................... Make consistent use of "Bot" These files contains "Robot" in the edit summaries, someone in irc said it should be Bot, therefore changing it. Also, convert 4 spaces in checkimages/en.json to tab. Change-Id: I1bfa034e6241fc4b3939aaaecd233105a68b5237 --- M basic/en.json M checkimages/en.json M piper/en.json M pywikibot/en.json M remove_edp_images/en.json 5 files changed, 24 insertions(+), 16 deletions(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve Siebrand: Looks good to me, approved jenkins-bot: Verified diff --git a/basic/en.json b/basic/en.json index 225de00..1d3f6c3 100644 --- a/basic/en.json +++ b/basic/en.json @@ -1,4 +1,8 @@ { - "@metadata": [], - "basic-changing": "Robot: Changing ..." + "@metadata": { + "authors": [ + "Revi" + ] + }, + "basic-changing": "Bot: Changing ..." } diff --git a/checkimages/en.json b/checkimages/en.json index 61d797e..f143c18 100644 --- a/checkimages/en.json +++ b/checkimages/en.json @@ -1,9 +1,10 @@ { - "@metadata": { - "authors": [ - "Filnik" - ] - }, + "@metadata": { + "authors": [ + "Filnik", + "Revi" + ] + }, "checkimages-source-tag-comment": "Bot: Marking newly uploaded untagged file", - "checkimages-source-notice-comment": "Robot: Requesting source information." + "checkimages-source-notice-comment": "Bot: Requesting source information." } diff --git a/piper/en.json b/piper/en.json index c42f3b9..5bf2a77 100644 --- a/piper/en.json +++ b/piper/en.json @@ -1,8 +1,9 @@ { "@metadata": { "authors": [ - "Ævar Arnfjörð Bjarmason" + "Ævar Arnfjörð Bjarmason", + "Revi" ] }, - "piper-edit-summary": "Robot: Piping the article text through %(filters)s" + "piper-edit-summary": "Bot: Piping the article text through %(filters)s" } diff --git a/pywikibot/en.json b/pywikibot/en.json index 0543378..89d9255 100644 --- a/pywikibot/en.json +++ b/pywikibot/en.json @@ -3,7 +3,8 @@ "authors": [ "Daniel Herding", "Huji", - "Siebrand" + "Siebrand", + "Revi" ] }, "pywikibot-enter-category-name": "Please enter the category name:", @@ -13,7 +14,7 @@ "pywikibot-enter-new-text": "Please enter the new text:", "pywikibot-enter-page-processing": "Which page should be processed?", "pywikibot-enter-xml-filename": "Please enter the XML dump's filename:", - "pywikibot-fixes-fckeditor": "Robot: Fixing rich-editor html", - "pywikibot-fixes-html": "Robot: Converting/fixing HTML", - "pywikibot-fixes-syntax": "Robot: Fixing wiki syntax" + "pywikibot-fixes-fckeditor": "Bot: Fixing rich-editor html", + "pywikibot-fixes-html": "Bot: Converting/fixing HTML", + "pywikibot-fixes-syntax": "Bot: Fixing wiki syntax" } diff --git a/remove_edp_images/en.json b/remove_edp_images/en.json index e334114..33821ab 100644 --- a/remove_edp_images/en.json +++ b/remove_edp_images/en.json @@ -1,8 +1,9 @@ { "@metadata": { "authors": [ - "Shizhao" + "Shizhao", + "Revi" ] }, - "remove_edp_images-edit-summary": "Robot: Fix EDP image use: [[%(title)s]]" + "remove_edp_images-edit-summary": "Bot: Fix EDP image use: [[%(title)s]]" } -- To view, visit https://gerrit.wikimedia.org/r/241267 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1bfa034e6241fc4b3939aaaecd233105a68b5237 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/i18n Gerrit-Branch: master Gerrit-Owner: Revi <reviwiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Siebrand <siebrand(a)kitano.nl> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] [IMPROV] Don't limit cryptography module on 2.6 - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [IMPROV] Don't limit cryptography module on 2.6 ...................................................................... [IMPROV] Don't limit cryptography module on 2.6 The cryptography module does complain on Python 2.6 that it's outdated. So it's using the newest version which doesn't issue that warning. But that could compromise the security as it won't include security patches. This is now not limiting that version anymore and installing the newest version on Python 2.6 as well. By default the deprecation warning is hidden but if the user still wishes to suppress this warning they can install version 0.8.2 or before. Bug: T108061 Change-Id: If5270a0b2214c2f3b124d50180a122bfeeadc3cb --- M requests-requirements.txt M requirements.txt M setup.py M tests/utils.py 4 files changed, 16 insertions(+), 13 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/requests-requirements.txt b/requests-requirements.txt index 2923433..778c9f6 100644 --- a/requests-requirements.txt +++ b/requests-requirements.txt @@ -1,9 +1,5 @@ requests -# Dependency of pyOpenSSL. Use specific version to avoid expected -# DeprecationWarning -cryptography<=0.8.2 ; python_version < '2.7' - # requests security extra # Bug T105767 on Python 2.7 release 9+ pyOpenSSL ; python_full_version < '2.7.9' or python_version >= '3' diff --git a/requirements.txt b/requirements.txt index c7db758..cad79bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,10 +21,6 @@ # mandatory; see README.conversion.txt requests -# Dependency of pyOpenSSL. Use specific version to avoid expected -# DeprecationWarning -cryptography<=0.8.2 ; python_version < '2.7' - # requests security extra pyOpenSSL ; python_full_version < '2.7.9' ndg-httpsclient ; python_full_version < '2.7.9' diff --git a/setup.py b/setup.py index 2a2d63f..6a120b1 100644 --- a/setup.py +++ b/setup.py @@ -107,11 +107,6 @@ if sys.version_info[0] == 2: if PY26: - # requests security extra includes pyOpenSSL. cryptography is the - # dependency of pyOpenSSL. 0.8.2 is the newest and compatible version - # for Python 2.6, which won't raise unexpected DeprecationWarning. - extra_deps['security'].append('cryptography<=0.8.2') - script_deps['replicate_wiki.py'] = ['argparse'] dependencies.append('future>=0.15.0') # provides collections backports diff --git a/tests/utils.py b/tests/utils.py index 654b91b..69f2fae 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,6 +44,9 @@ OSWIN32 = (sys.platform == 'win32') +PYTHON_26_CRYPTO_WARN = ('Python 2.6 is no longer supported by the Python core ' + 'team, please upgrade your Python.') + class DrySiteNote(RuntimeWarning): @@ -231,6 +234,12 @@ if PYTHON_VERSION >= (3, 5, 0): if str(entry.message) == ('inspect.getargspec() is deprecated, ' 'use inspect.signature() instead'): + return + # Avoid failures because cryptography is mentioning Python 2.6 + # is outdated + if PYTHON_VERSION < (2, 7): + if (isinstance(entry, DeprecationWarning) and + str(entry.message) == PYTHON_26_CRYPTO_WARN): return log.append(entry) @@ -557,9 +566,16 @@ """ Execute a command and capture outputs. + On Python 2.6 it adds an option to ignore the deprecation warning from + the cryptography package after the first entry of the command parameter. + @param command: executable to run and arguments to use @type command: list of unicode """ + if PYTHON_VERSION < (2, 7): + command.insert( + 1, '-W ignore:{0}:DeprecationWarning'.format(PYTHON_26_CRYPTO_WARN)) + # Any environment variables added on Windows must be of type # str() on Python 2. env = os.environ.copy() -- To view, visit https://gerrit.wikimedia.org/r/241251 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: If5270a0b2214c2f3b124d50180a122bfeeadc3cb Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] [FEAT] Test Python 3.5 on Travis - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [FEAT] Test Python 3.5 on Travis ...................................................................... [FEAT] Test Python 3.5 on Travis As Python 3.5 has been officially released we should test on this. There are minor issues with deprecations as `inspect.getargspec` has been deprecated. But apart from a deprecation message it still works and will only be broken with Python 3.6. Bug: T106209 Change-Id: Ic8256184d1c1dbf841def5ad3846155564e05b75 --- M .travis.yml M tests/utils.py 2 files changed, 7 insertions(+), 0 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/.travis.yml b/.travis.yml index 01b04ad..eaca16b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ - '2.7' - '3.3' - '3.4' + - '3.5' - '2.6' # OSX builds do not yet support Python diff --git a/tests/utils.py b/tests/utils.py index fb2fafb..654b91b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -227,6 +227,12 @@ else: skip_lines -= 1 + # Avoid failures because getargspec hasn't been removed yet: T106209 + if PYTHON_VERSION >= (3, 5, 0): + if str(entry.message) == ('inspect.getargspec() is deprecated, ' + 'use inspect.signature() instead'): + return + log.append(entry) log = super(WarningSourceSkipContextManager, self).__enter__() -- To view, visit https://gerrit.wikimedia.org/r/241147 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic8256184d1c1dbf841def5ad3846155564e05b75 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] [FEAT] tests: Use generic function to get path - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: [FEAT] tests: Use generic function to get path ...................................................................... [FEAT] tests: Use generic function to get path To access tests file there are various constants like `_data_dir`. But in most cases `os.path.join` is used on them so it's more sensible to supply a function which takes the filename and returns the combined result. Except for the `_root_dir` all instances of `_*_dir` have been replaced by their `join_*_path` counterpart. It also adds `join_pages_path` for `tests/pages/`. Change-Id: I409413a9a480f9bd149cb72a365e76cde7d8e3e7 --- M tests/__init__.py M tests/cache_tests.py M tests/data_ingestion_tests.py M tests/djvu_tests.py M tests/dry_api_tests.py M tests/fixes_tests.py M tests/http_tests.py M tests/pagegenerators_tests.py M tests/pwb_tests.py M tests/reflinks_tests.py M tests/replacebot_tests.py M tests/script_tests.py M tests/tools_tests.py M tests/upload_tests.py M tests/uploadbot_tests.py M tests/wikibase_tests.py M tests/xmlreader_tests.py 17 files changed, 92 insertions(+), 98 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/__init__.py b/tests/__init__.py index 290914d..4416e05 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -9,10 +9,11 @@ __version__ = '$Id$' +import functools import os import warnings -__all__ = ('requests', '_cache_dir', 'TestRequest', +__all__ = ('requests', 'TestRequest', 'patch_request', 'unpatch_request') # Verify that the unit tests have a base working environment: @@ -39,14 +40,33 @@ from pywikibot.data.api import Request as _original_Request from pywikibot.data.api import CachedRequest -_tests_dir = os.path.split(__file__)[0] -_cache_dir = os.path.join(_tests_dir, 'apicache') -_data_dir = os.path.join(_tests_dir, 'data') -_images_dir = os.path.join(_data_dir, 'images') +_root_dir = os.path.split(os.path.split(__file__)[0])[0] + + +def join_root_path(*names): + """Return a path relative to the root directory.""" + return os.path.join(_root_dir, *names) + + +def create_path_func(base_func, subpath): + """Return a function returning a path relative to the given directory.""" + func = functools.partial(base_func, subpath) + func.path = base_func.path + '/' + subpath + func.__doc__ = 'Return a path relative to `{0}/`.'.format(func.path) + return func + + +join_root_path.path = 'root' +join_tests_path = create_path_func(join_root_path, 'tests') +join_cache_path = create_path_func(join_tests_path, 'apicache') +join_data_path = create_path_func(join_tests_path, 'data') +join_pages_path = create_path_func(join_tests_path, 'pages') + +join_images_path = create_path_func(join_data_path, 'images') +join_xml_data_path = create_path_func(join_data_path, 'xml') # Find the root directory of the checkout -_root_dir = os.path.split(_tests_dir)[0] -_pwb_py = os.path.join(_root_dir, 'pwb.py') +_pwb_py = join_root_path('pwb.py') library_test_modules = [ 'python', @@ -141,7 +161,7 @@ def _unknown_test_modules(): """List tests which are to be executed.""" - dir_list = os.listdir(_tests_dir) + dir_list = os.listdir(join_tests_path()) all_test_list = [name[0:-9] for name in dir_list # strip '_tests.py' if name.endswith('_tests.py') and not name.startswith('_')] # skip __init__.py and _* @@ -219,7 +239,7 @@ CachedRequest._get_cache_dir = classmethod( - lambda cls, *args: cls._make_dir(_cache_dir)) + lambda cls, *args: cls._make_dir(join_cache_path())) # Travis-CI builds are set to retry twice, which aims to reduce the number diff --git a/tests/cache_tests.py b/tests/cache_tests.py index 70add5c..e4161f8 100644 --- a/tests/cache_tests.py +++ b/tests/cache_tests.py @@ -13,7 +13,7 @@ from pywikibot.site import BaseSite import scripts.maintenance.cache as cache -from tests import _cache_dir +from tests import join_cache_path from tests.aspects import unittest, TestCase @@ -37,7 +37,7 @@ def test_cache(self): """Test the apicache by doing _check_cache_entry over each entry.""" - cache.process_entries(_cache_dir, self._check_cache_entry) + cache.process_entries(join_cache_path(), self._check_cache_entry) if __name__ == '__main__': diff --git a/tests/data_ingestion_tests.py b/tests/data_ingestion_tests.py index ee745a9..16c3ea5 100644 --- a/tests/data_ingestion_tests.py +++ b/tests/data_ingestion_tests.py @@ -10,9 +10,7 @@ __version__ = '$Id$' -import os -from tests import _data_dir -from tests import _images_dir +from tests import join_data_path, join_images_path from tests.aspects import unittest, TestCase, ScriptMainTestCase from scripts import data_ingestion @@ -46,7 +44,7 @@ def test_downloadPhoto(self): """Test download from http://upload.wikimedia.org/.""" - with open(os.path.join(_images_dir, 'MP_sounds.png'), 'rb') as f: + with open(join_images_path('MP_sounds.png'), 'rb') as f: self.assertEqual(f.read(), self.obj.downloadPhoto().read()) def test_findDuplicateImages(self): @@ -82,7 +80,7 @@ def setUp(self): """Set up unit test.""" super(TestCSVReader, self).setUp() - with open(os.path.join(_data_dir, 'csv_ingestion.csv')) as fileobj: + with open(join_data_path('csv_ingestion.csv')) as fileobj: self.iterator = data_ingestion.CSVReader(fileobj, 'url', site=self.get_site()) self.obj = next(self.iterator) diff --git a/tests/djvu_tests.py b/tests/djvu_tests.py index cbb3eb6..5d8da3e 100644 --- a/tests/djvu_tests.py +++ b/tests/djvu_tests.py @@ -10,14 +10,13 @@ from __future__ import absolute_import, unicode_literals -import os import subprocess -from tests import _data_dir +from tests import join_data_path, create_path_func from tests.aspects import unittest, TestCase from pywikibot.tools.djvu import DjVuFile -_djvu_dir = 'djvu' +join_djvu_data_path = create_path_func(join_data_path, 'djvu') class TestDjVuFile(TestCase): @@ -26,9 +25,9 @@ net = False - file_djvu_not_existing = os.path.join(_data_dir, _djvu_dir, 'not_existing.djvu') - file_djvu = os.path.join(_data_dir, _djvu_dir, 'myfile.djvu') - file_djvu_wo_text = os.path.join(_data_dir, _djvu_dir, 'myfile_wo_text.djvu') + file_djvu_not_existing = join_djvu_data_path('not_existing.djvu') + file_djvu = join_djvu_data_path('myfile.djvu') + file_djvu_wo_text = join_djvu_data_path('myfile_wo_text.djvu') test_txt = 'A file with non-ASCII characters, \nlike é or ç' @classmethod diff --git a/tests/dry_api_tests.py b/tests/dry_api_tests.py index 400a6f5..1831672 100644 --- a/tests/dry_api_tests.py +++ b/tests/dry_api_tests.py @@ -10,7 +10,6 @@ __version__ = '$Id$' # -import os import datetime import pywikibot @@ -22,7 +21,7 @@ ) from pywikibot.family import Family -from tests import _images_dir +from tests import join_images_path from tests.utils import DummySiteinfo from tests.aspects import ( unittest, TestCase, DefaultDrySiteTestCase, SiteAttributeTestCase, @@ -272,7 +271,7 @@ def test_mime_file_payload(self): """Test Request._generate_MIME_part loads binary as binary.""" - local_filename = os.path.join(_images_dir, 'MP_sounds.png') + local_filename = join_images_path('MP_sounds.png') with open(local_filename, 'rb') as f: file_content = f.read() submsg = Request._generate_MIME_part( @@ -282,7 +281,7 @@ def test_mime_file_container(self): """Test Request._build_mime_request encodes binary.""" - local_filename = os.path.join(_images_dir, 'MP_sounds.png') + local_filename = join_images_path('MP_sounds.png') with open(local_filename, 'rb') as f: file_content = f.read() body = Request._build_mime_request({}, { @@ -304,7 +303,7 @@ site._userinfo = {'name': 'myusername', 'groups': []} req = Request(site=site, action="upload", file='MP_sounds.png', mime=True, - filename=os.path.join(_images_dir, 'MP_sounds.png')) + filename=join_images_path('MP_sounds.png')) self.assertEqual(req.mime, True) diff --git a/tests/fixes_tests.py b/tests/fixes_tests.py index a7eda6d..26650ae 100644 --- a/tests/fixes_tests.py +++ b/tests/fixes_tests.py @@ -8,12 +8,10 @@ from __future__ import absolute_import, unicode_literals __version__ = '$Id$' -# -import os from pywikibot import fixes -from tests import unittest, _data_dir +from tests import unittest, join_data_path from tests.aspects import TestCase @@ -37,14 +35,14 @@ """Test loading a fix file overwriting the fixes.""" fixes.fixes = {} old_fixes = fixes.fixes - fixes._load_file(os.path.join(_data_dir, 'set-fixes.py')) + fixes._load_file(join_data_path('set-fixes.py')) self.assertIsNot(fixes.fixes, old_fixes) def test_update_value(self): """Test loading a fix file changing the fixes.""" fixes.fixes = {} old_fixes = fixes.fixes - fixes._load_file(os.path.join(_data_dir, 'fixes.py')) + fixes._load_file(join_data_path('fixes.py')) self.assertIs(fixes.fixes, old_fixes) diff --git a/tests/http_tests.py b/tests/http_tests.py index b912376..03a5092 100644 --- a/tests/http_tests.py +++ b/tests/http_tests.py @@ -9,7 +9,6 @@ __version__ = '$Id$' -import os import re import warnings @@ -24,7 +23,7 @@ UnicodeType as unicode, ) -from tests import _images_dir +from tests import join_images_path from tests.aspects import unittest, TestCase, DeprecationTestCase @@ -371,7 +370,7 @@ """Set up test class.""" super(BinaryTestCase, cls).setUpClass() - with open(os.path.join(_images_dir, 'MP_sounds.png'), 'rb') as f: + with open(join_images_path('MP_sounds.png'), 'rb') as f: cls.png = f.read() def test_requests(self): diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index 10181a4..1e8cd20 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -12,7 +12,6 @@ import calendar import datetime import json -import os import sys from distutils.version import LooseVersion @@ -25,7 +24,7 @@ PreloadingGenerator, ) -from tests import _data_dir +from tests import join_data_path from tests.aspects import ( unittest, TestCase, @@ -275,7 +274,7 @@ ) def test_brackets(self): - filename = os.path.join(_data_dir, 'pagelist-brackets.txt') + filename = join_data_path('pagelist-brackets.txt') site = self.get_site() titles = list(pagegenerators.TextfilePageGenerator(filename, site)) self.assertEqual(len(titles), len(self.expected_titles)) @@ -285,7 +284,7 @@ self.assertPageTitlesEqual(titles, expected_titles) def test_lines(self): - filename = os.path.join(_data_dir, 'pagelist-lines.txt') + filename = join_data_path('pagelist-lines.txt') site = self.get_site() titles = list(pagegenerators.TextfilePageGenerator(filename, site)) self.assertEqual(len(titles), len(self.expected_titles)) diff --git a/tests/pwb_tests.py b/tests/pwb_tests.py index 84718e3..94535f3 100644 --- a/tests/pwb_tests.py +++ b/tests/pwb_tests.py @@ -15,14 +15,13 @@ __version__ = '$Id$' -import os import sys -from tests import _tests_dir +from tests import join_tests_path, create_path_func from tests.utils import execute, execute_pwb from tests.aspects import unittest, PwbTestCase -_pwb_tests_dir = os.path.join(_tests_dir, 'pwb') +join_pwb_tests_path = create_path_func(join_tests_path, 'pwb') class TestPwb(PwbTestCase): @@ -42,7 +41,7 @@ def _do_check(self, name): package_name = 'tests.pwb.' + name - script_path = os.path.join(_pwb_tests_dir, name + '.py') + script_path = join_pwb_tests_path(name + '.py') direct = execute([sys.executable, '-m', package_name]) vpwb = execute_pwb([script_path]) diff --git a/tests/reflinks_tests.py b/tests/reflinks_tests.py index 27023d0..603371f 100644 --- a/tests/reflinks_tests.py +++ b/tests/reflinks_tests.py @@ -9,14 +9,10 @@ __version__ = '$Id$' -import os - from scripts.reflinks import XmlDumpPageGenerator, ReferencesRobot, main -from tests import _data_dir +from tests import join_xml_data_path from tests.aspects import unittest, TestCase, ScriptMainTestCase - -_xml_data_dir = os.path.join(_data_dir, 'xml') class TestXMLPageGenerator(TestCase): @@ -31,7 +27,7 @@ def test_non_bare_ref_urls(self): """Test pages without bare references are not processed.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'article-pear-0.10.xml'), + xmlFilename=join_xml_data_path('article-pear-0.10.xml'), xmlStart=u'Pear', namespaces=[0, 1], site=self.get_site()) @@ -41,7 +37,7 @@ def test_simple_bare_refs(self): """Test simple bare references in multiple namespaces.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + xmlFilename=join_xml_data_path('dummy-reflinks.xml'), xmlStart=u'Fake page', namespaces=[0, 1], site=self.get_site()) @@ -52,7 +48,7 @@ def test_namespace_empty_list(self): """Test namespaces=[] processes all namespaces.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + xmlFilename=join_xml_data_path('dummy-reflinks.xml'), xmlStart=u'Fake page', namespaces=[], site=self.get_site()) @@ -64,7 +60,7 @@ def test_namespace_None(self): """Test namespaces=None processes all namespaces.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + xmlFilename=join_xml_data_path('dummy-reflinks.xml'), xmlStart=u'Fake page', namespaces=None, site=self.get_site()) @@ -76,7 +72,7 @@ def test_namespace_string_ids(self): """Test namespaces with ids as string.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + xmlFilename=join_xml_data_path('dummy-reflinks.xml'), xmlStart=u'Fake page', namespaces=["0", "1"], site=self.get_site()) @@ -87,7 +83,7 @@ def test_namespace_names(self): """Test namespaces with namespace names.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + xmlFilename=join_xml_data_path('dummy-reflinks.xml'), xmlStart=u'Fake page', namespaces=["Talk"], site=self.get_site()) @@ -99,7 +95,7 @@ def test_start_with_underscore(self): """Test with underscore in start page title.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + xmlFilename=join_xml_data_path('dummy-reflinks.xml'), xmlStart=u'Fake_page', namespaces=[0, 1], site=self.get_site()) @@ -110,7 +106,7 @@ def test_without_start(self): """Test without a start page title.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + xmlFilename=join_xml_data_path('dummy-reflinks.xml'), xmlStart=None, namespaces=[0, 1], site=self.get_site()) @@ -122,7 +118,7 @@ def test_start_prefix(self): """Test with a prefix as a start page title.""" gen = XmlDumpPageGenerator( - xmlFilename=os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + xmlFilename=join_xml_data_path('dummy-reflinks.xml'), xmlStart='Fake', namespaces=[0, 1], site=self.get_site()) @@ -160,14 +156,14 @@ def test_xml_simple(self): """Test the generator without any narrowing.""" - main('-xml:' + os.path.join(_xml_data_dir, 'dummy-reflinks.xml')) + main('-xml:' + join_xml_data_path('dummy-reflinks.xml')) gen = self.constructor_args[0] self.assertPageTitlesCountEqual(gen, [u'Fake page', u'Talk:Fake page'], site=self.get_site()) def test_xml_one_namespace(self): """Test the generator using one namespace id.""" - main('-xml:' + os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + main('-xml:' + join_xml_data_path('dummy-reflinks.xml'), '-namespace:1') gen = self.constructor_args[0] pages = list(gen) @@ -176,7 +172,7 @@ def test_xml_multiple_namespace_ids(self): """Test the generator using multiple separate namespaces parameters.""" - main('-xml:' + os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + main('-xml:' + join_xml_data_path('dummy-reflinks.xml'), '-namespace:0', '-namespace:1', '-xmlstart:Fake page') gen = self.constructor_args[0] self.assertPageTitlesCountEqual(gen, [u'Fake page', u'Talk:Fake page'], @@ -185,7 +181,7 @@ @unittest.expectedFailure def test_xml_multiple_namespace_ids_2(self): """Test the generator using multiple namespaces in one parameter.""" - main('-xml:' + os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + main('-xml:' + join_xml_data_path('dummy-reflinks.xml'), '-namespace:0,1', '-xmlstart:Fake page') gen = self.constructor_args[0] self.assertPageTitlesCountEqual(gen, [u'Fake page', u'Talk:Fake page'], @@ -194,7 +190,7 @@ @unittest.expectedFailure def test_xml_start_prefix(self): """Test the generator using a start partial page.""" - main('-xml:' + os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + main('-xml:' + join_xml_data_path('dummy-reflinks.xml'), '-namespace:1', '-xmlstart:Fake') gen = self.constructor_args[0] pages = list(gen) @@ -204,7 +200,7 @@ @unittest.expectedFailure def test_xml_start_underscore(self): """Test the generator using a start page with an underscore.""" - main('-xml:' + os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + main('-xml:' + join_xml_data_path('dummy-reflinks.xml'), '-namespace:1', '-xmlstart:Fake_page') gen = self.constructor_args[0] pages = list(gen) @@ -213,7 +209,7 @@ def test_xml_namespace_name(self): """Test the generator using a namespace name.""" - main('-xml:' + os.path.join(_xml_data_dir, 'dummy-reflinks.xml'), + main('-xml:' + join_xml_data_path('dummy-reflinks.xml'), '-namespace:Talk', '-xmlstart:Fake page') gen = self.constructor_args[0] pages = list(gen) diff --git a/tests/replacebot_tests.py b/tests/replacebot_tests.py index 8794795..ffebe76 100644 --- a/tests/replacebot_tests.py +++ b/tests/replacebot_tests.py @@ -8,8 +8,6 @@ from __future__ import absolute_import, unicode_literals __version__ = '$Id$' -# -import os import pywikibot @@ -17,13 +15,13 @@ from scripts import replace -from tests import _data_dir +from tests import join_data_path from tests.aspects import unittest from tests.bot_tests import TWNBotTestCase # Load only the custom fixes fixes.fixes.clear() -fixes._load_file(os.path.join(_data_dir, 'fixes.py')) +fixes._load_file(join_data_path('fixes.py')) class TestReplacementsMain(TWNBotTestCase): diff --git a/tests/script_tests.py b/tests/script_tests.py index 9dfcc94..6a2eb11 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -17,11 +17,11 @@ StringTypes, ) -from tests import _root_dir +from tests import join_root_path from tests.aspects import unittest, DefaultSiteTestCase, MetaTestCaseClass, PwbTestCase from tests.utils import allowed_failure, execute_pwb, add_metaclass -scripts_path = os.path.join(_root_dir, 'scripts') +scripts_path = join_root_path('scripts') # These dependencies are not always the package name which is in setup.py. # e.g. 'PIL.ImageTk' is a object provided by several different pypi packages, diff --git a/tests/tools_tests.py b/tests/tools_tests.py index acbb7ac..9b2445a 100644 --- a/tests/tools_tests.py +++ b/tests/tools_tests.py @@ -17,11 +17,9 @@ from pywikibot import tools -from tests import _data_dir +from tests import join_xml_data_path from tests.aspects import unittest, DeprecationTestCase, TestCase from tests.utils import expected_failure_if - -_xml_data_dir = os.path.join(_data_dir, 'xml') class ContextManagerWrapperTestCase(TestCase): @@ -87,7 +85,7 @@ def setUpClass(cls): """Define base_file and original_content.""" super(OpenArchiveTestCase, cls).setUpClass() - cls.base_file = os.path.join(_xml_data_dir, 'article-pyrus.xml') + cls.base_file = join_xml_data_path('article-pyrus.xml') with open(cls.base_file, 'rb') as f: cls.original_content = f.read() @@ -150,7 +148,7 @@ def setUpClass(cls): """Define base_file and original_content.""" super(OpenArchiveWriteTestCase, cls).setUpClass() - cls.base_file = os.path.join(_xml_data_dir, 'article-pyrus.xml') + cls.base_file = join_xml_data_path('article-pyrus.xml') with open(cls.base_file, 'rb') as f: cls.original_content = f.read() diff --git a/tests/upload_tests.py b/tests/upload_tests.py index 610a9d8..a7987d8 100644 --- a/tests/upload_tests.py +++ b/tests/upload_tests.py @@ -13,13 +13,11 @@ __version__ = '$Id$' -import os - import pywikibot from pywikibot.data.api import APIError -from tests import _images_dir +from tests import join_images_path from tests.aspects import unittest, TestCase @@ -32,22 +30,20 @@ family = 'wikipedia' code = 'test' - sounds_png = os.path.join(_images_dir, 'MP_sounds.png') - arrow_png = os.path.join(_images_dir, '1rightarrow.png') + sounds_png = join_images_path('MP_sounds.png') + arrow_png = join_images_path('1rightarrow.png') def test_png(self): """Test uploading a png using Site.upload.""" page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png') - local_filename = os.path.join(_images_dir, 'MP_sounds.png') - self.site.upload(page, source_filename=local_filename, + self.site.upload(page, source_filename=self.sounds_png, comment='pywikibot test', ignore_warnings=True) def test_png_chunked(self): """Test uploading a png in two chunks using Site.upload.""" page = pywikibot.FilePage(self.site, 'MP_sounds-pwb-chunked.png') - local_filename = os.path.join(_images_dir, 'MP_sounds.png') - self.site.upload(page, source_filename=local_filename, + self.site.upload(page, source_filename=self.sounds_png, comment='pywikibot test', ignore_warnings=True, chunk_size=1024) diff --git a/tests/uploadbot_tests.py b/tests/uploadbot_tests.py index a1da9de..67e95c3 100644 --- a/tests/uploadbot_tests.py +++ b/tests/uploadbot_tests.py @@ -17,7 +17,7 @@ import os from scripts import upload -from tests import _images_dir +from tests import join_images_path from tests.aspects import unittest, TestCase @@ -33,7 +33,7 @@ def test_png_list(self): """Test uploading a list of pngs using upload.py.""" image_list = [] - for directory_info in os.walk(_images_dir): + for directory_info in os.walk(join_images_path()): for dir_file in directory_info[2]: image_list.append(os.path.join(directory_info[0], dir_file)) bot = upload.UploadRobot(url=image_list, @@ -45,7 +45,7 @@ def test_png(self): """Test uploading a png using upload.py.""" - bot = upload.UploadRobot(url=[os.path.join(_images_dir, "MP_sounds.png")], + bot = upload.UploadRobot(url=[join_images_path("MP_sounds.png")], description="pywikibot upload.py script test", useFilename=None, keepFilename=True, verifyDescription=True, aborts=set(), diff --git a/tests/wikibase_tests.py b/tests/wikibase_tests.py index 248d324..406a42d 100644 --- a/tests/wikibase_tests.py +++ b/tests/wikibase_tests.py @@ -12,7 +12,6 @@ import copy import json -import os import pywikibot @@ -20,6 +19,7 @@ from pywikibot.page import WikibasePage, ItemPage from pywikibot.site import Namespace, NamespacesDict +from tests import join_pages_path from tests.aspects import ( unittest, TestCase, WikidataTestCase, @@ -650,7 +650,7 @@ super(TestLinks, self).setUp() self.wdp = pywikibot.ItemPage(self.get_repo(), 'Q60') self.wdp.id = 'Q60' - with open(os.path.join(os.path.split(__file__)[0], 'pages', 'Q60_only_sitelinks.wd')) as f: + with open(join_pages_path('Q60_only_sitelinks.wd')) as f: self.wdp._content = json.load(f) self.wdp.get() @@ -968,7 +968,7 @@ wikidata = self.get_repo() self.wdp = pywikibot.ItemPage(wikidata, 'Q60') self.wdp.id = 'Q60' - with open(os.path.join(os.path.split(__file__)[0], 'pages', 'Q60.wd')) as f: + with open(join_pages_path('Q60.wd')) as f: self.wdp._content = json.load(f) self.wdp.get() del self.wdp._content['id'] diff --git a/tests/xmlreader_tests.py b/tests/xmlreader_tests.py index 3f11ba8..00fe478 100644 --- a/tests/xmlreader_tests.py +++ b/tests/xmlreader_tests.py @@ -9,14 +9,10 @@ __version__ = '$Id$' -import os.path - from pywikibot import xmlreader -from tests import _data_dir +from tests import join_xml_data_path from tests.aspects import unittest, TestCase - -_xml_data_dir = os.path.join(_data_dir, 'xml') class XmlReaderTestCase(TestCase): @@ -28,7 +24,7 @@ def _get_entries(self, filename, **kwargs): """Get all entries via XmlDump.""" entries = [r for r in - xmlreader.XmlDump(os.path.join(_xml_data_dir, filename), + xmlreader.XmlDump(join_xml_data_path(filename), **kwargs).parse()] return entries @@ -62,8 +58,7 @@ """Test XmlDump correctly parsing whether a page is a redirect.""" pages = self._get_entries('article-pyrus.xml', allrevisions=True) pages = [r for r in - xmlreader.XmlDump(os.path.join(_xml_data_dir, - "article-pyrus.xml")).parse()] + xmlreader.XmlDump(join_xml_data_path('article-pyrus.xml')).parse()] self.assertTrue(pages[0].isredirect) def _compare(self, previous, variant, all_revisions): -- To view, visit https://gerrit.wikimedia.org/r/240893 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I409413a9a480f9bd149cb72a365e76cde7d8e3e7 Gerrit-PatchSet: 5 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] Remove implicit use of `pywikibot` - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Remove implicit use of `pywikibot` ...................................................................... Remove implicit use of `pywikibot` `pywikibot` was being used without an 'import pywikibot' and this wasnt a syntax error as 'import pywikibot.tools' or similar implicitly imports `pywikibot`. However this import may not always occur if the sub-module has already been loaded, such as if the package import fails but some sub-modules have been imported. Also remove the 'import pywikibot.tools' from pywikibot.page. Bug: T113161 Change-Id: Id01888d3b9d847927d4adce236153ac4b91a0c72 --- M pywikibot/__init__.py M pywikibot/comms/http.py M pywikibot/page.py 3 files changed, 66 insertions(+), 43 deletions(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index b75fa13..ba3aae3 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -26,19 +26,24 @@ from warnings import warn -# Use pywikibot. prefix for all in-package imports; this is to prevent -# confusion with similarly-named modules in version 1 framework, for users -# who want to continue using both +# logging must be imported first so that other modules can +# use these logging methods during the initialisation sequence. +from pywikibot.logging import ( + critical, debug, error, exception, log, output, stdout, warning +) from pywikibot import config2 as config from pywikibot.bot import ( - output, warning, error, critical, debug, stdout, exception, - input, input_choice, input_yn, inputChoice, handle_args, showHelp, ui, log, + input, input_choice, input_yn, inputChoice, handle_args, showHelp, ui, calledModuleName, Bot, CurrentPageBot, WikidataBot, # the following are flagged as deprecated on usage handleArgs, ) +from pywikibot.bot_choice import ( + QuitKeyboardInterrupt as _QuitKeyboardInterrupt, +) from pywikibot.data.api import UploadWarning as _UploadWarning +from pywikibot.diff import PatchManager from pywikibot.exceptions import ( Error, InvalidTitle, BadTitle, NoPage, NoMoveTarget, SectionError, SiteDefinitionError, NoSuchSite, UnknownSite, UnknownFamily, @@ -51,14 +56,25 @@ ServerError, FatalServerError, Server504Error, CaptchaError, SpamfilterError, CircularRedirect, InterwikiRedirectPage, WikiBaseError, CoordinateGlobeUnknownException, + DeprecatedPageNotFoundError as _DeprecatedPageNotFoundError, + _EmailUserError, ) -from pywikibot.tools import PY2, UnicodeMixin, redirect_func -from pywikibot.tools.formatter import color_format +from pywikibot.family import Family from pywikibot.i18n import translate -from pywikibot.diff import PatchManager +from pywikibot.site import BaseSite +from pywikibot.tools import ( + # __ to avoid conflict with ModuleDeprecationWrapper._deprecated + deprecated as __deprecated, + deprecate_arg as __deprecate_arg, + normalize_username, + redirect_func, + ModuleDeprecationWrapper as _ModuleDeprecationWrapper, + PY2, + UnicodeMixin, +) +from pywikibot.tools.formatter import color_format import pywikibot.textlib as textlib -import pywikibot.tools textlib_methods = ( 'unescape', 'replaceExcept', 'removeDisabledParts', 'removeHTMLParts', @@ -108,8 +124,8 @@ globals()[_name] = wrapped_func -deprecated = redirect_func(pywikibot.tools.deprecated) -deprecate_arg = redirect_func(pywikibot.tools.deprecate_arg) +deprecated = redirect_func(__deprecated) +deprecate_arg = redirect_func(__deprecate_arg) class Timestamp(datetime.datetime): @@ -564,14 +580,14 @@ # Iterate through all families and look, which does apply to # the given URL for fam in config.family_files: - family = pywikibot.family.Family.load(fam) + family = Family.load(fam) code = family.from_url(url) if code is not None: matched_sites += [(code, family)] if matched_sites: if len(matched_sites) > 1: - pywikibot.warning( + warning( 'Found multiple matches for URL "{0}": {1} (use first)' .format(url, ', '.join(str(s) for s in matched_sites))) _url_cache[url] = matched_sites[0] @@ -591,8 +607,8 @@ code = code or config.mylang fam = fam or config.family - if not isinstance(fam, pywikibot.family.Family): - fam = pywikibot.family.Family.load(fam) + if not isinstance(fam, Family): + fam = Family.load(fam) interface = interface or fam.interface(code) @@ -612,10 +628,10 @@ except ImportError: raise ValueError("Invalid interface name '%(interface)s'" % locals()) - if not issubclass(interface, pywikibot.site.BaseSite): + if not issubclass(interface, BaseSite): warning('Site called with interface=%s' % interface.__name__) - user = pywikibot.tools.normalize_username(user) + user = normalize_username(user) key = '%s:%s:%s:%s' % (interface.__name__, fam, code, user) if key not in _sites or not isinstance(_sites[key], interface): _sites[key] = interface(code=code, fam=fam, user=user, sysop=sysop) @@ -630,9 +646,10 @@ # alias for backwards-compability -getSite = pywikibot.tools.redirect_func(Site, old_name='getSite') +getSite = redirect_func(Site, old_name='getSite') +# These imports depend on Wb* classes above. from pywikibot.page import ( Page, FilePage, @@ -649,7 +666,7 @@ link_regex = re.compile(r'\[\[(?P<title>[^\]|[<>{}]*)(\|.*?)?\]\]') -(a)pywikibot.tools.deprecated("comment parameter for page saving method") +@__deprecated('comment parameter for page saving method') def setAction(s): """Set a summary to use for changed page submissions.""" config.default_edit_summary = s @@ -752,20 +769,20 @@ _putthread.setName('Put-Thread') _putthread.setDaemon(True) -wrapper = pywikibot.tools.ModuleDeprecationWrapper(__name__) +wrapper = _ModuleDeprecationWrapper(__name__) wrapper._add_deprecated_attr('ImagePage', FilePage) wrapper._add_deprecated_attr( 'cookie_jar', replacement_name='pywikibot.comms.http.cookie_jar') wrapper._add_deprecated_attr( - 'PageNotFound', pywikibot.exceptions.DeprecatedPageNotFoundError, + 'PageNotFound', _DeprecatedPageNotFoundError, warning_message=('{0}.{1} is deprecated, and no longer ' 'used by pywikibot; use http.fetch() instead.')) wrapper._add_deprecated_attr( - 'UserActionRefuse', pywikibot.exceptions._EmailUserError, + 'UserActionRefuse', _EmailUserError, warning_message='UserActionRefuse is deprecated; ' 'use UserRightsError and/or NotEmailableError instead.') wrapper._add_deprecated_attr( - 'QuitKeyboardInterrupt', pywikibot.bot.QuitKeyboardInterrupt, + 'QuitKeyboardInterrupt', _QuitKeyboardInterrupt, warning_message='pywikibot.QuitKeyboardInterrupt is deprecated; ' 'use pywikibot.bot.QuitKeyboardInterrupt instead.') wrapper._add_deprecated_attr( diff --git a/pywikibot/comms/http.py b/pywikibot/comms/http.py index 7a54e07..8db7213 100644 --- a/pywikibot/comms/http.py +++ b/pywikibot/comms/http.py @@ -44,11 +44,16 @@ from urllib2 import quote from pywikibot import config + +from pywikibot import __release__ +from pywikibot.bot import calledModuleName +from pywikibot.comms import threadedhttp from pywikibot.exceptions import ( FatalServerError, Server504Error, Server414Error ) -from pywikibot.comms import threadedhttp +from pywikibot.logging import critical, debug, error, log, warning from pywikibot.tools import deprecate_arg, issue_deprecation_warning, PY2 + import pywikibot.version # The error message for failed SSL certificate verification @@ -57,21 +62,21 @@ _logger = "comm.http" -if (isinstance(pywikibot.config2.socket_timeout, tuple) and +if (isinstance(config.socket_timeout, tuple) and StrictVersion(requests.__version__) < StrictVersion('2.4.0')): - pywikibot.warning('The configured timeout is a tuple but requests does not ' - 'support a tuple as a timeout. It uses the lower of the ' - 'two.') - pywikibot.config2.socket_timeout = min(pywikibot.config2.socket_timeout) + warning('The configured timeout is a tuple but requests does not ' + 'support a tuple as a timeout. It uses the lower of the ' + 'two.') + config.socket_timeout = min(config.socket_timeout) cookie_jar = cookielib.LWPCookieJar( config.datafilepath('pywikibot.lwp')) try: cookie_jar.load() except (IOError, cookielib.LoadError): - pywikibot.debug(u"Loading cookies failed.", _logger) + debug('Loading cookies failed.', _logger) else: - pywikibot.debug(u"Loaded cookies from file.", _logger) + debug('Loaded cookies from file.', _logger) session = requests.Session() session.cookies = cookie_jar @@ -84,17 +89,17 @@ if hasattr(sys, 'last_type'): # we quit because of an exception print(sys.last_type) - pywikibot.critical(message) + critical(message) else: - pywikibot.log(message) + log(message) - pywikibot.log('Network session closed.') + log('Network session closed.') atexit.register(_flush) USER_AGENT_PRODUCTS = { 'python': 'Python/' + '.'.join([str(i) for i in sys.version_info]), 'http_backend': 'requests/' + requests.__version__, - 'pwb': 'Pywikibot/' + pywikibot.__release__, + 'pwb': 'Pywikibot/' + __release__, } @@ -157,10 +162,12 @@ """ values = USER_AGENT_PRODUCTS.copy() - values['script'] = pywikibot.calledModuleName() + script_name = calledModuleName() + + values['script'] = script_name # TODO: script_product should add the script version, if known - values['script_product'] = pywikibot.calledModuleName() + values['script_product'] = script_name script_comments = [] username = '' @@ -275,8 +282,8 @@ if auth is not None and len(auth) == 4: if isinstance(requests_oauthlib, ImportError): warn('%s' % requests_oauthlib, ImportWarning) - pywikibot.error('OAuth authentication not supported: %s' - % requests_oauthlib) + error('OAuth authentication not supported: %s' + % requests_oauthlib) auth = None else: auth = requests_oauthlib.OAuth1(*auth) @@ -321,8 +328,7 @@ # HTTP status 207 is also a success status for Webdav FINDPROP, # used by the version module. if request.status not in (200, 207): - pywikibot.warning(u"Http response status %(status)s" - % {'status': request.data.status_code}) + warning('Http response status {0}'.format(request.data.status_code)) def _enqueue(uri, method="GET", body=None, headers=None, **kwargs): diff --git a/pywikibot/page.py b/pywikibot/page.py index 7f949fe..ec7430a 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -61,6 +61,7 @@ PYTHON_VERSION, PY2, MediaWikiVersion, UnicodeMixin, ComparableMixin, DotReadableDict, deprecated, deprecate_arg, deprecated_args, issue_deprecation_warning, + ModuleDeprecationWrapper as _ModuleDeprecationWrapper, first_upper, remove_last_args, _NotImplementedWarning, OrderedDict, Counter, ) @@ -2306,8 +2307,7 @@ self, step=step, total=total, content=content) -import pywikibot.tools -wrapper = pywikibot.tools.ModuleDeprecationWrapper(__name__) +wrapper = _ModuleDeprecationWrapper(__name__) wrapper._add_deprecated_attr('ImagePage', FilePage) -- To view, visit https://gerrit.wikimedia.org/r/239648 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Id01888d3b9d847927d4adce236153ac4b91a0c72 Gerrit-PatchSet: 8 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Shloub <shloub+wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] Family defined Site interface - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Family defined Site interface ...................................................................... Family defined Site interface Allow Family class to determine the appropriate Site class to be used. It is usable for supporting extensions, using Site classes composed of several APISite subclasses. It can also be used for the special case of Sites which are for a code that is no longer online. The Family.from_url cache now stores the Family object instead of the family name, to reduce the number of Family.load calls. Change-Id: Ia9f141b9e9f493aa2b1c42afcf4daa8049656c10 --- M pywikibot/__init__.py M pywikibot/families/wikidata_family.py M pywikibot/family.py M pywikibot/site.py M tests/aspects.py M tests/site_tests.py M tests/utils.py 7 files changed, 57 insertions(+), 17 deletions(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index 290a925..b75fa13 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -567,7 +567,7 @@ family = pywikibot.family.Family.load(fam) code = family.from_url(url) if code is not None: - matched_sites += [(code, fam)] + matched_sites += [(code, family)] if matched_sites: if len(matched_sites) > 1: @@ -590,7 +590,11 @@ # Fallback to config defaults code = code or config.mylang fam = fam or config.family - interface = interface or config.site_interface + + if not isinstance(fam, pywikibot.family.Family): + fam = pywikibot.family.Family.load(fam) + + interface = interface or fam.interface(code) # config.usernames is initialised with a dict for each family name family_name = str(fam) diff --git a/pywikibot/families/wikidata_family.py b/pywikibot/families/wikidata_family.py index b7a2786..2177b90 100644 --- a/pywikibot/families/wikidata_family.py +++ b/pywikibot/families/wikidata_family.py @@ -37,6 +37,10 @@ 'wikidata': ('wikidata', 'test') }) + def interface(self, code): + """Return 'DataSite'.""" + return 'DataSite' + def shared_data_repository(self, code, transcluded=False): """ Indicate Wikidata is both a repository and its own client. diff --git a/pywikibot/family.py b/pywikibot/family.py index 72753f0..921b7a6 100644 --- a/pywikibot/family.py +++ b/pywikibot/family.py @@ -1098,6 +1098,20 @@ def nice_get_address(self, code, title): return '%s%s' % (self.nicepath(code), title) + def interface(self, code): + """ + Return interface to use for code. + + @rtype: str or subclass of BaseSite + """ + if code in self.interwiki_removals: + if code in self.codes: + pywikibot.warn('Interwiki removal %s is in %s codes' + % (code, self)) + return 'RemovedSite' + + return config.site_interface + # List of codes which aren't returned by from_url; True returns None always _ignore_from_url = [] diff --git a/pywikibot/site.py b/pywikibot/site.py index 98de171..2d5bfc0 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -900,7 +900,8 @@ def __repr__(self): """Return internal representation.""" - return 'Site("%s", "%s")' % (self.code, self.family.name) + return '{0}("{1}", "{2}")'.format( + self.__class__.__name__, self.code, self.family) def __hash__(self): """Return hashable key.""" @@ -1729,6 +1730,15 @@ def __repr__(self): """Return a representation of the internal tokens dictionary.""" return self._tokens.__repr__() + + +class RemovedSite(BaseSite): + + """Site removed from a family.""" + + def __init__(self, code, fam, user=None, sysop=None): + """Constructor.""" + super(RemovedSite, self).__init__(code, fam, user, sysop) class NonMWAPISite(BaseSite): @@ -6450,10 +6460,6 @@ f.__doc__ = method.__doc__ return f return super(APISite, self).__getattr__(attr) - - def __repr__(self): - """Return internal representation.""" - return 'DataSite("%s", "%s")' % (self.code, self.family.name) @deprecated("pywikibot.PropertyPage") def _get_propertyitem(self, props, source, **params): diff --git a/tests/aspects.py b/tests/aspects.py index fa0e486..0503077 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -49,7 +49,7 @@ from pywikibot.exceptions import ServerError, NoUsername from pywikibot.family import WikimediaFamily from pywikibot.site import BaseSite -from pywikibot.tools import PY2 +from pywikibot.tools import PY2, StringTypes import tests @@ -106,6 +106,21 @@ sys.stdout.flush() result.addSuccess(self) + def assertMethod(self, method, *args): + """Generic method assertion.""" + if not method(*args): + self.fail('{0!r} ({1!r}) fails'.format(method, args)) + + def assertStringMethod(self, method, *args): + """ + Generic string method assertion. + + All args must be already converted to a string. + """ + for arg in args: + self.assertIsInstance(arg, StringTypes) + self.assertMethod(method, *args) + def assertPageInNamespaces(self, page, namespaces): """ Assert that Pages is in namespaces. diff --git a/tests/site_tests.py b/tests/site_tests.py index 85bdefd..73ff338 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -224,6 +224,11 @@ mysite_pickled = pickle.loads(mysite_str) self.assertEqual(mysite, mysite_pickled) + def test_repr(self): + """Test __repr__.""" + expect = 'Site("{0}", "{1}")'.format(self.code, self.family) + self.assertStringMethod(str.endswith, repr(self.site), expect) + def testBaseMethods(self): """Test cases for BaseSite methods.""" mysite = self.get_site() @@ -235,9 +240,6 @@ self.assertEqual(mysite.sitename(), "%s:%s" % (self.family, self.code)) - self.assertEqual(repr(mysite), - 'Site("%s", "%s")' - % (self.code, self.family)) self.assertIsInstance(mysite.linktrail(), basestring) self.assertIsInstance(mysite.redirect(), basestring) try: @@ -2757,6 +2759,7 @@ def test_removed_site(self): """Test Wikimedia offline site.""" site = pywikibot.Site('ru-sib', 'wikipedia') + self.assertIsInstance(site, pywikibot.site.RemovedSite) self.assertEqual(site.code, 'ru-sib') self.assertIsInstance(site.obsolete, bool) self.assertTrue(site.obsolete) diff --git a/tests/utils.py b/tests/utils.py index c12f5e2..fb2fafb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -364,12 +364,6 @@ def _build_namespaces(self): return Namespace.builtin_namespaces(case=self.siteinfo['case']) - def __repr__(self): - """Override default so warnings and errors indicate test is dry.""" - return "%s(%r, %r)" % (self.__class__.__name__, - self.code, - self.family.name) - @property def userinfo(self): """Return dry data.""" -- To view, visit https://gerrit.wikimedia.org/r/236404 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ia9f141b9e9f493aa2b1c42afcf4daa8049656c10 Gerrit-PatchSet: 12 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] Porting various Commons upload bots to core from compat. - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Porting various Commons upload bots to core from compat. ...................................................................... Porting various Commons upload bots to core from compat. Added scripts: 1) imagecopy.py 2) imagecopy_self.py 3) imageharvest.py 4) panoramiopicker.py Made change in pywikibot/config2.py so as to support functions of above scripts. Bug: T66856 Change-Id: I5ea3a2131badba22fdc5e99deb5c40a49f4f0998 --- M pywikibot/config2.py A scripts/imagecopy.py A scripts/imagecopy_self.py A scripts/imageharvest.py A scripts/panoramiopicker.py M tests/script_tests.py 6 files changed, 2,119 insertions(+), 0 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/config2.py b/pywikibot/config2.py index c5ad2c1..4cac315 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -655,6 +655,13 @@ 'reviewer': u'', # If so, under what reviewer name? } +# Using the Panoramio api +panoramio = { + 'review': False, # Do we use automatically make our uploads reviewed? + 'reviewer': u'', # If so, under what reviewer name? +} + + # ############# COPYRIGHT SETTINGS ############## # Enable/disable search engine in copyright.py script diff --git a/scripts/imagecopy.py b/scripts/imagecopy.py new file mode 100644 index 0000000..0450fc7 --- /dev/null +++ b/scripts/imagecopy.py @@ -0,0 +1,563 @@ +# -*- coding: utf-8 -*- +""" +Script to copy files from a local Wikimedia wiki to Wikimedia Commons. + +It uses CommonsHelper to not leave any information out and CommonSense +to automatically categorise the file. After copying, a NowCommons +template is added to the local wiki's file. It uses a local exclusion +list to skip files with templates not allow on Wikimedia Commons. If no +categories have been found, the file will be tagged on Commons. + +This bot uses a graphical interface and may not work from commandline +only environment. + +Requests for improvement for CommonsHelper output should be directed to +Magnus Manske at his talk page. Please be very specific in your request +(describe current output and expected output) and note an example file, +so he can test at: [[de:Benutzer Diskussion:Magnus Manske]]. You can +write him in German and English. + +Examples + +Work on a single image + python pwb.py imagecopy.py -page:Image:<imagename> +Work on the 100 newest images: + python pwb.py imagecopy.py -newimages:100 +Work on all images in a category:<cat> + python pwb.py imagecopy.py -cat:<cat> +Work on all images which transclude a template + python pwb.py imagecopy.py -transcludes:<template> + +See pagegenerators.py for more ways to get a list of images. +By default the bot works on your home wiki (set in user-config) + +Known issues/FIXMEs (no critical issues known): +* make it use pagegenerators.py +** Implemented in rewrite +* Some variable names are in Spanish, which makes the code harder to read. +** Almost all variables are now in English +* Depending on sorting within a file category, the "next batch" is sometimes + not working, leading to an endless loop +** Using pagegenerators now +* Different wikis can have different exclusion lists. A parameter for the + exclusion list Uploadbot.localskips.txt would probably be nice. +* Bot should probably use API instead of query.php +** Api? Query? Wikipedia.py! +* Should request alternative name if file name already exists on Commons +** Implemented in rewrite +* Exits after last file in category was processed, aborting all pending + threads. +** Implemented proper threading in rewrite +* Should take user-config.py as input for project and lang variables +** Implemented in rewrite +* Should require a Commons user to be present in user-config.py before + working +* Should probably have an input field for additional categories +* Should probably have an option to change uploadtext with file +* required i18n options for NowCommons template (f.e. {{subst:ncd}} on + en.wp. Currently needs customisation to work properly. Bot was tested + succesfully on nl.wp (12k+ files copied and deleted locally) and en.wp + (about 100 files copied and SieBot has bot approval for tagging {{ncd}} + with this bot) +** Implemented +* {{NowCommons|xxx}} requires the namespace prefix Image: on most wikis + and can be left out on others. This needs to be taken care of when + implementing i18n +** Implemented +* This bot should probably get a small tutorial at meta with a few + screenshots. +""" +# +# Based on upload.py by: +# (C) Rob W.W. Hooft, Andre Engels 2003-2007 +# (C) Wikipedian, Keichwa, Leogregianin, Rikwade, Misza13 2003-2007 +# +# New bot by: +# (C) Kyle/Orgullomoore, Siebrand Mazeland 2007-2008 +# +# Another rewrite by: +# (C) Multichill 2008-2011 +# (C) Pywikibot team, 2007-2015 +# +# Distributed under the terms of the MIT license. +# +from __future__ import absolute_import, unicode_literals + +__version__ = '$Id$' + +import codecs +import re +import socket +import threading +import webbrowser + +import pywikibot + +from pywikibot import pagegenerators, config, i18n + +from pywikibot.tools import PY2 + +from scripts import image, upload + +if not PY2: + import tkinter as Tkinter + + from urllib.parse import urlencode + from urllib.request import urlopen +else: + import Tkinter + + from urllib import urlencode, urlopen + +try: + from pywikibot.userinterfaces.gui import Tkdialog +except ImportError as _tk_error: + Tkdialog = None + +NL = '' + +nowCommonsTemplate = { + '_default': u'{{NowCommons|%s}}', + 'af': u'{{NowCommons|File:%s}}', + 'als': u'{{NowCommons|%s}}', + 'am': u'{{NowCommons|File:%s}}', + 'ang': u'{{NowCommons|File:%s}}', + 'ar': u'{{الآن كومنز|%s}}', + 'ast': u'{{EnCommons|File:%s}}', + 'az': u'{{NowCommons|%s}}', + 'bar': u'{{NowCommons|%s}}', + 'bg': u'{{NowCommons|%s}}', + 'bn': u'{{NowCommons|File:%s}}', + 'bs': u'{{NowCommons|%s}}', + 'ca': u'{{AraCommons|%s}}', + 'cs': u'{{NowCommons|%s}}', + 'cy': u'{{NowCommons|File:%s}}', + 'da': u'{{NowCommons|File:%s}}', + 'de': u'{{NowCommons|%s}}', + 'dsb': u'{{NowCommons|%s}}', + 'el': u'{{NowCommons|%s}}', + 'en': u'{{subst:ncd|%s}}', + 'eo': u'{{Nun en komunejo|%s}}', + 'es': u'{{EnCommons|File:%s}}', + 'et': u'{{NüüdCommonsis|File:%s}}', + 'fa': u'{{NowCommons|%s}}', + 'fi': u'{{NowCommons|%s}}', + 'fo': u'{{NowCommons|File:%s}}', + 'fr': u'{{Image sur Commons|%s}}', + 'fy': u'{{NowCommons|%s}}', + 'ga': u'{{Ag Cómhaoin|File:%s}}', + 'gl': u'{{EnCommons]|File:%s}}', + 'gv': u'{{NowCommons|File:%s}}', + 'he': u'{{גם בוויקישיתוף|%s}}', + 'hr': u'{{NowCommons|%s}}', + 'hsb': u'{{NowCommons|%s}}', + 'hu': u'{{Azonnali-commons|%s}}', + 'ia': u'{{NowCommons|File:%s}}', + 'id': u'{{NowCommons|File:%s}}', + 'ilo': u'{{NowCommons|File:%s}}', + 'io': u'{{NowCommons|%s}}', + 'is': u'{{NowCommons|%s}}', + 'it': u'{{NowCommons|%s}}', + 'ja': u'{{NowCommons|File:%s}}', + 'jv': u'{{NowCommons|File:%s}}', + 'ka': u'{{NowCommons|File:%s}}', + 'kn': u'{{NowCommons|File:%s}}', + 'ko': u'{{NowCommons|File:%s}}', + 'ku': u'{{NowCommons|%s}}', + 'lb': u'{{Elo op Commons|%s}}', + 'li': u'{{NowCommons|%s}}', + 'lt': u'{{NowCommons|File:%s}}', + 'lv': u'{{NowCommons|File:%s}}', + 'mk': u'{{NowCommons|File:%s}}', + 'mn': u'{{NowCommons|File:%s}}', + 'ms': u'{{NowCommons|%s}}', + 'nds-nl': u'{{NoenCommons|File:%s}}', + 'nl': u'{{NuCommons|%s}}', + 'nn': u'{{No på Commons|File:%s}}', + 'no': u'{{NowCommons|%s}}', + 'oc': u'{{NowCommons|File:%s}}', + 'pl': u'{{NowCommons|%s}}', + 'pt': u'{{NowCommons|%s}}', + 'ro': u'{{AcumCommons|File:%s}}', + 'ru': u'{{Перенесено на Викисклад|%s}}', + 'sa': u'{{NowCommons|File:%s}}', + 'scn': u'{{NowCommons|File:%s}}', + 'sh': u'{{NowCommons|File:%s}}', + 'sk': u'{{NowCommons|File:%s}}', + 'sl': u'{{OdslejZbirka|%s}}', + 'sq': u'{{NowCommons|File:%s}}', + 'sr': u'{{NowCommons|File:%s}}', + 'st': u'{{NowCommons|File:%s}}', + 'su': u'{{IlaharKiwari|File:%s}}', + 'sv': u'{{NowCommons|%s}}', + 'sw': u'{{NowCommons|%s}}', + 'ta': u'{{NowCommons|File:%s}}', + 'th': u'{{มีที่คอมมอนส์|File:%s}}', + 'tl': u'{{NasaCommons|File:%s}}', + 'tr': u'{{NowCommons|%s}}', + 'uk': u'{{NowCommons|File:%s}}', + 'ur': u'{{NowCommons|File:%s}}', + 'vec': u'{{NowCommons|%s}}', + 'vi': u'{{NowCommons|File:%s}}', + 'vo': u'{{InKobädikos|%s}}', + 'wa': u'{{NowCommons|%s}}', + 'zh': u'{{NowCommons|File:%s}}', + 'zh-min-nan': u'{{Commons ū|%s}}', + 'zh-yue': u'{{subst:Ncd|File:%s}}', +} + +moveToCommonsTemplate = { + 'ar': [u'نقل إلى كومنز'], + 'en': [u'Commons ok', u'Copy to Wikimedia Commons', u'Move to commons', + u'Movetocommons', u'To commons', + u'Copy to Wikimedia Commons by BotMultichill'], + 'fi': [u'Commonsiin'], + 'fr': [u'Image pour Commons'], + 'hsb': [u'Kopěruj do Wikimedia Commons'], + 'hu': [u'Commonsba'], + 'is': [u'Færa á Commons'], + 'ms': [u'Hantar ke Wikimedia Commons'], + 'nl': [u'Verplaats naar Wikimedia Commons', u'VNC'], + 'pl': [u'Do Commons', u'NaCommons', u'Na Commons'], + 'ru': [u'На Викисклад'], + 'sl': [u'Skopiraj v Zbirko'], + 'sr': [u'За оставу'], + 'sv': [u'Till Commons'], + 'zh': [u'Copy to Wikimedia Commons'], +} + + +def pageTextPost(url, parameters): + gotInfo = False + while not gotInfo: + try: + commonsHelperPage = urlopen( + "http://tools.wmflabs.org/commonshelper/index.php", parameters) + data = commonsHelperPage.read().decode('utf-8') + gotInfo = True + except IOError: + pywikibot.output(u'Got an IOError, let\'s try again') + except socket.timeout: + pywikibot.output(u'Got a timeout, let\'s try again') + return data + + +class imageTransfer(threading.Thread): + + """Facilitate transfer of image/file to commons.""" + + def __init__(self, imagePage, newname, category): + self.imagePage = imagePage + self.newname = newname + self.category = category + threading.Thread.__init__(self) + + def run(self): + tosend = {'language': self.imagePage.site.language().encode('utf-8'), + 'image': self.imagePage.title(withNamespace=False).encode('utf-8'), + 'newname': self.newname.encode('utf-8'), + 'project': self.imagePage.site.family.name.encode('utf-8'), + 'username': '', + 'commonsense': '1', + 'remove_categories': '1', + 'ignorewarnings': '1', + 'doit': 'Uitvoeren' + } + + tosend = urlencode(tosend) + pywikibot.output(tosend) + CH = pageTextPost('http://tools.wmflabs.org/commonshelper/index.php', + tosend) + pywikibot.output('Got CH desc.') + + tablock = CH.split('<textarea ')[1].split('>')[0] + CH = CH.split('<textarea ' + tablock + '>')[1].split('</textarea>')[0] + CH = CH.replace(u'×', u'×') + CH = self.fixAuthor(CH) + pywikibot.output(CH) + + # I want every picture to be tagged with the bottemplate so i can check my contributions later. + CH = u'\n\n{{BotMoveToCommons|' + self.imagePage.site.language() + \ + '.' + self.imagePage.site.family.name + \ + '|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}|day={{subst:CURRENTDAY}}}}' + \ + CH + + if self.category: + CH = CH.replace(u'{{subst:Unc}} ', u'') + CH += u'[[Category:' + self.category + u']]' + + bot = upload.UploadRobot(url=self.imagePage.fileUrl(), description=CH, + useFilename=self.newname, keepFilename=True, + verifyDescription=False, ignoreWarning=True, + targetSite=pywikibot.Site('commons', 'commons')) + bot.run() + + # Should check if the image actually was uploaded + if pywikibot.Page(pywikibot.Site('commons', 'commons'), + u'Image:' + self.newname).exists(): + # Get a fresh copy, force to get the page so we dont run into edit + # conflicts + imtxt = self.imagePage.get(force=True) + + # Remove the move to commons templates + if self.imagePage.site.language() in moveToCommonsTemplate: + for moveTemplate in moveToCommonsTemplate[self.imagePage.site.language()]: + imtxt = re.sub(u'(?i)\{\{' + moveTemplate + u'[^\}]*\}\}', + u'', imtxt) + + # add {{NowCommons}} + if self.imagePage.site.language() in nowCommonsTemplate: + addTemplate = nowCommonsTemplate[self.imagePage.site.language()] % self.newname + else: + addTemplate = nowCommonsTemplate['_default'] % self.newname + + commentText = i18n.twtranslate(self.imagePage.site, + 'commons-file-now-available', + {'localfile': self.imagePage.title(withNamespace=False), + 'commonsfile': self.newname}) + + pywikibot.showDiff(self.imagePage.get(), imtxt + addTemplate) + self.imagePage.put(imtxt + addTemplate, comment=commentText) + + self.gen = pagegenerators.FileLinksGenerator(self.imagePage) + self.preloadingGen = pagegenerators.PreloadingGenerator(self.gen) + + # If the image is uploaded under a different name, replace all instances + if self.imagePage.title(withNamespace=False) != self.newname: + moveSummary = i18n.twtranslate(self.imagePage.site, + 'commons-file-moved', + {'localfile': self.imagePage.title(withNamespace=False), + 'commonsfile': self.newname}) + + imagebot = image.ImageRobot(generator=self.preloadingGen, + oldImage=self.imagePage.title(withNamespace=False), + newImage=self.newname, + summary=moveSummary, always=True, + loose=True) + imagebot.run() + return + + def fixAuthor(self, pageText): + """Fix the author field in the information template.""" + informationRegex = re.compile( + u'\|Author\=Original uploader was (?P<author>\[\[:\w+:\w+:\w+\|\w+\]\] at \[.+\])') + selfRegex = re.compile( + u'\{\{self\|author\=(?P<author>\[\[:\w+:\w+:\w+\|\w+\]\] at \[.+\])\|') + + # Find the |Author=Original uploader was .... + informationMatch = informationRegex.search(pageText) + + # Find the {{self|author= + selfMatch = selfRegex.search(pageText) + + # Check if both are found and are equal + if (informationMatch and selfMatch): + if(informationMatch.group('author') == selfMatch.group('author')): + # Replace |Author=Original uploader was ... with |Author= ... + pageText = informationRegex.sub(r'|Author=\g<author>', pageText) + return pageText + + +# -label ok skip view +# textarea +archivo = config.datafilepath("Uploadbot.localskips.txt") +try: + open(archivo, 'r') +except IOError: + tocreate = open(archivo, 'w') + tocreate.write("{{NowCommons") + tocreate.close() + + +def getautoskip(): + """Get a list of templates to skip.""" + f = codecs.open(archivo, 'r', 'utf-8') + txt = f.read() + f.close() + toreturn = txt.split('{{')[1:] + return toreturn + + +class TkdialogIC(Tkdialog): + + """The dialog window for image info.""" + + def __init__(self, image_title, content, uploader, url, templates, + commonsconflict=0): + super(TkdialogIC, self).__init__() + self.root = Tkinter.Tk() + # "%dx%d%+d%+d" % (width, height, xoffset, yoffset) + # Always appear the same size and in the bottom-left corner + self.root.geometry("600x200+100-100") + self.root.title(image_title) + self.changename = '' + self.skip = 0 + self.url = url + self.uploader = "Unknown" + # uploader.decode('utf-8') + scrollbar = Tkinter.Scrollbar(self.root, orient=Tkinter.VERTICAL) + label = Tkinter.Label(self.root, text=u"Enter new name or leave blank.") + imageinfo = Tkinter.Label(self.root, text='Uploaded by %s.' % uploader) + textarea = Tkinter.Text(self.root) + textarea.insert(Tkinter.END, content.encode('utf-8')) + textarea.config(state=Tkinter.DISABLED, height=8, width=40, padx=0, pady=0, + wrap=Tkinter.WORD, yscrollcommand=scrollbar.set) + scrollbar.config(command=textarea.yview) + self.entry = Tkinter.Entry(self.root) + + self.templatelist = Tkinter.Listbox(self.root, bg="white", height=5) + + for template in templates: + self.templatelist.insert(Tkinter.END, template) + autoskip_button = Tkinter.Button(self.root, text="Add to AutoSkip", + command=self.add2_auto_skip) + browser_button = Tkinter.Button(self.root, text='View in browser', + command=self.open_in_browser) + skip_button = Tkinter.Button(self.root, text="Skip", command=self.skip_file) + ok_button = Tkinter.Button(self.root, text="OK", command=self.ok_file) + + # Start grid + label.grid(row=0) + ok_button.grid(row=0, column=1, rowspan=2) + skip_button.grid(row=0, column=2, rowspan=2) + browser_button.grid(row=0, column=3, rowspan=2) + + self.entry.grid(row=1) + + textarea.grid(row=2, column=1, columnspan=3) + scrollbar.grid(row=2, column=5) + self.templatelist.grid(row=2, column=0) + + autoskip_button.grid(row=3, column=0) + imageinfo.grid(row=3, column=1, columnspan=4) + + def ok_file(self): + """The user pressed the OK button.""" + self.changename = self.entry.get() + self.root.destroy() + + def getnewname(self): + """Activate the dialog and return the new name and if the image is skipped.""" + self.root.mainloop() + return (self.changename, self.skip) + + def open_in_browser(self): + """The user pressed the View in browser button.""" + webbrowser.open(self.url) + + def add2_auto_skip(self): + """The user pressed the Add to AutoSkip button.""" + templateid = int(self.templatelist.curselection()[0]) + template = self.templatelist.get(templateid) + with codecs.open(archivo, 'a', 'utf-8') as f: + f.write('{{' + template) + self.skip_file() + + +def doiskip(pagetext): + """Skip this image or not. + + Returns True if the image is on the skip list, otherwise False + """ + saltos = getautoskip() + # print saltos + for salto in saltos: + rex = u'\{\{\s*[' + salto[0].upper() + salto[0].lower() + ']' + \ + salto[1:] + '(\}\}|\|)' + # print rex + if re.search(rex, pagetext): + return True + return False + + +def main(*args): + """Process command line arguments and invoke bot.""" + generator = None + imagepage = None + always = False + category = u'' + # Load a lot of default generators + local_args = pywikibot.handle_args(args) + genFactory = pagegenerators.GeneratorFactory() + + for arg in local_args: + if arg == '-always': + always = True + elif arg.startswith('-cc:'): + category = arg[len('-cc:'):] + else: + genFactory.handleArg(arg) + + generator = genFactory.getCombinedGenerator() + if not generator: + pywikibot.bot.suggest_help(missing_generator=True) + return False + + pregenerator = pagegenerators.PreloadingGenerator(generator) + + for page in pregenerator: + skip = False + if page.exists() and (page.namespace() == 6) and ( + not page.isRedirectPage()): + imagepage = pywikibot.FilePage(page.site(), page.title()) + + # First do autoskip. + if doiskip(imagepage.get()): + pywikibot.output("Skipping " + page.title()) + skip = True + else: + # The first upload is last in the list. + try: + username = imagepage.getLatestUploader()[0] + except NotImplementedError: + # No API, using the page file instead + (datetime, username, resolution, size, + comment) = imagepage.getFileVersionHistory().pop() + if always: + newname = imagepage.title(withNamespace=False) + CommonsPage = pywikibot.Page(pywikibot.Site('commons', + 'commons'), + u'File:%s' % newname) + if CommonsPage.exists(): + skip = True + else: + while True: + # Do the TkdialogIC to accept/reject and change te name + (newname, skip) = TkdialogIC( + imagepage.title(withNamespace=False), + imagepage.get(), username, imagepage.permalink(), + imagepage.templates()).getnewname() + + if skip: + pywikibot.output('Skipping this image') + break + + # Did we enter a new name? + if len(newname) == 0: + # Take the old name + newname = imagepage.title(withNamespace=False) + else: + newname = newname.decode('utf-8') + + # Check if the image already exists + CommonsPage = pywikibot.Page( + pywikibot.Site('commons', 'commons'), + u'File:' + newname) + if not CommonsPage.exists(): + break + else: + pywikibot.output('Image already exists, pick another name or skip this image') + # We dont overwrite images, pick another name, go to the start of the loop + + if not skip: + imageTransfer(imagepage, newname, category).start() + + pywikibot.output(u'Still ' + str(threading.activeCount()) + u' active threads, lets wait') + for openthread in threading.enumerate(): + if openthread != threading.currentThread(): + openthread.join() + pywikibot.output(u'All threads are done') + + +if __name__ == "__main__": + main() diff --git a/scripts/imagecopy_self.py b/scripts/imagecopy_self.py new file mode 100644 index 0000000..73f458f --- /dev/null +++ b/scripts/imagecopy_self.py @@ -0,0 +1,1007 @@ +# -*- coding: utf-8 -*- +""" +Script to copy self published files from English Wikipedia to Wikimedia Commons. + +This bot is based on imagecopy.py and intended to be used to empty out +http://en.wikipedia.org/wiki/Category:Self-published_work + +This bot uses a graphical interface and may not work from commandline +only environment. + +Examples + +Work on a single file + python pwb.py imagecopy.py -page:file:<filename> +Work on all images in a category:<cat> + python pwb.py imagecopy.py -cat:<cat> +Work on all images which transclude a template + python pwb.py imagecopy.py -transcludes:<template> + +See pagegenerators.py for more ways to get a list of images. +By default the bot works on your home wiki (set in user-config) + +This is a first test version and should be used with care. + +Use -nochecktemplate if you don't want to add the check template. Be sure to +check it yourself. + +Todo: +*Queues with threads have to be implemented for the information collecting part + and for the upload part. +*Categories are now on a single line. Something like hotcat would be nice. + +""" +# +# Based on upload.py by: +# (C) Rob W.W. Hooft, Andre Engels 2003-2007 +# (C) Wikipedian, Keichwa, Leogregianin, Rikwade, Misza13 2003-2007 +# +# New bot by: +# (C) Kyle/Orgullomoore, Siebrand Mazeland 2007 +# +# Another rewrite by: +# (C) Multichill 2008 +# +# English Wikipedia specific bot by: +# (C) Multichill 2010-2012 +# +# (C) Pywikibot team, 2010-2015 +# +# Distributed under the terms of the MIT license. +# +from __future__ import absolute_import, unicode_literals + +__version__ = '$Id$' + +import re +import threading +import webbrowser + +from datetime import datetime + +import pywikibot + +from pywikibot import pagegenerators, i18n + +from pywikibot.tools import PY2 + +from scripts import imagerecat, image, upload + +if not PY2: + import tkinter as Tkinter + from queue import Queue +else: + import Tkinter + from Queue import Queue + +try: + from pywikibot.userinterfaces.gui import Tkdialog +except ImportError as _tk_error: + Tkdialog = None + +NL = '' + +nowCommonsTemplate = { + 'de': u'{{NowCommons|%s}}', + 'en': u'{{NowCommons|1=File:%s|date=~~~~~|reviewer={{subst:REVISIONUSER}}}}', + 'lb': u'{{Elo op Commons|%s}}', + 'nds-nl': u'{{NoenCommons|1=File:%s}}', + 'shared': u'{{NowCommons|1=File:%s|date=~~~~~|reviewer={{subst:REVISIONUSER}}}}', +} + +moveToCommonsTemplate = { + 'de': [u'NowCommons', u'NC', u'NCT', u'Nowcommons'], + 'en': [u'Commons ok', u'Copy to Wikimedia Commons', u'Move to commons', + u'Movetocommons', u'To commons', + u'Copy to Wikimedia Commons by BotMultichill'], + 'lb': [u'Move to commons'], + 'nds-nl': [u'Noar Commons', u'VNC'], + 'shared': [u'Move'], +} + +skipTemplates = { + 'de': [u'Löschprüfung', + u'NoCommons', + u'NowCommons', + u'NowCommons/Mängel', + u'NowCommons-Überprüft', + u'Wappenrecht', + ], + 'en': [u'Db-f1', + u'Db-f2', + u'Db-f3', + u'Db-f7', + u'Db-f8', + u'Db-f9', + u'Db-f10', + u'Do not move to Commons', + u'NowCommons', + u'CommonsNow', + u'Nowcommons', + u'NowCommonsThis', + u'Nowcommons2', + u'NCT', + u'Nowcommonsthis', + u'Moved to commons', + u'Now Commons', + u'Now at commons', + u'Db-nowcommons', + u'WikimediaCommons', + u'Now commons', + u'Di-no source', + u'Di-no license', + u'Di-no permission', + u'Di-orphaned fair use', + u'Di-no source no license', + u'Di-replaceable fair use', + u'Di-no fair use rationale', + u'Di-disputed fair use rationale', + u'Puf', + u'PUI', + u'Pui', + u'Ffd', + u'PD-user', # Only the self templates are supported for now. + u'Ticket Scan', + u'Non-free 2D art', + u'Non-free 3D art', + u'Non-free architectural work', + u'Non-free fair use in', + ], + 'lb': [u'Läschen', + ], + 'nds-nl': [u'Allinnig Wikipedie', + u'Bepark', + u'Gienidee', + u'NoenCommons', + u'NowCommons', + ], + 'shared': [u''], +} + + +licenseTemplates = { + 'de': [(u'\{\{Bild-CC-by-sa/3\.0/de\}\}[\s\r\n]*\{\{Bild-CC-by-sa/3\.0\}\}[\s\r\n]*\{\{Bild-GFDL-Neu\}\}', + u'{{Self|Cc-by-sa-3.0-de|Cc-by-sa-3.0|GFDL|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%' + + u'(family)s.org %(lang)s.%(family)s]}}'), + (u'\{\{Bild-GFDL\}\}[\s\r\n]*\{\{Bild-CC-by-sa/(\d\.\d)\}\}', + u'{{Self|GFDL|Cc-by-sa-3.0-migrated|Cc-by-sa-\\1|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang' + + u')s.%(family)s.org %(lang)s.%(family)s]}}'), + (u'\{\{Bild-GFDL\}\}', + u'{{Self|GFDL|Cc-by-sa-3.0-migrated|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s' + + u'.org %(lang)s.%(family)s]}}'), + (u'\{\{Bild-CC-by-sa/(\d\.\d)\}\}', + u'{{Self|Cc-by-sa-\\1|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org %(lang)s.' + + u'%(family)s]}}'), + (u'\{\{Bild-CC-by-sa/(\d\.\d)/de\}\}', + u'{{Self|Cc-by-sa-\\1-de|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org' + + u' %(lang)s.%(family)s]}}'), + (u'\{\{Bild-CC-by/(\d\.\d)\}\}', + u'{{Self|Cc-by-\\1|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org' + + u' %(lang)s.%(family)s]}}'), + (u'\{\{Bild-CC-by/(\d\.\d)/de\}\}', + u'{{Self|Cc-by-\\1-de|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org' + + u' %(lang)s.%(family)s]}}'), + ], + 'en': [(u'\{\{(self|self2)\|([^\}]+)\}\}', + u'{{Self|\\2|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org' + + u' %(lang)s.%(family)s]}}'), + (u'\{\{(GFDL-self|GFDL-self-no-disclaimers)\|([^\}]+)\}\}', + u'{{Self|GFDL|\\2|author=[[:%(lang)s:User:%(author)s|%(author)s]] at [http://%(lang)s.%(family)s.org' + + u'%(lang)s.%(family)s]}}'), + (u'\{\{GFDL-self-with-disclaimers\|([^\}]+)\}\}', + u'{{Self|GFDL-with-disclaimers|\\1|author=[[:%(lang)s:User:%(author)s|%(author)s]] at' + + u'[http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), + (u'\{\{PD-self(\|date=[^\}]+)?\}\}', + u'{{PD-user-w|%(lang)s|%(family)s|%(author)s}}'), + (u'\{\{Multilicense replacing placeholder(\|[^\}\|=]+=[^\}\|]+)*(?P<migration>\|[^\}\|=]+=[^\}\|]+)' + + u'(\|[^\}\|=]+=[^\}\|]+)*\}\}', + u'{{Self|GFDL|Cc-by-sa-2.5,2.0,1.0\\g<migration>|author=[[:%(lang)s:User:%(author)s|%(author)s]]' + + u' at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), + (u'\{\{Multilicense replacing placeholder new(\|class=[^\}]+)?\}\}', + u'{{Self|GFDL|Cc-by-sa-3.0,2.5,2.0,1.0|author=[[:%(lang)s:User:%(author)s|%(author)s]] at' + + u'[http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), + ], + 'lb': [(u'\{\{(self|self2)\|([^\}]+)\}\}', u'{{Self|\\2|author=[[:%(lang)s:User:%(author)s|%(author)s]]' + + u' at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]}}'), + ], + 'nds-nl': [(u'\{\{PD-eigenwark\}\}', u'{{PD-user-w|%(lang)s|%(family)s|%(author)s}}'), + ], + 'shared': [(u'\{\{(self|self2)\|([^\}]+)\}\}', u'{{Self|\\2|author=%(author)s at old wikivoyage shared}}'), + ], +} + +sourceGarbage = { + 'de': [u'==\s*Beschreibung,\sQuelle\s*==', + u'==\s*Beschrieving\s*==', + u'==\s*\[\[Wikipedia:Lizenzvorlagen für Bilder\|Lizenz\]\]\s*==', + ], + 'en': [u'==\s*Description\s*==', + u'==\s*Summary\s*==', + u'==\s*Licensing:?\s*==', + u'\{\{' + u'(Copy to Wikimedia Commons|Move to Commons|Move to commons|' + u'Move to Wikimedia Commons|Copy to commons|Mtc|MtC|MTC|CWC|CtWC|' + u'CTWC|Ctwc|Tocommons|Copy to Commons|To Commons|Movetocommons|' + u'Move to Wikimedia commons|Move-to-commons|Commons ok|ToCommons|' + u'To commons|MoveToCommons|Copy to wikimedia commons|' + u'Upload to commons|CopyToCommons|Copytocommons|MITC|MovetoCommons|' + u'Do move to Commons|Orphan image)' + u'(\|[^\}]+)?\}\}' + ], + 'lb': [u'==\s*Résumé\s*==', + u'==\s*Lizenz:\s*==', + ], + 'nds-nl': [u'==\s*Licentie\s*==', + u'\{\{DEFAULTSORT:\{\{PAGENAME\}\}\}\}', + ], + 'shared': [u'==\s*Beschreibung,\sQuelle\s*==', + u'==\s*Licensing:?\s*==', + ], +} + +informationTemplate = { + 'de': 'Information', + 'en': 'Information', + 'nds-nl': 'Information', + 'shared': 'Information', +} + +informationFields = { + 'de': { + u'anmerkungen': u'remarks', # FIXME: More flexible + u'beschreibung': u'description', + u'quelle': u'source', + u'datum': u'date', + u'urheber': u'author', + u'permission': u'permission', + u'andere Versione': u'other versions', + }, + 'en': { + u'location': u'remarks', + u'description': u'description', + u'source': u'source', + u'date': u'date', + u'author': u'author', + u'permission': u'permission', + u'other versions': u'other versions', + }, + 'nds-nl': { + u'location': u'remarks', + u'description': u'description', + u'source': u'source', + u'date': u'date', + u'author': u'author', + u'permission': u'permission', + u'other versions': u'other versions', + }, + 'shared': { + u'description': u'description', + u'source': u'source', + u'date': u'date', + u'author': u'author', + u'permission': u'permission', + u'other versions': u'other versions', + }, +} + + +def supportedSite(): + """Check if this site is supported.""" + site = pywikibot.Site() + lang = site.code + + lists = [ + nowCommonsTemplate, + moveToCommonsTemplate, + skipTemplates, + licenseTemplates, + sourceGarbage, + ] + for l in lists: + if not l.get(lang): + return False + return True + + +class imageFetcher(threading.Thread): + + """Tries to fetch information for all images in the generator.""" + + def __init__(self, pagegenerator, prefetchQueue): + self.pagegenerator = pagegenerator + self.prefetchQueue = prefetchQueue + imagerecat.initLists() + threading.Thread.__init__(self) + + def run(self): + for page in self.pagegenerator: + self.processImage(page) + self.prefetchQueue.put(None) + print(u'Fetched all images.') + return True + + def processImage(self, page): + """Work on a single image.""" + if page.exists() and (page.namespace() == 6) and \ + (not page.isRedirectPage()): + imagepage = pywikibot.FilePage(page.site(), page.title()) + + # First do autoskip. + if self.doiskip(imagepage): + pywikibot.output( + u'Skipping %s : Got a template on the skip list.' + % page.title()) + return False + + text = imagepage.get() + foundMatch = False + for (regex, replacement) in licenseTemplates[page.site.language()]: + match = re.search(regex, text, flags=re.IGNORECASE) + if match: + foundMatch = True + if not foundMatch: + pywikibot.output( + u'Skipping %s : No suitable license template was found.' + % page.title()) + return False + self.prefetchQueue.put(self.getNewFields(imagepage)) + + def doiskip(self, imagepage): + """Skip this image or not. + + Returns True if the image is on the skip list, otherwise False + + """ + for template in imagepage.templates(): + if template in skipTemplates[imagepage.site.language()]: + pywikibot.output( + u'Found %s which is on the template skip list' % template) + return True + return False + + def getNewFields(self, imagepage): + """Build a new description based on the imagepage.""" + if u'{{Information' in imagepage.get() or \ + u'{{information' in imagepage.get(): + (description, date, source, author, permission, + other_versions) = self.getNewFieldsFromInformation(imagepage) + else: + (description, date, source, + author) = self.getNewFieldsFromFreetext(imagepage) + permission = u'' + other_versions = u'' + + licensetemplate = self.getNewLicensetemplate(imagepage) + categories = self.getNewCategories(imagepage) + return {u'imagepage': imagepage, + u'filename': imagepage.title(withNamespace=False), + u'description': description, + u'date': date, + u'source': source, + u'author': author, + u'permission': permission, + u'other_versions': other_versions, + u'licensetemplate': licensetemplate, + u'categories': categories, + u'skip': False} + + def getNewFieldsFromInformation(self, imagepage): + """Try to extract fields from current information template for the new information template.""" + # fields = [u'location', u'description', u'source', u'date', u'author', u'permission', u'other versions'] + # FIXME: The implementation for German has to be checked for the "strange" fields + + description = u'' + source = u'' + date = u'' + author = u'' + permission = u'' + other_versions = u'' + contents = {} + + for key, value in informationFields.get( + imagepage.site.language()).items(): + contents[value] = u'' + + templates = imagepage.templatesWithParams() + + for (template, params) in templates: + if template == u'Information': + for param in params: + # Split at = + (field, sep, value) = param.partition(u'=') + # To lowercase, remove underscores and strip of spaces + field = field.lower().replace(u'_', u' ').strip() + # See if first part is in fields list + if field in informationFields.get(imagepage.site.language()).keys(): + # Ok, field is good, store it. + contents[informationFields.get(imagepage.site.language()).get(field)] = value.strip() + + # We now got the contents from the old information template. Let's get the info for the new one + + # Description + # FIXME: Add {{<lang>|<original text>}} if <lang is valid at Commons + if contents[u'description']: + description = self.convertLinks(contents[u'description'], + imagepage.site()) + if contents.get(u'remarks') and contents[u'remarks']: + if description == u'': + description = self.convertLinks(contents[u'remarks'], + imagepage.site()) + else: + description += u'<BR/>\n' + self.convertLinks( + contents[u'remarks'], imagepage.site()) + + # Source + source = self.getSource(imagepage, + source=self.convertLinks(contents[u'source'], + imagepage.site())) + + # Date + if contents[u'date']: + date = contents[u'date'] + else: + date = self.getUploadDate(imagepage) + + # Author + if not (contents[u'author'] == u'' or + contents[u'author'] == self.getAuthor(imagepage)): + author = self.convertLinks(contents[u'author'], imagepage.site()) + else: + author = self.getAuthorText(imagepage) + + # Permission + # Still have to filter out crap like "see below" or "yes" + if contents[u'permission']: + # Strip of the license temlate if it's in the permission section + for (regex, repl) in licenseTemplates[imagepage.site.language()]: + contents[u'permission'] = re.sub(regex, u'', + contents[u'permission'], + flags=re.IGNORECASE) + permission = self.convertLinks(contents[u'permission'], + imagepage.site()) + + # Other_versions + if contents[u'other versions']: + other_versions = self.convertLinks(contents[u'other versions'], + imagepage.site()) + + return (description, date, source, author, permission, other_versions) + + def getNewFieldsFromFreetext(self, imagepage): + """Try to extract fields from free text for the new information template.""" + text = imagepage.get() + # text = re.sub(u'== Summary ==', u'', text, re.IGNORECASE) + # text = re.sub(u'== Licensing ==', u'', text, re.IGNORECASE) + # text = re.sub(u'\{\{(self|self2)\|[^\}]+\}\}', u'', text, re.IGNORECASE) + + for toRemove in sourceGarbage[imagepage.site.language()]: + text = re.sub(toRemove, u'', text, flags=re.IGNORECASE) + + for (regex, repl) in licenseTemplates[imagepage.site.language()]: + text = re.sub(regex, u'', text, flags=re.IGNORECASE) + + text = pywikibot.removeCategoryLinks(text, imagepage.site()).strip() + + description = self.convertLinks(text.strip(), imagepage.site()) + date = self.getUploadDate(imagepage) + source = self.getSource(imagepage) + author = self.getAuthorText(imagepage) + return (description, date, source, author) + + def getUploadDate(self, imagepage): + """Get the original upload date for usage. + + The date is put in the date field of the new + information template. If we really have nothing better. + + """ + uploadtime = imagepage.getFileVersionHistory()[-1][0] + uploadDatetime = datetime.strptime(uploadtime, u'%Y-%m-%dT%H:%M:%SZ') + return (u'{{Date|' + str(uploadDatetime.year) + u'|' + str(uploadDatetime.month) + u'|' + str(uploadDatetime.day) + + u'}} (original upload date)') + + def getSource(self, imagepage, source=u''): + """Get the text to put in the source field of the new information template.""" + site = imagepage.site() + lang = site.code + family = site.family.name + if source == u'': + source = u'{{Own}}' + + return source.strip() + u'<BR />Transferred from [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' \ + % {u'lang': lang, u'family': family} + + def getAuthorText(self, imagepage): + """Get the original uploader to put in the author field of the new information template.""" + site = imagepage.site() + lang = site.code + family = site.family.name + + firstuploader = self.getAuthor(imagepage) + return (u'[[:%(lang)s:User:%(firstuploader)s|%(firstuploader)s]] at [http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' + % {u'lang': lang, u'family': family, + u'firstuploader': firstuploader}) + + def getAuthor(self, imagepage): + """Get the first uploader.""" + return imagepage.getFileVersionHistory()[-1][1].strip() + + def convertLinks(self, text, sourceSite): + """Convert links from the current wiki to Commons.""" + lang = sourceSite.code + family = sourceSite.family.name + conversions = [ + (u'\[\[([^\[\]\|]+)\|([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\2]]'), + (u'\[\[([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\1]]'), + ] + for (regex, replacement) in conversions: + text = re.sub(regex, replacement % {u'lang': lang, + u'family': family}, text) + return text + + def getNewLicensetemplate(self, imagepage): + """Get a license template to put on the image to be uploaded.""" + text = imagepage.get() + site = imagepage.site() + lang = site.code + family = site.family.name + result = u'' + for (regex, + replacement) in licenseTemplates[imagepage.site.language()]: + match = re.search(regex, text, flags=re.IGNORECASE) + if match: + result = re.sub(regex, replacement, match.group(0), + flags=re.IGNORECASE) + return result % {u'author': self.getAuthor(imagepage), + u'lang': lang, + u'family': family} + return result + + def getNewCategories(self, imagepage): + """Get categories for the image. + + Don't forget to filter. + + """ + result = u'' + (commonshelperCats, usage, + galleries) = imagerecat.getCommonshelperCats(imagepage) + newcats = imagerecat.applyAllFilters(commonshelperCats) + for newcat in newcats: + result += u'[[Category:' + newcat + u']] ' + return result + + +class userInteraction(threading.Thread): + + """Prompt all images to the user.""" + + def __init__(self, prefetchQueue, uploadQueue): + self.prefetchQueue = prefetchQueue + self.uploadQueue = uploadQueue + self.autonomous = False + threading.Thread.__init__(self) + + def run(self): + while True: + fields = self.prefetchQueue.get() + if fields: + self.processImage(fields) + else: + break + self.uploadQueue.put(None) + print(u'User worked on all images.') + return True + + def setAutonomous(self): + """Don't do any user interaction.""" + self.autonomous = True + return + + def processImage(self, fields): + """Work on a single image.""" + if self.autonomous: + # Check if the image already exists. Do nothing if the name is + # already taken. + CommonsPage = pywikibot.Page(pywikibot.Site('commons', + 'commons'), + u'File:' + fields.get('filename')) + if CommonsPage.exists(): + return False + else: + while True: + # Do the TkdialogICS to accept/reject and change te name + fields = TkdialogICS(fields).getnewmetadata() + + if fields.get('skip'): + pywikibot.output(u'Skipping %s : User pressed skip.' + % fields.get('imagepage').title()) + return False + + # Check if the image already exists + CommonsPage = pywikibot.Page(pywikibot.Site('commons', + 'commons'), + u'File:' + fields.get('filename')) + if not CommonsPage.exists(): + break + else: + pywikibot.output('Image already exists, pick another name ' + 'or skip this image') + # We dont overwrite images, pick another name, go to the start of the loop + + # Put the fields in the queue to be uploaded + self.uploadQueue.put(fields) + + +class TkdialogICS(Tkdialog): + + """The dialog window for image info.""" + + def __init__(self, fields): # imagepage, description, date, source, author, licensetemplate, categories): + self.root = Tkinter.Tk() + # "%dx%d%+d%+d" % (width, height, xoffset, yoffset) + # Always appear the same size and in the bottom-left corner + # FIXME : Base this on the screen size or make it possible for the user + # to configure this + + # Get all the relevant fields + super(TkdialogICS, self).__init__() + self.imagepage = fields.get('imagepage') + self.filename = fields.get('filename') + + self.description = fields.get('description') + self.date = fields.get('date') + self.source = fields.get('source') + self.author = fields.get('author') + self.permission = fields.get('permission') + self.other_versions = fields.get('other_versions') + + self.licensetemplate = fields.get('licensetemplate') + self.categories = fields.get('categories') + self.skip = False + + # Start building the page + self.root.geometry("1500x400+100-100") + self.root.title(self.filename) + + self.url = self.imagepage.permalink() + self.scrollbar = Tkinter.Scrollbar(self.root, orient=Tkinter.VERTICAL) + + self.old_description = Tkinter.Text(self.root) + self.old_description.insert(Tkinter.END, self.imagepage.get().encode('utf-8')) + self.old_description.config(state=Tkinter.DISABLED, height=8, width=140, padx=0, pady=0, wrap=Tkinter.WORD, + yscrollcommand=self.scrollbar.set) + + self.scrollbar.config(command=self.old_description.yview) + + self.old_description_label = Tkinter.Label(self.root, + text=u'The old description was : ') + self.new_description_label = Tkinter.Label(self.root, + text=u'The new fields are : ') + self.filename_label = Tkinter.Label(self.root, text=u'Filename : ') + self.information_description_label = Tkinter.Label(self.root, + text=u'Description : ') + self.information_date_label = Tkinter.Label(self.root, text=u'Date : ') + self.information_source_label = Tkinter.Label(self.root, text=u'Source : ') + self.information_author_label = Tkinter.Label(self.root, text=u'Author : ') + self.information_permission_label = Tkinter.Label(self.root, text=u'Permission : ') + self.information_other_versions_label = Tkinter.Label(self.root, text=u'Other versions : ') + + self.information_licensetemplate_label = Tkinter.Label(self.root, + text=u'License : ') + self.information_categories_label = Tkinter.Label(self.root, + text=u'Categories : ') + + self.filename_field = Tkinter.Entry(self.root) + self.information_description = Tkinter.Entry(self.root) + self.information_date = Tkinter.Entry(self.root) + self.information_source = Tkinter.Entry(self.root) + self.information_author = Tkinter.Entry(self.root) + self.information_permission = Tkinter.Entry(self.root) + self.information_other_versions = Tkinter.Entry(self.root) + self.information_licensetemplate = Tkinter.Entry(self.root) + self.information_categories = Tkinter.Entry(self.root) + + self.field_width = 120 + + self.filename_field.config(width=self.field_width) + self.information_description.config(width=self.field_width) + self.information_date.config(width=self.field_width) + self.information_source.config(width=self.field_width) + self.information_author.config(width=self.field_width) + self.information_permission.config(width=self.field_width) + self.information_other_versions.config(width=self.field_width) + self.information_licensetemplate.config(width=self.field_width) + self.information_categories.config(width=self.field_width) + + self.filename_field.insert(0, self.filename) + self.information_description.insert(0, self.description) + self.information_date.insert(0, self.date) + self.information_source.insert(0, self.source) + self.information_author.insert(0, self.author) + self.information_permission.insert(0, self.permission) + self.information_other_versions.insert(0, self.other_versions) + self.information_licensetemplate.insert(0, self.licensetemplate) + self.information_categories.insert(0, self.categories) + + self.browser_button = Tkinter.Button(self.root, text='View in browser', + command=self.open_in_browser) + self.skip_button = Tkinter.Button(self.root, text="Skip", command=self.skipFile) + self.ok_button = Tkinter.Button(self.root, text="OK", command=self.ok_file) + + # Start grid + self.old_description_label.grid(row=0, column=0, columnspan=3) + + self.old_description.grid(row=1, column=0, columnspan=3) + self.scrollbar.grid(row=1, column=3) + self.new_description_label.grid(row=2, column=0, columnspan=3) + + # All the labels for the new fields + self.filename_label.grid(row=3, column=0) + self.information_description_label.grid(row=4, column=0) + self.information_date_label.grid(row=5, column=0) + self.information_source_label.grid(row=6, column=0) + self.information_author_label.grid(row=7, column=0) + self.information_permission_label.grid(row=8, column=0) + self.information_other_versions_label.grid(row=9, column=0) + self.information_licensetemplate_label.grid(row=10, column=0) + self.information_categories_label.grid(row=11, column=0) + + # The new fields + self.filename_field.grid(row=3, column=1, columnspan=3) + self.information_description.grid(row=4, column=1, columnspan=3) + self.information_date.grid(row=5, column=1, columnspan=3) + self.information_source.grid(row=6, column=1, columnspan=3) + self.information_author.grid(row=7, column=1, columnspan=3) + self.information_permission.grid(row=8, column=1, columnspan=3) + self.information_other_versions.grid(row=9, column=1, columnspan=3) + self.information_licensetemplate.grid(row=10, column=1, columnspan=3) + self.information_categories.grid(row=11, column=1, columnspan=3) + + # The buttons at the bottom + self.ok_button.grid(row=12, column=3, rowspan=2) + self.skip_button.grid(row=12, column=2, rowspan=2) + self.browser_button.grid(row=12, column=1, rowspan=2) + + def ok_file(self): + """The user pressed the OK button.""" + self.filename = self.filename_field.get() + self.description = self.information_description.get() + self.date = self.information_date.get() + self.source = self.information_source.get() + self.author = self.information_author.get() + self.permission = self.information_permission.get() + self.other_versions = self.information_other_versions.get() + self.licensetemplate = self.information_licensetemplate.get() + self.categories = self.information_categories.get() + + self.root.destroy() + + def getnewmetadata(self): + """Activate the dialog and return the new name and if the image is skipped.""" + self.root.mainloop() + + return {u'imagepage': self.imagepage, + u'filename': self.filename, + u'description': self.description, + u'date': self.date, + u'source': self.source, + u'author': self.author, + u'permission': self.permission, + u'other_versions': self.other_versions, + u'licensetemplate': self.licensetemplate, + u'categories': self.categories, + u'skip': self.skip} + + def open_in_browser(self): + """The user pressed the View in browser button.""" + webbrowser.open(self.url) + + +class uploader(threading.Thread): + + """Upload all images.""" + + def __init__(self, uploadQueue): + self.uploadQueue = uploadQueue + self.checktemplate = True + threading.Thread.__init__(self) + + def run(self): + while True: # Change later + fields = self.uploadQueue.get() + if fields: + self.processImage(fields) + else: + break + return True + + def nochecktemplate(self): + """Don't want to add {{BotMoveToCommons}}.""" + self.checktemplate = False + return + + def processImage(self, fields): + """Work on a single image.""" + cid = self.buildNewImageDescription(fields) + pywikibot.output(cid) + bot = upload.UploadRobot(url=fields.get('imagepage').fileUrl(), + description=cid, useFilename=fields.get('filename'), + keepFilename=True, verifyDescription=False, + ignoreWarning=True, + targetSite=pywikibot.Site('commons', 'commons')) + bot.run() + + self.tagNowcommons(fields.get('imagepage'), fields.get('filename')) + self.replaceUsage(fields.get('imagepage'), fields.get('filename')) + + def buildNewImageDescription(self, fields): + """Build a new information template.""" + site = fields.get('imagepage').site() + lang = site.code + family = site.family.name + + cid = u'' + if self.checktemplate: + cid += u'\n{{BotMoveToCommons|%(lang)s.%(family)s|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}|day={{subst:CURRENTDAY}}}}\n' \ + % {u'lang': lang, u'family': family} + cid += u'== {{int:filedesc}} ==\n' + cid += u'{{Information\n' + cid += u'|description=%(description)s\n' % fields + cid += u'|date=%(date)s\n' % fields + cid += u'|source=%(source)s\n' % fields + cid += u'|author=%(author)s\n' % fields + cid += u'|permission=%(permission)s\n' % fields + cid += u'|other_versions=%(other_versions)s\n' % fields + cid += u'}}\n' + cid += u'== {{int:license}} ==\n' + cid += u'%(licensetemplate)s\n' % fields + cid += u'\n' + cid += self.getOriginalUploadLog(fields.get('imagepage')) + cid += u'__NOTOC__\n' + if fields.get('categories').strip() == u'': + cid = cid + u'{{Subst:Unc}}' + else: + cid = cid + u'%(categories)s\n' % fields + return cid + + def getOriginalUploadLog(self, imagepage): + """Get the original upload log to put at the bottom of the image description page at Commons.""" + filehistory = imagepage.getFileVersionHistory() + filehistory.reverse() + + site = imagepage.site() + lang = site.code + family = site.family.name + + sourceimage = imagepage.site.get_address( + imagepage.title()).replace(u'&redirect=no&useskin=monobook', u'') + + result = u'== {{Original upload log}} ==\n' + result += (u'The original description page is/was [http://%(lang)s.%(family)s.org%(sourceimage)s here]. All following' + + u'user names refer to %(lang)s.%(family)s.\n' + % {u'lang': lang, u'family': family, + u'sourceimage': sourceimage}) + for (timestamp, username, resolution, size, comment) in filehistory: + date = datetime.strptime( + timestamp, u'%Y-%m-%dT%H:%M:%SZ').strftime('%Y-%m-%d %H:%M') + result += (u'* %(date)s [[:%(lang)s:user:%(username)s|%(username)s]] %(resolution)s' + + u' (%(size)s bytes) \'\'<nowiki>%(comment)s</nowiki>\'\'\n' % { + u'lang': lang, + u'family': family, + u'date': date, + u'username': username, + u'resolution': resolution, + u'size': size, + u'comment': comment}) + + return result + + def tagNowcommons(self, imagepage, filename): + """Tagged the imag which has been moved to Commons for deletion.""" + if pywikibot.Page(pywikibot.Site('commons', 'commons'), + u'File:' + filename).exists(): + # Get a fresh copy, force to get the page so we dont run into edit + # conflicts + imtxt = imagepage.get(force=True) + + # Remove the move to commons templates + if imagepage.site.language() in moveToCommonsTemplate: + for moveTemplate in moveToCommonsTemplate[imagepage.site.language()]: + imtxt = re.sub(u'(?i)\{\{' + moveTemplate + + u'[^\}]*\}\}', u'', imtxt) + + # add {{NowCommons}} + if imagepage.site.language() in nowCommonsTemplate: + addTemplate = nowCommonsTemplate[ + imagepage.site.language()] % filename + else: + addTemplate = nowCommonsTemplate['_default'] % filename + + commentText = i18n.twtranslate( + imagepage.site(), 'commons-file-now-available', + {'localfile': imagepage.title(withNamespace=False), + 'commonsfile': filename}) + + pywikibot.showDiff(imagepage.get(), imtxt + addTemplate) + imagepage.put(imtxt + addTemplate, comment=commentText) + + def replaceUsage(self, imagepage, filename): + """If the image is uploaded under a different name, replace all usage.""" + if imagepage.title(withNamespace=False) != filename: + gen = pagegenerators.FileLinksGenerator(imagepage) + preloadingGen = pagegenerators.PreloadingGenerator(gen) + + moveSummary = i18n.twtranslate( + imagepage.site(), 'commons-file-moved', + {'localfile': imagepage.title(withNamespace=False), + 'commonsfile': filename}) + + imagebot = image.ImageRobot(generator=preloadingGen, + oldImage=imagepage.title(withNamespace=False), + newImage=filename, summary=moveSummary, + always=True, loose=True) + imagebot.run() + + +def main(*args): + generator = None + autonomous = False + checkTemplate = True + + # Load a lot of default generators + genFactory = pagegenerators.GeneratorFactory() + local_args = pywikibot.handle_args(args) + for arg in local_args: + if arg == '-nochecktemplate': + checkTemplate = False + elif arg == '-autonomous': + autonomous = True + else: + genFactory.handleArg(arg) + + generator = genFactory.getCombinedGenerator() + if not generator: + pywikibot.bot.suggest_help(missing_generator=True) + return False + + if not supportedSite(): + pywikibot.output(u'Sorry, this site is not supported (yet).') + return False + + pywikibot.warning(u'This is an experimental bot') + pywikibot.warning(u'It will only work on self published work images') + pywikibot.warning(u'This bot is still full of bugs') + pywikibot.warning(u'Use at your own risk!') + + pregenerator = pagegenerators.PreloadingGenerator(generator) + + prefetchQueue = Queue(maxsize=50) + uploadQueue = Queue(maxsize=200) + + imageFetcherThread = imageFetcher(pregenerator, prefetchQueue) + userInteractionThread = userInteraction(prefetchQueue, uploadQueue) + uploaderThread = uploader(uploadQueue) + + imageFetcherThread.daemon = False + userInteractionThread.daemon = False + uploaderThread.daemon = False + + if autonomous: + pywikibot.output(u'Bot is running in autonomous mode. There will be no ' + u'user interaction.') + userInteractionThread.setAutonomous() + + if not checkTemplate: + pywikibot.output(u'No check template will be added to the uploaded ' + u'files.') + uploaderThread.nochecktemplate() + + # Using the listed variables one may keep track of thread start status + # fetchDone = imageFetcherThread.start() + # userDone = userInteractionThread.start() + # uploadDone = uploaderThread.start() + + +if __name__ == "__main__": + main() diff --git a/scripts/imageharvest.py b/scripts/imageharvest.py new file mode 100644 index 0000000..c2d0705 --- /dev/null +++ b/scripts/imageharvest.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +""" +Bot for getting multiple images from an external site. + +It takes a URL as an argument and finds all images (and other files specified +by the extensions in 'fileformats') that URL is referring to, asking whether to +upload them. If further arguments are given, they are considered to be the text +that is common to the descriptions. + +A second use is to get a number of images that have URLs only differing in +numbers. To do this, use the command line option "-pattern", and give the URL +with the variable part replaced by '$' (if that character occurs in the URL +itself, you will have to change the bot code, my apologies). + +Other options: +-shown Choose images shown on the page as well as linked from it +-justshown Choose _only_ images shown on the page, not those linked +""" +# (C) Pywikibot team, 2004-2015 +# +# Distributed under the terms of the MIT license. +# +from __future__ import absolute_import, unicode_literals + +__version__ = '$Id$' + +import os + +import BeautifulSoup + +import pywikibot + +from pywikibot.tools import PY2 + +from scripts import upload + +if not PY2: + import urllib + from urllib.request import URLopener + + basestring = (str,) +else: + from urllib import URLopener + + +def get_imagelinks(url): + """Given a URL, get all images linked to by the page at that URL.""" + links = [] + uo = URLopener() + file = uo.open(url) + soup = BeautifulSoup.BeautifulSoup(file.read()) + file.close() + if not shown: + tagname = "a" + elif shown == "just": + tagname = "img" + else: + tagname = ["a", "img"] + + for tag in soup.findAll(tagname): + link = tag.get("src", tag.get("href", None)) + if link: + ext = os.path.splitext(link)[1].lower().strip('.') + if ext in fileformats: + links.append(urllib.basejoin(url, link)) + return links + + +def main(give_url, image_url, desc): + url = give_url + image_url = '' + if url == '': + if image_url: + url = pywikibot.input(u"What URL range should I check " + u"(use $ for the part that is changeable)") + else: + url = pywikibot.input(u"From what URL should I get the images?") + + if image_url: + minimum = 1 + maximum = 99 + answer = pywikibot.input( + u"What is the first number to check (default: 1)") + if answer: + minimum = int(answer) + answer = pywikibot.input( + u"What is the last number to check (default: 99)") + if answer: + maximum = int(answer) + + if not desc: + basicdesc = pywikibot.input( + u"What text should be added at the end of " + u"the description of each image from this url?") + else: + basicdesc = desc + + if image_url: + ilinks = [] + i = minimum + while i <= maximum: + ilinks += [url.replace("$", str(i))] + i += 1 + else: + ilinks = get_imagelinks(url) + + for image in ilinks: + if pywikibot.input_yn(u'Include image %s?' % image, default=False, automatic_quit=False): + desc = pywikibot.input(u"Give the description of this image:") + categories = [] + while True: + cat = pywikibot.input(u"Specify a category (or press enter to " + u"end adding categories)") + if not cat.strip(): + break + if ":" in cat: + categories.append(u"[[%s]]" % cat) + else: + categories.append(u"[[%s:%s]]" + % (mysite.namespace(14), cat)) + desc += "\r\n\r\n" + basicdesc + "\r\n\r\n" + \ + "\r\n".join(categories) + uploadBot = upload.UploadRobot(image, description=desc) + uploadBot.run() + elif answer == 's': + break + + +try: + url = u'' + image_url = False + shown = False + desc = [] + + for arg in pywikibot.handle_args(): + if arg == "-pattern": + image_url = True + elif arg == "-shown": + shown = True + elif arg == "-justshown": + shown = "just" + elif url == u'': + url = arg + else: + desc += [arg] + desc = ' '.join(desc) + + fileformats = ('jpg', 'jpeg', 'png', 'gif', 'svg', 'ogg') + mysite = pywikibot.Site() + main(url, image_url, desc) +finally: + pywikibot.stopme() diff --git a/scripts/panoramiopicker.py b/scripts/panoramiopicker.py new file mode 100644 index 0000000..d5927ed --- /dev/null +++ b/scripts/panoramiopicker.py @@ -0,0 +1,379 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Tool to copy a Panoramio set to Commons.""" +# +# (C) Multichill, 2010 +# (C) Pywikibot team, 2010-2015 +# +# Distributed under the terms of the MIT license. +# +from __future__ import absolute_import, unicode_literals + +__version__ = '$Id$' + +import base64 +import hashlib +import json +import re +import socket +import StringIO + +from BeautifulSoup import BeautifulSoup + +import pywikibot + +from pywikibot import config + +from pywikibot.tools import PY2 + +from scripts import imagerecat, upload + +if not PY2: + from urllib.request import urlopen +else: + from urllib import urlopen + +try: + from pywikibot.userinterfaces.gui import Tkdialog +except ImportError as _tk_error: + Tkdialog = None + + +def isAllowedLicense(photoInfo=None): + """ + Check if the image contains the right license. + + TODO: Maybe add more licenses + + """ + allowed = [u'by-sa'] + return photoInfo[u'license'] in allowed + + +def downloadPhoto(photoUrl=''): + """ + Download the photo and store it in a StrinIO.StringIO object. + + TODO: Add exception handling + + """ + imageFile = urlopen(photoUrl).read() + return StringIO.StringIO(imageFile) + + +def findDuplicateImages(photo=None, + site=pywikibot.Site(u'commons', u'commons')): + """Return list of duplicate images. + + Takes the photo, calculates the SHA1 hash and asks the mediawiki api + for a list of duplicates. + + TODO: Add exception handling, fix site thing + + """ + hashObject = hashlib.sha1() + hashObject.update(photo.getvalue()) + return site.allimages(sha1=base64.b16encode(hashObject.digest())) + + +def getLicense(photoInfo=None): + """Adding license to the Panoramio API with a beautiful soup hack.""" + photoInfo['license'] = u'c' + page = urlopen(photoInfo.get(u'photo_url')) + data = page.read() + soup = BeautifulSoup(data) + if soup.find("div", {'id': 'photo-info'}): + pointer = soup.find("div", {'id': 'photo-info'}) + if pointer.find("div", {'id': 'photo-details'}): + pointer = pointer.find("div", {'id': 'photo-details'}) + if pointer.find("ul", {'id': 'details'}): + pointer = pointer.find("ul", {'id': 'details'}) + if pointer.find("li", {'class': 'license by-sa'}): + photoInfo['license'] = u'by-sa' + # Does Panoramio have more license options? + + return photoInfo + + +def getFilename(photoInfo=None, site=pywikibot.Site(u'commons', u'commons'), + project=u'Panoramio'): + """Build a good filename for the upload. + + The name is based on the username and the title. Prevents naming collisions. + """ + username = photoInfo.get(u'owner_name') + title = photoInfo.get(u'photo_title') + if title: + title = cleanUpTitle(title) + else: + title = u'' + + if pywikibot.Page(site, u'File:%s - %s - %s.jpg' + % (project, username, title)).exists(): + i = 1 + while True: + if (pywikibot.Page(site, u'File:%s - %s - %s (%s).jpg' + % (project, username, title, str(i))).exists()): + i += 1 + else: + return u'%s - %s - %s (%s).jpg' % (project, username, title, + str(i)) + else: + return u'%s - %s - %s.jpg' % (project, username, title) + + +def cleanUpTitle(title): + """Clean up the title of a potential mediawiki page. + + Otherwise the title of the page might not be allowed by the software. + """ + title = title.strip() + title = re.sub(u"[<{\\[]", u"(", title) + title = re.sub(u"[>}\\]]", u")", title) + title = re.sub(u"[ _]?\$!\$", u"", title) + title = re.sub(u",:[ _]", u", ", title) + title = re.sub(u"[;:][ _]", u", ", title) + title = re.sub(u"[\t\n ]+", u" ", title) + title = re.sub(u"[\r\n ]+", u" ", title) + title = re.sub(u"[\n]+", u"", title) + title = re.sub(u"[?!]([.\"]|$)", u"\\1", title) + title = re.sub(u"[&#%?!]", u"^", title) + title = re.sub(u"[;]", u",", title) + title = re.sub(u"[/+\\\\:]", u"-", title) + title = re.sub(u"--+", u"-", title) + title = re.sub(u",,+", u",", title) + title = re.sub(u"[-,^]([.]|$)", u"\\1", title) + title = title.replace(u" ", u"_") + return title + + +def getDescription(photoInfo=None, panoramioreview=False, reviewer=u'', + override=u'', addCategory=u''): + """Build description for the image.""" + desc = u'' + desc += u'{{Information\n' + desc += u'|description=%(photo_title)s\n' + desc += u'|date=%(upload_date)s (upload date)\n' + desc += u'|source=[%(photo_url)s Panoramio]\n' + desc += u'|author=[%(owner_url)s?with_photo_id=%(photo_id)s %(owner_name)s] \n' + desc += u'|permission=\n' + desc += u'|other_versions=\n' + desc += u'|other_fields=\n' + desc += u'}}\n' + if photoInfo.get(u'latitude') and photoInfo.get(u'longitude'): + desc += u'{{Location dec|%(latitude)s|%(longitude)s|source:Panoramio}}\n' + desc += u'\n' + desc += u'=={{int:license-header}}==\n' + + if override: + desc += override + else: + if photoInfo.get(u'license') == u'by-sa': + desc += u'{{Cc-by-sa-3.0}}\n' + if panoramioreview: + desc += u'{{Panoramioreview|%s|{{subst:CURRENTYEAR}}-{{subst:CURRENTMONTH}}-{{subst:CURRENTDAY2}}}}\n' % (reviewer,) + else: + desc += u'{{Panoramioreview}}\n' + + desc += u'\n' + cats = u'' + if addCategory: + desc += u'\n[[Category:%s]]\n' % (addCategory,) + cats = True + + # Get categories based on location + if photoInfo.get(u'latitude') and photoInfo.get(u'longitude'): + cats = imagerecat.getOpenStreetMapCats(photoInfo.get(u'latitude'), + photoInfo.get(u'longitude')) + cats = imagerecat.applyAllFilters(cats) + for cat in cats: + desc += u'[[Category:%s]]\n' % (cat,) + if not cats: + desc += u'{{subst:Unc}}\n' + + return desc % photoInfo + + +def processPhoto(photoInfo=None, panoramioreview=False, reviewer=u'', + override=u'', addCategory=u'', autonomous=False): + """Process a single Panoramio photo.""" + if isAllowedLicense(photoInfo) or override: + # Should download the photo only once + photo = downloadPhoto(photoInfo.get(u'photo_file_url')) + + # Don't upload duplicate images, should add override option + duplicates = findDuplicateImages(photo) + if duplicates: + pywikibot.output(u'Found duplicate image at %s' % duplicates.pop()) + else: + filename = getFilename(photoInfo) + pywikibot.output(filename) + description = getDescription(photoInfo, panoramioreview, + reviewer, override, addCategory) + + pywikibot.output(description) + if not autonomous: + (newDescription, newFilename, skip) = Tkdialog( + description, photo, filename).show_dialog() + else: + newDescription = description + newFilename = filename + skip = False +# pywikibot.output(newPhotoDescription) +# if (pywikibot.Page(title=u'File:'+ filename, +# site=pywikibot.Site()).exists()): +# # I should probably check if the hash is the same and if not upload +# # it under a different name +# pywikibot.output(u'File:' + filename + u' already exists!') +# else: + # Do the actual upload + # Would be nice to check before I upload if the file is already at + # Commons + # Not that important for this program, but maybe for derived + # programs + if not skip: + bot = upload.UploadRobot(photoInfo.get(u'photo_file_url'), + description=newDescription, + useFilename=newFilename, + keepFilename=True, + verifyDescription=False) + bot.upload_image(debug=False) + return 1 + return 0 + + +def getPhotos(photoset=u'', start_id='', end_id='', interval=100): + """Loop over a set of Panoramio photos.""" + i = 0 + has_more = True + url = u'http://www.panoramio.com/map/get_panoramas.php?set=%s&from=%s&to=%s&size=or…' + while has_more: + gotInfo = False + maxtries = 10 + tries = 0 + while not gotInfo: + try: + if tries < maxtries: + tries += 1 + panoramioApiPage = urlopen(url % (photoset, i, i + interval)) + contents = panoramioApiPage.read().decode('utf-8') + gotInfo = True + i += interval + else: + break + except IOError: + pywikibot.output(u'Got an IOError, let\'s try again') + except socket.timeout: + pywikibot.output(u'Got a timeout, let\'s try again') + + print(contents) + metadata = json.loads(contents) + photos = metadata.get(u'photos') + for photo in photos: + yield photo + has_more = metadata.get(u'has_more') + return + + +def usage(): + """Print usage information. + + TODO : Need more. + """ + pywikibot.output( + u"Panoramiopicker is a tool to transfer Panaramio photos to Wikimedia " + u"Commons") + pywikibot.output(u"-set:<set_id>\n") + return + + +def main(*args): + """Process command line arguments and perform task.""" + site = pywikibot.Site(u'commons', u'commons') + config.family = site.family + config.lang = site.lang +# imagerecat.initLists() + + photoset = u'' # public (popular photos), full (all photos), user ID number + start_id = u'' + end_id = u'' + addCategory = u'' + autonomous = False + totalPhotos = 0 + uploadedPhotos = 0 + + # Do we mark the images as reviewed right away? + if config.panoramio['review']: + panoramioreview = config.panoramio['review'] + else: + panoramioreview = False + + # Set the Panoramio reviewer + if config.panoramio['reviewer']: + reviewer = config.panoramio['reviewer'] + elif 'commons' in config.sysopnames['commons']: + print(config.sysopnames['commons']) + reviewer = config.sysopnames['commons']['commons'] + elif 'commons' in config.usernames['commons']: + reviewer = config.usernames['commons']['commons'] + else: + reviewer = u'' + + # Should be renamed to overrideLicense or something like that + override = u'' + local_args = pywikibot.handle_args(args) + for arg in local_args: + if arg.startswith('-set'): + if len(arg) == 4: + photoset = pywikibot.input(u'What is the set?') + else: + photoset = arg[5:] + elif arg.startswith('-start_id'): + if len(arg) == 9: + start_id = pywikibot.input( + u'What is the id of the photo you want to start at?') + else: + start_id = arg[10:] + elif arg.startswith('-end_id'): + if len(arg) == 7: + end_id = pywikibot.input( + u'What is the id of the photo you want to end at?') + else: + end_id = arg[8:] + elif arg == '-panoramioreview': + panoramioreview = True + elif arg.startswith('-reviewer'): + if len(arg) == 9: + reviewer = pywikibot.input(u'Who is the reviewer?') + else: + reviewer = arg[10:] + elif arg.startswith('-override'): + if len(arg) == 9: + override = pywikibot.input(u'What is the override text?') + else: + override = arg[10:] + elif arg.startswith('-addcategory'): + if len(arg) == 12: + addCategory = pywikibot.input( + u'What category do you want to add?') + else: + addCategory = arg[13:] + elif arg == '-autonomous': + autonomous = True + + if photoset: + for photoInfo in getPhotos(photoset, start_id, end_id): + photoInfo = getLicense(photoInfo) + # time.sleep(10) + uploadedPhotos += processPhoto(photoInfo, panoramioreview, + reviewer, override, addCategory, + autonomous) + totalPhotos += 1 + else: + usage() + pywikibot.output(u'Finished running') + pywikibot.output(u'Total photos: ' + str(totalPhotos)) + pywikibot.output(u'Uploaded photos: ' + str(uploadedPhotos)) + +if __name__ == "__main__": + main() diff --git a/tests/script_tests.py b/tests/script_tests.py index 9dfcc94..0e94376 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -23,17 +23,26 @@ scripts_path = os.path.join(_root_dir, 'scripts') +if PY2: + TK_IMPORT = 'Tkinter' +else: + TK_IMPORT = 'tkinter' + # These dependencies are not always the package name which is in setup.py. # e.g. 'PIL.ImageTk' is a object provided by several different pypi packages, # and setup.py requests that 'Pillow' is installed to provide 'PIL.ImageTk'. # Here, it doesnt matter which pypi package was requested and installed. # Here, the name given to the module which will be imported is required. script_deps = { + 'imagecopy': [TK_IMPORT], + 'imagecopy_self': [TK_IMPORT], 'script_wui': ['crontab', 'lua'], # Note: package 'lunatic-python' provides module 'lua' 'flickrripper': ['flickrapi'], + 'imageharvest': ['BeautifulSoup'], 'match_images': ['PIL.ImageTk'], + 'panoramiopicker': ['BeautifulSoup'], 'states_redirect': ['pycountry'], 'patrol': ['mwlib'], 'weblinkchecker.py': ['memento_client'], @@ -138,8 +147,10 @@ 'imageuncat': 'WARNING: This script is primarily written for Wikimedia Commons', # script_input['interwiki'] above lists a title that should not exist 'interwiki': 'does not exist. Skipping.', + 'imageharvest': 'From what URL should I get the images', 'login': 'Logged in on ', 'pagefromfile': 'Please enter the file name', + 'panoramiopicker': 'Panoramiopicker is a tool to transfer Panaramio ', 'replace': 'Press Enter to use this automatic message', 'script_wui': 'Pre-loading all relevant page contents', 'shell': ('>>> ', 'Welcome to the'), -- To view, visit https://gerrit.wikimedia.org/r/196630 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I5ea3a2131badba22fdc5e99deb5c40a49f4f0998 Gerrit-PatchSet: 12 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Prianka <priyankajayaswal025(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

[Gerrit] Fix interwikiFormat support for Link - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Fix interwikiFormat support for Link ...................................................................... Fix interwikiFormat support for Link interwikiFormat is supposed to support a dictionary of Page or Link objects. There were several merges of core and compat 'textlib' to keep the two in sync, such as core's 8978bfb6, and the support for Link was broken somewhere along the way. Add tests for Link in addition to existing tests for Page. Also add a ValueError exception if it contains any other type. Change-Id: Ibb39ea8b3079083feec93249eb6abe4b997feb74 --- M pywikibot/textlib.py M tests/textlib_tests.py 2 files changed, 16 insertions(+), 5 deletions(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/textlib.py b/pywikibot/textlib.py index 18b9d7d..5998ac0 100644 --- a/pywikibot/textlib.py +++ b/pywikibot/textlib.py @@ -941,14 +941,15 @@ ar = interwikiSort(list(links.keys()), insite) s = [] for site in ar: - try: + if isinstance(links[site], pywikibot.Link): + links[site] = pywikibot.Page(links[site]) + if isinstance(links[site], pywikibot.Page): title = links[site].title(asLink=True, forceInterwiki=True, insite=insite) link = title.replace('[[:', '[[') s.append(link) - except AttributeError: - s.append(pywikibot.Site(site, insite.family).linkto( - links[site], othersite=insite)) + else: + raise ValueError('links dict must contain Page or Link objects') if insite.code in insite.family.interwiki_on_one_line: sep = u' ' else: diff --git a/tests/textlib_tests.py b/tests/textlib_tests.py index ff37a56..e50d5ef 100644 --- a/tests/textlib_tests.py +++ b/tests/textlib_tests.py @@ -104,7 +104,7 @@ cached = True - def test_interwiki_format(self): + def test_interwiki_format_Page(self): """Test formatting interwiki links using Page instances.""" interwikis = { 'de': pywikibot.Page(pywikibot.Link('de:German', self.site)), @@ -114,6 +114,16 @@ % {'LS': config.LS}, textlib.interwikiFormat(interwikis, self.site)) + def test_interwiki_format_Link(self): + """Test formatting interwiki links using Page instances.""" + interwikis = { + 'de': pywikibot.Link('de:German', self.site), + 'fr': pywikibot.Link('fr:French', self.site), + } + self.assertEqual('[[de:German]]%(LS)s[[fr:French]]%(LS)s' + % {'LS': config.LS}, + textlib.interwikiFormat(interwikis, self.site)) + class TestFormatCategory(DefaultDrySiteTestCase): -- To view, visit https://gerrit.wikimedia.org/r/240649 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ibb39ea8b3079083feec93249eb6abe4b997feb74 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

8 years, 7 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits September 2015