jenkins-bot has submitted this change and it was merged.
Change subject: DequeGenerator
......................................................................
DequeGenerator
Scripts may be written to dynamically populate a list of pages to work on.
Ideally this list would be created using a generator, however that can
require a large rewrite in the flow control.
To simplify the porting of scripts, this changeset adds classes
DequeGenerator and DequePreloadingGenerator classes that allow the page
list to be extended during iteration.
Implemented in makecat.
Change-Id: I9e67a4d9d0f1736071434fa84796f22732896088
---
M pywikibot/pagegenerators.py
M pywikibot/tools.py
M scripts/makecat.py
M tests/pagegenerators_tests.py
4 files changed, 72 insertions(+), 22 deletions(-)
Approvals:
XZise: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index eb8930a..b79dfda 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -28,7 +28,7 @@
import time
import pywikibot
from pywikibot import date, config, i18n
-from pywikibot.tools import deprecated_args
+from pywikibot.tools import deprecated_args, DequeGenerator
from pywikibot.comms import http
import pywikibot.data.wikidataquery as wdquery
@@ -1143,6 +1143,18 @@
yield i
+def DequePreloadingGenerator(generator, step=50):
+ assert(isinstance(generator, DequeGenerator))
+
+ while True:
+ page_count = min(len(generator), step)
+ if not page_count:
+ raise StopIteration
+
+ for page in PreloadingGenerator(generator, page_count):
+ yield page
+
+
def PreloadingItemGenerator(generator, step=50):
"""
Yield preloaded pages taken from another generator.
diff --git a/pywikibot/tools.py b/pywikibot/tools.py
index 6064d86..dcf4fd4 100644
--- a/pywikibot/tools.py
+++ b/pywikibot/tools.py
@@ -13,7 +13,7 @@
import time
import inspect
import re
-from collections import Mapping
+from collections import Mapping, deque
from distutils.version import Version
if sys.version_info[0] > 2:
@@ -320,6 +320,26 @@
EMPTY_DEFAULT = EmptyDefault()
+
+class DequeGenerator(deque):
+
+ """A generator that allows items to be added during
generating."""
+
+ def __iter__(self):
+ """Return the object which will be iterated."""
+ return self
+
+ def next(self):
+ """Python 3 iterator method."""
+ if len(self):
+ return self.popleft()
+ else:
+ raise StopIteration
+
+ def __next__(self):
+ """Python 3 iterator method."""
+ return self.next()
+
# Decorators
#
# Decorator functions without parameters are _invoked_ differently from
diff --git a/scripts/makecat.py b/scripts/makecat.py
index e523cff..42f4a7e 100644
--- a/scripts/makecat.py
+++ b/scripts/makecat.py
@@ -44,6 +44,7 @@
import codecs
import pywikibot
from pywikibot import date, pagegenerators, i18n, textlib
+from pywikibot.tools import DequeGenerator
def isdate(s):
@@ -197,7 +198,7 @@
removeparent = True
main = True
workingcatname = ''
- tocheck = []
+ tocheck = DequeGenerator()
for arg in pywikibot.handleArgs():
if arg.startswith('-nodate'):
skipdates = True
@@ -271,26 +272,14 @@
answer = workingcatname
pywikibot.output(u'' + answer)
pl = pywikibot.Page(mysite, answer)
- tocheck = []
checked[pl] = pl
include(pl)
- loaded = 0
- while tocheck:
- if loaded == 0:
- if len(tocheck) < 50:
- loaded = len(tocheck)
- else:
- loaded = 50
- tocheck = [x for x in pagegenerators.PreloadingGenerator(tocheck[:loaded])]
- if not checkbroken:
- if not tocheck[0].exists():
- pass
- else:
- asktoadd(tocheck[0])
- else:
- asktoadd(tocheck[0])
- tocheck = tocheck[1:]
- loaded -= 1
+
+ gen = pagegenerators.DequePreloadingGenerator(tocheck)
+
+ for page in gen:
+ if checkbroken or page.exists():
+ asktoadd(page)
finally:
try:
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index 44a99cd..dd85f1b 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -11,7 +11,12 @@
import pywikibot
from pywikibot import pagegenerators
-from tests.aspects import unittest, TestCase, WikidataTestCase
+from tests.aspects import (
+ unittest,
+ TestCase,
+ WikidataTestCase,
+ DefaultSiteTestCase,
+)
class TestPageGenerators(TestCase):
@@ -154,6 +159,30 @@
self.assertEqual(len(set(item['revid'] for item in items)), 4)
+class TestDequePreloadingGenerator(DefaultSiteTestCase):
+
+ """Test preloading generator on lists."""
+
+ def test_deque_preloading(self):
+ """Test pages being added to a
DequePreloadingGenerator."""
+ mainpage = self.get_mainpage()
+
+ pages = pywikibot.tools.DequeGenerator([mainpage])
+ gen = pagegenerators.DequePreloadingGenerator(pages)
+ pages_out = list()
+ for page in gen:
+ pages_out.append(page)
+ # Add a page to the generator
+ if page.namespace() == 0:
+ pages.extend([page.toggleTalkPage()])
+
+ self.assertTrue(all(isinstance(page, pywikibot.Page) for page in pages_out))
+ self.assertEqual(len(pages_out), 2)
+ self.assertEqual(pages_out[1].namespace(), 1)
+ self.assertIn(mainpage, pages_out)
+ self.assertIn(mainpage.toggleTalkPage(), pages_out)
+
+
class TestPreloadingItemGenerator(WikidataTestCase):
"""Test preloading item generator."""
--
To view, visit
https://gerrit.wikimedia.org/r/169519
To unsubscribe, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9e67a4d9d0f1736071434fa84796f22732896088
Gerrit-PatchSet: 4
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Nullzero <nullzero.free(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>