[Pywikibot-commits] [Gerrit] PreloadingItemGenerator: process normal pages - change (pywikibot/core)

jenkins-bot has submitted this change and it was merged. Change subject: PreloadingItemGenerator: process normal pages ...................................................................... PreloadingItemGenerator: process normal pages The item preloading generator only prefetches if the input generator yields WikibasePage, with a Site class that has method preloaditempages. This allows a Page which is in the 'wikibase-item' namespace to be be preloaded. Bug: 54999 Change-Id: I62140a91bb68a67a93dcf7bb70bfe284de672d02 --- M pywikibot/pagegenerators.py M tests/pagegenerators_tests.py M tests/wikibase_tests.py 3 files changed, 40 insertions(+), 1 deletion(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index 0ba02b4..2455532 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -1082,6 +1082,16 @@ """ sites = {} for page in generator: + if not isinstance(page, pywikibot.page.WikibasePage): + datasite = page.site.data_repository() + if page.namespace() != datasite.item_namespace: + pywikibot.output( + u'PreloadingItemGenerator skipping %s as it is not in %s' + % (page, datasite.item_namespace)) + continue + + page = pywikibot.ItemPage(datasite, page.title()) + site = page.site sites.setdefault(site, []).append(page) if len(sites[site]) >= step: diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index 96f574c..f7f372c 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -10,7 +10,7 @@ import pywikibot from pywikibot import pagegenerators -from tests.aspects import unittest, TestCase +from tests.aspects import unittest, TestCase, WikidataTestCase class TestPageGenerators(TestCase): @@ -153,6 +153,19 @@ self.assertEqual(len(set(item['revid'] for item in items)), 4) +class TestPreloadingItemGenerator(WikidataTestCase): + + """Test preloading item generator.""" + + def test_non_item_gen(self): + """Test TestPreloadingItemGenerator with ReferringPageGenerator.""" + site = self.get_site() + instance_of_page = pywikibot.Page(site, 'Property:P31') + ref_gen = pagegenerators.ReferringPageGenerator(instance_of_page, total=5) + gen = pagegenerators.PreloadingItemGenerator(ref_gen) + self.assertTrue(all(isinstance(item, pywikibot.ItemPage) for item in gen)) + + if __name__ == "__main__": try: unittest.main() diff --git a/tests/wikibase_tests.py b/tests/wikibase_tests.py index 7726433..6fce7df 100644 --- a/tests/wikibase_tests.py +++ b/tests/wikibase_tests.py @@ -631,6 +631,22 @@ self.assertEqual(response, self.data_out) +class TestPreloadingItemGenerator(TestCase): + + """Test preloading item generator.""" + + family = 'wikidata' + code = 'wikidata' + + def test_non_item_gen(self): + """Test TestPreloadingItemGenerator with ReferringPageGenerator.""" + site = self.get_site() + instance_of_page = pywikibot.Page(site, 'Property:P31') + ref_gen = pagegenerators.ReferringPageGenerator(instance_of_page, total=5) + gen = pagegenerators.PreloadingItemGenerator(ref_gen) + self.assertTrue(all(isinstance(item, pywikibot.ItemPage) for item in gen)) + + class TestNamespaces(WikidataTestCase): """Test cases to test namespaces of Wikibase entities.""" -- To view, visit https://gerrit.wikimedia.org/r/160438 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I62140a91bb68a67a93dcf7bb70bfe284de672d02 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg &lt;jayvdb(a)gmail.com&gt; Gerrit-Reviewer: Ladsgroup &lt;ladsgroup(a)gmail.com&gt; Gerrit-Reviewer: Merlijn van Deen &lt;valhallasw(a)arctus.nl&gt; Gerrit-Reviewer: XZise &lt;CommodoreFabianus(a)gmx.de&gt; Gerrit-Reviewer: jenkins-bot <>