Pywikipedia-svn November 2012

pywikipedia-svn@lists.wikimedia.org

10 participants
116 discussions

SVN: [10720] branches/rewrite/pywikibot/site.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10720 Revision: 10720 Author: xqt Date: 2012-11-14 06:45:31 +0000 (Wed, 14 Nov 2012) Log Message: ----------- some basic methods to read data from wikidata * CAUTION: The names and results of these methods could be changed in next future - get_item() - get one item by id - get_labels() - get labels of one item by id - get_decriptions() - get descriptions of one item by id - get_sitelinks() - get sitelinks of one item by id Modified Paths: -------------- branches/rewrite/pywikibot/site.py Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2012-11-13 22:13:13 UTC (rev 10719) +++ branches/rewrite/pywikibot/site.py 2012-11-14 06:45:31 UTC (rev 10720) @@ -1051,9 +1051,10 @@ def data_repository(self): """Return Site object for data repository e.g. wikidata.""" - code, fam = self.shared_data_repository(write) + code, fam = self.shared_data_repository() if bool(code or fam): - return pywikibot.Site(code, fam, self.username()) + return pywikibot.Site(code, fam, self.username(), + interface="DataSite") def is_image_repository(self): """Return True if Site object is the image repository.""" @@ -3186,6 +3187,95 @@ return wigen +class DataSite (APISite): + def __init__(self, code, fam=None, user=None, sysop=None): + APISite.__init__(self, code, fam, user, sysop) + self._namespaces[4] = [u"Wikidata", "Projekt"] + self._namespaces[5] = [u"Wikidata talk", "Projekt talk"] + self._namespaces[6] = [u"File", "Image"] + self._namespaces[7] = [u"File talk", "Image talk"] + self._namespaces[120] = [u"Property"] + self._namespaces[120] = [u"Property talk"] + self._namespaces[120] = [u"Query"] + self._namespaces[120] = [u"Query talk"] + return + + def __getattr__(self, attr): + """Calls to methods get_labels, get_descriptions, get_sitelinks""" + + if hasattr(self.__class__, attr): + return getattr(self.__class__, attr) + if attr.startswith("get_"): + props = attr.replace("get_", "") + if props in ['labels', 'descriptions', 'sitelinks']: + method = self._get_propertyitem + f = lambda *args, **params: \ + method(props, *args, **params) + if hasattr(method, "__doc__"): + f.__doc__ = method.__doc__ + return f + return super(APISite, self).__getattr__(attr) + + def _get_propertyitem(self, props, source, **params): + """generic method to get the data for multiple Wikibase items""" + wbdata = self.get_item(source, props=props, **params) + assert props in wbdata, \ + "API wbgetitems response lacks %s key" % props + return wbdata[props] + + def get_item(self, source, **params): + """get the data for multiple Wikibase items""" + if type(source) == int or \ + isinstance(source, basestring) and source.isdigit(): + ids = str(source) + wbrequest = api.Request(site=self, action="wbgetitems", ids=ids, + **params) + wbdata = wbrequest.submit() + print wbdata.keys() + assert 'success' in wbdata, \ + "API wbgetitems response lacks 'success' key" + assert wbdata['success'] == 1, \ + "API 'success' key ist not 1" + assert 'items' in wbdata, \ + "API wbgetitems response lacks 'items' key" + assert ids in wbdata['items'], \ + "API wbgetitems response lacks %s key" % ids + return wbdata['items'][ids] + else: + # not implemented yet + raise NotImplementedError + + # deprecated BaseSite methods + def fam(self): + raise NotImplementedError + def urlEncode(self, *args, **kwargs): + raise NotImplementedError + def getUrl(self, *args, **kwargs): + raise NotImplementedError + def linkto(self, *args, **kwargs): + raise NotImplementedError + def loggedInAs(self, *args, **kwargs): + raise NotImplementedError + def postData(self, *args, **kwargs): + raise NotImplementedError + def postForm(self, *args, **kwargs): + raise NotImplementedError + + # deprecated APISite methods + def isBlocked(self, *args, **kwargs): + raise NotImplementedError + def isAllowed(self, *args, **kwargs): + raise NotImplementedError + def prefixindex(self, *args, **kwargs): + raise NotImplementedError + def categories(self, *args, **kwargs): + raise NotImplementedError + def linksearch(self, *args, **kwargs): + raise NotImplementedError + def newimages(self, *args, **kwargs): + raise NotImplementedError + + #### METHODS NOT IMPLEMENTED YET #### class NotImplementedYet:

11 years, 6 months

SVN: [10719] branches/rewrite/pywikibot/site.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10719 Revision: 10719 Author: xqt Date: 2012-11-13 22:13:13 +0000 (Tue, 13 Nov 2012) Log Message: ----------- use __getattr__()ibuted site methods instead of the family methods; None is implizit returned Modified Paths: -------------- branches/rewrite/pywikibot/site.py Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2012-11-13 21:47:08 UTC (rev 10718) +++ branches/rewrite/pywikibot/site.py 2012-11-13 22:13:13 UTC (rev 10719) @@ -624,8 +624,6 @@ ## postForm: Post form data to an address at this site. ## postData: Post encoded form data to an http address at this site. ## -## shared_image_repository: Return tuple of image repositories used by this -## site. ## version: Return MediaWiki version string from Family file. ## versionnumber: Return int identifying the MediaWiki version. ## live_version: Return version number read from Special:Version. @@ -1034,13 +1032,13 @@ @property def has_image_repository(self): """Return True if site has a shared image repository like commons""" - code, fam = self.family.shared_image_repository(self.code) + code, fam = self.shared_image_repository() return bool(code or fam) @property def has_data_repository(self): """Return True if site has a shared image repository like wikidata""" - code, fam = self.family.shared_data_repository(self.code) + code, fam = self.shared_data_repository() return bool(code or fam) def image_repository(self): @@ -1049,8 +1047,6 @@ code, fam = self.shared_image_repository() if bool(code or fam): return pywikibot.Site(code, fam, self.username()) - else: - return None def data_repository(self): """Return Site object for data repository e.g. wikidata.""" @@ -1058,8 +1054,6 @@ code, fam = self.shared_data_repository(write) if bool(code or fam): return pywikibot.Site(code, fam, self.username()) - else: - return None def is_image_repository(self): """Return True if Site object is the image repository."""

11 years, 6 months

SVN: [10718] branches/rewrite/pywikibot/i18n.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10718 Revision: 10718 Author: xqt Date: 2012-11-13 21:47:08 +0000 (Tue, 13 Nov 2012) Log Message: ----------- enable i18n methods while using pwb.py and idle Modified Paths: -------------- branches/rewrite/pywikibot/i18n.py Modified: branches/rewrite/pywikibot/i18n.py =================================================================== --- branches/rewrite/pywikibot/i18n.py 2012-11-13 21:35:06 UTC (rev 10717) +++ branches/rewrite/pywikibot/i18n.py 2012-11-13 21:47:08 UTC (rev 10718) @@ -271,7 +271,7 @@ import table. """ package = twtitle.split("-")[0] - transdict = getattr(__import__("i18n", fromlist=[package]), package).msg + transdict = getattr(__import__("scripts.i18n", fromlist=[package]), package).msg code_needed = False # If a site is given instead of a code, use its language

11 years, 6 months

SVN: [10717] trunk/pywikipedia/tests

by valhallasw＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10717 Revision: 10717 Author: valhallasw Date: 2012-11-13 21:35:06 +0000 (Tue, 13 Nov 2012) Log Message: ----------- data_ingestion.py now for both rewrite /and/ trunk! Modified Paths: -------------- branches/rewrite/pywikibot/site.py trunk/pywikipedia/data_ingestion.py trunk/pywikipedia/tests/test_data_ingestion.py Added Paths: ----------- branches/rewrite/scripts/data_ingestion.py branches/rewrite/scripts/tests/ branches/rewrite/scripts/tests/data/ branches/rewrite/scripts/tests/data/MP_sounds.png branches/rewrite/scripts/tests/data/csv_ingestion.csv branches/rewrite/scripts/tests/test_data_ingestion.py branches/rewrite/scripts/tests/test_utils.py trunk/pywikipedia/tests/data/csv_ingestion.csv Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2012-11-13 21:25:31 UTC (rev 10716) +++ branches/rewrite/pywikibot/site.py 2012-11-13 21:35:06 UTC (rev 10717) @@ -2830,7 +2830,7 @@ return self.logevents(logtype="upload", total=number, start=lestart, end=leend, user=leuser, page=letitle) - def getImagesFromAnHash(self, hash_found=None): + def getFilesFromAnHash(self, hash_found=None): """Return all images that have the same hash. Useful to find duplicates or nowcommons. @@ -2846,6 +2846,11 @@ return [image.title(withNamespace=False) for image in self.allimages(sha1=hash_found)] + @deprecated('Site().getFilesFromAnHash') + def getImagesFromAnHash(self, hash_found=None): + return self.getFilesFromAnHash(self, hash_found) + + def upload(self, imagepage, source_filename=None, source_url=None, comment=None, watch=False, ignore_warnings=False): """Upload a file to the wiki. Added: branches/rewrite/scripts/data_ingestion.py =================================================================== --- branches/rewrite/scripts/data_ingestion.py (rev 0) +++ branches/rewrite/scripts/data_ingestion.py 2012-11-13 21:35:06 UTC (rev 10717) @@ -0,0 +1,283 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +A generic bot to do data ingestion (batch uploading) to Commons + +''' +import pywikibot +import posixpath, urlparse +import urllib +import hashlib, base64 +import StringIO + +class Photo(object): + ''' + Represents a Photo (or other file), with metadata, to upload to Commons. + + The constructor takes two parameters: URL (string) and metadata (dict with str:str key:value pairs) + that can be referred to from the title & template generation. + + + ''' + def __init__(self, URL, metadata): + self.URL = URL + self.metadata = metadata + self.metadata["_url"] = URL + self.metadata["_filename"] = filename = posixpath.split(urlparse.urlparse(URL)[2])[1] + self.metadata["_ext"] = ext = filename.split(".")[-1] + if ext == filename: + self.metadata["_ext"] = ext = None + self.contents = None + + def downloadPhoto(self): + ''' + Download the photo and store it in a StringIO.StringIO object. + + TODO: Add exception handling + ''' + if not self.contents: + imageFile=urllib.urlopen(self.URL).read() + self.contents = StringIO.StringIO(imageFile) + return self.contents + + def findDuplicateImages(self, site = pywikibot.getSite(u'commons', u'commons')): + ''' + Takes the photo, calculates the SHA1 hash and asks the mediawiki api for a list of duplicates. + + TODO: Add exception handling, fix site thing + ''' + hashObject = hashlib.sha1() + hashObject.update(self.downloadPhoto().getvalue()) + return site.getFilesFromAnHash(base64.b16encode(hashObject.digest())) + + def getTitle(self, fmt): + """ + Given a format string with %(name)s entries, returns the string formatted with metadata + """ + return fmt % self.metadata + + def getDescription(self, template, extraparams={}): + ''' + Generate a description for a file + ''' + + params = {} + params.update(self.metadata) + params.update(extraparams) + description = u'{{%s\n' % template + for key in sorted(params.keys()): + value = params[key] + if not key.startswith("_"): + description = description + (u'|%s=%s' % (key, self._safeTemplateValue(value))) + "\n" + description = description + u'}}' + + return description + + def _safeTemplateValue(self, value): + return value.replace("|", "{{!}}") + +def CSVReader(fileobj, urlcolumn, *args, **kwargs): + import csv + reader = csv.DictReader(fileobj, *args, **kwargs) + + for line in reader: + yield Photo(line[urlcolumn], line) + +class DataIngestionBot: + def __init__(self, reader, titlefmt, pagefmt, site=pywikibot.getSite(u'commons', u'commons')): + self.reader = reader + self.titlefmt = titlefmt + self.pagefmt = pagefmt + self.site = site + + def _doUpload(self, photo): + duplicates = photo.findDuplicateImages(self.site) + if duplicates: + pywikibot.output(u"Skipping duplicate of %r" % (duplicates, )) + return duplicates[0] + + title = photo.getTitle(self.titlefmt) + description = photo.getDescription(self.pagefmt) + + bot = upload.UploadRobot(url = photo.URL, + description = description, + useFilename = title, + keepFilename = True, + verifyDescription = False, + targetSite = self.site) + bot._contents = photo.downloadPhoto().getvalue() + bot._retrieved = True + bot.run() + + return title + + def doSingle(self): + return self._doUpload(self.reader.next()) + + def run(self): + for photo in self.reader: + self._doUpload(photo) + +if __name__=="__main__": + reader = CSVReader(open('tests/data/csv_ingestion.csv'), 'url') + bot = DataIngestionBot(reader, "%(name)s - %(set)s.%(_ext)s", ":user:valhallasw/test_template", pywikibot.getSite('test', 'test')) + bot.run() + +""" +class DataIngestionBot: + def __init__(self, configurationPage): + ''' + + ''' + self.site = configurationPage.site() + self.configuration = self.parseConfigurationPage(configurationPage) + + def parseConfigurationPage(self, configurationPage): + ''' + Expects a pywikibot.page object "configurationPage" which contains the configuration + ''' + configuration = {} + # Set a bunch of defaults + configuration['csvDialect']=u'excel' + configuration['csvDelimiter']=';' + configuration['csvEncoding']=u'Windows-1252' #FIXME: Encoding hell + + templates = configurationPage.templatesWithParams() + for (template, params) in templates: + if template==u'Data ingestion': + for param in params: + (field, sep, value) = param.partition(u'=') + + # Remove leading or trailing spaces + field = field.strip() + value = value.strip() + configuration[field] = value + print configuration + return configuration + + + def downloadPhoto(self, photoUrl = ''): + ''' + Download the photo and store it in a StrinIO.StringIO object. + + TODO: Add exception handling + ''' + imageFile=urllib.urlopen(photoUrl).read() + return StringIO.StringIO(imageFile) + + def findDuplicateImages(self, photo = None, site = pywikibot.getSite(u'commons', u'commons')): + ''' + Takes the photo, calculates the SHA1 hash and asks the mediawiki api for a list of duplicates. + + TODO: Add exception handling, fix site thing + ''' + hashObject = hashlib.sha1() + hashObject.update(photo.getvalue()) + return site.getFilesFromAnHash(base64.b16encode(hashObject.digest())) + + def getTitle(self, metadata): + ''' + Build a title. + Have titleFormat to indicate how the title would look. + We need to be able to strip off stuff if it's too long. configuration.get('maxTitleLength') + ''' + + #FIXME: Make this configurable. + title = self.configuration.get('titleFormat') % metadata + + description = metadata.get(u'dc:title') + identifier = metadata.get(u'dc:identifier') + + if len(description)>120: + description = description[0 : 120] + + title = u'%s - %s.jpg' % (description, identifier) + + return flickrripper.cleanUpTitle(title) + + def cleanDate(self, field): + ''' + A function to do date clean up. + ''' + # Empty, make it really empty + if field==u'-': + return u'' + # TODO: Circa + # TODO: Period + + return field + + def cleanEmptyField(self, field): + return field + + def procesFile(self, metadata): + # FIXME: Do some metadata enrichment + #metadata = getEuropeanaMetadata(metadata) + + fileLocation = metadata.get(self.configuration.get('sourceFileField')) + + photo = self.downloadPhoto(fileLocation) + duplicates = self.findDuplicateImages(photo) + + # We don't want to upload dupes + if duplicates: + pywikibot.output(u'Found duplicate image at %s' % duplicates.pop()) + # The file is at Commons so return True + return True + + # FIXME: Do some checking to see if the title already exists + + title = self.getTitle(metadata) + description = self.getDescription(metadata) + + + pywikibot.output(u'Preparing upload for %s.' % title) + pywikibot.output(description) + + bot = upload.UploadRobot(url=fileLocation, description=description, useFilename=title, keepFilename=True, verifyDescription=False, targetSite = self.site) + bot.run() + + def processCSV(self): + database = {} + + reader = csv.DictReader(open(self.configuration.get('csvFile'), "rb"), dialect=self.configuration.get('csvDialect'), delimiter=self.configuration.csvDelimiter) + # FIXME : Encoding problems http://docs.python.org/library/csv.html#csv-examples + for row in reader: + self.metadataCSV(row) + self.processFile(metadata) + + def run(self): + ''' + Do crap + ''' + if not self.configuration.get('sourceFormat'): + pywikibot.output(u'The field "sourceFormat" is not set') + return False + + if self.configuration.get('sourceFormat')==u'csv': + self.processCSV() + else: + pywikibot.output(u'%s is not a supported source format') + +def main(args): + generator = None; + + genFactory = pagegenerators.GeneratorFactory() + + for arg in pywikibot.handleArgs(): + genFactory.handleArg(arg) + + generator = genFactory.getCombinedGenerator() + if not generator: + return False + + for page in generator: + bot = DataIngestionBot(page) + bot.run() + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + finally: + print "All done!" +""" Added: branches/rewrite/scripts/tests/data/MP_sounds.png =================================================================== (Binary files differ) Property changes on: branches/rewrite/scripts/tests/data/MP_sounds.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/rewrite/scripts/tests/data/csv_ingestion.csv =================================================================== --- branches/rewrite/scripts/tests/data/csv_ingestion.csv (rev 0) +++ branches/rewrite/scripts/tests/data/csv_ingestion.csv 2012-11-13 21:35:06 UTC (rev 10717) @@ -0,0 +1,2 @@ +description.en,source,author,license,set,name,url +"""Sounds"" icon",http://commons.wikimedia.org/wiki/File:Sound-icon.svg,KDE artists | Silstor,LGPL,Crystal SVG icon set,Sound icon,http://upload.wikimedia.org/wikipedia/commons/f/fc/MP_sounds.png Property changes on: branches/rewrite/scripts/tests/data/csv_ingestion.csv ___________________________________________________________________ Added: svn:executable + * Added: branches/rewrite/scripts/tests/test_data_ingestion.py =================================================================== --- branches/rewrite/scripts/tests/test_data_ingestion.py (rev 0) +++ branches/rewrite/scripts/tests/test_data_ingestion.py 2012-11-13 21:35:06 UTC (rev 10717) @@ -0,0 +1,75 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +"""Unit tests for data_ingestion.py""" +__version__ = '$Id: test_userlib.py 9043 2011-03-13 10:25:08Z xqt $' + +import os +import unittest +import test_utils + +import pywikibot + +import data_ingestion + +class TestPhoto(unittest.TestCase): + def setUp(self): + self.obj = data_ingestion.Photo(URL='http://upload.wikimedia.org/wikipedia/commons/f/fc/MP_sounds.png', + metadata={'description.en': '"Sounds" icon', + 'source': 'http://commons.wikimedia.org/wiki/File:Sound-icon.svg', + 'author': 'KDE artists | Silstor', + 'license': 'LGPL', + 'set': 'Crystal SVG icon set', + 'name': 'Sound icon'} + ) + + def test_downloadPhoto(self): + f = open(os.path.join(os.path.split(__file__)[0], 'data', 'MP_sounds.png')) + self.assertEqual(f.read(), self.obj.downloadPhoto().read()) + + def test_findDuplicateImages(self): + duplicates = self.obj.findDuplicateImages() + self.assertIn('MP sounds.png', [dup.replace("_", " ") for dup in duplicates]) + + def test_getTitle(self): + self.assertEqual(self.obj.getTitle("%(name)s - %(set)s.%(_ext)s"), "Sound icon - Crystal SVG icon set.png") + + def test_getDescription(self): + self.assertEqual(self.obj.getDescription('CrystalTemplate'), +"""{{CrystalTemplate +|author=KDE artists {{!}} Silstor +|description.en="Sounds" icon +|license=LGPL +|name=Sound icon +|set=Crystal SVG icon set +|source=http://commons.wikimedia.org/wiki/File:Sound-icon.svg +}}""") + + +class TestCSVReader(unittest.TestCase): + def setUp(self): + fileobj = open(os.path.join(os.path.split(__file__)[0], 'data', 'csv_ingestion.csv')) + self.iterator = data_ingestion.CSVReader(fileobj, 'url') + self.obj = self.iterator.next() + + def test_PhotoURL(self): + self.assertEqual(self.obj.URL, 'http://upload.wikimedia.org/wikipedia/commons/f/fc/MP_sounds.png') + + def test_getTitle(self): + self.assertEqual(self.obj.getTitle("%(name)s - %(set)s.%(_ext)s"), "Sound icon - Crystal SVG icon set.png") + + def test_getDescription(self): + self.assertEqual(self.obj.getDescription('CrystalTemplate'), +"""{{CrystalTemplate +|author=KDE artists {{!}} Silstor +|description.en="Sounds" icon +|license=LGPL +|name=Sound icon +|set=Crystal SVG icon set +|source=http://commons.wikimedia.org/wiki/File:Sound-icon.svg +|url=http://upload.wikimedia.org/wikipedia/commons/f/fc/MP_sounds.png +}}""") + + +if __name__ == "__main__": + unittest.main() Added: branches/rewrite/scripts/tests/test_utils.py =================================================================== --- branches/rewrite/scripts/tests/test_utils.py (rev 0) +++ branches/rewrite/scripts/tests/test_utils.py 2012-11-13 21:35:06 UTC (rev 10717) @@ -0,0 +1,12 @@ +""" +Support module for PyWikipediaBot regression tests. +""" +__version__ = '$Id: test_utils.py 9197 2011-04-25 08:57:30Z xqt $' + +import sys + +# Add current directory and parent directory to module search path. +sys.path.insert(0, '..') +sys.path.insert(0, '.') + +del sys Modified: trunk/pywikipedia/data_ingestion.py =================================================================== --- trunk/pywikipedia/data_ingestion.py 2012-11-13 21:25:31 UTC (rev 10716) +++ trunk/pywikipedia/data_ingestion.py 2012-11-13 21:35:06 UTC (rev 10717) @@ -4,13 +4,11 @@ A generic bot to do data ingestion (batch uploading) to Commons ''' -import sys, os.path, glob, re, hashlib, base64, StringIO -import xml.etree.ElementTree -import wikipedia as pywikibot -import config, query, upload -import csv, urllib -import pagegenerators -import urlparse, posixpath +import pywikibot +import posixpath, urlparse +import urllib +import hashlib, base64 +import StringIO class Photo(object): ''' @@ -79,6 +77,7 @@ return value.replace("|", "{{!}}") def CSVReader(fileobj, urlcolumn, *args, **kwargs): + import csv reader = csv.DictReader(fileobj, *args, **kwargs) for line in reader: Added: trunk/pywikipedia/tests/data/csv_ingestion.csv =================================================================== --- trunk/pywikipedia/tests/data/csv_ingestion.csv (rev 0) +++ trunk/pywikipedia/tests/data/csv_ingestion.csv 2012-11-13 21:35:06 UTC (rev 10717) @@ -0,0 +1,2 @@ +description.en,source,author,license,set,name,url +"""Sounds"" icon",http://commons.wikimedia.org/wiki/File:Sound-icon.svg,KDE artists | Silstor,LGPL,Crystal SVG icon set,Sound icon,http://upload.wikimedia.org/wikipedia/commons/f/fc/MP_sounds.png Property changes on: trunk/pywikipedia/tests/data/csv_ingestion.csv ___________________________________________________________________ Added: svn:executable + * Modified: trunk/pywikipedia/tests/test_data_ingestion.py =================================================================== --- trunk/pywikipedia/tests/test_data_ingestion.py 2012-11-13 21:25:31 UTC (rev 10716) +++ trunk/pywikipedia/tests/test_data_ingestion.py 2012-11-13 21:35:06 UTC (rev 10717) @@ -8,8 +8,7 @@ import unittest import test_utils -import userlib -import wikipedia +import pywikibot import data_ingestion @@ -30,7 +29,7 @@ def test_findDuplicateImages(self): duplicates = self.obj.findDuplicateImages() - self.assertIn('MP_sounds.png', duplicates) + self.assertIn('MP sounds.png', [dup.replace("_", " ") for dup in duplicates]) def test_getTitle(self): self.assertEqual(self.obj.getTitle("%(name)s - %(set)s.%(_ext)s"), "Sound icon - Crystal SVG icon set.png")

11 years, 6 months

SVN: [10716] trunk/pywikipedia/query.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10716 Revision: 10716 Author: xqt Date: 2012-11-13 21:25:31 +0000 (Tue, 13 Nov 2012) Log Message: ----------- frompywikibot.support import deprecated, follow-up for r10715 Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2012-11-13 20:56:05 UTC (rev 10715) +++ trunk/pywikipedia/query.py 2012-11-13 21:25:31 UTC (rev 10716) @@ -26,9 +26,9 @@ # import time -import wikipedia as pywikibot +import pywikibot import config -from pywikibot import deprecate_arg +from pywikibot.support import deprecate_arg try: #For Python 2.6 newer import json

11 years, 6 months

SVN: [10715] trunk/pywikipedia

by valhallasw＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10715 Revision: 10715 Author: valhallasw Date: 2012-11-13 20:56:05 +0000 (Tue, 13 Nov 2012) Log Message: ----------- Added support for importing pywikibot instead of wikipedia. The used method is not very nice, but quite effective. Because wikipedia.py has many globals and dependencies, it's not easy to create a correct import chain. Therefore, instead of importing wikipedia in the pywikibot module, wikipedia.py monkey-patches /itself/ in there. Long story short: import pywikibot and be done with it. Modified Paths: -------------- trunk/pywikipedia/pagegenerators.py trunk/pywikipedia/pywikibot/__init__.py trunk/pywikipedia/pywikibot/i18n.py trunk/pywikipedia/wikipedia.py Added Paths: ----------- trunk/pywikipedia/pywikibot/support.py Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2012-11-12 20:30:37 UTC (rev 10714) +++ trunk/pywikipedia/pagegenerators.py 2012-11-13 20:56:05 UTC (rev 10715) @@ -28,7 +28,8 @@ import traceback import wikipedia as pywikibot import config -from pywikibot import deprecate_arg, i18n +from pywikibot import i18n +from pywikibot.support import deprecate_arg import date, catlib, userlib, query parameterHelp = u"""\ Modified: trunk/pywikipedia/pywikibot/__init__.py =================================================================== --- trunk/pywikipedia/pywikibot/__init__.py 2012-11-12 20:30:37 UTC (rev 10714) +++ trunk/pywikipedia/pywikibot/__init__.py 2012-11-13 20:56:05 UTC (rev 10715) @@ -1,185 +1,4 @@ -# -*- coding: utf-8 -*- -""" -The initialization file for the Pywikibot framework. -""" -# -# (C) Pywikipedia bot team, 2010-2011 -# -# Distributed under the terms of the MIT license. -# -__version__ = '$Id$' +# wikipedia.py will monkey-patch this module to look completely +# alike wikipedia itself... -import datetime -import difflib - -from exceptions import * -from i18n import translate -from textlib import * -from throttle import * - import wikipedia - -class Timestamp(datetime.datetime): - """Class for handling Mediawiki timestamps. - - This inherits from datetime.datetime, so it can use all of the methods - and operations of a datetime object. To ensure that the results of any - operation are also a Timestamp object, be sure to use only Timestamp - objects (and datetime.timedeltas) in any operation. - - Use Timestamp.fromISOformat() and Timestamp.fromtimestampformat() to - create Timestamp objects from Mediawiki string formats. - - Use Site.getcurrenttime() for the current time; this is more reliable - than using Timestamp.utcnow(). - - """ - mediawikiTSFormat = "%Y%m%d%H%M%S" - ISO8601Format = "%Y-%m-%dT%H:%M:%SZ" - - @classmethod - def fromISOformat(cls, ts): - """Convert an ISO 8601 timestamp to a Timestamp object.""" - return cls.strptime(ts, cls.ISO8601Format) - - @classmethod - def fromtimestampformat(cls, ts): - """Convert the internal MediaWiki timestamp format to a Timestamp object.""" - return cls.strptime(ts, cls.mediawikiTSFormat) - - def __str__(self): - """Return a string format recognized by the API""" - return self.strftime(self.ISO8601Format) - - def __add__(self, other): - newdt = datetime.datetime.__add__(self, other) - if isinstance(newdt, datetime.datetime): - return Timestamp(newdt.year, newdt.month, newdt.day, newdt.hour, - newdt.minute, newdt.second, newdt.microsecond, - newdt.tzinfo) - else: - return newdt - - def __sub__(self, other): - newdt = datetime.datetime.__sub__(self, other) - if isinstance(newdt, datetime.datetime): - return Timestamp(newdt.year, newdt.month, newdt.day, newdt.hour, - newdt.minute, newdt.second, newdt.microsecond, - newdt.tzinfo) - else: - return newdt - - -def deprecated(instead=None): - """Decorator to output a method deprecation warning. - - @param instead: if provided, will be used to specify the replacement - @type instead: string - """ - def decorator(method): - def wrapper(*args, **kwargs): - funcname = method.func_name - classname = args[0].__class__.__name__ - if instead: - wikipedia.output(u"%s.%s is DEPRECATED, use %s instead." - % (classname, funcname, instead)) - else: - wikipedia.output(u"%s.%s is DEPRECATED." % (classname, funcname)) - return method(*args, **kwargs) - wrapper.func_name = method.func_name - return wrapper - return decorator - -def deprecate_arg(old_arg, new_arg): - """Decorator to declare old_arg deprecated and replace it with new_arg""" - #_logger = "" - def decorator(method): - def wrapper(*__args, **__kw): - meth_name = method.__name__ - if old_arg in __kw: - if new_arg: - if new_arg in __kw: - wikipedia.output( -u"%(new_arg)s argument of %(meth_name)s replaces %(old_arg)s; cannot use both." - % locals()) - else: - wikipedia.output( -u"%(old_arg)s argument of %(meth_name)s is deprecated; use %(new_arg)s instead." - % locals()) - __kw[new_arg] = __kw[old_arg] - else: - wikipedia.output( - u"%(old_arg)s argument of %(meth_name)s is deprecated." - % locals()) - del __kw[old_arg] - return method(*__args, **__kw) - wrapper.__doc__ = method.__doc__ - wrapper.__name__ = method.__name__ - return wrapper - return decorator - -link_regex = re.compile(r'\[\[(?P<title>[^\]|[#<>{}]*)(\|.*?)?\]\]') - - -def showDiff(oldtext, newtext): - """ - Output a string showing the differences between oldtext and newtext. - The differences are highlighted (only on compatible systems) to show which - changes were made. - - """ - # This is probably not portable to non-terminal interfaces.... - # For information on difflib, see http://pydoc.org/2.3/difflib.html - color = { - '+': 'lightgreen', - '-': 'lightred', - } - diff = u'' - colors = [] - # This will store the last line beginning with + or -. - lastline = None - # For testing purposes only: show original, uncolored diff - # for line in difflib.ndiff(oldtext.splitlines(), newtext.splitlines()): - # print line - for line in difflib.ndiff(oldtext.splitlines(), newtext.splitlines()): - if line.startswith('?'): - # initialize color vector with None, which means default color - lastcolors = [None for c in lastline] - # colorize the + or - sign - lastcolors[0] = color[lastline[0]] - # colorize changed parts in red or green - for i in range(min(len(line), len(lastline))): - if line[i] != ' ': - lastcolors[i] = color[lastline[0]] - diff += lastline + '\n' - # append one None (default color) for the newline character - colors += lastcolors + [None] - elif lastline: - diff += lastline + '\n' - # colorize the + or - sign only - lastcolors = [None for c in lastline] - lastcolors[0] = color[lastline[0]] - colors += lastcolors + [None] - lastline = None - if line[0] in ('+', '-'): - lastline = line - # there might be one + or - line left that wasn't followed by a ? line. - if lastline: - diff += lastline + '\n' - # colorize the + or - sign only - lastcolors = [None for c in lastline] - lastcolors[0] = color[lastline[0]] - colors += lastcolors + [None] - - result = u'' - lastcolor = None - for i in range(len(diff)): - if colors[i] != lastcolor: - if lastcolor is None: - result += '\03{%s}' % colors[i] - else: - result += '\03{default}' - lastcolor = colors[i] - result += diff[i] - wikipedia.output(result) - Modified: trunk/pywikipedia/pywikibot/i18n.py =================================================================== --- trunk/pywikipedia/pywikibot/i18n.py 2012-11-12 20:30:37 UTC (rev 10714) +++ trunk/pywikipedia/pywikibot/i18n.py 2012-11-13 20:56:05 UTC (rev 10715) @@ -11,7 +11,7 @@ import re, sys import locale -from pywikibot import Error +from pywikibot.exceptions import Error import wikipedia as pywikibot import config Added: trunk/pywikipedia/pywikibot/support.py =================================================================== --- trunk/pywikipedia/pywikibot/support.py (rev 0) +++ trunk/pywikipedia/pywikibot/support.py 2012-11-13 20:56:05 UTC (rev 10715) @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +""" +The initialization file for the Pywikibot framework. +""" +# +# (C) Pywikipedia bot team, 2010-2011 +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id: __init__.py 9398 2011-07-16 16:17:18Z xqt $' + +import datetime +import difflib + +from exceptions import * +from i18n import translate +from textlib import * +from throttle import * + +import wikipedia + +class Timestamp(datetime.datetime): + """Class for handling Mediawiki timestamps. + + This inherits from datetime.datetime, so it can use all of the methods + and operations of a datetime object. To ensure that the results of any + operation are also a Timestamp object, be sure to use only Timestamp + objects (and datetime.timedeltas) in any operation. + + Use Timestamp.fromISOformat() and Timestamp.fromtimestampformat() to + create Timestamp objects from Mediawiki string formats. + + Use Site.getcurrenttime() for the current time; this is more reliable + than using Timestamp.utcnow(). + + """ + mediawikiTSFormat = "%Y%m%d%H%M%S" + ISO8601Format = "%Y-%m-%dT%H:%M:%SZ" + + @classmethod + def fromISOformat(cls, ts): + """Convert an ISO 8601 timestamp to a Timestamp object.""" + return cls.strptime(ts, cls.ISO8601Format) + + @classmethod + def fromtimestampformat(cls, ts): + """Convert the internal MediaWiki timestamp format to a Timestamp object.""" + return cls.strptime(ts, cls.mediawikiTSFormat) + + def __str__(self): + """Return a string format recognized by the API""" + return self.strftime(self.ISO8601Format) + + def __add__(self, other): + newdt = datetime.datetime.__add__(self, other) + if isinstance(newdt, datetime.datetime): + return Timestamp(newdt.year, newdt.month, newdt.day, newdt.hour, + newdt.minute, newdt.second, newdt.microsecond, + newdt.tzinfo) + else: + return newdt + + def __sub__(self, other): + newdt = datetime.datetime.__sub__(self, other) + if isinstance(newdt, datetime.datetime): + return Timestamp(newdt.year, newdt.month, newdt.day, newdt.hour, + newdt.minute, newdt.second, newdt.microsecond, + newdt.tzinfo) + else: + return newdt + + +def deprecated(instead=None): + """Decorator to output a method deprecation warning. + + @param instead: if provided, will be used to specify the replacement + @type instead: string + """ + def decorator(method): + def wrapper(*args, **kwargs): + funcname = method.func_name + classname = args[0].__class__.__name__ + if instead: + wikipedia.output(u"%s.%s is DEPRECATED, use %s instead." + % (classname, funcname, instead)) + else: + wikipedia.output(u"%s.%s is DEPRECATED." % (classname, funcname)) + return method(*args, **kwargs) + wrapper.func_name = method.func_name + return wrapper + return decorator + +def deprecate_arg(old_arg, new_arg): + """Decorator to declare old_arg deprecated and replace it with new_arg""" + #_logger = "" + def decorator(method): + def wrapper(*__args, **__kw): + meth_name = method.__name__ + if old_arg in __kw: + if new_arg: + if new_arg in __kw: + wikipedia.output( +u"%(new_arg)s argument of %(meth_name)s replaces %(old_arg)s; cannot use both." + % locals()) + else: + wikipedia.output( +u"%(old_arg)s argument of %(meth_name)s is deprecated; use %(new_arg)s instead." + % locals()) + __kw[new_arg] = __kw[old_arg] + else: + wikipedia.output( + u"%(old_arg)s argument of %(meth_name)s is deprecated." + % locals()) + del __kw[old_arg] + return method(*__args, **__kw) + wrapper.__doc__ = method.__doc__ + wrapper.__name__ = method.__name__ + return wrapper + return decorator + +link_regex = re.compile(r'\[\[(?P<title>[^\]|[#<>{}]*)(\|.*?)?\]\]') + + +def showDiff(oldtext, newtext): + """ + Output a string showing the differences between oldtext and newtext. + The differences are highlighted (only on compatible systems) to show which + changes were made. + + """ + # This is probably not portable to non-terminal interfaces.... + # For information on difflib, see http://pydoc.org/2.3/difflib.html + color = { + '+': 'lightgreen', + '-': 'lightred', + } + diff = u'' + colors = [] + # This will store the last line beginning with + or -. + lastline = None + # For testing purposes only: show original, uncolored diff + # for line in difflib.ndiff(oldtext.splitlines(), newtext.splitlines()): + # print line + for line in difflib.ndiff(oldtext.splitlines(), newtext.splitlines()): + if line.startswith('?'): + # initialize color vector with None, which means default color + lastcolors = [None for c in lastline] + # colorize the + or - sign + lastcolors[0] = color[lastline[0]] + # colorize changed parts in red or green + for i in range(min(len(line), len(lastline))): + if line[i] != ' ': + lastcolors[i] = color[lastline[0]] + diff += lastline + '\n' + # append one None (default color) for the newline character + colors += lastcolors + [None] + elif lastline: + diff += lastline + '\n' + # colorize the + or - sign only + lastcolors = [None for c in lastline] + lastcolors[0] = color[lastline[0]] + colors += lastcolors + [None] + lastline = None + if line[0] in ('+', '-'): + lastline = line + # there might be one + or - line left that wasn't followed by a ? line. + if lastline: + diff += lastline + '\n' + # colorize the + or - sign only + lastcolors = [None for c in lastline] + lastcolors[0] = color[lastline[0]] + colors += lastcolors + [None] + + result = u'' + lastcolor = None + for i in range(len(diff)): + if colors[i] != lastcolor: + if lastcolor is None: + result += '\03{%s}' % colors[i] + else: + result += '\03{default}' + lastcolor = colors[i] + result += diff[i] + wikipedia.output(result) + Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2012-11-12 20:30:37 UTC (rev 10714) +++ trunk/pywikipedia/wikipedia.py 2012-11-13 20:56:05 UTC (rev 10715) @@ -138,7 +138,7 @@ from BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, SoupStrainer import weakref # Splitting the bot into library parts -from pywikibot import * +from pywikibot.support import * import config, login, query, version # Check Unicode support (is this a wide or narrow python build?) @@ -8620,6 +8620,15 @@ of deprecating use of old Python versions for Pywikipedia. """ % sys.version) +# The following will monkey-patch the pywikibot module to contain the same +# functions and variables as wikipedia itself. This means we no longer have +# to import wikipedia as pywikibot - instead, we can just import pywikibot +# +# Alas, it ain't beautiful, but it does work. + +import pywikibot +pywikibot.__dict__.update(locals()) + if __name__ == '__main__': import doctest print 'Pywikipediabot %s' % version.getversion()

11 years, 6 months

SVN: [10714] trunk/pywikipedia/pywikibot/comms/http.py

by drtrigon＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10714 Revision: 10714 Author: drtrigon Date: 2012-11-12 20:30:37 +0000 (Mon, 12 Nov 2012) Log Message: ----------- bug fix; patch to solve bug #3579749 (follow up for r10440, r10711 and r10712) Modified Paths: -------------- trunk/pywikipedia/pywikibot/comms/http.py Modified: trunk/pywikipedia/pywikibot/comms/http.py =================================================================== --- trunk/pywikipedia/pywikibot/comms/http.py 2012-11-12 20:27:20 UTC (rev 10713) +++ trunk/pywikipedia/pywikibot/comms/http.py 2012-11-12 20:30:37 UTC (rev 10714) @@ -48,7 +48,7 @@ return lambda *args, **kwds: self._call(name, attr, *args, **kwds) else: # do call to buffer data from parent and return - return self._call(name, attr, *args, **kwds) + return self._call(name, (lambda *args, **kwds: attr)) def _call(self, name, attr, *args, **kwds): if name not in self._buffer:

11 years, 6 months

SVN: [10713] trunk/pywikipedia/wikipedia.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10713 Revision: 10713 Author: xqt Date: 2012-11-12 20:27:20 +0000 (Mon, 12 Nov 2012) Log Message: ----------- doc string for Page.linkedPages() adopted from rewrite branch see bug #3586421 Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2012-11-11 20:03:42 UTC (rev 10712) +++ trunk/pywikipedia/wikipedia.py 2012-11-12 20:27:20 UTC (rev 10713) @@ -2680,10 +2680,16 @@ allDone = True return cats - def linkedPages(self, withImageLinks = False): + def linkedPages(self, withImageLinks=False): """Return a list of Pages that this Page links to. - Excludes interwiki and category links, and also image links by default. + Only returns pages from "normal" internal links. Category links are + omitted unless prefixed with ":". Image links are omitted when parameter + withImageLinks is False. Embedded templates are omitted (but links + within them are returned). All interwiki and external links are omitted. + + @param withImageLinks: include Image links + @return: a list of Page objects. """ result = [] try:

11 years, 6 months

SVN: [10700] branches/rewrite/scripts/i18n

by Merlijn van Deen

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10700 Revision: 10700 Author: siebrand Date: 2012-11-11 12:18:44 +0000 (Sun, 11 Nov 2012) Log Message: ----------- Update language and key ordering. Now the same as that for MediaWiki extension files. Modified Paths: -------------- branches/rewrite/scripts/i18n/add_text.py branches/rewrite/scripts/i18n/archivebot.py branches/rewrite/scripts/i18n/basic.py branches/rewrite/scripts/i18n/capitalize_redirects.py branches/rewrite/scripts/i18n/catall.py branches/rewrite/scripts/i18n/category.py branches/rewrite/scripts/i18n/category_redirect.py branches/rewrite/scripts/i18n/clean_sandbox.py branches/rewrite/scripts/i18n/commons.py branches/rewrite/scripts/i18n/cosmetic_changes.py branches/rewrite/scripts/i18n/delete.py branches/rewrite/scripts/i18n/djvutext.py branches/rewrite/scripts/i18n/editarticle.py branches/rewrite/scripts/i18n/featured.py branches/rewrite/scripts/i18n/fixing_redirects.py branches/rewrite/scripts/i18n/interwiki.py branches/rewrite/scripts/i18n/isbn.py branches/rewrite/scripts/i18n/movepages.py branches/rewrite/scripts/i18n/ndashredir.py branches/rewrite/scripts/i18n/noreferences.py branches/rewrite/scripts/i18n/pywikibot.py branches/rewrite/scripts/i18n/redirect.py branches/rewrite/scripts/i18n/replace.py branches/rewrite/scripts/i18n/solve_disambiguation.py branches/rewrite/scripts/i18n/spellcheck.py branches/rewrite/scripts/i18n/table2wiki.py branches/rewrite/scripts/i18n/template.py branches/rewrite/scripts/i18n/thirdparty.py branches/rewrite/scripts/i18n/unlink.py branches/rewrite/scripts/i18n/welcome.py [Message clipped]

11 years, 6 months

SVN: [10712] trunk/pywikipedia/pywikibot/comms/http.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10712 Revision: 10712 Author: xqt Date: 2012-11-11 20:03:42 +0000 (Sun, 11 Nov 2012) Log Message: ----------- revert r10711 Modified Paths: -------------- trunk/pywikipedia/pywikibot/comms/http.py Modified: trunk/pywikipedia/pywikibot/comms/http.py =================================================================== --- trunk/pywikipedia/pywikibot/comms/http.py 2012-11-11 19:57:35 UTC (rev 10711) +++ trunk/pywikipedia/pywikibot/comms/http.py 2012-11-11 20:03:42 UTC (rev 10712) @@ -40,7 +40,7 @@ self._parent = addinfourl self._buffer = {} - def __getattr__(self, name, *args, **kwds): + def __getattr__(self, name): # raise same exception as parent if attribute does not exist attr = getattr(self._parent, name) if callable(attr):

11 years, 6 months

← Newer
1
2
3
4
5
6
7
...
12
Older →

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn November 2012