Pywikipedia-svn April 2009

pywikipedia-svn@lists.wikimedia.org

8 participants
162 discussions

SVN: [6720] branches/rewrite/pywikibot/comms/http.py

by russblau＠svn.wikimedia.org

Revision: 6720 Author: russblau Date: 2009-04-25 16:21:08 +0000 (Sat, 25 Apr 2009) Log Message: ----------- Change order of tests Modified Paths: -------------- branches/rewrite/pywikibot/comms/http.py Modified: branches/rewrite/pywikibot/comms/http.py =================================================================== --- branches/rewrite/pywikibot/comms/http.py 2009-04-25 15:47:36 UTC (rev 6719) +++ branches/rewrite/pywikibot/comms/http.py 2009-04-25 16:21:08 UTC (rev 6720) @@ -99,13 +99,13 @@ request.lock.acquire() #TODO: do some error correcting stuff - if request.data[0].status == 504: - raise Server504Error("Server %s timed out" % site.hostname()) - #if all else fails if isinstance(request.data, Exception): raise request.data + if request.data[0].status == 504: + raise Server504Error("Server %s timed out" % site.hostname()) + if request.data[0].status != 200: pywikibot.output(u"Http response status %(status)s" % {'status': request.data[0].status},

15 years, 1 month

SVN: [6719] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6719 Author: nicdumz Date: 2009-04-25 15:47:36 +0000 (Sat, 25 Apr 2009) Log Message: ----------- Fixing NameError from r6711 Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-25 15:11:30 UTC (rev 6718) +++ trunk/pywikipedia/interwiki.py 2009-04-25 15:47:36 UTC (rev 6719) @@ -1323,6 +1323,7 @@ if globalvar.autonomous and not globalvar.force and len(removing) > 0: for rmsite in removing: if rmsite != page.site(): # Sometimes sites have an erroneous link to itself as an interwiki + rmPage = old[rmsite] ########## # temporary hard-coded special case to get rid of thousands of broken links to the Lombard Wikipedia, # where useless bot-created articles were mass-deleted. See for example:

15 years, 1 month

SVN: [6718] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6718 Author: nicdumz Date: 2009-04-25 15:11:30 +0000 (Sat, 25 Apr 2009) Log Message: ----------- Using sets instead of lists to do a set union and then a set difference Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-25 15:00:56 UTC (rev 6717) +++ trunk/pywikipedia/interwiki.py 2009-04-25 15:11:30 UTC (rev 6718) @@ -1186,7 +1186,15 @@ lclSite = self.originPage.site() lclSiteDone = False frgnSiteDone = False - for siteCode in lclSite.family.languages_by_size + [s for s in lclSite.family.langs if (not s in lclSite.family.languages_by_size and not s in lclSite.family.obsolete)]: + + # XXX Do we really need to make an union here? + # we should have sorted(languages_by_size) = sorted(langs) ?! + langBySize = set(lclSite.family.languages_by_size) + allLangs = set(lcl.family.langs) + + langToCheck = (langBySize + allLangs).difference(lclSite.family.obsolete) + + for siteCode in langToCheck: site = wikipedia.getSite(code = siteCode) if (not lclSiteDone and site == lclSite) or (not frgnSiteDone and site != lclSite and site in new): if site == lclSite:

15 years, 1 month

SVN: [6717] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6717 Author: nicdumz Date: 2009-04-25 15:00:56 +0000 (Sat, 25 Apr 2009) Log Message: ----------- updatedSites is a list. It needs to be converted explicitely to a set before using set differences Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-25 14:55:09 UTC (rev 6716) +++ trunk/pywikipedia/interwiki.py 2009-04-25 15:00:56 UTC (rev 6717) @@ -1426,7 +1426,7 @@ expectedPages = set(new.itervalues()) expectedSites = set(new) try: - for site in expectedSites - updatedSites: + for site in expectedSites - set(updatedSites): page = new[site] if not page.section(): try:

15 years, 1 month

SVN: [6716] trunk/pywikipedia/wikipedia.py

by nicdumz＠svn.wikimedia.org

Revision: 6716 Author: nicdumz Date: 2009-04-25 14:55:09 +0000 (Sat, 25 Apr 2009) Log Message: ----------- Remember the results of Page.interwiki in ._interwikis Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-04-25 14:51:44 UTC (rev 6715) +++ trunk/pywikipedia/wikipedia.py 2009-04-25 14:55:09 UTC (rev 6716) @@ -1731,6 +1731,9 @@ interwiki links in the page text. """ + if hasattr(self, "_interwikis"): + return self._interwikis + result = [] ll = getLanguageLinks(self.get(), insite=self.site(), pageLink=self.aslink()) @@ -1754,6 +1757,8 @@ output( u"ERROR: link from %s to [[%s:%s]] is improperly formatted?" % (self.aslink(), newSite, newTitle)) + + self._interwikis = result return result def categories(self, get_redirect=False):

15 years, 1 month

SVN: [6715] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6715 Author: nicdumz Date: 2009-04-25 14:51:44 +0000 (Sat, 25 Apr 2009) Log Message: ----------- Using set difference instead of a for + inner test on each element Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-25 14:43:14 UTC (rev 6714) +++ trunk/pywikipedia/interwiki.py 2009-04-25 14:51:44 UTC (rev 6715) @@ -1426,8 +1426,9 @@ expectedPages = set(new.itervalues()) expectedSites = set(new) try: - for site, page in new.iteritems(): - if site not in updatedSites and not page.section(): + for site in expectedSites - updatedSites: + page = new[site] + if not page.section(): try: linkedPages = set(page.interwiki()) except wikipedia.NoPage: @@ -1438,8 +1439,8 @@ linkedPagesDict = {} for linkedPage in linkedPages: linkedPagesDict[linkedPage.site()] = linkedPage - for expectedPage in expectedPages: - if expectedPage != page and expectedPage not in linkedPages: + for expectedPage in expectedPages - linkedPages: + if expectedPage != page: try: linkedPage = linkedPagesDict[expectedPage.site()] wikipedia.output(u"WARNING: %s: %s does not link to %s but to %s" % (page.site().family.name, page.aslink(True), expectedPage.aslink(True), linkedPage.aslink(True)))

15 years, 1 month

SVN: [6714] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6714 Author: nicdumz Date: 2009-04-25 14:43:14 +0000 (Sat, 25 Apr 2009) Log Message: ----------- new already is a Site->Page dict. No need to iterate on its values to get the Page's sites... Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-25 14:34:52 UTC (rev 6713) +++ trunk/pywikipedia/interwiki.py 2009-04-25 14:43:14 UTC (rev 6714) @@ -1423,8 +1423,8 @@ """ # use sets because searching an element is faster than in lists - expectedPages = set(new.values()) - expectedSites = set([page.site() for page in expectedPages]) + expectedPages = set(new.itervalues()) + expectedSites = set(new) try: for site, page in new.iteritems(): if site not in updatedSites and not page.section():

15 years, 1 month

SVN: [6713] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6713 Author: nicdumz Date: 2009-04-25 14:34:52 +0000 (Sat, 25 Apr 2009) Log Message: ----------- Removing 2 levels of identation with 2 early return statement Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-25 14:26:37 UTC (rev 6712) +++ trunk/pywikipedia/interwiki.py 2009-04-25 14:34:52 UTC (rev 6713) @@ -1330,86 +1330,89 @@ if not mods: wikipedia.output(u'No changes needed' ) + return False + + wikipedia.output(u"Changes to be made: %s" % mods) + oldtext = page.get() + newtext = wikipedia.replaceLanguageLinks(oldtext, new, site = page.site()) + if newtext == oldtext: + return False + if globalvar.debug: + wikipedia.showDiff(oldtext, newtext) + + # wikipedia.output(u"NOTE: Replace %s" % page.aslink()) + # Determine whether we need permission to submit + ask = False + if removing and removing != [page.site()]: # Allow for special case of a self-pointing interwiki link + self.problem('Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True)), createneed = False) + ask = True + if globalvar.force: + ask = False + if globalvar.confirm: + ask = True + # If we need to ask, do so + if ask: + if globalvar.autonomous: + # If we cannot ask, deny permission + answer = 'n' + else: + answer = wikipedia.inputChoice(u'Submit?', ['Yes', 'No', 'Give up'], ['y', 'n', 'g']) else: - wikipedia.output(u"Changes to be made: %s" % mods) - oldtext = page.get() - newtext = wikipedia.replaceLanguageLinks(oldtext, new, site = page.site()) - if globalvar.debug: - wikipedia.showDiff(oldtext, newtext) - if newtext != oldtext: - # wikipedia.output(u"NOTE: Replace %s" % page.aslink()) - # Determine whether we need permission to submit - ask = False - if removing and removing != [page.site()]: # Allow for special case of a self-pointing interwiki link - self.problem('Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True)), createneed = False) - ask = True - if globalvar.force: - ask = False - if globalvar.confirm: - ask = True - # If we need to ask, do so - if ask: - if globalvar.autonomous: - # If we cannot ask, deny permission - answer = 'n' - else: - answer = wikipedia.inputChoice(u'Submit?', ['Yes', 'No', 'Give up'], ['y', 'n', 'g']) + # If we do not need to ask, allow + answer = 'y' + # If we got permission to submit, do so + if answer == 'y': + # Check whether we will have to wait for wikipedia. If so, make + # another get-query first. + if bot: + while wikipedia.get_throttle.waittime() + 2.0 < wikipedia.put_throttle.waittime(): + wikipedia.output(u"NOTE: Performing a recursive query first to save time....") + qdone = bot.oneQuery() + if not qdone: + # Nothing more to do + break + wikipedia.output(u"NOTE: Updating live wiki...") + timeout=60 + while 1: + try: + status, reason, data = page.put(newtext, comment = wikipedia.translate(page.site().lang, msg)[0] + mods) + except wikipedia.LockedPage: + wikipedia.output(u'Page %s is locked. Skipping.' % (page.title(),)) + raise SaveError + except wikipedia.EditConflict: + wikipedia.output(u'ERROR putting page: An edit conflict occurred. Giving up.') + raise SaveError + except (wikipedia.SpamfilterError), error: + wikipedia.output(u'ERROR putting page: %s blacklisted by spamfilter. Giving up.' % (error.url,)) + raise SaveError + except (wikipedia.PageNotSaved), error: + wikipedia.output(u'ERROR putting page: %s' % (error.args,)) + raise SaveError + except (socket.error, IOError), error: + if timeout>3600: + raise + wikipedia.output(u'ERROR putting page: %s' % (error.args,)) + wikipedia.output(u'Sleeping %i seconds before trying again.' % (timeout,)) + timeout *= 2 + time.sleep(timeout) + except wikipedia.ServerError: + if timeout>3600: + raise + wikipedia.output(u'ERROR putting page: ServerError.') + wikipedia.output(u'Sleeping %i seconds before trying again.' % (timeout,)) + timeout *= 2 + time.sleep(timeout) else: - # If we do not need to ask, allow - answer = 'y' - # If we got permission to submit, do so - if answer == 'y': - # Check whether we will have to wait for wikipedia. If so, make - # another get-query first. - if bot: - while wikipedia.get_throttle.waittime() + 2.0 < wikipedia.put_throttle.waittime(): - wikipedia.output(u"NOTE: Performing a recursive query first to save time....") - qdone = bot.oneQuery() - if not qdone: - # Nothing more to do - break - wikipedia.output(u"NOTE: Updating live wiki...") - timeout=60 - while 1: - try: - status, reason, data = page.put(newtext, comment = wikipedia.translate(page.site().lang, msg)[0] + mods) - except wikipedia.LockedPage: - wikipedia.output(u'Page %s is locked. Skipping.' % (page.title(),)) - raise SaveError - except wikipedia.EditConflict: - wikipedia.output(u'ERROR putting page: An edit conflict occurred. Giving up.') - raise SaveError - except (wikipedia.SpamfilterError), error: - wikipedia.output(u'ERROR putting page: %s blacklisted by spamfilter. Giving up.' % (error.url,)) - raise SaveError - except (wikipedia.PageNotSaved), error: - wikipedia.output(u'ERROR putting page: %s' % (error.args,)) - raise SaveError - except (socket.error, IOError), error: - if timeout>3600: - raise - wikipedia.output(u'ERROR putting page: %s' % (error.args,)) - wikipedia.output(u'Sleeping %i seconds before trying again.' % (timeout,)) - timeout *= 2 - time.sleep(timeout) - except wikipedia.ServerError: - if timeout>3600: - raise - wikipedia.output(u'ERROR putting page: ServerError.') - wikipedia.output(u'Sleeping %i seconds before trying again.' % (timeout,)) - timeout *= 2 - time.sleep(timeout) - else: - break - if str(status) == '302': - return True - else: - wikipedia.output(u'%s %s' % (status, reason)) - elif answer == 'g': - raise GiveUpOnPage - else: - raise LinkMustBeRemoved('Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True))) - return False + break + if str(status) == '302': + return True + else: + wikipedia.output(u'%s %s' % (status, reason)) + return False + elif answer == 'g': + raise GiveUpOnPage + else: + raise LinkMustBeRemoved('Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True))) def reportBacklinks(self, new, updatedSites): """

15 years, 1 month

SVN: [6712] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6712 Author: nicdumz Date: 2009-04-25 14:26:37 +0000 (Sat, 25 Apr 2009) Log Message: ----------- Condition testing........ Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-25 11:42:04 UTC (rev 6711) +++ trunk/pywikipedia/interwiki.py 2009-04-25 14:26:37 UTC (rev 6712) @@ -1331,8 +1331,7 @@ if not mods: wikipedia.output(u'No changes needed' ) else: - if mods: - wikipedia.output(u"Changes to be made: %s" % mods) + wikipedia.output(u"Changes to be made: %s" % mods) oldtext = page.get() newtext = wikipedia.replaceLanguageLinks(oldtext, new, site = page.site()) if globalvar.debug:

15 years, 1 month

SVN: [6711] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6711 Author: nicdumz Date: 2009-04-25 11:42:04 +0000 (Sat, 25 Apr 2009) Log Message: ----------- * Translating message only once and store the 3 different messages instead of translating 3 times the message. * Correcting usage of return values of compareLang since 'removing' is now a Site dictionary, and not a Page dictionary. Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-25 11:30:32 UTC (rev 6710) +++ trunk/pywikipedia/interwiki.py 2009-04-25 11:42:04 UTC (rev 6711) @@ -1313,17 +1313,17 @@ # When running in autonomous mode without -force switch, make sure we don't remove any items, but allow addition of the new ones if globalvar.autonomous and not globalvar.force and len(removing) > 0: - for rmPage in removing: - if rmPage.site() != page.site(): # Sometimes sites have an erroneous link to itself as an interwiki + for rmsite in removing: + if rmsite != page.site(): # Sometimes sites have an erroneous link to itself as an interwiki ########## # temporary hard-coded special case to get rid of thousands of broken links to the Lombard Wikipedia, # where useless bot-created articles were mass-deleted. See for example: # http://meta.wikimedia.org/wiki/Proposals_for_closing_projects/Closure_of_Lo… - if rmPage.site() == wikipedia.getSite('lmo', 'wikipedia'): + if rmsite == wikipedia.getSite('lmo', 'wikipedia'): wikipedia.output('Found bad link to %s. As many lmo pages were deleted, it is assumed that it can be safely removed.' % rmPage.aslink()) else: ########## - new[rmPage.site()] = old[rmPage.site()] + new[rmsite] = old[rmsite] wikipedia.output(u"WARNING: %s is either deleted or has a mismatching disambiguation state." % rmPage.aslink(True)) # Re-Check what needs to get done mods, adding, removing, modifying = compareLanguages(old, new, insite = page.site()) @@ -1341,8 +1341,8 @@ # wikipedia.output(u"NOTE: Replace %s" % page.aslink()) # Determine whether we need permission to submit ask = False - if removing and removing != [page]: # Allow for special case of a self-pointing interwiki link - self.problem('Found incorrect link to %s in %s'% (",".join([x.site().lang for x in removing]), page.aslink(True)), createneed = False) + if removing and removing != [page.site()]: # Allow for special case of a self-pointing interwiki link + self.problem('Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True)), createneed = False) ask = True if globalvar.force: ask = False @@ -1409,7 +1409,7 @@ elif answer == 'g': raise GiveUpOnPage else: - raise LinkMustBeRemoved('Found incorrect link to %s in %s'% (",".join([x.site().lang for x in removing]), page.aslink(True))) + raise LinkMustBeRemoved('Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True))) return False def reportBacklinks(self, new, updatedSites): @@ -1682,12 +1682,14 @@ # Use short format, just the language code fmt = lambda d, site: site.lang + _, add, rem, mod = wikipedia.translate(insite.lang, msg) + if adding: - mods += " %s: %s" % (wikipedia.translate(insite.lang, msg)[1], ", ".join([fmt(new, x) for x in adding])) + mods += " %s: %s" % (add, ", ".join([fmt(new, x) for x in adding])) if removing: - mods += " %s: %s" % (wikipedia.translate(insite.lang, msg)[2], ", ".join([fmt(old, x) for x in removing])) + mods += " %s: %s" % (rem, ", ".join([fmt(old, x) for x in removing])) if modifying: - mods += " %s: %s" % (wikipedia.translate(insite.lang, msg)[3], ", ".join([fmt(new, x) for x in modifying])) + mods += " %s: %s" % (mod, ", ".join([fmt(new, x) for x in modifying])) return mods, adding, removing, modifying def readWarnfile(filename, bot):

15 years, 1 month

← Newer
1
...
4
5
6
7
8
9
10
...
17
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn April 2009