Pywikipedia-svn November 2009

pywikipedia-svn@lists.wikimedia.org

9 participants
146 discussions

SVN: [7592] trunk/pywikipedia/cosmetic_changes.py

by xqt＠svn.wikimedia.org

Revision: 7592 Author: xqt Date: 2009-11-04 13:49:07 +0000 (Wed, 04 Nov 2009) Log Message: ----------- translations for tk-wiki Modified Paths: -------------- trunk/pywikipedia/cosmetic_changes.py Modified: trunk/pywikipedia/cosmetic_changes.py =================================================================== --- trunk/pywikipedia/cosmetic_changes.py 2009-11-04 13:22:17 UTC (rev 7591) +++ trunk/pywikipedia/cosmetic_changes.py 2009-11-04 13:49:07 UTC (rev 7592) @@ -93,6 +93,7 @@ 'sr': u'Бот козметичке промене', 'sv': u'Bot: Kosmetiska ändringar', 'th': u'บอต ปรับแต่งให้อ่านง่าย', + 'tk': u'Bot: kosmetik üýtgeşme', 'tl': u'robot Kosmetiko pagbabago', 'tr': u'Bot Kozmetik değişiklikler', 'uk': u'робот косметичні зміни', @@ -149,6 +150,7 @@ 'sr': u'; козметичке промене', 'sv': u'; kosmetiska ändringar', 'th': u'; ปรับแต่งให้อ่านง่าย', + 'tk': u'; kosmetik üýtgeşme', 'tl': u'; Kosmetiko pagbabago', 'tr': u'; Kozmetik değişiklikler', 'uk': u'; косметичні зміни',

14 years, 6 months

SVN: [7591] trunk/pywikipedia/wikipedia.py

by alexsh＠svn.wikimedia.org

Revision: 7591 Author: alexsh Date: 2009-11-04 13:22:17 +0000 (Wed, 04 Nov 2009) Log Message: ----------- * site().getUrl(): change all HTTP process to use urllib2. * handle and combine Site Authentication, proxy handle and Proxy Authentication in the bottom. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-11-04 09:05:42 UTC (rev 7590) +++ trunk/pywikipedia/wikipedia.py 2009-11-04 13:22:17 UTC (rev 7591) @@ -123,7 +123,7 @@ __version__ = '$Id$' import os, sys -import httplib, socket, urllib, urllib2 +import httplib, socket, urllib, urllib2, cookielib import traceback import time, threading, Queue import math @@ -5464,69 +5464,59 @@ Returns the HTML text of the page converted to unicode. """ + if retry is None: retry = config.retry_on_fail - if self.hostname() in config.authenticate.keys(): - uo = authenticateURLopener + headers = {'User-agent': useragent,} + + if self.cookies(sysop = sysop): + headers['Cookie'] = self.cookies(sysop = sysop) + if compress: + headers['Accept-encoding'] = 'gzip' + + if refer: + headers['Refer'] = refer + + if no_hostname: # This allow users to parse also toolserver's script + url = path # and other useful pages without using some other functions. else: - if config.proxy['host'] and type(config.proxy['auth']) == tuple: - proxyHandle = {'http':'http://%s:%s@%s' % (config.proxy['auth'][0], config.proxy['auth'][1], config.proxy['host'] )} - elif config.proxy['host']: - proxyHandle = {'http':'http://%s' % config.proxy['host'] } - else: - proxyHandle = None - - uo = MyURLopener(proxies = proxyHandle) - if self.cookies(sysop = sysop): - uo.addheader('Cookie', self.cookies(sysop = sysop)) - if compress: - uo.addheader('Accept-encoding', 'gzip') - if no_hostname == True: # This allow users to parse also toolserver's script - url = path # and other useful pages without using some other functions. - else: url = '%s://%s%s' % (self.protocol(), self.hostname(), path) - if refer: - uo.addheader('Refer', refer) data = self.urlEncode(data) - + # Try to retrieve the page until it was successfully loaded (just in # case the server is down or overloaded). # Wait for retry_idle_time minutes (growing!) between retries. retry_idle_time = 1 while True: try: - if self.hostname() in config.authenticate.keys(): - request = urllib2.Request(url, data) - request.add_header('User-agent', useragent) - opener = urllib2.build_opener() - f = opener.open(request) - else: - f = uo.open(url, data) + request = urllib2.Request(url, data, headers) + f = MyURLopener.open(request) # read & info can raise socket.error text = f.read() headers = f.info() - break except KeyboardInterrupt: raise + except urllib2.HTTPError, e: + if e.code in [401, 404]: + raise PageNotFound(u'Page %s could not be retrieved. Check your family file ?' % url) + output(u"Result:%s %s" % (e.code, e.msg)) + raise except Exception, e: + output(u'%s' %e) if retry: - # We assume that the server is down. Wait some time, then try again. - output(u"%s" % e) - output(u"""\ -WARNING: Could not open '%s'. Maybe the server or -your connection is down. Retrying in %i minutes...""" + output(u"""WARNING: Could not open '%s'. Maybe the server or\n your connection is down. Retrying in %i minutes...""" % (url, retry_idle_time)) time.sleep(retry_idle_time * 60) # Next time wait longer, but not longer than half an hour retry_idle_time *= 2 if retry_idle_time > 30: retry_idle_time = 30 - else: - raise - + continue + + raise if cookie_only: return headers.get('set-cookie', '') contentType = headers.get('content-type', '') @@ -5569,8 +5559,8 @@ if back_response: return f, text - else: - return text + + return text def _getUserData(self, text, sysop = False, force = True): """ @@ -8082,48 +8072,37 @@ s = time.strptime(tz, "%Y-%m-%dT%H:%M:%SZ") return int(time.strftime("%Y%m%d%H%M%S", s)) -class MyURLopener(urllib.FancyURLopener): - version="PythonWikipediaBot/1.0" +# Site Cookies handler +COOKIEFILE = config.datafilepath('login-data', 'cookies.lwp') +cj = cookielib.LWPCookieJar() +if os.path.isfile(COOKIEFILE): + cj.load(COOKIEFILE) - def http_error_default(self, url, fp, errcode, errmsg, headers): - if errcode == 401 or errcode == 404: - raise PageNotFound(u'Page %s could not be retrieved. Check your family file ?' % url) - else: - return urllib.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers) +cookieProcessor = urllib2.HTTPCookieProcessor(cj) +MyURLopener = urllib2.build_opener(cookieProcessor) + +if config.proxy['host']: + proxyHandler = urllib2.ProxyHandler({'http':'http://%s/' % config.proxy['host'] }) + + MyURLopener = urllib2.build_opener(cookieProcessor, proxyHandler) + if config.proxy['auth']: + proxyAuth = urllib2.HTTPPasswordMgrWithDefaultRealm() + proxyAuth.add_password(None, config.proxy['host'], config.proxy['auth'][0], config.proxy['auth'][1]) + proxyAuthHandler = urllib2.ProxyBasicAuthHandler(proxyAuth) - def open_http(self, url, data=None): - ret = urllib.FancyURLopener.open_http(self, url, data) - if hasattr(self, 'http_code'): - ret.status = self.http_code - del self.http_code - else: - ret.status = 200 - return ret + MyURLopener = urllib2.build_opener(cookieProcessor, proxyHandler, proxyAuthHandler) - - -# Special opener in case we are using a site with authentication if config.authenticate: - import urllib2, cookielib - COOKIEFILE = config.datafilepath('login-data', 'cookies.lwp') - cj = cookielib.LWPCookieJar() - if os.path.isfile(COOKIEFILE): - cj.load(COOKIEFILE) passman = urllib2.HTTPPasswordMgrWithDefaultRealm() for site in config.authenticate: passman.add_password(None, site, config.authenticate[site][0], config.authenticate[site][1]) authhandler = urllib2.HTTPBasicAuthHandler(passman) + + MyURLopener = urllib2.build_opener(cookieProcessor, authhandler) if config.proxy['host']: - proxyHandle = urllib2.ProxyHandler({'http':'http://%s' % config.proxy['host'] }) + MyURLopener = urllib2.build_opener(cookieProcessor, authhandler, proxyHandler) if config.proxy['auth']: - proxyAuth = urllib2.HTTPPasswordMgr() - proxyAuth.add_password(None, config.proxy['host'], config.proxy['auth'][0], config.proxy['auth'][1]) - proxyAuthHandle = urllib2.ProxyBasicAuthHandler(proxyAuth) - else: - proxyHandle = None - proxyAuthHandle = None - authenticateURLopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj), proxyHandle, proxyAuthHandle, authhandler) - urllib2.install_opener(authenticateURLopener) + MyURLopener = urllib2.build_opener(cookieProcessor, authhandler, proxyHandler, proxyAuthHandler) if __name__ == '__main__': import doctest

14 years, 6 months

SVN: [7590] trunk/pywikipedia/clean_sandbox.py

by xqt＠svn.wikimedia.org

Revision: 7590 Author: xqt Date: 2009-11-04 09:05:42 +0000 (Wed, 04 Nov 2009) Log Message: ----------- Keyboard interrupt handling Modified Paths: -------------- trunk/pywikipedia/clean_sandbox.py Modified: trunk/pywikipedia/clean_sandbox.py =================================================================== --- trunk/pywikipedia/clean_sandbox.py 2009-11-04 08:41:30 UTC (rev 7589) +++ trunk/pywikipedia/clean_sandbox.py 2009-11-04 09:05:42 UTC (rev 7590) @@ -147,7 +147,7 @@ translatedMsg = wikipedia.translate(mySite, msg) if text.strip() == translatedContent.strip(): wikipedia.output(u'The sandbox is still clean, no change necessary.') - elif text.find(translatedContent.strip()) <> 0: + elif text.find(translatedContent.strip()) <> 0 and not 'subst:' in translatedContent: sandboxPage.put(translatedContent, translatedMsg) wikipedia.output(u'Standard content was changed, sandbox cleaned.') else: @@ -186,7 +186,10 @@ return bot = SandboxBot(hours, no_repeat, delay) - bot.run() + try: + bot.run() + except KeyboardInterrupt: + wikipedia.output('\nQuitting program...') if __name__ == "__main__": try:

14 years, 6 months

SVN: [7589] trunk/pywikipedia/wikipedia.py

by xqt＠svn.wikimedia.org

Revision: 7589 Author: xqt Date: 2009-11-04 08:41:30 +0000 (Wed, 04 Nov 2009) Log Message: ----------- Do not unescape pagetext coming from API (bugfix for #2886547) Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-11-03 20:36:09 UTC (rev 7588) +++ trunk/pywikipedia/wikipedia.py 2009-11-04 08:41:30 UTC (rev 7589) @@ -782,7 +782,7 @@ self._isWatched = False #cannot handle in API in my research for now. - pagetext = unescape(pageInfo['revisions'][0]['*']) + pagetext = pageInfo['revisions'][0]['*'] pagetext = pagetext.rstrip() if self.site().lang == 'eo': pagetext = decodeEsperantoX(pagetext)

14 years, 6 months

SVN: [7588] trunk/pywikipedia/mysql_autoconnection.py

by btongminh＠svn.wikimedia.org

Revision: 7588 Author: btongminh Date: 2009-11-03 20:36:09 +0000 (Tue, 03 Nov 2009) Log Message: ----------- Close the database the clean way, instead of the hack try..except Modified Paths: -------------- trunk/pywikipedia/mysql_autoconnection.py Modified: trunk/pywikipedia/mysql_autoconnection.py =================================================================== --- trunk/pywikipedia/mysql_autoconnection.py 2009-11-03 11:59:53 UTC (rev 7587) +++ trunk/pywikipedia/mysql_autoconnection.py 2009-11-03 20:36:09 UTC (rev 7588) @@ -36,6 +36,7 @@ self.max_retries = max_retries self.current_retry = 0 self.callback = callback + self.database = None self.conn_args = conn_args self.conn_kwargs = conn_kwargs @@ -83,7 +84,8 @@ self.current_retry = 0 self.connected = False try: - self.database.close() + if self.database: + self.database.close() except: pass

14 years, 7 months

SVN: [7587] trunk/pywikipedia/wikipedia.py

by alexsh＠svn.wikimedia.org

Revision: 7587 Author: alexsh Date: 2009-11-03 11:59:53 +0000 (Tue, 03 Nov 2009) Log Message: ----------- handle response code in MyURLopener for python 2.5, thanks for sbrhsieh(a)ptt.cc. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-11-03 11:56:50 UTC (rev 7586) +++ trunk/pywikipedia/wikipedia.py 2009-11-03 11:59:53 UTC (rev 7587) @@ -8090,6 +8090,15 @@ raise PageNotFound(u'Page %s could not be retrieved. Check your family file ?' % url) else: return urllib.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers) + + def open_http(self, url, data=None): + ret = urllib.FancyURLopener.open_http(self, url, data) + if hasattr(self, 'http_code'): + ret.status = self.http_code + del self.http_code + else: + ret.status = 200 + return ret

14 years, 7 months

SVN: [7586] trunk/pywikipedia/welcome.py

by alexsh＠svn.wikimedia.org

Revision: 7586 Author: alexsh Date: 2009-11-03 11:56:50 +0000 (Tue, 03 Nov 2009) Log Message: ----------- comment typo Modified Paths: -------------- trunk/pywikipedia/welcome.py Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-11-03 11:02:03 UTC (rev 7585) +++ trunk/pywikipedia/welcome.py 2009-11-03 11:56:50 UTC (rev 7586) @@ -206,12 +206,12 @@ # The page where the bot will save the log (e.g. Wikipedia:Welcome log). # -# Log disable project: da, de, en, he, id, ka, pdc, pt, ru, vo. -# ATTENTION: these languages in wikipedia is disabled to log welcome users, not necessary to set parameter for them. +# ATTENTION: "Log disabled comment is listed the projects not to log welcomed users, and no necessary to set deatils. logbook = { 'commons': {'_default': u'Project:Welcome log', }, 'wikipedia': { '_default': None, + # Log disabled: da, de, en, he, id, ka, pdc, pt, ru, vo. 'ar': u'Project:سجل الترحيب', 'fa': u'Project:سیاهه خوشامد', 'fr': u'Wikipedia:Prise de décision/Accueil automatique des nouveaux par un robot/log',

14 years, 7 months

SVN: [7585] trunk/pywikipedia/cosmetic_changes.py

by xqt＠svn.wikimedia.org

Revision: 7585 Author: xqt Date: 2009-11-03 11:02:03 +0000 (Tue, 03 Nov 2009) Log Message: ----------- validHTML feature by jandudik (#2890674) Modified Paths: -------------- trunk/pywikipedia/cosmetic_changes.py Modified: trunk/pywikipedia/cosmetic_changes.py =================================================================== --- trunk/pywikipedia/cosmetic_changes.py 2009-11-03 09:57:15 UTC (rev 7584) +++ trunk/pywikipedia/cosmetic_changes.py 2009-11-03 11:02:03 UTC (rev 7585) @@ -372,7 +372,7 @@ return text def validXhtml(self, text): - text = pywikibot.replaceExcept(text, r'<br>', r'<br />', ['comment', 'math', 'nowiki', 'pre']) + text = pywikibot.replaceExcept(text, r'(?i)<br[ /]*>', r'<br />', ['comment', 'math', 'nowiki', 'pre']) return text def removeUselessSpaces(self, text):

14 years, 7 months

SVN: [7584] trunk/pywikipedia/cosmetic_changes.py

by xqt＠svn.wikimedia.org

Revision: 7584 Author: xqt Date: 2009-11-03 09:57:15 +0000 (Tue, 03 Nov 2009) Log Message: ----------- Bugfix for missing newline Modified Paths: -------------- trunk/pywikipedia/cosmetic_changes.py Modified: trunk/pywikipedia/cosmetic_changes.py =================================================================== --- trunk/pywikipedia/cosmetic_changes.py 2009-11-03 07:18:19 UTC (rev 7583) +++ trunk/pywikipedia/cosmetic_changes.py 2009-11-03 09:57:15 UTC (rev 7584) @@ -337,6 +337,8 @@ newLink = ' ' + newLink if hadTrailingSpaces: newLink = newLink + ' ' + if newline: + newLink = newline + newLink return newLink # don't change anything return match.group()

14 years, 7 months

SVN: [7583] trunk/pywikipedia/clean_sandbox.py

by xqt＠svn.wikimedia.org

Revision: 7583 Author: xqt Date: 2009-11-03 07:18:19 +0000 (Tue, 03 Nov 2009) Log Message: ----------- +als +bar Modified Paths: -------------- trunk/pywikipedia/clean_sandbox.py Modified: trunk/pywikipedia/clean_sandbox.py =================================================================== --- trunk/pywikipedia/clean_sandbox.py 2009-11-02 18:43:03 UTC (rev 7582) +++ trunk/pywikipedia/clean_sandbox.py 2009-11-03 07:18:19 UTC (rev 7583) @@ -20,6 +20,7 @@ # (C) Wikipedian, 2006-2007 # (C) Andre Engels, 2007 # (C) Siebrand Mazeland, 2007 +# (C) xqt, 2009 # # Distributed under the terms of the MIT license. # @@ -30,7 +31,9 @@ import time content = { + 'als':u'{{subst:/Vorlage}}', 'ar': u'{{من فضلك اترك هذا السطر ولا تعدله (عنوان ساحة التجربة)}}\n', + 'bar':u'{{Bitte erst NACH dieser Zeile schreiben! (Begrüßungskasten)}}\r\n', 'cs': u'{{subst:/uhrabat}}', 'da': u'{{subst:Sandkasse tekst}}', 'de': u'{{Bitte erst NACH dieser Zeile schreiben! (Begrüßungskasten)}}\r\n', @@ -55,7 +58,9 @@ } msg = { + 'als':u'Bötli: Sandchaschte iigebnet.', 'ar': u'روبوت: هذه الصفحة سيتم تفريغها تلقائيا', + 'bar':u'Bot: Spielwiesn gmaht.', 'cs': u'Uhrabání pískoviště', 'da': u'Bot: Nyt sand (fra[[Skabelon:Sandkasse tekst]])', 'de': u'Bot: Setze Spielwiese zurück.', @@ -79,7 +84,9 @@ } sandboxTitle = { + 'als':u'Wikipedia:Sandchaschte', 'ar': u'ويكيبيديا:ساحة التجربة', + 'bar':u'Wikipedia:Spielwiese', 'cs': u'Wikipedie:Pískoviště', 'da': u'Wikipedia:Sandkassen', 'de': u'Wikipedia:Spielwiese',

14 years, 7 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn November 2009