Pywikipedia-svn April 2013

pywikipedia-svn@lists.wikimedia.org

9 participants
175 discussions

SVN: [11336] trunk/pywikipedia/maintenance/preferences.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11336 Revision: 11336 Author: xqt Date: 2013-04-04 07:50:38 +0000 (Thu, 04 Apr 2013) Log Message: ----------- remove deprecated parameter getSite() Modified Paths: -------------- trunk/pywikipedia/maintenance/preferences.py Modified: trunk/pywikipedia/maintenance/preferences.py =================================================================== --- trunk/pywikipedia/maintenance/preferences.py 2013-04-04 07:29:29 UTC (rev 11335) +++ trunk/pywikipedia/maintenance/preferences.py 2013-04-04 07:50:38 UTC (rev 11336) @@ -143,7 +143,7 @@ def set_for(lang, family, keys, values, verbose = False): import wikipedia - site = wikipedia.getSite(lang, family, persistent_http = True) + site = wikipedia.getSite(lang, family) prefs = Preferences(site) for key, value in zip(keys, values): prev = unicode(prefs.get(key, ''))

11 years, 2 months

SVN: [11335] branches/rewrite/pywikibot/textlib.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11335 Revision: 11335 Author: xqt Date: 2013-04-04 07:29:29 +0000 (Thu, 04 Apr 2013) Log Message: ----------- update from trunk r11333 Modified Paths: -------------- branches/rewrite/pywikibot/textlib.py Modified: branches/rewrite/pywikibot/textlib.py =================================================================== --- branches/rewrite/pywikibot/textlib.py 2013-04-04 06:30:35 UTC (rev 11334) +++ branches/rewrite/pywikibot/textlib.py 2013-04-04 07:29:29 UTC (rev 11335) @@ -19,6 +19,7 @@ from HTMLParser import HTMLParser import config2 as config +TEMP_REGEX = re.compile('{{(msg:)?(?P<name>[^{\|]+?)(\|(?P<params>[^{]+?))?}}') def unescape(s): """Replace escaped HTML-special characters by their originals""" @@ -75,14 +76,6 @@ # source code readability. # TODO: handle nested tables. 'table': re.compile(r'(?ims)^{\|.*?^\|}|<table>.*?</table>'), - # templates with parameters often have whitespace that is used to - # improve wiki source code readability. - # 'template': re.compile(r'(?s){{.*?}}'), - # The regex above fails on nested templates. This regex can handle - # templates cascaded up to level 2, but no deeper. For arbitrary - # depth, we'd need recursion which can't be done in Python's re. - # After all, the language of correct parenthesis words is not regular. - 'template': re.compile(r'(?s){{(({{.*?}})?.*?)*}}'), 'hyperlink': compileLinkR(), 'gallery': re.compile(r'(?is)<gallery.*?>.*?</gallery>'), # this matches internal wikilinks, but also interwiki, categories, and @@ -107,12 +100,15 @@ old = re.compile(old) dontTouchRegexes = [] + except_templates = False for exc in exceptions: if isinstance(exc, basestring): # assume it's a reference to the exceptionRegexes dictionary # defined above. if exc in exceptionRegexes: dontTouchRegexes.append(exceptionRegexes[exc]) + elif exc == 'template': + except_templates = True else: # nowiki, noinclude, includeonly, timeline, math ond other # extensions @@ -125,6 +121,35 @@ else: # assume it's a regular expression dontTouchRegexes.append(exc) + + # mark templates + # don't care about mw variables and parser functions + if except_templates: + marker1 = findmarker(text) + marker2 = findmarker(text, u'##', u'#') + Rvalue = re.compile('{{{.+?}}}') + Rmarker1 = re.compile('%(mark)s(\d+)%(mark)s' % {'mark': marker1}) + Rmarker2 = re.compile('%(mark)s(\d+)%(mark)s' % {'mark': marker2}) + values = {} + count = 0 + for m in Rvalue.finditer(text): + count += 1 + item = m.group() + text = text.replace(item, '%s%d%s' % (marker2, count, marker2)) + values[count] = item + inside = {} + count = 0 + while TEMP_REGEX.search(text) is not None: + for m in TEMP_REGEX.finditer(text): + count += 1 + item = m.group() + text = text.replace(item, '%s%d%s' % (marker1, count, marker1)) + + for m2 in Rmarker1.finditer(item): + item = item.replace(m2.group(), inside[int(m2.group(1))]) + for m2 in Rmarker2.finditer(item): + item = item.replace(m2.group(), values[int(m2.group(1))]) + inside[count] = item index = 0 markerpos = len(text) while True: @@ -194,6 +219,12 @@ index = match.start() + len(replacement) markerpos = match.start() + len(replacement) text = text[:markerpos] + marker + text[markerpos:] + + if except_templates: # restore templates from dict + for m2 in Rmarker1.finditer(text): + text = text.replace(m2.group(), inside[int(m2.group(1))]) + for m2 in Rmarker2.finditer(text): + text = text.replace(m2.group(), values[int(m2.group(1))]) return text @@ -831,7 +862,7 @@ #---------------------------------- def extract_templates_and_params(text): - """Return list of template calls found in text. + """Return a list of templates found in text. Return value is a list of tuples. There is one tuple for each use of a template in the page, with the template title as the first entry and a @@ -840,6 +871,8 @@ with an integer value corresponding to its position among the unnnamed parameters, and if this results multiple parameters with the same name only the last value provided will be returned. + @param text: The wikitext from which templates are extracted + @type text: unicode or string """ # remove commented-out stuff etc. @@ -858,8 +891,6 @@ marker4 = findmarker(thistxt, u'§§', u'§') result = [] - Rtemplate = re.compile( - ur'{{(msg:)?(?P<name>[^{\|]+?)(\|(?P<params>[^{]+?))?}}') Rmath = re.compile(ur'<math>[^<]+</math>') Rvalue = re.compile(r'{{{.+?}}}') Rmarker = re.compile(ur'%s(\d+)%s' % (marker, marker)) @@ -886,8 +917,8 @@ inside = {} count = 0 - while Rtemplate.search(thistxt) is not None: - for m in Rtemplate.finditer(thistxt): + while TEMP_REGEX.search(thistxt) is not None: + for m in TEMP_REGEX.finditer(thistxt): # Make sure it is not detected again count += 1 text = m.group() @@ -909,6 +940,35 @@ # Doesn't detect templates whose name changes, # or templates whose name contains math tags continue + + # {{#if: }} + if name.startswith('#'): + continue + +## TODO: merged from wikipedia.py - implement the following +## if self.site().isInterwikiLink(name): +## continue +## # {{DEFAULTSORT:...}} +## defaultKeys = self.site().versionnumber() > 13 and \ +## self.site().getmagicwords('defaultsort') +## # It seems some wikis does not have this magic key +## if defaultKeys: +## found = False +## for key in defaultKeys: +## if name.startswith(key): +## found = True +## break +## if found: continue +## +## try: +## name = Page(self.site(), name).title() +## except InvalidTitle: +## if name: +## output( +## u"Page %s contains invalid template name {{%s}}." +## % (self.title(), name.strip())) +## continue + # Parameters paramString = m.group('params') params = {}

11 years, 2 months

SVN: [11334] trunk/pywikipedia/catlib.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11334 Revision: 11334 Author: xqt Date: 2013-04-04 06:30:35 +0000 (Thu, 04 Apr 2013) Log Message: ----------- sortkeyprefix breaks with mw 1.16 bugfix for bug #3603953 Modified Paths: -------------- trunk/pywikipedia/catlib.py Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2013-04-04 05:54:15 UTC (rev 11333) +++ trunk/pywikipedia/catlib.py 2013-04-04 06:30:35 UTC (rev 11334) @@ -66,12 +66,15 @@ """Subclass of Page that has some special tricks that only work for category: pages""" - def __init__(self, site, title = None, insite = None, sortKey = None, sortKeyPrefix = None): - wikipedia.Page.__init__(self, site = site, title = title, insite = insite, defaultNamespace = 14) + def __init__(self, site, title=None, insite=None, sortKey=None, + sortKeyPrefix=None): + wikipedia.Page.__init__(self, site=site, title=title, insite=insite, + defaultNamespace=14) self.sortKey = sortKey self.sortKeyPrefix = sortKeyPrefix if self.namespace() != 14: - raise ValueError(u'BUG: %s is not in the category namespace!' % title) + raise ValueError(u'BUG: %s is not in the category namespace!' + % title) self.completelyCached = False self.articleCache = [] self.subcatCache = [] @@ -84,7 +87,8 @@ """ if self.sortKey: - titleWithSortKey = '%s|%s' % (self.title(savetitle=True), self.sortKey) + titleWithSortKey = '%s|%s' % (self.title(savetitle=True), + self.sortKey) else: titleWithSortKey = self.title(savetitle=True) if not noInterwiki and (forceInterwiki @@ -92,9 +96,11 @@ if self.site().family != wikipedia.getSite().family \ and self.site().family.name != self.site().lang: return '[[%s:%s:%s]]' % (self.site().family.name, - self.site().lang, self.title(savetitle=True)) + self.site().lang, + self.title(savetitle=True)) else: - return '[[%s:%s]]' % (self.site().lang, self.title(savetitle=True)) + return '[[%s:%s]]' % (self.site().lang, + self.title(savetitle=True)) elif textlink: return '[[:%s]]' % self.title(savetitle=True) else: @@ -159,8 +165,9 @@ # contents of subcategory are cached by calling # this method recursively; therefore, do not cache # them again - for item in page._getAndCacheContents(newrecurse, purge, cache=cache, - sortby=sortby, sortdir=sortdir): + for item in page._getAndCacheContents( + newrecurse, purge, cache=cache, + sortby=sortby, sortdir=sortdir): yield item if not startFrom: self.completelyCached = True @@ -207,9 +214,11 @@ 'action': 'query', 'list': 'categorymembers', 'cmtitle': self.title(), - 'cmprop': ['title', 'ids', 'sortkey', 'sortkeyprefix', 'timestamp'], + 'cmprop': ['title', 'ids', 'sortkey', 'timestamp'], #'': '', } + if self.site().versionnumber() > 16: + params['cmprop'].append('sortkeyprefix') if sortby: params['cmsort'] = sortby if sortdir: @@ -223,15 +232,17 @@ if currentPageOffset: params.update(currentPageOffset) wikipedia.output('Getting [[%s]] list from %s...' - % (self.title(), "%s=%s" % currentPageOffset.popitem())) + % (self.title(), + "%s=%s" % currentPageOffset.popitem())) else: msg = 'Getting [[%s]] list' % self.title() + # category sort keys are uppercase if startFrom: - startFrom = startFrom.upper() # category sort keys are uppercase + startFrom = startFrom.upper() params['cmstartsortkey'] = startFrom msg += ' starting at %s' % startFrom if endsort: - endsort = endsort.upper() # category sort keys are uppercase + endsort = endsort.upper() params['cmendsortkey'] = endsort msg += ' ending at %s' % endsort wikipedia.output(msg + u'...') @@ -246,11 +257,20 @@ count += 1 # For MediaWiki versions where subcats look like articles if memb['ns'] == 14: - yield SUBCATEGORY, Category(self.site(), memb['title'], sortKey=memb['sortkey'], sortKeyPrefix=memb['sortkeyprefix']) + if 'sortkeyprefix' in memb: + sortKeyPrefix = memb['sortkeyprefix'] + else: + sortKeyPrefix = None + yield SUBCATEGORY, \ + Category(self.site(), memb['title'], + sortKey=memb['sortkey'], + sortKeyPrefix=sortKeyPrefix) elif memb['ns'] == 6: - yield ARTICLE, wikipedia.ImagePage(self.site(), memb['title']) + yield ARTICLE, wikipedia.ImagePage(self.site(), + memb['title']) else: - yield ARTICLE, wikipedia.Page(self.site(), memb['title'], defaultNamespace=memb['ns']) + yield ARTICLE, wikipedia.Page(self.site(), memb['title'], + defaultNamespace=memb['ns']) if count >= params['cmlimit']: break # try to find a link to the next list page

11 years, 2 months

SVN: [11333] trunk/pywikipedia/pywikibot/textlib.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11333 Revision: 11333 Author: xqt Date: 2013-04-04 05:54:15 +0000 (Thu, 04 Apr 2013) Log Message: ----------- enable nested templates handling for textlib.replaceExcept() The old implementation could only handle templates cascaded up to level 2 and in some circumstances it fails into an infinite loop. Now we use a similar code of textlib.extract_templates_and_params() resp. templatesWithParams() to hide and restore the templates. MediaWiki variables and parser functions are handled as templates. Bugfix for bug #3603994, bug #2819291, bug #3158761 Modified Paths: -------------- trunk/pywikipedia/pywikibot/textlib.py Modified: trunk/pywikipedia/pywikibot/textlib.py =================================================================== --- trunk/pywikipedia/pywikibot/textlib.py 2013-04-03 22:39:10 UTC (rev 11332) +++ trunk/pywikipedia/pywikibot/textlib.py 2013-04-04 05:54:15 UTC (rev 11333) @@ -19,6 +19,7 @@ from HTMLParser import HTMLParser import config +TEMP_REGEX = re.compile('{{(msg:)?(?P<name>[^{\|]+?)(\|(?P<params>[^{]+?))?}}') def unescape(s): """Replace escaped HTML-special characters by their originals""" @@ -75,14 +76,6 @@ # source code readability. # TODO: handle nested tables. 'table': re.compile(r'(?ims)^{\|.*?^\|}|<table>.*?</table>'), - # templates with parameters often have whitespace that is used to - # improve wiki source code readability. - # 'template': re.compile(r'(?s){{.*?}}'), - # The regex above fails on nested templates. This regex can handle - # templates cascaded up to level 2, but no deeper. For arbitrary - # depth, we'd need recursion which can't be done in Python's re. - # After all, the language of correct parenthesis words is not regular. - 'template': re.compile(r'(?s){{(({{.*?}})?.*?)*}}'), 'hyperlink': compileLinkR(), 'gallery': re.compile(r'(?is)<gallery.*?>.*?</gallery>'), # this matches internal wikilinks, but also interwiki, categories, and @@ -107,12 +100,15 @@ old = re.compile(old) dontTouchRegexes = [] + except_templates = False for exc in exceptions: if isinstance(exc, basestring): # assume it's a reference to the exceptionRegexes dictionary # defined above. if exc in exceptionRegexes: dontTouchRegexes.append(exceptionRegexes[exc]) + elif exc == 'template': + except_templates = True else: # nowiki, noinclude, includeonly, timeline, math ond other # extensions @@ -125,6 +121,35 @@ else: # assume it's a regular expression dontTouchRegexes.append(exc) + + # mark templates + # don't care about mw variables and parser functions + if except_templates: + marker1 = findmarker(text) + marker2 = findmarker(text, u'##', u'#') + Rvalue = re.compile('{{{.+?}}}') + Rmarker1 = re.compile('%(mark)s(\d+)%(mark)s' % {'mark': marker1}) + Rmarker2 = re.compile('%(mark)s(\d+)%(mark)s' % {'mark': marker2}) + values = {} + count = 0 + for m in Rvalue.finditer(text): + count += 1 + item = m.group() + text = text.replace(item, '%s%d%s' % (marker2, count, marker2)) + values[count] = item + inside = {} + count = 0 + while TEMP_REGEX.search(text) is not None: + for m in TEMP_REGEX.finditer(text): + count += 1 + item = m.group() + text = text.replace(item, '%s%d%s' % (marker1, count, marker1)) + + for m2 in Rmarker1.finditer(item): + item = item.replace(m2.group(), inside[int(m2.group(1))]) + for m2 in Rmarker2.finditer(item): + item = item.replace(m2.group(), values[int(m2.group(1))]) + inside[count] = item index = 0 markerpos = len(text) while True: @@ -194,6 +219,12 @@ index = match.start() + len(replacement) markerpos = match.start() + len(replacement) text = text[:markerpos] + marker + text[markerpos:] + + if except_templates: # restore templates from dict + for m2 in Rmarker1.finditer(text): + text = text.replace(m2.group(), inside[int(m2.group(1))]) + for m2 in Rmarker2.finditer(text): + text = text.replace(m2.group(), values[int(m2.group(1))]) return text @@ -863,8 +894,6 @@ marker4 = findmarker(thistxt, u'§§', u'§') result = [] - Rtemplate = re.compile( - ur'{{(msg:)?(?P<name>[^{\|]+?)(\|(?P<params>[^{]+?))?}}') Rmath = re.compile(ur'<math>[^<]+</math>') Rvalue = re.compile(r'{{{.+?}}}') Rmarker = re.compile(ur'%s(\d+)%s' % (marker, marker)) @@ -891,8 +920,8 @@ inside = {} count = 0 - while Rtemplate.search(thistxt) is not None: - for m in Rtemplate.finditer(thistxt): + while TEMP_REGEX.search(thistxt) is not None: + for m in TEMP_REGEX.finditer(thistxt): # Make sure it is not detected again count += 1 text = m.group()

11 years, 2 months

SVN: [11332] trunk/pywikipedia/commonsdelinker/image_replacer.py

by siebrand＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11332 Revision: 11332 Author: siebrand Date: 2013-04-03 22:39:10 +0000 (Wed, 03 Apr 2013) Log Message: ----------- Remove persistent_http = True Why do we deprecate and not even fix the tools in the repo itself? Not cool... Thanks to Betacommand for the help on IRC. Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/image_replacer.py Modified: trunk/pywikipedia/commonsdelinker/image_replacer.py =================================================================== --- trunk/pywikipedia/commonsdelinker/image_replacer.py 2013-04-03 20:03:54 UTC (rev 11331) +++ trunk/pywikipedia/commonsdelinker/image_replacer.py 2013-04-03 22:39:10 UTC (rev 11332) @@ -45,7 +45,7 @@ self.disallowed_replacements = [(re.compile(i[0], re.I), re.compile(i[1], re.I)) for i in self.config.get('disallowed_replacements', ())] - self.site = wikipedia.getSite(persistent_http = True) + self.site = wikipedia.getSite() self.site.forceLogin() self.database = connect_database() @@ -220,7 +220,7 @@ if self.config.get('replacer_run_once', False): self.run_once() return - + while True: self.run_once() time.sleep(self.config['replacer_timeout']) @@ -242,11 +242,11 @@ def process_reports(self): end_time = time.time() + self.config['replacer_timeout'] - + while self.reporters and time.time() < end_time: report = self.reporters[0] del self.reporters[0] - + self.report(report) def report(self, (old_image, new_image, user, comment, not_ok)):

11 years, 2 months

SVN: [11331] trunk/pywikipedia/flickrripper.py

by legoktm＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11331 Revision: 11331 Author: legoktm Date: 2013-04-03 20:03:54 +0000 (Wed, 03 Apr 2013) Log Message: ----------- Check that the api_secret key is set before checking it's value Modified Paths: -------------- trunk/pywikipedia/flickrripper.py Modified: trunk/pywikipedia/flickrripper.py =================================================================== --- trunk/pywikipedia/flickrripper.py 2013-04-03 20:01:45 UTC (rev 11330) +++ trunk/pywikipedia/flickrripper.py 2013-04-03 20:03:54 UTC (rev 11331) @@ -501,7 +501,7 @@ 'Any flickr user can get a key at http://www.flickr.com/services/api/keys/apply/') return - if config.flickr['api_secret']: + if 'api_secret' in config.flickr and config.flickr['api_secret']: flickr = flickrapi.FlickrAPI(config.flickr['api_key'], config.flickr['api_secret']) (token, frob) = flickr.get_token_part_one(perms='read') if not token: # The user still hasn't authorised this app yet, get_token_part_one() will have spawn a browser window

11 years, 2 months

SVN: [11330] trunk/pywikipedia/wikipedia.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11330 Revision: 11330 Author: xqt Date: 2013-04-03 20:01:45 +0000 (Wed, 03 Apr 2013) Log Message: ----------- to keep compatible we call extract_templates_and_params(thistxt, asList=True) to get the parameter list. NEW: The parameter items are strip()ped! Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2013-04-03 19:59:51 UTC (rev 11329) +++ trunk/pywikipedia/wikipedia.py 2013-04-03 20:01:45 UTC (rev 11330) @@ -2842,7 +2842,7 @@ output(u"""\ thistxt argument of templatesWithParams is deprecated. Please use textlib method extract_templates_and_params() instead.""") - return extract_templates_and_params(thistxt) + return extract_templates_and_params(thistxt, asList=True) def getRedirectTarget(self): """Return a Page object for the target this Page redirects to.

11 years, 2 months

SVN: [11329] trunk/pywikipedia/pywikibot/textlib.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11329 Revision: 11329 Author: xqt Date: 2013-04-03 19:59:51 +0000 (Wed, 03 Apr 2013) Log Message: ----------- return dictionary as default for extract_templates_and_params() Modified Paths: -------------- trunk/pywikipedia/pywikibot/textlib.py Modified: trunk/pywikipedia/pywikibot/textlib.py =================================================================== --- trunk/pywikipedia/pywikibot/textlib.py 2013-04-03 19:53:22 UTC (rev 11328) +++ trunk/pywikipedia/pywikibot/textlib.py 2013-04-03 19:59:51 UTC (rev 11329) @@ -828,14 +828,14 @@ # Functions dealing with templates #---------------------------------- -def extract_templates_and_params(text, asDict=False): +def extract_templates_and_params(text, asList=False): """Return a list of templates found in text. Return value is a list of tuples. There is one tuple for each use of a template in the page, with the template title as the first entry and either a list of parameters or a dict of parameters as the second entry - which depends on asDict method parameter. - If asDict is True the parameters is a dict, and they are indexed by strings; + which depends on asList method parameter. + If asList is False the parameters is a dict, and they are indexed by strings; as in MediaWiki, an unnamed parameter is given a parameter name with an integer value corresponding to its position among the unnamed parameters, and if this results multiple parameters with the same name, only the last @@ -843,8 +843,8 @@ @param text: The wikitext from which templates are extracted @type text: unicode or string - @param asDict: If True, return parameters as list, else as dict - @type asDict: bool + @param asList: If True, return parameters as list, else as dict + @type asList: bool """ # remove commented-out stuff etc. @@ -961,7 +961,7 @@ markedParams = paramString.split('|') # Replace markers for param in markedParams: - if asDict and "=" in param: + if not asList and "=" in param: param_name, param_val = param.split("=", 1) else: param_name = unicode(numbered_param) @@ -982,10 +982,10 @@ params[param_name.strip()] = param_val.strip() # Add it to the result - if asDict: + if asList: + result.append((name, params.values())) + else: result.append((name, params)) - else: - result.append((name, params.values())) return result

11 years, 2 months

SVN: [11328] trunk/pywikipedia/subster.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11328 Revision: 11328 Author: xqt Date: 2013-04-03 19:53:22 +0000 (Wed, 03 Apr 2013) Log Message: ----------- partially revert r11327 Modified Paths: -------------- trunk/pywikipedia/subster.py Modified: trunk/pywikipedia/subster.py =================================================================== --- trunk/pywikipedia/subster.py 2013-04-03 19:50:12 UTC (rev 11327) +++ trunk/pywikipedia/subster.py 2013-04-03 19:53:22 UTC (rev 11328) @@ -328,8 +328,7 @@ # 0.2.) check for 'simple' mode and get additional params if param['simple']: p = self.site.getExpandedString(param['simple']) - param.update( - pywikibot.extract_templates_and_params(p, asDict=True)[0][1]) + param.update(pywikibot.extract_templates_and_params(p)[0][1]) # 0.5.) check cron/date if param['cron']:

11 years, 2 months

SVN: [11327] trunk/pywikipedia/subster.py

by xqt＠svn.wikimedia.org

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11327 Revision: 11327 Author: xqt Date: 2013-04-03 19:50:12 +0000 (Wed, 03 Apr 2013) Log Message: ----------- extract_templates_and_params() call with asDict=True; strip trailing white space, some PEP8 changes Modified Paths: -------------- trunk/pywikipedia/subster.py Modified: trunk/pywikipedia/subster.py =================================================================== --- trunk/pywikipedia/subster.py 2013-04-03 19:39:46 UTC (rev 11326) +++ trunk/pywikipedia/subster.py 2013-04-03 19:50:12 UTC (rev 11327) @@ -267,7 +267,7 @@ for item in params: # 1st stage: main/general content substitution # 1.) - 5.) subst templates - metadata = { 'bot-error': unicode(False), + metadata = { 'bot-error': unicode(False), 'bot-error-traceback': u'', } # DRTRIGON-132 try: (substed_content, tags, md) = self.subTemplate(substed_content, item) @@ -318,16 +318,18 @@ Returns a tuple containig the new content with tags substituted and a list of those tags. + """ substed_tags = [] # DRTRIGON-73 - metadata = { 'mw-signature': u'~~~~', - 'mw-timestamp': u'~~~~~', } # DRTRIGON-132 + metadata = {'mw-signature': u'~~~~', + 'mw-timestamp': u'~~~~~',} # DRTRIGON-132 # 0.2.) check for 'simple' mode and get additional params if param['simple']: p = self.site.getExpandedString(param['simple']) - param.update( pywikibot.extract_templates_and_params(p)[0][1] ) + param.update( + pywikibot.extract_templates_and_params(p, asDict=True)[0][1]) # 0.5.) check cron/date if param['cron']: @@ -337,9 +339,14 @@ param['cron'] = '* * ' + param['cron'] entry = crontab.CronTab(param['cron']) # find the delay from midnight (does not return 0.0 - but next) - delay = entry.next(datetime.datetime.now().replace(hour=0,minute=0,second=0,microsecond=0)-datetime.timedelta(microseconds=1)) + delay = entry.next(datetime.datetime.now().replace(hour=0, + minute=0, + second=0, + microsecond=0)- \ + datetime.timedelta(microseconds=1)) - pywikibot.output(u'CRON delay for execution: %.3f (<= %i)' % (delay, bot_config['CRONMaxDelay'])) + pywikibot.output(u'CRON delay for execution: %.3f (<= %i)' + % (delay, bot_config['CRONMaxDelay'])) if not (delay <= bot_config['CRONMaxDelay']): return (content, substed_tags, metadata) @@ -348,7 +355,8 @@ # (security: check url not to point to a local file on the server, # e.g. 'file://' - same as used in xsalt.py) secure = False - for item in [u'http://', u'https://', u'mail://', u'local://', u'wiki://']: + for item in [u'http://', u'https://', u'mail://', u'local://', + u'wiki://']: secure = secure or (param['url'][:len(item)] == item) param['zip'] = ast.literal_eval(param['zip']) if not secure: @@ -356,19 +364,24 @@ if (param['url'][:7] == u'wiki://'): url = param['url'][7:].strip('[]') # enable wiki-links if ast.literal_eval(param['expandtemplates']): # DRTRIGON-93 (only with 'wiki://') - external_buffer = pywikibot.Page(self.site, url).get(expandtemplates=True) + external_buffer = pywikibot.Page(self.site, + url).get(expandtemplates=True) else: external_buffer = self.load( pywikibot.Page(self.site, url) ) elif (param['url'][:7] == u'mail://'): # DRTRIGON-101 url = param['url'].replace(u'{{@}}', u'@') # e.g. nlwiki - mbox = SubsterMailbox(pywikibot.config.datafilepath(bot_config['data_path'], bot_config['mbox_file'], '')) + mbox = SubsterMailbox( + pywikibot.config.datafilepath(bot_config['data_path'], + bot_config['mbox_file'], '')) external_buffer = mbox.find_data(url) mbox.close() elif (param['url'][:8] == u'local://'): # DRTRIGON-131 if (param['url'][8:] == u'cache/state_bots'): # filename hard-coded - d = shelve.open(pywikibot.config.datafilepath('cache', 'state_bots')) - external_buffer = pprint.pformat(ast.literal_eval(pprint.pformat(d))) + d = shelve.open(pywikibot.config.datafilepath('cache', + 'state_bots')) + external_buffer = pprint.pformat( + ast.literal_eval(pprint.pformat(d))) d.close() else: external_buffer = u'n/a' @@ -378,12 +391,13 @@ # on page, if the user placed them, else use the conventional mode. # http://www.diveintopython.net/http_web_services/etags.html f_url, external_buffer = http.request(self.site, param['url'], - no_hostname = True, + no_hostname = True, back_response = True) headers = f_url.headers # same like 'f_url.info()' #if param['zip']: if ('text/' not in headers['content-type']): - pywikibot.output(u'Source is of non-text content-type, using raw data instead.') + pywikibot.output(u'Source is of non-text content-type, ' + u'using raw data instead.') external_buffer = f_url.read() del f_url # free some memory (no need to keep copy) @@ -413,7 +427,8 @@ if external_data: # not None external_data = external_data.groups() - pywikibot.output(u'Groups found by regex: %i' % len(external_data)) + pywikibot.output(u'Groups found by regex: %i' + % len(external_data)) # DRTRIGON-114: Support for named groups in regexs if regex.groupindex: @@ -474,7 +489,7 @@ def subTag(self, content, value, external_data=u'~~~~', count=1): """Substitute one single tag (of a template) in content. - + Can also be (ab)used to check for presence of a tag. """ substed_tags = []

11 years, 2 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn April 2013