http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11291
Revision: 11291
Author: drtrigon
Date: 2013-03-29 10:38:05 +0000 (Fri, 29 Mar 2013)
Log Message:
-----------
bug fix; windows compatiblity for r11251
bug fix; correct familiy recognition (data_repository) at command line, see r11247 and r11255
improvement; code and syntax clean up (change 'WD' to 'data')
Modified Paths:
--------------
trunk/pywikipedia/subster.py
Modified: trunk/pywikipedia/subster.py
===================================================================
--- trunk/pywikipedia/subster.py 2013-03-29 08:49:35 UTC (rev 11290)
+++ trunk/pywikipedia/subster.py 2013-03-29 10:38:05 UTC (rev 11291)
@@ -161,12 +161,12 @@
# modification of timezone to be in sync with wiki
os.environ['TZ'] = 'Europe/Amsterdam'
-
- # windows doesn't have that attribute
- # TODO: fix it
if hasattr(time, "tzset"):
time.tzset()
- pywikibot.output(u'Setting process TimeZone (TZ): %s' % str(time.tzname)) # ('CET', 'CEST')
+ pywikibot.output(u'Setting process TimeZone (TZ): %s' % str(time.tzname)) # ('CET', 'CEST')
+ else:
+ # e.g. windows doesn't have that attribute
+ pywikibot.output(u'WARNING: This operating system has NO SUPPORT for setting TimeZone by code! Before running this script, please set the TimeZone manually to one approriate for use with the Wikipedia language and region you intend to.')
# init constants
self._bot_config = bot_config
@@ -239,11 +239,10 @@
**flags )
# DRTRIGON-130: data repository (wikidata) output to items
- if self.site.is_data_repository() or\
- (self.site.family.name == 'wikidata'): # (work-a-round)
- data = self.WD_convertContent(substed_content)
- #print self.WD_save(dataoutpage, dic[u'claims'], {u'p32': data}, summary)
- self.WD_save(page, data)
+ if self.site.is_data_repository():
+ data = self.data_convertContent(substed_content)
+ #print self.data_save(dataoutpage, dic[u'claims'], {u'p32': data}, summary)
+ self.data_save(page, data)
else:
pywikibot.output(u'NOTHING TO DO!')
@@ -506,7 +505,7 @@
pywikibot.output(u''.join(diff))
pywikibot.output(u'--- ' * 15)
- def WD_convertContent(self, substed_content):
+ def data_convertContent(self, substed_content):
"""Converts the substed content to Wikidata format in order to save.
(1 line of wiki text is converted to 1 claim/statement)
@@ -528,7 +527,7 @@
return res
-# def WD_save(self, outpage, dic, data, comment=None):
+# def data_save(self, outpage, dic, data, comment=None):
# """Stores the content to Wikidata.
#
# @param dic: Original content.
@@ -586,7 +585,7 @@
#
# return changed
- def WD_save(self, page, data):
+ def data_save(self, page, data):
"""Stores the content to Wikidata.
@param page: Page containing template.
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11290
Revision: 11290
Author: xqt
Date: 2013-03-29 08:49:35 +0000 (Fri, 29 Mar 2013)
Log Message:
-----------
check whether all language codes are listed at wikistats. Otherwise extend the new list with the difference.
Modified Paths:
--------------
trunk/pywikipedia/maintenance/wikimedia_sites.py
Modified: trunk/pywikipedia/maintenance/wikimedia_sites.py
===================================================================
--- trunk/pywikipedia/maintenance/wikimedia_sites.py 2013-03-29 08:45:29 UTC (rev 11289)
+++ trunk/pywikipedia/maintenance/wikimedia_sites.py 2013-03-29 08:49:35 UTC (rev 11290)
@@ -13,7 +13,7 @@
import sys, re
sys.path.insert(1, '..')
-import wikipedia
+import pywikibot
import codecs
familiesDict = {
@@ -31,13 +31,13 @@
if not families:
families = familiesDict.keys()
for family in families:
- wikipedia.output('Checking family %s:' % family)
+ pywikibot.output('Checking family %s:' % family)
- original = wikipedia.Family(family).languages_by_size
- obsolete = wikipedia.Family(family).obsolete
+ original = pywikibot.Family(family).languages_by_size
+ obsolete = pywikibot.Family(family).obsolete
url = 'http://s23.org/wikistats/%s' % familiesDict[family]
- uo = wikipedia.MyURLopener
+ uo = pywikibot.MyURLopener
f = uo.open(url)
text = f.read()
@@ -53,9 +53,11 @@
continue
new.append(lang)
if original == new:
- wikipedia.output(u'The lists match!')
+ pywikibot.output(u'The lists match!')
else:
- wikipedia.output(u"The lists don't match, the new list is:")
+ pywikibot.output(u"The lists don't match, the new list is:")
+ missing = set(original) - set(new)
+ new += missing
text = u' self.languages_by_size = [\r\n'
line = ' ' * 11
for lang in new:
@@ -63,15 +65,20 @@
line += u" '%s'," % lang
else:
text += u'%s\r\n' % line
- line = ' '
+ line = ' ' * 11
line += u" '%s'," % lang
text += u'%s\r\n' % line
text += u' ]'
- wikipedia.output(text)
+ pywikibot.output(text)
+ if missing:
+ pywikibot.output(u"WARNING: ['%s'] not listed at wikistats.\n"
+ u"Now listed as last item\n"
+ % "', '".join(missing))
family_file_name = '../families/%s_family.py' % family
family_file = codecs.open(family_file_name, 'r', 'utf8')
old_text = family_text = family_file.read()
- old = re.findall(ur'(?msu)^ {8}self.languages_by_size.+?\]', family_text)[0]
+ old = re.findall(ur'(?msu)^ {8}self.languages_by_size.+?\]',
+ family_text)[0]
family_text = family_text.replace(old, text)
family_file = codecs.open(family_file_name, 'w', 'utf8')
family_file.write(family_text)
@@ -80,9 +87,9 @@
if __name__ == '__main__':
try:
fam = []
- for arg in wikipedia.handleArgs():
+ for arg in pywikibot.handleArgs():
if arg in familiesDict.keys() and arg not in fam:
fam.append(arg)
update_family(fam)
finally:
- wikipedia.stopme()
+ pywikibot.stopme()
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11286
Revision: 11286
Author: xqt
Date: 2013-03-28 17:08:58 +0000 (Thu, 28 Mar 2013)
Log Message:
-----------
remove obsolete interwiki message on top of iw links
Modified Paths:
--------------
trunk/pywikipedia/add_text.py
Modified: trunk/pywikipedia/add_text.py
===================================================================
--- trunk/pywikipedia/add_text.py 2013-03-28 17:03:30 UTC (rev 11285)
+++ trunk/pywikipedia/add_text.py 2013-03-28 17:08:58 UTC (rev 11286)
@@ -87,9 +87,7 @@
'¶ms;': pagegenerators.parameterHelp,
}
-nn_iw_msg = u'<!--interwiki (no, sv, da first; then other languages alphabetically by name)-->'
-
class NoEnoughData(pywikibot.Error):
""" Error class for when the user doesn't specified all the data needed """
@@ -240,16 +238,6 @@
interwikiInside = pywikibot.getLanguageLinks(newtext, site)
# Removing the interwiki
newtext = pywikibot.removeLanguageLinks(newtext, site)
- # nn got a message between the categories and the iw's
- # and they want to keep it there, first remove it
- hasCommentLine = False
- if (site.language() == u'nn'):
- regex = re.compile(
- '(<!-- ?interwiki \(no(?:/nb)?, ?sv, ?da first; then other languages alphabetically by name\) ?-->)')
- found = regex.findall(newtext)
- if found:
- hasCommentLine = True
- newtext = regex.sub('', newtext)
# Adding the text
newtext += u"\n%s" % addText
@@ -257,9 +245,6 @@
newtext = pywikibot.replaceCategoryLinks(newtext,
categoriesInside, site,
True)
- #Put the nn iw message back
- if site.language() == u'nn' and (interwikiInside or hasCommentLine):
- newtext = newtext + u'\r\n\r\n' + nn_iw_msg
# Dealing the stars' issue
allstars = []
starstext = pywikibot.removeDisabledParts(text)
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11284
Revision: 11284
Author: xqt
Date: 2013-03-28 16:25:40 +0000 (Thu, 28 Mar 2013)
Log Message:
-----------
remove obsolete interwiki message on top of iw links
Modified Paths:
--------------
trunk/pywikipedia/cosmetic_changes.py
Modified: trunk/pywikipedia/cosmetic_changes.py
===================================================================
--- trunk/pywikipedia/cosmetic_changes.py 2013-03-28 15:33:20 UTC (rev 11283)
+++ trunk/pywikipedia/cosmetic_changes.py 2013-03-28 16:25:40 UTC (rev 11284)
@@ -52,8 +52,8 @@
cosmetic_changes_disable['wikipedia'] = ('de', 'en', 'fr')
"""
#
-# (C) xqt, 2009-2012
-# (C) Pywikipedia bot team, 2006-2012
+# (C) xqt, 2009-2013
+# (C) Pywikipedia bot team, 2006-2013
#
# Distributed under the terms of the MIT license.
#
@@ -77,14 +77,6 @@
'&warning;': warning,
}
-# Interwiki message on top of iw links
-# 2nd line is a regex if needed
-msg_interwiki = {
- 'fr': u'<!-- Autres langues -->',
- 'nn': (u'<!--interwiki (no, sv, da first; then other languages alphabetically by name)-->',
- u'(<!-- ?interwiki \(no(?:/nb)?, ?sv, ?da first; then other languages alphabetically by name\) ?-->)')
-}
-
# This is from interwiki.py;
# move it to family file and implement global instances
moved_links = {
@@ -248,7 +240,6 @@
categories = None
interwikiLinks = None
allstars = []
- hasCommentLine = False
# The PyWikipediaBot is no longer allowed to touch categories on the
# German Wikipedia. See
@@ -289,21 +280,6 @@
text = regex.sub('', text)
allstars += found
- # nn got a message between the categories and the iw's
- # and they want to keep it there, first remove it
- if self.site.lang in msg_interwiki:
- iw_msg = msg_interwiki[self.site.lang]
- if isinstance(iw_msg, tuple):
- iw_reg = iw_msg[1]
- iw_msg = iw_msg[0]
- else:
- iw_reg = u'(%s)' % iw_msg
- regex = re.compile(iw_reg)
- found = regex.findall(text)
- if found:
- hasCommentLine = True
- text = regex.sub('', text)
-
# Adding categories
if categories:
##Sorting categories in alphabetic order. beta test only on Persian Wikipedia, TODO fix bug for sorting
@@ -316,13 +292,6 @@
# categories.insert(0, name)
text = pywikibot.replaceCategoryLinks(text, categories,
site=self.site)
- # Put the iw message back
- if not self.talkpage and \
- ((interwikiLinks or hasCommentLine) and
- self.site.language() == 'nn' or
- (interwikiLinks and hasCommentLine) and
- self.site.language() == 'fr'):
- text += '\r\n\r\n' + iw_msg
# Adding stars templates
if allstars:
text = text.strip() + self.site.family.interwiki_text_separator
@@ -331,6 +300,7 @@
text += '%s\r\n' % element.strip()
if pywikibot.verbose:
pywikibot.output(u'%s' % element.strip())
+
# Adding the interwiki
if interwikiLinks:
text = pywikibot.replaceLanguageLinks(text, interwikiLinks,
@@ -397,7 +367,8 @@
if not aliases: continue
text = pywikibot.replaceExcept(
text,
- r'\[\[(?P<left>.+?:.+?\..+?\|) *(' + '|'.join(aliases) + ') *(?P<right>(\|.*?)?\]\])',
+ r'\[\[(?P<left>.+?:.+?\..+?\|) *(' + '|'.join(aliases) + \
+ ') *(?P<right>(\|.*?)?\]\])',
r'[[\g<left>' + aliases[0] + '\g<right>', exceptions)
return text
@@ -686,14 +657,16 @@
r'\1----\2', exceptions)
# horizontal line with attributes; can't be done with wiki syntax
# so we only make it XHTML compliant
- text = pywikibot.replaceExcept(text, r'(?i)<hr ([^>/]+?)>', r'<hr \1 />',
+ text = pywikibot.replaceExcept(text, r'(?i)<hr ([^>/]+?)>',
+ r'<hr \1 />',
exceptions)
# a header where only spaces are in the same line
for level in range(1, 7):
equals = '\\1%s \\2 %s\\3' % ("=" * level, "=" * level)
text = pywikibot.replaceExcept(
text,
- r'(?i)([\r\n]) *<h%d> *([^<]+?) *</h%d> *([\r\n])' % (level, level),
+ r'(?i)([\r\n]) *<h%d> *([^<]+?) *</h%d> *([\r\n])'
+ % (level, level),
r'%s' % equals,
exceptions)
# TODO: maybe we can make the bot replace <p> tags with \r\n's.
@@ -701,7 +674,8 @@
def fixReferences(self, text):
#http://en.wikipedia.org/wiki/User:AnomieBOT/source/tasks/OrphanReferenceFixer.pm
- exceptions = ['nowiki', 'comment', 'math', 'pre', 'source', 'startspace']
+ exceptions = ['nowiki', 'comment', 'math', 'pre', 'source',
+ 'startspace']
# it should be name = " or name=" NOT name ="
text = re.sub(r'(?i)<ref +name(= *| *=)"', r'<ref name="', text)
@@ -713,7 +687,8 @@
return text
def fixStyle(self, text):
- exceptions = ['nowiki', 'comment', 'math', 'pre', 'source', 'startspace']
+ exceptions = ['nowiki', 'comment', 'math', 'pre', 'source',
+ 'startspace']
# convert prettytable to wikitable class
if self.site.language in ('de', 'en'):
text = pywikibot.replaceExcept(text,
@@ -722,16 +697,21 @@
return text
def fixTypo(self, text):
- exceptions = ['nowiki', 'comment', 'math', 'pre', 'source', 'startspace', 'gallery', 'hyperlink', 'interwiki', 'link']
+ exceptions = ['nowiki', 'comment', 'math', 'pre', 'source',
+ 'startspace', 'gallery', 'hyperlink', 'interwiki', 'link']
# change <number> ccm -> <number> cm³
- text = pywikibot.replaceExcept(text, ur'(\d)\s* ccm', ur'\1 cm³', exceptions)
- text = pywikibot.replaceExcept(text, ur'(\d)\s*ccm', ur'\1 cm³', exceptions)
+ text = pywikibot.replaceExcept(text, ur'(\d)\s* ccm',
+ ur'\1 cm³', exceptions)
+ text = pywikibot.replaceExcept(text, ur'(\d)\s*ccm', ur'\1 cm³',
+ exceptions)
# Solve wrong Nº sign with °C or °F
# additional exception requested on fr-wiki for this stuff
pattern = re.compile(u'«.*?»', re.UNICODE)
exceptions.append(pattern)
- text = pywikibot.replaceExcept(text, ur'(\d)\s* [º°]([CF])', ur'\1 °\2', exceptions)
- text = pywikibot.replaceExcept(text, ur'(\d)\s*[º°]([CF])', ur'\1 °\2', exceptions)
+ text = pywikibot.replaceExcept(text, ur'(\d)\s* [º°]([CF])',
+ ur'\1 °\2', exceptions)
+ text = pywikibot.replaceExcept(text, ur'(\d)\s*[º°]([CF])',
+ ur'\1 °\2', exceptions)
text = pywikibot.replaceExcept(text, ur'º([CF])', ur'°\1', exceptions)
return text