http://www.mediawiki.org/wiki/Special:Code/pywikipedia/9557
Revision: 9557
Author: xqt
Date: 2011-09-25 20:29:51 +0000 (Sun, 25 Sep 2011)
Log Message:
-----------
update from trunk r9556
Modified Paths:
--------------
branches/rewrite/scripts/template.py
Modified: branches/rewrite/scripts/template.py
===================================================================
--- branches/rewrite/scripts/template.py 2011-09-25 20:21:45 UTC (rev 9556)
+++ branches/rewrite/scripts/template.py 2011-09-25 20:29:51 UTC (rev 9557)
@@ -5,9 +5,9 @@
Syntax: python template.py [-remove] [xml[:filename]] oldTemplate [newTemplate]
-Specify the template on the command line. The program will
-pick up the template page, and look for all pages using it. It will
-then automatically loop over them, and replace the template.
+Specify the template on the command line. The program will pick up the template
+page, and look for all pages using it. It will then automatically loop over
+them, and replace the template.
Command line options:
@@ -17,14 +17,16 @@
article. This is done by changing {{...}} or {{msg:...}} into
{{subst:...}}
+-assubst Replaces the first argument as old template with the second
+ argument as new template but substitutes it like -subst does.
+ Using both options -remove and -subst in the same command line has
+ the same effect.
+
-xml retrieve information from a local dump
(http://download.wikimedia.org). If this argument isn\'t given,
info will be loaded from the maintenance page of the live wiki.
argument can also be given as "-xml:filename.xml".
--namespace: Only process templates in the given namespace number (may be used
- multiple times).
-
-user: Only process pages edited by a given user
-skipuser: Only process pages not edited by a given user
@@ -64,18 +66,16 @@
If you have a template called [[Template:test]] and want to substitute it only
on pages in the User: and User talk: namespaces, do:
- python template.py test -namespace:2 -namespace:3
+ python template.py test -subst -namespace:2 -namespace:3
-Note that, on the English Wikipedia, User: is namespace 2 and User talk: is
-namespace 3. This may differ on other projects so make sure to find out the
-appropriate namespace numbers.
+Note that -namespace: is a global pywikipedia parameter
This next example substitutes the template lived with a supplied edit summary.
It only performs substitutions in main article namespace and doesn't prompt to
start replacing. Note that -putthrottle: is a global pywikipedia parameter.
- python template.py -putthrottle:30 -namespace:0 lived -always
+ python template.py -putthrottle:30 -namespace:0 lived -subst -always
-summary:"ROBOT: Substituting {{lived}}, see [[WP:SUBST]]."
@@ -255,12 +255,16 @@
r'(?P<parameters>\s*\|.+?|) *}}',
re.DOTALL)
- if self.remove:
- replacements.append((templateRegex, ''))
+ if self.subst and self.remove:
+ replacements.append((templateRegex,
+ '{{subst:%s\g<parameters>}}' % new))
+ exceptions['inside-tags']=['ref', 'gallery']
elif self.subst:
replacements.append((templateRegex,
'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags']=['ref', 'gallery']
+ elif self.remove:
+ replacements.append((templateRegex, ''))
else:
replacements.append((templateRegex,
'{{%s\g<parameters>}}' % new))
@@ -292,6 +296,8 @@
remove = True
elif arg == '-subst':
subst = True
+ elif arg == '-assubst':
+ subst = remove = True
elif arg == ('-always'):
acceptAll = True
elif arg.startswith('-xml'):
@@ -300,11 +306,6 @@
u'Please enter the XML dump\'s filename: ')
else:
xmlfilename = arg[5:]
- elif arg.startswith('-namespace:'):
- try:
- namespaces.append(int(arg[len('-namespace:'):]))
- except ValueError:
- namespaces.append(arg[len('-namespace:'):])
elif arg.startswith('-category:'):
addedCat = arg[len('-category:'):]
elif arg.startswith('-summary:'):
@@ -322,7 +323,7 @@
pywikibot.Page(pywikibot.Site(), arg,
ns=10).title(withNamespace=False))
- if subst or remove:
+ if subst ^ remove:
for templateName in templateNames:
templates[templateName] = None
else:
@@ -331,7 +332,7 @@
templates[templateNames[i]] = templateNames[i + 1]
except IndexError:
pywikibot.output(
-u'Unless using -subst or -remove, you must give an even number of template names.')
+u'Unless using solely -subst or -remove, you must give an even number of template names.')
return
oldTemplates = []
@@ -351,8 +352,6 @@
gen = pagegenerators.CombinedPageGenerator(gens)
gen = pagegenerators.DuplicateFilterPageGenerator(gen)
- if namespaces:
- gen = pagegenerators.NamespaceFilterPageGenerator(gen, namespaces)
if user:
gen = UserEditFilterGenerator(gen, user, timestamp, skip)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/9556
Revision: 9556
Author: xqt
Date: 2011-09-25 20:21:45 +0000 (Sun, 25 Sep 2011)
Log Message:
-----------
new option -assubst for replacing a old template by a new one substituting it. both options -subst and -remove has the same effect. (feature request bug #3150628, patch of bug #3163473 modified)
Modified Paths:
--------------
trunk/pywikipedia/template.py
Modified: trunk/pywikipedia/template.py
===================================================================
--- trunk/pywikipedia/template.py 2011-09-25 18:48:53 UTC (rev 9555)
+++ trunk/pywikipedia/template.py 2011-09-25 20:21:45 UTC (rev 9556)
@@ -5,9 +5,9 @@
Syntax: python template.py [-remove] [xml[:filename]] oldTemplate [newTemplate]
-Specify the template on the command line. The program will
-pick up the template page, and look for all pages using it. It will
-then automatically loop over them, and replace the template.
+Specify the template on the command line. The program will pick up the template
+page, and look for all pages using it. It will then automatically loop over
+them, and replace the template.
Command line options:
@@ -17,6 +17,11 @@
article. This is done by changing {{...}} or {{msg:...}} into
{{subst:...}}
+-assubst Replaces the first argument as old template with the second
+ argument as new template but substitutes it like -subst does.
+ Using both options -remove and -subst in the same command line has
+ the same effect.
+
-xml retrieve information from a local dump
(http://download.wikimedia.org). If this argument isn\'t given,
info will be loaded from the maintenance page of the live wiki.
@@ -250,12 +255,16 @@
r'(?P<parameters>\s*\|.+?|) *}}',
re.DOTALL)
- if self.remove:
- replacements.append((templateRegex, ''))
+ if self.subst and self.remove:
+ replacements.append((templateRegex,
+ '{{subst:%s\g<parameters>}}' % new))
+ exceptions['inside-tags']=['ref', 'gallery']
elif self.subst:
replacements.append((templateRegex,
'{{subst:%s\g<parameters>}}' % old))
exceptions['inside-tags']=['ref', 'gallery']
+ elif self.remove:
+ replacements.append((templateRegex, ''))
else:
replacements.append((templateRegex,
'{{%s\g<parameters>}}' % new))
@@ -287,6 +296,8 @@
remove = True
elif arg == '-subst':
subst = True
+ elif arg == '-assubst':
+ subst = remove = True
elif arg == ('-always'):
acceptAll = True
elif arg.startswith('-xml'):
@@ -312,7 +323,7 @@
pywikibot.Page(pywikibot.getSite(), arg,
defaultNamespace=10).titleWithoutNamespace())
- if subst or remove:
+ if subst ^ remove:
for templateName in templateNames:
templates[templateName] = None
else:
@@ -321,7 +332,7 @@
templates[templateNames[i]] = templateNames[i + 1]
except IndexError:
pywikibot.output(
-u'Unless using -subst or -remove, you must give an even number of template names.')
+u'Unless using solely -subst or -remove, you must give an even number of template names.')
return
oldTemplates = []
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/9555
Revision: 9555
Author: xqt
Date: 2011-09-25 18:48:53 +0000 (Sun, 25 Sep 2011)
Log Message:
-----------
use pagegenerators -namespace option
Modified Paths:
--------------
trunk/pywikipedia/template.py
Modified: trunk/pywikipedia/template.py
===================================================================
--- trunk/pywikipedia/template.py 2011-09-25 18:41:52 UTC (rev 9554)
+++ trunk/pywikipedia/template.py 2011-09-25 18:48:53 UTC (rev 9555)
@@ -22,9 +22,6 @@
info will be loaded from the maintenance page of the live wiki.
argument can also be given as "-xml:filename.xml".
--namespace: Only process templates in the given namespace number (may be used
- multiple times).
-
-user: Only process pages edited by a given user
-skipuser: Only process pages not edited by a given user
@@ -66,7 +63,9 @@
python template.py test -subst -namespace:2 -namespace:3
+Note that -namespace: is a global pywikipedia parameter
+
This next example substitutes the template lived with a supplied edit summary.
It only performs substitutions in main article namespace and doesn't prompt to
start replacing. Note that -putthrottle: is a global pywikipedia parameter.
@@ -296,11 +295,6 @@
u'Please enter the XML dump\'s filename: ')
else:
xmlfilename = arg[5:]
- elif arg.startswith('-namespace:'):
- try:
- namespaces.append(int(arg[len('-namespace:'):]))
- except ValueError:
- namespaces.append(arg[len('-namespace:'):])
elif arg.startswith('-category:'):
addedCat = arg[len('-category:'):]
elif arg.startswith('-summary:'):
@@ -347,8 +341,6 @@
gen = pagegenerators.CombinedPageGenerator(gens)
gen = pagegenerators.DuplicateFilterPageGenerator(gen)
- if namespaces:
- gen = pagegenerators.NamespaceFilterPageGenerator(gen, namespaces)
if user:
gen = UserEditFilterGenerator(gen, user, timestamp, skip)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/9554
Revision: 9554
Author: xqt
Date: 2011-09-25 18:41:52 +0000 (Sun, 25 Sep 2011)
Log Message:
-----------
-subst option is always needed for substituting a template
Modified Paths:
--------------
trunk/pywikipedia/template.py
Modified: trunk/pywikipedia/template.py
===================================================================
--- trunk/pywikipedia/template.py 2011-09-25 17:24:13 UTC (rev 9553)
+++ trunk/pywikipedia/template.py 2011-09-25 18:41:52 UTC (rev 9554)
@@ -64,18 +64,14 @@
If you have a template called [[Template:test]] and want to substitute it only
on pages in the User: and User talk: namespaces, do:
- python template.py test -namespace:2 -namespace:3
+ python template.py test -subst -namespace:2 -namespace:3
-Note that, on the English Wikipedia, User: is namespace 2 and User talk: is
-namespace 3. This may differ on other projects so make sure to find out the
-appropriate namespace numbers.
-
This next example substitutes the template lived with a supplied edit summary.
It only performs substitutions in main article namespace and doesn't prompt to
start replacing. Note that -putthrottle: is a global pywikipedia parameter.
- python template.py -putthrottle:30 -namespace:0 lived -always
+ python template.py -putthrottle:30 -namespace:0 lived -subst -always
-summary:"ROBOT: Substituting {{lived}}, see [[WP:SUBST]]."
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/9553
Revision: 9553
Author: xqt
Date: 2011-09-25 17:24:13 +0000 (Sun, 25 Sep 2011)
Log Message:
-----------
update from rewrite branch
Modified Paths:
--------------
trunk/pywikipedia/wikipedia.py
Modified: trunk/pywikipedia/wikipedia.py
===================================================================
--- trunk/pywikipedia/wikipedia.py 2011-09-25 17:20:50 UTC (rev 9552)
+++ trunk/pywikipedia/wikipedia.py 2011-09-25 17:24:13 UTC (rev 9553)
@@ -115,7 +115,7 @@
"""
from __future__ import generators
#
-# (C) Pywikipedia bot team, 2003-2010
+# (C) Pywikipedia bot team, 2003-2011
#
# Distributed under the terms of the MIT license.
#
@@ -7625,6 +7625,8 @@
default_code = site.language()
default_family = site.family
+# Command line parsing and help
+
def calledModuleName():
"""Return the name of the module calling this function.
@@ -7633,18 +7635,17 @@
"""
# get commandline arguments
- args = sys.argv
- try:
- # clip off the '.py' filename extension
- return os.path.basename(args[0][:args[0].rindex('.')])
- except ValueError:
- return os.path.basename(args[0])
+ called = sys.argv[0].strip()
+ if ".py" in called: # could end with .pyc, .pyw, etc. on some platforms
+ # clip off the '.py?' filename extension
+ called = called[:called.rindex('.py')]
+ return os.path.basename(called)
-def decodeArg(arg):
- if sys.platform=='win32':
- if config.console_encoding == 'cp850':
+def _decodeArg(arg):
+ if sys.platform == 'win32':
+ if config.console_encoding in ('cp437', 'cp850'):
# Western Windows versions give parameters encoded as windows-1252
- # even though the console encoding is cp850.
+ # even though the console encoding is cp850 or cp437.
return unicode(arg, 'windows-1252')
elif config.console_encoding == 'cp852':
# Central/Eastern European Windows versions give parameters encoded
@@ -7669,20 +7670,19 @@
"""
global default_code, default_family, verbose, debug
- # get commandline arguments
+ # get commandline arguments if necessary
if not args:
args = sys.argv[1:]
# get the name of the module calling this function. This is
# required because the -help option loads the module's docstring and because
# the module name will be used for the filename of the log.
- # TODO: check if the following line is platform-independent
moduleName = calledModuleName()
nonGlobalArgs = []
+ do_help = False
for arg in args:
- arg = decodeArg(arg)
+ arg = _decodeArg(arg)
if arg == '-help':
- showHelp(moduleName)
- sys.exit(0)
+ do_help = True
elif arg.startswith('-dir:'):
pass # config_dir = arg[5:] // currently handled in wikipediatools.py - possibly before this routine is called.
elif arg.startswith('-family:'):
@@ -7734,9 +7734,69 @@
""")
if verbose:
output(u'Pywikipediabot %s' % (version.getversion()))
- output(u'Python %s' % (sys.version))
+ output(u'Python %s' % sys.version)
+
+ if do_help:
+ showHelp()
+ sys.exit(0)
return nonGlobalArgs
+def showHelp(moduleName=None):
+ # the parameter moduleName is deprecated and should be left out.
+ moduleName = moduleName or calledModuleName()
+ try:
+ moduleName = moduleName[moduleName.rindex("\\")+1:]
+ except ValueError: # There was no \ in the module name, so presumably no problem
+ pass
+
+ globalHelp = u'''
+Global arguments available for all bots:
+
+-dir:PATH Read the bot's configuration data from directory given by
+ PATH, instead of from the default directory.
+
+-lang:xx Set the language of the wiki you want to work on, overriding
+ the configuration in user-config.py. xx should be the
+ language code.
+
+-family:xyz Set the family of the wiki you want to work on, e.g.
+ wikipedia, wiktionary, wikitravel, ...
+ This will override the configuration in user-config.py.
+
+-daemonize:xyz Immediately return control to the terminal and redirect
+ stdout and stderr to xyz (only use for bots that require
+ no input from stdin).
+
+-help Show this help text.
+
+-log Enable the logfile. Logs will be stored in the logs
+ subdirectory.
+
+-log:xyz Enable the logfile, using 'xyz' as the filename.
+
+-nolog Disable the logfile (if it is enabled by default).
+
+-putthrottle:n Set the minimum time (in seconds) the bot will wait between
+-pt:n saving pages.
+
+-verbose Have the bot provide additional output that may be
+-v useful in debugging.
+
+-cosmeticchanges Toggles the cosmetic_changes setting made in config.py or
+-cc user_config.py to its inverse and overrules it. All other
+ settings and restrictions are untouched.
+'''# % moduleName
+ output(globalHelp, toStdout=True)
+ try:
+ exec('import %s as module' % moduleName)
+ helpText = module.__doc__.decode('utf-8')
+ if hasattr(module, 'docuReplacements'):
+ for key, value in module.docuReplacements.iteritems():
+ helpText = helpText.replace(key, value.strip('\n\r'))
+ output(helpText, toStdout=True)
+ except:
+ output(u'Sorry, no help available for %s' % moduleName)
+
#########################
# Interpret configuration
#########################
@@ -7777,7 +7837,7 @@
"""
modname = os.path.basename(sys.argv[0])
# put quotation marks around all parameters
- args = [decodeArg(modname)] + [decodeArg('"%s"' % s) for s in sys.argv[1:]]
+ args = [_decodeArg(modname)] + [_decodeArg('"%s"' % s) for s in sys.argv[1:]]
commandLogFilename = config.datafilepath('logs', 'commands.log')
try:
commandLogFile = codecs.open(commandLogFilename, 'a', 'utf-8')
@@ -7825,7 +7885,8 @@
output_lock = threading.Lock()
input_lock = threading.Lock()
output_cache = []
-def output(text, decoder = None, newline = True, toStdout = False):
+
+def output(text, decoder=None, newline=True, toStdout=False, **kwargs):
"""Output a message to the user via the userinterface.
Works like print, but uses the encoding used by the user's console
@@ -7923,63 +7984,7 @@
return data
-def showHelp(moduleName = None):
- # the parameter moduleName is deprecated and should be left out.
- moduleName = moduleName or sys.argv[0][:sys.argv[0].rindex('.')]
- try:
- moduleName = moduleName[moduleName.rindex("\\")+1:]
- except ValueError: # There was no \ in the module name, so presumably no problem
- pass
- globalHelp =u'''
-Global arguments available for all bots:
-
--dir:PATH Read the bot's configuration data from directory given by
- PATH, instead of from the default directory.
-
--lang:xx Set the language of the wiki you want to work on, overriding
- the configuration in user-config.py. xx should be the
- language code.
-
--family:xyz Set the family of the wiki you want to work on, e.g.
- wikipedia, wiktionary, wikitravel, ...
- This will override the configuration in user-config.py.
-
--daemonize:xyz Immediately returns control to the terminal and redirects
- stdout and stderr to xyz (only use for bots that require
- no input from stdin).
-
--help Shows this help text.
-
--log Enable the logfile. Logs will be stored in the logs
- subdirectory.
-
--log:xyz Enable the logfile, using xyz as the filename.
-
--nolog Disable the logfile (if it is enabled by default).
-
--putthrottle:n Set the minimum time (in seconds) the bot will wait between
--pt:n saving pages.
-
--verbose Have the bot provide additional output that may be useful in
--v debugging.
-
--cosmeticchanges Toggles the cosmetic_changes setting made in config.py or
--cc user_config.py to its inverse and overrules it. All other
- settings and restrictions are untouched.
-'''
- output(globalHelp, toStdout = True)
- try:
- exec('import %s as module' % moduleName)
- helpText = module.__doc__.decode('utf-8')
- if hasattr(module, 'docuReplacements'):
- for key, value in module.docuReplacements.iteritems():
- helpText = helpText.replace(key, value.strip('\n\r'))
- output(helpText, toStdout = True)
- except:
- raise
- output(u'Sorry, no help available for %s' % moduleName)
-
page_put_queue = Queue.Queue(config.max_queue_size)
def async_put():
"""Daemon; take pages from the queue and try to save them on the wiki."""