Revision: 5315
Author: btongminh
Date: 2008-05-06 10:38:43 +0000 (Tue, 06 May 2008)
Log Message:
-----------
Fix starvation in case we don't have any databases
Modified Paths:
--------------
trunk/pywikipedia/commonsdelinker/checkusage.py
Modified: trunk/pywikipedia/commonsdelinker/checkusage.py
===================================================================
--- trunk/pywikipedia/commonsdelinker/checkusage.py 2008-05-06 10:37:01 UTC (rev 5314)
+++ trunk/pywikipedia/commonsdelinker/checkusage.py 2008-05-06 10:38:43 UTC (rev 5315)
@@ -374,11 +374,13 @@
def close(self):
+ if getattr(self, 'http'):
+ self.http.close()
+ if not hasattr(self, 'databases'): return
for connection, cursor in self.databases.itervalues():
try:
connection.close()
except:
pass
- if self.http:
- self.http.close()
+
\ No newline at end of file
Feature Requests item #1930465, was opened at 2008-03-31 21:23
Message generated for change (Comment added) made by alleborgo
You can respond by visiting:
https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1930465&group_…
Please note that this message will contain a full copy of the comment thread,
including the initial issue submission, for this request,
not just the latest update.
Category: None
Group: None
>Status: Closed
Priority: 5
Private: No
Submitted By: Nobody/Anonymous (nobody)
>Assigned to: Alleborgo (alleborgo)
Summary: Some translation
Initial Comment:
New indonesian (id:) disambiguation:
{{disingkat}}
cs summary for replace.py:
'cs':u'Robot automaticky nahradil text: %s',
----------------------------------------------------------------------
>Comment By: Alleborgo (alleborgo)
Date: 2008-05-06 02:21
Message:
Logged In: YES
user_id=1898409
Originator: NO
Committed in r5312.
Thank you very much!
----------------------------------------------------------------------
You can respond by visiting:
https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1930465&group_…
Revision: 5310
Author: russblau
Date: 2008-05-05 21:47:18 +0000 (Mon, 05 May 2008)
Log Message:
-----------
debugging
Modified Paths:
--------------
branches/rewrite/pywikibot/login.py
branches/rewrite/pywikibot/site.py
branches/rewrite/pywikibot/throttle.py
Modified: branches/rewrite/pywikibot/login.py
===================================================================
--- branches/rewrite/pywikibot/login.py 2008-05-05 21:46:16 UTC (rev 5309)
+++ branches/rewrite/pywikibot/login.py 2008-05-05 21:47:18 UTC (rev 5310)
@@ -44,6 +44,7 @@
#
__version__='$Id$'
+import logging
import re
import urllib2
import config
Modified: branches/rewrite/pywikibot/site.py
===================================================================
--- branches/rewrite/pywikibot/site.py 2008-05-05 21:46:16 UTC (rev 5309)
+++ branches/rewrite/pywikibot/site.py 2008-05-05 21:47:18 UTC (rev 5310)
@@ -578,6 +578,9 @@
def preloadpages(self, pagelist, groupsize=60):
"""Return a generator to a list of preloaded pages.
+ Note that [at least in current implementation] pages may be iterated
+ in a different order than in the underlying pagelist.
+
@param pagelist: an iterable that returns Page objects
@param groupsize: how many Pages to query at a time
@type groupsize: int
@@ -585,7 +588,9 @@
"""
from pywikibot.tools import itergroup
for sublist in itergroup(pagelist, groupsize):
- pageids = [str(p._pageid) for p in sublist if hasattr(p, "_pageid")]
+ pageids = [str(p._pageid) for p in sublist
+ if hasattr(p, "_pageid")
+ and p._pageid > 0]
cache = dict((p.title(withSection=False), p) for p in sublist)
rvgen = api.PropertyGenerator("revisions|info")
if len(pageids) == len(sublist):
@@ -595,12 +600,21 @@
rvgen.request["titles"] = "|".join(cache.keys())
rvgen.request[u"rvprop"] = \
u"ids|flags|timestamp|user|comment|content"
+ logging.info(u"Retrieving %s pages from %s."
+ % (len(cache), self)
+ )
for pagedata in rvgen:
- if pagedata['title'] not in cache:
- raise Error(
+ try:
+ if pagedata['title'] not in cache:
+ raise Error(
u"preloadpages: Query returned unexpected title '%s'"
- % pagedata['title']
- )
+ % pagedata['title']
+ )
+ except KeyError:
+ logging.debug("No 'title' in %s" % pagedata)
+ logging.debug("pageids=%s" % pageids)
+ logging.debug("titles=%s" % cache.keys())
+ continue
page = cache[pagedata['title']]
api.update_page(page, pagedata)
if 'revisions' in pagedata: # true if page exists
Modified: branches/rewrite/pywikibot/throttle.py
===================================================================
--- branches/rewrite/pywikibot/throttle.py 2008-05-05 21:46:16 UTC (rev 5309)
+++ branches/rewrite/pywikibot/throttle.py 2008-05-05 21:47:18 UTC (rev 5310)
@@ -245,6 +245,12 @@
# account for any time we waited while acquiring the lock
wait = delay - (time.time() - started)
if wait > 0:
+ if wait > config.noisysleep:
+ logging.warn(u"Sleeping for %.1f seconds, %s"
+ % (wait,
+ time.strftime("%Y-%m-%d %H:%M:%S",
+ time.localtime()))
+ )
time.sleep(wait)
finally:
self.lock.release()
Revision: 5309
Author: russblau
Date: 2008-05-05 21:46:16 +0000 (Mon, 05 May 2008)
Log Message:
-----------
Change default user file location; WARNING to testers - this may require you to relocate files. Either use the PYWIKIBOT2_DIR environ variable or create a ~/.pywikbot directory and copy your user-config.py into it.
Modified Paths:
--------------
branches/rewrite/pywikibot/config.py
Modified: branches/rewrite/pywikibot/config.py
===================================================================
--- branches/rewrite/pywikibot/config.py 2008-05-05 20:57:46 UTC (rev 5308)
+++ branches/rewrite/pywikibot/config.py 2008-05-05 21:46:16 UTC (rev 5309)
@@ -79,32 +79,31 @@
This is determined in the following order -
1. If the script was called with a -dir: argument, use the directory
provided in this argument
- 2. If the user has a PYWIKIBOT_DIR environment variable, use the value
+ 2. If the user has a PYWIKIBOT2_DIR environment variable, use the value
of it
- 3. If the script was started from a directory that contains a
- user-config.py file, use this directory as the base
- 4. If all else fails, use the directory from which this module was
- loaded.
+ 3. Use (and if necessary create) a 'pywikibot' folder (Windows) or
+ '.pywikibot' directory (Unix and similar) under the user's home
+ directory.
"""
+ NAME = "pywikibot"
for arg in __sys.argv[1:]:
if arg.startswith("-dir:"):
base_dir = arg[5:]
__sys.argv.remove(arg)
break
else:
- if os.environ.has_key("PYWIKIBOT_DIR"):
- base_dir = os.environ["PYWIKIBOT_DIR"]
+ if os.environ.has_key("PYWIKIBOT2_DIR"):
+ base_dir = os.environ["PYWIKIBOT2_DIR"]
else:
- if os.path.exists('user-config.py'):
- base_dir = '.'
- else:
- try:
- base_dir = os.path.split(
- __sys.modules['wikipediatools'].__file__)[0]
- except KeyError:
- print __sys.modules
- base_dir = '.'
+ is_windows = __sys.platform == 'win32'
+ home = os.path.expanduser("~")
+ if is_windows:
+ base_dir = os.path.join(home, "Application Data", NAME)
+ else:
+ base_dir = os.path.join(home, "."+NAME)
+ if not os.path.isdir(base_dir):
+ os.makedirs(base_dir, mode=0700)
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
# make sure this path is valid and that it contains user-config file
@@ -116,11 +115,11 @@
return base_dir
_base_dir = _get_base_dir()
-_RfamilyFile = re.compile('(?P<name>.+)_family.py$')
-for _filename in os.listdir(os.path.join(_base_dir, 'families')):
- _m = _RfamilyFile.match(_filename)
- if _m:
- familyName = _m.group('name')
+# families/ is a subdirectory of the directory in which config.py is found
+for _filename in os.listdir(
+ os.path.join(os.path.dirname(__file__), 'families')):
+ if _filename.endswith("_family.py"):
+ familyName = _filename[ : -len("_family.py")]
usernames[familyName] = {}
sysopnames[familyName] = {}
disambiguation_comment[familyName] = {}
@@ -526,13 +525,12 @@
"""Return an absolute path to a data file in a standard location.
Argument(s) are zero or more directory names, optionally followed by a
- data file name. The return path is offset to the "data" subdirectory of
- config.base_dir. Any directories in the path that do not already exist
- are created.
+ data file name. The return path is offset to config.base_dir. Any
+ directories in the path that do not already exist are created.
"""
import os
- return makepath(os.path.join(os.path.join(base_dir, "data"), *filename))
+ return makepath(os.path.join(base_dir, *filename))
def shortpath(path):
"""Return a file path relative to config.base_dir."""
Revision: 5307
Author: russblau
Date: 2008-05-05 18:29:26 +0000 (Mon, 05 May 2008)
Log Message:
-----------
Method name changes (get... loads and saves info from wiki; page... iterates information related to a particular page)
Modified Paths:
--------------
branches/rewrite/pywikibot/page.py
branches/rewrite/pywikibot/site.py
Modified: branches/rewrite/pywikibot/page.py
===================================================================
--- branches/rewrite/pywikibot/page.py 2008-05-05 14:32:15 UTC (rev 5306)
+++ branches/rewrite/pywikibot/page.py 2008-05-05 18:29:26 UTC (rev 5307)
@@ -447,7 +447,7 @@
# to implement those methods in the site interface and then combine
# the results for this method, or to implement this method and then
# split up the results for the others.
- return self.site().getreferences(
+ return self.site().pagereferences(
self, follow_redirects, redirectsOnly,
withTemplateInclusion, onlyTemplateInclusion)
@@ -460,11 +460,11 @@
omit redirects; if None, do not filter
"""
- return self.site().getbacklinks(self, followRedirects, filterRedirects)
+ return self.site().pagebacklinks(self, followRedirects, filterRedirects)
def embeddedin(self):
"""Yield all pages that embed this page as a template."""
- return self.site().getembeddedin(self)
+ return self.site().page_embeddedin(self)
def canBeEdited(self):
"""Return bool indicating whether this page can be edited.
@@ -574,7 +574,7 @@
@return: a generator that yields Page objects.
"""
- return self.site().getlinks(self)
+ return self.site().pagelinks(self)
def interwiki(self):
"""Iterate interwiki links in the page text.
@@ -582,7 +582,7 @@
@return: a generator that yields Link objects.
"""
- return self.site().getinterwiki(self)
+ return self.site().pageinterwiki(self)
def langlinks(self):
"""Iterate all interlanguage links on this page.
@@ -593,7 +593,7 @@
@return: a generator that yields Link objects.
"""
- return self.site().getlanglinks(self)
+ return self.site().pagelanglinks(self)
def imagelinks(self, followRedirects=None, loose=None):
"""Iterate ImagePage objects for images displayed on this Page.
@@ -609,7 +609,7 @@
if loose is not None:
logging.debug(
u"Page.imagelinks(loose) option is deprecated.")
- return self.site().getimages(self)
+ return self.site().pageimages(self)
def templates(self):
"""Iterate Page objects for templates used on this Page.
@@ -619,7 +619,7 @@
a normal link.
"""
- return self.site().gettemplates(self)
+ return self.site().pagetemplates(self)
def templatesWithParams(self):
"""Iterate templates used on this Page.
@@ -644,7 +644,7 @@
if nofollow_redirects is not None:
logging.debug(
u"Page.categories(nofollow_redirects) option is deprecated.")
- return self.site().getcategories(self, withSortKey=withSortKey)
+ return self.site().pagecategories(self, withSortKey=withSortKey)
def extlinks(self):
"""Iterate all external URLs (not interwiki links) from this page.
@@ -652,7 +652,7 @@
@return: a generator that yields unicode objects containing URLs.
"""
- return self.site().getextlinks(self)
+ return self.site().page_extlinks(self)
def getRedirectTarget(self):
"""Return a Page object for the target this Page redirects to.
@@ -664,7 +664,7 @@
if not self.isRedirectPage():
raise pywikibot.IsNotRedirectPage
if not isinstance(self._redir, Page):
- self.site().getredirtarget(self)
+ self.site().pageredirtarget(self)
return self._redir
def getVersionHistory(self, forceReload=False, reverseOrder=False,
@@ -1100,7 +1100,7 @@
recurse = recurse - 1
if not hasattr(self, "_subcats"):
self._subcats = []
- for member in self.site().getcategorymembers(self, namespaces=[14]):
+ for member in self.site().pagecategorymembers(self, namespaces=[14]):
subcat = Category(self.site(), member.title())
self._subcats.append(subcat)
yield subcat
@@ -1127,7 +1127,7 @@
"""
namespaces = [x for x in self.site().namespaces().keys()
if x>=0 and x!=14]
- for member in self.site().getcategorymembers(self,
+ for member in self.site().pagecategorymembers(self,
namespaces=namespaces):
yield member
if recurse:
Modified: branches/rewrite/pywikibot/site.py
===================================================================
--- branches/rewrite/pywikibot/site.py 2008-05-05 14:32:15 UTC (rev 5306)
+++ branches/rewrite/pywikibot/site.py 2008-05-05 18:29:26 UTC (rev 5307)
@@ -540,7 +540,7 @@
self.getpageinfo(page)
return bool(page._redir)
- def getredirtarget(self, page):
+ def pageredirtarget(self, page):
"""Return Page object for the redirect target of page."""
if not hasattr(page, "_redir"):
self.getpageinfo(page)
@@ -554,13 +554,13 @@
result = query.submit()
if "query" not in result or "redirects" not in result["query"]:
raise RuntimeError(
- "getredirtarget: No 'redirects' found for page %s."
+ "pageredirtarget: No 'redirects' found for page %s."
% title)
redirmap = dict((item['from'], item['to'])
for item in result['query']['redirects'])
if title not in redirmap:
raise RuntimeError(
- "getredirtarget: 'redirects' contains no key for page %s."
+ "pageredirtarget: 'redirects' contains no key for page %s."
% title)
if "pages" not in result['query']:
# no "pages" element indicates a circular redirect
@@ -569,7 +569,7 @@
# there should be only one value in 'pages', and it is the target
if pagedata['title'] not in redirmap.values():
raise RuntimeError(
- "getredirtarget: target page '%s' not found in 'redirects'"
+ "pageredirtarget: target page '%s' not found in 'redirects'"
% pagedata['title'])
target = pywikibot.Page(self, pagedata['title'], pagedata['ns'])
api.update_page(target, pagedata)
@@ -620,7 +620,7 @@
# following group of methods map more-or-less directly to API queries
- def getbacklinks(self, page, followRedirects=False, filterRedirects=None,
+ def pagebacklinks(self, page, followRedirects=False, filterRedirects=None,
namespaces=None):
"""Iterate all pages that link to the given page.
@@ -646,7 +646,7 @@
blgen.request["gblredirect"] = ""
return blgen
- def getembeddedin(self, page, filterRedirects=None, namespaces=None):
+ def page_embeddedin(self, page, filterRedirects=None, namespaces=None):
"""Iterate all pages that embedded the given page as a template.
@param page: The Page to get inclusions for.
@@ -667,20 +667,20 @@
or "nonredirects"
return eigen
- def getreferences(self, page, followRedirects, filterRedirects,
+ def pagereferences(self, page, followRedirects, filterRedirects,
withTemplateInclusion, onlyTemplateInclusion):
- """Convenience method combining getbacklinks and getembeddedin."""
+ """Convenience method combining pagebacklinks and page_embeddedin."""
if onlyTemplateInclusion:
- return self.getembeddedin(page)
+ return self.page_embeddedin(page)
if not withTemplateInclusion:
- return self.getbacklinks(page, follow_redirects)
+ return self.pagebacklinks(page, follow_redirects)
import itertools
- return itertools.chain(self.getbacklinks(
+ return itertools.chain(self.pagebacklinks(
page, followRedirects, filterRedirects),
- self.getembeddedin(page, filterRedirects)
+ self.page_embeddedin(page, filterRedirects)
)
- def getlinks(self, page, namespaces=None):
+ def pagelinks(self, page, namespaces=None):
"""Iterate internal wikilinks contained (or transcluded) on page."""
plgen = api.PageGenerator("links")
if hasattr(page, "_pageid"):
@@ -693,7 +693,7 @@
for ns in namespaces)
return plgen
- def getcategories(self, page, withSortKey=False):
+ def pagecategories(self, page, withSortKey=False):
"""Iterate categories to which page belongs."""
# Sortkey doesn't work with generator; FIXME or deprecate
clgen = api.CategoryPageGenerator("categories")
@@ -704,13 +704,13 @@
clgen.request['titles'] = cltitle
return clgen
- def getimages(self, page):
+ def pageimages(self, page):
"""Iterate images used (not just linked) on the page."""
imtitle = page.title(withSection=False).encode(self.encoding())
imgen = api.ImagePageGenerator("images", titles=imtitle)
return imgen
- def gettemplates(self, page, namespaces=None):
+ def pagetemplates(self, page, namespaces=None):
"""Iterate templates transcluded (not just linked) on the page."""
tltitle = page.title(withSection=False).encode(self.encoding())
tlgen = api.PageGenerator("templates", titles=tltitle)
@@ -719,7 +719,7 @@
for ns in namespaces)
return tlgen
- def getcategorymembers(self, category, namespaces=None):
+ def pagecategorymembers(self, category, namespaces=None):
"""Iterate members of specified category.
@param category: The Category to iterate.
@@ -873,11 +873,11 @@
if latest:
page._revid = revision.revid
- def getinterwiki(self, page):
+ def pageinterwiki(self, page):
# TODO
raise NotImplementedError
- def getlanglinks(self, page):
+ def pagelanglinks(self, page):
"""Iterate all interlanguage links on page, yielding Link objects."""
lltitle = page.title(withSection=False)
llquery = api.PropertyGenerator("langlinks",
@@ -894,7 +894,7 @@
yield pywikibot.Link(linkdata['*'],
source=pywikibot.Site(linkdata['lang']))
- def getextlinks(self, page):
+ def page_extlinks(self, page):
"""Iterate all external links on page, yielding URL strings."""
eltitle = page.title(withSection=False)
elquery = api.PropertyGenerator("extlinks",