jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/351758 )
Change subject: weblinkchecker.py: Retry get_memento_info if there is a ConnectionError
......................................................................
weblinkchecker.py: Retry get_memento_info if there is a ConnectionError
`get_memento_info` sometimes fails to connect to the server. Retrying
should mitigate the issue.
Skip the related test on ConnectionError. `max_retries` is reduced to 1 during
tests and thus the test might still fail.
Bug: T164365
Change-Id: I0db18b7a3e5fa67de4132f73c58a15b76725edc3
---
M scripts/weblinkchecker.py
M tests/weblinkchecker_tests.py
2 files changed, 23 insertions(+), 6 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py
index 9fd0274..e261164 100755
--- a/scripts/weblinkchecker.py
+++ b/scripts/weblinkchecker.py
@@ -113,6 +113,7 @@
import time
from functools import partial
+from time import sleep
from warnings import warn
try:
@@ -122,7 +123,9 @@
import pywikibot
-from pywikibot import comms, i18n, config, pagegenerators, textlib, weblib
+from pywikibot import (
+ comms, i18n, config, pagegenerators, textlib, weblib, config2,
+)
from pywikibot.bot import ExistingPageBot, SingleSiteBot
from pywikibot.pagegenerators import (
@@ -188,7 +191,18 @@
if timegate_uri:
mc.timegate_uri = timegate_uri
- memento_info = mc.get_memento_info(url, when)
+ retry_count = 0
+ while retry_count <= config2.max_retries:
+ try:
+ memento_info = mc.get_memento_info(url, when)
+ break
+ except requests.ConnectionError as e:
+ error = e
+ retry_count += 1
+ sleep(config2.retry_wait)
+ else:
+ raise error
+
mementos = memento_info.get('mementos')
if not mementos:
raise Exception(
diff --git a/tests/weblinkchecker_tests.py b/tests/weblinkchecker_tests.py
index be84bab..63171ac 100644
--- a/tests/weblinkchecker_tests.py
+++ b/tests/weblinkchecker_tests.py
@@ -9,6 +9,8 @@
import datetime
+from requests import ConnectionError as RequestsConnectionError
+
from pywikibot.tools import PY2
if not PY2:
from urllib.parse import urlparse
@@ -31,10 +33,11 @@
when = datetime.datetime.now()
else:
when = datetime.datetime.strptime(date_string, '%Y%m%d')
- return weblinkchecker._get_closest_memento_url(
- url,
- when,
- self.timegate_uri)
+ try:
+ return weblinkchecker._get_closest_memento_url(
+ url, when, self.timegate_uri)
+ except RequestsConnectionError as e:
+ self.skipTest(e)
class WeblibTestMementoInternetArchive(MementoTestCase,
weblib_tests.TestInternetArchive):
--
To view, visit
https://gerrit.wikimedia.org/r/351758
To unsubscribe, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0db18b7a3e5fa67de4132f73c58a15b76725edc3
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Dalba <dalba.wiki(a)gmail.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>