sergey/hosts
1
0
Fork 0
mirror of https://github.com/StevenBlack/hosts.git synced 2026-07-01 02:36:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Retry to get file from URL once failed

Browse Source 
(Resolves #1038)
This commit is contained in:
Max G
2020-07-11 14:09:51 +03:00
parent 0a96dcad34
commit f26168194b
2 changed files with 16 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
testUpdateHostsFile.py
View File

@@ -1617,14 +1617,14 @@ class GetFileByUrl(BaseStdout):
url = b"www.google.com" url = b"www.google.com"
expected = "www.google.com" expected = "www.google.com"
actual = get_file_by_url(url) actual = get_file_by_url(url, delay=0)
self.assertEqual(actual, expected) self.assertEqual(actual, expected)
@mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_fail) @mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_fail)
def test_read_url_fail(self, _): def test_read_url_fail(self, _):
url = b"www.google.com" url = b"www.google.com"
self.assertIsNone(get_file_by_url(url)) self.assertIsNone(get_file_by_url(url, delay=0))
expected = "Problem getting file:" expected = "Problem getting file:"
output = sys.stdout.getvalue() output = sys.stdout.getvalue()
@@ -1634,7 +1634,7 @@ class GetFileByUrl(BaseStdout):
@mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_read_fail) @mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_read_fail)
def test_read_url_read_fail(self, _): def test_read_url_read_fail(self, _):
url = b"www.google.com" url = b"www.google.com"
self.assertIsNone(get_file_by_url(url)) self.assertIsNone(get_file_by_url(url, delay=0))
expected = "Problem getting file:" expected = "Problem getting file:"
output = sys.stdout.getvalue() output = sys.stdout.getvalue()
@@ -1644,7 +1644,7 @@ class GetFileByUrl(BaseStdout):
@mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_decode_fail) @mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_decode_fail)
def test_read_url_decode_fail(self, _): def test_read_url_decode_fail(self, _):
url = b"www.google.com" url = b"www.google.com"
self.assertIsNone(get_file_by_url(url)) self.assertIsNone(get_file_by_url(url, delay=0))
expected = "Problem getting file:" expected = "Problem getting file:"
output = sys.stdout.getvalue() output = sys.stdout.getvalue()

19
updateHostsFile.py
View File

@@ -1461,7 +1461,7 @@ def maybe_copy_example_file(file_path):
shutil.copyfile(example_file_path, file_path) shutil.copyfile(example_file_path, file_path)
def get_file_by_url(url): def get_file_by_url(url, retries=3, delay=10):
""" """
Get a file data located at a particular URL. Get a file data located at a particular URL.
@@ -1482,12 +1482,17 @@ def get_file_by_url(url):
format we have to encode or decode data before parsing it to UTF-8. format we have to encode or decode data before parsing it to UTF-8.
""" """
try: while retries:
with urlopen(url) as f: try:
soup = BeautifulSoup(f.read(), "lxml").get_text() with urlopen(url) as f:
return "\n".join(list(map(domain_to_idna, soup.split("\n")))) soup = BeautifulSoup(f.read(), "lxml").get_text()
except Exception: return "\n".join(list(map(domain_to_idna, soup.split("\n"))))
print("Problem getting file: ", url) except Exception as e:
if 'failure in name resolution' in str(e):
print('No internet connection! Retrying in {} seconds'.format(delay))
time.sleep(delay)
retries -= 1
return print("Problem getting file: ", url)
def write_data(f, data): def write_data(f, data):