Skip to content

Commit

Permalink
expand retryer to include image reading (#236)
Browse files Browse the repository at this point in the history
* expand retryer to include image reading

* update tests
  • Loading branch information
martinfleis committed Mar 4, 2024
1 parent 192e6de commit afc96d6
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 13 deletions.
25 changes: 14 additions & 11 deletions contextily/tile.py
Expand Up @@ -14,7 +14,7 @@

import numpy as np
import rasterio as rio
from PIL import Image
from PIL import Image, UnidentifiedImageError
from joblib import Memory as _Memory
from joblib import Parallel, delayed
from rasterio.transform import from_origin
Expand Down Expand Up @@ -288,11 +288,7 @@ def _process_source(source):


def _fetch_tile(tile_url, wait, max_retries):
request = _retryer(tile_url, wait, max_retries)
with io.BytesIO(request.content) as image_stream:
image = Image.open(image_stream).convert("RGBA")
array = np.asarray(image)
image.close()
array = _retryer(tile_url, wait, max_retries)
return array


Expand Down Expand Up @@ -412,7 +408,7 @@ def _warper(img, transform, s_crs, t_crs, resampling):

def _retryer(tile_url, wait, max_retries):
"""
Retry a url many times in attempt to get a tile
Retry a url many times in attempt to get a tile and read the image
Arguments
---------
Expand All @@ -428,25 +424,32 @@ def _retryer(tile_url, wait, max_retries):
Returns
-------
request object containing the web response.
array of the tile
"""
try:
request = requests.get(tile_url, headers={"user-agent": USER_AGENT})
request.raise_for_status()
except requests.HTTPError:
with io.BytesIO(request.content) as image_stream:
image = Image.open(image_stream).convert("RGBA")
array = np.asarray(image)
image.close()

return array

except (requests.HTTPError, UnidentifiedImageError):
if request.status_code == 404:
raise requests.HTTPError(
"Tile URL resulted in a 404 error. "
"Double-check your tile url:\n{}".format(tile_url)
)
elif request.status_code == 104:
elif request.status_code == 104 or request.status_code == 200:
if max_retries > 0:
os.wait(wait)
max_retries -= 1
request = _retryer(tile_url, wait, max_retries)
else:
raise requests.HTTPError("Connection reset by peer too many times.")
return request



def howmany(w, s, e, n, zoom, verbose=True, ll=False):
Expand Down
4 changes: 2 additions & 2 deletions tests/test_cx.py
Expand Up @@ -127,7 +127,7 @@ def test_warp_tiles():
)
assert wimg[100, 100, :].tolist() == [247, 246, 241, 255]
assert wimg[100, 200, :].tolist() == [246, 246, 241, 255]
assert wimg[20, 120, :].tolist() == [139, 128, 148, 255]
assert wimg[20, 120, :].tolist() == [139, 128, 149, 255]


@pytest.mark.network
Expand All @@ -144,7 +144,7 @@ def test_warp_img_transform():
wimg, _ = cx.warp_img_transform(img, rtr.transform, rtr.crs, "epsg:4326")
assert wimg[:, 100, 100].tolist() == [247, 246, 241, 255]
assert wimg[:, 100, 200].tolist() == [246, 246, 241, 255]
assert wimg[:, 20, 120].tolist() == [139, 128, 148, 255]
assert wimg[:, 20, 120].tolist() == [139, 128, 149, 255]


def test_howmany():
Expand Down

0 comments on commit afc96d6

Please sign in to comment.