Retry downloading media if they fail with 404/500+

Trying to fetch a media right after it was uploaded can sometimes lead
to a  404
This commit is contained in:
miruka 2020-07-18 22:26:23 -04:00
parent 46c763fdfd
commit 11fb32bff6

View File

@ -12,6 +12,7 @@ from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Optional from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Optional
from urllib.parse import urlparse from urllib.parse import urlparse
from .errors import MatrixError
from PIL import Image as PILImage from PIL import Image as PILImage
@ -133,8 +134,20 @@ class Media:
async def create(self) -> Path: async def create(self) -> Path:
"""Download and cache the media file to disk.""" """Download and cache the media file to disk."""
retries = 0
while True:
try:
async with CONCURRENT_DOWNLOADS_LIMIT: async with CONCURRENT_DOWNLOADS_LIMIT:
data = await self._get_remote_data() data = await self._get_remote_data()
except MatrixError as err:
if err.http_code != 404 and err.http_code < 500:
raise
else:
break
await asyncio.sleep(min(30, 0.2 * (2 ** (min(1000, retries) - 1))))
retries += 1
self.local_path.parent.mkdir(parents=True, exist_ok=True) self.local_path.parent.mkdir(parents=True, exist_ok=True)