From 11fb32bff6f3e29d0d5310533d81c9ba34b6572e Mon Sep 17 00:00:00 2001 From: miruka Date: Sat, 18 Jul 2020 22:26:23 -0400 Subject: [PATCH] Retry downloading media if they fail with 404/500+ Trying to fetch a media right after it was uploaded can sometimes lead to a 404 --- src/backend/media_cache.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/backend/media_cache.py b/src/backend/media_cache.py index e2e49c23..c33c961f 100644 --- a/src/backend/media_cache.py +++ b/src/backend/media_cache.py @@ -12,6 +12,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Optional from urllib.parse import urlparse +from .errors import MatrixError from PIL import Image as PILImage @@ -133,8 +134,20 @@ class Media: async def create(self) -> Path: """Download and cache the media file to disk.""" - async with CONCURRENT_DOWNLOADS_LIMIT: - data = await self._get_remote_data() + retries = 0 + + while True: + try: + async with CONCURRENT_DOWNLOADS_LIMIT: + data = await self._get_remote_data() + except MatrixError as err: + if err.http_code != 404 and err.http_code < 500: + raise + else: + break + + await asyncio.sleep(min(30, 0.2 * (2 ** (min(1000, retries) - 1)))) + retries += 1 self.local_path.parent.mkdir(parents=True, exist_ok=True)