diff --git a/src/backend/backend.py b/src/backend/backend.py index df9ce73a..2f123325 100644 --- a/src/backend/backend.py +++ b/src/backend/backend.py @@ -68,8 +68,8 @@ class Backend: - `("", "rooms")`: rooms our account `user_id` is part of; - - `("", "uploads")`: ongoing or failed file uploads for - our account `user_id`; + - `("", "transfers")`: ongoing or failed file + uploads/downloads for our account `user_id`; - `("", "", "members")`: members in the room `room_id` that our account `user_id` is part of; @@ -322,7 +322,7 @@ class Backend: self.models["accounts"].pop(user_id, None) self.models["matching_accounts"].pop(user_id, None) - self.models[user_id, "uploads"].clear() + self.models[user_id, "transfers"].clear() for room_id in self.models[user_id, "rooms"]: self.models["all_rooms"].pop(room_id, None) diff --git a/src/backend/errors.py b/src/backend/errors.py index 1e75b66e..60b2fa6a 100644 --- a/src/backend/errors.py +++ b/src/backend/errors.py @@ -90,6 +90,11 @@ class MatrixTooLarge(MatrixError): m_code: str = "M_TOO_LARGE" +@dataclass +class MatrixBadGateway(MatrixError): + http_code: int = 502 + m_code: Optional[str] = None + @dataclass class MatrixBadGateway(MatrixError): http_code: int = 502 diff --git a/src/backend/matrix_client.py b/src/backend/matrix_client.py index 9b1d55c5..59e1c39b 100644 --- a/src/backend/matrix_client.py +++ b/src/backend/matrix_client.py @@ -9,7 +9,6 @@ import io import logging as log import platform import re -import sys import textwrap import traceback from contextlib import suppress @@ -42,8 +41,8 @@ from .errors import ( from .html_markdown import HTML_PROCESSOR as HTML from .media_cache import Media, Thumbnail from .models.items import ( - ZERO_DATE, Account, Event, Member, Room, TypeSpecifier, Upload, - UploadStatus, + ZERO_DATE, Account, Event, Member, Room, Transfer, TransferStatus, + TypeSpecifier, ) from .models.model_store import ModelStore from .nio_callbacks import NioCallbacks @@ -55,11 +54,6 @@ from .pyotherside_events import ( if TYPE_CHECKING: from .backend import Backend -if sys.version_info >= (3, 7): - current_task = asyncio.current_task -else: - current_task = asyncio.Task.current_task - CryptDict = Dict[str, Any] PathCallable = Union[ str, Path, Callable[[], Coroutine[None, None, Union[str, Path]]], @@ -190,8 +184,8 @@ class MatrixClient(nio.AsyncClient): self.sync_task: Optional[asyncio.Future] = None self.start_task: Optional[asyncio.Future] = None - self.upload_monitors: Dict[UUID, nio.TransferMonitor] = {} - self.upload_tasks: Dict[UUID, asyncio.Task] = {} + self.transfer_monitors: Dict[UUID, nio.TransferMonitor] = {} + self.transfer_tasks: Dict[UUID, asyncio.Task] = {} self.send_message_tasks: Dict[UUID, asyncio.Task] = {} self._presence: str = "" @@ -628,23 +622,23 @@ class MatrixClient(nio.AsyncClient): await self._send_message(room_id, content, tx_id) - async def toggle_pause_upload( + async def toggle_pause_transfer( self, room_id: str, uuid: Union[str, UUID], ) -> None: if isinstance(uuid, str): uuid = UUID(uuid) - pause = not self.upload_monitors[uuid].pause + pause = not self.transfer_monitors[uuid].pause - self.upload_monitors[uuid].pause = pause - self.models[room_id, "uploads"][str(uuid)].paused = pause + self.transfer_monitors[uuid].pause = pause + self.models[room_id, "transfers"][str(uuid)].paused = pause - async def cancel_upload(self, uuid: Union[str, UUID]) -> None: + async def cancel_transfer(self, uuid: Union[str, UUID]) -> None: if isinstance(uuid, str): uuid = UUID(uuid) - self.upload_tasks[uuid].cancel() + self.transfer_tasks[uuid].cancel() async def send_clipboard_image( @@ -691,9 +685,9 @@ class MatrixClient(nio.AsyncClient): try: await self._send_file(item_uuid, room_id, path, reply_to_event_id) except (nio.TransferCancelledError, asyncio.CancelledError): - self.upload_monitors.pop(item_uuid, None) - self.upload_tasks.pop(item_uuid, None) - self.models[room_id, "uploads"].pop(str(item_uuid), None) + self.transfer_monitors.pop(item_uuid, None) + self.transfer_tasks.pop(item_uuid, None) + self.models[room_id, "transfers"].pop(str(item_uuid), None) async def _send_file( @@ -708,10 +702,10 @@ class MatrixClient(nio.AsyncClient): # TODO: this function is way too complex, and most of it should be # refactored into nio. - self.upload_tasks[item_uuid] = current_task() # type: ignore + self.transfer_tasks[item_uuid] = utils.current_task() # type: ignore - upload_item = Upload(item_uuid) - self.models[room_id, "uploads"][str(item_uuid)] = upload_item + transfer = Transfer(item_uuid, is_upload=True) + self.models[room_id, "transfers"][str(item_uuid)] = transfer transaction_id = uuid4() path = Path(await path() if callable(path) else path) @@ -726,18 +720,18 @@ class MatrixClient(nio.AsyncClient): # This error will be caught again by the try block later below size = 0 - upload_item.set_fields( - status=UploadStatus.Uploading, filepath=path, total_size=size, + transfer.set_fields( + status=TransferStatus.Transfering, filepath=path, total_size=size, ) monitor = nio.TransferMonitor(size) - self.upload_monitors[item_uuid] = monitor + self.transfer_monitors[item_uuid] = monitor def on_transferred(transferred: int) -> None: - upload_item.uploaded = transferred + transfer.transferred = transferred def on_speed_changed(speed: float) -> None: - upload_item.set_fields( + transfer.set_fields( speed = speed, time_left = monitor.remaining_time or timedelta(0), ) @@ -761,8 +755,8 @@ class MatrixClient(nio.AsyncClient): raise nio.TransferCancelledError() except (MatrixError, OSError) as err: - upload_item.set_fields( - status = UploadStatus.Error, + transfer.set_fields( + status = TransferStatus.Error, error = type(err), error_args = err.args, ) @@ -771,8 +765,8 @@ class MatrixClient(nio.AsyncClient): while True: await asyncio.sleep(0.1) - upload_item.status = UploadStatus.Caching - local_media = await Media.from_existing_file( + transfer.status = TransferStatus.Caching + local_media = await Media.from_existing_file( self.backend.media_cache, self.user_id, url, path, ) @@ -787,7 +781,7 @@ class MatrixClient(nio.AsyncClient): "body": path.name, "info": { "mimetype": mime, - "size": upload_item.total_size, + "size": transfer.total_size, }, } @@ -822,20 +816,20 @@ class MatrixClient(nio.AsyncClient): thumb_ext = "png" if thumb_info.mime == "image/png" else "jpg" thumb_name = f"{path.stem}_thumbnail.{thumb_ext}" - upload_item.set_fields( - status = UploadStatus.Uploading, + transfer.set_fields( + status = TransferStatus.Transfering, filepath = Path(thumb_name), total_size = len(thumb_data), ) try: - upload_item.total_size = thumb_info.size + transfer.total_size = thumb_info.size monitor = nio.TransferMonitor(thumb_info.size) monitor.on_transferred = on_transferred monitor.on_speed_changed = on_speed_changed - self.upload_monitors[item_uuid] = monitor + self.transfer_monitors[item_uuid] = monitor thumb_url, _, thumb_crypt_dict = await self.upload( lambda *_: thumb_data, @@ -851,7 +845,7 @@ class MatrixClient(nio.AsyncClient): except MatrixError as err: log.warning(f"Failed uploading thumbnail {path}: {err}") else: - upload_item.status = UploadStatus.Caching + transfer.status = TransferStatus.Caching await Thumbnail.from_bytes( self.backend.media_cache, @@ -907,9 +901,9 @@ class MatrixClient(nio.AsyncClient): content["msgtype"] = "m.file" content["filename"] = path.name - del self.upload_monitors[item_uuid] - del self.upload_tasks[item_uuid] - del self.models[room_id, "uploads"][str(upload_item.id)] + del self.transfer_monitors[item_uuid] + del self.transfer_tasks[item_uuid] + del self.models[room_id, "transfers"][str(transfer.id)] if reply_to_event_id: await self.send_text( @@ -1004,7 +998,7 @@ class MatrixClient(nio.AsyncClient): """Send a message event with `content` dict to a room.""" self.send_message_tasks[transaction_id] = \ - current_task() # type: ignore + utils.current_task() # type: ignore async with self.backend.send_locks[room_id]: await self.room_send( @@ -2068,13 +2062,13 @@ class MatrixClient(nio.AsyncClient): avatar_size = (48, 48) - avatar_path = await self.backend.media_cache.get_thumbnail( + avatar_path = await Thumbnail( + cache = self.backend.media_cache, client_user_id = self.user_id, mxc = mxc, title = f"user_{user_id}.notification", - width = avatar_size[0], - height = avatar_size[1], - ) + wanted_size = avatar_size, + ).get() image_data = None create = False diff --git a/src/backend/media_cache.py b/src/backend/media_cache.py index 28136c2e..3e9a58fc 100644 --- a/src/backend/media_cache.py +++ b/src/backend/media_cache.py @@ -13,11 +13,14 @@ from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Optional from urllib.parse import urlparse +from uuid import uuid4 import nio from PIL import Image as PILImage -from .utils import Size, atomic_write +from .models.items import Transfer, TransferStatus +from .models.model import Model +from .utils import Size, atomic_write, current_task if TYPE_CHECKING: from .backend import Backend @@ -25,8 +28,6 @@ if TYPE_CHECKING: if sys.version_info < (3, 8): import pyfastcopy # noqa -CryptDict = Optional[Dict[str, Any]] - CONCURRENT_DOWNLOADS_LIMIT = asyncio.BoundedSemaphore(8) ACCESS_LOCKS: DefaultDict[str, asyncio.Lock] = DefaultDict(asyncio.Lock) @@ -47,45 +48,35 @@ class MediaCache: self.downloads_dir.mkdir(parents=True, exist_ok=True) - async def get_media( - self, - client_user_id: str, - mxc: str, - title: str, - crypt_dict: CryptDict = None, - ) -> Path: - """Return `Media.get()`'s result. Intended for QML.""" - - return await Media(self, client_user_id, mxc, title, crypt_dict).get() + async def get_media(self, *args) -> Path: + """Return `Media(self, ...).get()`'s result. Intended for QML.""" + return await Media(self, *args).get() - async def get_thumbnail( - self, - client_user_id: str, - mxc: str, - title: str, - width: int, - height: int, - crypt_dict: CryptDict = None, - ) -> Path: - """Return `Thumbnail.get()`'s result. Intended for QML.""" - + async def get_thumbnail(self, width: float, height: float, *args) -> Path: + """Return `Thumbnail(self, ...).get()`'s result. Intended for QML.""" # QML sometimes pass float sizes, which matrix API doesn't like. size = (round(width), round(height)) - - thumb = Thumbnail(self, client_user_id, mxc, title, crypt_dict, size) - return await thumb.get() + return await Thumbnail( + self, *args, wanted_size=size, # type: ignore + ).get() @dataclass class Media: - """A matrix media file.""" + """A matrix media file that is downloaded or has yet to be. - cache: "MediaCache" = field() - client_user_id: str = field() - mxc: str = field() - title: str = field() - crypt_dict: CryptDict = field(repr=False) + If the `room_id` is not set, no `Transfer` model item will be registered + while this media is being downloaded. + """ + + cache: "MediaCache" = field() + client_user_id: str = field() + mxc: str = field() + title: str = field() + room_id: Optional[str] = None + filesize: Optional[int] = None + crypt_dict: Optional[Dict[str, Any]] = field(default=None, repr=False) def __post_init__(self) -> None: @@ -155,12 +146,39 @@ class Media: async def _get_remote_data(self) -> bytes: """Return the file's data from the matrix server, decrypt if needed.""" - parsed = urlparse(self.mxc) + client = self.cache.backend.clients[self.client_user_id] - resp = await self.cache.backend.clients[self.client_user_id].download( - server_name = parsed.netloc, - media_id = parsed.path.lstrip("/"), - ) + transfer: Optional[Transfer] = None + model: Optional[Model] = None + + if self.room_id: + model = self.cache.backend.models[self.room_id, "transfers"] + transfer = Transfer( + id = uuid4(), + is_upload = False, + filepath = self.local_path, + total_size = self.filesize or 0, + status = TransferStatus.Transfering, + ) + assert model is not None + client.transfer_tasks[transfer.id] = current_task() # type: ignore + model[str(transfer.id)] = transfer + + try: + parsed = urlparse(self.mxc) + resp = await client.download( + server_name = parsed.netloc, + media_id = parsed.path.lstrip("/"), + ) + except (nio.TransferCancelledError, asyncio.CancelledError): + if transfer and model: + del model[str(transfer.id)] + del client.transfer_tasks[transfer.id] + raise + + if transfer and model: + del model[str(transfer.id)] + del client.transfer_tasks[transfer.id] return await self._decrypt(resp.body) @@ -196,8 +214,13 @@ class Media: """Copy an existing file to cache and return a `Media` for it.""" media = cls( - cache, client_user_id, mxc, existing.name, {}, **kwargs, - ) # type: ignore + cache = cache, + client_user_id = client_user_id, + mxc = mxc, + title = existing.name, + filesize = existing.stat().st_size, + **kwargs, + ) media.local_path.parent.mkdir(parents=True, exist_ok=True) if not media.local_path.exists() or overwrite: @@ -221,8 +244,8 @@ class Media: """Create a cached file from bytes data and return a `Media` for it.""" media = cls( - cache, client_user_id, mxc, filename, {}, **kwargs, - ) # type: ignore + cache, client_user_id, mxc, filename, filesize=len(data), **kwargs, + ) media.local_path.parent.mkdir(parents=True, exist_ok=True) if not media.local_path.exists() or overwrite: @@ -237,14 +260,9 @@ class Media: @dataclass class Thumbnail(Media): - """The thumbnail of a matrix media, which is a media itself.""" + """A matrix media's thumbnail, which is downloaded or has yet to be.""" - cache: "MediaCache" = field() - client_user_id: str = field() - mxc: str = field() - title: str = field() - crypt_dict: CryptDict = field(repr=False) - wanted_size: Size = field() + wanted_size: Size = (800, 600) server_size: Optional[Size] = field(init=False, repr=False, default=None) diff --git a/src/backend/models/items.py b/src/backend/models/items.py index 07b18b12..2208bf02 100644 --- a/src/backend/models/items.py +++ b/src/backend/models/items.py @@ -286,37 +286,38 @@ class Member(ModelItem): ) -class UploadStatus(AutoStrEnum): +class TransferStatus(AutoStrEnum): """Enum describing the status of an upload operation.""" - Preparing = auto() - Uploading = auto() - Caching = auto() - Error = auto() + Preparing = auto() + Transfering = auto() + Caching = auto() + Error = auto() @dataclass(eq=False) -class Upload(ModelItem): - """Represent a running or failed file upload operation.""" +class Transfer(ModelItem): + """Represent a running or failed file upload/download operation.""" - id: UUID = field() - filepath: Path = Path("-") + id: UUID = field() + is_upload: bool = field() + filepath: Path = Path("-") - total_size: int = 0 - uploaded: int = 0 - speed: float = 0 - time_left: timedelta = timedelta(0) - paused: bool = False + total_size: int = 0 + transferred: int = 0 + speed: float = 0 + time_left: timedelta = timedelta(0) + paused: bool = False - status: UploadStatus = UploadStatus.Preparing + status: TransferStatus = TransferStatus.Preparing error: OptionalExceptionType = type(None) error_args: Tuple[Any, ...] = () start_date: datetime = field(init=False, default_factory=datetime.now) - def __lt__(self, other: "Upload") -> bool: - """Sort by the start date, from newest upload to oldest.""" + def __lt__(self, other: "Transfer") -> bool: + """Sort by the start date, from newest transfer to oldest.""" return (self.start_date, self.id) > (other.start_date, other.id) diff --git a/src/backend/nio_callbacks.py b/src/backend/nio_callbacks.py index ee2f5c27..40b1739b 100644 --- a/src/backend/nio_callbacks.py +++ b/src/backend/nio_callbacks.py @@ -197,6 +197,8 @@ class NioCallbacks: client_user_id = self.user_id, mxc = ev.url, title = ev.body, + room_id = room.room_id, + filesize = info.get("size") or 0, crypt_dict = media_crypt_dict, ).get_local() except FileNotFoundError: diff --git a/src/backend/utils.py b/src/backend/utils.py index 3ca0175e..c9f90326 100644 --- a/src/backend/utils.py +++ b/src/backend/utils.py @@ -38,8 +38,10 @@ from .pcn.section import Section if sys.version_info >= (3, 7): from contextlib import asynccontextmanager + current_task = asyncio.current_task else: from async_generator import asynccontextmanager + current_task = asyncio.Task.current_task Size = Tuple[int, int] BytesOrPIL = Union[bytes, PILImage.Image] diff --git a/src/gui/Base/HMxcImage.qml b/src/gui/Base/HMxcImage.qml index 46473389..4e9acb31 100644 --- a/src/gui/Base/HMxcImage.qml +++ b/src/gui/Base/HMxcImage.qml @@ -9,6 +9,9 @@ HImage { property string clientUserId property string mxc property string title + property var roomId: undefined // undefined or string + property var fileSize: undefined // undefined or int (bytes) + property string sourceOverride: "" property bool thumbnail: true property var cryptDict: ({}) @@ -39,10 +42,10 @@ HImage { } const method = image.thumbnail ? "get_thumbnail" : "get_media" - const args = - image.thumbnail ? - [clientUserId, image.mxc, image.title, w, h, cryptDict] : - [clientUserId, image.mxc, image.title, cryptDict] + let args = [ + clientUserId, image.mxc, image.title, roomId, fileSize, cryptDict, + ] + if (image.thumbnail) args = [w, h, ...args] getFutureId = py.callCoro("media_cache." + method, args, path => { if (! image) return diff --git a/src/gui/Pages/Chat/FileTransfer/Transfer.qml b/src/gui/Pages/Chat/FileTransfer/Transfer.qml index 0197c93c..d3de5980 100644 --- a/src/gui/Pages/Chat/FileTransfer/Transfer.qml +++ b/src/gui/Pages/Chat/FileTransfer/Transfer.qml @@ -12,7 +12,7 @@ HColumnLayout { property bool cancelPending: false property int msLeft: model.time_left - property int uploaded: model.uploaded + property int transferred: model.transferred readonly property int speed: model.speed readonly property int totalSize: model.total_size readonly property string status: model.status @@ -21,12 +21,12 @@ HColumnLayout { function cancel() { cancelPending = true // Python will delete this model item on cancel - py.callClientCoro(chat.userId, "cancel_upload", [model.id]) + py.callClientCoro(chat.userId, "cancel_transfer", [model.id]) } function toggle_pause() { py.callClientCoro( - chat.userId, "toggle_pause_upload", [chat.roomId, model.id], + chat.userId, "toggle_pause_transfer", [chat.roomId, model.id], ) } @@ -36,7 +36,7 @@ HColumnLayout { HRowLayout { HIcon { - svgName: "uploading" + svgName: model.is_upload ? "uploading" : "downloading" colorize: cancelPending || transfer.status === "Error" ? theme.colors.negativeBackground : @@ -70,7 +70,7 @@ HColumnLayout { status === "Preparing" ? qsTr("Preparing file...") : - status === "Uploading" ? + status === "Transfering" ? fileName : status === "Caching" ? @@ -120,8 +120,12 @@ HColumnLayout { speed ? qsTr("%1/s").arg(CppUtils.formattedBytes(speed)) : "", - qsTr("%1/%2").arg(CppUtils.formattedBytes(uploaded)) - .arg(CppUtils.formattedBytes(totalSize)), + transferred && totalSize ? + qsTr("%1/%2").arg(CppUtils.formattedBytes(transferred)) + .arg(CppUtils.formattedBytes(totalSize)) : + transferred || totalSize ? + CppUtils.formattedBytes(transferred || totalSize) : + "", ] HLabel { @@ -131,7 +135,7 @@ HColumnLayout { rightPadding: leftPadding Layout.preferredWidth: - status === "Uploading" ? implicitWidth : 0 + status === "Transfering" ? implicitWidth : 0 Behavior on Layout.preferredWidth { HNumberAnimation {} } } @@ -142,7 +146,7 @@ HColumnLayout { padded: false icon.name: transfer.paused ? - "upload-resume" : "upload-pause" + "transfer-resume" : "transfer-pause" icon.color: transfer.paused ? theme.colors.positiveBackground : @@ -153,8 +157,9 @@ HColumnLayout { onClicked: transfer.toggle_pause() + // TODO: pausing downloads Layout.preferredWidth: - status === "Uploading" ? + status === "Transfering" && model.is_upload ? theme.baseElementsHeight : 0 Layout.fillHeight: true @@ -163,7 +168,7 @@ HColumnLayout { } HButton { - icon.name: "upload-cancel" + icon.name: "transfer-cancel" icon.color: theme.colors.negativeBackground onClicked: transfer.cancel() padded: false @@ -183,8 +188,8 @@ HColumnLayout { HProgressBar { id: progressBar visible: Layout.maximumHeight !== 0 - indeterminate: status !== "Uploading" - value: uploaded + indeterminate: status !== "Transfering" || ! totalSize || ! transferred + value: transferred to: totalSize // TODO: bake this in hprogressbar diff --git a/src/gui/Pages/Chat/FileTransfer/TransferList.qml b/src/gui/Pages/Chat/FileTransfer/TransferList.qml index c7c2571f..1bd655de 100644 --- a/src/gui/Pages/Chat/FileTransfer/TransferList.qml +++ b/src/gui/Pages/Chat/FileTransfer/TransferList.qml @@ -25,7 +25,7 @@ Rectangle { id: transferList anchors.fill: parent - model: ModelStore.get(chat.roomId, "uploads") + model: ModelStore.get(chat.roomId, "transfers") delegate: Transfer { width: transferList.width } } } diff --git a/src/gui/Pages/Chat/Timeline/EventList.qml b/src/gui/Pages/Chat/Timeline/EventList.qml index 98bb3e0e..e0b6881c 100644 --- a/src/gui/Pages/Chat/Timeline/EventList.qml +++ b/src/gui/Pages/Chat/Timeline/EventList.qml @@ -489,19 +489,16 @@ Rectangle { return } - print("Downloading " + event.media_url + " ...") - const args = [ chat.userId, event.media_url, event.media_title, + chat.roomId, + event.media_size, JSON.parse(event.media_crypt_dict), ] - py.callCoro("media_cache.get_media", args, path => { - print("Done: " + path) - callback(path) - }) + py.callCoro("media_cache.get_media", args, callback) } function openMediaExternally(event) { diff --git a/src/icons/thin/downloading.svg b/src/icons/thin/downloading.svg new file mode 100644 index 00000000..8aa55a76 --- /dev/null +++ b/src/icons/thin/downloading.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/icons/thin/upload-cancel.svg b/src/icons/thin/transfer-cancel.svg similarity index 100% rename from src/icons/thin/upload-cancel.svg rename to src/icons/thin/transfer-cancel.svg diff --git a/src/icons/thin/upload-pause.svg b/src/icons/thin/transfer-pause.svg similarity index 100% rename from src/icons/thin/upload-pause.svg rename to src/icons/thin/transfer-pause.svg diff --git a/src/icons/thin/upload-resume.svg b/src/icons/thin/transfer-resume.svg similarity index 100% rename from src/icons/thin/upload-resume.svg rename to src/icons/thin/transfer-resume.svg diff --git a/src/icons/thin/uploading.svg b/src/icons/thin/uploading.svg index c4cde5b5..c59377c6 100644 --- a/src/icons/thin/uploading.svg +++ b/src/icons/thin/uploading.svg @@ -1,3 +1 @@ - - - + \ No newline at end of file